/[escript]/trunk/scripts/prepare.py
ViewVC logotype

Contents of /trunk/scripts/prepare.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2249 - (show annotations)
Fri Feb 6 00:11:51 2009 UTC (11 years, 7 months ago) by jfenwick
File MIME type: text/x-python
File size: 6216 byte(s)
Fixed some pathing issues.

1
2 import shutil, os, datetime, sys
3
4
5 SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk"
6 NUMJs=4
7 TOPDIR=str(datetime.date.today())
8 ERRMAIL="j.fenwick1@uq.edu.au"
9 EXECUTELOCATION="/scratch/jfenwick/AUTOTESTS"
10 OUTSIDEDIR=os.getcwd()
11
12 #Settings for actual tests appear below the class declarations
13
14
15 def failure(msg):
16 print "Terminating - Error "+str(msg)
17 print "Should be sending mail to "+str(ERRMAIL)
18 sys.exit(1)
19
20 def progress(msg):
21 print msg
22
23 class TestConfiguration(object):
24 def __init__(self, name, opts, omp, mpi, binexec, pythonexec):
25 self.name=name
26 self.opts=opts
27 self.omp=omp
28 self.mpi=mpi
29 self.binexec=binexec
30 self.pythonexec=pythonexec
31
32 def getHeader():
33 res="#!/bin/bash\n"
34 res=res+'MAIL_RECIPIENTS="'+ERRMAIL+'"\n'
35 res=res+"function report()\n{\n"
36 res=res+" NOW=`date '+%Y/%m/%d %H:%M'`\n"
37 res=res+" cat > $LOGDIR/message << END_MSG\n"
38 res=res+"Sucessful configurations:\n"
39 res=res+"$SUCCESSFUL\n\n"
40 res=res+"Failed on configuration:\n"
41 res=res+"$ATTEMPTING\n\n"
42 res=res+"Tests ran from $START until $NOW.\n"
43 res=res+"Log files can be found in $TOP.\n"
44 res=res+"This mail was sent from $SCRIPTNAME, running as $USER on `hostname`.\n"
45 res=res+"END_MSG\n"
46 res=res+"}\n"
47 res=res+"function progress()\n{\n"
48 res=res+" echo $1\n"
49 res=res+" echo $1 >> $PROGRESSFILE\n"
50 res=res+"}\n"
51 res=res+"function failure()\n{\n echo $1\n"
52 res=res+" report\n"
53 res=res+" touch $LOGDIR/Failure\n"
54 res=res+" exit 1\n}\nTOP=`pwd`\nLOGDIR=$TOP/Logs\nPROGRESSFILE=$LOGDIR/progress\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n"
55 res=res+". /usr/share/modules/init/sh #So the module command works\n"
56 res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n"
57 res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n"
58 res=res+"SCRIPTNAME=$0\n"
59 return res
60
61 def toString(self):
62 runcount=1
63 for o in self.omp:
64 for m in self.mpi:
65 cmd="bash utest.sh 'mpiexec -np"+str(m)+"' $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1"
66 res="cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
67 res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
68 res=res+"TESTROOT=`pwd`\n"
69 res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
70 res=res+"mkdir $TESTLOGDIR\n"
71 res=res+"export OMP_NUM_THREADS="+str(o)+"\n"
72 res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
73 res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
74 res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi='+str(m)+'"\n'
75 res=res+'ATTEMPTING=$RUNNAME\n'
76 res=res+'progress "Starting '+cmd+'"\n'
77 res=res+cmd+' || failure "'+cmd+'"\n'
78 res=res+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n'
79 res=res+'progress "completed '+cmd+'"\n'
80 res=res+'ATTEMPTING=None\n'
81 res=res+"export OMP_NUM_THREADS=1\n"
82 res=res+"cd $TOP\n"
83 res=res+"rm -rf "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
84 ++runcount
85 if len(self.mpi)==0:
86 cmd="bash utest.sh '' python $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1"
87 res="cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
88 res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
89 res=res+"TESTROOT=`pwd`\n"
90 res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
91 res=res+"mkdir $TESTLOGDIR\n"
92 res=res+"export OMP_NUM_THREADS="+str(o)+"\n"
93 res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
94 res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
95 res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi=n/a"\n'
96 res=res+'ATTEMPTING=$RUNNAME\n'
97 res=res+'progress "Starting '+cmd+'"\n'
98 res=res+cmd+" || failure \""+cmd+"\" \n"
99 res=res+'ATTEMPTING=None\n'
100 res=res+'progress "completed '+cmd+'"\n'
101 res=res+"cd $TOP\n"
102 res=res+"rm -rf "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
103 ++runcount
104 res=res+"\ncd $TOP\n\n"
105 return res
106
107 def getFooter():
108 res="\ntouch $LOGDIR/Success\n"
109 res=res+"report"
110 return res
111
112 getHeader=staticmethod(getHeader)
113 getFooter=staticmethod(getFooter)
114
115 #Test settings
116 testconfs=[]
117 testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python"))
118 testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI"))
119
120
121 try:
122 os.mkdir(TOPDIR)
123 os.chdir(TOPDIR)
124 except OSError:
125 failure("Unable to create top directory "+TOPDIR+" does it exist already?")
126 sys.exit(1)
127
128 try:
129 os.mkdir("Logs")
130 except OSError:
131 failure("Unable to create Logs directory ")
132 sys.exit(1)
133
134 coresult=os.system("svn export "+SVNURL+" src")
135 if coresult!=0:
136 failure("Unable to export working copy")
137 sys.exit(1)
138
139
140
141
142 dir=os.getcwd()
143 for conf in testconfs:
144 progress("Creating "+conf.name+"_src")
145 # Yes I know copytree exists. No I don't trust it (It was having problems doing the copy).
146 res=os.system("cp -r src "+conf.name+"_src")
147 if res!=0:
148 failure("Error copying src to "+conf.name+"_src")
149 # shutil.copytree("src",conf.name+"_src")
150 os.chdir(conf.name+"_src")
151 cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests"
152 progress(cmdstr)
153 res=os.system(cmdstr)
154 os.chdir(str(dir))
155 if res!=0:
156 failure("running scons build failed for "+conf.name+"_src")
157
158 progress("Builds complete")
159 progress("Removing export copy")
160 shutil.rmtree("src",ignore_errors=True)
161 progress("Building test file")
162
163 try:
164 testfile=open("dotests.sh","w")
165 testfile.write(TestConfiguration.getHeader())
166 for c in testconfs:
167 testfile.write(c.toString())
168 testfile.close()
169 except IOError:
170 failure("Creating testfile")
171
172 progress("Building test file complete")
173 progress("Copying files to exec area")
174 os.chdir(OUTSIDEDIR)
175 try:
176 shutil.copytree(TOPDIR,EXECUTELOCATION)
177 except IOError, OSError:
178 failure("copying to work area")
179 progress("Copy to exec area complete")
180
181 print "Should be submitting this test now"
182
183 os.chdir(EXECUTELOCATION)
184 os.chdir(TOPDIR)
185 try:
186 res=os.system("bash dotests.sh")
187 except OSError:
188 failure("Running tests")
189
190 try:
191 shutil.copytree(EXECUTELOCATION+"/Logs",OUTSIDEDIR)
192 except OSError:
193 failure("Log copy failed")
194
195
196
197 raise "Test not submitted - not cleaned up either"
198 #Now we build the script and submit it pbs (that behaviour should be optional?)

  ViewVC Help
Powered by ViewVC 1.1.26