/[escript]/trunk/scripts/prepare.py
ViewVC logotype

Contents of /trunk/scripts/prepare.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2269 - (show annotations)
Fri Feb 13 03:55:28 2009 UTC (10 years, 7 months ago) by jfenwick
File MIME type: text/x-python
File size: 9432 byte(s)
Updating prepare.py

1 #!/sw/apps/python/x86_64/gcc-4.1.2/python-2.4.4/bin/python
2 import shutil, os, datetime, sys, os.path, time
3
4 #This script does not use the python, platform independent path manipulation stuff.
5 #It probably should
6
7 SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk"
8 NUMJs=4
9 TOPDIR=str(datetime.date.today())
10 ERRMAIL="j.fenwick1@uq.edu.au"
11 EXECUTELOCATION="/scratch/jfenwick/AUTOTESTS"
12 OUTSIDEDIR='/data1/jfenwick/EscriptDev'
13 TESTSLEEP=30*60
14
15 SRCMSG="This message was sent by prepare.py running as "+str(os.environ['USER'])+" on "+str(os.environ['HOSTNAME']+"\n")
16
17 #Settings for actual tests appear below the class declarations
18
19
20 def failure(msg):
21 print "Terminating - Error "+str(msg)
22 print "Should be sending mail to "+str(ERRMAIL)
23 mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n"
24 mailcmd=mailcmd+"Error preparing for test run:\n"+msg+"\n"
25 mailcmd=mailcmd+SRCMSG
26 mailcmd=mailcmd+"ENDMSG\n"
27 os.system(mailcmd)
28 sys.exit(1)
29
30 def progress(msg):
31 print msg
32
33 class TestConfiguration(object):
34 def __init__(self, name, opts, omp, mpi, binexec, pythonexec):
35 self.name=name
36 self.opts=opts
37 self.omp=omp
38 self.mpi=mpi
39 self.binexec=binexec
40 self.pythonexec=pythonexec
41
42 def getHeader():
43 res="#!/bin/bash\n"
44 res=res+'MAIL_RECIPIENTS="'+ERRMAIL+'"\n'
45 res=res+"function report()\n{\n"
46 res=res+" NOW=`date '+%Y/%m/%d %H:%M'`\n"
47 res=res+" cat > $LOGDIR/message << END_MSG\n"
48 res=res+"Sucessful configurations:\n"
49 res=res+"$SUCCESSFUL\n\n"
50 res=res+"Failed on configuration:\n"
51 res=res+"$ATTEMPTING\n\n"
52 res=res+"Tests ran from $START until $NOW.\n"
53 res=res+"Log files can be found in $FINALLOGDIR.\n"
54 res=res+"END_MSG\n"
55 res=res+"}\n"
56 res=res+"function progress()\n{\n"
57 res=res+" echo $1\n"
58 res=res+" echo $1 >> $PROGRESSFILE\n"
59 res=res+"}\n"
60 res=res+"function failure()\n{\n echo $1\n"
61 res=res+" report\n"
62 res=res+" touch $LOGDIR/Failure\n"
63 res=res+" if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
64 res=res+" exit 1\n}\n"
65 res=res+"cd "+EXECUTELOCATION+"/"+TOPDIR+"\n"
66 res=res+"TOP=`pwd`\nLOGDIR=$TOP/Logs\nPROGRESSFILE=$LOGDIR/progress\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n"
67 res=res+". /usr/share/modules/init/sh #So the module command works\n"
68 res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n"
69 res=res+"module load netpbm\n"
70 res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n"
71 res=res+"SCRIPTNAME=$0\n"
72 res=res+"START=`date '+%Y/%m/%d %H:%M'`\n"
73 res=res+"TESTLOGDIR=$LOGDIR\n"
74 res=res+"FINALLOGDIR="+OUTSIDEDIR+"/"+TOPDIR+"_Logs\n"
75 res=res+"MPICOM='mpirun -np ' # Use this one for non-pbs jobs\n"
76 res=res+"MPICOM='mpiexec -n '\n"
77 return res
78
79 def toString(self):
80 runcount=1
81 res=""
82 print "Processing "+self.name
83 for o in self.omp:
84 print "o="+str(o)
85 for m in self.mpi:
86 print " m="+str(m)
87 cmd="bash utest.sh \"$MPICOM "+str(m)+"\" $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1"
88 res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
89 res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
90 res=res+"TESTROOT=`pwd`\n"
91 res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
92 res=res+"mkdir $TESTLOGDIR\n"
93 res=res+"export OMP_NUM_THREADS="+str(o)+"\n"
94 res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
95 res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
96 res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi='+str(m)+'"\n'
97 res=res+'ATTEMPTING=$RUNNAME\n'
98 res=res+'progress "Starting '+cmd+'"\n'
99 res=res+cmd+' || failure "'+cmd+'"\n'
100 res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
101 res=res+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n'
102 res=res+'progress "completed '+cmd+'"\n'
103 res=res+'ATTEMPTING=None\n'
104 res=res+"export OMP_NUM_THREADS=1\n"
105 res=res+"cd $TOP\n"
106 res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n"
107 runcount=runcount+1
108 if len(self.mpi)==0:
109 print " m=()"
110 cmd="bash utest.sh '' python $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1"
111 res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
112 res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
113 res=res+"TESTROOT=`pwd`\n"
114 res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
115 res=res+"mkdir $TESTLOGDIR\n"
116 res=res+"export OMP_NUM_THREADS="+str(o)+"\n"
117 res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
118 res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
119 res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi=n/a"\n'
120 res=res+'ATTEMPTING=$RUNNAME\n'
121 res=res+'progress "Starting '+cmd+'"\n'
122 res=res+cmd+" || failure \""+cmd+"\" \n"
123 res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
124 res=res+'ATTEMPTING=None\n'
125 res=res+'progress "completed '+cmd+'"\n'
126 res=res+"cd $TOP\n"
127 res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n"
128 runcount=runcount+1
129 res=res+"rm -rf "+self.name+"_src\n"
130 res=res+"\ncd $TOP\n\n"
131 return res
132
133 def getFooter():
134 res="\ntouch $LOGDIR/Success\n"
135 res=res+"report"
136 return res
137
138 getHeader=staticmethod(getHeader)
139 getFooter=staticmethod(getFooter)
140
141 #Test settings
142 testconfs=[]
143 testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python"))
144 testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI"))
145
146
147 os.chdir(OUTSIDEDIR)
148 LOGDIR=OUTSIDEDIR+"/"+TOPDIR+"_Logs"
149
150 if os.path.exists(LOGDIR):
151 failure("Logs directory for "+TOPDIR+" already exists.")
152 sys.exit(1)
153
154 try:
155 os.mkdir(TOPDIR)
156 os.chdir(TOPDIR)
157 except OSError:
158 failure("Unable to create top directory "+TOPDIR+" does it exist already?")
159 sys.exit(1)
160
161 try:
162 os.mkdir("Logs")
163 except OSError:
164 failure("Unable to create Logs directory ")
165 sys.exit(1)
166
167 coresult=os.system("svn export "+SVNURL+" src")
168 if coresult!=0:
169 failure("Unable to export working copy")
170 sys.exit(1)
171
172
173
174
175 dir=os.getcwd()
176 for conf in testconfs:
177 progress("Creating "+conf.name+"_src")
178 # res=os.system("cp -r src "+conf.name+"_src")
179 # if res!=0:
180 # failure("Error copying src to "+conf.name+"_src")
181 try:
182 shutil.copytree("src",conf.name+"_src")
183 except Error:
184 failure("copying src to "+conf.name+"_src")
185 os.chdir(conf.name+"_src")
186 cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests"
187 progress(cmdstr)
188 res=os.system(cmdstr)
189 os.chdir(str(dir))
190 if res!=0:
191 failure("running scons build failed for "+conf.name+"_src")
192
193 progress("Builds complete")
194 progress("Removing export copy")
195 shutil.rmtree("src",ignore_errors=True)
196 progress("Building test file")
197
198 try:
199 testfile=open("dotests.sh","w")
200 testfile.write(TestConfiguration.getHeader())
201 for c in testconfs:
202 testfile.write(c.toString())
203 testfile.write(TestConfiguration.getFooter())
204 testfile.close()
205 import stat
206 os.chmod("dotests.sh",stat.S_IEXEC|stat.S_IREAD)
207 except IOError:
208 failure("Creating testfile")
209
210 progress("Building test file complete")
211 progress("Copying files to exec area")
212 os.chdir(OUTSIDEDIR)
213 #try:
214 #shutil.copytree(TOPDIR,EXECUTELOCATION+"/"+TOPDIR)
215 #except OSError:
216 # failure("copying to work area")
217 res=os.system("cp -r "+TOPDIR+" "+EXECUTELOCATION+"/"+TOPDIR)
218 if res!=0:
219 failure("copying work area")
220 progress("Copy to exec area complete")
221
222 print "Submitting test"
223
224 ######### test section
225
226 os.chdir(EXECUTELOCATION)
227 os.chdir(TOPDIR)
228
229
230 try:
231 # res=os.system("bash dotests.sh")
232 res=os.system("qsub -l select=1:ncpus=8:mem=31gb dotests.sh")
233 except OSError:
234 failure("Submitting tests")
235
236 os.chdir(OUTSIDEDIR)
237
238 print "Sleeping for "+str(TESTSLEEP)+" seconds to wait for results."
239 time.sleep(TESTSLEEP)
240 print "Waking up."
241
242 ######### end test section
243
244 try:
245 shutil.copytree(EXECUTELOCATION+"/"+TOPDIR+"/Logs",OUTSIDEDIR+"/"+TOPDIR+"_Logs")
246 except OSError:
247 failure("Log copy failed")
248
249 cleanupfailure=False
250
251 try:
252 shutil.rmtree(EXECUTELOCATION+"/"+TOPDIR)
253 shutil.rmtree(OUTSIDEDIR+"/"+TOPDIR)
254 except OSError:
255 cleanupfailure=True
256
257 #Now we sum up
258 #if Logs/Failure or Logs/Success does not exist send message about tests not completing.
259 if not os.path.exists(LOGDIR+"/Success") and not os.path.exists(LOGDIR+"/Failure"):
260 mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n"
261 mailcmd=mailcmd+"For some reason no Success or failure is recorded for unit tests in "+LOGDIR+"\n"
262 mailcmd=mailcmd+"\nAlso: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"\n"
263 mailcmd=mailcmd+SRCMSG
264 mailcmd=mailcmd+"ENDMSG\n"
265 os.system(mailcmd)
266 sys.exit(1)
267
268
269 if os.path.exists(LOGDIR+"/Failure"):
270 os.system("echo 'Also: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"' >> "+LOGDIR+"/message\n")
271 os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n")
272 mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests failed' "+ERRMAIL+"\n"
273 os.system(mailcmd)
274 sys.exit(1)
275 #so we must have succeeded
276
277 if cleanupfailure:
278 os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n")
279 mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests cleanup failed - tests succeeded' "+ERRMAIL+"\n"
280 res=os.system(mailcmd)
281 sys.exit(res)
282 else:
283 os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n")
284 mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests succeeded' "+ERRMAIL+"\n"
285 res=os.system(mailcmd)
286 sys.exit(res)
287

  ViewVC Help
Powered by ViewVC 1.1.26