/[escript]/trunk/scripts/prepare.py
ViewVC logotype

Annotation of /trunk/scripts/prepare.py

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2270 - (hide annotations)
Mon Feb 16 04:43:28 2009 UTC (10 years, 7 months ago) by jfenwick
Original Path: branches/arrexp_2137_win_merge/scripts/prepare.py
File MIME type: text/x-python
File size: 9432 byte(s)
Bringing branch up to 2269

1 jfenwick 2270 #!/sw/apps/python/x86_64/gcc-4.1.2/python-2.4.4/bin/python
2 jfenwick 2266 import shutil, os, datetime, sys, os.path, time
3    
4     #This script does not use the python, platform independent path manipulation stuff.
5     #It probably should
6    
7     SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk"
8     NUMJs=4
9     TOPDIR=str(datetime.date.today())
10     ERRMAIL="j.fenwick1@uq.edu.au"
11     EXECUTELOCATION="/scratch/jfenwick/AUTOTESTS"
12 jfenwick 2270 OUTSIDEDIR='/data1/jfenwick/EscriptDev'
13 jfenwick 2266 TESTSLEEP=30*60
14    
15     SRCMSG="This message was sent by prepare.py running as "+str(os.environ['USER'])+" on "+str(os.environ['HOSTNAME']+"\n")
16    
17     #Settings for actual tests appear below the class declarations
18    
19    
20     def failure(msg):
21     print "Terminating - Error "+str(msg)
22     print "Should be sending mail to "+str(ERRMAIL)
23     mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n"
24     mailcmd=mailcmd+"Error preparing for test run:\n"+msg+"\n"
25     mailcmd=mailcmd+SRCMSG
26     mailcmd=mailcmd+"ENDMSG\n"
27     os.system(mailcmd)
28     sys.exit(1)
29    
30     def progress(msg):
31     print msg
32    
33     class TestConfiguration(object):
34     def __init__(self, name, opts, omp, mpi, binexec, pythonexec):
35     self.name=name
36     self.opts=opts
37     self.omp=omp
38     self.mpi=mpi
39     self.binexec=binexec
40     self.pythonexec=pythonexec
41    
42     def getHeader():
43     res="#!/bin/bash\n"
44     res=res+'MAIL_RECIPIENTS="'+ERRMAIL+'"\n'
45     res=res+"function report()\n{\n"
46     res=res+" NOW=`date '+%Y/%m/%d %H:%M'`\n"
47     res=res+" cat > $LOGDIR/message << END_MSG\n"
48     res=res+"Sucessful configurations:\n"
49     res=res+"$SUCCESSFUL\n\n"
50     res=res+"Failed on configuration:\n"
51     res=res+"$ATTEMPTING\n\n"
52     res=res+"Tests ran from $START until $NOW.\n"
53     res=res+"Log files can be found in $FINALLOGDIR.\n"
54     res=res+"END_MSG\n"
55     res=res+"}\n"
56     res=res+"function progress()\n{\n"
57     res=res+" echo $1\n"
58     res=res+" echo $1 >> $PROGRESSFILE\n"
59     res=res+"}\n"
60     res=res+"function failure()\n{\n echo $1\n"
61     res=res+" report\n"
62     res=res+" touch $LOGDIR/Failure\n"
63     res=res+" if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
64     res=res+" exit 1\n}\n"
65     res=res+"cd "+EXECUTELOCATION+"/"+TOPDIR+"\n"
66     res=res+"TOP=`pwd`\nLOGDIR=$TOP/Logs\nPROGRESSFILE=$LOGDIR/progress\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n"
67     res=res+". /usr/share/modules/init/sh #So the module command works\n"
68     res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n"
69 jfenwick 2270 res=res+"module load netpbm\n"
70 jfenwick 2266 res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n"
71     res=res+"SCRIPTNAME=$0\n"
72     res=res+"START=`date '+%Y/%m/%d %H:%M'`\n"
73     res=res+"TESTLOGDIR=$LOGDIR\n"
74     res=res+"FINALLOGDIR="+OUTSIDEDIR+"/"+TOPDIR+"_Logs\n"
75 jfenwick 2270 res=res+"MPICOM='mpirun -np ' # Use this one for non-pbs jobs\n"
76     res=res+"MPICOM='mpiexec -n '\n"
77 jfenwick 2266 return res
78    
79     def toString(self):
80     runcount=1
81     res=""
82     print "Processing "+self.name
83     for o in self.omp:
84     print "o="+str(o)
85     for m in self.mpi:
86     print " m="+str(m)
87 jfenwick 2270 cmd="bash utest.sh \"$MPICOM "+str(m)+"\" $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1"
88 jfenwick 2266 res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
89     res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
90     res=res+"TESTROOT=`pwd`\n"
91     res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
92     res=res+"mkdir $TESTLOGDIR\n"
93     res=res+"export OMP_NUM_THREADS="+str(o)+"\n"
94     res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
95     res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
96     res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi='+str(m)+'"\n'
97     res=res+'ATTEMPTING=$RUNNAME\n'
98     res=res+'progress "Starting '+cmd+'"\n'
99     res=res+cmd+' || failure "'+cmd+'"\n'
100     res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
101     res=res+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n'
102     res=res+'progress "completed '+cmd+'"\n'
103     res=res+'ATTEMPTING=None\n'
104     res=res+"export OMP_NUM_THREADS=1\n"
105     res=res+"cd $TOP\n"
106     res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n"
107     runcount=runcount+1
108     if len(self.mpi)==0:
109     print " m=()"
110     cmd="bash utest.sh '' python $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1"
111     res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
112     res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
113     res=res+"TESTROOT=`pwd`\n"
114     res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
115     res=res+"mkdir $TESTLOGDIR\n"
116     res=res+"export OMP_NUM_THREADS="+str(o)+"\n"
117     res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
118     res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
119     res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi=n/a"\n'
120     res=res+'ATTEMPTING=$RUNNAME\n'
121     res=res+'progress "Starting '+cmd+'"\n'
122     res=res+cmd+" || failure \""+cmd+"\" \n"
123     res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
124     res=res+'ATTEMPTING=None\n'
125     res=res+'progress "completed '+cmd+'"\n'
126     res=res+"cd $TOP\n"
127     res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n"
128     runcount=runcount+1
129     res=res+"rm -rf "+self.name+"_src\n"
130     res=res+"\ncd $TOP\n\n"
131     return res
132    
133     def getFooter():
134     res="\ntouch $LOGDIR/Success\n"
135     res=res+"report"
136     return res
137    
138     getHeader=staticmethod(getHeader)
139     getFooter=staticmethod(getFooter)
140    
141     #Test settings
142     testconfs=[]
143     testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python"))
144     testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI"))
145    
146 jfenwick 2270
147     os.chdir(OUTSIDEDIR)
148 jfenwick 2266 LOGDIR=OUTSIDEDIR+"/"+TOPDIR+"_Logs"
149    
150     if os.path.exists(LOGDIR):
151     failure("Logs directory for "+TOPDIR+" already exists.")
152     sys.exit(1)
153    
154     try:
155     os.mkdir(TOPDIR)
156     os.chdir(TOPDIR)
157     except OSError:
158     failure("Unable to create top directory "+TOPDIR+" does it exist already?")
159     sys.exit(1)
160    
161     try:
162     os.mkdir("Logs")
163     except OSError:
164     failure("Unable to create Logs directory ")
165     sys.exit(1)
166    
167     coresult=os.system("svn export "+SVNURL+" src")
168     if coresult!=0:
169     failure("Unable to export working copy")
170     sys.exit(1)
171    
172    
173    
174    
175     dir=os.getcwd()
176     for conf in testconfs:
177     progress("Creating "+conf.name+"_src")
178     # res=os.system("cp -r src "+conf.name+"_src")
179     # if res!=0:
180     # failure("Error copying src to "+conf.name+"_src")
181     try:
182     shutil.copytree("src",conf.name+"_src")
183     except Error:
184     failure("copying src to "+conf.name+"_src")
185     os.chdir(conf.name+"_src")
186     cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests"
187     progress(cmdstr)
188     res=os.system(cmdstr)
189     os.chdir(str(dir))
190     if res!=0:
191     failure("running scons build failed for "+conf.name+"_src")
192    
193     progress("Builds complete")
194     progress("Removing export copy")
195     shutil.rmtree("src",ignore_errors=True)
196     progress("Building test file")
197    
198     try:
199     testfile=open("dotests.sh","w")
200     testfile.write(TestConfiguration.getHeader())
201     for c in testconfs:
202     testfile.write(c.toString())
203     testfile.write(TestConfiguration.getFooter())
204     testfile.close()
205     import stat
206     os.chmod("dotests.sh",stat.S_IEXEC|stat.S_IREAD)
207     except IOError:
208     failure("Creating testfile")
209    
210     progress("Building test file complete")
211     progress("Copying files to exec area")
212     os.chdir(OUTSIDEDIR)
213 jfenwick 2270 #try:
214     #shutil.copytree(TOPDIR,EXECUTELOCATION+"/"+TOPDIR)
215     #except OSError:
216     # failure("copying to work area")
217     res=os.system("cp -r "+TOPDIR+" "+EXECUTELOCATION+"/"+TOPDIR)
218     if res!=0:
219     failure("copying work area")
220 jfenwick 2266 progress("Copy to exec area complete")
221    
222     print "Submitting test"
223    
224     ######### test section
225    
226     os.chdir(EXECUTELOCATION)
227     os.chdir(TOPDIR)
228    
229    
230     try:
231     # res=os.system("bash dotests.sh")
232     res=os.system("qsub -l select=1:ncpus=8:mem=31gb dotests.sh")
233     except OSError:
234     failure("Submitting tests")
235    
236     os.chdir(OUTSIDEDIR)
237    
238     print "Sleeping for "+str(TESTSLEEP)+" seconds to wait for results."
239     time.sleep(TESTSLEEP)
240     print "Waking up."
241    
242 jfenwick 2270 ######### end test section
243 jfenwick 2266
244     try:
245     shutil.copytree(EXECUTELOCATION+"/"+TOPDIR+"/Logs",OUTSIDEDIR+"/"+TOPDIR+"_Logs")
246     except OSError:
247     failure("Log copy failed")
248    
249     cleanupfailure=False
250    
251     try:
252     shutil.rmtree(EXECUTELOCATION+"/"+TOPDIR)
253     shutil.rmtree(OUTSIDEDIR+"/"+TOPDIR)
254     except OSError:
255     cleanupfailure=True
256    
257     #Now we sum up
258     #if Logs/Failure or Logs/Success does not exist send message about tests not completing.
259     if not os.path.exists(LOGDIR+"/Success") and not os.path.exists(LOGDIR+"/Failure"):
260     mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n"
261     mailcmd=mailcmd+"For some reason no Success or failure is recorded for unit tests in "+LOGDIR+"\n"
262     mailcmd=mailcmd+"\nAlso: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"\n"
263     mailcmd=mailcmd+SRCMSG
264     mailcmd=mailcmd+"ENDMSG\n"
265     os.system(mailcmd)
266     sys.exit(1)
267    
268    
269     if os.path.exists(LOGDIR+"/Failure"):
270     os.system("echo 'Also: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"' >> "+LOGDIR+"/message\n")
271     os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n")
272     mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests failed' "+ERRMAIL+"\n"
273     os.system(mailcmd)
274     sys.exit(1)
275     #so we must have succeeded
276    
277     if cleanupfailure:
278     os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n")
279     mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests cleanup failed - tests succeeded' "+ERRMAIL+"\n"
280     res=os.system(mailcmd)
281     sys.exit(res)
282     else:
283     os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n")
284     mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests succeeded' "+ERRMAIL+"\n"
285     res=os.system(mailcmd)
286     sys.exit(res)
287    

  ViewVC Help
Powered by ViewVC 1.1.26