/[escript]/trunk/scripts/prepare.py
ViewVC logotype

Diff of /trunk/scripts/prepare.py

Parent Directory Parent Directory | Revision Log Revision Log | View Patch Patch

revision 2249 by jfenwick, Fri Feb 6 00:11:51 2009 UTC revision 2250 by jfenwick, Fri Feb 6 06:25:21 2009 UTC
# Line 1  Line 1 
1    
2  import shutil, os, datetime, sys  import shutil, os, datetime, sys, os.path, time
3    
4    #This script does not use the python, platform independent path manipulation stuff.
5    #It probably should
6    
7  SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk"  SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk"
8  NUMJs=4  NUMJs=4
# Line 8  TOPDIR=str(datetime.date.today()) Line 10  TOPDIR=str(datetime.date.today())
10  ERRMAIL="j.fenwick1@uq.edu.au"  ERRMAIL="j.fenwick1@uq.edu.au"
11  EXECUTELOCATION="/scratch/jfenwick/AUTOTESTS"  EXECUTELOCATION="/scratch/jfenwick/AUTOTESTS"
12  OUTSIDEDIR=os.getcwd()  OUTSIDEDIR=os.getcwd()
13    TESTSLEEP=30*60
14    
15  #Settings for actual tests appear below the class declarations  #Settings for actual tests appear below the class declarations
16    
# Line 15  OUTSIDEDIR=os.getcwd() Line 18  OUTSIDEDIR=os.getcwd()
18  def failure(msg):  def failure(msg):
19      print "Terminating - Error "+str(msg)      print "Terminating - Error "+str(msg)
20      print "Should be sending mail to "+str(ERRMAIL)      print "Should be sending mail to "+str(ERRMAIL)
21        mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n"
22        mailcmd=mailcmd+"Error preparing for test run:\n"+msg+"\n"
23        mailcmd=mailcmd+"ENDMSG\n"
24        os.system(mailcmd)
25      sys.exit(1)      sys.exit(1)
26    
27  def progress(msg):  def progress(msg):
# Line 40  class TestConfiguration(object): Line 47  class TestConfiguration(object):
47      res=res+"Failed on configuration:\n"      res=res+"Failed on configuration:\n"
48      res=res+"$ATTEMPTING\n\n"      res=res+"$ATTEMPTING\n\n"
49      res=res+"Tests ran from $START until $NOW.\n"      res=res+"Tests ran from $START until $NOW.\n"
50      res=res+"Log files can be found in $TOP.\n"      res=res+"Log files can be found in $FINALLOGDIR.\n"
51      res=res+"This mail was sent from $SCRIPTNAME, running as $USER on `hostname`.\n"      res=res+"This mail was sent from $SCRIPTNAME, running as $USER on `hostname`.\n"
52      res=res+"END_MSG\n"      res=res+"END_MSG\n"
53      res=res+"}\n"      res=res+"}\n"
# Line 51  class TestConfiguration(object): Line 58  class TestConfiguration(object):
58      res=res+"function failure()\n{\n  echo $1\n"      res=res+"function failure()\n{\n  echo $1\n"
59      res=res+"  report\n"      res=res+"  report\n"
60      res=res+"  touch $LOGDIR/Failure\n"      res=res+"  touch $LOGDIR/Failure\n"
61      res=res+"  exit 1\n}\nTOP=`pwd`\nLOGDIR=$TOP/Logs\nPROGRESSFILE=$LOGDIR/progress\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n"      res=res+"  if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
62        res=res+"  exit 1\n}\n"
63        res=res+"cd "+EXECUTELOCATION+"/"+TOPDIR+"\n"
64        res=res+"TOP=`pwd`\nLOGDIR=$TOP/Logs\nPROGRESSFILE=$LOGDIR/progress\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n"
65      res=res+". /usr/share/modules/init/sh       #So the module command works\n"      res=res+". /usr/share/modules/init/sh       #So the module command works\n"
66      res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n"      res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n"
67      res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n"      res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n"
68      res=res+"SCRIPTNAME=$0\n"      res=res+"SCRIPTNAME=$0\n"
69        res=res+"START=`date '+%Y/%m/%d %H:%M'`\n"
70        res=res+"TESTLOGDIR=$LOGDIR\n"
71        res=res+"FINALLOGDIR="+OUTSIDEDIR+"/"+TOPDIR+"_Logs\n"
72      return res      return res
73    
74      def toString(self):      def toString(self):
75      runcount=1      runcount=1
76        res=""
77        print "Processing "+self.name
78      for o in self.omp:      for o in self.omp:
79          for m in self.mpi:                print "o="+str(o)
80            for m in self.mpi:
81            print "   m="+str(m)        
82          cmd="bash utest.sh 'mpiexec -np"+str(m)+"' $TESTROOT/lib/pythonMPI  >$TESTLOGDIR/output 2>&1"          cmd="bash utest.sh 'mpiexec -np"+str(m)+"' $TESTROOT/lib/pythonMPI  >$TESTLOGDIR/output 2>&1"
83          res="cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"          res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
84          res=res+"cd "+self.name+"_test"+str(runcount)+"\n"          res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
85          res=res+"TESTROOT=`pwd`\n"          res=res+"TESTROOT=`pwd`\n"
86          res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"          res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
# Line 75  class TestConfiguration(object): Line 92  class TestConfiguration(object):
92          res=res+'ATTEMPTING=$RUNNAME\n'          res=res+'ATTEMPTING=$RUNNAME\n'
93          res=res+'progress "Starting '+cmd+'"\n'          res=res+'progress "Starting '+cmd+'"\n'
94          res=res+cmd+' || failure "'+cmd+'"\n'          res=res+cmd+' || failure "'+cmd+'"\n'
95            res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
96          res=res+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n'          res=res+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n'
97          res=res+'progress "completed '+cmd+'"\n'          res=res+'progress "completed '+cmd+'"\n'
98          res=res+'ATTEMPTING=None\n'          res=res+'ATTEMPTING=None\n'
99          res=res+"export OMP_NUM_THREADS=1\n"          res=res+"export OMP_NUM_THREADS=1\n"
100          res=res+"cd $TOP\n"          res=res+"cd $TOP\n"
101          res=res+"rm -rf "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"          res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n"
102          ++runcount          runcount=runcount+1
103          if len(self.mpi)==0:          if len(self.mpi)==0:
104            print "   m=()"
105          cmd="bash utest.sh '' python $TESTROOT/lib/pythonMPI  >$TESTLOGDIR/output 2>&1"          cmd="bash utest.sh '' python $TESTROOT/lib/pythonMPI  >$TESTLOGDIR/output 2>&1"
106          res="cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"          res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
107          res=res+"cd "+self.name+"_test"+str(runcount)+"\n"          res=res+"cd "+self.name+"_test"+str(runcount)+"\n"
108          res=res+"TESTROOT=`pwd`\n"          res=res+"TESTROOT=`pwd`\n"
109          res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"          res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n"
# Line 96  class TestConfiguration(object): Line 115  class TestConfiguration(object):
115          res=res+'ATTEMPTING=$RUNNAME\n'          res=res+'ATTEMPTING=$RUNNAME\n'
116          res=res+'progress "Starting '+cmd+'"\n'          res=res+'progress "Starting '+cmd+'"\n'
117          res=res+cmd+" || failure \""+cmd+"\" \n"          res=res+cmd+" || failure \""+cmd+"\" \n"
118            res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n"
119          res=res+'ATTEMPTING=None\n'          res=res+'ATTEMPTING=None\n'
120          res=res+'progress "completed '+cmd+'"\n'          res=res+'progress "completed '+cmd+'"\n'
121          res=res+"cd $TOP\n"          res=res+"cd $TOP\n"
122          res=res+"rm -rf "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"          res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n"
123          ++runcount          runcount=runcount+1
124        res=res+"rm -rf "+self.name+"_src\n"
125      res=res+"\ncd $TOP\n\n"      res=res+"\ncd $TOP\n\n"
126      return res      return res
127    
# Line 117  testconfs=[] Line 138  testconfs=[]
138  testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python"))  testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python"))
139  testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI"))  testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI"))
140    
141    LOGDIR=OUTSIDEDIR+"/"+TOPDIR+"_Logs"
142    
143    if os.path.exists(LOGDIR):
144        failure("Logs directory for "+TOPDIR+" already exists.")
145        sys.exit(1)
146    
147  try:  try:
148      os.mkdir(TOPDIR)      os.mkdir(TOPDIR)
# Line 142  if coresult!=0: Line 168  if coresult!=0:
168  dir=os.getcwd()  dir=os.getcwd()
169  for conf in testconfs:  for conf in testconfs:
170      progress("Creating "+conf.name+"_src")      progress("Creating "+conf.name+"_src")
171  # Yes I know copytree exists. No I don't trust it (It was having problems doing the copy).  #    res=os.system("cp -r src "+conf.name+"_src")
172      res=os.system("cp -r src "+conf.name+"_src")  #    if res!=0:
173      if res!=0:  #   failure("Error copying src to "+conf.name+"_src")
174      failure("Error copying src to "+conf.name+"_src")      try:
175  #    shutil.copytree("src",conf.name+"_src")          shutil.copytree("src",conf.name+"_src")
176        except Error:
177        failure("copying src to "+conf.name+"_src")
178      os.chdir(conf.name+"_src")      os.chdir(conf.name+"_src")
179      cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests"      cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests"
180      progress(cmdstr)      progress(cmdstr)
# Line 165  try: Line 193  try:
193      testfile.write(TestConfiguration.getHeader())      testfile.write(TestConfiguration.getHeader())
194      for c in testconfs:      for c in testconfs:
195          testfile.write(c.toString())          testfile.write(c.toString())
196        testfile.write(TestConfiguration.getFooter())
197      testfile.close()      testfile.close()
198        import stat
199        os.chmod("dotests.sh",stat.S_IEXEC|stat.S_IREAD)
200  except IOError:  except IOError:
201      failure("Creating testfile")      failure("Creating testfile")
202        
203  progress("Building test file complete")  progress("Building test file complete")
204  progress("Copying files to exec area")  progress("Copying files to exec area")
205  os.chdir(OUTSIDEDIR)  os.chdir(OUTSIDEDIR)
206  try:  try:
207      shutil.copytree(TOPDIR,EXECUTELOCATION)      shutil.copytree(TOPDIR,EXECUTELOCATION+"/"+TOPDIR)
208  except IOError, OSError:  except Error:
209      failure("copying to work area")      failure("copying to work area")
210  progress("Copy to exec area complete")  progress("Copy to exec area complete")
211    
212  print "Should be submitting this test now"  print "Submitting test"
213    
214    ######### test section
215    
216  os.chdir(EXECUTELOCATION)  os.chdir(EXECUTELOCATION)
217  os.chdir(TOPDIR)  os.chdir(TOPDIR)
218    
219    
220  try:  try:
221      res=os.system("bash dotests.sh")  #   res=os.system("bash dotests.sh")
222        res=os.system("qsub -l select=1:ncpus=8:mem=31gb dotests.sh")
223  except OSError:  except OSError:
224      failure("Running tests")      failure("Submitting tests")
225    
226    os.chdir(OUTSIDEDIR)
227    
228    print "Sleeping for "+str(TESTSLEEP)+" seconds to wait for results."
229    time.sleep(TESTSLEEP)
230    print "Waking up."
231    
232    ########################
233    
234  try:  try:
235      shutil.copytree(EXECUTELOCATION+"/Logs",OUTSIDEDIR)      shutil.copytree(EXECUTELOCATION+"/"+TOPDIR+"/Logs",OUTSIDEDIR+"/"+TOPDIR+"_Logs")
236  except OSError:  except OSError:
237      failure("Log copy failed")      failure("Log copy failed")
238    
239    cleanupfailure=False
240    
241    try:
242        shutil.rmtree(EXECUTELOCATION+"/"+TOPDIR)
243        shutil.rmtree(OUTSIDEDIR+"/"+TOPDIR)
244    except OSError:
245        cleanupfailure=True
246    
247    #Now we sum up
248    #if Logs/Failure or Logs/Success does not exist send message about tests not completing.
249    if not os.path.exists(LOGDIR+"/Success") and  not os.path.exists(LOGDIR+"/Failure"):
250        mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n"
251        mailcmd=mailcmd+"For some reason no Success or failure is recorded for unit tests in "+LOGDIR+"\n"
252        mailcmd=mailcmd+"\nAlso: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"\n"
253        mailcmd=mailcmd+"ENDMSG\n"
254        os.system(mailcmd)
255        sys.exit(1)
256    
257    
258    if os.path.exists(LOGDIR+"/Failure"):
259        os.system("echo 'Also: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"' >> "+LOGDIR+"/message\n")
260        mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests failed' "+ERRMAIL+"\n"
261        os.system(mailcmd)
262        sys.exit(1)
263    #so we must have succeeded
264    
265    if cleanupfailure:
266        mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests cleanup failed - tests succeeded' "+ERRMAIL+"\n"
267        res=os.system(mailcmd)
268        sys.exit(res)
269    
 raise "Test not submitted - not cleaned up either"  
 #Now we build the script and submit it pbs (that behaviour should be optional?)  

Legend:
Removed from v.2249  
changed lines
  Added in v.2250

  ViewVC Help
Powered by ViewVC 1.1.26