| 1 |
|
| 2 |
import shutil, os, datetime, sys, os.path, time |
| 3 |
|
| 4 |
#This script does not use the python, platform independent path manipulation stuff. |
| 5 |
#It probably should |
| 6 |
|
| 7 |
SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk" |
| 8 |
NUMJs=4 |
| 9 |
TOPDIR=str(datetime.date.today()) |
| 10 |
ERRMAIL="j.fenwick1@uq.edu.au" |
| 11 |
EXECUTELOCATION="/scratch/jfenwick/AUTOTESTS" |
| 12 |
OUTSIDEDIR=os.getcwd() |
| 13 |
TESTSLEEP=30*60 |
| 14 |
|
| 15 |
SRCMSG="This message was sent by prepare.py running as "+str(os.environ['USER'])+" on "+str(os.environ['HOSTNAME']+"\n") |
| 16 |
|
| 17 |
#Settings for actual tests appear below the class declarations |
| 18 |
|
| 19 |
|
| 20 |
def failure(msg): |
| 21 |
print "Terminating - Error "+str(msg) |
| 22 |
print "Should be sending mail to "+str(ERRMAIL) |
| 23 |
mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n" |
| 24 |
mailcmd=mailcmd+"Error preparing for test run:\n"+msg+"\n" |
| 25 |
mailcmd=mailcmd+SRCMSG |
| 26 |
mailcmd=mailcmd+"ENDMSG\n" |
| 27 |
os.system(mailcmd) |
| 28 |
sys.exit(1) |
| 29 |
|
| 30 |
def progress(msg): |
| 31 |
print msg |
| 32 |
|
| 33 |
class TestConfiguration(object): |
| 34 |
def __init__(self, name, opts, omp, mpi, binexec, pythonexec): |
| 35 |
self.name=name |
| 36 |
self.opts=opts |
| 37 |
self.omp=omp |
| 38 |
self.mpi=mpi |
| 39 |
self.binexec=binexec |
| 40 |
self.pythonexec=pythonexec |
| 41 |
|
| 42 |
def getHeader(): |
| 43 |
res="#!/bin/bash\n" |
| 44 |
res=res+'MAIL_RECIPIENTS="'+ERRMAIL+'"\n' |
| 45 |
res=res+"function report()\n{\n" |
| 46 |
res=res+" NOW=`date '+%Y/%m/%d %H:%M'`\n" |
| 47 |
res=res+" cat > $LOGDIR/message << END_MSG\n" |
| 48 |
res=res+"Sucessful configurations:\n" |
| 49 |
res=res+"$SUCCESSFUL\n\n" |
| 50 |
res=res+"Failed on configuration:\n" |
| 51 |
res=res+"$ATTEMPTING\n\n" |
| 52 |
res=res+"Tests ran from $START until $NOW.\n" |
| 53 |
res=res+"Log files can be found in $FINALLOGDIR.\n" |
| 54 |
res=res+"END_MSG\n" |
| 55 |
res=res+"}\n" |
| 56 |
res=res+"function progress()\n{\n" |
| 57 |
res=res+" echo $1\n" |
| 58 |
res=res+" echo $1 >> $PROGRESSFILE\n" |
| 59 |
res=res+"}\n" |
| 60 |
res=res+"function failure()\n{\n echo $1\n" |
| 61 |
res=res+" report\n" |
| 62 |
res=res+" touch $LOGDIR/Failure\n" |
| 63 |
res=res+" if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n" |
| 64 |
res=res+" exit 1\n}\n" |
| 65 |
res=res+"cd "+EXECUTELOCATION+"/"+TOPDIR+"\n" |
| 66 |
res=res+"TOP=`pwd`\nLOGDIR=$TOP/Logs\nPROGRESSFILE=$LOGDIR/progress\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n" |
| 67 |
res=res+". /usr/share/modules/init/sh #So the module command works\n" |
| 68 |
res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n" |
| 69 |
res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n" |
| 70 |
res=res+"SCRIPTNAME=$0\n" |
| 71 |
res=res+"START=`date '+%Y/%m/%d %H:%M'`\n" |
| 72 |
res=res+"TESTLOGDIR=$LOGDIR\n" |
| 73 |
res=res+"FINALLOGDIR="+OUTSIDEDIR+"/"+TOPDIR+"_Logs\n" |
| 74 |
return res |
| 75 |
|
| 76 |
def toString(self): |
| 77 |
runcount=1 |
| 78 |
res="" |
| 79 |
print "Processing "+self.name |
| 80 |
for o in self.omp: |
| 81 |
print "o="+str(o) |
| 82 |
for m in self.mpi: |
| 83 |
print " m="+str(m) |
| 84 |
cmd="bash utest.sh 'mpiexec -np"+str(m)+"' $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1" |
| 85 |
res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n" |
| 86 |
res=res+"cd "+self.name+"_test"+str(runcount)+"\n" |
| 87 |
res=res+"TESTROOT=`pwd`\n" |
| 88 |
res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n" |
| 89 |
res=res+"mkdir $TESTLOGDIR\n" |
| 90 |
res=res+"export OMP_NUM_THREADS="+str(o)+"\n" |
| 91 |
res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n" |
| 92 |
res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n" |
| 93 |
res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi='+str(m)+'"\n' |
| 94 |
res=res+'ATTEMPTING=$RUNNAME\n' |
| 95 |
res=res+'progress "Starting '+cmd+'"\n' |
| 96 |
res=res+cmd+' || failure "'+cmd+'"\n' |
| 97 |
res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n" |
| 98 |
res=res+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n' |
| 99 |
res=res+'progress "completed '+cmd+'"\n' |
| 100 |
res=res+'ATTEMPTING=None\n' |
| 101 |
res=res+"export OMP_NUM_THREADS=1\n" |
| 102 |
res=res+"cd $TOP\n" |
| 103 |
res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n" |
| 104 |
runcount=runcount+1 |
| 105 |
if len(self.mpi)==0: |
| 106 |
print " m=()" |
| 107 |
cmd="bash utest.sh '' python $TESTROOT/lib/pythonMPI >$TESTLOGDIR/output 2>&1" |
| 108 |
res=res+"cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n" |
| 109 |
res=res+"cd "+self.name+"_test"+str(runcount)+"\n" |
| 110 |
res=res+"TESTROOT=`pwd`\n" |
| 111 |
res=res+"TESTLOGDIR=$LOGDIR/"+self.name+"_test"+str(runcount)+"\n" |
| 112 |
res=res+"mkdir $TESTLOGDIR\n" |
| 113 |
res=res+"export OMP_NUM_THREADS="+str(o)+"\n" |
| 114 |
res=res+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n" |
| 115 |
res=res+"export PYTHONPATH=`pwd`:$OLDPYTH\n" |
| 116 |
res=res+'RUNNAME="'+self.name+' omp='+str(o)+' mpi=n/a"\n' |
| 117 |
res=res+'ATTEMPTING=$RUNNAME\n' |
| 118 |
res=res+'progress "Starting '+cmd+'"\n' |
| 119 |
res=res+cmd+" || failure \""+cmd+"\" \n" |
| 120 |
res=res+"if [ -f stdout_cpu_0001.out ];then cp std_cpu_* $TESTLOGDIR;fi\n" |
| 121 |
res=res+'ATTEMPTING=None\n' |
| 122 |
res=res+'progress "completed '+cmd+'"\n' |
| 123 |
res=res+"cd $TOP\n" |
| 124 |
res=res+"rm -rf "+self.name+"_test"+str(runcount)+"\n\n" |
| 125 |
runcount=runcount+1 |
| 126 |
res=res+"rm -rf "+self.name+"_src\n" |
| 127 |
res=res+"\ncd $TOP\n\n" |
| 128 |
return res |
| 129 |
|
| 130 |
def getFooter(): |
| 131 |
res="\ntouch $LOGDIR/Success\n" |
| 132 |
res=res+"report" |
| 133 |
return res |
| 134 |
|
| 135 |
getHeader=staticmethod(getHeader) |
| 136 |
getFooter=staticmethod(getFooter) |
| 137 |
|
| 138 |
#Test settings |
| 139 |
testconfs=[] |
| 140 |
testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python")) |
| 141 |
testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI")) |
| 142 |
|
| 143 |
LOGDIR=OUTSIDEDIR+"/"+TOPDIR+"_Logs" |
| 144 |
|
| 145 |
if os.path.exists(LOGDIR): |
| 146 |
failure("Logs directory for "+TOPDIR+" already exists.") |
| 147 |
sys.exit(1) |
| 148 |
|
| 149 |
try: |
| 150 |
os.mkdir(TOPDIR) |
| 151 |
os.chdir(TOPDIR) |
| 152 |
except OSError: |
| 153 |
failure("Unable to create top directory "+TOPDIR+" does it exist already?") |
| 154 |
sys.exit(1) |
| 155 |
|
| 156 |
try: |
| 157 |
os.mkdir("Logs") |
| 158 |
except OSError: |
| 159 |
failure("Unable to create Logs directory ") |
| 160 |
sys.exit(1) |
| 161 |
|
| 162 |
coresult=os.system("svn export "+SVNURL+" src") |
| 163 |
if coresult!=0: |
| 164 |
failure("Unable to export working copy") |
| 165 |
sys.exit(1) |
| 166 |
|
| 167 |
|
| 168 |
|
| 169 |
|
| 170 |
dir=os.getcwd() |
| 171 |
for conf in testconfs: |
| 172 |
progress("Creating "+conf.name+"_src") |
| 173 |
# res=os.system("cp -r src "+conf.name+"_src") |
| 174 |
# if res!=0: |
| 175 |
# failure("Error copying src to "+conf.name+"_src") |
| 176 |
try: |
| 177 |
shutil.copytree("src",conf.name+"_src") |
| 178 |
except Error: |
| 179 |
failure("copying src to "+conf.name+"_src") |
| 180 |
os.chdir(conf.name+"_src") |
| 181 |
cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests" |
| 182 |
progress(cmdstr) |
| 183 |
res=os.system(cmdstr) |
| 184 |
os.chdir(str(dir)) |
| 185 |
if res!=0: |
| 186 |
failure("running scons build failed for "+conf.name+"_src") |
| 187 |
|
| 188 |
progress("Builds complete") |
| 189 |
progress("Removing export copy") |
| 190 |
shutil.rmtree("src",ignore_errors=True) |
| 191 |
progress("Building test file") |
| 192 |
|
| 193 |
try: |
| 194 |
testfile=open("dotests.sh","w") |
| 195 |
testfile.write(TestConfiguration.getHeader()) |
| 196 |
for c in testconfs: |
| 197 |
testfile.write(c.toString()) |
| 198 |
testfile.write(TestConfiguration.getFooter()) |
| 199 |
testfile.close() |
| 200 |
import stat |
| 201 |
os.chmod("dotests.sh",stat.S_IEXEC|stat.S_IREAD) |
| 202 |
except IOError: |
| 203 |
failure("Creating testfile") |
| 204 |
|
| 205 |
progress("Building test file complete") |
| 206 |
progress("Copying files to exec area") |
| 207 |
os.chdir(OUTSIDEDIR) |
| 208 |
try: |
| 209 |
shutil.copytree(TOPDIR,EXECUTELOCATION+"/"+TOPDIR) |
| 210 |
except Error: |
| 211 |
failure("copying to work area") |
| 212 |
progress("Copy to exec area complete") |
| 213 |
|
| 214 |
print "Submitting test" |
| 215 |
|
| 216 |
######### test section |
| 217 |
|
| 218 |
os.chdir(EXECUTELOCATION) |
| 219 |
os.chdir(TOPDIR) |
| 220 |
|
| 221 |
|
| 222 |
try: |
| 223 |
# res=os.system("bash dotests.sh") |
| 224 |
res=os.system("qsub -l select=1:ncpus=8:mem=31gb dotests.sh") |
| 225 |
except OSError: |
| 226 |
failure("Submitting tests") |
| 227 |
|
| 228 |
os.chdir(OUTSIDEDIR) |
| 229 |
|
| 230 |
print "Sleeping for "+str(TESTSLEEP)+" seconds to wait for results." |
| 231 |
time.sleep(TESTSLEEP) |
| 232 |
print "Waking up." |
| 233 |
|
| 234 |
######################## |
| 235 |
|
| 236 |
try: |
| 237 |
shutil.copytree(EXECUTELOCATION+"/"+TOPDIR+"/Logs",OUTSIDEDIR+"/"+TOPDIR+"_Logs") |
| 238 |
except OSError: |
| 239 |
failure("Log copy failed") |
| 240 |
|
| 241 |
cleanupfailure=False |
| 242 |
|
| 243 |
try: |
| 244 |
shutil.rmtree(EXECUTELOCATION+"/"+TOPDIR) |
| 245 |
shutil.rmtree(OUTSIDEDIR+"/"+TOPDIR) |
| 246 |
except OSError: |
| 247 |
cleanupfailure=True |
| 248 |
|
| 249 |
#Now we sum up |
| 250 |
#if Logs/Failure or Logs/Success does not exist send message about tests not completing. |
| 251 |
if not os.path.exists(LOGDIR+"/Success") and not os.path.exists(LOGDIR+"/Failure"): |
| 252 |
mailcmd="cat << ENDMSG |mail -s 'Esys unit tests failed to execute properly' "+ERRMAIL+"\n" |
| 253 |
mailcmd=mailcmd+"For some reason no Success or failure is recorded for unit tests in "+LOGDIR+"\n" |
| 254 |
mailcmd=mailcmd+"\nAlso: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"\n" |
| 255 |
mailcmd=mailcmd+SRCMSG |
| 256 |
mailcmd=mailcmd+"ENDMSG\n" |
| 257 |
os.system(mailcmd) |
| 258 |
sys.exit(1) |
| 259 |
|
| 260 |
|
| 261 |
if os.path.exists(LOGDIR+"/Failure"): |
| 262 |
os.system("echo 'Also: the cleanup of work areas failed. "+EXECUTELOCATION+"/"+TOPDIR+" or "+OUTSIDEDIR+"/"+TOPDIR+"' >> "+LOGDIR+"/message\n") |
| 263 |
os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n") |
| 264 |
mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests failed' "+ERRMAIL+"\n" |
| 265 |
os.system(mailcmd) |
| 266 |
sys.exit(1) |
| 267 |
#so we must have succeeded |
| 268 |
|
| 269 |
if cleanupfailure: |
| 270 |
os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n") |
| 271 |
mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests cleanup failed - tests succeeded' "+ERRMAIL+"\n" |
| 272 |
res=os.system(mailcmd) |
| 273 |
sys.exit(res) |
| 274 |
else: |
| 275 |
os.system("echo '"+SRCMSG+"' >> "+LOGDIR+"/message\n") |
| 276 |
mailcmd="cat "+LOGDIR+"/message | mail -s 'Esys unit tests succeeded' "+ERRMAIL+"\n" |
| 277 |
res=os.system(mailcmd) |
| 278 |
sys.exit(res) |
| 279 |
|
Parent Directory
| 
Revision Log