trunk/scripts/prepare.py


import shutil, os, datetime, sys


SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk"
NUMJs=4
TOPDIR=str(datetime.date.today())
ERRMAIL="j.fenwick1@uq.edu.au"

def failure(msg):
    print "Terminating - Error "+str(msg)
    print "Should be sending mail to "+str(ERRMAIL)

def progress(msg):
    print msg

class TestConfiguration(object):
    def __init__(self, name, opts, omp, mpi, binexec, pythonexec):
    self.name=name
    self.opts=opts
    self.omp=omp
    self.mpi=mpi
    self.binexec=binexec
    self.pythonexec=pythonexec

    def getHeader():
    res="#!/bin/bash\n"
    res=res+'MAIL_RECIPIENTS="'+ERRMAIL+'"\n'
    res=res+"function report()\n{\n"
    res=res+"   NOW=`date '+%Y/%m/%d %H:%M'`\n"
    res=res+"cat <<END_MSG | mail -s \"Esys tests `date` $TESTSTATE\" $MAIL_RECIPIENTS\n"
    res=res+"Sucessful configurations:\n"
    res=res+"$SUCCESSFUL\n\n"
    res=res+"Failed on configuration:\n"
    res=res+"$ATTEMPTED\n\n"
    res=res+"Tests ran from $START until $NOW.\n"
    res=res+"Log files can be found in $TOP.\n"
    res=res+"This mail was sent from $SCRIPTNAME, running as $USER on `hostname`.\n"
    res=res+"END_MSG\n"
    res=res+"}\n"
    res=res+"function progress()\n{\n"
    res=res+"  echo $1\n"
    res=res+"  cat $1 >> $LOGFILE\n"
    res=res+"}\n"
    res=res+"function failure()\n{\n  echo $1\n"
    res=res+"  report\n"
    res=res+"  exit 1\n}\nTOP=`pwd`\nLOGFILE=$TOP/Logs\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n"
    res=res+". /usr/share/modules/init/sh       #So the module command works\n"
    res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n"
    res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n"
    return res

    def toString(self):
    runcount=1
    ref="cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n" 
    ref=ref+"cd "+self.name+"_test"+str(runcount)+"\n"
    ref=ref+"TESTROOT=`pwd`\n"
    for o in self.omp:
        for m in self.mpi:
        cmd="bash utest.sh 'mpiexec -np"+str(m)+"' $TESTROOT/lib/pythonMPI"
        ref=ref+"export OMP_NUM_THREADS="+str(o)+"\n"
        ref=ref+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
        ref=ref+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
        ref=ref+'RUNNAME="'+self.name+' omp='+str(o)+' mpi='+str(m)+'"\n'
        ref=ref+'ATTEMPTING=$RUNNAME'
        ref=ref+'progress "Starting '+cmd+'"'
        ref=ref+cmd+' || failure "'+cmd+'"\n'
        ref=ref+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n'
        ref=ref+'completed "'+cmd+'"'
        ref=ref+'ATTEMPTING=None\n'
        ref=ref+"export OMP_NUM_THREADS=1\n"
        ++runcount
        if len(self.mpi)==0:
        cmd="bash utest.sh '' python"
        ref=ref+"export OMP_NUM_THREADS="+str(o)+"\n"
        ref=ref+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
        ref=ref+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
        ref=ref+'progress "Starting '+cmd+'"' 
        ref=ref+cmd+" || failure \""+cmd+"\" \n" 
        ref=ref+'completed "'+cmd+'"'
        ++runcount
    ref=ref+"\ncd $TOP\n\n"
    return ref
 
    getHeader=staticmethod(getHeader)


try:
    os.mkdir(TOPDIR)
    os.chdir(TOPDIR)
except OSError:
    failure("Unable to create top directory "+TOPDIR+" does it exist already?")
    sys.exit(1)

try:
    os.mkdir("Logs")
except OSError:
    failure("Unable to create Logs directory ")
    sys.exit(1)

coresult=os.system("svn export "+SVNURL+" src")
if coresult!=0:
    failure("Unable to export working copy")
    sys.exit(1)

testconfs=[]
testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python"))
testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI"))


dir=os.getcwd()
for conf in testconfs:
    progress("Creating "+conf.name+"_src")
# Yes I know copytree exists. No I don't trust it (It was having problems doing the copy).
    res=os.system("cp -r src "+conf.name+"_src")
    if res!=0:
    failure("Error copying src to "+conf.name+"_src")
#    shutil.copytree("src",conf.name+"_src")
    os.chdir(conf.name+"_src")
    cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests"
    progress(cmdstr)
    res=os.system(cmdstr)
    os.chdir(str(dir))
    if res!=0:
    failure("Error running scons build failed for "+conf.name+"_src")
    
progress("Builds complete")
#print "Removing export copy"
#shutil.rmtree("src",ignore_errors=True)
progress("Building test file")

try:
    testfile=open("dotests.sh","w")
    testfile.write(TestConfiguration.getHeader())
    for c in testconfs:
        testfile.write(c.toString())
    testfile.close()
except IOError:
    failure("Creating testfile")
    
progress("Building test file complete")

print "Should be submitting this test now"
raise "Test not submitted"
#Now we build the script and submit it pbs (that behaviour should be optional?)
1	jfenwick	2242
2			import shutil, os, datetime, sys
3
4
5			SVNURL="https://shake200.esscc.uq.edu.au/svn/esys13/trunk"
6			NUMJs=4
7			TOPDIR=str(datetime.date.today())
8			ERRMAIL="j.fenwick1@uq.edu.au"
9
10			def failure(msg):
11			print "Terminating - Error "+str(msg)
12			print "Should be sending mail to "+str(ERRMAIL)
13
14			def progress(msg):
15			print msg
16
17			class TestConfiguration(object):
18			def __init__(self, name, opts, omp, mpi, binexec, pythonexec):
19			self.name=name
20			self.opts=opts
21			self.omp=omp
22			self.mpi=mpi
23			self.binexec=binexec
24			self.pythonexec=pythonexec
25
26			def getHeader():
27			res="#!/bin/bash\n"
28			res=res+'MAIL_RECIPIENTS="'+ERRMAIL+'"\n'
29			res=res+"function report()\n{\n"
30			res=res+" NOW=`date '+%Y/%m/%d %H:%M'`\n"
31			res=res+"cat <<END_MSG \| mail -s \"Esys tests `date` $TESTSTATE\" $MAIL_RECIPIENTS\n"
32			res=res+"Sucessful configurations:\n"
33			res=res+"$SUCCESSFUL\n\n"
34			res=res+"Failed on configuration:\n"
35			res=res+"$ATTEMPTED\n\n"
36			res=res+"Tests ran from $START until $NOW.\n"
37			res=res+"Log files can be found in $TOP.\n"
38			res=res+"This mail was sent from $SCRIPTNAME, running as $USER on `hostname`.\n"
39			res=res+"END_MSG\n"
40			res=res+"}\n"
41			res=res+"function progress()\n{\n"
42			res=res+" echo $1\n"
43			res=res+" cat $1 >> $LOGFILE\n"
44			res=res+"}\n"
45			res=res+"function failure()\n{\n echo $1\n"
46			res=res+" report\n"
47			res=res+" exit 1\n}\nTOP=`pwd`\nLOGFILE=$TOP/Logs\nOLDPYTH=$PYTHONPATH\nOLDLD=$LD_LIBRARY_PATH\n"
48			res=res+". /usr/share/modules/init/sh #So the module command works\n"
49			res=res+"module load subversion-1.3.1\nmodule load escript/current\nmodule load pbs\nmodule load mayavi/gcc-4.1.2/mayavi-1.5\n"
50			res=res+"module load mplayer/gcc-4.1.2/mplayer-1.0rc2\n\n"
51			return res
52
53			def toString(self):
54			runcount=1
55			ref="cp -r "+self.name+"_src "+self.name+"_test"+str(runcount)+"\n"
56			ref=ref+"cd "+self.name+"_test"+str(runcount)+"\n"
57			ref=ref+"TESTROOT=`pwd`\n"
58			for o in self.omp:
59			for m in self.mpi:
60			cmd="bash utest.sh 'mpiexec -np"+str(m)+"' $TESTROOT/lib/pythonMPI"
61			ref=ref+"export OMP_NUM_THREADS="+str(o)+"\n"
62			ref=ref+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
63			ref=ref+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
64			ref=ref+'RUNNAME="'+self.name+' omp='+str(o)+' mpi='+str(m)+'"\n'
65			ref=ref+'ATTEMPTING=$RUNNAME'
66			ref=ref+'progress "Starting '+cmd+'"'
67			ref=ref+cmd+' \|\| failure "'+cmd+'"\n'
68			ref=ref+'SUCCESSFUL="$SUCCESSFUL, $RUNNAME"\n'
69			ref=ref+'completed "'+cmd+'"'
70			ref=ref+'ATTEMPTING=None\n'
71			ref=ref+"export OMP_NUM_THREADS=1\n"
72			++runcount
73			if len(self.mpi)==0:
74			cmd="bash utest.sh '' python"
75			ref=ref+"export OMP_NUM_THREADS="+str(o)+"\n"
76			ref=ref+"export LD_LIBRARY_PATH=`pwd`/lib:$OLDLD\n"
77			ref=ref+"export PYTHONPATH=`pwd`:$OLDPYTH\n"
78			ref=ref+'progress "Starting '+cmd+'"'
79			ref=ref+cmd+" \|\| failure \""+cmd+"\" \n"
80			ref=ref+'completed "'+cmd+'"'
81			++runcount
82			ref=ref+"\ncd $TOP\n\n"
83			return ref
84
85			getHeader=staticmethod(getHeader)
86
87
88			try:
89			os.mkdir(TOPDIR)
90			os.chdir(TOPDIR)
91			except OSError:
92			failure("Unable to create top directory "+TOPDIR+" does it exist already?")
93			sys.exit(1)
94
95			try:
96			os.mkdir("Logs")
97			except OSError:
98			failure("Unable to create Logs directory ")
99			sys.exit(1)
100
101			coresult=os.system("svn export "+SVNURL+" src")
102			if coresult!=0:
103			failure("Unable to export working copy")
104			sys.exit(1)
105
106			testconfs=[]
107			testconfs.append(TestConfiguration("OMPNoMPI","",omp=(1,8),mpi=(),binexec="",pythonexec="python"))
108			testconfs.append(TestConfiguration("MPI","usempi=yes",omp=(1,),mpi=(1,8),binexec="mpiexec -np ",pythonexec="lib/pythonMPI"))
109
110
111			dir=os.getcwd()
112			for conf in testconfs:
113			progress("Creating "+conf.name+"_src")
114			# Yes I know copytree exists. No I don't trust it (It was having problems doing the copy).
115			res=os.system("cp -r src "+conf.name+"_src")
116			if res!=0:
117			failure("Error copying src to "+conf.name+"_src")
118			# shutil.copytree("src",conf.name+"_src")
119			os.chdir(conf.name+"_src")
120			cmdstr="scons -j"+str(NUMJs)+" "+conf.opts+" install_all build_tests build_py_tests"
121			progress(cmdstr)
122			res=os.system(cmdstr)
123			os.chdir(str(dir))
124			if res!=0:
125			failure("Error running scons build failed for "+conf.name+"_src")
126
127			progress("Builds complete")
128			#print "Removing export copy"
129			#shutil.rmtree("src",ignore_errors=True)
130			progress("Building test file")
131
132			try:
133			testfile=open("dotests.sh","w")
134			testfile.write(TestConfiguration.getHeader())
135			for c in testconfs:
136			testfile.write(c.toString())
137			testfile.close()
138			except IOError:
139			failure("Creating testfile")
140
141			progress("Building test file complete")
142
143			print "Should be submitting this test now"
144			raise "Test not submitted"
145			#Now we build the script and submit it pbs (that behaviour should be optional?)