--- trunk/autotest-scons 2005/12/13 05:17:32 344 +++ trunk/autotest-scons 2007/01/25 03:31:14 937 @@ -1,160 +1,135 @@ -#!/bin/bash +#!/bin/sh + +# +# Submit a PBS job to run the tests +# +# Usage: autotest-scons # Short testing, only C++ tests +# Usage: autotest-scons run_tests # Short testing, only C++ tests +# Usage: autotest-scons all_tests # Long testing, C++ and python tests +# +# Copy this somewhere and run it every day via cron +# +# This file should be maintained in SVN as esys13/trunk/autotest-scons, +# be sure to copy it there and commit after modification +# + +# Which tests should we run? +target='run_tests' # Default scons target that runs the tests +test "x$1" != "x" && target="$1" +requested_cpus=2 +test "x$target" = "xall_tests" && requested_cpus=4 + +MAIL_RECIPIENTS="l.gross@uq.edu.au matt@esscc.uq.edu.au robert.woodcock@csiro.au Peter.Hornby@csiro.au k.steube@uq.edu.au" +# MAIL_RECIPIENTS="k.steube@uq.edu.au" + +# This time is when the job was submitted. If it waits in the queue it might run on another day. +# This date is also used to create the sandbox directory where the tests are run. +RunDate=`date '+%Y_%m_%d'` # Time stamp for log file names + +WorkDir=/raid3/ksteube/AutoTests +cd $WorkDir + +# Where to put the output of scons run_tests +out_file="$WorkDir/Logs/$RunDate.test.output" + +# Save the name of this script +SCRIPT_NAME=$0 -# $Id: autotest-scons 162 2005-11-11 00:09:59Z svn $ -# An explicit testing script for esys using the scons build system +# Below here \$variable means I want the variable interpreted when PBS runs the job, not when the job is submitted +# Similarly for \`...\` + +# Write the PBS script to run the tests +cat << EOF > Logs/$RunDate.pbs.script.sh +#!/bin/bash -# list of users to email test results to -MAIL_RECIPIENTS="jgs@esscc.uq.edu.au gross@esscc.uq.edu.au cochrane@esscc.uq.edu.au elspeth@esscc.uq.edu.au matt@esscc.uq.edu.au" +#PBS -q q80 +#PBS -N autotest +#PBS -l ncpus=$requested_cpus +#PBS -o $RunDate.pbs.stdout +#PBS -e $RunDate.pbs.stderr +#PBS -W umask=022 + +echo "PBS_JOBNAME \$PBS_JOBNAME" +echo "PBS_JOBID \$PBS_JOBID" +echo "PBS_QUEUE \$PBS_QUEUE" +echo "NCPUS \$NCPUS" +echo "USER \$USER" +echo "PBS_O_HOST \$PBS_O_HOST" +echo "PBS_O_WORKDIR \$PBS_O_WORKDIR" +echo "PBS_O_SHELL \$PBS_O_SHELL" +date +echo "" +echo "" + + + +START=\`date '+%Y/%m/%d %H:%M'\` + +finish () { + date + # Clean up the sandbox + cd $WorkDir + /bin/rm -rf sandbox.$RunDate + END=\`date '+%Y/%m/%d %H:%M'\` + cat << END_MSG | mail -s "ESYS_TESTS $target $RunDate \$1" $MAIL_RECIPIENTS +\$2. +The tests ran from \$START to \$END on \$NCPUS CPUs +See the log files in $WorkDir/Logs/$RunDate* +This mail was sent by $SCRIPT_NAME +running as \$USER on \`hostname\`. +END_MSG + test "x\$1" = "FAILURE" && exit 1 + exit 0 +} + + +cd $WorkDir || finish FAILURE "Could not cd to WorkDir $WorkDir" + +# Create an empty out_file +cat /dev/null > $out_file || finish FAILURE "Could not create out_file $out_file" + +umask 022 + +test -d sandbox.$RunDate && finish FAILURE "Today's sandbox already exists" +mkdir sandbox.$RunDate || finish FAILURE "Could not mkdir sandbox" +cd sandbox.$RunDate || finish FAILURE "Could not cd to sandbox" + +echo "Checking out esys13/trunk" +svn checkout svn+ssh://shake200.esscc.uq.edu.au/home/www_svn/repos/esys13/trunk >> $out_file 2>&1 || finish FAILURE "Could not check out esys13/trunk" + +# Load modules +. /opt/modules/default/init/bash +module use /raid2/matt/modules/modulefiles +module use /raid2/toolspp4/modulefiles/gcc-3.3.6 +module use /usr/share/modules/modulefiles +module load esys/env # Recommended modules +module load doxygen/1.4.6 +module load boost/1.33.0/python-2.4.1 + +# How many threads? One per CPU. +export OMP_NUM_THREADS=\$NCPUS + +# Run the tests +echo "Running the tests $target" +cd trunk || finish FAILURE "Could not cd to trunk" +scons -j \$NCPUS $target >> $out_file 2>&1 || finish FAILURE "Could not run scons $target" + +echo "Cleaning up after the tests" + +# Delete files older than 21 days +find $WorkDir -atime +21 -exec rm -f {} \; + +finish SUCCESS "Successfully ran 'scons $target' on \`hostname\`" + +EOF + +# cd to the logs area so the PBS logs are deposited there +cd $WorkDir/Logs + +# Submit the job +. /opt/modules/default/init/sh +module load pbspro -# modules to test -TESTS="escript bruce finley" +# Submit the job +qsub -S /bin/bash $RunDate.pbs.script.sh > /dev/null -# the python tests to run -FINLEY_PYTESTS="ImportTest.py finleyTest.py SimpleSolve.py GradTest.py RecTest.py test_linearPDEsOnFinley.py test_generators.py test_visualization_interface.py test_utilOnFinley.py test_symbolsOnFinley.py" -ESCRIPT_PYTESTS="ImportTest.py BinaryOps.py UnaryOps.py SliceGetting.py SliceSetting.py MiscTests.py ArchiveTests.py escriptTest.py newEscriptTests.py DataVariableTests.py test_xml.py" -BRUCE_PYTESTS="ImportTest.py BruceTest.py test_utilOnBruce.py test_symbolsOnBruce.py" - -PWD=`pwd` -echo working in: $PWD - -# define compiler executable and library locations -export PATH=/opt/intel/cmplrs/80.058/intel_cc_80/bin:$PATH -export LD_LIBRARY_PATH=/opt/intel/cmplrs/80.058/intel_cc_80/lib - -# define python location -export PATH=/raid2/tools/python-2.3.4/bin:$PATH - -# set library and openmp settings -export LD_LIBRARY_PATH=$PWD/lib:/raid2/tools/boost/lib:$LD_LIBRARY_PATH -export PYTHONPATH=$PWD -export OMP_SCHEDULE="dynamic" -export OMP_NUM_THREADS=4 -export OMP_DYNAMIC=TRUE -export OMP_NESTED=FALSE - -FAIL=0 - -echo cd $1 -cd $1 -if [ $? != 0 ] -then - echo couldnt cd $1 - echo "couldnt cd $1" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - exit 1 -fi - -echo running svn update -svn update -if [ $? != 0 ] -then - echo svn update failed - echo "svn update failed" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - exit 2 -fi - -echo running scons debug=1 -scons debug=1 -if [ $? != 0 ] -then - echo scons failed - echo "scons failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - exit 3 -fi - -echo running bruce unit_tests -cd bruce/test -./unit_test-scons -if [ $? != 0 ] -then - echo bruce unit_test failed - echo "bruce unit_test failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - FAIL=1 -fi -cd ../.. - -echo running escript unit_tests -cd escript/test -./unit_test-scons -if [ $? != 0 ] -then - echo escript unit_test failed - echo "escript unit_test failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - FAIL=1 -fi -cd ../.. - -echo running finley unit_tests -cd finley/test -./unit_test-scons -if [ $? != 0 ] -then - echo finley unit_test failed - echo "finley unit_test failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - FAIL=1 -fi -cd ../.. - -echo running esysUtils unit_tests -cd esysUtils/test -./unit_test-scons -if [ $? != 0 ] -then - echo esysUtils unit_test failed - echo "esysUtils unit_test failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - FAIL=1 -fi -cd ../.. - -for module in $TESTS -do - echo "running python tests for: $module" - cd $module/test/python - if [ "$module" == "finley" ] - then - for pytest in $FINLEY_PYTESTS - do - echo "Running python test: $pytest" - python $pytest - if [ $? != 0 ] - then - echo Python Testing FAILED for $pytest in module $module - echo "finley py_test: $pytest failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - FAIL=1 - fi - done - elif [ "$module" == "escript" ] - then - for pytest in $ESCRIPT_PYTESTS - do - echo "Running python test: $pytest" - python $pytest - if [ $? != 0 ] - then - echo Python Testing FAILED for $pytest in module $module - echo "escript py_test: $pytest failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - FAIL=1 - fi - done - elif [ "$module" == "bruce" ] - then - for pytest in $BRUCE_PYTESTS - do - echo "Running python test: $pytest" - python $pytest - if [ $? != 0 ] - then - echo Python Testing FAILED for $pytest in module $module - echo "bruce py_test: $pytest failed - see autotest-scons logfile" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS - FAIL=1 - fi - done - fi - cd ../../.. -done - -if [ $FAIL == 0 ] -then - echo success - echo "success" | mail -s "esys autotest-scons results" $MAIL_RECIPIENTS -fi -exit 0