1 |
ksteube |
774 |
#!/bin/sh |
2 |
|
|
|
3 |
gross |
707 |
# |
4 |
ksteube |
774 |
# Submit a PBS job to run the tests |
5 |
gross |
707 |
# |
6 |
ksteube |
774 |
# Usage: autotest-scons # Short testing, only C++ tests |
7 |
|
|
# Usage: autotest-scons run_tests # Short testing, only C++ tests |
8 |
|
|
# Usage: autotest-scons all_tests # Long testing, C++ and python tests |
9 |
gross |
707 |
# |
10 |
ksteube |
774 |
# Copy this somewhere and run it every day via cron |
11 |
gross |
707 |
# |
12 |
ksteube |
774 |
# This file should be maintained in SVN as esys13/trunk/autotest-scons, |
13 |
|
|
# be sure to copy it there and commit after modification |
14 |
|
|
# |
15 |
jgs |
318 |
|
16 |
ksteube |
774 |
# Which tests should we run? |
17 |
|
|
target='run_tests' # Default scons target that runs the tests |
18 |
|
|
test "x$1" != "x" && target="$1" |
19 |
|
|
requested_cpus=2 |
20 |
|
|
test "x$target" = "xall_tests" && requested_cpus=4 |
21 |
jgs |
318 |
|
22 |
ksteube |
937 |
MAIL_RECIPIENTS="l.gross@uq.edu.au matt@esscc.uq.edu.au robert.woodcock@csiro.au Peter.Hornby@csiro.au k.steube@uq.edu.au" |
23 |
|
|
# MAIL_RECIPIENTS="k.steube@uq.edu.au" |
24 |
jgs |
321 |
|
25 |
ksteube |
937 |
# This time is when the job was submitted. If it waits in the queue it might run on another day. |
26 |
|
|
# This date is also used to create the sandbox directory where the tests are run. |
27 |
ksteube |
774 |
RunDate=`date '+%Y_%m_%d'` # Time stamp for log file names |
28 |
elspeth |
721 |
|
29 |
ksteube |
774 |
WorkDir=/raid3/ksteube/AutoTests |
30 |
|
|
cd $WorkDir |
31 |
svn |
346 |
|
32 |
ksteube |
774 |
# Where to put the output of scons run_tests |
33 |
|
|
out_file="$WorkDir/Logs/$RunDate.test.output" |
34 |
elspeth |
721 |
|
35 |
ksteube |
774 |
# Save the name of this script |
36 |
|
|
SCRIPT_NAME=$0 |
37 |
jgs |
321 |
|
38 |
ksteube |
774 |
# Below here \$variable means I want the variable interpreted when PBS runs the job, not when the job is submitted |
39 |
|
|
# Similarly for \`...\` |
40 |
|
|
|
41 |
|
|
# Write the PBS script to run the tests |
42 |
|
|
cat << EOF > Logs/$RunDate.pbs.script.sh |
43 |
|
|
#!/bin/bash |
44 |
|
|
|
45 |
ksteube |
937 |
#PBS -q q80 |
46 |
ksteube |
774 |
#PBS -N autotest |
47 |
|
|
#PBS -l ncpus=$requested_cpus |
48 |
|
|
#PBS -o $RunDate.pbs.stdout |
49 |
|
|
#PBS -e $RunDate.pbs.stderr |
50 |
|
|
#PBS -W umask=022 |
51 |
|
|
|
52 |
|
|
echo "PBS_JOBNAME \$PBS_JOBNAME" |
53 |
|
|
echo "PBS_JOBID \$PBS_JOBID" |
54 |
|
|
echo "PBS_QUEUE \$PBS_QUEUE" |
55 |
|
|
echo "NCPUS \$NCPUS" |
56 |
|
|
echo "USER \$USER" |
57 |
|
|
echo "PBS_O_HOST \$PBS_O_HOST" |
58 |
|
|
echo "PBS_O_WORKDIR \$PBS_O_WORKDIR" |
59 |
|
|
echo "PBS_O_SHELL \$PBS_O_SHELL" |
60 |
|
|
date |
61 |
|
|
echo "" |
62 |
|
|
echo "" |
63 |
|
|
|
64 |
|
|
|
65 |
|
|
|
66 |
|
|
START=\`date '+%Y/%m/%d %H:%M'\` |
67 |
|
|
|
68 |
|
|
finish () { |
69 |
ksteube |
979 |
# state will be 'FAILURE' or 'SUCCESS' |
70 |
|
|
state="\$1" |
71 |
ksteube |
774 |
date |
72 |
|
|
# Clean up the sandbox |
73 |
|
|
cd $WorkDir |
74 |
|
|
/bin/rm -rf sandbox.$RunDate |
75 |
|
|
END=\`date '+%Y/%m/%d %H:%M'\` |
76 |
ksteube |
979 |
cat << END_MSG | mail -s "ESYS_TESTS $target $RunDate \$state" $MAIL_RECIPIENTS |
77 |
ksteube |
774 |
\$2. |
78 |
|
|
The tests ran from \$START to \$END on \$NCPUS CPUs |
79 |
|
|
See the log files in $WorkDir/Logs/$RunDate* |
80 |
|
|
This mail was sent by $SCRIPT_NAME |
81 |
|
|
running as \$USER on \`hostname\`. |
82 |
|
|
END_MSG |
83 |
ksteube |
979 |
if [ "x\$state" = "xFAILURE" ]; then |
84 |
|
|
touch Logs/$RunDate.FAILURE |
85 |
|
|
exit 1 |
86 |
|
|
fi |
87 |
ksteube |
774 |
exit 0 |
88 |
|
|
} |
89 |
|
|
|
90 |
|
|
|
91 |
|
|
cd $WorkDir || finish FAILURE "Could not cd to WorkDir $WorkDir" |
92 |
|
|
|
93 |
|
|
# Create an empty out_file |
94 |
|
|
cat /dev/null > $out_file || finish FAILURE "Could not create out_file $out_file" |
95 |
|
|
|
96 |
|
|
umask 022 |
97 |
|
|
|
98 |
|
|
test -d sandbox.$RunDate && finish FAILURE "Today's sandbox already exists" |
99 |
|
|
mkdir sandbox.$RunDate || finish FAILURE "Could not mkdir sandbox" |
100 |
|
|
cd sandbox.$RunDate || finish FAILURE "Could not cd to sandbox" |
101 |
|
|
|
102 |
|
|
echo "Checking out esys13/trunk" |
103 |
|
|
svn checkout svn+ssh://shake200.esscc.uq.edu.au/home/www_svn/repos/esys13/trunk >> $out_file 2>&1 || finish FAILURE "Could not check out esys13/trunk" |
104 |
|
|
|
105 |
|
|
# Load modules |
106 |
|
|
. /opt/modules/default/init/bash |
107 |
|
|
module use /raid2/matt/modules/modulefiles |
108 |
|
|
module use /raid2/toolspp4/modulefiles/gcc-3.3.6 |
109 |
|
|
module use /usr/share/modules/modulefiles |
110 |
|
|
module load esys/env # Recommended modules |
111 |
|
|
module load doxygen/1.4.6 |
112 |
robwdcock |
683 |
module load boost/1.33.0/python-2.4.1 |
113 |
ksteube |
995 |
module load gmsh-1.65.0 |
114 |
jgs |
318 |
|
115 |
ksteube |
774 |
# How many threads? One per CPU. |
116 |
|
|
export OMP_NUM_THREADS=\$NCPUS |
117 |
svn |
344 |
|
118 |
ksteube |
774 |
# Run the tests |
119 |
|
|
echo "Running the tests $target" |
120 |
|
|
cd trunk || finish FAILURE "Could not cd to trunk" |
121 |
|
|
scons -j \$NCPUS $target >> $out_file 2>&1 || finish FAILURE "Could not run scons $target" |
122 |
elspeth |
721 |
|
123 |
ksteube |
774 |
echo "Cleaning up after the tests" |
124 |
jgs |
319 |
|
125 |
ksteube |
979 |
# Delete files older than 60 days |
126 |
|
|
find $WorkDir -atime +60 -exec rm -f {} \; |
127 |
elspeth |
721 |
|
128 |
ksteube |
774 |
finish SUCCESS "Successfully ran 'scons $target' on \`hostname\`" |
129 |
svn |
320 |
|
130 |
ksteube |
774 |
EOF |
131 |
jgs |
318 |
|
132 |
ksteube |
774 |
# cd to the logs area so the PBS logs are deposited there |
133 |
|
|
cd $WorkDir/Logs |
134 |
jgs |
365 |
|
135 |
ksteube |
774 |
# Submit the job |
136 |
|
|
. /opt/modules/default/init/sh |
137 |
|
|
module load pbspro |
138 |
ksteube |
937 |
|
139 |
|
|
# Submit the job |
140 |
ksteube |
774 |
qsub -S /bin/bash $RunDate.pbs.script.sh > /dev/null |
141 |
|
|
|