Annotation of /trunk/bin/run-escript

#!/bin/bash

# Escript/Finley wrapper for python
# Sets LD_LIBRARY_PATH and PYTHONPATH and then runs either python or the MPI launcher

#Extra paths can be configured about a page further down
#Search for EXTRA_PATH=""

#set to 1 if performing this is a standalone build and ../../pkg contains the relevant tools
STANDALONE=0

#set to 1 if this is part of a packaged build (.deb) and files will be installed in standard locations
#rather than everything in a single directory
#Do not use this together with $STANDALONE
STDLOCATION=0

#Now we find the location of this script
#Note that this location should be absolute but does not need to be unique
scriptdir=""
CURDIR=`pwd`

#Environment vars which control operations:
# ESCRIPT_NUM_NODES, ESCRIPT_NUM_PROCS, ESCRIPT_NUM_THREADS, ESCRIPT_HOSTFILE, ESCRIPT_CREATESTDFILES

HOSTFILE=/tmp/escript.$USER.$$
HOSTFILE2=/tmp/escript2.$USER.$$

#Begin finding ESCRIPT_ROOT
if [ $STDLOCATION -ne 0 ]
then
    ESCRIPT_ROOT=/usr/lib/escript
else
  #We don't know the escript root so we need to work it out from the invocation
  #Need to match if the name contains /
  if [[ $0 =~ / ]]
  then
      # We are not using the PATH to find the script
      cd `dirname $0`
      scriptdir=`pwd`
      cd $CURDIR
  else
      # name does not contain / therefore we are using 
      tscriptdir=`which $0`
      if [ $? != 0 ]
      then
          echo "Error! Unable to determine script directory. Exiting."
          exit 1
      fi
      scriptdir=`dirname $tscriptdir`
  fi

  cd $scriptdir/..
  ESCRIPT_ROOT=`pwd`
  cd ..
  ESCRIPT_PARENT=`pwd`
  cd $CURDIR

fi
##### End finding ESCRIPT_ROOT  ########

if [ ! -r $ESCRIPT_ROOT/lib/buildvars ]; then
    echo "Error! Unable to read escript build information. Exiting."
    exit 1
fi

PYTHON_MPI_NULL="$ESCRIPT_ROOT/lib/pythonMPI"
PYTHON_MPI_REDIRECT="$ESCRIPT_ROOT/lib/pythonMPIredirect"
PYTHON_CMD=python

# if possible please express paths relative to $ESCRIPT_ROOT unless
# they are in an unrelated location

EXTRA_DYLD_LIBRARY_PATH=""
EXTRA_PATH=$ESCRIPT_ROOT/bin
EXTRA_PYTHONPATH=$ESCRIPT_ROOT
EXTRA_LD_LIBRARY_PATH=$ESCRIPT_ROOT/lib

if [ $STANDALONE -eq 1 ]
then
    EXTRA_PATH=$ESCRIPT_PARENT/pkg/python/bin:$EXTRA_PATH
    EXTRA_LD_LIBRARY_PATH=$ESCRIPT_PARENT/pkg/boost/lib:$ESCRIPT_PARENT/pkg/netcdf/lib/:$EXTRA_LD_LIBRARY_PATH
    EXTRA_LD_LIBRARY_PATH=$EXTRA_LD_LIBRARY_PATH
    EXTRA_LD_LIBRARY_PATH=$ESCRIPT_PARENT/pkg/python/lib:$EXTRA_LD_LIBRARY_PATH
    EXTRA_PYTHONPATH=$ESCRIPT_PARENT/pkg/numpy/lib/python2.6/site-packages:$ESCRIPT_PARENT/pkg/matplotlib/lib/python2.6/site-packages:$EXTRA_PYTHONPATH
fi


HELP_TEXT="
Usage: escript [options] script.py [arguments...]
    -n nn       number of nodes to use
    -p np       number of MPI processes to spawn
    -t nt       number of OpenMP threads to use
    -f file     name of MPI hostfile
    -c      print compile information for escript and exit
    -V      print escript version and exit
    -i      interactive mode 
    -b      do not invoke python (run non-python programs)
    -e      print export statements for environment and exit
    -o      redirect output from MPI to files
    -v      print diagnostics
    -x      ..reserved for future use ..
    script.py   Your python script
    arguments...    The optional command-line arguments to your script
"

if [ "$1" = "--help" ]; then
  echo "$HELP_TEXT"
  exit 0
fi
#=======================================================================================

# Avoid bug in hybrid runs with MPT MPI


# Parse the command-line options
# option e should not be followed by a :
while getopts 'bn:p:t:f:h:ecVviox' option
do
    case "$option" in
      "b")  DOBINARY=yes
        ;;
      "n")  ESCRIPT_NUM_NODES=$OPTARG
        ;;
      "p")  ESCRIPT_NUM_PROCS=$OPTARG
        ;;
      "t")  ESCRIPT_NUM_THREADS=$OPTARG
        ;;
      "f")  ESCRIPT_HOSTFILE=$OPTARG
        ;;
      "c")  cat $ESCRIPT_ROOT/lib/buildvars
        exit 0
        ;;
      "V")  echo "escript-development(build "`grep svn_revision $ESCRIPT_ROOT/lib/buildvars |cut -d= -f2`")"
        exit 0
        ;;
      "h")  echo "$HELPTEXT"
        exit 0
        ;;
      "i")  DOINTERACTIVE=yes
        ;;
      "e")  echo "export LD_LIBRARY_PATH=$EXTRA_LD_LIBRARY_PATH:\$LD_LIBRARY_PATH"
        echo "export PYTHONPATH=$EXTRA_PYTHONPATH:\$PYTHONPATH"
        echo "export PATH=$EXTRA_PATH:\$PATH"
        if [ "`uname`" = "Darwin" ]
        then
            echo "export DYLD_LIBRARY_PATH=$EXTRA_DYLD_LIBRARY_PATH:$EXTRA_LD_LIBRARY_PATH:\$DYLD_LIBRARY_PATH"
        fi
        exit 0
        ;;
      "o")  ESCRIPT_CREATESTDFILES="yes"
        ;;
      "v")  ESCRIPT_VERBOSE="yes"
        ;;
      "x")  echo "-x not implemented yet"
        exit 1
        ;;
      ?)    echo "$HELP_TEXT"
        exit 1
        ;;
    esac
done
shift `expr $OPTIND - 1`
#==============================================
#
#   Read the MPI_FLAVOUR from the buildvars
#
MPI_FLAVOUR=`grep mpi_flavour $ESCRIPT_ROOT/lib/buildvars |cut -d= -f2`
WITH_OPENMP=`grep useopenmp $ESCRIPT_ROOT/lib/buildvars |cut -d= -f2`

if [ ! -z $ESCRIPT_VERBOSE ]; then
    echo "MPI flavour is $MPI_FLAVOUR."
    if [ "$WITH_OPENMP" = "y" ]; then echo "OpenMP enabled."; fi
fi

#
#  extend path variables
#
export PATH=$EXTRA_PATH:$PATH
export LD_LIBRARY_PATH=$EXTRA_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
export PYTHONPATH=$EXTRA_PYTHONPATH:$PYTHONPATH
EXPORT_ENV="PATH,LD_LIBRARY_PATH,PYTHONPATH"
if [ "`uname`" = "Darwin" ]
then
    export DYLD_LIBRARY_PATH=$EXTRA_DYLD_LIBRARY_PATH:$EXTRA_LD_LIBRARY_PATH:$DYLD_LIBRARY_PATH
    EXPORT_ENV="$EXPORT_ENV,DYLD_LIBRARY_PATH"
fi
if [ ! -z $ESCRIPT_VERBOSE ] 
then 
    echo "PATH = $PATH "
    echo "LD_LIBRARY_PATH = $LD_LIBRARY_PATH "
    echo "PYTHONPATH = $PYTHONPATH "
    if [ ! -z $DYLD_LIBRARY_PATH ]; then echo "DYLD_LIBRARY_PATH = $DYLD_LIBRARY_PATH "; fi
fi
#==============================================
#
#  Ensure the variables have sensible values
#
if [ "$MPI_FLAVOUR" = "none" ] 
then
    if [ ! -z $ESCRIPT_NUM_NODES ]
    then
        echo "Warning: MPI disabled but number of nodes set. Option ignored."
    fi
    if [ ! -z $ESCRIPT_NUM_PROCS ]
    then
        echo "Warning: MPI disabled but number of processors per node set. Option ignored."
    fi
    if [ ! -z $ESCRIPT_HOSTFILE ]
    then
        echo "Warning: MPI disabled but host file is given. Option ignored."
    fi
    ESCRIPT_NUM_NODES=1
    ESCRIPT_NUM_PROCS=1
else
    # use the PBS_NODEFILE if not otherwise specified
    if [[ ( ! -z $PBS_NODEFILE ) && ( -z $ESCRIPT_HOSTFILE ) ]]
    then
    ESCRIPT_HOSTFILE=$PBS_NODEFILE

    fi

    if [ ! -z $ESCRIPT_HOSTFILE ] 
    then
        if [ -f $ESCRIPT_HOSTFILE ]
        then
            cat $ESCRIPT_HOSTFILE | sort -u > $HOSTFILE
            NUM_HOSTS=`cat $HOSTFILE | wc -l`
            if [ ! -z $ESCRIPT_NUM_NODES ] 
            then
                if [ $NUM_HOSTS -ne $ESCRIPT_NUM_NODES ]
                then
                    echo "number of hosts selected in the host file $ESCRIPT_HOSTFILE needs to match the requested number of nodes $ESCRIPT_NUM_NODES."
                    exit 1
                fi
             else
                ESCRIPT_NUM_NODES=$NUM_HOSTS
             fi
        else
           echo "cannot find hostfile $ESCRIPT_HOSTFILE."
           exit 1
        fi
    else
        HOSTFILE=''
    fi

    if [ -z $ESCRIPT_NUM_NODES ]
    then
      ESCRIPT_NUM_NODES=1
    fi

    if [ -z $ESCRIPT_NUM_PROCS ]
    then
      ESCRIPT_NUM_PROCS=1
    fi

    if [ ! -z $ESCRIPT_VERBOSE ]
    then 
          echo "ESCRIPT_NUM_NODES = $ESCRIPT_NUM_NODES "
          echo "ESCRIPT_NUM_PROCS = $ESCRIPT_NUM_PROCS "
    fi
fi

if [ "$WITH_OPENMP" = "y" ]
then
   if [ -z $ESCRIPT_NUM_THREADS ]
   then
        ESCRIPT_NUM_THREADS=$OMP_NUM_THREADS
        if [ -z $ESCRIPT_NUM_THREADS ]
        then
           ESCRIPT_NUM_THREADS=1
        fi
   fi
   if [ ! -z $ESCRIPT_VERBOSE ]
   then
        echo "ESCRIPT_NUM_THREADS is $ESCRIPT_NUM_THREADS."
   fi
else
   if [[ ( ! -z $ESCRIPT_NUM_THREADS ) && ( $ESCRIPT_NUM_THREADS != 1 ) ]]
   then
       echo "Warning: OpenMP is disabled but number of threads requested is $ESCRIPT_NUM_THREADS!=1. Running without threads."
   fi
   ESCRIPT_NUM_THREADS=1
fi
#
# Now we compute total number of Processes
#
(( TOTPROC=$ESCRIPT_NUM_NODES * $ESCRIPT_NUM_PROCS))
if [ $? -ne 0 ]     #Some compute error 
then            #This could happen if the args were not a number
  echo "expression of total number of processors = $ESCRIPT_NUM_NODES * $ESCRIPT_NUM_PROCS is not numerical."
  exit 1
fi

#
# Test to ensure people aren't trying to combine interactive and multi-process
#
if [[ ( ( ! -z $DOINTERACTIVE ) || ( $# -eq 0 ) ) && ( $TOTPROC -gt 1) ]]
then
  echo "Interactive mode cannot be used with more than one process"
  exit 1
fi

if [ $TOTPROC -gt 1 ] 
then
   if [ "$ESCRIPT_CREATESTDFILES" = "yes" ]
   then
       PYTHON_MPI=$PYTHON_MPI_REDIRECT
   else
       PYTHON_MPI=$PYTHON_MPI_NULL
   fi
else
    PYTHON_MPI=$PYTHON_MPI_NULL
fi
#=========================================================================================================
# Must have at least one command-line arg: the python script
if [ $# -eq 0 ]
then
  if [ ! -z $DOBINARY ]
  then
    echo "No program to run was specified. Exiting."
    exit 1
  fi
fi
if [ -z "$@" ]
then
   if [ ! -z $DOBINARY ]
   then
       echo "Program must be given."
       exit 1
   else
      DOINTERACTIVE="yes"
   fi
fi

#=========================================================================================================
if [ ! -z $DOBINARY ]
then
    EXEC_CMD="$@"
else
    if [ "$MPI_FLAVOUR" = "none" ]
    then
        # Check to see if the python version we were compiled with matches the one of EXEC_CMD
        # We should only do this check if they are running a python script
        if [ -f $ESCRIPT_ROOT/lib/pyversion ]
        then
           compversion=`cat $ESCRIPT_ROOT/lib/pyversion`
           intversion=`$PYTHON_CMD -V 2>&1`
           if [ "$compversion" != "$intversion" ]
           then
              echo "Python versions do not match. Escript was compiled for "$compversion"."
              echo "Current version of Python appears to be "$intversion"."
              exit 1
           fi
        fi
    if [ ! -z $DOINTERACTIVE ]
        then
           EXEC_CMD="$PYTHON_CMD -i $@"
        else
           EXEC_CMD="$PYTHON_CMD $@"
        fi
    else
    if [ ! -z $DOINTERACTIVE ]
        then
           EXEC_CMD="$PYTHON_MPI -i $@"
        else
           EXEC_CMD="$PYTHON_MPI $@"
        fi
    fi
fi
if [ ! -z $ESCRIPT_VERBOSE ]; then echo "Command to be executed is \"$EXEC_CMD\"."; fi
#===============================================================================================
#
#   now we start to spwan things:
#
if [ "$WITH_OPENMP" = "y" ]
then
   export OMP_NUM_THREADS=$ESCRIPT_NUM_THREADS
   EXPORT_ENV="$EXPORT_ENV,OMP_NUM_THREADS"
fi
EXIT_CODE=1
#=============== no MPI ===================================
if [ "$MPI_FLAVOUR" = "none" ] 
then
   $EXEC_CMD 
   EXIT_CODE=$?
#=============== OpenMPI ===================================
elif [ "$MPI_FLAVOUR" = "OPENMPI" ] 
then 
   if [ ! -z $HOSTFILE ] 
   then
      HOST_LIST=`awk 'BEGIN{S=""}{if (S == "") { S = $0 } else {S = S "," $0}}END{print S}' $HOSTFILE`
      CMD="mpirun --gmca mpi_warn_on_fork 0 -x ${EXPORT_ENV//,/ -x } --bynode -np $TOTPROC --host $HOST_LIST $EXEC_CMD"
   else
      CMD="mpirun --gmca mpi_warn_on_fork 0 -x ${EXPORT_ENV//,/ -x } -np $TOTPROC $EXEC_CMD"
   fi 
   if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
   $CMD
   EXIT_CODE=$?

#=============== Intel MPI ===================================
elif [ "$MPI_FLAVOUR" = "INTELMPI" ]
then

   if [ "$WITH_OPENMP" = "y" ]
   then
       export I_MPI_PIN_DOMAIN=omp
       EXPORT_ENV="$EXPORT_ENV, I_MPI_PIN_DOMAIN"
   fi

   if [ ! -z $HOSTFILE ] 
   then
      mpdboot -n $ESCRIPT_NUM_NODES -r ssh -f $HOSTFILE
      if [ $? -ne 0 ] 
      then
         echo "mpdboot with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes failed."
         exit 1
      else 
          if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes."; fi
      fi
   else
      mpdboot -n 1 -r ssh 
      if [ $? -ne 0 ] 
      then
         echo "mpdboot failed."
         exit 1
      else 
          if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started."; fi
      fi
   fi
   CMD="mpiexec -perhost $ESCRIPT_NUM_PROCS -envall -n $TOTPROC $EXEC_CMD"
   if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
   $CMD
   EXIT_CODE=$?
   mpdallexit
   if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdallexit executed."; fi
#=============== SGI's MPIMPT ===================================
elif [ "$MPI_FLAVOUR" = "MPT" ]
then
   export MPI_NUM_MEMORY_REGIONS=0
   EXPORT_ENV="$EXPORT_ENV,MPI_NUM_MEMORY_REGIONS"
   if [ ! -z $HOSTFILE ] 
   then
      HOST_LIST=`awk 'BEGIN{S=""}{if (S == "") { S = $0 } else {S = S "," $0}}END{print S}' $HOSTFILE`
      CMD="mpirun $HOST_LIST -np $ESCRIPT_NUM_PROCS $EXEC_CMD"
   else
      CMD="mpirun -np $TOTPROC $EXEC_CMD"
   fi 
   if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
   $CMD
   EXIT_CODE=$?
#=============== MPICH ===================================
elif [ "$MPI_FLAVOUR" = "MPICH" ]
then
   if [ ! -z $HOSTFILE ] 
   then
      touch $HOSTFILE2
      for (( i=1;i<=$ESCRIPT_NUM_PROCS;i+=1 )) ; do cat $HOSTFILE >> $HOSTFILE2  ; done
      CMD="mpirun -machinefile $HOSTFILE2  -np $TOTPROC $EXEC_CMD"
   else
      CMD="mpirun -np $TOTPROC $EXEC_CMD"
   fi 
   if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
   $CMD
   EXIT_CODE=$?
#=============== MPICH2 ===================================
elif [ "$MPI_FLAVOUR" = "MPICH2" ]
then
   if [ ! -z $HOSTFILE ] 
   then
         mpdboot -n $ESCRIPT_NUM_NODES -r ssh -f $HOSTFILE
         if [ $? -ne 0 ] 
         then
            echo "mpdboot with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes failed."
            exit 1
         else 
             if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes."; fi
         fi
   else
      mpdboot -n 1 -r ssh 
      if [ $? -ne 0 ] 
      then
         echo "mpdboot failed."
         exit 1
      else 
          if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started."; fi
      fi
   fi
   CMD="mpiexec -genvlist $EXPORT_ENV -np $TOTPROC $EXEC_CMD"
   if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
   $CMD
   EXIT_CODE=$?
   mpdallexit
   if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdallexit executed."; fi
else
   echo "unknown MPI flavour '$MPI_FLAVOUR'."
fi
exit $EXIT_CODE
1	jfenwick	2409	#!/bin/bash
2
3			# Escript/Finley wrapper for python
4			# Sets LD_LIBRARY_PATH and PYTHONPATH and then runs either python or the MPI launcher
5
6			#Extra paths can be configured about a page further down
7			#Search for EXTRA_PATH=""
8
9	jfenwick	2568	#set to 1 if performing this is a standalone build and ../../pkg contains the relevant tools
10	jfenwick	2409	STANDALONE=0
11
12			#set to 1 if this is part of a packaged build (.deb) and files will be installed in standard locations
13			#rather than everything in a single directory
14			#Do not use this together with $STANDALONE
15			STDLOCATION=0
16
17			#Now we find the location of this script
18			#Note that this location should be absolute but does not need to be unique
19			scriptdir=""
20			CURDIR=`pwd`
21
22			#Environment vars which control operations:
23			# ESCRIPT_NUM_NODES, ESCRIPT_NUM_PROCS, ESCRIPT_NUM_THREADS, ESCRIPT_HOSTFILE, ESCRIPT_CREATESTDFILES
24
25	jfenwick	3047	HOSTFILE=/tmp/escript.$USER.$$
26			HOSTFILE2=/tmp/escript2.$USER.$$
27	jfenwick	2409
28			#Begin finding ESCRIPT_ROOT
29	caltinay	2806	if [ $STDLOCATION -ne 0 ]
30	jfenwick	2409	then
31			ESCRIPT_ROOT=/usr/lib/escript
32			else
33			#We don't know the escript root so we need to work it out from the invocation
34			#Need to match if the name contains /
35			if [[ $0 =~ / ]]
36			then
37			# We are not using the PATH to find the script
38			cd `dirname $0`
39			scriptdir=`pwd`
40			cd $CURDIR
41			else
42			# name does not contain / therefore we are using
43			tscriptdir=`which $0`
44			if [ $? != 0 ]
45			then
46	caltinay	2806	echo "Error! Unable to determine script directory. Exiting."
47			exit 1
48	jfenwick	2409	fi
49			scriptdir=`dirname $tscriptdir`
50			fi
51
52			cd $scriptdir/..
53			ESCRIPT_ROOT=`pwd`
54			cd ..
55			ESCRIPT_PARENT=`pwd`
56			cd $CURDIR
57
58			fi
59			##### End finding ESCRIPT_ROOT ########
60
61	caltinay	2806	if [ ! -r $ESCRIPT_ROOT/lib/buildvars ]; then
62			echo "Error! Unable to read escript build information. Exiting."
63			exit 1
64			fi
65
66	jfenwick	2409	PYTHON_MPI_NULL="$ESCRIPT_ROOT/lib/pythonMPI"
67			PYTHON_MPI_REDIRECT="$ESCRIPT_ROOT/lib/pythonMPIredirect"
68			PYTHON_CMD=python
69
70			# if possible please express paths relative to $ESCRIPT_ROOT unless
71			# they are in an unrelated location
72
73			EXTRA_DYLD_LIBRARY_PATH=""
74			EXTRA_PATH=$ESCRIPT_ROOT/bin
75			EXTRA_PYTHONPATH=$ESCRIPT_ROOT
76			EXTRA_LD_LIBRARY_PATH=$ESCRIPT_ROOT/lib
77
78	caltinay	2806	if [ $STANDALONE -eq 1 ]
79	jfenwick	2409	then
80	jfenwick	2530	EXTRA_PATH=$ESCRIPT_PARENT/pkg/python/bin:$EXTRA_PATH
81			EXTRA_LD_LIBRARY_PATH=$ESCRIPT_PARENT/pkg/boost/lib:$ESCRIPT_PARENT/pkg/netcdf/lib/:$EXTRA_LD_LIBRARY_PATH
82	jfenwick	2541	EXTRA_LD_LIBRARY_PATH=$EXTRA_LD_LIBRARY_PATH
83	jfenwick	2530	EXTRA_LD_LIBRARY_PATH=$ESCRIPT_PARENT/pkg/python/lib:$EXTRA_LD_LIBRARY_PATH
84	jfenwick	2541	EXTRA_PYTHONPATH=$ESCRIPT_PARENT/pkg/numpy/lib/python2.6/site-packages:$ESCRIPT_PARENT/pkg/matplotlib/lib/python2.6/site-packages:$EXTRA_PYTHONPATH
85	jfenwick	2409	fi
86
87
88
89			HELP_TEXT="
90			Usage: escript [options] script.py [arguments...]
91			-n nn number of nodes to use
92			-p np number of MPI processes to spawn
93			-t nt number of OpenMP threads to use
94			-f file name of MPI hostfile
95			-c print compile information for escript and exit
96			-V print escript version and exit
97			-i interactive mode
98			-b do not invoke python (run non-python programs)
99			-e print export statements for environment and exit
100			-o redirect output from MPI to files
101			-v print diagnostics
102			-x ..reserved for future use ..
103			script.py Your python script
104			arguments... The optional command-line arguments to your script
105			"
106
107			if [ "$1" = "--help" ]; then
108			echo "$HELP_TEXT"
109			exit 0
110			fi
111			#=======================================================================================
112
113			# Avoid bug in hybrid runs with MPT MPI
114
115
116			# Parse the command-line options
117			# option e should not be followed by a :
118			while getopts 'bn:p:t:f:h:ecVviox' option
119			do
120			case "$option" in
121			"b") DOBINARY=yes
122			;;
123			"n") ESCRIPT_NUM_NODES=$OPTARG
124			;;
125			"p") ESCRIPT_NUM_PROCS=$OPTARG
126			;;
127			"t") ESCRIPT_NUM_THREADS=$OPTARG
128			;;
129			"f") ESCRIPT_HOSTFILE=$OPTARG
130			;;
131			"c") cat $ESCRIPT_ROOT/lib/buildvars
132			exit 0
133			;;
134	jfenwick	2875	"V") echo "escript-development(build "`grep svn_revision $ESCRIPT_ROOT/lib/buildvars \|cut -d= -f2`")"
135	jfenwick	2409	exit 0
136			;;
137			"h") echo "$HELPTEXT"
138			exit 0
139			;;
140			"i") DOINTERACTIVE=yes
141			;;
142			"e") echo "export LD_LIBRARY_PATH=$EXTRA_LD_LIBRARY_PATH:\$LD_LIBRARY_PATH"
143			echo "export PYTHONPATH=$EXTRA_PYTHONPATH:\$PYTHONPATH"
144			echo "export PATH=$EXTRA_PATH:\$PATH"
145	caltinay	2806	if [ "`uname`" = "Darwin" ]
146	jfenwick	2409	then
147			echo "export DYLD_LIBRARY_PATH=$EXTRA_DYLD_LIBRARY_PATH:$EXTRA_LD_LIBRARY_PATH:\$DYLD_LIBRARY_PATH"
148			fi
149			exit 0
150			;;
151			"o") ESCRIPT_CREATESTDFILES="yes"
152			;;
153			"v") ESCRIPT_VERBOSE="yes"
154			;;
155			"x") echo "-x not implemented yet"
156			exit 1
157			;;
158			?) echo "$HELP_TEXT"
159			exit 1
160			;;
161			esac
162			done
163			shift `expr $OPTIND - 1`
164			#==============================================
165			#
166			# Read the MPI_FLAVOUR from the buildvars
167			#
168			MPI_FLAVOUR=`grep mpi_flavour $ESCRIPT_ROOT/lib/buildvars \|cut -d= -f2`
169			WITH_OPENMP=`grep useopenmp $ESCRIPT_ROOT/lib/buildvars \|cut -d= -f2`
170
171	caltinay	2806	if [ ! -z $ESCRIPT_VERBOSE ]; then
172			echo "MPI flavour is $MPI_FLAVOUR."
173			if [ "$WITH_OPENMP" = "y" ]; then echo "OpenMP enabled."; fi
174			fi
175
176	jfenwick	2409	#
177			# extend path variables
178			#
179			export PATH=$EXTRA_PATH:$PATH
180			export LD_LIBRARY_PATH=$EXTRA_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
181			export PYTHONPATH=$EXTRA_PYTHONPATH:$PYTHONPATH
182			EXPORT_ENV="PATH,LD_LIBRARY_PATH,PYTHONPATH"
183	caltinay	2806	if [ "`uname`" = "Darwin" ]
184	jfenwick	2409	then
185			export DYLD_LIBRARY_PATH=$EXTRA_DYLD_LIBRARY_PATH:$EXTRA_LD_LIBRARY_PATH:$DYLD_LIBRARY_PATH
186			EXPORT_ENV="$EXPORT_ENV,DYLD_LIBRARY_PATH"
187			fi
188	caltinay	2806	if [ ! -z $ESCRIPT_VERBOSE ]
189	jfenwick	2409	then
190			echo "PATH = $PATH "
191			echo "LD_LIBRARY_PATH = $LD_LIBRARY_PATH "
192			echo "PYTHONPATH = $PYTHONPATH "
193			if [ ! -z $DYLD_LIBRARY_PATH ]; then echo "DYLD_LIBRARY_PATH = $DYLD_LIBRARY_PATH "; fi
194			fi
195			#==============================================
196			#
197			# Ensure the variables have sensible values
198			#
199	caltinay	2806	if [ "$MPI_FLAVOUR" = "none" ]
200	jfenwick	2409	then
201			if [ ! -z $ESCRIPT_NUM_NODES ]
202			then
203			echo "Warning: MPI disabled but number of nodes set. Option ignored."
204			fi
205			if [ ! -z $ESCRIPT_NUM_PROCS ]
206			then
207			echo "Warning: MPI disabled but number of processors per node set. Option ignored."
208			fi
209			if [ ! -z $ESCRIPT_HOSTFILE ]
210			then
211			echo "Warning: MPI disabled but host file is given. Option ignored."
212			fi
213			ESCRIPT_NUM_NODES=1
214			ESCRIPT_NUM_PROCS=1
215			else
216	jfenwick	3056	# use the PBS_NODEFILE if not otherwise specified
217			if [[ ( ! -z $PBS_NODEFILE ) && ( -z $ESCRIPT_HOSTFILE ) ]]
218			then
219			ESCRIPT_HOSTFILE=$PBS_NODEFILE
220
221			fi
222
223	jfenwick	2409	if [ ! -z $ESCRIPT_HOSTFILE ]
224			then
225			if [ -f $ESCRIPT_HOSTFILE ]
226			then
227			cat $ESCRIPT_HOSTFILE \| sort -u > $HOSTFILE
228			NUM_HOSTS=`cat $HOSTFILE \| wc -l`
229			if [ ! -z $ESCRIPT_NUM_NODES ]
230			then
231			if [ $NUM_HOSTS -ne $ESCRIPT_NUM_NODES ]
232			then
233			echo "number of hosts selected in the host file $ESCRIPT_HOSTFILE needs to match the requested number of nodes $ESCRIPT_NUM_NODES."
234			exit 1
235			fi
236			else
237			ESCRIPT_NUM_NODES=$NUM_HOSTS
238			fi
239			else
240			echo "cannot find hostfile $ESCRIPT_HOSTFILE."
241			exit 1
242			fi
243			else
244			HOSTFILE=''
245			fi
246
247			if [ -z $ESCRIPT_NUM_NODES ]
248			then
249			ESCRIPT_NUM_NODES=1
250			fi
251
252			if [ -z $ESCRIPT_NUM_PROCS ]
253			then
254			ESCRIPT_NUM_PROCS=1
255			fi
256
257	caltinay	2806	if [ ! -z $ESCRIPT_VERBOSE ]
258	jfenwick	2409	then
259			echo "ESCRIPT_NUM_NODES = $ESCRIPT_NUM_NODES "
260			echo "ESCRIPT_NUM_PROCS = $ESCRIPT_NUM_PROCS "
261			fi
262			fi
263
264	caltinay	2806	if [ "$WITH_OPENMP" = "y" ]
265	jfenwick	2409	then
266			if [ -z $ESCRIPT_NUM_THREADS ]
267			then
268			ESCRIPT_NUM_THREADS=$OMP_NUM_THREADS
269			if [ -z $ESCRIPT_NUM_THREADS ]
270			then
271			ESCRIPT_NUM_THREADS=1
272			fi
273			fi
274			if [ ! -z $ESCRIPT_VERBOSE ]
275			then
276			echo "ESCRIPT_NUM_THREADS is $ESCRIPT_NUM_THREADS."
277			fi
278			else
279			if [[ ( ! -z $ESCRIPT_NUM_THREADS ) && ( $ESCRIPT_NUM_THREADS != 1 ) ]]
280			then
281			echo "Warning: OpenMP is disabled but number of threads requested is $ESCRIPT_NUM_THREADS!=1. Running without threads."
282			fi
283			ESCRIPT_NUM_THREADS=1
284			fi
285			#
286			# Now we compute total number of Processes
287			#
288			(( TOTPROC=$ESCRIPT_NUM_NODES * $ESCRIPT_NUM_PROCS))
289			if [ $? -ne 0 ] #Some compute error
290			then #This could happen if the args were not a number
291			echo "expression of total number of processors = $ESCRIPT_NUM_NODES * $ESCRIPT_NUM_PROCS is not numerical."
292			exit 1
293			fi
294
295			#
296			# Test to ensure people aren't trying to combine interactive and multi-process
297			#
298			if [[ ( ( ! -z $DOINTERACTIVE ) \|\| ( $# -eq 0 ) ) && ( $TOTPROC -gt 1) ]]
299			then
300			echo "Interactive mode cannot be used with more than one process"
301			exit 1
302			fi
303
304			if [ $TOTPROC -gt 1 ]
305			then
306	caltinay	2806	if [ "$ESCRIPT_CREATESTDFILES" = "yes" ]
307	jfenwick	2409	then
308			PYTHON_MPI=$PYTHON_MPI_REDIRECT
309			else
310			PYTHON_MPI=$PYTHON_MPI_NULL
311			fi
312			else
313			PYTHON_MPI=$PYTHON_MPI_NULL
314			fi
315			#=========================================================================================================
316			# Must have at least one command-line arg: the python script
317			if [ $# -eq 0 ]
318			then
319			if [ ! -z $DOBINARY ]
320			then
321			echo "No program to run was specified. Exiting."
322			exit 1
323			fi
324			fi
325	jfenwick	3059	if [ -z "$@" ]
326	gross	2867	then
327			if [ ! -z $DOBINARY ]
328			then
329			echo "Program must be given."
330			exit 1
331			else
332			DOINTERACTIVE="yes"
333			fi
334			fi
335	jfenwick	2409
336			#=========================================================================================================
337			if [ ! -z $DOBINARY ]
338			then
339			EXEC_CMD="$@"
340			else
341	caltinay	2806	if [ "$MPI_FLAVOUR" = "none" ]
342	jfenwick	2409	then
343			# Check to see if the python version we were compiled with matches the one of EXEC_CMD
344			# We should only do this check if they are running a python script
345			if [ -f $ESCRIPT_ROOT/lib/pyversion ]
346			then
347			compversion=`cat $ESCRIPT_ROOT/lib/pyversion`
348			intversion=`$PYTHON_CMD -V 2>&1`
349			if [ "$compversion" != "$intversion" ]
350			then
351			echo "Python versions do not match. Escript was compiled for "$compversion"."
352			echo "Current version of Python appears to be "$intversion"."
353			exit 1
354			fi
355			fi
356			if [ ! -z $DOINTERACTIVE ]
357			then
358			EXEC_CMD="$PYTHON_CMD -i $@"
359			else
360			EXEC_CMD="$PYTHON_CMD $@"
361			fi
362			else
363	gross	2865	if [ ! -z $DOINTERACTIVE ]
364			then
365			EXEC_CMD="$PYTHON_MPI -i $@"
366			else
367			EXEC_CMD="$PYTHON_MPI $@"
368			fi
369	jfenwick	2409	fi
370			fi
371			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "Command to be executed is \"$EXEC_CMD\"."; fi
372			#===============================================================================================
373			#
374			# now we start to spwan things:
375			#
376	caltinay	2806	if [ "$WITH_OPENMP" = "y" ]
377	jfenwick	2409	then
378			export OMP_NUM_THREADS=$ESCRIPT_NUM_THREADS
379			EXPORT_ENV="$EXPORT_ENV,OMP_NUM_THREADS"
380			fi
381			EXIT_CODE=1
382			#=============== no MPI ===================================
383	caltinay	2806	if [ "$MPI_FLAVOUR" = "none" ]
384	jfenwick	2409	then
385			$EXEC_CMD
386			EXIT_CODE=$?
387			#=============== OpenMPI ===================================
388	caltinay	2806	elif [ "$MPI_FLAVOUR" = "OPENMPI" ]
389	jfenwick	2409	then
390			if [ ! -z $HOSTFILE ]
391			then
392			HOST_LIST=`awk 'BEGIN{S=""}{if (S == "") { S = $0 } else {S = S "," $0}}END{print S}' $HOSTFILE`
393	gross	2905	CMD="mpirun --gmca mpi_warn_on_fork 0 -x ${EXPORT_ENV//,/ -x } --bynode -np $TOTPROC --host $HOST_LIST $EXEC_CMD"
394	jfenwick	2409	else
395	gross	2905	CMD="mpirun --gmca mpi_warn_on_fork 0 -x ${EXPORT_ENV//,/ -x } -np $TOTPROC $EXEC_CMD"
396	jfenwick	2409	fi
397			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
398			$CMD
399			EXIT_CODE=$?
400
401			#=============== Intel MPI ===================================
402	caltinay	2806	elif [ "$MPI_FLAVOUR" = "INTELMPI" ]
403	jfenwick	2409	then
404	gross	2510
405	caltinay	2806	if [ "$WITH_OPENMP" = "y" ]
406	gross	2510	then
407			export I_MPI_PIN_DOMAIN=omp
408			EXPORT_ENV="$EXPORT_ENV, I_MPI_PIN_DOMAIN"
409			fi
410
411	jfenwick	2409	if [ ! -z $HOSTFILE ]
412			then
413			mpdboot -n $ESCRIPT_NUM_NODES -r ssh -f $HOSTFILE
414			if [ $? -ne 0 ]
415			then
416			echo "mpdboot with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes failed."
417			exit 1
418			else
419			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes."; fi
420			fi
421	gross	2866	else
422			mpdboot -n 1 -r ssh
423			if [ $? -ne 0 ]
424			then
425			echo "mpdboot failed."
426			exit 1
427			else
428			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started."; fi
429			fi
430	jfenwick	2409	fi
431			CMD="mpiexec -perhost $ESCRIPT_NUM_PROCS -envall -n $TOTPROC $EXEC_CMD"
432			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
433			$CMD
434			EXIT_CODE=$?
435	gross	2866	mpdallexit
436			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdallexit executed."; fi
437	jfenwick	2409	#=============== SGI's MPIMPT ===================================
438	caltinay	2806	elif [ "$MPI_FLAVOUR" = "MPT" ]
439	jfenwick	2409	then
440			export MPI_NUM_MEMORY_REGIONS=0
441			EXPORT_ENV="$EXPORT_ENV,MPI_NUM_MEMORY_REGIONS"
442			if [ ! -z $HOSTFILE ]
443			then
444			HOST_LIST=`awk 'BEGIN{S=""}{if (S == "") { S = $0 } else {S = S "," $0}}END{print S}' $HOSTFILE`
445	gross	2866	CMD="mpirun $HOST_LIST -np $ESCRIPT_NUM_PROCS $EXEC_CMD"
446	jfenwick	2409	else
447			CMD="mpirun -np $TOTPROC $EXEC_CMD"
448			fi
449			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
450			$CMD
451			EXIT_CODE=$?
452			#=============== MPICH ===================================
453	caltinay	2806	elif [ "$MPI_FLAVOUR" = "MPICH" ]
454	jfenwick	2409	then
455			if [ ! -z $HOSTFILE ]
456			then
457			touch $HOSTFILE2
458			for (( i=1;i<=$ESCRIPT_NUM_PROCS;i+=1 )) ; do cat $HOSTFILE >> $HOSTFILE2 ; done
459			CMD="mpirun -machinefile $HOSTFILE2 -np $TOTPROC $EXEC_CMD"
460			else
461			CMD="mpirun -np $TOTPROC $EXEC_CMD"
462			fi
463			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
464			$CMD
465			EXIT_CODE=$?
466			#=============== MPICH2 ===================================
467	caltinay	2806	elif [ "$MPI_FLAVOUR" = "MPICH2" ]
468	jfenwick	2409	then
469			if [ ! -z $HOSTFILE ]
470			then
471			mpdboot -n $ESCRIPT_NUM_NODES -r ssh -f $HOSTFILE
472			if [ $? -ne 0 ]
473			then
474			echo "mpdboot with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes failed."
475			exit 1
476			else
477			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started with host file $ESCRIPT_HOSTFILE for $ESCRIPT_NUM_NODES nodes."; fi
478			fi
479	gross	2866	else
480			mpdboot -n 1 -r ssh
481			if [ $? -ne 0 ]
482			then
483			echo "mpdboot failed."
484			exit 1
485			else
486			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdboot was started."; fi
487			fi
488	jfenwick	2409	fi
489			CMD="mpiexec -genvlist $EXPORT_ENV -np $TOTPROC $EXEC_CMD"
490			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "MPI command is \"$CMD\"."; fi
491			$CMD
492			EXIT_CODE=$?
493	gross	2866	mpdallexit
494			if [ ! -z $ESCRIPT_VERBOSE ]; then echo "mpdallexit executed."; fi
495	jfenwick	2409	else
496	caltinay	2806	echo "unknown MPI flavour '$MPI_FLAVOUR'."
497	jfenwick	2409	fi
498			exit $EXIT_CODE
Name	Value
svn:executable	*
svn:mergeinfo	/branches/lapack2681/bin/escript:2682-2741 /branches/restext/bin/escript:2610-2624 /branches/stage3.0/bin/escript:2569-2590 /release/3.0/bin/escript:2591-2601