# “defq” is the default and route only queue with
# targets: common and ib queues
#PBS -q defq
# Ask for 20 minutes wall clock’s time for the whole job
#PBS -l walltime=00:20:00
# Ask for 5 minutes CPU time for the whole job
#PBS -l cput=00:05:00
# Ask for 10 CPUes in total
#PBS -l nodes=10
## Ask for 10 hosts x 4 cpus
##PBS -l nodes=wn001.jinr.ru:ppn=4+wn002.jinr.ru:ppn=4+wn003.jinr.ru:ppn=4+wn004.jinr.ru:ppn=4+wn005.jinr.ru:ppn=4+wn006.jinr.ru:ppn=4+wn007.jinr.ru:ppn=4+wn008.jinr.ru:ppn=4+wn009.jinr.ru:ppn=4+wn010.jinr.ru:ppn=4
# Job is not restartable
#PBS -r n
# No checkpoint for the job (not implemented on linux)
#PBS -c n
#
# start of the real jobs from here
#
echo “==================================================”
# Print user name
myname=`whoami` 2>&1
echo “whoami=$myname”
# Print execution host (mother superior)
echo “host=”`hostname -f 2>&1`
# Print current directory
echo “pwd=”`pwd 2>&1`
# Go to the original directory where qsub was invoked.
# It’s important for mpi jobs to run from the directory
# where all executables and data files can be found on all
# involved hosts.
echo “cd \$PBS_O_WORKDIR”
cd $PBS_O_WORKDIR 2>&1
echo “pwd=”`pwd 2>&1`
# Check for PAG
echo “id=”`id 2>&1`
# Check for my procs and environment
echo “==================== klist ======================”
klist -5 2>&1 | grep -Ev ‘^[[:space:]]*$’
echo “==================== tokens ======================”
tokens 2>&1 | grep -Ev ‘^[[:space:]]*$’
echo “============== my processes ======================”
ps -ef | grep $myname | grep -v ‘grep ‘ | grep -v ‘ps -ef’
echo “============ /tmp/tkt* /tmp/krb5cc* ============”
/bin/ls -lt /tmp/tkt* /tmp/krb5cc* 2>&1 | grep $myname | head -5
echo “==================================================”
# Build the executable.
if test -e myprog ; then
echo “mv -f myprog myprog.bak”
mv -f myprog myprog.bak 2>&1
fi
echo “mpicc: “`which mpicc 2>&1`
echo “mpicc -i-dynamic -o myprog myprog.c”
mpicc -i-dynamic -o myprog myprog.c 2>&1
# And run it in all allocated nodes
echo “mpiexec: “`which mpiexec 2>&1`
if test -x myprog ; then
# is infiniband loaded on the node
if test X”`/sbin/lsmod | grep ^ib_core\ `” != “X” ; then
# with infiniband – default
echo “ulimit -l 262144”
ulimit -l 262144 2>&1
echo “================= run program ====================”
echo “mpiexec ./myprog”
mpiexec ./myprog 2>&1
else
# no infiniband – disable openib
echo “================= run program ====================”
echo “mpiexec –mca btl ^openib ./myprog”
mpiexec –mca btl ^openib ./myprog 2>&1
fi
else
echo “myprog (executable) not found in “`pwd`
fi
echo “==================================================”
# That’s all
echo “done”