forked from jswhit/replay_scripts
-
Notifications
You must be signed in to change notification settings - Fork 0
/
runmpi
executable file
·47 lines (47 loc) · 2.26 KB
/
runmpi
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#!/bin/sh
export OMP_NUM_THREADS=${OMP_NUM_THREADS:-1}
[ -z "$nprocs" ] && echo "Need to set nprocs" && exit 1;
[ -z "$machine" ] && echo "Need to set machine" && exit 1;
[ -z "$PGM" ] && echo "Need to set PGM" && exit 1;
if [ ! -z $SLURM_JOB_ID ]; then
[ -z "$mpitaskspernode" ] && echo "Need to set mpitaskspernode" && exit 1;
# use srun
#export OMP_PROC_BIND=spread
#export OMP_PLACES=threads
totcores=`expr $nprocs \* $OMP_NUM_THREADS`
totnodes=`python -c "import math; print int(math.ceil(float(${totcores})/${corespernode}))"`
count=`python -c "import math; print int(math.floor(float(${corespernode})/${mpitaskspernode}))"`
#mpitaskspernode=`expr $totcores \/ $totnodes`
mpitaskspernode=`python -c "import math; print int(math.ceil(float(${nprocs})/${totnodes}))"`
# -c: cpus per mpi task (number of threads per mpi task)
# -n: total number of mpi tasks
# -N: number of nodes to run on
# --ntasks-per-node: mpi tasks on each node
echo "running srun -N $totnodes -n $nprocs -c $count --ntasks-per-node=$mpitaskspernode --exclusive --cpu-bind=cores --verbose $PGM"
eval srun -N $totnodes -n $nprocs -c $count --ntasks-per-node=$mpitaskspernode --exclusive --cpu-bind=cores --verbose $PGM
rc=$?
elif [ "$machine" == 'wcoss' ] ; then
[ -z "$mpitaskspernode" ] && echo "Need to set mpitaskspernode" && exit 1;
echo "running aprun -n $nprocs -N $mpitaskspernode -d ${OMP_NUM_THREADS} --cc depth $PGM"
eval aprun -n $nprocs -N $mpitaskspernode -d ${OMP_NUM_THREADS} --cc depth $PGM
rc=$?
elif [ "$machine" == 'gaea' ] ; then
[ -z "$mpitaskspernode" ] && echo "Need to set mpitaskspernode" && exit 1;
# use aprun
echo "running aprun -n $nprocs -N $mpitaskspernode -d ${OMP_NUM_THREADS} --cc depth $PGM"
eval aprun -n $nprocs -N $mpitaskspernode -d ${OMP_NUM_THREADS} --cc depth $PGM
rc=$?
elif [ "$machine" == 'theia' ]; then
# HOSTFILE env var must be set
[ -z "$HOSTFILE" ] && echo "Need to set HOSTFILE" && exit 1;
cat $HOSTFILE
echo "running mpirun -np $nprocs -machinefile $HOSTFILE $PGM"
echo "OMP_NUM_THREADS = $OMP_NUM_THREADS"
eval mpirun -np $nprocs -machinefile $HOSTFILE $PGM
rc=$?
else
echo "machine must be 'wcoss', 'theia', or 'gaea', got $machine"
rc=1
fi
echo "exiting runmpi..."
exit $rc