#!/bin/bash
#
# This script is a sample slurm job submission for the basic app template.
set -e

#  --partition=$jobQueue \
#  --accunt=$jobAccount \
#  --qos=$jobQOS \
#  --nodes=$nodeCount \
#  --ntasks=$taskCount \
#  --ntasks-per-node=$no
###intel2018###
source /data/intel/bin/compilervars.sh intel64
source /data/intel/impi/2018.1.148/bin64/mpivars.sh
source /data/intel/mkl/bin/mklvars.sh intel64

if [ -z "$SUB_PROJECT" ]; then
    SUB_PROJECT_OPT=""
else
    SUB_PROJECT_OPT="--wckey=$SUB_PROJECT"
fi

export HOST_NUM=${HOST_NUM}
export MPI_PER_NODE=${MPI_PER_NODE}

WORK_DIR=`dirname ${EXE_FILE}`

cd ${WORK_DIR}
echo `pwd` > /tmp/test.log
NS=`python getNs.py ${HOST_NUM} ${MEM_PER_NODE} ${TOTAL_MEM_PERCENT}`


echo "HPLinpack benchmark input file
Innovative Computing Laboratory, University of Tennessee
HPL.out      output file name (if any)
8            device out (6=stdout,7=stderr,file)
1            # of problems sizes (N)
${NS}        Ns
1            # of NBs
${NB}      NBs
1            PMAP process mapping (0=Row-,1=Column-major)
1            # of process grids (P x Q)
${PS}           Ps
${QS}          Qs
16.0         threshold
1            # of panel fact
2 1 0        PFACTs (0=left, 1=Crout, 2=Right)
1            # of recursive stopping criterium
2            NBMINs (>= 1)
1            # of panels in recursion
2            NDIVs
1            # of recursive panel fact.
1 0 2        RFACTs (0=left, 1=Crout, 2=Right)
1            # of broadcast
0            BCASTs (0=1rg,1=1rM,2=2rg,3=2rM,4=Lng,5=LnM)
1            # of lookahead depth
0            DEPTHs (>=0)
0            SWAP (0=bin-exch,1=long,2=mix)
1            swapping threshold
1            L1 in (0=transposed,1=no-transposed) form
1            U  in (0=transposed,1=no-transposed) form
0            Equilibration (0=no,1=yes)
8            memory alignment in double (> 0)" > HPL.dat

sbatch \
  --mail-type=ALL \
  --partition=${PARTITION} \
  --job-name=${JOB_NAME} \
  --nodes=${HOST_NUM} \
  $SUB_PROJECT_OPT \
  -o slurm-%j.out \
  -e slurm-%j.err \
  ${EXE_FILE}
