#!/bin/bash --login

# ChemShell SLURM batch script
# you.lu@stfc.ac.uk

#SBATCH -o $CHEMSH_JOBNAME.o$SLURM_JOBID
#SBATCH -e $CHEMSH_JOBNAME.e$SLURM_JOBID

echo "Node list is" $SLURM_NODELIST


export I_MPI_PMI_LIBRARY=/usr/lib64/libpmi.so
#export I_MPI_FALLBACK_DEVICE=0
#export ALLINEA_MPI_INIT_PENDING=1
#export ALLINEA_MPI_INIT=MPI_Init

# some things that might be useful
echo
echo SLURM parameters:
echo
echo SLURM_JOBID         = $SLURM_JOBID
echo SLURM_JOB_NODELIST  = $SLURM_JOB_NODELIST
echo SLURM_NNODES        = $SLURM_NNODES
echo SLURMTMPDIR         = $SLURMTMPDIR
echo SLURM_ARRAYID       = $SLURM_ARRAYID
echo SLURM_ARRAY_JOB_ID  = $SLURM_ARRAY_JOB_ID
echo SLURM_ARRAY_TASK_ID = $SLURM_ARRAY_TASK_ID

NPROCS=`srun --nodes=${SLURM_NNODES} bash -c 'hostname' | wc -l`

echo
echo ChemShell runtime parameters:
echo
echo working directory = $SLURM_SUBMIT_DIR
echo PYTHONPATH        = $PYTHONPATH
echo NPROCS            = $NPROCS
echo WRAPPER           = $CHEMSH_WRAPPER
echo EXEC              = $CHEMSH_EXEC
echo NWORKGROUPS       = $CHEMSH_NWORKGROUPS
echo INPUT             = $CHEMSH_INPUT
echo OUTPUT            = $CHEMSH_OUTPUT
echo Print level       = $CHEMSH_DEBUG

# TWK 17/7/2023 Test driver mode
if [[ "$CHEMSH_TEST_DRIVER" != "" ]]
then
    echo
    echo Running the ChemShell test suite:
    echo
    echo CHEMSH_TEST_DRIVER = $CHEMSH_TEST_DRIVER
    echo CHEMSH_TESTSETS = $CHEMSH_TESTSETS
    echo
    CHEMSH_TEST="${CHEMSH_TEST_DRIVER} --sets ${CHEMSH_TESTSETS} --exe "
else
    CHEMSH_TEST=""
fi

# use srun preferably to use the allocated resources in a job step
echo Executing command:
echo $CHEMSH_TEST $CHEMSH_WRAPPER mpirun -n $CHEMSH_NPROCS $CHEMSH_EXEC -nwg $CHEMSH_NWORKGROUPS -npmm $CHEMSH_NPROCS_MM -dbg $CHEMSH_DEBUG $CHEMSH_INPUT \> $CHEMSH_OUTPUT

# YL 05/12/2018: srun fails somehow
#$CHEMSH_WRAPPER srun -n $CHEMSH_NPROCS $CHEMSH_EXEC -nwg $CHEMSH_NWORKGROUPS $CHEMSH_INPUT > $CHEMSH_OUTPUT
$CHEMSH_TEST $CHEMSH_WRAPPER mpirun -n $CHEMSH_NPROCS $CHEMSH_EXEC -nwg $CHEMSH_NWORKGROUPS -npmm $CHEMSH_NPROCS_MM -dbg $CHEMSH_DEBUG $CHEMSH_INPUT > $CHEMSH_OUTPUT

