#!/bin/bash

# ChemShell SLURM batch script
# you.lu@stfc.ac.uk

#SBATCH -o $CHEMSH_JOBNAME.o$SLURM_JOBID
#SBATCH -e $CHEMSH_JOBNAME.e$SLURM_JOBID

echo "Node list is" $SLURM_NODELIST

# number of OpenMP threads per MPI process
export OMP_NUM_THREADS=$CHEMSH_NTHREADS

# see: https://www.cfd-online.com/Forums/openfoam-solving/84084-decomposed-case-2-cores-not-working-2.html#post292700
export OMPI_MCA_btl_tcp_if_exclude=lo,virbr0

#export SLURM_MPI_TYPE=pmix_v3
#export I_MPI_PROCESS_MANAGER=hydra
#export I_MPI_FABRICS=shm:tmi
#export I_MPI_FALLBACK=0
#export I_MPI_PMI_LIBRARY=

# some things that might be useful
echo
echo SLURM parameters:
echo
echo SLURM_JOBID         = $SLURM_JOBID
echo SLURM_JOB_NODELIST  = $SLURM_JOB_NODELIST
echo SLURM_NNODES        = $SLURM_NNODES
echo SLURMTMPDIR         = $SLURMTMPDIR
echo SLURM_ARRAYID       = $SLURM_ARRAYID
echo SLURM_ARRAY_JOB_ID  = $SLURM_ARRAY_JOB_ID
echo SLURM_ARRAY_TASK_ID = $SLURM_ARRAY_TASK_ID

NPROCS=`srun --nodes=${SLURM_NNODES} bash -c 'hostname' | wc -l`

echo
echo ChemShell runtime parameters:
echo
echo working directory = $SLURM_SUBMIT_DIR
echo MPIEXEC           = $MPI_EXEC
echo PYTHONPATH        = $PYTHONPATH
echo NPROCS            = $CHEMSH_NPROCS
echo NPROCS_MM         = $CHEMSH_NPROCS_MM
echo NTHREADS          = $CHEMSH_NTHREADS
echo WRAPPER           = $CHEMSH_WRAPPER
echo EXEC              = $CHEMSH_EXEC
echo NWORKGROUPS       = $CHEMSH_NWORKGROUPS
echo CHEMSH_GA_MEMORY  = $CHEMSH_GA_MEMORY
echo INPUT             = $CHEMSH_INPUT
echo OUTPUT            = $CHEMSH_OUTPUT
echo Print level       = $CHEMSH_DEBUG
echo

# TWK 17/7/2023 Test driver mode
if [[ "$CHEMSH_TEST_DRIVER" != "" ]]
then
    echo
    echo Running the ChemShell test suite:
    echo
    echo CHEMSH_TEST_DRIVER = $CHEMSH_TEST_DRIVER
    echo CHEMSH_TESTSETS = $CHEMSH_TESTSETS
    echo
    CHEMSH_TEST="${CHEMSH_TEST_DRIVER} --sets ${CHEMSH_TESTSETS} --exe "
else
    CHEMSH_TEST=""
fi

# the output will go to .err
module list
echo
echo Executing command:
echo $CHEMSH_TEST $CHEMSH_WRAPPER $MPI_EXEC -n $CHEMSH_NPROCS $CHEMSH_EXEC -nwg $CHEMSH_NWORKGROUPS -npmm $CHEMSH_NPROCS_MM -gm $CHEMSH_GA_MEMORY -dbg $CHEMSH_DEBUG $CHEMSH_INPUT \> $CHEMSH_OUTPUT

$CHEMSH_TEST $CHEMSH_WRAPPER $MPI_EXEC -n $CHEMSH_NPROCS $CHEMSH_EXEC -nwg $CHEMSH_NWORKGROUPS -npmm $CHEMSH_NPROCS_MM -gm $CHEMSH_GA_MEMORY -dbg $CHEMSH_DEBUG $CHEMSH_INPUT > $CHEMSH_OUTPUT
