#!/bin/bash

# This file is part of ABEL
# Copyright 2025, The ABEL Authors
# Authors: C.A.Lindstrøm(1), J.B.B.Chen(1), O.G.Finnerud(1), D.Kalvik(1), E.Hørlyk(1), A.Huebl(2), K.N.Sjobak(1), E.Adli(1)
# Affiliations: 1) University of Oslo, 2) LBNL
# License: GPL-3.0-or-later

#SBATCH -A $project_name
#SBATCH -J "HiPACE++"
#SBATCH -o hipace-%j.out
#SBATCH -t 00:30:00
#SBATCH --partition=$partition_name
#SBATCH --mem=$memory
#SBATCH --nodes=$num_nodes
#SBATCH --ntasks-per-node=$num_tasks_per_node
#SBATCH --gpus-per-node=$num_tasks_per_node

export MPICH_GPU_SUPPORT_ENABLED=1
export FI_MR_CACHE_MONITOR=memhooks  # alternative cache monitor
export ROCFFT_RTC_CACHE_PATH=/dev/null
export OMP_NUM_THREADS=1

cat << EOF > select_gpu
#!/bin/bash

export ROCR_VISIBLE_DEVICES=\$$SLURM_LOCALID
exec \$$*
EOF

chmod +x ./select_gpu

CPU_BIND="map_cpu:49,57,17,25,1,9,33,41"

#Don't touch the next line, it is used by the
# local runner to find the input filename
# filename_input "$filename_input"

srun --cpu-bind=$${CPU_BIND} ./select_gpu $hipace_binary_path $filename_input
rm -rf ./select_gpu
