Merge pull request #342 from koarakawaii/main-scripts_update_for_TwnI…

…II_and_forerunner Add/Update submit scripts and configure files for Forerunner/TaiwaniaIII
gamer-project · Dec 28, 2024 · acd94c9 · acd94c9
2 parents 73f2af4 + 31ee4c4
commit acd94c9
Show file tree

Hide file tree

Showing 8 changed files with 270 additions and 21 deletions.
diff --git a/configs/forerunnerI_gnu.config b/configs/forerunnerI_gnu.config
@@ -0,0 +1,50 @@
+# forerunnerI
+### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found:
+###     module use  /home/d07222009/module_CALAB
+###     module load gnu_13.2.0/gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.0  gnu_13.2.0/openucx/1.18.0
+###
+CUDA_PATH
+FFTW2_PATH
+FFTW3_PATH      $(FFTW3_PATH)
+MPI_PATH        $(MPI_PATH)
+HDF5_PATH       $(HDF5_PATH)
+GRACKLE_PATH
+GSL_PATH        $(GSL_PATH)
+LIBYT_PATH
+CUFFTDX_PATH
+
+# compilers
+CXX     g++-13.2.0
+CXX_MPI mpicxx
+
+# flags
+CXXFLAG -g
+CXXFLAG -O3
+#CXXFLAG -std=c++11
+#CXXFLAG -Ofast
+CXXFLAG -Wall
+CXXFLAG -Wextra
+CXXFLAG -Wno-unused-variable
+CXXFLAG -Wno-unused-parameter
+CXXFLAG -Wno-maybe-uninitialized
+CXXFLAG -Wno-unused-but-set-variable
+CXXFLAG -Wno-unused-function
+CXXFLAG -Wno-unused-result
+CXXFLAG -Wno-implicit-fallthrough
+CXXFLAG -Wno-parentheses
+CXXFLAG -Wno-unknown-pragmas
+
+OPENMPFLAG -fopenmp
+
+LIBFLAG
+
+NVCCFLAG_COM -O3
+#NVCCFLAG_COM -use_fast_math
+NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true
+NVCCFLAG_POT -Xptxas -dlcm=ca
+
+# for debugging
+#CXXFLAG -fstack-protector-all
+#CXXFLAG -fstack-protector-strong
+#CXXFLAG -fsanitize=undefined -fsanitize=address
+#LIBFLAG -fsanitize=undefined -fsanitize=address
diff --git a/configs/forerunnerI_intel.config b/configs/forerunnerI_intel.config
@@ -0,0 +1,42 @@
+# forerunnerI
+### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found:
+###     module use  /home/d07222009/module_CALAB
+###     module load intel/2024_01_46 oneapi_2024/fftw/3.3.10 oneapi_2024/gsl/2.8.0 oneapi_2024/hdf5/1.14.4 oneapi_2024/openmpi/5.0.0 oneapi_2024/openucx/1.18.0
+###
+CUDA_PATH
+FFTW2_PATH
+FFTW3_PATH      $(FFTW3_PATH)
+MPI_PATH        $(MPI_PATH)
+HDF5_PATH       $(HDF5_PATH)
+GRACKLE_PATH
+GSL_PATH        $(GSL_PATH)
+LIBYT_PATH
+CUFFTDX_PATH
+
+# compilers
+CXX     icpx
+CXX_MPI mpicxx
+
+# flags
+# for warning flags fro oneapi, see: https://www.intel.com/content/dam/develop/external/us/en/documents/oneapi_dpcpp_cpp_compiler.pdf
+CXXFLAG -g
+CXXFLAG -O3
+CXXFLAG -fp-model precise -fstack-protector-all
+#CXXFLAG -std=c++11
+#CXXFLAG -gxx-name=YOUR_G++
+CXXFLAG -Werror -Wfatal-errors -Woverflow
+CXXFLAG -Wno-uninitialized -Wno-absolute-value -Wno-unknown-pragmas -diag-disable 3180 -diag-disable 10441
+
+OPENMPFLAG -qopenmp
+
+LIBFLAG -limf
+
+NVCCFLAG_COM -O3
+#NVCCFLAG_COM -use_fast_math
+NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true
+NVCCFLAG_POT -Xptxas -dlcm=ca
+
+# for debugging
+#CXXFLAG -fstack-protector-all
+#CXXFLAG -fstack-protector-strong  # somehow it can capture issues not detected by -fstack-protector-all
+#LIBFLAG -lssp
diff --git a/configs/taiwania3_gnu.config b/configs/taiwania3_gnu.config
@@ -0,0 +1,51 @@
+# TaiwaniaIII
+### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found:
+###     module use  /home/d07222009/module_CALAB
+###     module load gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.5  gnu_13.2.0/openucx/1.18.0
+###
+CUDA_PATH
+FFTW2_PATH
+FFTW3_PATH      $(FFTW3_PATH)
+MPI_PATH        $(MPI_PATH)
+HDF5_PATH       $(HDF5_PATH)
+GRACKLE_PATH
+GSL_PATH        $(GSL_PATH)
+LIBYT_PATH
+CUFFTDX_PATH
+
+# compilers
+CXX     g++
+CXX_MPI mpicxx
+
+# flags
+CXXFLAG -g
+CXXFLAG -O3
+CXXFLAG -std=c++11
+#CXXFLAG -Ofast
+CXXFLAG -Wall
+CXXFLAG -Wextra
+CXXFLAG -Wno-unused-variable
+CXXFLAG -Wno-unused-parameter
+CXXFLAG -Wno-maybe-uninitialized
+CXXFLAG -Wno-unused-but-set-variable
+CXXFLAG -Wno-unused-function
+CXXFLAG -Wno-unused-result
+CXXFLAG -Wno-implicit-fallthrough
+CXXFLAG -Wno-parentheses
+CXXFLAG -Wno-unknown-pragmas
+CXXFLAG -Wno-cast-function-type
+
+OPENMPFLAG -fopenmp
+
+LIBFLAG
+
+NVCCFLAG_COM -O3
+#NVCCFLAG_COM -use_fast_math
+NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true
+NVCCFLAG_POT -Xptxas -dlcm=ca
+
+# for debugging
+#CXXFLAG -fstack-protector-all
+#CXXFLAG -fstack-protector-strong
+#CXXFLAG -fsanitize=undefined -fsanitize=address
+#LIBFLAG -fsanitize=undefined -fsanitize=address
diff --git a/configs/taiwania3_intel.config b/configs/taiwania3_intel.config
@@ -0,0 +1,41 @@
+# TaiwaniaIII
+### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found:
+###     module use  /home/d07222009/module_CALAB
+###     module load intel/2024 intel_2024/fftw/3.3.10 intel_2024/gsl/2.8.0 intel_2024/hdf5/1.14.4 intel_2024/openmpi/5.0.5 intel_2024/openucx/1.18.0
+###
+CUDA_PATH
+FFTW2_PATH
+FFTW3_PATH      $(FFTW3_PATH)
+MPI_PATH        $(MPI_PATH)
+HDF5_PATH       $(HDF5_PATH)
+GRACKLE_PATH
+GSL_PATH        $(GSL_PATH)
+LIBYT_PATH
+CUFFTDX_PATH
+
+# compilers
+CXX     icpx
+CXX_MPI mpicxx
+
+# flags
+CXXFLAG -g
+CXXFLAG -O3
+CXXFLAG -fp-model precise -fstack-protector-all
+#CXXFLAG -std=c++11
+#CXXFLAG -gxx-name=YOUR_G++
+CXXFLAG -Werror -Wfatal-errors -Woverflow
+CXXFLAG -Wno-uninitialized -Wno-absolute-value -Wno-unknown-pragmas -diag-disable 3180 -diag-disable 10441
+
+OPENMPFLAG -qopenmp
+
+LIBFLAG -limf
+
+NVCCFLAG_COM -O3
+#NVCCFLAG_COM -use_fast_math
+NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true
+NVCCFLAG_POT -Xptxas -dlcm=ca
+
+# for debugging
+#CXXFLAG -fstack-protector-all
+#CXXFLAG -fstack-protector-strong  # somehow it can capture issues not detected by -fstack-protector-all
+#LIBFLAG -lssp
diff --git a/example/queue/submit_forerunnerI_gnu.job b/example/queue/submit_forerunnerI_gnu.job
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#################################################################
+#       OpenMPI(compiled by GNU) job script example             #
+#################################################################
+
+#SBATCH --account=ACCOUNT                               # (-A) Account/project number
+#SBATCH --job-name=JOB_NAME                             # (-J) Job name
+#SBATCH --partition=ct448                               # (-p) Specific slurm partition
+#SBATCH --nodes=2                                       # (-N) Maximum number of nodes to be allocated
+#SBATCH --ntasks=32                                     # (-n) Number of total MPI tasks (i.e. processes)
+#SBATCH --cpus-per-task=7                               # (-c) Number of cores per MPI task
+#SBATCH --ntasks-per-node=16                            # Maximum number of tasks on each node
+#SBATCH --mem=482000M                                   # Memory limit per compute node for the job. Do not use with mem-per-cpu flag.
+#SBATCH --time=2:00:00                                  # (-t) Wall time limit (days-hrs:min:sec)
+##SBATCH -o job.%j.out
+##SBATCH -e job.%j.err
+##SBATCH --mail-type=BEGIN,END,FAIL                      # Mail events (NONE, BEGIN, END, FAIL, ALL)
+##SBATCH --mail-user=EMAIL_ADDRESS                       # Where to send mail.  Set this to your email address
+##SBATCH --exclude=icpnp[101-102,255-256]                # Example for excluding specified nodes
+
+LOG_FILE=log
+
+module purge
+module use /home/d07222009/module_CALAB
+module load gnu_13.2.0/gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.0 gnu_13.2.0/openucx/1.18.0
+module list 1>>$LOG_FILE 2>&1
+
+# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option
+# There are 8 NUMA nodes on each node, 4 per socket
+mpirun -map-by ppr:2:numa:pe=7 --report-bindings ./gamer 1>>$LOG_FILE 2>&1
+echo "=============================================================" >> $LOG_FILE
diff --git a/example/queue/submit_forerunnerI_intel.job b/example/queue/submit_forerunnerI_intel.job
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#################################################################
+#       OpenMPI(compiled by OneAPI) job script example          #
+#################################################################
+
+#SBATCH --account=ACCOUNT                               # (-A) Account/project number
+#SBATCH --job-name=JOB_NAME                             # (-J) Job name
+#SBATCH --partition=ct448                               # (-p) Specific slurm partition
+#SBATCH --nodes=2                                       # (-N) Maximum number of nodes to be allocated
+#SBATCH --ntasks=32                                     # (-n) Number of total MPI tasks (i.e. processes)
+#SBATCH --cpus-per-task=7                               # (-c) Number of cores per MPI task
+#SBATCH --ntasks-per-node=16                            # Maximum number of tasks on each node
+#SBATCH --mem=482000M                                   # Memory limit per compute node for the job. Do not use with mem-per-cpu flag.
+#SBATCH --time=2:00:00                                  # (-t) Wall time limit (days-hrs:min:sec)
+##SBATCH -o job.%j.out
+##SBATCH -e job.%j.err
+##SBATCH --mail-type=BEGIN,END,FAIL                      # Mail events (NONE, BEGIN, END, FAIL, ALL)
+##SBATCH --mail-user=EMAIL_ADDRESS                       # Where to send mail.  Set this to your email address
+##SBATCH --exclude=icpnp[101-102,255-256]                # Example for excluding specified nodes
+
+LOG_FILE=log
+
+module purge
+module use /home/d07222009/module_CALAB
+module load intel/2024_01_46 oneapi_2024/fftw/3.3.10 oneapi_2024/gsl/2.8.0 oneapi_2024/hdf5/1.14.4 oneapi_2024/openmpi/5.0.0 oneapi_2024/openucx/1.18.0
+module list 1>>$LOG_FILE 2>&1
+
+# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option
+# There are 8 NUMA nodes on each node, 4 per socket
+mpirun -map-by ppr:2:numa:pe=7 --report-bindings ./gamer 1>>$LOG_FILE 2>&1
+echo "=============================================================" >> $LOG_FILE
diff --git a/example/queue/submit_taiwania3_gnu.job b/example/queue/submit_taiwania3_gnu.job
@@ -1,9 +1,9 @@
+#!/bin/bash
+
 ###############################################
 #         GNU MPI job script example          #
 ###############################################
 
-#!/bin/bash
-
 #SBATCH --account=ACCOUNT                                   # (-A) Account/project number
 #SBATCH --job-name=JOB_NAME                                 # (-J) Job name
 #SBATCH --partition=ctest                                   # (-p) Specific slurm partition
@@ -13,19 +13,20 @@
 #SBATCH --cpus-per-task=14                                  # (-c) Number of cores per MPI task
 #SBATCH --mem=162400M                                       # Memory limit per compute node for the job. Do not use with mem-per-cpu flag.
 #SBATCH --time=00:30:00                                     # (-t) Wall time limit (days-hrs:min:sec)
-##SBATCH -o log_taiwania_III
+##SBATCH -o log
 ##SBATCH -e job.%j.err
 ##SBATCH --mail-type=BEGIN,END,FAIL                         # Mail events (NONE, BEGIN, END, FAIL, ALL)
 ##SBATCH --mail-user=EMAIL_ADDRESS                          # Where to send mail.  Set this to your email address
 #SBATCH --exclude=cpn[3001-3120,3241-3360]                  # Exclude large-memory nodes
 
-LOG_FILE=log_taiwania_III_gnu_9.4.0
+LOG_FILE=log
 
 module purge
-module load compiler/gcc/9.4.0 OpenMPI/4.1.1
-module list >> $LOG_FILE
-
-export LD_LIBRARY_PATH="/opt/ohpc/Taiwania3/libs/gcc485/ompi410/hdf5-1.12/lib:FFTW_PATH/lib:$LD_LIBRARY_PATH"
+module use /home/d07222009/module_CALAB
+module load gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.5  gnu_13.2.0/openucx/1.18.0
+module list 1>>$LOG_FILE 2>&1
 
-mpirun -map-by ppr:2:socket:pe=14 --report-bindings ./gamer 1>>$LOG_FILE 2>&1
-echo "=============================================================" >> $LOG_FILE
+# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option
+# There are 2 NUMA nodes on each node, 1 per socket
+mpirun -map-by ppr:2:numa:pe=14 --report-bindings ./gamer 1>>$LOG_FILE 2>&1
+echo "=============================================================" 1>>$LOG_FILE 2>&1
diff --git a/example/queue/submit_taiwania3_intel.job b/example/queue/submit_taiwania3_intel.job
@@ -1,9 +1,9 @@
+#!/bin/bash
+
 ###############################################
 #       Intel MPI job script example          #
 ###############################################
 
-#!/bin/bash
-
 #SBATCH --account=ACCOUNT                               # (-A) Account/project number
 #SBATCH --job-name=JOB_NAME                             # (-J) Job name
 #SBATCH --partition=ctest                               # (-p) Specific slurm partition
@@ -13,20 +13,20 @@
 #SBATCH --cpus-per-task=14                              # (-c) Number of cores per MPI task
 #SBATCH --mem=162400M                                   # Memory limit per compute node for the job. Do not use with mem-per-cpu flag.
 #SBATCH --time=00:30:00                                 # (-t) Wall time limit (days-hrs:min:sec)
-##SBATCH -o log_taiwania_III
+##SBATCH -o log
 ##SBATCH -e job.%j.err
 #SBATCH --mail-type=BEGIN,END,FAIL                      # Mail events (NONE, BEGIN, END, FAIL, ALL)
 #SBATCH --mail-user=EMAIL_ACCOUNT                       # Where to send mail.  Set this to your email address
 #SBATCH --exclude=cpn[3001-3120,3241-3360]              # Exclude large-memory nodes
 
-LOG_FILE=log_taiwania_III_intel_2018
+LOG_FILE=log
 
 module purge
-module load compiler/intel/2018u4 IntelMPI/2018u4
-module list >> LOG_FILE
-export LD_LIBRARY_PATH="FFTW_PATH/lib:$LD_LIBRARY_PATH"
-export UCX_TLS="ud,dc,shm,self"
+module use /home/d07222009/module_CALAB
+module load intel/2024 intel_2024/fftw/3.3.10 intel_2024/gsl/2.8.0 intel_2024/hdf5/1.14.4 intel_2024/openmpi/5.0.5 intel_2024/openucx/1.18.0
+module list 1>>$LOG_FILE 2>&1
 
-mpiexec.hydra -bootstrap slurm -n $SLURM_NTASKS ./gamer >> $LOG_FILE 2>&1
-#mpirun -map-by ppr:2:socket:pe=14 -print-rank-map ./gamer 1>>$LOG_FILE 2>&1
-echo "=============================================================" >> $LOG_FILE
+# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option
+# There are 2 NUMA nodes on each node, 1 per socket
+mpirun -map-by ppr:2:numa:pe=14 --report-bindings ./gamer 1>>$LOG_FILE 2>&1
+echo "=============================================================" 1>>$LOG_FILE 2>&1