diff --git a/configs/forerunnerI_gnu.config b/configs/forerunnerI_gnu.config new file mode 100644 index 0000000000..7c19d94743 --- /dev/null +++ b/configs/forerunnerI_gnu.config @@ -0,0 +1,50 @@ +# forerunnerI +### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found: +### module use /home/d07222009/module_CALAB +### module load gnu_13.2.0/gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.0 gnu_13.2.0/openucx/1.18.0 +### +CUDA_PATH +FFTW2_PATH +FFTW3_PATH $(FFTW3_PATH) +MPI_PATH $(MPI_PATH) +HDF5_PATH $(HDF5_PATH) +GRACKLE_PATH +GSL_PATH $(GSL_PATH) +LIBYT_PATH +CUFFTDX_PATH + +# compilers +CXX g++-13.2.0 +CXX_MPI mpicxx + +# flags +CXXFLAG -g +CXXFLAG -O3 +#CXXFLAG -std=c++11 +#CXXFLAG -Ofast +CXXFLAG -Wall +CXXFLAG -Wextra +CXXFLAG -Wno-unused-variable +CXXFLAG -Wno-unused-parameter +CXXFLAG -Wno-maybe-uninitialized +CXXFLAG -Wno-unused-but-set-variable +CXXFLAG -Wno-unused-function +CXXFLAG -Wno-unused-result +CXXFLAG -Wno-implicit-fallthrough +CXXFLAG -Wno-parentheses +CXXFLAG -Wno-unknown-pragmas + +OPENMPFLAG -fopenmp + +LIBFLAG + +NVCCFLAG_COM -O3 +#NVCCFLAG_COM -use_fast_math +NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true +NVCCFLAG_POT -Xptxas -dlcm=ca + +# for debugging +#CXXFLAG -fstack-protector-all +#CXXFLAG -fstack-protector-strong +#CXXFLAG -fsanitize=undefined -fsanitize=address +#LIBFLAG -fsanitize=undefined -fsanitize=address diff --git a/configs/forerunnerI_intel.config b/configs/forerunnerI_intel.config new file mode 100644 index 0000000000..a13575e4cc --- /dev/null +++ b/configs/forerunnerI_intel.config @@ -0,0 +1,42 @@ +# forerunnerI +### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found: +### module use /home/d07222009/module_CALAB +### module load intel/2024_01_46 oneapi_2024/fftw/3.3.10 oneapi_2024/gsl/2.8.0 oneapi_2024/hdf5/1.14.4 oneapi_2024/openmpi/5.0.0 oneapi_2024/openucx/1.18.0 +### +CUDA_PATH +FFTW2_PATH +FFTW3_PATH $(FFTW3_PATH) +MPI_PATH $(MPI_PATH) +HDF5_PATH $(HDF5_PATH) +GRACKLE_PATH +GSL_PATH $(GSL_PATH) +LIBYT_PATH +CUFFTDX_PATH + +# compilers +CXX icpx +CXX_MPI mpicxx + +# flags +# for warning flags fro oneapi, see: https://www.intel.com/content/dam/develop/external/us/en/documents/oneapi_dpcpp_cpp_compiler.pdf +CXXFLAG -g +CXXFLAG -O3 +CXXFLAG -fp-model precise -fstack-protector-all +#CXXFLAG -std=c++11 +#CXXFLAG -gxx-name=YOUR_G++ +CXXFLAG -Werror -Wfatal-errors -Woverflow +CXXFLAG -Wno-uninitialized -Wno-absolute-value -Wno-unknown-pragmas -diag-disable 3180 -diag-disable 10441 + +OPENMPFLAG -qopenmp + +LIBFLAG -limf + +NVCCFLAG_COM -O3 +#NVCCFLAG_COM -use_fast_math +NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true +NVCCFLAG_POT -Xptxas -dlcm=ca + +# for debugging +#CXXFLAG -fstack-protector-all +#CXXFLAG -fstack-protector-strong # somehow it can capture issues not detected by -fstack-protector-all +#LIBFLAG -lssp diff --git a/configs/taiwania3_gnu.config b/configs/taiwania3_gnu.config new file mode 100644 index 0000000000..b4c5b1f49d --- /dev/null +++ b/configs/taiwania3_gnu.config @@ -0,0 +1,51 @@ +# TaiwaniaIII +### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found: +### module use /home/d07222009/module_CALAB +### module load gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.5 gnu_13.2.0/openucx/1.18.0 +### +CUDA_PATH +FFTW2_PATH +FFTW3_PATH $(FFTW3_PATH) +MPI_PATH $(MPI_PATH) +HDF5_PATH $(HDF5_PATH) +GRACKLE_PATH +GSL_PATH $(GSL_PATH) +LIBYT_PATH +CUFFTDX_PATH + +# compilers +CXX g++ +CXX_MPI mpicxx + +# flags +CXXFLAG -g +CXXFLAG -O3 +CXXFLAG -std=c++11 +#CXXFLAG -Ofast +CXXFLAG -Wall +CXXFLAG -Wextra +CXXFLAG -Wno-unused-variable +CXXFLAG -Wno-unused-parameter +CXXFLAG -Wno-maybe-uninitialized +CXXFLAG -Wno-unused-but-set-variable +CXXFLAG -Wno-unused-function +CXXFLAG -Wno-unused-result +CXXFLAG -Wno-implicit-fallthrough +CXXFLAG -Wno-parentheses +CXXFLAG -Wno-unknown-pragmas +CXXFLAG -Wno-cast-function-type + +OPENMPFLAG -fopenmp + +LIBFLAG + +NVCCFLAG_COM -O3 +#NVCCFLAG_COM -use_fast_math +NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true +NVCCFLAG_POT -Xptxas -dlcm=ca + +# for debugging +#CXXFLAG -fstack-protector-all +#CXXFLAG -fstack-protector-strong +#CXXFLAG -fsanitize=undefined -fsanitize=address +#LIBFLAG -fsanitize=undefined -fsanitize=address diff --git a/configs/taiwania3_intel.config b/configs/taiwania3_intel.config new file mode 100644 index 0000000000..2d3465c989 --- /dev/null +++ b/configs/taiwania3_intel.config @@ -0,0 +1,41 @@ +# TaiwaniaIII +### Caution!! Load modules under module_CALAB on TwnIII first before compiling, such that FFTW3_PATH, MPI_PATH, HDF5_PATH and GSL_PATH can be found: +### module use /home/d07222009/module_CALAB +### module load intel/2024 intel_2024/fftw/3.3.10 intel_2024/gsl/2.8.0 intel_2024/hdf5/1.14.4 intel_2024/openmpi/5.0.5 intel_2024/openucx/1.18.0 +### +CUDA_PATH +FFTW2_PATH +FFTW3_PATH $(FFTW3_PATH) +MPI_PATH $(MPI_PATH) +HDF5_PATH $(HDF5_PATH) +GRACKLE_PATH +GSL_PATH $(GSL_PATH) +LIBYT_PATH +CUFFTDX_PATH + +# compilers +CXX icpx +CXX_MPI mpicxx + +# flags +CXXFLAG -g +CXXFLAG -O3 +CXXFLAG -fp-model precise -fstack-protector-all +#CXXFLAG -std=c++11 +#CXXFLAG -gxx-name=YOUR_G++ +CXXFLAG -Werror -Wfatal-errors -Woverflow +CXXFLAG -Wno-uninitialized -Wno-absolute-value -Wno-unknown-pragmas -diag-disable 3180 -diag-disable 10441 + +OPENMPFLAG -qopenmp + +LIBFLAG -limf + +NVCCFLAG_COM -O3 +#NVCCFLAG_COM -use_fast_math +NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true +NVCCFLAG_POT -Xptxas -dlcm=ca + +# for debugging +#CXXFLAG -fstack-protector-all +#CXXFLAG -fstack-protector-strong # somehow it can capture issues not detected by -fstack-protector-all +#LIBFLAG -lssp diff --git a/example/queue/submit_forerunnerI_gnu.job b/example/queue/submit_forerunnerI_gnu.job new file mode 100644 index 0000000000..63a1cc1540 --- /dev/null +++ b/example/queue/submit_forerunnerI_gnu.job @@ -0,0 +1,32 @@ +#!/bin/bash + +################################################################# +# OpenMPI(compiled by GNU) job script example # +################################################################# + +#SBATCH --account=ACCOUNT # (-A) Account/project number +#SBATCH --job-name=JOB_NAME # (-J) Job name +#SBATCH --partition=ct448 # (-p) Specific slurm partition +#SBATCH --nodes=2 # (-N) Maximum number of nodes to be allocated +#SBATCH --ntasks=32 # (-n) Number of total MPI tasks (i.e. processes) +#SBATCH --cpus-per-task=7 # (-c) Number of cores per MPI task +#SBATCH --ntasks-per-node=16 # Maximum number of tasks on each node +#SBATCH --mem=482000M # Memory limit per compute node for the job. Do not use with mem-per-cpu flag. +#SBATCH --time=2:00:00 # (-t) Wall time limit (days-hrs:min:sec) +##SBATCH -o job.%j.out +##SBATCH -e job.%j.err +##SBATCH --mail-type=BEGIN,END,FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) +##SBATCH --mail-user=EMAIL_ADDRESS # Where to send mail. Set this to your email address +##SBATCH --exclude=icpnp[101-102,255-256] # Example for excluding specified nodes + +LOG_FILE=log + +module purge +module use /home/d07222009/module_CALAB +module load gnu_13.2.0/gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.0 gnu_13.2.0/openucx/1.18.0 +module list 1>>$LOG_FILE 2>&1 + +# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option +# There are 8 NUMA nodes on each node, 4 per socket +mpirun -map-by ppr:2:numa:pe=7 --report-bindings ./gamer 1>>$LOG_FILE 2>&1 +echo "=============================================================" >> $LOG_FILE diff --git a/example/queue/submit_forerunnerI_intel.job b/example/queue/submit_forerunnerI_intel.job new file mode 100644 index 0000000000..c2c5c2b888 --- /dev/null +++ b/example/queue/submit_forerunnerI_intel.job @@ -0,0 +1,32 @@ +#!/bin/bash + +################################################################# +# OpenMPI(compiled by OneAPI) job script example # +################################################################# + +#SBATCH --account=ACCOUNT # (-A) Account/project number +#SBATCH --job-name=JOB_NAME # (-J) Job name +#SBATCH --partition=ct448 # (-p) Specific slurm partition +#SBATCH --nodes=2 # (-N) Maximum number of nodes to be allocated +#SBATCH --ntasks=32 # (-n) Number of total MPI tasks (i.e. processes) +#SBATCH --cpus-per-task=7 # (-c) Number of cores per MPI task +#SBATCH --ntasks-per-node=16 # Maximum number of tasks on each node +#SBATCH --mem=482000M # Memory limit per compute node for the job. Do not use with mem-per-cpu flag. +#SBATCH --time=2:00:00 # (-t) Wall time limit (days-hrs:min:sec) +##SBATCH -o job.%j.out +##SBATCH -e job.%j.err +##SBATCH --mail-type=BEGIN,END,FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) +##SBATCH --mail-user=EMAIL_ADDRESS # Where to send mail. Set this to your email address +##SBATCH --exclude=icpnp[101-102,255-256] # Example for excluding specified nodes + +LOG_FILE=log + +module purge +module use /home/d07222009/module_CALAB +module load intel/2024_01_46 oneapi_2024/fftw/3.3.10 oneapi_2024/gsl/2.8.0 oneapi_2024/hdf5/1.14.4 oneapi_2024/openmpi/5.0.0 oneapi_2024/openucx/1.18.0 +module list 1>>$LOG_FILE 2>&1 + +# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option +# There are 8 NUMA nodes on each node, 4 per socket +mpirun -map-by ppr:2:numa:pe=7 --report-bindings ./gamer 1>>$LOG_FILE 2>&1 +echo "=============================================================" >> $LOG_FILE diff --git a/example/queue/submit_taiwania3_gnu.job b/example/queue/submit_taiwania3_gnu.job index fe789002c2..87b02f081a 100644 --- a/example/queue/submit_taiwania3_gnu.job +++ b/example/queue/submit_taiwania3_gnu.job @@ -1,9 +1,9 @@ +#!/bin/bash + ############################################### # GNU MPI job script example # ############################################### -#!/bin/bash - #SBATCH --account=ACCOUNT # (-A) Account/project number #SBATCH --job-name=JOB_NAME # (-J) Job name #SBATCH --partition=ctest # (-p) Specific slurm partition @@ -13,19 +13,20 @@ #SBATCH --cpus-per-task=14 # (-c) Number of cores per MPI task #SBATCH --mem=162400M # Memory limit per compute node for the job. Do not use with mem-per-cpu flag. #SBATCH --time=00:30:00 # (-t) Wall time limit (days-hrs:min:sec) -##SBATCH -o log_taiwania_III +##SBATCH -o log ##SBATCH -e job.%j.err ##SBATCH --mail-type=BEGIN,END,FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) ##SBATCH --mail-user=EMAIL_ADDRESS # Where to send mail. Set this to your email address #SBATCH --exclude=cpn[3001-3120,3241-3360] # Exclude large-memory nodes -LOG_FILE=log_taiwania_III_gnu_9.4.0 +LOG_FILE=log module purge -module load compiler/gcc/9.4.0 OpenMPI/4.1.1 -module list >> $LOG_FILE - -export LD_LIBRARY_PATH="/opt/ohpc/Taiwania3/libs/gcc485/ompi410/hdf5-1.12/lib:FFTW_PATH/lib:$LD_LIBRARY_PATH" +module use /home/d07222009/module_CALAB +module load gcc/13.2.0 gnu_13.2.0/fftw/3.3.10 gnu_13.2.0/gsl/2.8.0 gnu_13.2.0/hdf5/1.14.4 gnu_13.2.0/openmpi/5.0.5 gnu_13.2.0/openucx/1.18.0 +module list 1>>$LOG_FILE 2>&1 -mpirun -map-by ppr:2:socket:pe=14 --report-bindings ./gamer 1>>$LOG_FILE 2>&1 -echo "=============================================================" >> $LOG_FILE +# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option +# There are 2 NUMA nodes on each node, 1 per socket +mpirun -map-by ppr:2:numa:pe=14 --report-bindings ./gamer 1>>$LOG_FILE 2>&1 +echo "=============================================================" 1>>$LOG_FILE 2>&1 diff --git a/example/queue/submit_taiwania3_intel.job b/example/queue/submit_taiwania3_intel.job index 74b419a679..b42e4a7947 100644 --- a/example/queue/submit_taiwania3_intel.job +++ b/example/queue/submit_taiwania3_intel.job @@ -1,9 +1,9 @@ +#!/bin/bash + ############################################### # Intel MPI job script example # ############################################### -#!/bin/bash - #SBATCH --account=ACCOUNT # (-A) Account/project number #SBATCH --job-name=JOB_NAME # (-J) Job name #SBATCH --partition=ctest # (-p) Specific slurm partition @@ -13,20 +13,20 @@ #SBATCH --cpus-per-task=14 # (-c) Number of cores per MPI task #SBATCH --mem=162400M # Memory limit per compute node for the job. Do not use with mem-per-cpu flag. #SBATCH --time=00:30:00 # (-t) Wall time limit (days-hrs:min:sec) -##SBATCH -o log_taiwania_III +##SBATCH -o log ##SBATCH -e job.%j.err #SBATCH --mail-type=BEGIN,END,FAIL # Mail events (NONE, BEGIN, END, FAIL, ALL) #SBATCH --mail-user=EMAIL_ACCOUNT # Where to send mail. Set this to your email address #SBATCH --exclude=cpn[3001-3120,3241-3360] # Exclude large-memory nodes -LOG_FILE=log_taiwania_III_intel_2018 +LOG_FILE=log module purge -module load compiler/intel/2018u4 IntelMPI/2018u4 -module list >> LOG_FILE -export LD_LIBRARY_PATH="FFTW_PATH/lib:$LD_LIBRARY_PATH" -export UCX_TLS="ud,dc,shm,self" +module use /home/d07222009/module_CALAB +module load intel/2024 intel_2024/fftw/3.3.10 intel_2024/gsl/2.8.0 intel_2024/hdf5/1.14.4 intel_2024/openmpi/5.0.5 intel_2024/openucx/1.18.0 +module list 1>>$LOG_FILE 2>&1 -mpiexec.hydra -bootstrap slurm -n $SLURM_NTASKS ./gamer >> $LOG_FILE 2>&1 -#mpirun -map-by ppr:2:socket:pe=14 -print-rank-map ./gamer 1>>$LOG_FILE 2>&1 -echo "=============================================================" >> $LOG_FILE +# See: https://docs.open-mpi.org/en/v5.0.x/man-openmpi/man1/mpirun.1.html#the-map-by-option +# There are 2 NUMA nodes on each node, 1 per socket +mpirun -map-by ppr:2:numa:pe=14 --report-bindings ./gamer 1>>$LOG_FILE 2>&1 +echo "=============================================================" 1>>$LOG_FILE 2>&1