From 52bd203dacec50e511d1b6c5eaf5af182ee8a404 Mon Sep 17 00:00:00 2001 From: Salman Chen Date: Sat, 15 Feb 2025 23:33:40 +0800 Subject: [PATCH 1/8] Update fomalhaut.config Chore: Update CUDA and HDF5 path for Fomalhaut --- configs/fomalhaut.config | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/fomalhaut.config b/configs/fomalhaut.config index 5cde54533c..7b3fcc4dc6 100644 --- a/configs/fomalhaut.config +++ b/configs/fomalhaut.config @@ -1,8 +1,8 @@ # NTHU-fomalhaut (openmpi-gnu) -CUDA_PATH /usr/local/cuda-10.0 +CUDA_PATH /cluster/software/cuda/10.1.105/gcc--8.3.0 # For CUDA 10 FFTW2_PATH /home/hyschive/software/fftw/2.1.5-openmpi-gnu MPI_PATH /storage/app/gnu/openmpi-3.1.3-gcc8 -HDF5_PATH /storage/app/gnu/hdf5-1.8.21-gcc8 +HDF5_PATH /cluster/software/hdf5/1.10.5/gcc--8.3.0/serial # compilers CXX g++ @@ -35,5 +35,5 @@ NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true NVCCFLAG_POT -Xptxas -dlcm=ca #gpu -GPU_COMPUTE_CAPABILITY 860 # GeForce RTX 3080 and RTX A4000 -#GPU_COMPUTE_CAPABILITY 750 # GeForce RTX 2080 Ti +# GPU_COMPUTE_CAPABILITY 860 # GeForce RTX 3080 and RTX A4000 (use CUDA 11) +GPU_COMPUTE_CAPABILITY 750 # GeForce RTX 2080 Ti (default with CUDA 10) From 60caafc39d77f6158342b321f9bc4079b0626e1d Mon Sep 17 00:00:00 2001 From: Salman Chen Date: Fri, 21 Feb 2025 13:48:09 +0800 Subject: [PATCH 2/8] Chore: Update fomalhaut.config with recommended values --- configs/fomalhaut.config | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/configs/fomalhaut.config b/configs/fomalhaut.config index 7b3fcc4dc6..5f5bd7d941 100644 --- a/configs/fomalhaut.config +++ b/configs/fomalhaut.config @@ -1,8 +1,13 @@ # NTHU-fomalhaut (openmpi-gnu) -CUDA_PATH /cluster/software/cuda/10.1.105/gcc--8.3.0 # For CUDA 10 -FFTW2_PATH /home/hyschive/software/fftw/2.1.5-openmpi-gnu -MPI_PATH /storage/app/gnu/openmpi-3.1.3-gcc8 -HDF5_PATH /cluster/software/hdf5/1.10.5/gcc--8.3.0/serial +CUDA_PATH /cluster/software/cuda-10.0 +#CUDA_PATH /cluster/software/cuda/11.6/gcc--8.3.0 +FFTW2_PATH /cluster/software/fftw/2.1.5/gcc--8.3.0/openmpi--3.1.4 +FFTW3_PATH /cluster/software/fftw/3.3.8/gcc--8.3.0/openmpi--3.1.4 +MPI_PATH /cluster/software/openmpi/3.1.4/gcc--8.3.0 +HDF5_PATH /cluster/software/hdf5-parallel/1.8.21/gcc--8.3.0/openmpi--3.1.4 +GRACKLE_PATH +GSL_PATH +LIBYT_PATH # compilers CXX g++ @@ -34,6 +39,6 @@ NVCCFLAG_COM -O3 NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true NVCCFLAG_POT -Xptxas -dlcm=ca -#gpu -# GPU_COMPUTE_CAPABILITY 860 # GeForce RTX 3080 and RTX A4000 (use CUDA 11) -GPU_COMPUTE_CAPABILITY 750 # GeForce RTX 2080 Ti (default with CUDA 10) +# GPU +GPU_COMPUTE_CAPABILITY 750 # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) +#GPU_COMPUTE_CAPABILITY 860 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) From ce13ac22a1cddf6a797e03c8247d724d4a949fb3 Mon Sep 17 00:00:00 2001 From: salmanhiro Date: Mon, 24 Feb 2025 20:05:25 +0800 Subject: [PATCH 3/8] Chore: Add options for SLURM job in fomalhaut example, fix typings --- configs/fomalhaut.config | 6 +++--- example/queue/submit_fomalhaut.job | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/configs/fomalhaut.config b/configs/fomalhaut.config index 5f5bd7d941..b90ac5dc19 100644 --- a/configs/fomalhaut.config +++ b/configs/fomalhaut.config @@ -1,8 +1,8 @@ # NTHU-fomalhaut (openmpi-gnu) CUDA_PATH /cluster/software/cuda-10.0 -#CUDA_PATH /cluster/software/cuda/11.6/gcc--8.3.0 +# CUDA_PATH /cluster/software/cuda/11.6/gcc--8.3.0 FFTW2_PATH /cluster/software/fftw/2.1.5/gcc--8.3.0/openmpi--3.1.4 -FFTW3_PATH /cluster/software/fftw/3.3.8/gcc--8.3.0/openmpi--3.1.4 +# FFTW3_PATH /cluster/software/fftw/3.3.8/gcc--8.3.0/openmpi--3.1.4 MPI_PATH /cluster/software/openmpi/3.1.4/gcc--8.3.0 HDF5_PATH /cluster/software/hdf5-parallel/1.8.21/gcc--8.3.0/openmpi--3.1.4 GRACKLE_PATH @@ -41,4 +41,4 @@ NVCCFLAG_POT -Xptxas -dlcm=ca # GPU GPU_COMPUTE_CAPABILITY 750 # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) -#GPU_COMPUTE_CAPABILITY 860 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) +# GPU_COMPUTE_CAPABILITY 860 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) diff --git a/example/queue/submit_fomalhaut.job b/example/queue/submit_fomalhaut.job index 52c8b285a9..9263c6a1cb 100644 --- a/example/queue/submit_fomalhaut.job +++ b/example/queue/submit_fomalhaut.job @@ -12,6 +12,8 @@ #Load module module purge -module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/2.1.5 hdf5-parallel/1.8.21 +module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/2.1.5 hdf5-parallel/1.8.21 # # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) +# module load gcc/8.3.0 cuda/11.0 openmpi/3.1.4 fftw/2.1.5 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) + mpirun -np 2 -map-by ppr:1:socket:pe=18 --report-bindings ./gamer 1>>log 2>&1 From 5e2c6e69061a598c39d5b5f059d1ebffbe01eb8e Mon Sep 17 00:00:00 2001 From: salmanhiro Date: Tue, 25 Feb 2025 12:26:34 +0800 Subject: [PATCH 4/8] Fix: Comments format --- configs/fomalhaut.config | 8 ++++---- example/queue/submit_fomalhaut.job | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/configs/fomalhaut.config b/configs/fomalhaut.config index b90ac5dc19..046562be4c 100644 --- a/configs/fomalhaut.config +++ b/configs/fomalhaut.config @@ -1,8 +1,8 @@ # NTHU-fomalhaut (openmpi-gnu) CUDA_PATH /cluster/software/cuda-10.0 -# CUDA_PATH /cluster/software/cuda/11.6/gcc--8.3.0 +#CUDA_PATH /cluster/software/cuda/11.6/gcc--8.3.0 FFTW2_PATH /cluster/software/fftw/2.1.5/gcc--8.3.0/openmpi--3.1.4 -# FFTW3_PATH /cluster/software/fftw/3.3.8/gcc--8.3.0/openmpi--3.1.4 +#FFTW3_PATH /cluster/software/fftw/3.3.8/gcc--8.3.0/openmpi--3.1.4 MPI_PATH /cluster/software/openmpi/3.1.4/gcc--8.3.0 HDF5_PATH /cluster/software/hdf5-parallel/1.8.21/gcc--8.3.0/openmpi--3.1.4 GRACKLE_PATH @@ -39,6 +39,6 @@ NVCCFLAG_COM -O3 NVCCFLAG_FLU -Xptxas -dlcm=ca -prec-div=false -ftz=true NVCCFLAG_POT -Xptxas -dlcm=ca -# GPU +# gpu GPU_COMPUTE_CAPABILITY 750 # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) -# GPU_COMPUTE_CAPABILITY 860 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) +#GPU_COMPUTE_CAPABILITY 860 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) diff --git a/example/queue/submit_fomalhaut.job b/example/queue/submit_fomalhaut.job index 9263c6a1cb..fa9f82654a 100644 --- a/example/queue/submit_fomalhaut.job +++ b/example/queue/submit_fomalhaut.job @@ -12,8 +12,8 @@ #Load module module purge -module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/2.1.5 hdf5-parallel/1.8.21 # # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) -# module load gcc/8.3.0 cuda/11.0 openmpi/3.1.4 fftw/2.1.5 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) +module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) +#module load gcc/8.3.0 cuda/11.0 openmpi/3.1.4 fftw/2.1.5 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) mpirun -np 2 -map-by ppr:1:socket:pe=18 --report-bindings ./gamer 1>>log 2>&1 From ee4380faca8a8e0315a65225ed7f88fced1faaeb Mon Sep 17 00:00:00 2001 From: salmanhiro Date: Tue, 25 Feb 2025 12:28:15 +0800 Subject: [PATCH 5/8] Chore: Change FFTW to 3.3.8 (default in fomalhaut) --- example/queue/submit_fomalhaut.job | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/queue/submit_fomalhaut.job b/example/queue/submit_fomalhaut.job index fa9f82654a..ae4ad9a521 100644 --- a/example/queue/submit_fomalhaut.job +++ b/example/queue/submit_fomalhaut.job @@ -13,7 +13,7 @@ #Load module module purge module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) -#module load gcc/8.3.0 cuda/11.0 openmpi/3.1.4 fftw/2.1.5 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) +#module load gcc/8.3.0 cuda/11.0 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) mpirun -np 2 -map-by ppr:1:socket:pe=18 --report-bindings ./gamer 1>>log 2>&1 From 4ee90e5ae9675e0070a3f26d098acbd25047802f Mon Sep 17 00:00:00 2001 From: salmanhiro Date: Tue, 25 Feb 2025 12:29:56 +0800 Subject: [PATCH 6/8] Chore: Change default FFTW --- configs/fomalhaut.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/fomalhaut.config b/configs/fomalhaut.config index 046562be4c..92ebacf2db 100644 --- a/configs/fomalhaut.config +++ b/configs/fomalhaut.config @@ -1,8 +1,8 @@ # NTHU-fomalhaut (openmpi-gnu) CUDA_PATH /cluster/software/cuda-10.0 #CUDA_PATH /cluster/software/cuda/11.6/gcc--8.3.0 -FFTW2_PATH /cluster/software/fftw/2.1.5/gcc--8.3.0/openmpi--3.1.4 -#FFTW3_PATH /cluster/software/fftw/3.3.8/gcc--8.3.0/openmpi--3.1.4 +#FFTW2_PATH /cluster/software/fftw/2.1.5/gcc--8.3.0/openmpi--3.1.4 +FFTW3_PATH /cluster/software/fftw/3.3.8/gcc--8.3.0/openmpi--3.1.4 MPI_PATH /cluster/software/openmpi/3.1.4/gcc--8.3.0 HDF5_PATH /cluster/software/hdf5-parallel/1.8.21/gcc--8.3.0/openmpi--3.1.4 GRACKLE_PATH From 449a6b39d3906caf1c51e8a47401f02800993102 Mon Sep 17 00:00:00 2001 From: salmanhiro Date: Tue, 25 Feb 2025 12:36:23 +0800 Subject: [PATCH 7/8] Fix: Change cuda 11 version --- example/queue/submit_fomalhaut.job | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/queue/submit_fomalhaut.job b/example/queue/submit_fomalhaut.job index ae4ad9a521..677484ebbc 100644 --- a/example/queue/submit_fomalhaut.job +++ b/example/queue/submit_fomalhaut.job @@ -13,7 +13,7 @@ #Load module module purge module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) -#module load gcc/8.3.0 cuda/11.0 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) +#module load gcc/8.3.0 cuda/11.6 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04) mpirun -np 2 -map-by ppr:1:socket:pe=18 --report-bindings ./gamer 1>>log 2>&1 From 1c343af4cf23911fe6df1f43f43d212f43d1def4 Mon Sep 17 00:00:00 2001 From: salmanhiro Date: Tue, 25 Feb 2025 12:51:25 +0800 Subject: [PATCH 8/8] Fix: Remove duplicate # --- example/queue/submit_fomalhaut.job | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/queue/submit_fomalhaut.job b/example/queue/submit_fomalhaut.job index 677484ebbc..f347d6f54c 100644 --- a/example/queue/submit_fomalhaut.job +++ b/example/queue/submit_fomalhaut.job @@ -12,7 +12,7 @@ #Load module module purge -module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) +module load gcc/8.3.0 cuda/10.0 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # GeForce RTX 2080 Ti (CUDA 10, g01 and g02) #module load gcc/8.3.0 cuda/11.6 openmpi/3.1.4 fftw/3.3.8 hdf5-parallel/1.8.21 # RTX A4000 and GeForce RTX 3080 (CUDA 11, g03 and g04)