Docs: update instructions for Adastra supercomputer (CINES, France)

Docs/source/install/hpc/adastra.rst

@@ -31,18 +31,26 @@ If you are new to this system, **please see the following resources**:
 Preparation
 -----------
 
+The following instructions will install WarpX in the ``$SHAREDHOMEDIR`` directory,
+which is shared among all the members of a given project. Due to the inode
+quota enforced for this machine, a shared installation of WarpX is advised.
+
 Use the following commands to download the WarpX source code:
 
 .. code-block:: bash
 
-   git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+   # If you have multiple projects, activate the project that you want to use with:
+   #
+   # myproject -a YOUR_PROJECT_NAME
+   #
+   git clone https://github.com/ECP-WarpX/WarpX.git $SHAREDHOMEDIR/src/warpx
 
-We use system software modules, add environment hints and further dependencies via the file ``$HOME/adastra_warpx.profile``.
+We use system software modules, add environment hints and further dependencies via the file ``$SHAREDHOMEDIR/adastra_warpx.profile``.
 Create it now:
 
 .. code-block:: bash
 
-   cp $HOME/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $HOME/adastra_warpx.profile
+   cp $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $SHAREDHOMEDIR/adastra_warpx.profile
 
 .. dropdown:: Script Details
    :color: light
@@ -53,23 +61,23 @@ Create it now:
       :language: bash
 
 Edit the 2nd line of this script, which sets the ``export proj=""`` variable using a text editor
-such as ``nano``, ``emacs``, or ``vim`` (all available by default on
-Adastra login nodes).
+such as ``nano``, ``emacs``, or ``vim`` (all available by default on Adastra login nodes) and
+uncomment the 3rd line (which sets ``$proj`` as the active project).
 
 .. important::
 
    Now, and as the first step on future logins to Adastra, activate these environment settings:
 
    .. code-block:: bash
 
-      source $HOME/adastra_warpx.profile
+      source $SHAREDHOMEDIR/adastra_warpx.profile
 
 Finally, since Adastra does not yet provide software modules for some of our dependencies, install them once:
 
 .. code-block:: bash
 
-   bash $HOME/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh
-   source $HOME/sw/adastra/gpu/venvs/warpx-adastra/bin/activate
+   bash $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh
+   source $SHAREDHOMEDIR/sw/adastra/gpu/venvs/warpx-adastra/bin/activate
 
 .. dropdown:: Script Details
    :color: light
@@ -89,13 +97,13 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd $SHAREDHOMEDIR/src/warpx
    rm -rf build_adastra
 
    cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
    cmake --build build_adastra -j 16
 
-The WarpX application executables are now in ``$HOME/src/warpx/build_adastra/bin/``.
+The WarpX application executables are now in ``$SHAREDHOMEDIR/src/warpx/build_adastra/bin/``.
 Additionally, the following commands will install WarpX as a Python module:
 
 .. code-block:: bash
@@ -119,7 +127,7 @@ If you already installed WarpX in the past and want to update it, start by getti
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd $SHAREDHOMEDIR/src/warpx
 
    # read the output of this command - does it look ok?
    git status

Tools/machines/adastra-cines/adastra_warpx.profile.example

@@ -1,30 +1,33 @@
-# please set your project account
+# please set your project account and uncomment the following two lines
 #export proj=your_project_id
+#myproject -a $proj
 
 # required dependencies
+module purge
+module load cpe/23.12
 module load craype-accel-amd-gfx90a craype-x86-trento
 module load PrgEnv-cray
+module load CCE-GPU-3.0.0
 module load amd-mixed/5.2.3
-module load CPE-23.02-cce-15.0.1-GPU-softs
 
 # optional: for PSATD in RZ geometry support
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH
-export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH
+export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH
 
 # optional: for QED lookup table generation support
-module load boost/1.81.0-mpi-python3
+module load boost/1.83.0-mpi-python3
 
 # optional: for openPMD support
 module load cray-hdf5-parallel
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
 
 export PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3/bin:${PATH}
 
 # optional: for Python bindings or libEnsemble
-module load cray-python/3.9.13.1
+module load cray-python/3.11.5
 
 # fix system defaults: do not escape $ with a \ on tab completion
 shopt -s direxpand
@@ -49,7 +52,4 @@ export AMREX_AMD_ARCH=gfx90a
 # compiler environment hints
 export CC=$(which cc)
 export CXX=$(which CC)
-export FC=$(which ftn)
-export CFLAGS="-I${ROCM_PATH}/include"
-export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
-export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64"
+export FC=$(which amdflang)

Tools/machines/adastra-cines/install_dependencies.sh

@@ -20,7 +20,7 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo
 
 # Remove old dependencies #####################################################
 #
-SW_DIR="${HOME}/sw/adastra/gpu"
+SW_DIR="${SHAREDHOMEDIR}/sw/adastra/gpu"
 rm -rf ${SW_DIR}
 mkdir -p ${SW_DIR}
 
@@ -34,62 +34,62 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true
 #
 
 # BLAS++ (for PSATD+RZ)
-if [ -d $HOME/src/blaspp ]
+if [ -d $SHAREDHOMEDIR/src/blaspp ]
 then
-  cd $HOME/src/blaspp
+  cd $SHAREDHOMEDIR/src/blaspp
   git fetch --prune
   git checkout master
   git pull
   cd -
 else
-  git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp
+  git clone https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp
 fi
-rm -rf $HOME/src/blaspp-adastra-gpu-build
-CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master
-cmake --build $HOME/src/blaspp-adastra-gpu-build --target install --parallel 16
-rm -rf $HOME/src/blaspp-adastra-gpu-build
+rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build
+CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master
+cmake --build $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build --target install --parallel 16
+rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build
 
 # LAPACK++ (for PSATD+RZ)
-if [ -d $HOME/src/lapackpp ]
+if [ -d $SHAREDHOMEDIR/src/lapackpp ]
 then
-  cd $HOME/src/lapackpp
+  cd $SHAREDHOMEDIR/src/lapackpp
   git fetch --prune
   git checkout master
   git pull
   cd -
 else
-  git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp
+  git clone https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp
 fi
-rm -rf $HOME/src/lapackpp-adastra-gpu-build
-CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master
-cmake --build $HOME/src/lapackpp-adastra-gpu-build --target install --parallel 16
-rm -rf $HOME/src/lapackpp-adastra-gpu-build
+rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build
+CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master
+cmake --build $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build --target install --parallel 16
+rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build
 
 # c-blosc (I/O compression, for OpenPMD)
-if [ -d $HOME/src/c-blosc ]
+if [ -d $SHAREDHOMEDIR/src/c-blosc ]
 then
   # git repository is already there
   :
 else
-  git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc
+  git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $SHAREDHOMEDIR/src/c-blosc
 fi
-rm -rf $HOME/src/c-blosc-ad-build
-cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/c-blosc-1.21.1
-cmake --build $HOME/src/c-blosc-ad-build --target install --parallel 16
-rm -rf $HOME/src/c-blosc-ad-build
+rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build
+cmake -S $SHAREDHOMEDIR/src/c-blosc -B $SHAREDHOMEDIR/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1
+cmake --build $SHAREDHOMEDIR/src/c-blosc-ad-build --target install --parallel 16
+rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build
 
 # ADIOS2 v. 2.8.3 (for OpenPMD)
-if [ -d $HOME/src/adios2 ]
+if [ -d $SHAREDHOMEDIR/src/adios2 ]
 then
   # git repository is already there
   :
 else
-  git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2
+  git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $SHAREDHOMEDIR/src/adios2
 fi
-rm -rf $HOME/src/adios2-ad-build
-cmake -S $HOME/src/adios2 -B $HOME/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/adios2-2.8.3
-cmake --build $HOME/src/adios2-ad-build --target install -j 16
-rm -rf $HOME/src/adios2-ad-build
+rm -rf $SHAREDHOMEDIR/src/adios2-ad-build
+cmake -S $SHAREDHOMEDIR/src/adios2 -B $SHAREDHOMEDIR/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3
+cmake --build $SHAREDHOMEDIR/src/adios2-ad-build --target install -j 16
+rm -rf $SHAREDHOMEDIR/src/adios2-ad-build
 
 
 # Python ######################################################################
@@ -114,9 +114,9 @@ python3 -m pip install --upgrade openpmd-api
 python3 -m pip install --upgrade matplotlib
 python3 -m pip install --upgrade yt
 # install or update WarpX dependencies such as picmistandard
-python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt
+python3 -m pip install --upgrade -r $SHAREDHOMEDIR/src/warpx/requirements.txt
 # optional: for libEnsemble
-python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt
+python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/LibEnsemble/requirements.txt
 # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch)
 #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2
-#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt
+#python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/optimas/requirements.txt

Tools/machines/adastra-cines/submit.sh

@@ -1,22 +1,26 @@
 #!/bin/bash
-#SBATCH --job-name=warpx
 #SBATCH --account=<account_to_charge>
+#SBATCH --job-name=warpx
 #SBATCH --constraint=MI250
-#SBATCH --ntasks-per-node=8 --cpus-per-task=8 --gpus-per-node=8
-#SBATCH --threads-per-core=1 # --hint=nomultithread
+#SBATCH --nodes=2
 #SBATCH --exclusive
 #SBATCH --output=%x-%j.out
 #SBATCH --time=00:10:00
-#SBATCH --nodes=2
 
 module purge
 
-# Architecture
+# A CrayPE environment version
+module load cpe/23.12
+# An architecture
 module load craype-accel-amd-gfx90a craype-x86-trento
 # A compiler to target the architecture
 module load PrgEnv-cray
 # Some architecture related libraries and tools
-module load amd-mixed
+module load CCE-GPU-3.0.0
+module load amd-mixed/5.2.3
+
+date
+module list
 
 export MPICH_GPU_SUPPORT_ENABLED=1
 
@@ -36,4 +40,5 @@ export OMP_NUM_THREADS=1
 export WARPX_NMPI_PER_NODE=8
 export TOTAL_NMPI=$(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} ))
 srun -N${SLURM_JOB_NUM_NODES} -n${TOTAL_NMPI} --ntasks-per-node=${WARPX_NMPI_PER_NODE} \
+     --cpus-per-task=8 --threads-per-core=1 --gpu-bind=closest \
     ./warpx inputs > output.txt