Merge branch 'development' into deposition_vandb

JustinRayAngus · JustinRayAngus · commit 139957b41c72 · 2024-02-01T15:20:33.000-08:00
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -43,7 +43,15 @@ set_cxx17_superbuild()
 # this is an optional tool that stores compiled object files; allows fast
 # re-builds even with "make clean" in between. Mainly used to store AMReX
 # objects
-set_ccache()
+if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
+    set(WarpX_CCACHE_DEFAULT ON)
+else()
+    set(WarpX_CCACHE_DEFAULT OFF)  # we are a subproject in a superbuild
+endif()
+option(WarpX_CCACHE "Enable ccache for faster rebuilds" ${WarpX_CCACHE_DEFAULT})
+if(WarpX_CCACHE)
+    set_ccache()
+endif()
 
 
 # Output Directories ##########################################################
@@ -149,6 +157,7 @@ endif()
 # this defined the variable BUILD_TESTING which is ON by default
 #include(CTest)
 
+
 # Dependencies ################################################################
 #
 
diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst
@@ -116,7 +116,7 @@ By default, the most important dependencies of WarpX are automatically downloade
 CMake Option                  Default & Values                               Description
 ============================= ============================================== ===========================================================
 ``BUILD_SHARED_LIBS``         ON/**OFF**                                     `Build shared libraries for dependencies <https://cmake.org/cmake/help/latest/variable/BUILD_SHARED_LIBS.html>`__
-``CCACHE_PROGRAM``            First found ``ccache`` executable.             Set to ``-DCCACHE_PROGRAM=NO`` to disable CCache.
+``WarpX_CCACHE``              **ON**/OFF                                     Search and use CCache to speed up rebuilds.
 ``AMReX_CUDA_PTX_VERBOSE``    ON/**OFF**                                     Print CUDA code generation statistics from ``ptxas``.
 ``WarpX_amrex_src``           *None*                                         Path to AMReX source directory (preferred if set)
 ``WarpX_amrex_repo``          ``https://github.com/AMReX-Codes/amrex.git``   Repository URI to pull and build AMReX from
diff --git a/Docs/source/install/hpc/adastra.rst b/Docs/source/install/hpc/adastra.rst
@@ -31,18 +31,26 @@ If you are new to this system, **please see the following resources**:
 Preparation
 -----------
 
+The following instructions will install WarpX in the ``$SHAREDHOMEDIR`` directory,
+which is shared among all the members of a given project. Due to the inode
+quota enforced for this machine, a shared installation of WarpX is advised.
+
 Use the following commands to download the WarpX source code:
 
 .. code-block:: bash
 
-   git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+   # If you have multiple projects, activate the project that you want to use with:
+   #
+   # myproject -a YOUR_PROJECT_NAME
+   #
+   git clone https://github.com/ECP-WarpX/WarpX.git $SHAREDHOMEDIR/src/warpx
 
-We use system software modules, add environment hints and further dependencies via the file ``$HOME/adastra_warpx.profile``.
+We use system software modules, add environment hints and further dependencies via the file ``$SHAREDHOMEDIR/adastra_warpx.profile``.
 Create it now:
 
 .. code-block:: bash
 
-   cp $HOME/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $HOME/adastra_warpx.profile
+   cp $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/adastra_warpx.profile.example $SHAREDHOMEDIR/adastra_warpx.profile
 
 .. dropdown:: Script Details
    :color: light
@@ -53,23 +61,23 @@ Create it now:
       :language: bash
 
 Edit the 2nd line of this script, which sets the ``export proj=""`` variable using a text editor
-such as ``nano``, ``emacs``, or ``vim`` (all available by default on
-Adastra login nodes).
+such as ``nano``, ``emacs``, or ``vim`` (all available by default on Adastra login nodes) and
+uncomment the 3rd line (which sets ``$proj`` as the active project).
 
 .. important::
 
    Now, and as the first step on future logins to Adastra, activate these environment settings:
 
    .. code-block:: bash
 
-      source $HOME/adastra_warpx.profile
+      source $SHAREDHOMEDIR/adastra_warpx.profile
 
 Finally, since Adastra does not yet provide software modules for some of our dependencies, install them once:
 
 .. code-block:: bash
 
-   bash $HOME/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh
-   source $HOME/sw/adastra/gpu/venvs/warpx-adastra/bin/activate
+   bash $SHAREDHOMEDIR/src/warpx/Tools/machines/adastra-cines/install_dependencies.sh
+   source $SHAREDHOMEDIR/sw/adastra/gpu/venvs/warpx-adastra/bin/activate
 
 .. dropdown:: Script Details
    :color: light
@@ -89,13 +97,13 @@ Use the following :ref:`cmake commands <building-cmake>` to compile the applicat
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd $SHAREDHOMEDIR/src/warpx
    rm -rf build_adastra
 
    cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3"
    cmake --build build_adastra -j 16
 
-The WarpX application executables are now in ``$HOME/src/warpx/build_adastra/bin/``.
+The WarpX application executables are now in ``$SHAREDHOMEDIR/src/warpx/build_adastra/bin/``.
 Additionally, the following commands will install WarpX as a Python module:
 
 .. code-block:: bash
@@ -119,7 +127,7 @@ If you already installed WarpX in the past and want to update it, start by getti
 
 .. code-block:: bash
 
-   cd $HOME/src/warpx
+   cd $SHAREDHOMEDIR/src/warpx
 
    # read the output of this command - does it look ok?
    git status
diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H
@@ -1142,9 +1142,11 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition<PIdx>& GetPosition,
  *        particles positions are determined and in how the particle gamma is calculated.
  *
  * \tparam depos_order  deposition order
+ * \param xp_n,yp_n,zp_n  Pointer to arrays of particle position at time level n.
  * \param GetPosition  A functor for returning the particle position.
  * \param wp           Pointer to array of particle weights.
- * \param uxp,uyp,uzp  Pointer to arrays of particle momentum.
+ * \param uxp_n,uyp_n,uzp_n  Pointer to arrays of particle momentum at time level n.
+ * \param uxp_nph,uyp_nph,uzp_nph  Pointer to arrays of particle momentum at time level n + 1/2.
  * \param ion_lev      Pointer to array of particle ionization level. This is
                        required to have the charge of each macroparticle
                        since q is a scalar. For non-ionizable species,
diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example
@@ -1,30 +1,33 @@
-# please set your project account
+# please set your project account and uncomment the following two lines
 #export proj=your_project_id
+#myproject -a $proj
 
 # required dependencies
+module purge
+module load cpe/23.12
 module load craype-accel-amd-gfx90a craype-x86-trento
 module load PrgEnv-cray
+module load CCE-GPU-3.0.0
 module load amd-mixed/5.2.3
-module load CPE-23.02-cce-15.0.1-GPU-softs
 
 # optional: for PSATD in RZ geometry support
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH
-export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH
-export LD_LIBRARY_PATH=${HOME}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH
+export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH
+export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH
 
 # optional: for QED lookup table generation support
-module load boost/1.81.0-mpi-python3
+module load boost/1.83.0-mpi-python3
 
 # optional: for openPMD support
 module load cray-hdf5-parallel
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
-export CMAKE_PREFIX_PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
+export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
 
 export PATH=${HOME}/sw/adastra/gpu/adios2-2.8.3/bin:${PATH}
 
 # optional: for Python bindings or libEnsemble
-module load cray-python/3.9.13.1
+module load cray-python/3.11.5
 
 # fix system defaults: do not escape $ with a \ on tab completion
 shopt -s direxpand
@@ -49,7 +52,4 @@ export AMREX_AMD_ARCH=gfx90a
 # compiler environment hints
 export CC=$(which cc)
 export CXX=$(which CC)
-export FC=$(which ftn)
-export CFLAGS="-I${ROCM_PATH}/include"
-export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
-export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64"
+export FC=$(which amdflang)
diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh
@@ -20,7 +20,7 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo
 
 # Remove old dependencies #####################################################
 #
-SW_DIR="${HOME}/sw/adastra/gpu"
+SW_DIR="${SHAREDHOMEDIR}/sw/adastra/gpu"
 rm -rf ${SW_DIR}
 mkdir -p ${SW_DIR}
 
@@ -34,62 +34,62 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true
 #
 
 # BLAS++ (for PSATD+RZ)
-if [ -d $HOME/src/blaspp ]
+if [ -d $SHAREDHOMEDIR/src/blaspp ]
 then
-  cd $HOME/src/blaspp
+  cd $SHAREDHOMEDIR/src/blaspp
   git fetch --prune
   git checkout master
   git pull
   cd -
 else
-  git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp
+  git clone https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp
 fi
-rm -rf $HOME/src/blaspp-adastra-gpu-build
-CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master
-cmake --build $HOME/src/blaspp-adastra-gpu-build --target install --parallel 16
-rm -rf $HOME/src/blaspp-adastra-gpu-build
+rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build
+CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master
+cmake --build $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build --target install --parallel 16
+rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build
 
 # LAPACK++ (for PSATD+RZ)
-if [ -d $HOME/src/lapackpp ]
+if [ -d $SHAREDHOMEDIR/src/lapackpp ]
 then
-  cd $HOME/src/lapackpp
+  cd $SHAREDHOMEDIR/src/lapackpp
   git fetch --prune
   git checkout master
   git pull
   cd -
 else
-  git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp
+  git clone https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp
 fi
-rm -rf $HOME/src/lapackpp-adastra-gpu-build
-CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master
-cmake --build $HOME/src/lapackpp-adastra-gpu-build --target install --parallel 16
-rm -rf $HOME/src/lapackpp-adastra-gpu-build
+rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build
+CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master
+cmake --build $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build --target install --parallel 16
+rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build
 
 # c-blosc (I/O compression, for OpenPMD)
-if [ -d $HOME/src/c-blosc ]
+if [ -d $SHAREDHOMEDIR/src/c-blosc ]
 then
   # git repository is already there
   :
 else
-  git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc
+  git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $SHAREDHOMEDIR/src/c-blosc
 fi
-rm -rf $HOME/src/c-blosc-ad-build
-cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/c-blosc-1.21.1
-cmake --build $HOME/src/c-blosc-ad-build --target install --parallel 16
-rm -rf $HOME/src/c-blosc-ad-build
+rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build
+cmake -S $SHAREDHOMEDIR/src/c-blosc -B $SHAREDHOMEDIR/src/c-blosc-ad-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1
+cmake --build $SHAREDHOMEDIR/src/c-blosc-ad-build --target install --parallel 16
+rm -rf $SHAREDHOMEDIR/src/c-blosc-ad-build
 
 # ADIOS2 v. 2.8.3 (for OpenPMD)
-if [ -d $HOME/src/adios2 ]
+if [ -d $SHAREDHOMEDIR/src/adios2 ]
 then
   # git repository is already there
   :
 else
-  git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2
+  git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $SHAREDHOMEDIR/src/adios2
 fi
-rm -rf $HOME/src/adios2-ad-build
-cmake -S $HOME/src/adios2 -B $HOME/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/adastra/gpu/adios2-2.8.3
-cmake --build $HOME/src/adios2-ad-build --target install -j 16
-rm -rf $HOME/src/adios2-ad-build
+rm -rf $SHAREDHOMEDIR/src/adios2-ad-build
+cmake -S $SHAREDHOMEDIR/src/adios2 -B $SHAREDHOMEDIR/src/adios2-ad-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3
+cmake --build $SHAREDHOMEDIR/src/adios2-ad-build --target install -j 16
+rm -rf $SHAREDHOMEDIR/src/adios2-ad-build
 
 
 # Python ######################################################################
@@ -114,9 +114,9 @@ python3 -m pip install --upgrade openpmd-api
 python3 -m pip install --upgrade matplotlib
 python3 -m pip install --upgrade yt
 # install or update WarpX dependencies such as picmistandard
-python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt
+python3 -m pip install --upgrade -r $SHAREDHOMEDIR/src/warpx/requirements.txt
 # optional: for libEnsemble
-python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt
+python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/LibEnsemble/requirements.txt
 # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch)
 #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2
-#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt
+#python3 -m pip install -r $SHAREDHOMEDIR/src/warpx/Tools/optimas/requirements.txt
diff --git a/Tools/machines/adastra-cines/submit.sh b/Tools/machines/adastra-cines/submit.sh
@@ -1,22 +1,26 @@
 #!/bin/bash
-#SBATCH --job-name=warpx
 #SBATCH --account=<account_to_charge>
+#SBATCH --job-name=warpx
 #SBATCH --constraint=MI250
-#SBATCH --ntasks-per-node=8 --cpus-per-task=8 --gpus-per-node=8
-#SBATCH --threads-per-core=1 # --hint=nomultithread
+#SBATCH --nodes=2
 #SBATCH --exclusive
 #SBATCH --output=%x-%j.out
 #SBATCH --time=00:10:00
-#SBATCH --nodes=2
 
 module purge
 
-# Architecture
+# A CrayPE environment version
+module load cpe/23.12
+# An architecture
 module load craype-accel-amd-gfx90a craype-x86-trento
 # A compiler to target the architecture
 module load PrgEnv-cray
 # Some architecture related libraries and tools
-module load amd-mixed
+module load CCE-GPU-3.0.0
+module load amd-mixed/5.2.3
+
+date
+module list
 
 export MPICH_GPU_SUPPORT_ENABLED=1
 
@@ -36,4 +40,5 @@ export OMP_NUM_THREADS=1
 export WARPX_NMPI_PER_NODE=8
 export TOTAL_NMPI=$(( ${SLURM_JOB_NUM_NODES} * ${WARPX_NMPI_PER_NODE} ))
 srun -N${SLURM_JOB_NUM_NODES} -n${TOTAL_NMPI} --ntasks-per-node=${WARPX_NMPI_PER_NODE} \
+     --cpus-per-task=8 --threads-per-core=1 --gpu-bind=closest \
     ./warpx inputs > output.txt
diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example
@@ -2,9 +2,9 @@
 #export proj=<yourProject>
 
 # required dependencies
-module load LUMI/23.03  partition/G
+module load LUMI/23.09  partition/G
 module load rocm/5.2.3  # waiting for 5.5 for next bump
-module load buildtools/23.03
+module load buildtools/23.09
 
 # optional: just an additional text editor
 module load nano
@@ -16,16 +16,16 @@ export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH
 export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH
 
 # optional: for QED lookup table generation support
-module load Boost/1.81.0-cpeCray-23.03
+module load Boost/1.82.0-cpeCray-23.09
 
 # optional: for openPMD support
-module load cray-hdf5/1.12.2.3
+module load cray-hdf5/1.12.2.7
 export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH
 export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH
 export PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3/bin:${PATH}
 
 # optional: for Python bindings or libEnsemble
-module load cray-python/3.9.13.1
+module load cray-python/3.10.10
 
 # an alias to request an interactive batch node for one hour
 #   for paralle execution, start on the batch node: srun <command>
@@ -41,9 +41,13 @@ export MPICH_GPU_SUPPORT_ENABLED=1
 export AMREX_AMD_ARCH=gfx90a
 
 # compiler environment hints
-export CC=$(which cc)
-export CXX=$(which CC)
-export FC=$(which ftn)
+# Warning: using the compiler wrappers cc and CC
+#          instead of amdclang and amdclang++
+#          currently results in a significant
+#          loss of performances
+export CC=$(which amdclang)
+export CXX=$(which amdclang++)
+export FC=$(which amdflang)
 export CFLAGS="-I${ROCM_PATH}/include"
 export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed"
 export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64"
diff --git a/Tools/machines/lumi-csc/submit.sh b/Tools/machines/lumi-csc/submit.sh
