From 5a7c32577fd921a8bdda6edae7c6aa1abfff1c86 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 20 Feb 2024 08:47:52 +0100 Subject: [PATCH] Docs: update instructions for Fugaku supercomputer (Riken, Japan) (#4656) * update instructions for the Fugaku supercomputer * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add missing instruction * update Fugaku instructions * fix bug * fix bug * fix bug * fix bug * fix bug * fix bug * fix issue with Fujitsu compiler * update install_dependencies script to save some space * delete BinaryCollision move constructor * update compilation instructions * update submission script --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/install/hpc/fugaku.rst | 67 +++++++++---------- .../BinaryCollision/BinaryCollision.H | 6 +- .../fugaku-riken/fugaku_warpx.profile.example | 31 +++++++++ .../fugaku-riken/install_dependencies.sh | 53 +++++++++++++++ Tools/machines/fugaku-riken/submit.sh | 6 ++ 5 files changed, 126 insertions(+), 37 deletions(-) create mode 100644 Tools/machines/fugaku-riken/fugaku_warpx.profile.example create mode 100755 Tools/machines/fugaku-riken/install_dependencies.sh diff --git a/Docs/source/install/hpc/fugaku.rst b/Docs/source/install/hpc/fugaku.rst index e70aeefa504..5e5fc57c3b0 100644 --- a/Docs/source/install/hpc/fugaku.rst +++ b/Docs/source/install/hpc/fugaku.rst @@ -14,8 +14,10 @@ If you are new to this system, **please see the following resources**: * `Fugaku user guide `__ -Installation ------------- +.. _building-fugaku-preparation: + +Preparation +----------- Use the following commands to download the WarpX source code and switch to the correct branch: @@ -31,45 +33,42 @@ Compiling WarpX on Fugaku is more practical on a compute node. Use the following pjsub --interact -L "elapse=02:00:00" -L "node=1" --sparam "wait-time=300" --mpi "max-proc-per-node=48" --all-mount-gfscache -Then, load ``cmake`` and ``ninja`` using ``spack``: +We use system software modules, add environment hints and further dependencies via the file ``$HOME/fugaku_warpx.profile``. +Create it now, modify it if needed, and source it (it will take few minutes): .. code-block:: bash - . /vol0004/apps/oss/spack/share/spack/setup-env.sh - spack load cmake@3.21.4%fj@4.8.1 arch=linux-rhel8-a64fx + cp $HOME/src/warpx/Tools/machines/fugaku-riken/fugaku_warpx.profile.example $HOME/fugaku_warpx.profile + source $HOME/fugaku_warpx.profile - # optional: faster builds - spack load ninja@1.11.1%fj@4.8.1 +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - # avoid harmless warning messages "[WARN] xos LPG [...]" - export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH + .. literalinclude:: ../../../../Tools/machines/fugaku-riken/fugaku_warpx.profile.example + :language: bash - -At this point we need to download and compile the libraries required for OpenPMD support: +Finally, since Fugaku does not yet provide software modules for some of our dependencies, install them once: .. code-block:: bash - export CC=$(which mpifcc) - export CXX=$(which mpiFCC) - export CFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" - export CXXFLAGS="${CFLAGS}" + bash $HOME/src/warpx/Tools/machines/fugaku-riken/install_dependencies.sh + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - export CMAKE_PREFIX_PATH=${HOME}/sw/a64fx-fj490/c-blosc-1.21.1-install:$CMAKE_PREFIX_PATH - export CMAKE_PREFIX_PATH=${HOME}/sw/a64fx-fj490/adios2-2.8.3-install:$CMAKE_PREFIX_PATH + .. literalinclude:: ../../../../Tools/machines/fugaku-riken/install_dependencies.sh + :language: bash - # c-blosc (I/O compression) - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git src/c-blosc - rm -rf src/c-blosc-fg-build - cmake -S src/c-blosc -B src/c-blosc-fg-build -DBUILD_SHARED_LIBS=OFF -DBUILD_SHARED=OFF -DBUILD_STATIC=ON -DBUILD_TESTS=OFF -DBUILD_FUZZERS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/a64fx-fj490/c-blosc-1.21.1-install - cmake --build src/c-blosc-fg-build --target install --parallel 24 +.. _building-fugaku-compilation: - # ADIOS2 - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git src/adios2 - rm -rf src/adios2-fg-build - cmake -S src/adios2 -B src/adios2-fg-build -DBUILD_SHARED_LIBS=OFF -DADIOS2_USE_Blosc=ON -DBUILD_TESTING=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/a64fx-fj490/adios2-2.8.3-install - cmake --build src/adios2-fg-build --target install -j 24 +Compilation +----------- -Finally, ``cd`` into the directory ``$HOME/src/warpx`` and use the following commands to compile: +Use the following :ref:`cmake commands ` to compile the application executable: .. code-block:: bash @@ -82,16 +81,14 @@ Finally, ``cd`` into the directory ``$HOME/src/warpx`` and use the following com export CXXFLAGS="-Nclang" cmake -S . -B build -DWarpX_COMPUTE=OMP \ - -DWarpX_DIMS="1;2;3" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_FLAGS_RELEASE="-Ofast -mllvm -polly -mllvm -polly-parallel" \ - -DAMReX_DIFFERENT_COMPILER=ON \ - -DWarpX_MPI_THREAD_MULTIPLE=OFF + -DWarpX_DIMS="1;2;3" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_FLAGS_RELEASE="-Ofast" \ + -DAMReX_DIFFERENT_COMPILER=ON \ + -DWarpX_MPI_THREAD_MULTIPLE=OFF cmake --build build -j 48 -The general :ref:`cmake compile-time options ` apply as usual. - **That's it!** A 3D WarpX executable is now in ``build/bin/`` and :ref:`can be run ` with a :ref:`3D example inputs file `. diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index c69f07acdb2..78381f22258 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -110,8 +110,10 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) noexcept = default; - BinaryCollision& operator= ( BinaryCollision&& ) noexcept = default; + + + BinaryCollision ( BinaryCollision&& ) = delete; + BinaryCollision& operator= ( BinaryCollision&& ) = delete; /** Perform the collisions * diff --git a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example new file mode 100644 index 00000000000..a430579a7d5 --- /dev/null +++ b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example @@ -0,0 +1,31 @@ +. /vol0004/apps/oss/spack/share/spack/setup-env.sh + +# required dependencies +spack load cmake@3.24.3%fj@4.10.0 arch=linux-rhel8-a64fx + +# avoid harmless warning messages "[WARN] xos LPG [...]" +export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH + +# optional: faster builds +spack load ninja@1.11.1%fj@4.10.0 + +# optional: for PSATD +spack load fujitsu-fftw + +# optional: for QED lookup table generation support +spack load boost@1.80.0%fj@4.8.1/zc5pwgc + +# optional: for openPMD support +spack load hdf5@1.12.2%fj@4.8.1/im6lxev +export CMAKE_PREFIX_PATH=${HOME}/sw/fugaku/a64fx/c-blosc-1.21.1-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/fugaku/a64fx/adios2-2.8.3-install:$CMAKE_PREFIX_PATH + +# compiler environment hints +export CC=$(which mpifcc) +export CXX=$(which mpiFCC) +export FC=$(which mpifrt) +export CFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" +export CXXFLAGS="-O3 -Nclang -Nlibomp -Klib -g -DNDEBUG" + +# avoid harmless warning messages "[WARN] xos LPG [...]" +export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH diff --git a/Tools/machines/fugaku-riken/install_dependencies.sh b/Tools/machines/fugaku-riken/install_dependencies.sh new file mode 100755 index 00000000000..3ceb45e4558 --- /dev/null +++ b/Tools/machines/fugaku-riken/install_dependencies.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Copyright 2023 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl, Luca Fedeli +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Remove old dependencies ##################################################### +# +SRC_DIR="${HOME}/src/" +SW_DIR="${HOME}/sw/fugaku/a64fx/" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} + +# General extra dependencies ################################################## +# + +# c-blosc (I/O compression) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - +else + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi + rm -rf ${SRC_DIR}/c-blosc-fugaku-build + cmake -S ${SRC_DIR}/c-blosc -B ${SRC_DIR}/c-blosc-fugaku-build -DBUILD_SHARED_LIBS=OFF -DBUILD_SHARED=OFF -DBUILD_STATIC=ON -DBUILD_TESTS=OFF -DBUILD_FUZZERS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1-install + cmake --build ${SRC_DIR}/c-blosc-fugaku-build --target install --parallel 48 + rm -rf ${SRC_DIR}/c-blosc-fugaku-build + +# ADIOS2 (I/O) +if [ -d ${SRC_DIR}/c-blosc ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - +else + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${SRC_DIR}/adios2-fugaku-build +cmake -S ${SRC_DIR}/adios2 -B ${SRC_DIR}/adios2-fugaku-build -DBUILD_SHARED_LIBS=OFF -DADIOS2_USE_Blosc=ON -DBUILD_TESTING=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3-install +cmake --build ${SRC_DIR}/adios2-fugaku-build --target install -j 48 +rm -rf ${SRC_DIR}/adios2-fugaku-build diff --git a/Tools/machines/fugaku-riken/submit.sh b/Tools/machines/fugaku-riken/submit.sh index caf254ba236..3768ed62f46 100644 --- a/Tools/machines/fugaku-riken/submit.sh +++ b/Tools/machines/fugaku-riken/submit.sh @@ -18,6 +18,12 @@ export INPUT="i.3d" export XOS_MMM_L_PAGING_POLICY=demand:demand:demand +# Add HDF5 library path to LD_LIBRARY_PATH +# This is done manually to avoid calling spack during the run, +# since this would take a significant amount of time. +export LD_LIBRARY_PATH=/vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/hdf5-1.12.2-im6lxevf76cu6cbzspi4itgz3l4gncjj/lib:$LD_LIBRARY_PATH + +# Broadcast WarpX executable to all the nodes llio_transfer ${EXE} mpiexec -stdout-proc ./output.%j/%/1000r/stdout -stderr-proc ./output.%j/%/1000r/stderr -n ${MPI_RANKS} ${EXE} ${INPUT}