CI: Upgrade HIP Workflows, Fix New Bug (#5707)

- [x] Update HIP CI workflow to run on Ubuntu 24.04
- [x] Add HIP version to CLI, default 6.3.2
- [x] Fix bug caused by using `std::optional` on device

---------

Co-authored-by: Edoardo Zoni <ezoni@lbl.gov>
Co-authored-by: David Grote <grote1@llnl.gov>

Author: 3 people

.github/workflows/dependencies/hip.sh

@@ -28,7 +28,9 @@ sudo apt-key add rocm.gpg.key
 
 source /etc/os-release # set UBUNTU_CODENAME: focal or jammy or ...
 
-echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-latest} ${UBUNTU_CODENAME} main" \
+VERSION=${1-6.3.2}
+
+echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${VERSION} ${UBUNTU_CODENAME} main" \
   | sudo tee /etc/apt/sources.list.d/rocm.list
 echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \
   | sudo tee -a /etc/profile.d/rocm.sh
@@ -50,12 +52,16 @@ sudo apt-get install -y --no-install-recommends \
     libzstd-dev     \
     ninja-build     \
     openmpi-bin     \
-    rocm-dev        \
-    rocfft-dev      \
-    rocprim-dev     \
-    rocsparse-dev   \
-    rocrand-dev     \
-    hiprand-dev
+    rocm-dev${VERSION}        \
+    roctracer-dev${VERSION}   \
+    rocprofiler-dev${VERSION} \
+    rocrand-dev${VERSION}     \
+    rocfft-dev${VERSION}      \
+    rocprim-dev${VERSION}     \
+    rocsparse-dev${VERSION}
+
+# hiprand-dev is a new package that does not exist in old versions
+sudo apt-get install -y --no-install-recommends hiprand-dev${VERSION} || true
 
 # ccache
 $(dirname "$0")/ccache.sh
@@ -69,11 +75,6 @@ which clang++
 export CXX=$(which clang++)
 export CC=$(which clang)
 
-# "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
-#   https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
-#   https://github.com/open-mpi/ompi/issues/9317
-export LDFLAGS="-lopen-pal"
-
 # cmake-easyinstall
 #
 sudo curl -L -o /usr/local/bin/cmake-easyinstall https://raw.githubusercontent.com/ax3l/cmake-easyinstall/main/cmake-easyinstall

.github/workflows/hip.yml

@@ -16,7 +16,7 @@ concurrency:
 jobs:
   build_hip_3d_sp:
     name: HIP 3D SP
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     env:
       CXXFLAGS: "-Werror -Wno-deprecated-declarations -Wno-error=pass-failed"
       CMAKE_GENERATOR: Ninja
@@ -25,7 +25,7 @@ jobs:
     - uses: actions/checkout@v4
     - name: install dependencies
       shell: bash
-      run: .github/workflows/dependencies/hip.sh
+      run: .github/workflows/dependencies/hip.sh 6.3.2
     - name: CCache Cache
       uses: actions/cache@v4
       with:
@@ -48,11 +48,6 @@ jobs:
         export CXX=$(which clang++)
         export CC=$(which clang)
 
-        # "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
-        #   https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
-        #   https://github.com/open-mpi/ompi/issues/9317
-        export LDFLAGS="-lopen-pal"
-
         cmake -S . -B build_sp \
           -DCMAKE_VERBOSE_MAKEFILE=ON \
           -DAMReX_AMD_ARCH=gfx900     \
@@ -75,7 +70,7 @@ jobs:
 
   build_hip_2d_dp:
     name: HIP 2D DP
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     env:
       CXXFLAGS: "-Werror -Wno-deprecated-declarations -Wno-error=pass-failed"
       CMAKE_GENERATOR: Ninja
@@ -84,7 +79,7 @@ jobs:
     - uses: actions/checkout@v4
     - name: install dependencies
       shell: bash
-      run: .github/workflows/dependencies/hip.sh
+      run: .github/workflows/dependencies/hip.sh 6.3.2
     - name: CCache Cache
       uses: actions/cache@v4
       with:
@@ -107,11 +102,6 @@ jobs:
         export CXX=$(which clang++)
         export CC=$(which clang)
 
-        # "mpic++ --showme" forgets open-pal in Ubuntu 20.04 + OpenMPI 4.0.3
-        #   https://bugs.launchpad.net/ubuntu/+source/openmpi/+bug/1941786
-        #   https://github.com/open-mpi/ompi/issues/9317
-        export LDFLAGS="-lopen-pal"
-
         cmake -S . -B build_2d \
           -DCMAKE_VERBOSE_MAKEFILE=ON \
           -DAMReX_AMD_ARCH=gfx900     \

Source/AcceleratorLattice/LatticeElementFinder.H

@@ -114,6 +114,9 @@ struct LatticeElementFinderDevice
                                     AcceleratorLattice const& accelerator_lattice,
                                     LatticeElementFinder const & h_finder);
 
+    /* Whether the class has been initialized */
+    bool m_initialized = false;
+
     /* Size and location of the index lookup table */
     amrex::Real m_zmin;
     amrex::Real m_dz;

Source/AcceleratorLattice/LatticeElementFinder.cpp

@@ -92,6 +92,8 @@ LatticeElementFinderDevice::InitLatticeElementFinderDevice (WarpXParIter const&
                                                             LatticeElementFinder const & h_finder)
 {
 
+    m_initialized = true;
+
     auto& warpx = WarpX::GetInstance();
 
     int const lev = a_pti.GetLevel();

Source/Particles/Gather/GetExternalFields.H

@@ -15,8 +15,6 @@
 #include <AMReX_Parser.H>
 #include <AMReX_REAL.H>
 
-#include <optional>
-
 
 /** \brief Functor class that assigns external
  *         field values (E and B) to particles.
@@ -56,10 +54,10 @@ struct GetExternalEBField
     const amrex::ParticleReal* AMREX_RESTRICT m_uy = nullptr;
     const amrex::ParticleReal* AMREX_RESTRICT m_uz = nullptr;
 
-    std::optional<LatticeElementFinderDevice> d_lattice_element_finder;
+    LatticeElementFinderDevice d_lattice_element_finder;
 
     [[nodiscard]] AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
-    bool isNoOp () const { return (m_Etype == None && m_Btype == None && !d_lattice_element_finder.has_value()); }
+    bool isNoOp () const { return (m_Etype == None && m_Btype == None && !d_lattice_element_finder.m_initialized); }
 
     AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
     void operator () (long i,
@@ -72,10 +70,9 @@ struct GetExternalEBField
     {
         using namespace amrex::literals;
 
-        if (d_lattice_element_finder) {
-            // Note that the "*" is needed since d_lattice_element_finder is optional
-            (*d_lattice_element_finder)(i, field_Ex, field_Ey, field_Ez,
-                                           field_Bx, field_By, field_Bz);
+        if (d_lattice_element_finder.m_initialized) {
+            d_lattice_element_finder(i, field_Ex, field_Ey, field_Ez,
+                                        field_Bx, field_By, field_Bz);
         }
 
         if (m_Etype == None && m_Btype == None) { return; }