From d0d95d9c5d47088df2268a9b574cc6a6fd10c2bd Mon Sep 17 00:00:00 2001 From: AlexanderSinn Date: Wed, 26 Feb 2025 20:45:55 +0100 Subject: [PATCH] Replace Elixir with Async Arena --- Src/Base/AMReX_GpuControl.cpp | 4 +-- Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H | 27 +++++++------------ .../MLMG/AMReX_MLNodeLaplacian_sten.cpp | 9 +++---- .../MLMG/AMReX_MLNodeLaplacian_sync.cpp | 27 +++++++------------ Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp | 9 +++---- .../MLMG/AMReX_MLTensorOp_grad.cpp | 9 +++---- 6 files changed, 29 insertions(+), 56 deletions(-) diff --git a/Src/Base/AMReX_GpuControl.cpp b/Src/Base/AMReX_GpuControl.cpp index 4c862b417c5..ddc5c6714f8 100644 --- a/Src/Base/AMReX_GpuControl.cpp +++ b/Src/Base/AMReX_GpuControl.cpp @@ -6,8 +6,8 @@ namespace amrex::Gpu { #if defined(AMREX_USE_GPU) bool in_launch_region = true; bool in_graph_region = false; -bool in_single_stream_region = false; -bool in_nosync_region = false; +bool in_single_stream_region = true; +bool in_nosync_region = true; #endif } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H index e7d5f2659b3..b6b9c3b82b5 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H @@ -1350,12 +1350,9 @@ MLCellLinOpT::reflux (int crse_amrlev, MF& res, const MF& crse_sol, const MF if (fluxreg.CrseHasWork(mfi)) { const Box& tbx = mfi.tilebox(); - AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp);, - flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp);, - flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp);); - AMREX_D_TERM(Elixir elifx = flux[0].elixir();, - Elixir elify = flux[1].elixir();, - Elixir elifz = flux[2].elixir();); + AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp,The_Async_Arena());, + flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp,The_Async_Arena());, + flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp,The_Async_Arena());); FFlux(crse_amrlev, mfi, pflux, crse_sol[mfi], Location::FaceCentroid); fluxreg.CrseAdd(mfi, cpflux, crse_dx, dt, RunOn::Gpu); } @@ -1371,12 +1368,9 @@ MLCellLinOpT::reflux (int crse_amrlev, MF& res, const MF& crse_sol, const MF { const Box& tbx = mfi.tilebox(); const int face_only = true; - AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp);, - flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp);, - flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp);); - AMREX_D_TERM(Elixir elifx = flux[0].elixir();, - Elixir elify = flux[1].elixir();, - Elixir elifz = flux[2].elixir();); + AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp,The_Async_Arena());, + flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp,The_Async_Arena());, + flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp,The_Async_Arena());); FFlux(fine_amrlev, mfi, pflux, fine_sol[mfi], Location::FaceCentroid, face_only); fluxreg.FineAdd(mfi, cpflux, fine_dx, dt, RunOn::Gpu); } @@ -1410,12 +1404,9 @@ MLCellLinOpT::compFlux (int amrlev, const Array& fluxes, for (MFIter mfi(sol, mfi_info); mfi.isValid(); ++mfi) { const Box& tbx = mfi.tilebox(); - AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp);, - flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp);, - flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp);); - AMREX_D_TERM(Elixir elifx = flux[0].elixir();, - Elixir elify = flux[1].elixir();, - Elixir elifz = flux[2].elixir();); + AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp,The_Async_Arena());, + flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp,The_Async_Arena());, + flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp,The_Async_Arena());); FFlux(amrlev, mfi, pflux, sol[mfi], loc); for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const Box& nbx = mfi.nodaltilebox(idim); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sten.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sten.cpp index db83ee8c979..938b74479f8 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sten.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sten.cpp @@ -115,12 +115,10 @@ MLNodeLaplacian::buildStencil () { const Box& btmp = ccbxg1 & sgfab_orig.box(); - cnfab.resize(ccbxg1, ncomp_c); - Elixir cneli = cnfab.elixir(); + cnfab.resize(ccbxg1, ncomp_c, The_Async_Arena()); Array4 const& cnarr = cnfab.array(); - sgfab.resize(ccbxg1); - Elixir sgeli = sgfab.elixir(); + sgfab.resize(ccbxg1, 1, The_Async_Arena()); Array4 const& sgarr = sgfab.array(); AMREX_HOST_DEVICE_FOR_3D(ccbxg1, i, j, k, @@ -162,8 +160,7 @@ MLNodeLaplacian::buildStencil () { const Box& btmp = ccbxg1 & sgfab_orig.box(); - sgfab.resize(ccbxg1); - Elixir sgeli = sgfab.elixir(); + sgfab.resize(ccbxg1, 1, The_Async_Arena()); Array4 const& sgarr = sgfab.array(); AMREX_HOST_DEVICE_FOR_3D(ccbxg1, i, j, k, diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sync.cpp b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sync.cpp index 1ae0205f986..701c4ca8b42 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sync.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sync.cpp @@ -134,8 +134,7 @@ MLNodeLaplacian::compSyncResidualCoarse (MultiFab& sync_resid, const MultiFab& a { const Box& ccvbx = amrex::enclosedCells(mfi.validbox()); - u.resize(ccbxg1, AMREX_SPACEDIM); - Elixir ueli = u.elixir(); + u.resize(ccbxg1, AMREX_SPACEDIM, The_Async_Arena()); Array4 const& uarr = u.array(); Box b = ccbxg1 & ccvbx; @@ -169,8 +168,7 @@ MLNodeLaplacian::compSyncResidualCoarse (MultiFab& sync_resid, const MultiFab& a } }); - rhs.resize(bx); - Elixir rhseli = rhs.elixir(); + rhs.resize(bx, 1, The_Async_Arena()); Array4 const& rhsarr = rhs.array(); Array4 const& dmskarr = dmsk.const_array(mfi); @@ -248,13 +246,11 @@ MLNodeLaplacian::compSyncResidualCoarse (MultiFab& sync_resid, const MultiFab& a Box const& sgbx = amrex::grow(amrex::enclosedCells(stbx),1); constexpr int ncomp_s = (AMREX_SPACEDIM == 2) ? 5 : 9; - sten.resize(stbx,ncomp_s); - Elixir steneli = sten.elixir(); + sten.resize(stbx,ncomp_s, The_Async_Arena()); Array4 const& stenarr = sten.array(); constexpr int ncomp_c = (AMREX_SPACEDIM == 2) ? 6 : 27; - cn.resize(sgbx,ncomp_c+1); - Elixir cneli = cn.elixir(); + cn.resize(sgbx,ncomp_c+1, The_Async_Arena()); Array4 const& cnarr = cn.array(); Array4 const& sgarr = cn.array(ncomp_c); @@ -395,8 +391,7 @@ MLNodeLaplacian::compSyncResidualFine (MultiFab& sync_resid, const MultiFab& phi const Box& bxg1 = amrex::grow(bx,1); const Box& ccbxg1 = amrex::enclosedCells(bxg1); - u.resize(ccbxg1, AMREX_SPACEDIM); - Elixir ueli = u.elixir(); + u.resize(ccbxg1, AMREX_SPACEDIM, The_Async_Arena()); Array4 const& uarr = u.array(); Box ovlp = ccvbx & ccbxg1; @@ -430,8 +425,7 @@ MLNodeLaplacian::compSyncResidualFine (MultiFab& sync_resid, const MultiFab& phi } }); - tmpmask.resize(bx); - Elixir tmeli = tmpmask.elixir(); + tmpmask.resize(bx, 1, The_Async_Arena()); Array4 const& tmpmaskarr = tmpmask.array(); Array4 const& dmskarr = dmsk.const_array(mfi); AMREX_HOST_DEVICE_FOR_3D(bx, i, j, k, @@ -439,8 +433,7 @@ MLNodeLaplacian::compSyncResidualFine (MultiFab& sync_resid, const MultiFab& phi tmpmaskarr(i,j,k) = 1-dmskarr(i,j,k); }); - rhs.resize(bx); - Elixir rhseli = rhs.elixir(); + rhs.resize(bx, 1, The_Async_Arena()); Array4 const& rhsarr = rhs.array(); #ifdef AMREX_USE_EB @@ -516,13 +509,11 @@ MLNodeLaplacian::compSyncResidualFine (MultiFab& sync_resid, const MultiFab& phi Box const& sgbx = amrex::grow(amrex::enclosedCells(stbx),1); constexpr int ncomp_s = (AMREX_SPACEDIM == 2) ? 5 : 9; - sten.resize(stbx,ncomp_s); - Elixir steneli = sten.elixir(); + sten.resize(stbx,ncomp_s, The_Async_Arena()); Array4 const& stenarr = sten.array(); constexpr int ncomp_c = (AMREX_SPACEDIM == 2) ? 6 : 27; - cn.resize(sgbx,ncomp_c+1); - Elixir cneli = cn.elixir(); + cn.resize(sgbx,ncomp_c+1, The_Async_Arena()); Array4 const& cnarr = cn.array(); Array4 const& sgarr = cn.array(ncomp_c); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp index a0af307ff9c..8b6a836c701 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp @@ -243,12 +243,9 @@ MLTensorOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode bc AMREX_D_TERM(Box const xbx = amrex::surroundingNodes(bx,0);, Box const ybx = amrex::surroundingNodes(bx,1);, Box const zbx = amrex::surroundingNodes(bx,2);); - AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM);, - fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM);, - fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM);); - AMREX_D_TERM(Elixir fxeli = fluxfab_tmp[0].elixir();, - Elixir fyeli = fluxfab_tmp[1].elixir();, - Elixir fzeli = fluxfab_tmp[2].elixir();); + AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM, The_Async_Arena());, + fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM, The_Async_Arena());, + fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM, The_Async_Arena());); AMREX_D_TERM(Array4 const fxfab = fluxfab_tmp[0].array();, Array4 const fyfab = fluxfab_tmp[1].array();, Array4 const fzfab = fluxfab_tmp[2].array();); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp index 0fa5a151454..acea55f938a 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp @@ -48,12 +48,9 @@ MLTensorOp::compFlux (int amrlev, const Array& fluxes, AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, Box const ybx = mfi.nodaltilebox(1);, Box const zbx = mfi.nodaltilebox(2);); - AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM);, - fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM);, - fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM);); - AMREX_D_TERM(Elixir fxeli = fluxfab_tmp[0].elixir();, - Elixir fyeli = fluxfab_tmp[1].elixir();, - Elixir fzeli = fluxfab_tmp[2].elixir();); + AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM, The_Async_Arena());, + fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM, The_Async_Arena());, + fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM, The_Async_Arena());); AMREX_D_TERM(Array4 const fxfab = fluxfab_tmp[0].array();, Array4 const fyfab = fluxfab_tmp[1].array();, Array4 const fzfab = fluxfab_tmp[2].array(););