Skip to content

Commit

Permalink
Replace Elixir with Async Arena
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexanderSinn
AlexanderSinn committed Feb 26, 2025
1 parent cacff2f commit d0d95d9
Show file tree
Hide file tree
Showing 6 changed files with 29 additions and 56 deletions.
4 changes: 2 additions & 2 deletions Src/Base/AMReX_GpuControl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@ namespace amrex::Gpu {
#if defined(AMREX_USE_GPU)
bool in_launch_region = true;
bool in_graph_region = false;
bool in_single_stream_region = false;
bool in_nosync_region = false;
bool in_single_stream_region = true;
bool in_nosync_region = true;
#endif

}
27 changes: 9 additions & 18 deletions Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1350,12 +1350,9 @@ MLCellLinOpT<MF>::reflux (int crse_amrlev, MF& res, const MF& crse_sol, const MF
if (fluxreg.CrseHasWork(mfi))
{
const Box& tbx = mfi.tilebox();
AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp);,
flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp);,
flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp););
AMREX_D_TERM(Elixir elifx = flux[0].elixir();,
Elixir elify = flux[1].elixir();,
Elixir elifz = flux[2].elixir(););
AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp,The_Async_Arena());,
flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp,The_Async_Arena());,
flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp,The_Async_Arena()););
FFlux(crse_amrlev, mfi, pflux, crse_sol[mfi], Location::FaceCentroid);
fluxreg.CrseAdd(mfi, cpflux, crse_dx, dt, RunOn::Gpu);
}
Expand All @@ -1371,12 +1368,9 @@ MLCellLinOpT<MF>::reflux (int crse_amrlev, MF& res, const MF& crse_sol, const MF
{
const Box& tbx = mfi.tilebox();
const int face_only = true;
AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp);,
flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp);,
flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp););
AMREX_D_TERM(Elixir elifx = flux[0].elixir();,
Elixir elify = flux[1].elixir();,
Elixir elifz = flux[2].elixir(););
AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp,The_Async_Arena());,
flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp,The_Async_Arena());,
flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp,The_Async_Arena()););
FFlux(fine_amrlev, mfi, pflux, fine_sol[mfi], Location::FaceCentroid, face_only);
fluxreg.FineAdd(mfi, cpflux, fine_dx, dt, RunOn::Gpu);
}
Expand Down Expand Up @@ -1410,12 +1404,9 @@ MLCellLinOpT<MF>::compFlux (int amrlev, const Array<MF*,AMREX_SPACEDIM>& fluxes,
for (MFIter mfi(sol, mfi_info); mfi.isValid(); ++mfi)
{
const Box& tbx = mfi.tilebox();
AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp);,
flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp);,
flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp););
AMREX_D_TERM(Elixir elifx = flux[0].elixir();,
Elixir elify = flux[1].elixir();,
Elixir elifz = flux[2].elixir(););
AMREX_D_TERM(flux[0].resize(amrex::surroundingNodes(tbx,0),ncomp,The_Async_Arena());,
flux[1].resize(amrex::surroundingNodes(tbx,1),ncomp,The_Async_Arena());,
flux[2].resize(amrex::surroundingNodes(tbx,2),ncomp,The_Async_Arena()););
FFlux(amrlev, mfi, pflux, sol[mfi], loc);
for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) {
const Box& nbx = mfi.nodaltilebox(idim);
Expand Down
9 changes: 3 additions & 6 deletions Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sten.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,12 +115,10 @@ MLNodeLaplacian::buildStencil ()
{
const Box& btmp = ccbxg1 & sgfab_orig.box();

cnfab.resize(ccbxg1, ncomp_c);
Elixir cneli = cnfab.elixir();
cnfab.resize(ccbxg1, ncomp_c, The_Async_Arena());
Array4<Real> const& cnarr = cnfab.array();

sgfab.resize(ccbxg1);
Elixir sgeli = sgfab.elixir();
sgfab.resize(ccbxg1, 1, The_Async_Arena());
Array4<Real> const& sgarr = sgfab.array();

AMREX_HOST_DEVICE_FOR_3D(ccbxg1, i, j, k,
Expand Down Expand Up @@ -162,8 +160,7 @@ MLNodeLaplacian::buildStencil ()
{
const Box& btmp = ccbxg1 & sgfab_orig.box();

sgfab.resize(ccbxg1);
Elixir sgeli = sgfab.elixir();
sgfab.resize(ccbxg1, 1, The_Async_Arena());
Array4<Real> const& sgarr = sgfab.array();

AMREX_HOST_DEVICE_FOR_3D(ccbxg1, i, j, k,
Expand Down
27 changes: 9 additions & 18 deletions Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sync.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,7 @@ MLNodeLaplacian::compSyncResidualCoarse (MultiFab& sync_resid, const MultiFab& a
{
const Box& ccvbx = amrex::enclosedCells(mfi.validbox());

u.resize(ccbxg1, AMREX_SPACEDIM);
Elixir ueli = u.elixir();
u.resize(ccbxg1, AMREX_SPACEDIM, The_Async_Arena());
Array4<Real> const& uarr = u.array();

Box b = ccbxg1 & ccvbx;
Expand Down Expand Up @@ -169,8 +168,7 @@ MLNodeLaplacian::compSyncResidualCoarse (MultiFab& sync_resid, const MultiFab& a
}
});

rhs.resize(bx);
Elixir rhseli = rhs.elixir();
rhs.resize(bx, 1, The_Async_Arena());
Array4<Real> const& rhsarr = rhs.array();
Array4<int const> const& dmskarr = dmsk.const_array(mfi);

Expand Down Expand Up @@ -248,13 +246,11 @@ MLNodeLaplacian::compSyncResidualCoarse (MultiFab& sync_resid, const MultiFab& a
Box const& sgbx = amrex::grow(amrex::enclosedCells(stbx),1);

constexpr int ncomp_s = (AMREX_SPACEDIM == 2) ? 5 : 9;
sten.resize(stbx,ncomp_s);
Elixir steneli = sten.elixir();
sten.resize(stbx,ncomp_s, The_Async_Arena());
Array4<Real> const& stenarr = sten.array();

constexpr int ncomp_c = (AMREX_SPACEDIM == 2) ? 6 : 27;
cn.resize(sgbx,ncomp_c+1);
Elixir cneli = cn.elixir();
cn.resize(sgbx,ncomp_c+1, The_Async_Arena());
Array4<Real> const& cnarr = cn.array();
Array4<Real> const& sgarr = cn.array(ncomp_c);

Expand Down Expand Up @@ -395,8 +391,7 @@ MLNodeLaplacian::compSyncResidualFine (MultiFab& sync_resid, const MultiFab& phi
const Box& bxg1 = amrex::grow(bx,1);
const Box& ccbxg1 = amrex::enclosedCells(bxg1);

u.resize(ccbxg1, AMREX_SPACEDIM);
Elixir ueli = u.elixir();
u.resize(ccbxg1, AMREX_SPACEDIM, The_Async_Arena());
Array4<Real> const& uarr = u.array();

Box ovlp = ccvbx & ccbxg1;
Expand Down Expand Up @@ -430,17 +425,15 @@ MLNodeLaplacian::compSyncResidualFine (MultiFab& sync_resid, const MultiFab& phi
}
});

tmpmask.resize(bx);
Elixir tmeli = tmpmask.elixir();
tmpmask.resize(bx, 1, The_Async_Arena());
Array4<int> const& tmpmaskarr = tmpmask.array();
Array4<int const> const& dmskarr = dmsk.const_array(mfi);
AMREX_HOST_DEVICE_FOR_3D(bx, i, j, k,
{
tmpmaskarr(i,j,k) = 1-dmskarr(i,j,k);
});

rhs.resize(bx);
Elixir rhseli = rhs.elixir();
rhs.resize(bx, 1, The_Async_Arena());
Array4<Real> const& rhsarr = rhs.array();

#ifdef AMREX_USE_EB
Expand Down Expand Up @@ -516,13 +509,11 @@ MLNodeLaplacian::compSyncResidualFine (MultiFab& sync_resid, const MultiFab& phi
Box const& sgbx = amrex::grow(amrex::enclosedCells(stbx),1);

constexpr int ncomp_s = (AMREX_SPACEDIM == 2) ? 5 : 9;
sten.resize(stbx,ncomp_s);
Elixir steneli = sten.elixir();
sten.resize(stbx,ncomp_s, The_Async_Arena());
Array4<Real> const& stenarr = sten.array();

constexpr int ncomp_c = (AMREX_SPACEDIM == 2) ? 6 : 27;
cn.resize(sgbx,ncomp_c+1);
Elixir cneli = cn.elixir();
cn.resize(sgbx,ncomp_c+1, The_Async_Arena());
Array4<Real> const& cnarr = cn.array();
Array4<Real> const& sgarr = cn.array(ncomp_c);

Expand Down
9 changes: 3 additions & 6 deletions Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,12 +243,9 @@ MLTensorOp::apply (int amrlev, int mglev, MultiFab& out, MultiFab& in, BCMode bc
AMREX_D_TERM(Box const xbx = amrex::surroundingNodes(bx,0);,
Box const ybx = amrex::surroundingNodes(bx,1);,
Box const zbx = amrex::surroundingNodes(bx,2););
AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM);,
fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM);,
fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM););
AMREX_D_TERM(Elixir fxeli = fluxfab_tmp[0].elixir();,
Elixir fyeli = fluxfab_tmp[1].elixir();,
Elixir fzeli = fluxfab_tmp[2].elixir(););
AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM, The_Async_Arena());,
fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM, The_Async_Arena());,
fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM, The_Async_Arena()););
AMREX_D_TERM(Array4<Real> const fxfab = fluxfab_tmp[0].array();,
Array4<Real> const fyfab = fluxfab_tmp[1].array();,
Array4<Real> const fzfab = fluxfab_tmp[2].array(););
Expand Down
9 changes: 3 additions & 6 deletions Src/LinearSolvers/MLMG/AMReX_MLTensorOp_grad.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,9 @@ MLTensorOp::compFlux (int amrlev, const Array<MultiFab*,AMREX_SPACEDIM>& fluxes,
AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);,
Box const ybx = mfi.nodaltilebox(1);,
Box const zbx = mfi.nodaltilebox(2););
AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM);,
fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM);,
fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM););
AMREX_D_TERM(Elixir fxeli = fluxfab_tmp[0].elixir();,
Elixir fyeli = fluxfab_tmp[1].elixir();,
Elixir fzeli = fluxfab_tmp[2].elixir(););
AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM, The_Async_Arena());,
fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM, The_Async_Arena());,
fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM, The_Async_Arena()););
AMREX_D_TERM(Array4<Real> const fxfab = fluxfab_tmp[0].array();,
Array4<Real> const fyfab = fluxfab_tmp[1].array();,
Array4<Real> const fzfab = fluxfab_tmp[2].array(););
Expand Down

0 comments on commit d0d95d9

Please sign in to comment.