Skip to content

Commit 8ec5eb6

Browse files
Clean up charge deposition code duplication (#4113)
1 parent be07243 commit 8ec5eb6

File tree

2 files changed

+65
-92
lines changed

2 files changed

+65
-92
lines changed

Source/Particles/WarpXParticleContainer.H

+4
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,10 @@ public:
207207
const bool apply_boundary_and_scale_volume = false,
208208
const bool interpolate_across_levels = true,
209209
const int icomp = 0);
210+
void DepositCharge (std::unique_ptr<amrex::MultiFab>& rho, const int lev,
211+
const bool local = false, const bool reset = false,
212+
const bool apply_boundary_and_scale_volume = false,
213+
const int icomp = 0);
210214

211215
std::unique_ptr<amrex::MultiFab> GetChargeDensity(int lev, bool local = false);
212216

Source/Particles/WarpXParticleContainer.cpp

+61-92
Original file line numberDiff line numberDiff line change
@@ -976,58 +976,9 @@ WarpXParticleContainer::DepositCharge (amrex::Vector<std::unique_ptr<amrex::Mult
976976
int const finest_level = rho.size() - 1;
977977
for (int lev = 0; lev <= finest_level; ++lev)
978978
{
979-
// Reset the rho array if reset is True
980-
int const nc = WarpX::ncomps;
981-
if (reset) rho[lev]->setVal(0., icomp*nc, nc, rho[lev]->nGrowVect());
982-
983-
// Loop over particle tiles and deposit charge on each level
984-
#ifdef AMREX_USE_OMP
985-
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
986-
{
987-
const int thread_num = omp_get_thread_num();
988-
#else
989-
const int thread_num = 0;
990-
#endif
991-
for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti)
992-
{
993-
const long np = pti.numParticles();
994-
auto const & wp = pti.GetAttribs(PIdx::w);
995-
996-
int* AMREX_RESTRICT ion_lev = nullptr;
997-
if (do_field_ionization)
998-
{
999-
ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr();
1000-
}
1001-
1002-
DepositCharge(pti, wp, ion_lev, rho[lev].get(), icomp, 0, np, thread_num, lev, lev);
1003-
}
1004-
#ifdef AMREX_USE_OMP
1005-
}
1006-
#endif
1007-
1008-
#ifdef WARPX_DIM_RZ
1009-
if (apply_boundary_and_scale_volume)
1010-
{
1011-
WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho[lev].get(), lev);
1012-
}
1013-
#endif
1014-
1015-
// Exchange guard cells
1016-
if (!local) {
1017-
// Possible performance optimization:
1018-
// pass less than `rho[lev]->nGrowVect()` in the fifth input variable `dst_ng`
1019-
ablastr::utils::communication::SumBoundary(
1020-
*rho[lev], 0, rho[lev]->nComp(), rho[lev]->nGrowVect(), rho[lev]->nGrowVect(),
1021-
WarpX::do_single_precision_comms, m_gdb->Geom(lev).periodicity());
1022-
}
1023-
1024-
#ifndef WARPX_DIM_RZ
1025-
if (apply_boundary_and_scale_volume)
1026-
{
1027-
// Reflect density over PEC boundaries, if needed.
1028-
WarpX::GetInstance().ApplyRhofieldBoundary(lev, rho[lev].get(), PatchType::fine);
1029-
}
1030-
#endif
979+
DepositCharge (
980+
rho[lev], lev, local, reset, apply_boundary_and_scale_volume, icomp
981+
);
1031982
}
1032983

1033984
// Now that the charge has been deposited at each level,
@@ -1053,72 +1004,90 @@ WarpXParticleContainer::DepositCharge (amrex::Vector<std::unique_ptr<amrex::Mult
10531004
}
10541005
}
10551006

1056-
std::unique_ptr<MultiFab>
1057-
WarpXParticleContainer::GetChargeDensity (int lev, bool local)
1007+
void
1008+
WarpXParticleContainer::DepositCharge (std::unique_ptr<amrex::MultiFab>& rho,
1009+
const int lev, const bool local, const bool reset,
1010+
const bool apply_boundary_and_scale_volume,
1011+
const int icomp)
10581012
{
1059-
const auto& gm = m_gdb->Geom(lev);
1060-
const auto& ba = m_gdb->ParticleBoxArray(lev);
1061-
const auto& dm = m_gdb->DistributionMap(lev);
1062-
BoxArray nba = ba;
1063-
1064-
#ifdef WARPX_DIM_RZ
1065-
const bool is_PSATD_RZ =
1066-
(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD);
1067-
#else
1068-
const bool is_PSATD_RZ = false;
1069-
#endif
1070-
if( !is_PSATD_RZ )
1071-
nba.surroundingNodes();
1072-
1073-
// Number of guard cells for local deposition of rho
1074-
const WarpX& warpx = WarpX::GetInstance();
1075-
const int ng_rho = warpx.get_ng_depos_rho().max();
1076-
1077-
auto rho = std::make_unique<MultiFab>(nba,dm,WarpX::ncomps,ng_rho);
1078-
rho->setVal(0.0);
1013+
// Reset the rho array if reset is True
1014+
int const nc = WarpX::ncomps;
1015+
if (reset) rho->setVal(0., icomp*nc, nc, rho->nGrowVect());
10791016

1017+
// Loop over particle tiles and deposit charge on each level
10801018
#ifdef AMREX_USE_OMP
10811019
#pragma omp parallel if (amrex::Gpu::notInLaunchRegion())
10821020
{
1083-
#endif
1084-
#ifdef AMREX_USE_OMP
1085-
const int thread_num = omp_get_thread_num();
1021+
const int thread_num = omp_get_thread_num();
10861022
#else
1087-
const int thread_num = 0;
1023+
const int thread_num = 0;
10881024
#endif
1025+
for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti)
1026+
{
1027+
const long np = pti.numParticles();
1028+
auto const & wp = pti.GetAttribs(PIdx::w);
10891029

1090-
for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti)
1030+
int* AMREX_RESTRICT ion_lev = nullptr;
1031+
if (do_field_ionization)
10911032
{
1092-
const long np = pti.numParticles();
1093-
auto& wp = pti.GetAttribs(PIdx::w);
1094-
1095-
const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)?
1096-
pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr;
1097-
1098-
DepositCharge(pti, wp, ion_lev, rho.get(), 0, 0, np,
1099-
thread_num, lev, lev);
1033+
ion_lev = pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr();
11001034
}
1035+
1036+
DepositCharge(pti, wp, ion_lev, rho.get(), icomp, 0, np, thread_num, lev, lev);
1037+
}
11011038
#ifdef AMREX_USE_OMP
11021039
}
11031040
#endif
11041041

11051042
#ifdef WARPX_DIM_RZ
1106-
WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho.get(), lev);
1043+
if (apply_boundary_and_scale_volume)
1044+
{
1045+
WarpX::GetInstance().ApplyInverseVolumeScalingToChargeDensity(rho.get(), lev);
1046+
}
11071047
#endif
11081048

1109-
if (!local) {
1049+
// Exchange guard cells
1050+
if ( !local ) {
11101051
// Possible performance optimization:
11111052
// pass less than `rho->nGrowVect()` in the fifth input variable `dst_ng`
11121053
ablastr::utils::communication::SumBoundary(
11131054
*rho, 0, rho->nComp(), rho->nGrowVect(), rho->nGrowVect(),
1114-
WarpX::do_single_precision_comms, gm.periodicity());
1055+
WarpX::do_single_precision_comms,
1056+
m_gdb->Geom(lev).periodicity()
1057+
);
11151058
}
11161059

11171060
#ifndef WARPX_DIM_RZ
1118-
// Reflect density over PEC boundaries, if needed.
1119-
WarpX::GetInstance().ApplyRhofieldBoundary(lev, rho.get(), PatchType::fine);
1061+
if (apply_boundary_and_scale_volume)
1062+
{
1063+
// Reflect density over PEC boundaries, if needed.
1064+
WarpX::GetInstance().ApplyRhofieldBoundary(lev, rho.get(), PatchType::fine);
1065+
}
11201066
#endif
1067+
}
1068+
1069+
std::unique_ptr<MultiFab>
1070+
WarpXParticleContainer::GetChargeDensity (int lev, bool local)
1071+
{
1072+
const auto& ba = m_gdb->ParticleBoxArray(lev);
1073+
const auto& dm = m_gdb->DistributionMap(lev);
1074+
BoxArray nba = ba;
1075+
1076+
#ifdef WARPX_DIM_RZ
1077+
const bool is_PSATD_RZ =
1078+
(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD);
1079+
#else
1080+
const bool is_PSATD_RZ = false;
1081+
#endif
1082+
if( !is_PSATD_RZ )
1083+
nba.surroundingNodes();
1084+
1085+
// Number of guard cells for local deposition of rho
1086+
const WarpX& warpx = WarpX::GetInstance();
1087+
const int ng_rho = warpx.get_ng_depos_rho().max();
11211088

1089+
auto rho = std::make_unique<MultiFab>(nba, dm, WarpX::ncomps,ng_rho);
1090+
DepositCharge(rho, lev, local, true, true, 0);
11221091
return rho;
11231092
}
11241093

0 commit comments

Comments
 (0)