Remove temporary particle container (#5571)

Close #5056.

Co-authored-by: Andrew Myers <atmyers2@gmail.com>

Source/Diagnostics/BTDiagnostics.cpp

@@ -1460,7 +1460,7 @@ BTDiagnostics::InitializeParticleBuffer ()
         m_totalParticles_in_buffer[i].resize(m_output_species_names.size());
         for (int isp = 0; isp < m_particles_buffer[i].size(); ++isp) {
             m_totalParticles_in_buffer[i][isp] = 0;
-            m_particles_buffer[i][isp] = std::make_unique<PinnedMemoryParticleContainer>(WarpX::GetInstance().GetParGDB());
+            m_particles_buffer[i][isp] = std::make_unique<PinnedMemoryParticleContainer>(mpc.GetParticleContainer(isp).make_alike<amrex::PinnedArenaAllocator>());
             const int idx = mpc.getSpeciesID(m_output_species_names[isp]);
 
             // SoA component names

Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.H

@@ -27,12 +27,12 @@ struct SelectParticles
      * \brief Constructor of SelectParticles functor.
      *
      * @param[in] a_pti              WarpX particle iterator
-     * @param[in] tmp_particle_data  temporary particle data
      * @param[in] current_z_boost    current z-position of the slice in boosted frame
      * @param[in] old_z_boost        previous z-position of the slice in boosted frame
      * @param[in] a_offset           index offset for particles to be selected
      */
-    SelectParticles( const WarpXParIter& a_pti, TmpParticles& tmp_particle_data,
+    SelectParticles (const WarpXParticleContainer& pc,
+                     WarpXParIter& a_pti,
                      amrex::Real current_z_boost, amrex::Real old_z_boost,
                      int a_offset = 0);
 
@@ -83,15 +83,15 @@ struct LorentzTransformParticles
      * \brief Constructor of the LorentzTransformParticles functor.
      *
      * @param[in] a_pti              WarpX particle iterator
-     * @param[in] tmp_particle_data  temporary particle data
      * @param[in] t_boost            time in boosted frame
      * @param[in] dt                 timestep in boosted-frame
      * @param[in] t_lab              time in lab-frame
      * @param[in] a_offset           index offset for particles to be transformed
      */
-    LorentzTransformParticles ( const WarpXParIter& a_pti, TmpParticles& tmp_particle_data,
-                                amrex::Real t_boost, amrex::Real dt,
-                                amrex::Real t_lab, int a_offset = 0);
+    LorentzTransformParticles (const WarpXParticleContainer& pc,
+                               WarpXParIter& a_pti,
+                               amrex::Real t_boost, amrex::Real dt,
+                               amrex::Real t_lab, int a_offset = 0);
 
     /**
      * \brief Functor call. This method computes the Lorentz-transform for particle
@@ -133,8 +133,12 @@ struct LorentzTransformParticles
         const amrex::Real weight_new = (m_t_lab - t_old_p)
                                      / (t_new_p - t_old_p);
         // weighted sum of old and new values
+#if (AMREX_SPACEDIM >= 2)
         const amrex::ParticleReal xp = m_xpold[i_src] * weight_old + xpnew * weight_new;
+#endif
+#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ)
         const amrex::ParticleReal yp = m_ypold[i_src] * weight_old + ypnew * weight_new;
+#endif
         const amrex::ParticleReal zp = z_old_p * weight_old + z_new_p * weight_new;
         const amrex::ParticleReal uxp = m_uxpold[i_src] * weight_old
                                       + m_uxpnew[i_src] * weight_new;
@@ -153,19 +157,24 @@ struct LorentzTransformParticles
 #elif defined (WARPX_DIM_XZ)
         dst.m_rdata[PIdx::x][i_dst] = xp;
         dst.m_rdata[PIdx::z][i_dst] = zp;
-        amrex::ignore_unused(yp);
 #elif defined (WARPX_DIM_1D_Z)
         dst.m_rdata[PIdx::z][i_dst] = zp;
-        amrex::ignore_unused(xp, yp);
-#else
-        amrex::ignore_unused(xp, yp, zp);
 #endif
         dst.m_rdata[PIdx::w][i_dst] = m_wpnew[i_src];
         dst.m_rdata[PIdx::ux][i_dst] = uxp;
         dst.m_rdata[PIdx::uy][i_dst] = uyp;
         dst.m_rdata[PIdx::uz][i_dst] = uzp;
     }
 
+    amrex::Real m_t_boost;
+    amrex::Real m_dt;
+    amrex::Real m_t_lab;
+    amrex::Real m_gammaboost;
+    amrex::Real m_betaboost;
+    amrex::Real m_Phys_c;
+    amrex::Real m_inv_c2;
+    amrex::Real m_uzfrm;
+
     GetParticlePosition<PIdx> m_get_position;
 
     amrex::ParticleReal* AMREX_RESTRICT m_xpold = nullptr;
@@ -180,15 +189,6 @@ struct LorentzTransformParticles
     const amrex::ParticleReal* AMREX_RESTRICT m_uypnew = nullptr;
     const amrex::ParticleReal* AMREX_RESTRICT m_uzpnew = nullptr;
     const amrex::ParticleReal* AMREX_RESTRICT m_wpnew = nullptr;
-
-    amrex::Real m_gammaboost;
-    amrex::Real m_betaboost;
-    amrex::Real m_Phys_c;
-    amrex::Real m_uzfrm;
-    amrex::Real m_inv_c2;
-    amrex::Real m_t_boost;
-    amrex::Real m_dt;
-    amrex::Real m_t_lab;
 };
 
 /**

Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp

@@ -15,29 +15,36 @@
 #include <AMReX_Print.H>
 #include <AMReX_BaseFwd.H>
 
-SelectParticles::SelectParticles (const WarpXParIter& a_pti, TmpParticles& tmp_particle_data,
-                                  amrex::Real current_z_boost, amrex::Real old_z_boost,
-                                  int a_offset)
+SelectParticles::SelectParticles (
+    const WarpXParticleContainer& /*pc*/,
+    WarpXParIter& a_pti,
+    amrex::Real current_z_boost,
+    amrex::Real old_z_boost,
+    int a_offset
+)
     : m_current_z_boost(current_z_boost), m_old_z_boost(old_z_boost)
 {
     m_get_position = GetParticlePosition<PIdx>(a_pti, a_offset);
 
-    const auto lev = a_pti.GetLevel();
-    const auto index = a_pti.GetPairIndex();
-
-    zpold = tmp_particle_data[lev][index][TmpIdx::zold].dataPtr();
+    zpold = a_pti.GetAttribs("z_n_btd").dataPtr();
 }
 
 
-LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti,
-                                TmpParticles& tmp_particle_data,
-                                amrex::Real t_boost, amrex::Real dt,
-                                amrex::Real t_lab, int a_offset)
-    : m_t_boost(t_boost), m_dt(dt), m_t_lab(t_lab)
+LorentzTransformParticles::LorentzTransformParticles (
+    const WarpXParticleContainer& /*pc*/,
+    WarpXParIter& a_pti,
+    amrex::Real t_boost,
+    amrex::Real dt,
+    amrex::Real t_lab,
+    int a_offset
+)
+    : m_t_boost(t_boost), m_dt(dt), m_t_lab(t_lab),
+      m_gammaboost(WarpX::gamma_boost), m_betaboost(WarpX::beta_boost),
+      m_Phys_c(PhysConst::c), m_inv_c2(amrex::Real(1.0)/(m_Phys_c * m_Phys_c)),
+      m_uzfrm(-m_gammaboost*m_betaboost*m_Phys_c)
 {
     using namespace amrex::literals;
 
-    if (tmp_particle_data.empty()) { return; }
     m_get_position = GetParticlePosition<PIdx>(a_pti, a_offset);
 
     const auto& attribs = a_pti.GetAttribs();
@@ -46,21 +53,16 @@ LorentzTransformParticles::LorentzTransformParticles ( const WarpXParIter& a_pti
     m_uypnew = attribs[PIdx::uy].dataPtr();
     m_uzpnew = attribs[PIdx::uz].dataPtr();
 
-    const auto lev = a_pti.GetLevel();
-    const auto index = a_pti.GetPairIndex();
-
-    m_xpold = tmp_particle_data[lev][index][TmpIdx::xold].dataPtr();
-    m_ypold = tmp_particle_data[lev][index][TmpIdx::yold].dataPtr();
-    m_zpold = tmp_particle_data[lev][index][TmpIdx::zold].dataPtr();
-    m_uxpold = tmp_particle_data[lev][index][TmpIdx::uxold].dataPtr();
-    m_uypold = tmp_particle_data[lev][index][TmpIdx::uyold].dataPtr();
-    m_uzpold = tmp_particle_data[lev][index][TmpIdx::uzold].dataPtr();
-
-    m_betaboost = WarpX::beta_boost;
-    m_gammaboost = WarpX::gamma_boost;
-    m_Phys_c = PhysConst::c;
-    m_inv_c2 = 1._rt/(m_Phys_c * m_Phys_c);
-    m_uzfrm = -m_gammaboost*m_betaboost*m_Phys_c;
+#if (AMREX_SPACEDIM >= 2)
+    m_xpold = a_pti.GetAttribs("x_n_btd").dataPtr();
+#endif
+#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ)
+    m_ypold = a_pti.GetAttribs("y_n_btd").dataPtr();
+#endif
+    m_zpold = a_pti.GetAttribs("z_n_btd").dataPtr();
+    m_uxpold = a_pti.GetAttribs("ux_n_btd").dataPtr();
+    m_uypold = a_pti.GetAttribs("uy_n_btd").dataPtr();
+    m_uzpold = a_pti.GetAttribs("uz_n_btd").dataPtr();
 }
 
 /**
@@ -84,7 +86,6 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst
     auto &warpx = WarpX::GetInstance();
     // get particle slice
     const int nlevs = std::max(0, m_pc_src->finestLevel()+1);
-    auto tmp_particle_data = m_pc_src->getTmpParticleData();
     for (int lev = 0; lev < nlevs; ++lev) {
         const amrex::Real t_boost = warpx.gett_new(0);
         const amrex::Real dt = warpx.getdt(0);
@@ -107,13 +108,19 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst
 
                 auto index = std::make_pair(pti.index(), pti.LocalTileIndex());
 
-                const auto GetParticleFilter = SelectParticles(pti, tmp_particle_data,
-                                               m_current_z_boost[i_buffer],
-                                               m_old_z_boost[i_buffer]);
+                const auto GetParticleFilter = SelectParticles(
+                    *m_pc_src,
+                    pti,
+                    m_current_z_boost[i_buffer],
+                    m_old_z_boost[i_buffer]
+                );
                 const auto GetParticleLorentzTransform = LorentzTransformParticles(
-                                                         pti, tmp_particle_data,
-                                                         t_boost, dt,
-                                                         m_t_lab[i_buffer]);
+                    *m_pc_src,
+                    pti,
+                    t_boost,
+                    dt,
+                    m_t_lab[i_buffer]
+                );
 
                 long const np = pti.numParticles();
 

Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp

@@ -213,7 +213,7 @@ FlushFormatCheckpoint::CheckpointParticles (
         real_names.resize(pc->NumRealComps() - AMREX_SPACEDIM);
         write_real_comps.resize(pc->NumRealComps() - AMREX_SPACEDIM);
 
-        // note, skip the required compnent names here
+        // note, skip the required component names here
         auto rnames = pc->GetRealSoANames();
         for (std::size_t index = PIdx::nattribs; index < rnames.size(); ++index) {
             std::size_t const i = index - AMREX_SPACEDIM;

Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp

@@ -362,7 +362,15 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir,
         Vector<int> int_flags;
         Vector<int> real_flags;
 
-        // note: positions skipped here, since we reconstruct a plotfile SoA from them
+#if !defined (WARPX_DIM_1D_Z)
+        real_names.push_back("position_x");
+#endif
+#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ)
+        real_names.push_back("position_y");
+#endif
+#if !defined(WARPX_DIM_RZ)
+        real_names.push_back("position_z");
+#endif
         real_names.push_back("weight");
         real_names.push_back("momentum_x");
         real_names.push_back("momentum_y");
@@ -373,31 +381,30 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir,
 #endif
 
         // get the names of the extra real comps
-        real_names.resize(tmp.NumRealComps() - AMREX_SPACEDIM);
+        real_names.resize(tmp.NumRealComps());
+        real_flags = part_diag.m_plot_flags;
+        real_flags.resize(tmp.NumRealComps());
 
-        // note, skip the required compnent names here
+        // note, skip the required component names here
         auto rnames = tmp.GetRealSoANames();
         for (std::size_t index = PIdx::nattribs; index < rnames.size(); ++index) {
-            real_names[index - AMREX_SPACEDIM] = rnames[index];
+            real_names[index] = rnames[index];
+            real_flags[index] = tmp.h_redistribute_real_comp[index];
         }
 
-        // plot any "extra" fields by default
-        real_flags = part_diag.m_plot_flags;
-        real_flags.resize(tmp.NumRealComps(), 1);
-
         //   note: skip the mandatory AMREX_SPACEDIM positions for pure SoA
+        real_names.erase(real_names.begin(), real_names.begin() + AMREX_SPACEDIM);
         real_flags.erase(real_flags.begin(), real_flags.begin() + AMREX_SPACEDIM);
 
-        // and the names
+        // and the int comps
         int_names.resize(tmp.NumIntComps());
+        int_flags.resize(tmp.NumIntComps());
         auto inames = tmp.GetIntSoANames();
         for (std::size_t index = 0; index < inames.size(); ++index) {
             int_names[index] = inames[index];
+            int_flags[index] = tmp.h_redistribute_int_comp[index];
         }
 
-        // plot by default
-        int_flags.resize(tmp.NumIntComps(), 1);
-
         const auto mass = pc->AmIA<PhysicalSpecies::photon>() ? PhysConst::m_e : pc->getMass();
         RandomFilter const random_filter(part_diag.m_do_random_filter,
                                          part_diag.m_random_fraction);
@@ -428,6 +435,7 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir,
             tmp.copyParticles(*pinned_pc, true);
             particlesConvertUnits(ConvertDirection::WarpX_to_SI, &tmp, mass);
         }
+
         // real_names contains a list of all particle attributes.
         // real_flags & int_flags are 1 or 0, whether quantity is dumped or not.
         tmp.WritePlotFile(

Source/Diagnostics/WarpXOpenPMD.cpp

@@ -641,20 +641,21 @@ for (const auto & particle_diag : particle_diags) {
         real_names[i] = detail::snakeToCamel(rn[i]);
     }
 
-    // plot any "extra" fields by default
     amrex::Vector<int> real_flags = particle_diag.m_plot_flags;
-    real_flags.resize(tmp.NumRealComps(), 1);
+    real_flags.resize(tmp.NumRealComps());
+    for (size_t index = PIdx::nattribs; index < rn.size(); ++index) {
+        real_flags[index] = tmp.h_redistribute_real_comp[index];
+    }
+
 
-    // and the int names
+    // and the int components
+    amrex::Vector<int> int_flags(tmp.NumIntComps());
     for (size_t i = 0; i < in.size(); ++i)
     {
         int_names[i] = detail::snakeToCamel(in[i]);
+        int_flags[i] = tmp.h_redistribute_int_comp[i];
     }
 
-    // plot by default
-    amrex::Vector<int> int_flags;
-    int_flags.resize(tmp.NumIntComps(), 1);
-
     // real_names contains a list of all real particle attributes.
     // real_flags is 1 or 0, whether quantity is dumped or not.
     DumpToFile(&tmp,

Source/Particles/MultiParticleContainer.H

@@ -257,6 +257,13 @@ public:
      */
     void SetDoBackTransformedParticles (const std::string& species_name, bool do_back_transformed_particles);
 
+    /** Get value of private member m_do_back_transformed_particles
+     */
+    [[nodiscard]] bool getDoBackTransformedParticles () const
+    {
+        return m_do_back_transformed_particles;
+    }
+
     [[nodiscard]] int nSpeciesDepositOnMainGrid () const
     {
         bool const onMainGrid = true;

Source/Particles/MultiParticleContainer.cpp

@@ -855,7 +855,24 @@ MultiParticleContainer::SetDoBackTransformedParticles (const std::string& specie
         if (species_names_list[i] == species_name) {
            found = true;
            auto& pc = allcontainers[i];
+           const bool old_do_btd = pc->GetDoBackTransformedParticles();
            pc->SetDoBackTransformedParticles(do_back_transformed_particles);
+
+           if ((!old_do_btd) && do_back_transformed_particles) {
+               // Set comm to false so that the attributes are not communicated
+               // nor written to the checkpoint files
+               int const comm = 0;
+#if (AMREX_SPACEDIM >= 2)
+               pc->AddRealComp("x_n_btd", comm);
+#endif
+#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ)
+               pc->AddRealComp("y_n_btd", comm);
+#endif
+               pc->AddRealComp("z_n_btd", comm);
+               pc->AddRealComp("ux_n_btd", comm);
+               pc->AddRealComp("uy_n_btd", comm);
+               pc->AddRealComp("uz_n_btd", comm);
+           }
         }
     }
     WARPX_ALWAYS_ASSERT_WITH_MESSAGE(

Source/Particles/ParticleBoundaryBuffer.cpp

@@ -384,11 +384,11 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC
                 if (!buffer[i].isDefined())
                 {
                     buffer[i] = pc.make_alike<amrex::PinnedArenaAllocator>();
-                    buffer[i].AddIntComp("stepScraped", false);
-                    buffer[i].AddRealComp("deltaTimeScraped", false);
-                    buffer[i].AddRealComp("nx", false);
-                    buffer[i].AddRealComp("ny", false);
-                    buffer[i].AddRealComp("nz", false);
+                    buffer[i].AddIntComp("stepScraped", true);
+                    buffer[i].AddRealComp("deltaTimeScraped", true);
+                    buffer[i].AddRealComp("nx", true);
+                    buffer[i].AddRealComp("ny", true);
+                    buffer[i].AddRealComp("nz", true);
                 }
 
                 auto& species_buffer = buffer[i];
@@ -480,11 +480,11 @@ void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries (
             if (!buffer[i].isDefined())
             {
                 buffer[i] = pc.make_alike<amrex::PinnedArenaAllocator>();
-                buffer[i].AddIntComp("stepScraped", false);
-                buffer[i].AddRealComp("deltaTimeScraped", false);
-                buffer[i].AddRealComp("nx", false);
-                buffer[i].AddRealComp("ny", false);
-                buffer[i].AddRealComp("nz", false);
+                buffer[i].AddIntComp("stepScraped", true);
+                buffer[i].AddRealComp("deltaTimeScraped", true);
+                buffer[i].AddRealComp("nx", true);
+                buffer[i].AddRealComp("ny", true);
+                buffer[i].AddRealComp("nz", true);
 
             }