From 5d68b2b05b5677840a722c5d68c055555f1d4a63 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 31 Jan 2024 15:29:42 -0800 Subject: [PATCH 1/2] Fix BTD/Scrape Flush Count with Filters Move the counting of already flushed particles for writers that call the I/O backends multiple time per data set, e.g., BTD and boundary scraping, into the I/O backend. Currently, filtering is done as the first step in I/O backends and thus the previous count outside of the I/O backends was over-counting particles that might still get filtered out. Offset should be a `long`: Overflow risk is very high for pure `int`. Also, counter is `unsigned`, so `unsigned long` for now. --- Source/Diagnostics/BTDiagnostics.H | 2 - Source/Diagnostics/BTDiagnostics.cpp | 21 ++----- .../BoundaryScrapingDiagnostics.cpp | 8 ++- Source/Diagnostics/Diagnostics.H | 2 +- Source/Diagnostics/Diagnostics.cpp | 10 ++++ Source/Diagnostics/FlushFormats/FlushFormat.H | 4 +- .../FlushFormats/FlushFormatAscent.H | 4 +- .../FlushFormats/FlushFormatAscent.cpp | 3 +- .../FlushFormats/FlushFormatCheckpoint.H | 4 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 3 +- .../FlushFormats/FlushFormatOpenPMD.H | 4 +- .../FlushFormats/FlushFormatOpenPMD.cpp | 5 +- .../FlushFormats/FlushFormatPlotfile.H | 6 +- .../FlushFormats/FlushFormatPlotfile.cpp | 18 ++++-- .../FlushFormats/FlushFormatSensei.H | 4 +- .../FlushFormats/FlushFormatSensei.cpp | 4 +- Source/Diagnostics/FullDiagnostics.cpp | 16 +++++- Source/Diagnostics/WarpXOpenPMD.H | 10 ++-- Source/Diagnostics/WarpXOpenPMD.cpp | 56 +++++++++---------- 19 files changed, 104 insertions(+), 80 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index ab894da69c2..f6c44c777ea 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -399,8 +399,6 @@ private: lab-frame data. */ void InitializeParticleFunctors () override; - /** Update total number of particles flushed for all species for ith snapshot */ - void UpdateTotalParticlesFlushed(int i_buffer); /** Reset total number of particles in the particle buffer to 0 for ith snapshot */ void ResetTotalParticlesInBuffer(int i_buffer); /** Clear particle data stored in the particle buffer */ diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 0e517e8190c..01b128462c7 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -1065,12 +1065,13 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) } } m_flush_format->WriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - labtime, m_output_species[i_buffer], nlev_output, file_name, m_file_min_digits, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + labtime, + m_totalParticles_flushed_already.at(i_buffer), + m_output_species.at(i_buffer), nlev_output, file_name, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, - use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter[i_buffer], - m_max_buffer_multifabs[i_buffer], m_geom_snapshot[i_buffer][0], isLastBTDFlush, - m_totalParticles_flushed_already[i_buffer]); + use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter.at(i_buffer), + m_max_buffer_multifabs.at(i_buffer), m_geom_snapshot.at(i_buffer).at(0), isLastBTDFlush); // Rescaling the box for plotfile after WriteToFile. This is because, for plotfiles, when writing particles, amrex checks if the particles are within the bounds defined by the box. However, in BTD, particles can be (at max) 1 cell outside the bounds of the geometry. So we keep a one-cell bigger box for plotfile when writing out the particle data and rescale after. if (m_format == "plotfile") { @@ -1104,7 +1105,6 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) NullifyFirstFlush(i_buffer); // if particles are selected for output then update and reset counters if (!m_output_species_names.empty()) { - UpdateTotalParticlesFlushed(i_buffer); ResetTotalParticlesInBuffer(i_buffer); ClearParticleBuffer(i_buffer); } @@ -1489,15 +1489,6 @@ BTDiagnostics::PrepareParticleDataForOutput() } } -void -BTDiagnostics::UpdateTotalParticlesFlushed(int i_buffer) -{ - for (int isp = 0; isp < m_totalParticles_flushed_already[i_buffer].size(); ++isp) { - m_totalParticles_flushed_already[i_buffer][isp] += static_cast( - m_particles_buffer[i_buffer][isp]->TotalNumberOfParticles()); - } -} - void BTDiagnostics::ResetTotalParticlesInBuffer(int i_buffer) { diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index c85dbd6b226..3c7157c2aed 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -157,11 +157,13 @@ BoundaryScrapingDiagnostics::Flush (int i_buffer, bool /* force_flush */) const std::string file_prefix = m_file_prefix + "/particles_at_" + particle_buffer.boundaryName(i_buffer); m_flush_format->WriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - warpx.gett_new(0), m_output_species[i_buffer], nlev_output, file_prefix, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_totalParticles_flushed_already.at(i_buffer), + m_output_species.at(i_buffer), nlev_output, file_prefix, m_file_min_digits, false, false, use_pinned_pc, isBTD, warpx.getistep(0), bufferID, numBTDBuffers, geom, - isLastBTD, m_totalParticles_flushed_already[i_buffer]); + isLastBTD); // Now that the data has been written out, clear out the buffer particle_buffer.clearParticles(i_buffer); diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 53ce319d747..69ac8724a78 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -313,7 +313,7 @@ protected: * The first vector is for total number of snapshots and second vector loops * over the total number of species selected for diagnostics. */ - amrex::Vector< amrex::Vector > m_totalParticles_flushed_already; + amrex::Vector< amrex::Vector > m_totalParticles_flushed_already; /** Vector of total number of particles in the buffer, per species, per snapshot. * The first vector is for total number of snapshots and second vector loops * over the total number of species selected for diagnostics. diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 3b5daabaffa..d7691f38a88 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -516,6 +516,16 @@ Diagnostics::InitBaseData () // allocate vector of particle buffers m_output_species.resize(m_num_buffers); + + // Initialize total number of particles flushed + m_totalParticles_flushed_already.resize(m_num_buffers); + for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { + int const n_species = static_cast(m_output_species_names.size()); + m_totalParticles_flushed_already[i_buffer].resize(n_species); + for (int i_species=0; i_species& mf, amrex::Vector& geom, amrex::Vector iteration, double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, @@ -24,8 +25,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const = 0; + bool isLastBTDFlush = false) const = 0; FlushFormat () = default; virtual ~FlushFormat() = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H index 228e4bc5cf6..e84df31fff2 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H @@ -33,6 +33,7 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, @@ -41,8 +42,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false ) const override; #ifdef AMREX_USE_ASCENT /** \brief Do in-situ visualization for particle data. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index 980047e3b46..ad00f11c54a 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -15,13 +15,14 @@ FlushFormatAscent::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, + amrex::Vector& /* totalParticlesFlushedAlready*/, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, const bool /*use_pinned_pc*/, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, - bool /*isLastBTDFlush*/, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool /*isLastBTDFlush*/) const { #ifdef AMREX_USE_ASCENT WARPX_PROFILE("FlushFormatAscent::WriteToFile()"); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H index f6aad226d75..292441314d8 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H @@ -20,6 +20,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, @@ -28,8 +29,7 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const final; + bool isLastBTDFlush = false) const final; void CheckpointParticles (const std::string& dir, const amrex::Vector& particle_diags) const; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 5f59cd723da..e2c28e1ba96 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -31,6 +31,7 @@ FlushFormatCheckpoint::WriteToFile ( const amrex::Vector& /*mf*/, amrex::Vector& geom, const amrex::Vector iteration, const double /*time*/, + amrex::Vector& /* totalParticlesFlushedAlready*/, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool /*plot_raw_fields*/, @@ -39,7 +40,7 @@ FlushFormatCheckpoint::WriteToFile ( bool /*isBTD*/, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, const amrex::Geometry& /*full_BTD_snapshot*/, - bool /*isLastBTDFlush*/, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool /*isLastBTDFlush*/) const { WARPX_PROFILE("FlushFormatCheckpoint::WriteToFile()"); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H index 88380407f5e..ae603d3580c 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H @@ -32,6 +32,7 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int output_levels, std::string prefix, int file_min_digits, bool plot_raw_fields, @@ -40,8 +41,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false ) const override; ~FlushFormatOpenPMD () override = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index 3b7006243e7..71558d69e2b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -120,13 +120,14 @@ FlushFormatOpenPMD::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int output_levels, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, const bool use_pinned_pc, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& full_BTD_snapshot, - bool isLastBTDFlush, const amrex::Vector& totalParticlesFlushedAlready) const + bool isLastBTDFlush) const { WARPX_PROFILE("FlushFormatOpenPMD::WriteToFile()"); const std::string& filename = amrex::Concatenate(prefix, iteration[0], file_min_digits); @@ -164,7 +165,7 @@ FlushFormatOpenPMD::WriteToFile ( // particles: all (reside only on locally finest level) m_OpenPMDPlotWriter->WriteOpenPMDParticles( - particle_diags, static_cast(time), use_pinned_pc, isBTD, isLastBTDFlush, totalParticlesFlushedAlready); + particle_diags, static_cast(time), totalParticlesFlushedAlready, use_pinned_pc, isBTD, isLastBTDFlush); // signal that no further updates will be written to this iteration m_OpenPMDPlotWriter->CloseStep(isBTD, isLastBTDFlush); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index 486dcc3b5ee..9579030b1ac 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -27,6 +27,7 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, @@ -35,8 +36,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false) const override; /** Write general info of the run into the plotfile */ void WriteJobInfo(const std::string& dir) const; @@ -49,11 +49,13 @@ public: * \param[in] dir name of output directory * \param[in] particle_diags Each element of this vector handles output of 1 species. * \param[in] time the simulation time on the coarsest level + * \param[inout] totalParticlesFlushedAlready already flushed particles per species * \param[in] isBTD whether this is a back-transformed diagnostic */ void WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, amrex::Real time, + amrex::Vector& totalParticlesFlushedAlready, bool isBTD = false) const; FlushFormatPlotfile () = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index df73ed34c94..afe784cf3cb 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -59,13 +59,14 @@ FlushFormatPlotfile::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, const bool /*use_pinned_pc*/, bool isBTD, int snapshotID, int bufferID, int numBuffers, const amrex::Geometry& /*full_BTD_snapshot*/, - bool isLastBTDFlush, const amrex::Vector& /* totalParticlesFlushedAlready*/) const + bool isLastBTDFlush) const { WARPX_PROFILE("FlushFormatPlotfile::WriteToFile()"); auto & warpx = WarpX::GetInstance(); @@ -99,7 +100,7 @@ FlushFormatPlotfile::WriteToFile ( WriteAllRawFields(plot_raw_fields, nlev, filename, plot_raw_fields_guards); - WriteParticles(filename, particle_diags, static_cast(time), isBTD); + WriteParticles(filename, particle_diags, static_cast(time), totalParticlesFlushedAlready, isBTD); WriteJobInfo(filename); @@ -340,10 +341,15 @@ FlushFormatPlotfile::WriteWarpXHeader( void FlushFormatPlotfile::WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, - const amrex::Real time, bool isBTD) const + const amrex::Real time, + amrex::Vector& totalParticlesFlushedAlready, + bool isBTD) const { - + int i = 0; for (const auto& part_diag : particle_diags) { + // only BTD writes multiple times into the same step, zero out for other methods + if (!isBTD) { totalParticlesFlushedAlready.at(i) = 0; } + WarpXParticleContainer* pc = part_diag.getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = part_diag.getPinnedParticleContainer(); auto tmp = isBTD ? @@ -418,6 +424,10 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, dir, part_diag.getSpeciesName(), real_flags, int_flags, real_names, int_names); + + // keep book of filtered-and-written particles + totalParticlesFlushedAlready[i] += tmp.TotalNumberOfParticles(false, true); + i++; } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index 54eb7099ba4..36a181069f6 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -53,6 +53,7 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, @@ -61,8 +62,7 @@ public: bool isBTD = false, int snapshotID = -1, int bufferID = 1, int numBuffers = 1, const amrex::Geometry& full_BTD_snapshot = amrex::Geometry(), - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector() ) const override; + bool isLastBTDFlush = false) const override; /** \brief Do in-situ visualization for particle data. * \param[in] particle_diags Each element of this vector handles output of 1 species. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index e162b8b3121..f18201ef782 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -48,13 +48,13 @@ FlushFormatSensei::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, + amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, const bool use_pinned_pc, bool isBTD, int /*snapshotID*/, int /*bufferID*/, int /*numBuffers*/, - const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/, - const amrex::Vector& totalParticlesFlushedAlready) const + const amrex::Geometry& /*full_BTD_snapshot*/, bool /*isLastBTDFlush*/) const { amrex::ignore_unused( geom, nlev, prefix, file_min_digits, diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 4f1e47a2a52..455e361c2d5 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -75,6 +75,16 @@ FullDiagnostics::InitializeParticleBuffer () m_output_species[i_buffer].push_back(ParticleDiag(m_diag_name, species, mpc.GetParticleContainerPtr(idx))); } } + + // Initialize total number of particles flushed + m_totalParticles_flushed_already.resize(m_num_buffers); + for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { + int const n_species = static_cast(m_output_species_names.size()); + m_totalParticles_flushed_already[i_buffer].resize(n_species); + for (int i_species=0; i_speciesWriteToFile( - m_varnames, m_mf_output[i_buffer], m_geom_output[i_buffer], warpx.getistep(), - warpx.gett_new(0), m_output_species[i_buffer], nlev_output, m_file_prefix, + m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + warpx.gett_new(0), + m_totalParticles_flushed_already.at(i_buffer), + m_output_species.at(i_buffer), nlev_output, m_file_prefix, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); FlushRaw(); diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index e3b7b893d0a..3356d5a63ce 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -123,10 +123,10 @@ public: void WriteOpenPMDParticles ( const amrex::Vector& particle_diags, amrex::Real time, + amrex::Vector& totalParticlesFlushedAlready, bool use_pinned_pc = false, bool isBTD = false, - bool isLastBTDFlush = false, - const amrex::Vector& totalParticlesFlushedAlready = amrex::Vector()); + bool isLastBTDFlush = false); /** Write out all openPMD fields for all active MR levels * @@ -290,9 +290,9 @@ private: * @param[in] int_comp_names The int attribute names, from WarpX * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles + * @param[inout] ParticleFlushOffset previously flushed number of particles in BTD * @param[in] isBTD is this a backtransformed diagnostics (BTD) write? * @param[in] isLastBTDFlush is this the last time we will flush this BTD station? - * @param[in] ParticleFlushOffset previously flushed number of particles in BTD */ void DumpToFile (ParticleContainer* pc, const std::string& name, @@ -303,9 +303,9 @@ private: const amrex::Vector& int_comp_names, amrex::ParticleReal charge, amrex::ParticleReal mass, + unsigned long & ParticleFlushOffset, bool isBTD = false, - bool isLastBTDFlush = false, - int ParticleFlushOffset = 0); + bool isLastBTDFlush = false); /** Get the openPMD-api filename for openPMD::Series * diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 71d96a47927..a5029fdb243 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -519,9 +519,12 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) void WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& particle_diags, - const amrex::Real time, const bool use_pinned_pc, - const bool isBTD, const bool isLastBTDFlush, - const amrex::Vector& totalParticlesFlushedAlready) + const amrex::Real time, + amrex::Vector& totalParticlesFlushedAlready, + const bool use_pinned_pc, + const bool isBTD, + const bool isLastBTDFlush +) { WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()"); @@ -618,31 +621,16 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. - { - if (isBTD) { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - totalParticlesFlushedAlready[i] - ); - } else { - DumpToFile(&tmp, - particle_diags[i].getSpeciesName(), - m_CurrentStep, - real_flags, - int_flags, - real_names, int_names, - pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush, - 0 - ); - } - } + DumpToFile(&tmp, + particle_diags.at(i).getSpeciesName(), + m_CurrentStep, + real_flags, + int_flags, + real_names, int_names, + pc->getCharge(), pc->getMass(), + totalParticlesFlushedAlready.at(i), + isBTD, isLastBTDFlush + ); } } @@ -656,9 +644,11 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, const amrex::Vector& int_comp_names, amrex::ParticleReal const charge, amrex::ParticleReal const mass, + unsigned long & ParticleFlushOffset, const bool isBTD, - const bool isLastBTDFlush, - int ParticleFlushOffset) { + const bool isLastBTDFlush +) +{ WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD: series must be initialized"); AMREX_ALWAYS_ASSERT(write_real_comp.size() == pc->NumRealComps()); @@ -666,6 +656,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc->NumRealComps()); AMREX_ALWAYS_ASSERT(int_comp_names.size() == pc->NumIntComps()); + // only BTD writes multiple times into the same step, zero out for other methods + if (!isBTD) { ParticleFlushOffset = 0; } + WarpXParticleCounter counter(pc); auto const num_dump_particles = counter.GetTotalNumParticles(); @@ -864,6 +857,9 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, } m_Series->flush(); + + // keep book of filtered-and-written particles + ParticleFlushOffset += num_dump_particles; } void From 24837249999afa465337bd8d5b2bab3516eeca13 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 1 Feb 2024 14:16:37 -0800 Subject: [PATCH 2/2] Simplify: Remove `m_totalParticles_flushed_already` Less state we can forget in checkpoint-restart and that we have to transfer across API boundaries. --- Source/Diagnostics/BTDiagnostics.cpp | 12 ++++------- .../BoundaryScrapingDiagnostics.cpp | 13 ++---------- Source/Diagnostics/Diagnostics.H | 5 ----- Source/Diagnostics/Diagnostics.cpp | 10 ---------- Source/Diagnostics/FlushFormats/FlushFormat.H | 1 - .../FlushFormats/FlushFormatAscent.H | 1 - .../FlushFormats/FlushFormatAscent.cpp | 1 - .../FlushFormats/FlushFormatCheckpoint.H | 1 - .../FlushFormats/FlushFormatCheckpoint.cpp | 1 - .../FlushFormats/FlushFormatOpenPMD.H | 1 - .../FlushFormats/FlushFormatOpenPMD.cpp | 3 +-- .../FlushFormats/FlushFormatPlotfile.H | 3 --- .../FlushFormats/FlushFormatPlotfile.cpp | 12 +---------- .../FlushFormats/FlushFormatSensei.H | 1 - .../FlushFormats/FlushFormatSensei.cpp | 5 ++--- Source/Diagnostics/FullDiagnostics.cpp | 11 ---------- Source/Diagnostics/OpenPMDHelpFunction.H | 18 +++++++++++++++++ Source/Diagnostics/OpenPMDHelpFunction.cpp | 20 +++++++++++++++++++ Source/Diagnostics/WarpXOpenPMD.H | 2 -- Source/Diagnostics/WarpXOpenPMD.cpp | 12 +++-------- 20 files changed, 51 insertions(+), 82 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 01b128462c7..f7965cd2688 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -129,7 +129,6 @@ void BTDiagnostics::DerivedInitData () } } m_particles_buffer.resize(m_num_buffers); - m_totalParticles_flushed_already.resize(m_num_buffers); m_totalParticles_in_buffer.resize(m_num_buffers); // check that simulation can fill all BTD snapshots @@ -1067,7 +1066,6 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) m_flush_format->WriteToFile( m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), labtime, - m_totalParticles_flushed_already.at(i_buffer), m_output_species.at(i_buffer), nlev_output, file_name, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter.at(i_buffer), @@ -1271,10 +1269,10 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) InterleaveSpeciesHeader(recent_species_Header,snapshot_species_Header, m_output_species_names[i], m_buffer_flush_counter[i_snapshot]); if (BufferSpeciesHeader.m_total_particles == 0) { continue; } - if (m_totalParticles_flushed_already[i_snapshot][i]==0) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, - std::string("Renaming ").append(recent_ParticleHdrFilename).append(" to ").append(snapshot_ParticleHdrFilename).append(" has failed")); + if (!amrex::FileExists(snapshot_ParticleHdrFilename)) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + std::rename(recent_ParticleHdrFilename.c_str(), snapshot_ParticleHdrFilename.c_str()) == 0, + std::string("Renaming ").append(recent_ParticleHdrFilename).append(" to ").append(snapshot_ParticleHdrFilename).append(" has failed")); } else { InterleaveParticleDataHeader(recent_ParticleHdrFilename, snapshot_ParticleHdrFilename); @@ -1435,10 +1433,8 @@ BTDiagnostics::InitializeParticleBuffer () const MultiParticleContainer& mpc = warpx.GetPartContainer(); for (int i = 0; i < m_num_buffers; ++i) { m_particles_buffer[i].resize(m_output_species_names.size()); - m_totalParticles_flushed_already[i].resize(m_output_species_names.size()); m_totalParticles_in_buffer[i].resize(m_output_species_names.size()); for (int isp = 0; isp < m_particles_buffer[i].size(); ++isp) { - m_totalParticles_flushed_already[i][isp] = 0; m_totalParticles_in_buffer[i][isp] = 0; m_particles_buffer[i][isp] = std::make_unique(WarpX::GetInstance().GetParGDB()); const int idx = mpc.getSpeciesID(m_output_species_names[isp]); diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 3c7157c2aed..11ffce02f09 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -102,15 +102,6 @@ BoundaryScrapingDiagnostics::InitializeParticleBuffer () m_output_species[i_buffer].push_back(ParticleDiag(m_diag_name, species_name, pc, bnd_buffer)); } } - // Initialize total number of particles flushed - m_totalParticles_flushed_already.resize(m_num_buffers); - for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - int const n_species = static_cast(m_output_species_names.size()); - m_totalParticles_flushed_already[i_buffer].resize(n_species); - for (int i_species=0; i_speciesWriteToFile( m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), warpx.gett_new(0), - m_totalParticles_flushed_already.at(i_buffer), - m_output_species.at(i_buffer), nlev_output, file_prefix, + m_output_species.at(i_buffer), + nlev_output, file_prefix, m_file_min_digits, false, false, use_pinned_pc, isBTD, warpx.getistep(0), bufferID, numBTDBuffers, geom, isLastBTD); diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index 69ac8724a78..c0d2a9f0d53 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -309,11 +309,6 @@ protected: /** Vector of pointers to functors to compute particle output per species*/ amrex::Vector< std::unique_ptr > m_all_particle_functors; - /** Vector of total number of particles previously flushed, per species, per snapshot. - * The first vector is for total number of snapshots and second vector loops - * over the total number of species selected for diagnostics. - */ - amrex::Vector< amrex::Vector > m_totalParticles_flushed_already; /** Vector of total number of particles in the buffer, per species, per snapshot. * The first vector is for total number of snapshots and second vector loops * over the total number of species selected for diagnostics. diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index d7691f38a88..3b5daabaffa 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -516,16 +516,6 @@ Diagnostics::InitBaseData () // allocate vector of particle buffers m_output_species.resize(m_num_buffers); - - // Initialize total number of particles flushed - m_totalParticles_flushed_already.resize(m_num_buffers); - for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - int const n_species = static_cast(m_output_species_names.size()); - m_totalParticles_flushed_already[i_buffer].resize(n_species); - for (int i_species=0; i_species& mf, amrex::Vector& geom, amrex::Vector iteration, double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H index e84df31fff2..9d8d3fcd7d2 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.H @@ -33,7 +33,6 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp index ad00f11c54a..abfba37cd15 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatAscent.cpp @@ -15,7 +15,6 @@ FlushFormatAscent::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, - amrex::Vector& /* totalParticlesFlushedAlready*/, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H index 292441314d8..5c26ac97f61 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.H @@ -20,7 +20,6 @@ class FlushFormatCheckpoint final : public FlushFormatPlotfile const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index e2c28e1ba96..d77437fb931 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -31,7 +31,6 @@ FlushFormatCheckpoint::WriteToFile ( const amrex::Vector& /*mf*/, amrex::Vector& geom, const amrex::Vector iteration, const double /*time*/, - amrex::Vector& /* totalParticlesFlushedAlready*/, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool /*plot_raw_fields*/, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H index ae603d3580c..141760ac2a3 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.H @@ -32,7 +32,6 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int output_levels, std::string prefix, int file_min_digits, bool plot_raw_fields, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp index 71558d69e2b..e0c8c4ef2d6 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatOpenPMD.cpp @@ -120,7 +120,6 @@ FlushFormatOpenPMD::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int output_levels, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, @@ -165,7 +164,7 @@ FlushFormatOpenPMD::WriteToFile ( // particles: all (reside only on locally finest level) m_OpenPMDPlotWriter->WriteOpenPMDParticles( - particle_diags, static_cast(time), totalParticlesFlushedAlready, use_pinned_pc, isBTD, isLastBTDFlush); + particle_diags, static_cast(time), use_pinned_pc, isBTD, isLastBTDFlush); // signal that no further updates will be written to this iteration m_OpenPMDPlotWriter->CloseStep(isBTD, isLastBTDFlush); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H index 9579030b1ac..c62056b8907 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.H @@ -27,7 +27,6 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, @@ -49,13 +48,11 @@ public: * \param[in] dir name of output directory * \param[in] particle_diags Each element of this vector handles output of 1 species. * \param[in] time the simulation time on the coarsest level - * \param[inout] totalParticlesFlushedAlready already flushed particles per species * \param[in] isBTD whether this is a back-transformed diagnostic */ void WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, amrex::Real time, - amrex::Vector& totalParticlesFlushedAlready, bool isBTD = false) const; FlushFormatPlotfile () = default; diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index afe784cf3cb..970d9a504d2 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -59,7 +59,6 @@ FlushFormatPlotfile::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, @@ -100,7 +99,7 @@ FlushFormatPlotfile::WriteToFile ( WriteAllRawFields(plot_raw_fields, nlev, filename, plot_raw_fields_guards); - WriteParticles(filename, particle_diags, static_cast(time), totalParticlesFlushedAlready, isBTD); + WriteParticles(filename, particle_diags, static_cast(time), isBTD); WriteJobInfo(filename); @@ -342,14 +341,9 @@ void FlushFormatPlotfile::WriteParticles(const std::string& dir, const amrex::Vector& particle_diags, const amrex::Real time, - amrex::Vector& totalParticlesFlushedAlready, bool isBTD) const { - int i = 0; for (const auto& part_diag : particle_diags) { - // only BTD writes multiple times into the same step, zero out for other methods - if (!isBTD) { totalParticlesFlushedAlready.at(i) = 0; } - WarpXParticleContainer* pc = part_diag.getParticleContainer(); PinnedMemoryParticleContainer* pinned_pc = part_diag.getPinnedParticleContainer(); auto tmp = isBTD ? @@ -424,10 +418,6 @@ FlushFormatPlotfile::WriteParticles(const std::string& dir, dir, part_diag.getSpeciesName(), real_flags, int_flags, real_names, int_names); - - // keep book of filtered-and-written particles - totalParticlesFlushedAlready[i] += tmp.TotalNumberOfParticles(false, true); - i++; } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index 36a181069f6..d2ec9a5a4e0 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -53,7 +53,6 @@ public: const amrex::Vector& mf, amrex::Vector& geom, amrex::Vector iteration, double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, std::string prefix, int file_min_digits, bool plot_raw_fields, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index f18201ef782..348e1da4a00 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -48,7 +48,6 @@ FlushFormatSensei::WriteToFile ( const amrex::Vector& mf, amrex::Vector& geom, const amrex::Vector iteration, const double time, - amrex::Vector& totalParticlesFlushedAlready, const amrex::Vector& particle_diags, int nlev, const std::string prefix, int file_min_digits, bool plot_raw_fields, bool plot_raw_fields_guards, @@ -59,8 +58,8 @@ FlushFormatSensei::WriteToFile ( amrex::ignore_unused( geom, nlev, prefix, file_min_digits, plot_raw_fields, plot_raw_fields_guards, - use_pinned_pc, - totalParticlesFlushedAlready); + use_pinned_pc + ); #ifndef AMREX_USE_SENSEI_INSITU amrex::ignore_unused(varnames, mf, iteration, time, particle_diags, diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 455e361c2d5..fd329a38220 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -75,16 +75,6 @@ FullDiagnostics::InitializeParticleBuffer () m_output_species[i_buffer].push_back(ParticleDiag(m_diag_name, species, mpc.GetParticleContainerPtr(idx))); } } - - // Initialize total number of particles flushed - m_totalParticles_flushed_already.resize(m_num_buffers); - for (int i_buffer = 0; i_buffer < m_num_buffers; ++i_buffer) { - int const n_species = static_cast(m_output_species_names.size()); - m_totalParticles_flushed_already[i_buffer].resize(n_species); - for (int i_species=0; i_speciesWriteToFile( m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), warpx.gett_new(0), - m_totalParticles_flushed_already.at(i_buffer), m_output_species.at(i_buffer), nlev_output, m_file_prefix, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards); diff --git a/Source/Diagnostics/OpenPMDHelpFunction.H b/Source/Diagnostics/OpenPMDHelpFunction.H index 9db4b9fb194..d2f2c4f9f9d 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.H +++ b/Source/Diagnostics/OpenPMDHelpFunction.H @@ -14,7 +14,25 @@ #include +/** Determine the preferred file ending if unspecified + * + * @return file ending without the "." + */ std::string WarpXOpenPMDFileType (); +#ifdef WARPX_USE_OPENPMD +/** Determine how many particles were already written in this species and step + * + * This checks for a particle species the current size of the id attribute, if it exists, + * and if it does it takes its extent as the number of particles already on disk. + * + * Note that this checks declared size, not necessarily written size. + * + * @return exisitng extent of the "id" attribute or zero. + */ +unsigned long +num_already_flushed (openPMD::ParticleSpecies & currSpecies); +#endif + #endif // WARPX_OPENPMDHELPFUNCTION_H_ diff --git a/Source/Diagnostics/OpenPMDHelpFunction.cpp b/Source/Diagnostics/OpenPMDHelpFunction.cpp index a898c97b6b4..6170249b52b 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.cpp +++ b/Source/Diagnostics/OpenPMDHelpFunction.cpp @@ -27,3 +27,23 @@ WarpXOpenPMDFileType () #endif // WARPX_USE_OPENPMD return openPMDFileType; } + +#ifdef WARPX_USE_OPENPMD +unsigned long +num_already_flushed (openPMD::ParticleSpecies & currSpecies) +{ + const auto *const scalar = openPMD::RecordComponent::SCALAR; + + unsigned long ParticleFlushOffset = 0; + + if (currSpecies.contains("id")) { + if (currSpecies["id"].contains(scalar)) { + if (!currSpecies["id"][scalar].empty()) { + ParticleFlushOffset = currSpecies["id"][scalar].getExtent().at(0); + } + } + } + + return ParticleFlushOffset; +} +#endif diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 3356d5a63ce..4597dacd9ae 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -123,7 +123,6 @@ public: void WriteOpenPMDParticles ( const amrex::Vector& particle_diags, amrex::Real time, - amrex::Vector& totalParticlesFlushedAlready, bool use_pinned_pc = false, bool isBTD = false, bool isLastBTDFlush = false); @@ -303,7 +302,6 @@ private: const amrex::Vector& int_comp_names, amrex::ParticleReal charge, amrex::ParticleReal mass, - unsigned long & ParticleFlushOffset, bool isBTD = false, bool isLastBTDFlush = false); diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index a5029fdb243..64411ecf6e4 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -520,7 +520,6 @@ WarpXOpenPMDPlot::Init (openPMD::Access access, bool isBTD) void WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector& particle_diags, const amrex::Real time, - amrex::Vector& totalParticlesFlushedAlready, const bool use_pinned_pc, const bool isBTD, const bool isLastBTDFlush @@ -628,7 +627,6 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { int_flags, real_names, int_names, pc->getCharge(), pc->getMass(), - totalParticlesFlushedAlready.at(i), isBTD, isLastBTDFlush ); } @@ -644,7 +642,6 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, const amrex::Vector& int_comp_names, amrex::ParticleReal const charge, amrex::ParticleReal const mass, - unsigned long & ParticleFlushOffset, const bool isBTD, const bool isLastBTDFlush ) @@ -656,15 +653,15 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc->NumRealComps()); AMREX_ALWAYS_ASSERT(int_comp_names.size() == pc->NumIntComps()); - // only BTD writes multiple times into the same step, zero out for other methods - if (!isBTD) { ParticleFlushOffset = 0; } - WarpXParticleCounter counter(pc); auto const num_dump_particles = counter.GetTotalNumParticles(); openPMD::Iteration currIteration = GetIteration(iteration, isBTD); openPMD::ParticleSpecies currSpecies = currIteration.particles[name]; + // only BTD writes multiple times into the same step, zero for other methods + unsigned long ParticleFlushOffset = isBTD ? num_already_flushed(currSpecies) : 0; + // prepare data structures the first time BTD has non-zero particles // we set some of them to zero extent, so we need to time that well bool const is_first_flush_with_particles = num_dump_particles > 0 && ParticleFlushOffset == 0; @@ -857,9 +854,6 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, } m_Series->flush(); - - // keep book of filtered-and-written particles - ParticleFlushOffset += num_dump_particles; } void