From 01416dc0765b2e56e6239b79ea2be84032f3d5d4 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Mon, 19 Feb 2024 11:13:16 -0800 Subject: [PATCH 1/4] redo metadata for fields for incomplete BTD flush --- Source/Diagnostics/BTDiagnostics.cpp | 64 +++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 16617159662..7a4f032254b 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -308,7 +308,7 @@ BTDiagnostics::DoDump (int step, int i_buffer, bool force_flush) // was already fully written and buffer was reset to zero size or that // lab snapshot was not even started to be backtransformed yet const auto do_forced_flush = (force_flush && !buffer_empty(i_buffer)); - + if (do_forced_flush) amrex::Print() << "forcing flush \n"; return is_buffer_full || last_z_slice_filled || do_forced_flush; } @@ -1015,6 +1015,66 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) bool const isBTD = true; double const labtime = m_t_lab[i_buffer]; + amrex::Vector< amrex::MultiFab > out; + out.resize(nlev_output); + for(int lev = 0; lev < nlev_output; ++lev) { + if (!force_flush) { + out[lev] = amrex::MultiFab(m_mf_output[i_buffer][lev].boxArray(), + m_mf_output[i_buffer][lev].DistributionMap(), + m_mf_output[i_buffer][lev].nComp(), 0); + out[lev].ParallelCopy( m_mf_output[i_buffer][lev], 0, 0, m_mf_output[i_buffer][lev].nComp() ); + } else { + auto const& ba = m_mf_output[i_buffer][lev].boxArray(); + auto const& dm = m_mf_output[i_buffer][lev].DistributionMap(); + amrex::BoxList bl(ba.ixType()); + amrex::Vector proc; + amrex::Vector imap; + for (int i = 0; i < int(ba.size()); ++i) { + auto b = ba[i]; + int new_small = b.bigEnd(m_moving_window_dir) + - (m_buffer_counter[i_buffer]-1); + b.setSmall(m_moving_window_dir, + std::max(b.smallEnd(m_moving_window_dir),new_small)); + if (b.ok()) { + bl.push_back(b); + proc.push_back(dm[i]); + if (dm[i] == amrex::ParallelDescriptor::MyProc()) { + imap.push_back(i); + } + } + } + out[lev] = amrex::MultiFab(amrex::BoxArray(std::move(bl)), + amrex::DistributionMapping(std::move(proc)), + m_mf_output[i_buffer][lev].nComp(),0); + for (amrex::MFIter mfi(out[lev], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + auto const& bx = mfi.tilebox(); + (out[lev])[mfi].template copy + ( (m_mf_output[i_buffer][lev])[imap[mfi.LocalIndex()]], + bx, 0, bx, 0, m_mf_output[i_buffer][lev].nComp() ); + } + amrex::BoxArray newba = out[lev].boxArray(); + amrex::Box newbbox = newba.minimalBox(); + //int new_small = std::max(m_snapshot_box[i_buffer].smallEnd(m_moving_window_dir), + // newbbox.smallEnd(m_moving_window_dir)); + int ncellsflushed = m_buffer_flush_counter[i_buffer]*m_buffer_size; + int currentncells = newbbox.bigEnd(m_moving_window_dir) - newbbox.smallEnd(m_moving_window_dir); + int new_small = newbbox.smallEnd(m_moving_window_dir); + m_snapshot_box[i_buffer].setSmall(m_moving_window_dir, new_small); + auto ref_ratio = amrex::IntVect(1); + if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } + const amrex::Real new_zlo = m_snapshot_domain_lab[i_buffer].hi(m_moving_window_dir) - + (currentncells + ncellsflushed) * + dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]); + m_snapshot_domain_lab[i_buffer].setLo(m_moving_window_dir, new_zlo); + amrex::Vector BTdiag_periodicity(AMREX_SPACEDIM, 0); + m_geom_snapshot[i_buffer][lev].define( m_snapshot_box[i_buffer], + &m_snapshot_domain_lab[i_buffer], + amrex::CoordSys::cartesian, + BTdiag_periodicity.data() ); + } + } + amrex::Vector vba; amrex::Vector vdmap; amrex::Vector vgeom; @@ -1064,7 +1124,7 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) } } m_flush_format->WriteToFile( - m_varnames, m_mf_output.at(i_buffer), m_geom_output.at(i_buffer), warpx.getistep(), + m_varnames, out, m_geom_output.at(i_buffer), warpx.getistep(), labtime, m_output_species.at(i_buffer), nlev_output, file_name, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, From 5241f9debabdb312f0b124dcf564896d7a242e5b Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Thu, 22 Feb 2024 17:21:17 -0800 Subject: [PATCH 2/4] compute geom for all flushes --- Source/Diagnostics/BTDiagnostics.cpp | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 7a4f032254b..756e680687a 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -1018,12 +1018,12 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) amrex::Vector< amrex::MultiFab > out; out.resize(nlev_output); for(int lev = 0; lev < nlev_output; ++lev) { - if (!force_flush) { - out[lev] = amrex::MultiFab(m_mf_output[i_buffer][lev].boxArray(), - m_mf_output[i_buffer][lev].DistributionMap(), - m_mf_output[i_buffer][lev].nComp(), 0); - out[lev].ParallelCopy( m_mf_output[i_buffer][lev], 0, 0, m_mf_output[i_buffer][lev].nComp() ); - } else { + //if (!force_flush) { + //out[lev] = amrex::MultiFab(m_mf_output[i_buffer][lev].boxArray(), + // m_mf_output[i_buffer][lev].DistributionMap(), + // m_mf_output[i_buffer][lev].nComp(), 0); + //out[lev].ParallelCopy( m_mf_output[i_buffer][lev], 0, 0, m_mf_output[i_buffer][lev].nComp() ); + //} else { auto const& ba = m_mf_output[i_buffer][lev].boxArray(); auto const& dm = m_mf_output[i_buffer][lev].DistributionMap(); amrex::BoxList bl(ba.ixType()); @@ -1066,13 +1066,15 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) const amrex::Real new_zlo = m_snapshot_domain_lab[i_buffer].hi(m_moving_window_dir) - (currentncells + ncellsflushed) * dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]); + amrex::Print() << " old z lo : " << m_snapshot_domain_lab[i_buffer].lo(m_moving_window_dir) << " \n"; m_snapshot_domain_lab[i_buffer].setLo(m_moving_window_dir, new_zlo); + amrex::Print() << " new z lo : " << m_snapshot_domain_lab[i_buffer].lo(m_moving_window_dir) << " \n"; amrex::Vector BTdiag_periodicity(AMREX_SPACEDIM, 0); m_geom_snapshot[i_buffer][lev].define( m_snapshot_box[i_buffer], &m_snapshot_domain_lab[i_buffer], amrex::CoordSys::cartesian, BTdiag_periodicity.data() ); - } + //} } amrex::Vector vba; @@ -1123,6 +1125,7 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) } } } + amrex::Print() << " m_buffer_flush_counter : " << m_buffer_flush_counter[i_buffer] << "\n"; m_flush_format->WriteToFile( m_varnames, out, m_geom_output.at(i_buffer), warpx.getistep(), labtime, From 5be28a89e6c722b1a07e422c38faa4dc57796142 Mon Sep 17 00:00:00 2001 From: RevathiJambunathan Date: Thu, 22 Feb 2024 17:41:32 -0800 Subject: [PATCH 3/4] local copies of geom, box, domain --- Source/Diagnostics/BTDiagnostics.cpp | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 756e680687a..a81f9019588 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -1017,6 +1017,7 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) amrex::Vector< amrex::MultiFab > out; out.resize(nlev_output); + amrex::Vector new_geom(nlev_output); for(int lev = 0; lev < nlev_output; ++lev) { //if (!force_flush) { //out[lev] = amrex::MultiFab(m_mf_output[i_buffer][lev].boxArray(), @@ -1060,21 +1061,28 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) int ncellsflushed = m_buffer_flush_counter[i_buffer]*m_buffer_size; int currentncells = newbbox.bigEnd(m_moving_window_dir) - newbbox.smallEnd(m_moving_window_dir); int new_small = newbbox.smallEnd(m_moving_window_dir); - m_snapshot_box[i_buffer].setSmall(m_moving_window_dir, new_small); + amrex::Box new_snapshot_box = m_snapshot_box[i_buffer]; + new_snapshot_box.setSmall(m_moving_window_dir, new_small); auto ref_ratio = amrex::IntVect(1); if (lev > 0 ) { ref_ratio = WarpX::RefRatio(lev-1); } const amrex::Real new_zlo = m_snapshot_domain_lab[i_buffer].hi(m_moving_window_dir) - (currentncells + ncellsflushed) * dz_lab(warpx.getdt(lev), ref_ratio[m_moving_window_dir]); amrex::Print() << " old z lo : " << m_snapshot_domain_lab[i_buffer].lo(m_moving_window_dir) << " \n"; - m_snapshot_domain_lab[i_buffer].setLo(m_moving_window_dir, new_zlo); +// m_snapshot_domain_lab[i_buffer].setLo(m_moving_window_dir, new_zlo); + amrex::RealBox new_snapshot_lab = m_snapshot_domain_lab[i_buffer]; + new_snapshot_lab.setLo(m_moving_window_dir, new_zlo); amrex::Print() << " new z lo : " << m_snapshot_domain_lab[i_buffer].lo(m_moving_window_dir) << " \n"; amrex::Vector BTdiag_periodicity(AMREX_SPACEDIM, 0); - m_geom_snapshot[i_buffer][lev].define( m_snapshot_box[i_buffer], - &m_snapshot_domain_lab[i_buffer], - amrex::CoordSys::cartesian, - BTdiag_periodicity.data() ); - //} +// m_geom_snapshot[i_buffer][lev].define( new_snapshot_box, +// &m_snapshot_domain_lab[i_buffer], +// amrex::CoordSys::cartesian, +// BTdiag_periodicity.data() ); + new_geom[lev].define( new_snapshot_box, + &m_snapshot_domain_lab[i_buffer], + amrex::CoordSys::cartesian, + BTdiag_periodicity.data() ); + //} } amrex::Vector vba; @@ -1132,7 +1140,7 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) m_output_species.at(i_buffer), nlev_output, file_name, m_file_min_digits, m_plot_raw_fields, m_plot_raw_fields_guards, use_pinned_pc, isBTD, i_buffer, m_buffer_flush_counter.at(i_buffer), - m_max_buffer_multifabs.at(i_buffer), m_geom_snapshot.at(i_buffer).at(0), isLastBTDFlush); + m_max_buffer_multifabs.at(i_buffer), new_geom.at(0), isLastBTDFlush); // Rescaling the box for plotfile after WriteToFile. This is because, for plotfiles, when writing particles, amrex checks if the particles are within the bounds defined by the box. However, in BTD, particles can be (at max) 1 cell outside the bounds of the geometry. So we keep a one-cell bigger box for plotfile when writing out the particle data and rescale after. if (m_format == "plotfile") { From 97141016e1405542877a4c52ab13b926202978dc Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 22 Feb 2024 18:36:35 -0800 Subject: [PATCH 4/4] BTDDiags: Pass Full Size Again until Last Flush --- Source/Diagnostics/BTDiagnostics.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index a81f9019588..84a91b06cdc 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -1019,12 +1019,13 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) out.resize(nlev_output); amrex::Vector new_geom(nlev_output); for(int lev = 0; lev < nlev_output; ++lev) { - //if (!force_flush) { - //out[lev] = amrex::MultiFab(m_mf_output[i_buffer][lev].boxArray(), - // m_mf_output[i_buffer][lev].DistributionMap(), - // m_mf_output[i_buffer][lev].nComp(), 0); - //out[lev].ParallelCopy( m_mf_output[i_buffer][lev], 0, 0, m_mf_output[i_buffer][lev].nComp() ); - //} else { + if (!force_flush) { + out[lev] = amrex::MultiFab(m_mf_output[i_buffer][lev].boxArray(), + m_mf_output[i_buffer][lev].DistributionMap(), + m_mf_output[i_buffer][lev].nComp(), 0); + out[lev].ParallelCopy( m_mf_output[i_buffer][lev], 0, 0, m_mf_output[i_buffer][lev].nComp() ); + new_geom[lev] = m_geom_snapshot[i_buffer][lev]; + } else { auto const& ba = m_mf_output[i_buffer][lev].boxArray(); auto const& dm = m_mf_output[i_buffer][lev].DistributionMap(); amrex::BoxList bl(ba.ixType()); @@ -1082,7 +1083,7 @@ BTDiagnostics::Flush (int i_buffer, bool force_flush) &m_snapshot_domain_lab[i_buffer], amrex::CoordSys::cartesian, BTdiag_periodicity.data() ); - //} + } } amrex::Vector vba;