From 529723f812600d97cbc41ce676c3a7337911311c Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Tue, 4 Mar 2025 07:14:41 -0800 Subject: [PATCH] Synchronize after copying in 2D reduced diagnostics (#5726) Add `Gpu::streamSynchronize();` after copying `TableData` from either host-to-device and device-to-host in the `ParticleHistogram2D` and `DifferentialLuminosity2D` reduced diagnostics. We have observed unstable behaviors with the `DifferentialLuminosity2D` diagnostic and @RemiLehe suggested this could be a reason. Marked as bug, unless otherwise recommended. --------- Co-authored-by: Remi Lehe --- Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp | 1 + Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp index b3968b9fb02..09029185593 100644 --- a/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp +++ b/Source/Diagnostics/ReducedDiags/DifferentialLuminosity2D.cpp @@ -315,6 +315,7 @@ void DifferentialLuminosity2D::ComputeDiags (int step) // Copy data from GPU memory m_h_data_2D.copy(m_d_data_2D); + Gpu::streamSynchronize(); // reduced sum over mpi ranks const int size = static_cast (m_d_data_2D.size()); diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index 8dd19186b25..a39c5fac855 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -149,6 +149,8 @@ void ParticleHistogram2D::ComputeDiags (int step) } d_data_2D.copy(m_h_data_2D); + // Gpu::streamSynchronize() is not needed, because there is a sync in WarpXParIter constructor later + auto d_table = d_data_2D.table(); // get a reference to WarpX instance @@ -244,6 +246,7 @@ void ParticleHistogram2D::ComputeDiags (int step) // Copy data from GPU memory m_h_data_2D.copy(d_data_2D); + Gpu::streamSynchronize(); // reduced sum over mpi ranks const int size = static_cast (d_data_2D.size());