diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index a1e7f5affda..28bfaaf57a7 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -131,7 +131,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 028638564f7be0694b9898f8d4088cdbf9a6f9f5 && cd - + cd ../amrex && git checkout --detach 3734079379bb6b2a3850d197241f6b2c3b3bfa7d && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index d6030743524..c850970bca6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -54,7 +54,14 @@ jobs: run: | $env:PATH += ';C:/Program Files (x86)/WarpX/bin/' - python3 Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py + $job = Start-ThreadJob -Name "test" -ScriptBlock { & "python3" "Examples/Tests/gaussian_beam/inputs_test_3d_gaussian_beam_picmi.py" } + Start-Sleep -Seconds 120 + Stop-Job $job.Id + + $jobOutput = $job | Receive-Job + $jobOutput + + cat Backtrace.0 # JSON writes are currently very slow (50min) with MSVC # --diagformat=openpmd diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 66570644bdc..09156072cad 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -90,6 +90,11 @@ Scientific works in laser-ion acceleration and laser-matter interaction. Physical Review Research **6**, 033148, 2024. `DOI:10.1103/PhysRevResearch.6.033148 `__ +#. Zaïm N, Sainte-Marie A, Fedeli L, Bartoli P, Huebl A, Leblanc A, Vay J-L, Vincenti H. + **Light-matter interaction near the Schwinger limit using tightly focused doppler-boosted lasers**. + Physical Review Letters **132**, 175002, 2024. + `DOI:10.1103/PhysRevLett.132.175002 `__ + #. Knight B, Gautam C, Stoner C, Egner B, Smith J, Orban C, Manfredi J, Frische K, Dexter M, Chowdhury E, Patnaik A (2023). **Detailed Characterization of a kHz-rate Laser-Driven Fusion at a Thin Liquid Sheet with a Neutron Detection Suite**. High Power Laser Science and Engineering, 1-13, 2023. @@ -110,6 +115,11 @@ Scientific works in laser-ion acceleration and laser-matter interaction. Phys. Rev. Accel. Beams **25**, 093402, 2022. `DOI:10.1103/PhysRevAccelBeams.25.093402 `__ +#. Fedeli L, Sainte-Marie A, Zaïm N, Thévenet M, Vay J-L, Myers A, Quéré F, Vincenti H. + **Probing strong-field QED with Doppler-boosted PetaWatt-class lasers**. + Physical Review Letters **127**, 114801, 2021. + `DOI:10.1103/PhysRevLett.127.114801 `__ + Particle Accelerator & Beam Physics *********************************** diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index c9e986381ec..dc684fc39ed 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -666,11 +666,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Initialize multi-J loop: // 1) Prepare E,B,F,G fields in spectral space - PSATDForwardTransformEB( - m_fields.get_mr_levels_alldirs("Efield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level), - m_fields.get_mr_levels_alldirs("Efield_cp", finest_level), - m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level) ); + PSATDForwardTransformEB(); if (WarpX::do_dive_cleaning) { PSATDForwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDForwardTransformG(); } @@ -681,8 +677,10 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0) && rho_in_time == RhoInTime::Linear) { - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; // Deposit rho at relative time -dt // (dt[0] denotes the time step on mesh refinement level 0) @@ -690,7 +688,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Filter, exchange boundary, and interpolate across levels SyncRho(); // Forward FFT of rho - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_new); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_new); } // 4) Deposit J at relative time -dt with time step dt @@ -706,9 +704,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // of guard cells. SyncCurrent("current_fp"); // Forward FFT of J - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs( "current_fp", finest_level), - m_fields.get_mr_levels_alldirs( "current_cp", finest_level) ); + PSATDForwardTransformJ("current_fp", "current_cp"); } // Number of depositions for multi-J scheme @@ -742,16 +738,16 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // of guard cells. SyncCurrent("current_fp"); // Forward FFT of J - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs( "current_fp", finest_level), - m_fields.get_mr_levels_alldirs( "current_cp", finest_level) ); + PSATDForwardTransformJ("current_fp", "current_cp"); // Deposit new rho // (after checking that pointer to rho_fp on MR level 0 is not null) if (m_fields.has("rho_fp", 0)) { - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_cp", finest_level); + ablastr::fields::MultiLevelScalarField const rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; // Move rho from new to old if rho is linear in time if (rho_in_time == RhoInTime::Linear) { PSATDMoveRhoNewToRhoOld(); } @@ -762,7 +758,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) SyncRho(); // Forward FFT of rho const int rho_idx = (rho_in_time == RhoInTime::Linear) ? rho_new : rho_mid; - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_idx); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_idx); } if (WarpX::current_correction) @@ -778,10 +774,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (the relative time reached here coincides with an integer full time step) if (i_deposit == n_deposit-1) { - PSATDBackwardTransformEB(m_fields.get_mr_levels_alldirs("Efield_fp", max_level), - m_fields.get_mr_levels_alldirs("Bfield_fp", max_level), - m_fields.get_mr_levels_alldirs("Efield_cp", max_level), - m_fields.get_mr_levels_alldirs("Bfield_cp", max_level)); + PSATDBackwardTransformEB(); if (WarpX::do_dive_cleaning) { PSATDBackwardTransformF(); } if (WarpX::do_divb_cleaning) { PSATDBackwardTransformG(); } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 43a964c1806..fd1f55142f2 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -100,55 +100,85 @@ namespace { } } -void WarpX::PSATDForwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp) +void WarpX::PSATDForwardTransformEB () { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; + const std::string Efield_fp_string = "Efield_fp"; + const std::string Efield_cp_string = "Efield_cp"; + const std::string Bfield_fp_string = "Bfield_fp"; + const std::string Bfield_cp_string = "Bfield_cp"; + for (int lev = 0; lev <= finest_level; ++lev) { - ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp[lev], Idx.Ex, Idx.Ey, Idx.Ez); - ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp[lev], Idx.Bx, Idx.By, Idx.Bz); + if (m_fields.has_vector(Efield_fp_string, lev)) { + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, Idx.Ex, Idx.Ey, Idx.Ez); + } + if (m_fields.has_vector(Bfield_fp_string, lev)) { + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, Idx.Bx, Idx.By, Idx.Bz); + } if (spectral_solver_cp[lev]) { - ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp[lev], Idx.Ex, Idx.Ey, Idx.Ez); - ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp[lev], Idx.Bx, Idx.By, Idx.Bz); + if (m_fields.has_vector(Efield_cp_string, lev)) { + ablastr::fields::VectorField const E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, Idx.Ex, Idx.Ey, Idx.Ez); + } + if (m_fields.has_vector(Bfield_cp_string, lev)) { + ablastr::fields::VectorField const B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, Idx.Bx, Idx.By, Idx.Bz); + } } } } -void WarpX::PSATDBackwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp) +void WarpX::PSATDBackwardTransformEB () { const SpectralFieldIndex& Idx = spectral_solver_fp[0]->m_spectral_index; + const std::string Efield_fp_string = "Efield_fp"; + const std::string Efield_cp_string = "Efield_cp"; + const std::string Bfield_fp_string = "Bfield_fp"; + const std::string Bfield_cp_string = "Bfield_cp"; + for (int lev = 0; lev <= finest_level; ++lev) { - BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp[lev], - Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); - BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp[lev], - Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + if (m_fields.has_vector(Efield_fp_string, lev)) { + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], E_fp, + Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); + } + if (m_fields.has_vector(Bfield_fp_string, lev)) { + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], B_fp, + Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + } if (spectral_solver_cp[lev]) { - BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp[lev], - Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); - BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp[lev], - Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + if (m_fields.has_vector(Efield_cp_string, lev)) { + ablastr::fields::VectorField const E_cp = m_fields.get_alldirs(Efield_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], E_cp, + Idx.Ex, Idx.Ey, Idx.Ez, m_fill_guards_fields); + } + if (m_fields.has_vector(Bfield_cp_string, lev)) { + ablastr::fields::VectorField const B_cp = m_fields.get_alldirs(Bfield_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], B_cp, + Idx.Bx, Idx.By, Idx.Bz, m_fill_guards_fields); + } } } // Damp the fields in the guard cells for (int lev = 0; lev <= finest_level; ++lev) { - DampFieldsInGuards(lev, E_fp[lev], B_fp[lev]); + if (m_fields.has_vector(Efield_fp_string, lev) && m_fields.has_vector(Bfield_fp_string, lev)) { + ablastr::fields::VectorField const E_fp = m_fields.get_alldirs(Efield_fp_string, lev); + ablastr::fields::VectorField const B_fp = m_fields.get_alldirs(Bfield_fp_string, lev); + DampFieldsInGuards(lev, E_fp, B_fp); + } } } @@ -280,10 +310,12 @@ WarpX::PSATDBackwardTransformG () } void WarpX::PSATDForwardTransformJ ( - const ablastr::fields::MultiLevelVectorField& J_fp, - const ablastr::fields::MultiLevelVectorField& J_cp, + std::string const & J_fp_string, + std::string const & J_cp_string, const bool apply_kspace_filter) { + if (!m_fields.has_vector(J_fp_string, 0)) { return; } + SpectralFieldIndex Idx; int idx_jx, idx_jy, idx_jz; @@ -295,7 +327,10 @@ void WarpX::PSATDForwardTransformJ ( idx_jy = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jy_new) : static_cast(Idx.Jy_mid); idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); - ForwardTransformVect(lev, *spectral_solver_fp[lev], J_fp[lev], idx_jx, idx_jy, idx_jz); + if (m_fields.has_vector(J_fp_string, lev)) { + ablastr::fields::VectorField J_fp = m_fields.get_alldirs(J_fp_string, lev); + ForwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, idx_jx, idx_jy, idx_jz); + } if (spectral_solver_cp[lev]) { @@ -305,7 +340,10 @@ void WarpX::PSATDForwardTransformJ ( idx_jy = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jy_new) : static_cast(Idx.Jy_mid); idx_jz = (J_in_time == JInTime::Linear) ? static_cast(Idx.Jz_new) : static_cast(Idx.Jz_mid); - ForwardTransformVect(lev, *spectral_solver_cp[lev], J_cp[lev], idx_jx, idx_jy, idx_jz); + if (m_fields.has_vector(J_cp_string, lev)) { + ablastr::fields::VectorField J_cp = m_fields.get_alldirs(J_cp_string, lev); + ForwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, idx_jx, idx_jy, idx_jz); + } } } @@ -341,9 +379,11 @@ void WarpX::PSATDForwardTransformJ ( } void WarpX::PSATDBackwardTransformJ ( - ablastr::fields::MultiLevelVectorField const & J_fp, - ablastr::fields::MultiLevelVectorField const & J_cp) + std::string const & J_fp_string, + std::string const & J_cp_string) { + if (!m_fields.has_vector(J_fp_string, 0)) { return; } + SpectralFieldIndex Idx; int idx_jx, idx_jy, idx_jz; @@ -357,8 +397,11 @@ void WarpX::PSATDBackwardTransformJ ( idx_jy = static_cast(Idx.Jy_mid); idx_jz = static_cast(Idx.Jz_mid); - BackwardTransformVect(lev, *spectral_solver_fp[lev], J_fp[lev], - idx_jx, idx_jy, idx_jz, m_fill_guards_current); + if (m_fields.has_vector(J_fp_string, lev)) { + ablastr::fields::VectorField J_fp = m_fields.get_alldirs(J_fp_string, lev); + BackwardTransformVect(lev, *spectral_solver_fp[lev], J_fp, + idx_jx, idx_jy, idx_jz, m_fill_guards_current); + } if (spectral_solver_cp[lev]) { @@ -370,26 +413,35 @@ void WarpX::PSATDBackwardTransformJ ( idx_jy = static_cast(Idx.Jy_mid); idx_jz = static_cast(Idx.Jz_mid); - BackwardTransformVect(lev, *spectral_solver_cp[lev], J_cp[lev], - idx_jx, idx_jy, idx_jz, m_fill_guards_current); + if (m_fields.has_vector(J_cp_string, lev)) { + ablastr::fields::VectorField J_cp = m_fields.get_alldirs(J_cp_string, lev); + BackwardTransformVect(lev, *spectral_solver_cp[lev], J_cp, + idx_jx, idx_jy, idx_jz, m_fill_guards_current); + } } } } void WarpX::PSATDForwardTransformRho ( - ablastr::fields::MultiLevelScalarField const & charge_fp, - ablastr::fields::MultiLevelScalarField const & charge_cp, + std::string const & charge_fp_string, + std::string const & charge_cp_string, const int icomp, const int dcomp, const bool apply_kspace_filter) { - if (charge_fp[0] == nullptr) { return; } + if (!m_fields.has(charge_fp_string, 0)) { return; } for (int lev = 0; lev <= finest_level; ++lev) { - if (charge_fp[lev]) { spectral_solver_fp[lev]->ForwardTransform(lev, *charge_fp[lev], dcomp, icomp); } + if (m_fields.has(charge_fp_string, lev)) { + amrex::MultiFab const & charge_fp = *m_fields.get(charge_fp_string, lev); + spectral_solver_fp[lev]->ForwardTransform(lev, charge_fp, dcomp, icomp); + } if (spectral_solver_cp[lev]) { - if (charge_cp[lev]) { spectral_solver_cp[lev]->ForwardTransform(lev, *charge_cp[lev], dcomp, icomp); } + if (m_fields.has(charge_cp_string, lev)) { + amrex::MultiFab const & charge_cp = *m_fields.get(charge_cp_string, lev); + spectral_solver_cp[lev]->ForwardTransform(lev, charge_cp, dcomp, icomp); + } } } @@ -669,53 +721,56 @@ WarpX::PushPSATD () const int rho_old = spectral_solver_fp[0]->m_spectral_index.rho_old; const int rho_new = spectral_solver_fp[0]->m_spectral_index.rho_new; - const ablastr::fields::MultiLevelScalarField rho_fp = m_fields.get_mr_levels("rho_fp", finest_level); - const ablastr::fields::MultiLevelScalarField rho_cp = m_fields.get_mr_levels("rho_fp", finest_level); + + std::string const rho_fp_string = "rho_fp"; + std::string const rho_cp_string = "rho_cp"; + const ablastr::fields::MultiLevelVectorField current_fp = m_fields.get_mr_levels_alldirs("current_fp", finest_level); - const ablastr::fields::MultiLevelVectorField current_cp = m_fields.get_mr_levels_alldirs("current_cp", finest_level); - const ablastr::fields::MultiLevelVectorField current_buf = m_fields.get_mr_levels_alldirs("current_buf", finest_level); + std::string current_fp_string = "current_fp"; + std::string const current_cp_string = "current_cp"; if (fft_periodic_single_box) { if (current_correction) { // FFT of J and rho - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); // Correct J in k-space PSATDCurrentCorrection(); // Inverse FFT of J - PSATDBackwardTransformJ(current_fp, current_cp); + PSATDBackwardTransformJ(current_fp_string, current_cp_string); } else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // FFT of D and rho (if used) // TODO Replace current_cp with current_cp_vay once Vay deposition is implemented with MR - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + current_fp_string = "current_fp_vay"; + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); // Compute J from D in k-space PSATDVayDeposition(); // Inverse FFT of J, subtract cumulative sums of D - PSATDBackwardTransformJ(current_fp, current_cp); + current_fp_string = "current_fp"; + PSATDBackwardTransformJ(current_fp_string, current_cp_string); // TODO Cumulative sums need to be fixed with periodic single box PSATDSubtractCurrentPartialSumsAvg(); // FFT of J after subtraction of cumulative sums - PSATDForwardTransformJ(current_fp, current_cp); + PSATDForwardTransformJ(current_fp_string, current_cp_string); } else // no current correction, no Vay deposition { // FFT of J and rho (if used) - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); } } else // no periodic single box @@ -727,20 +782,20 @@ WarpX::PushPSATD () // In RZ geometry, do not apply filtering here, since it is // applied in the subsequent calls to these functions (below) const bool apply_kspace_filter = false; - PSATDForwardTransformJ(current_fp, current_cp, apply_kspace_filter); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old, apply_kspace_filter); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new, apply_kspace_filter); + PSATDForwardTransformJ(current_fp_string, current_cp_string, apply_kspace_filter); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old, apply_kspace_filter); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new, apply_kspace_filter); #else - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); #endif // Correct J in k-space PSATDCurrentCorrection(); // Inverse FFT of J - PSATDBackwardTransformJ(current_fp, current_cp); + PSATDBackwardTransformJ(current_fp_string, current_cp_string); // Synchronize J and rho SyncCurrent("current_fp"); @@ -749,14 +804,15 @@ WarpX::PushPSATD () else if (current_deposition_algo == CurrentDepositionAlgo::Vay) { // FFT of D - PSATDForwardTransformJ( - m_fields.get_mr_levels_alldirs("current_fp_vay", finest_level), current_cp); + current_fp_string = "current_fp_vay"; + PSATDForwardTransformJ(current_fp_string, current_cp_string); // Compute J from D in k-space PSATDVayDeposition(); // Inverse FFT of J, subtract cumulative sums of D - PSATDBackwardTransformJ(current_fp, current_cp); + current_fp_string = "current_fp"; + PSATDBackwardTransformJ(current_fp_string, current_cp_string); PSATDSubtractCurrentPartialSumsAvg(); // Synchronize J and rho (if used). @@ -770,18 +826,13 @@ WarpX::PushPSATD () } // FFT of J and rho (if used) - PSATDForwardTransformJ(current_fp, current_cp); - PSATDForwardTransformRho(rho_fp, rho_cp, 0, rho_old); - PSATDForwardTransformRho(rho_fp, rho_cp, 1, rho_new); + PSATDForwardTransformJ(current_fp_string, current_cp_string); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 0, rho_old); + PSATDForwardTransformRho(rho_fp_string, rho_cp_string, 1, rho_new); } - auto Efield_fp = m_fields.get_mr_levels_alldirs("Efield_fp", finest_level); - auto Bfield_fp = m_fields.get_mr_levels_alldirs("Bfield_fp", finest_level); - auto Efield_cp = m_fields.get_mr_levels_alldirs("Efield_cp", finest_level); - auto Bfield_cp = m_fields.get_mr_levels_alldirs("Bfield_cp", finest_level); - // FFT of E and B - PSATDForwardTransformEB(Efield_fp, Bfield_fp, Efield_cp, Bfield_cp ); + PSATDForwardTransformEB(); #ifdef WARPX_DIM_RZ if (pml_rz[0]) { pml_rz[0]->PushPSATD(0); } @@ -795,10 +846,7 @@ WarpX::PushPSATD () PSATDPushSpectralFields(); // Inverse FFT of E, B, F, and G - PSATDBackwardTransformEB( m_fields.get_mr_levels_alldirs("Efield_fp",finest_level), - Bfield_fp, - m_fields.get_mr_levels_alldirs("Efield_cp",finest_level), - Bfield_cp); + PSATDBackwardTransformEB(); if (WarpX::fft_do_time_averaging) { auto Efield_avg_fp = m_fields.get_mr_levels_alldirs("Efield_avg_fp", finest_level); auto Bfield_avg_fp = m_fields.get_mr_levels_alldirs("Bfield_avg_fp", finest_level); diff --git a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp index 69ca47d9ad7..13752458b9e 100644 --- a/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp +++ b/Source/FieldSolver/WarpXPushFieldsHybridPIC.cpp @@ -22,7 +22,6 @@ using namespace amrex; void WarpX::HybridPICEvolveFields () { using ablastr::fields::Direction; - using ablastr::fields::va2vm; WARPX_PROFILE("WarpX::HybridPICEvolveFields()"); @@ -192,8 +191,6 @@ void WarpX::HybridPICEvolveFields () void WarpX::HybridPICDepositInitialRhoAndJ () { - using ablastr::fields::va2vm; - ablastr::fields::MultiLevelScalarField rho_fp_temp = m_fields.get_mr_levels("hybrid_rho_fp_temp", finest_level); ablastr::fields::MultiLevelVectorField current_fp_temp = m_fields.get_mr_levels_alldirs("hybrid_current_fp_temp", finest_level); mypc->DepositCharge(rho_fp_temp, 0._rt); diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 5fa213cd8c1..34a2fb18095 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -20,6 +20,7 @@ #include "WarpXSumGuardCells.H" #include "Particles/MultiParticleContainer.H" +#include #include #include diff --git a/Source/Python/MultiFabRegister.cpp b/Source/Python/MultiFabRegister.cpp index da0712a8909..f345f667743 100644 --- a/Source/Python/MultiFabRegister.cpp +++ b/Source/Python/MultiFabRegister.cpp @@ -99,7 +99,8 @@ void init_MultiFabRegister (py::module & m) .def("alloc_like", &MultiFabRegister::alloc_like, py::arg("other_name"), - py::arg("other_level") + py::arg("other_level"), + py::arg("level") ) .def("has", diff --git a/Source/WarpX.H b/Source/WarpX.H index d9626fec52a..9525acc8bc5 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1650,40 +1650,14 @@ private: /** * \brief Forward FFT of E,B on all mesh refinement levels - * - * \param E_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch electric field to be transformed - * \param B_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch magnetic field to be transformed - * \param E_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch electric field to be transformed - * \param B_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch magnetic field to be transformed - */ - void PSATDForwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp); + */ + void PSATDForwardTransformEB (); /** * \brief Backward FFT of E,B on all mesh refinement levels, * with field damping in the guard cells (if needed) - * - * \param E_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch electric field to be transformed - * \param B_fp Vector of three-dimensional arrays (for each level) - * storing the fine patch magnetic field to be transformed - * \param E_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch electric field to be transformed - * \param B_cp Vector of three-dimensional arrays (for each level) - * storing the coarse patch magnetic field to be transformed - */ - void PSATDBackwardTransformEB ( - const ablastr::fields::MultiLevelVectorField& E_fp, - const ablastr::fields::MultiLevelVectorField& B_fp, - const ablastr::fields::MultiLevelVectorField& E_cp, - const ablastr::fields::MultiLevelVectorField& B_cp); + */ + void PSATDBackwardTransformEB (); /** * \brief Backward FFT of averaged E,B on all mesh refinement levels @@ -1715,8 +1689,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformJ ( - ablastr::fields::MultiLevelVectorField const& J_fp, - ablastr::fields::MultiLevelVectorField const& J_cp, + std::string const & J_fp_string, + std::string const & J_cp_string, bool apply_kspace_filter=true); /** @@ -1728,8 +1702,8 @@ private: * storing the coarse patch current to be transformed */ void PSATDBackwardTransformJ ( - ablastr::fields::MultiLevelVectorField const & J_fp, - ablastr::fields::MultiLevelVectorField const & J_cp); + std::string const & J_fp_string, + std::string const & J_cp_string); /** * \brief Forward FFT of rho on all mesh refinement levels, @@ -1743,8 +1717,8 @@ private: * (only used in RZ geometry to avoid double filtering) */ void PSATDForwardTransformRho ( - ablastr::fields::MultiLevelScalarField const & charge_fp, - ablastr::fields::MultiLevelScalarField const & charge_cp, + std::string const & charge_fp_string, + std::string const & charge_cp_string, int icomp, int dcomp, bool apply_kspace_filter=true); /** diff --git a/Source/ablastr/fields/MultiFabRegister.H b/Source/ablastr/fields/MultiFabRegister.H index 7599aa68d07..7237fd6a575 100644 --- a/Source/ablastr/fields/MultiFabRegister.H +++ b/Source/ablastr/fields/MultiFabRegister.H @@ -241,27 +241,26 @@ namespace ablastr::fields std::optional initial_value = std::nullopt ); - /** title + /** Allocate a new MultiFab (field) with the same size and distribution as another. * - * body body - * body + * @todo this is reserved for future use and not yet implemented. + * @todo we will also add an overload with dir. * - * @param other_name ... - * @param other_level ... + * @param new_name new name + * @param other_name the other multifab + * @param level the MR level to represent */ void alloc_like ( - const std::string& other_name, - int other_level + std::string const & new_name, + std::string const & other_name, + int level ); - /** title - * - * body body - * body + /** Check if a scalar MultiFab (field) is registered. * - * @param name ... - * @param level ... + * @param name the name to check if registered + * @param level the MR level to check * @return true if contained, otherwise false */ [[nodiscard]] bool @@ -270,14 +269,11 @@ namespace ablastr::fields int level ) const; - /** title + /** Check if a MultiFab that is part of a vector/tensor field is registered. * - * body body - * body - * - * @param name ... - * @param dir ... - * @param level ... + * @param name the name to check if registered + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to check * @return true if contained, otherwise false */ [[nodiscard]] bool @@ -287,46 +283,55 @@ namespace ablastr::fields int level ) const; - /** title + /** Check if a MultiFab vector field is registered. * - * body body - * body + * @param name the name to check if registered + * @param level the MR level to check + * @return true if contained, otherwise false + */ + [[nodiscard]] bool + has_vector ( + std::string const & name, + int level + ) const; + + /** Return a scalar MultiFab (field). * - * @param name ... - * @param level ... - * @return ... + * This throws a runtime error if the requested field is not present. + * + * @param name the name to check if registered + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ - amrex::MultiFab* + [[nodiscard]] amrex::MultiFab* get ( std::string name, int level ); - /** title + /** Return a MultiFab that is part of a vector/tensor field. * - * body body - * body + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param dir ... - * @param level ... - * @return ... + * @param name the name to check if registered + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ - amrex::MultiFab* + [[nodiscard]] amrex::MultiFab* get ( std::string name, Direction dir, int level ); - /** title + /** Return a scalar MultiFab (field). * - * body body - * body + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param level ... - * @return ... + * @param name the name to check if registered + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ [[nodiscard]] amrex::MultiFab const * get ( @@ -334,15 +339,14 @@ namespace ablastr::fields int level ) const; - /** title + /** Return a MultiFab that is part of a vector/tensor field. * - * body body - * body + * This throws a runtime error if the requested field is not present. * - * @param name ... - * @param dir ... - * @param level ... - * @return ... + * @param name the name to check if registered + * @param dir the field component for vector fields ("direction" of the unit vector) + * @param level the MR level to check + * @return a non-owning pointer to the MultiFab (field) */ [[nodiscard]] amrex::MultiFab const * get ( @@ -359,7 +363,7 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ - MultiLevelScalarField + [[nodiscard]] MultiLevelScalarField get_mr_levels ( const std::string& name, int finest_level @@ -378,7 +382,7 @@ namespace ablastr::fields * @param level ... * @return ... */ - VectorField + [[nodiscard]] VectorField get_alldirs ( const std::string&, int level @@ -398,7 +402,7 @@ namespace ablastr::fields * @param finest_level ... * @return ... */ - MultiLevelVectorField + [[nodiscard]] MultiLevelVectorField get_mr_levels_alldirs ( const std::string& name, int finest_level @@ -508,11 +512,11 @@ namespace ablastr::fields private: amrex::MultiFab * internal_get ( - const std::string& key + const std::string& key ); [[nodiscard]] amrex::MultiFab const * internal_get ( - const std::string& key + const std::string& key ) const; /** data storage: ownership and lifetime control */ @@ -531,26 +535,6 @@ namespace ablastr::fields const std::array< std::unique_ptr, 3 > & old_vectorfield ); - /** TODO: temporary, remove me - * - * Convert get_mr_levels_alldirs type to legacy amrex::Vector, 3 > >. - * - * @return - */ - MultiLevelVectorField - va2vm ( - const amrex::Vector, 3 > >& old_vector_on_levels - ); - - /** TODO: temporary, remove me - * - * @return - */ - MultiLevelScalarField - va2vm ( - const amrex::Vector >& old_scalar_on_levels - ); - } // namespace ablastr::fields #endif // ABLASTR_FIELDS_MF_REGISTER_H diff --git a/Source/ablastr/fields/MultiFabRegister.cpp b/Source/ablastr/fields/MultiFabRegister.cpp index f5e19c19db1..4abed6b372d 100644 --- a/Source/ablastr/fields/MultiFabRegister.cpp +++ b/Source/ablastr/fields/MultiFabRegister.cpp @@ -119,8 +119,9 @@ namespace ablastr::fields void MultiFabRegister::alloc_like ( - const std::string& /* other_name */, - int /* other_level */ + std::string const & /* new_name */, + std::string const & /* other_name */, + int /* level */ ) { // other_name = mf_name(other_name, other_level); @@ -328,6 +329,23 @@ namespace ablastr::fields return m_mf_register.count(name) > 0; } + bool + MultiFabRegister::has_vector ( + std::string const & name, + int level + ) const + { + const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; + + unsigned long count = 0; + for (const Direction& dir : all_dirs) { + std::string const internal_name = mf_name(name, dir, level); + count += m_mf_register.count(internal_name); + } + + return count == 3; + } + amrex::MultiFab* MultiFabRegister::internal_get ( const std::string& key @@ -622,49 +640,4 @@ namespace ablastr::fields } return field_on_level; } - - MultiLevelVectorField - va2vm ( - const amrex::Vector, 3 > >& old_vector_on_levels - ) - { - auto const finest_level = static_cast(old_vector_on_levels.size() - 1u); - - MultiLevelVectorField field_on_level; - field_on_level.reserve(finest_level+1); - - const std::vector all_dirs = {Direction{0}, Direction{1}, Direction{2}}; - - for (int lvl = 0; lvl <= finest_level; lvl++) - { - // insert a new level - field_on_level.push_back(VectorField{}); - - // insert components - for (const auto dir : {0, 1, 2}) - { - field_on_level[lvl][Direction{dir}] = old_vector_on_levels[lvl][dir].get(); - } - } - return field_on_level; - } - - MultiLevelScalarField - va2vm ( - const amrex::Vector >& old_scalar_on_levels - ) - { - auto const finest_level = static_cast(old_scalar_on_levels.size() - 1u); - - MultiLevelScalarField field_on_level; - field_on_level.reserve(finest_level+1); - - for (int lvl = 0; lvl <= finest_level; lvl++) - { - // insert a scalar field on a level - field_on_level.push_back(ScalarField{old_scalar_on_levels[lvl].get()}); - - } - return field_on_level; - } } // namespace ablastr::fields diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index e3682b69ff5..72642b575e8 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -279,7 +279,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "028638564f7be0694b9898f8d4088cdbf9a6f9f5" +set(WarpX_amrex_branch "3734079379bb6b2a3850d197241f6b2c3b3bfa7d" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)")