Skip to content

Commit 6969114

Browse files
dpgrotedpgrote
authored
[WIP]Add diagnostic file output for the implicit solvers (#5464)
This PR adds the option to write out information about the implicit solvers to a file during the simulation. Note that this PR uses `std::filesystem` directly. This does not work easily for older gcc compilers, from before version 9.1. As a side effect, this PR removes builds using the older gcc and bumps up the comment in the dependencies documentation. This should not be an issue since all of the installs on HPC systems use newer versions.
1 parent 85482ff commit 6969114

File tree

11 files changed

+114
-49
lines changed

11 files changed

+114
-49
lines changed

‎Docs/source/install/dependencies.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ Dependencies
66
WarpX depends on the following popular third party software.
77
Please see installation instructions below.
88

9-
- a mature `C++17 <https://en.wikipedia.org/wiki/C%2B%2B17>`__ compiler, e.g., GCC 8.4+, Clang 7, NVCC 11.0, MSVC 19.15 or newer
9+
- a mature `C++17 <https://en.wikipedia.org/wiki/C%2B%2B17>`__ compiler, e.g., GCC 9.1+, Clang 7, NVCC 11.0, MSVC 19.15 or newer
1010
- `CMake 3.24.0+ <https://cmake.org>`__
1111
- `Git 2.18+ <https://git-scm.com>`__
1212
- `AMReX <https://amrex-codes.github.io>`__: we automatically download and compile a copy of AMReX

‎Examples/Tests/implicit/inputs_test_2d_theta_implicit_jfnk_vandb

+2
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,15 @@ implicit_evolve.particle_tolerance = 1.0e-12
5252
#picard.relative_tolerance = 0.0 #1.0e-12
5353
#picard.absolute_tolerance = 0.0 #1.0e-24
5454
#picard.require_convergence = false
55+
#picard.diagnostic_file = "diags/picard_solver.txt"
5556

5657
implicit_evolve.nonlinear_solver = "newton"
5758
newton.verbose = true
5859
newton.max_iterations = 20
5960
newton.relative_tolerance = 1.0e-12
6061
newton.absolute_tolerance = 0.0
6162
newton.require_convergence = false
63+
newton.diagnostic_file = "diags/newton_solver.txt"
6264

6365
gmres.verbose_int = 2
6466
gmres.max_iterations = 1000

‎Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ void SemiImplicitEM::OneStep ( amrex::Real start_time,
8080
// Solve nonlinear system for Eg at t_{n+1/2}
8181
// Particles will be advanced to t_{n+1/2}
8282
m_E.Copy(m_Eold); // initial guess for Eg^{n+1/2}
83-
m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt );
83+
m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt, a_step );
8484

8585
// Update WarpX owned Efield_fp to t_{n+1/2}
8686
m_WarpX->SetElectricFieldAndApplyBCs( m_E, half_time );

‎Source/FieldSolver/ImplicitSolvers/StrangImplicitSpectralEM.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ void StrangImplicitSpectralEM::OneStep ( amrex::Real start_time,
8181

8282
// Solve nonlinear system for E at t_{n+1/2}
8383
// Particles will be advanced to t_{n+1/2}
84-
m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt );
84+
m_nlsolver->Solve( m_E, m_Eold, half_time, 0.5_rt*m_dt, a_step );
8585

8686
// Update WarpX owned Efield_fp and Bfield_fp to t_{n+1/2}
8787
UpdateWarpXFields( m_E, half_time );

‎Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ void ThetaImplicitEM::OneStep ( const amrex::Real start_time,
106106
// Solve nonlinear system for Eg at t_{n+theta}
107107
// Particles will be advanced to t_{n+1/2}
108108
m_E.Copy(m_Eold); // initial guess for Eg^{n+theta}
109-
m_nlsolver->Solve( m_E, m_Eold, start_time, m_theta*m_dt );
109+
m_nlsolver->Solve( m_E, m_Eold, start_time, m_theta*m_dt, a_step );
110110

111111
// Update WarpX owned Efield_fp and Bfield_fp to t_{n+theta}
112112
UpdateWarpXFields( m_E, start_time );

‎Source/NonlinearSolvers/NewtonSolver.H

+57-2
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
#include <AMReX_ParmParse.H>
1717

1818
#include <vector>
19+
#include <istream>
20+
#include <filesystem>
1921

2022
/**
2123
* \brief Newton method to solve nonlinear equation of form:
@@ -45,7 +47,8 @@ public:
4547
void Solve ( Vec& a_U,
4648
const Vec& a_b,
4749
amrex::Real a_time,
48-
amrex::Real a_dt ) const override;
50+
amrex::Real a_dt,
51+
int a_step) const override;
4952

5053
void GetSolverParams ( amrex::Real& a_rtol,
5154
amrex::Real& a_atol,
@@ -200,6 +203,34 @@ void NewtonSolver<Vec,Ops>::Define ( const Vec& a_U,
200203

201204
this->m_is_defined = true;
202205

206+
// Create diagnostic file and write header
207+
if (!this->m_diagnostic_file.empty() && amrex::ParallelDescriptor::IOProcessor()) {
208+
209+
std::filesystem::path const diagnostic_path(this->m_diagnostic_file);
210+
std::filesystem::path const diagnostic_dir = diagnostic_path.parent_path();
211+
if (!diagnostic_dir.empty()) {
212+
std::filesystem::create_directories(diagnostic_dir);
213+
}
214+
215+
std::ofstream diagnostic_file{this->m_diagnostic_file, std::ofstream::out | std::ofstream::trunc};
216+
int c = 0;
217+
diagnostic_file << "#";
218+
diagnostic_file << "[" << c++ << "]step()";
219+
diagnostic_file << " ";
220+
diagnostic_file << "[" << c++ << "]time(s)";
221+
diagnostic_file << " ";
222+
diagnostic_file << "[" << c++ << "]iters";
223+
diagnostic_file << " ";
224+
diagnostic_file << "[" << c++ << "]norm_abs";
225+
diagnostic_file << " ";
226+
diagnostic_file << "[" << c++ << "]norm_rel";
227+
diagnostic_file << " ";
228+
diagnostic_file << "[" << c++ << "]gmres_iters";
229+
diagnostic_file << " ";
230+
diagnostic_file << "[" << c++ << "]gmres_last_res";
231+
diagnostic_file << "\n";
232+
diagnostic_file.close();
233+
}
203234
}
204235

205236
template <class Vec, class Ops>
@@ -211,6 +242,7 @@ void NewtonSolver<Vec,Ops>::ParseParameters ()
211242
pp_newton.query("relative_tolerance", m_rtol);
212243
pp_newton.query("max_iterations", m_maxits);
213244
pp_newton.query("require_convergence", m_require_convergence);
245+
pp_newton.query("diagnostic_file", this->m_diagnostic_file);
214246

215247
const amrex::ParmParse pp_gmres("gmres");
216248
pp_gmres.query("verbose_int", m_gmres_verbose_int);
@@ -227,7 +259,8 @@ template <class Vec, class Ops>
227259
void NewtonSolver<Vec,Ops>::Solve ( Vec& a_U,
228260
const Vec& a_b,
229261
amrex::Real a_time,
230-
amrex::Real a_dt ) const
262+
amrex::Real a_dt,
263+
int a_step) const
231264
{
232265
BL_PROFILE("NewtonSolver::Solve()");
233266
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
@@ -248,6 +281,7 @@ void NewtonSolver<Vec,Ops>::Solve ( Vec& a_U,
248281
amrex::Real norm_rel = 0.;
249282

250283
int iter;
284+
int linear_solver_iters = 0;
251285
for (iter = 0; iter < m_maxits;) {
252286

253287
// Compute residual: F(U) = U - b - R(U)
@@ -293,6 +327,7 @@ void NewtonSolver<Vec,Ops>::Solve ( Vec& a_U,
293327
// Solve linear system for Newton step [Jac]*dU = F
294328
m_dU.zero();
295329
m_linear_solver->solve( m_dU, m_F, m_gmres_rtol, m_gmres_atol );
330+
linear_solver_iters += m_linear_solver->getNumIters();
296331

297332
// Update solution
298333
a_U -= m_dU;
@@ -321,6 +356,26 @@ void NewtonSolver<Vec,Ops>::Solve ( Vec& a_U,
321356
}
322357
}
323358

359+
if (!this->m_diagnostic_file.empty() && amrex::ParallelDescriptor::IOProcessor()) {
360+
std::ofstream diagnostic_file{this->m_diagnostic_file, std::ofstream::out | std::ofstream::app};
361+
diagnostic_file << std::setprecision(14);
362+
diagnostic_file << a_step;
363+
diagnostic_file << " ";;
364+
diagnostic_file << a_time;
365+
diagnostic_file << " ";;
366+
diagnostic_file << iter;
367+
diagnostic_file << " ";;
368+
diagnostic_file << norm_abs;
369+
diagnostic_file << " ";;
370+
diagnostic_file << norm_rel;
371+
diagnostic_file << " ";;
372+
diagnostic_file << linear_solver_iters;
373+
diagnostic_file << " ";;
374+
diagnostic_file << m_linear_solver->getResidualNorm();
375+
diagnostic_file << "\n";
376+
diagnostic_file.close();
377+
}
378+
324379
}
325380

326381
template <class Vec, class Ops>

‎Source/NonlinearSolvers/NonlinearSolver.H

+3-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,8 @@ public:
5959
virtual void Solve ( Vec&,
6060
const Vec&,
6161
amrex::Real,
62-
amrex::Real ) const = 0;
62+
amrex::Real,
63+
int) const = 0;
6364

6465
/**
6566
* \brief Print parameters used by the nonlinear solver.
@@ -81,6 +82,7 @@ protected:
8182

8283
bool m_is_defined = false;
8384
mutable bool m_verbose = true;
85+
std::string m_diagnostic_file;
8486

8587
};
8688

‎Source/NonlinearSolvers/PicardSolver.H

+48-3
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
#include "Utils/TextMsg.H"
1414

1515
#include <vector>
16+
#include <istream>
17+
#include <filesystem>
1618

1719
/**
1820
* \brief Picard fixed-point iteration method to solve nonlinear
@@ -42,7 +44,8 @@ public:
4244
void Solve ( Vec& a_U,
4345
const Vec& a_b,
4446
amrex::Real a_time,
45-
amrex::Real a_dt ) const override;
47+
amrex::Real a_dt,
48+
int a_step) const override;
4649

4750
void GetSolverParams ( amrex::Real& a_rtol,
4851
amrex::Real& a_atol,
@@ -114,6 +117,31 @@ void PicardSolver<Vec,Ops>::Define ( const Vec& a_U,
114117

115118
this->m_is_defined = true;
116119

120+
// Create diagnostic file and write header
121+
if (!this->m_diagnostic_file.empty() && amrex::ParallelDescriptor::IOProcessor()) {
122+
123+
std::filesystem::path const diagnostic_path(this->m_diagnostic_file);
124+
std::filesystem::path const diagnostic_dir = diagnostic_path.parent_path();
125+
if (!diagnostic_dir.empty()) {
126+
std::filesystem::create_directories(diagnostic_dir);
127+
}
128+
129+
std::ofstream diagnostic_file{this->m_diagnostic_file, std::ofstream::out | std::ofstream::trunc};
130+
int c = 0;
131+
diagnostic_file << "#";
132+
diagnostic_file << "[" << c++ << "]step()";
133+
diagnostic_file << " ";
134+
diagnostic_file << "[" << c++ << "]time(s)";
135+
diagnostic_file << " ";
136+
diagnostic_file << "[" << c++ << "]iters";
137+
diagnostic_file << " ";
138+
diagnostic_file << "[" << c++ << "]norm_abs";
139+
diagnostic_file << " ";
140+
diagnostic_file << "[" << c++ << "]norm_rel";
141+
diagnostic_file << "\n";
142+
diagnostic_file.close();
143+
}
144+
117145
}
118146

119147
template <class Vec, class Ops>
@@ -125,14 +153,15 @@ void PicardSolver<Vec,Ops>::ParseParameters ()
125153
pp_picard.query("relative_tolerance", m_rtol);
126154
pp_picard.query("max_iterations", m_maxits);
127155
pp_picard.query("require_convergence", m_require_convergence);
128-
156+
pp_picard.query("diagnostic_file", this->m_diagnostic_file);
129157
}
130158

131159
template <class Vec, class Ops>
132160
void PicardSolver<Vec,Ops>::Solve ( Vec& a_U,
133161
const Vec& a_b,
134162
amrex::Real a_time,
135-
amrex::Real a_dt ) const
163+
amrex::Real a_dt,
164+
int a_step) const
136165
{
137166
BL_PROFILE("PicardSolver::Solve()");
138167
WARPX_ALWAYS_ASSERT_WITH_MESSAGE(
@@ -213,6 +242,22 @@ void PicardSolver<Vec,Ops>::Solve ( Vec& a_U,
213242
}
214243
}
215244

245+
if (!this->m_diagnostic_file.empty() && amrex::ParallelDescriptor::IOProcessor()) {
246+
std::ofstream diagnostic_file{this->m_diagnostic_file, std::ofstream::out | std::ofstream::app};
247+
diagnostic_file << std::setprecision(14);
248+
diagnostic_file << a_step;
249+
diagnostic_file << " ";
250+
diagnostic_file << a_time;
251+
diagnostic_file << " ";
252+
diagnostic_file << iter;
253+
diagnostic_file << " ";
254+
diagnostic_file << norm_abs;
255+
diagnostic_file << " ";
256+
diagnostic_file << norm_rel;
257+
diagnostic_file << "\n";
258+
diagnostic_file.close();
259+
}
260+
216261
}
217262

218263
#endif

‎Tools/machines/desktop/spack-ubuntu-cuda.yaml

-13
Original file line numberDiff line numberDiff line change
@@ -120,19 +120,6 @@ spack:
120120
modules: []
121121
environment: {}
122122
extra_rpaths: []
123-
- compiler:
124-
spec: gcc@8.3.0
125-
paths:
126-
cc: /usr/bin/gcc
127-
cxx: /usr/bin/g++
128-
f77: /usr/bin/gfortran
129-
fc: /usr/bin/gfortran
130-
flags: {}
131-
operating_system: debian10
132-
target: x86_64
133-
modules: []
134-
environment: {}
135-
extra_rpaths: []
136123

137124
# binary caches
138125
mirrors:

‎Tools/machines/desktop/spack-ubuntu-openmp.yaml

-13
Original file line numberDiff line numberDiff line change
@@ -110,19 +110,6 @@ spack:
110110
modules: []
111111
environment: {}
112112
extra_rpaths: []
113-
- compiler:
114-
spec: gcc@8.3.0
115-
paths:
116-
cc: /usr/bin/gcc
117-
cxx: /usr/bin/g++
118-
f77: /usr/bin/gfortran
119-
fc: /usr/bin/gfortran
120-
flags: {}
121-
operating_system: debian10
122-
target: x86_64
123-
modules: []
124-
environment: {}
125-
extra_rpaths: []
126113

127114
# binary caches
128115
mirrors:

‎Tools/machines/desktop/spack-ubuntu-rocm.yaml

-13
Original file line numberDiff line numberDiff line change
@@ -114,19 +114,6 @@ spack:
114114
modules: []
115115
environment: {}
116116
extra_rpaths: []
117-
- compiler:
118-
spec: gcc@8.3.0
119-
paths:
120-
cc: /usr/bin/gcc
121-
cxx: /usr/bin/g++
122-
f77: /usr/bin/gfortran
123-
fc: /usr/bin/gfortran
124-
flags: {}
125-
operating_system: debian10
126-
target: x86_64
127-
modules: []
128-
environment: {}
129-
extra_rpaths: []
130117

131118
# binary caches
132119
mirrors:

0 commit comments

Comments
 (0)