Skip to content

Commit 5a0f854

Browse files
mjrenomjreno
authored
refactor(netcdf): utility package input adjustments (#2106)
* netcdf improved error handling and utl-ncf input adjustments * escape underscores * add autotest assert for netcdf data model * add baseline testing for utl-ncf options * remove in_record attribute from chunk params * set chunk_time param optional --------- Co-authored-by: mjreno <mjreno@IGSAAA071L01144.gs.doi.net>
1 parent 8d5b621 commit 5a0f854

16 files changed

+234
-159
lines changed

‎autotest/test_netcdf_gwe_cnd.py

+5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
xa = pytest.importorskip("xarray")
2626
xu = pytest.importorskip("xugrid")
27+
nc = pytest.importorskip("netCDF4")
2728

2829

2930
def build_models(idx, test, export, gridded_input):
@@ -57,6 +58,10 @@ def check_output(idx, test, export, gridded_input):
5758

5859
name = "gwe-" + test.name
5960

61+
# verify format of generated netcdf file
62+
with nc.Dataset(test.workspace / f"{name}.nc") as ds:
63+
assert ds.data_model == "NETCDF4"
64+
6065
if gridded_input == "netcdf":
6166
# re-run the simulation with model netcdf input
6267
input_fname = f"{name}.nc"

‎autotest/test_netcdf_gwf_disv.py

+20-1
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414

1515
xa = pytest.importorskip("xarray")
1616
xu = pytest.importorskip("xugrid")
17+
nc = pytest.importorskip("netCDF4")
1718

1819
wkt = (
1920
'PROJCS["NAD83 / UTM zone 18N", '
@@ -53,7 +54,15 @@ def build_models(idx, test, export, gridded_input):
5354
gwf.name_file.nc_mesh2d_filerecord = f"{name}.nc"
5455

5556
# netcdf config
56-
ncf = flopy.mf6.ModflowUtlncf(gwf.disv, ogc_wkt=wkt, filename=f"{name}.disv.ncf")
57+
ncf = flopy.mf6.ModflowUtlncf(
58+
gwf.disv,
59+
deflate=9,
60+
shuffle=True,
61+
chunk_time=1,
62+
chunk_face=3,
63+
wkt=wkt,
64+
filename=f"{name}.disv.ncf",
65+
)
5766

5867
# output control
5968
oc = flopy.mf6.ModflowGwfoc(
@@ -72,6 +81,16 @@ def check_output(idx, test, export, gridded_input):
7281

7382
name = test.name
7483

84+
# verify format of generated netcdf file
85+
with nc.Dataset(test.workspace / f"{name}.nc") as ds:
86+
assert ds.data_model == "NETCDF4"
87+
cmpr = ds.variables["head_l1"].filters()
88+
chnk = ds.variables["head_l1"].chunking()
89+
assert cmpr["shuffle"]
90+
assert cmpr["complevel"] == 9
91+
assert chnk == [1, 3]
92+
assert ds.variables["projection"].getncattr("wkt").lower() == wkt.lower()
93+
7594
if gridded_input == "netcdf":
7695
# re-run the simulation with model netcdf input
7796
input_fname = f"{name}.nc"

‎autotest/test_netcdf_gwf_lak_wetlakbedarea02.py

+5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
xa = pytest.importorskip("xarray")
2626
xu = pytest.importorskip("xugrid")
27+
nc = pytest.importorskip("netCDF4")
2728

2829

2930
def build_models(idx, test, export, gridded_input):
@@ -61,6 +62,10 @@ def check_output(idx, test, export, gridded_input):
6162
name = cases[idx]
6263
gwfname = "gwf-" + name
6364

65+
# verify format of generated netcdf file
66+
with nc.Dataset(test.workspace / f"{gwfname}.nc") as ds:
67+
assert ds.data_model == "NETCDF4"
68+
6469
if gridded_input == "netcdf":
6570
# re-run the simulation with model netcdf input
6671
input_fname = f"{gwfname}.nc"

‎autotest/test_netcdf_gwf_rch01.py

+5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
xa = pytest.importorskip("xarray")
2626
xu = pytest.importorskip("xugrid")
27+
nc = pytest.importorskip("netCDF4")
2728

2829

2930
def build_models(idx, test, export, gridded_input):
@@ -58,6 +59,10 @@ def check_output(idx, test, export, gridded_input):
5859

5960
name = "rch"
6061

62+
# verify format of generated netcdf file
63+
with nc.Dataset(test.workspace / f"{name}.nc") as ds:
64+
assert ds.data_model == "NETCDF4"
65+
6166
if gridded_input == "netcdf":
6267
# re-run the simulation with model netcdf input
6368
input_fname = f"{name}.nc"

‎autotest/test_netcdf_gwf_rch03.py

+5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
xa = pytest.importorskip("xarray")
2626
xu = pytest.importorskip("xugrid")
27+
nc = pytest.importorskip("netCDF4")
2728

2829

2930
def build_models(idx, test, export, gridded_input):
@@ -59,6 +60,10 @@ def check_output(idx, test, export, gridded_input):
5960

6061
name = "rch"
6162

63+
# verify format of generated netcdf file
64+
with nc.Dataset(test.workspace / f"{name}.nc") as ds:
65+
assert ds.data_model == "NETCDF4"
66+
6267
if gridded_input == "netcdf":
6368
# re-run the simulation with model netcdf input
6469
input_fname = f"{name}.nc"

‎autotest/test_netcdf_gwf_sto01.py

+39-7
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
xa = pytest.importorskip("xarray")
1414
xu = pytest.importorskip("xugrid")
15+
nc = pytest.importorskip("netCDF4")
1516

1617
htol = [None for _ in range(len(cases))]
1718

@@ -52,22 +53,53 @@ def build_models(idx, test, export, gridded_input):
5253

5354
if export == "ugrid":
5455
gwf.name_file.nc_mesh2d_filerecord = f"{name}.nc"
56+
ncf = flopy.mf6.ModflowUtlncf(
57+
gwf.dis,
58+
deflate=5,
59+
shuffle=True,
60+
chunk_time=1,
61+
chunk_face=10,
62+
wkt=wkt,
63+
filename=f"{name}.dis.ncf",
64+
)
5565
elif export == "structured":
5666
gwf.name_file.nc_structured_filerecord = f"{name}.nc"
57-
58-
# netcdf config
59-
ncf = flopy.mf6.ModflowUtlncf(
60-
gwf.dis,
61-
ogc_wkt=wkt,
62-
filename=f"{name}.dis.ncf",
63-
)
67+
ncf = flopy.mf6.ModflowUtlncf(
68+
gwf.dis,
69+
deflate=5,
70+
shuffle=True,
71+
chunk_time=1,
72+
chunk_z=1,
73+
chunk_y=5,
74+
chunk_x=5,
75+
wkt=wkt,
76+
filename=f"{name}.dis.ncf",
77+
)
6478

6579
return sim, dummy
6680

6781

6882
def check_output(idx, test, export, gridded_input):
6983
from test_gwf_sto01 import check_output as check
7084

85+
# verify format of generated netcdf file
86+
with nc.Dataset(test.workspace / "gwf_sto01.nc") as ds:
87+
assert ds.data_model == "NETCDF4"
88+
if export == "structured":
89+
cmpr = ds.variables["head"].filters()
90+
chnk = ds.variables["head"].chunking()
91+
assert chnk == [1, 1, 5, 5]
92+
assert (
93+
ds.variables["projection"].getncattr("crs_wkt").lower() == wkt.lower()
94+
)
95+
elif export == "ugrid":
96+
cmpr = ds.variables["head_l1"].filters()
97+
chnk = ds.variables["head_l1"].chunking()
98+
assert chnk == [1, 10]
99+
assert ds.variables["projection"].getncattr("wkt").lower() == wkt.lower()
100+
assert cmpr["shuffle"]
101+
assert cmpr["complevel"] == 5
102+
71103
if gridded_input == "netcdf":
72104
# re-run the simulation with model netcdf input
73105
input_fname = "gwf_sto01.nc"

‎autotest/test_netcdf_gwf_vsc03_sfr.py

+5
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424

2525
xa = pytest.importorskip("xarray")
2626
xu = pytest.importorskip("xugrid")
27+
nc = pytest.importorskip("netCDF4")
2728

2829

2930
def build_models(idx, test, export, gridded_input):
@@ -61,6 +62,10 @@ def check_output(idx, test, export, gridded_input):
6162

6263
name = "gwf-" + test.name
6364

65+
# verify format of generated netcdf file
66+
with nc.Dataset(test.workspace / f"{name}.nc") as ds:
67+
assert ds.data_model == "NETCDF4"
68+
6469
if gridded_input == "netcdf":
6570
# re-run the simulation with model netcdf input
6671
input_fname = f"{name}.nc"

‎autotest/test_netcdf_gwt_dsp01.py

+33-6
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
xa = pytest.importorskip("xarray")
1414
xu = pytest.importorskip("xugrid")
15+
nc = pytest.importorskip("netCDF4")
1516

1617

1718
def build_models(idx, test, export, gridded_input):
@@ -29,14 +30,26 @@ def build_models(idx, test, export, gridded_input):
2930

3031
if export == "ugrid":
3132
gwt.name_file.nc_mesh2d_filerecord = f"{gwtname}.nc"
33+
ncf = flopy.mf6.ModflowUtlncf(
34+
gwt.dis,
35+
deflate=3,
36+
shuffle=False,
37+
chunk_time=1,
38+
chunk_face=5,
39+
filename=f"{gwtname}.dis.ncf",
40+
)
3241
elif export == "structured":
3342
gwt.name_file.nc_structured_filerecord = f"{gwtname}.nc"
34-
35-
# netcdf config
36-
ncf = flopy.mf6.ModflowUtlncf(
37-
gwt.dis,
38-
filename=f"{gwtname}.dis.ncf",
39-
)
43+
ncf = flopy.mf6.ModflowUtlncf(
44+
gwt.dis,
45+
deflate=3,
46+
shuffle=False,
47+
chunk_time=1,
48+
chunk_z=1,
49+
chunk_y=1,
50+
chunk_x=20,
51+
filename=f"{gwtname}.dis.ncf",
52+
)
4053

4154
oc = flopy.mf6.ModflowGwtoc(
4255
gwt,
@@ -56,6 +69,20 @@ def check_output(idx, test, export, gridded_input):
5669
name = cases[idx]
5770
gwtname = "gwt_" + name
5871

72+
# verify format of generated netcdf file
73+
with nc.Dataset(test.workspace / f"{gwtname}.nc") as ds:
74+
assert ds.data_model == "NETCDF4"
75+
if export == "structured":
76+
cmpr = ds.variables["concentration"].filters()
77+
chnk = ds.variables["concentration"].chunking()
78+
assert chnk == [1, 1, 1, 20]
79+
elif export == "ugrid":
80+
cmpr = ds.variables["concentration_l1"].filters()
81+
chnk = ds.variables["concentration_l1"].chunking()
82+
assert chnk == [1, 5]
83+
assert not cmpr["shuffle"]
84+
assert cmpr["complevel"] == 3
85+
5986
if gridded_input == "netcdf":
6087
# re-run the simulation with model netcdf input
6188
input_fname = f"{gwtname}.nc"

‎autotest/test_netcdf_gwt_prudic2004t2.py

+5
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212

1313
xa = pytest.importorskip("xarray")
1414
xu = pytest.importorskip("xugrid")
15+
nc = pytest.importorskip("netCDF4")
1516

1617

1718
def build_models(idx, test, export, gridded_input):
@@ -47,6 +48,10 @@ def check_output(idx, test, export, gridded_input):
4748
name = test.name
4849
gwtname = "gwt_" + name
4950

51+
# verify format of generated netcdf file
52+
with nc.Dataset(test.workspace / f"{gwtname}.nc") as ds:
53+
assert ds.data_model == "NETCDF4"
54+
5055
if gridded_input == "netcdf":
5156
# re-run the simulation with model netcdf input
5257
input_fname = f"{gwtname}.nc"

‎doc/mf6io/mf6ivar/dfn/utl-ncf.dfn

+10-32
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
# --------------------- utl ncf options ---------------------
55

66
block options
7-
name ogc_wkt
7+
name wkt
88
type string
99
shape lenbigline
1010
reader urword
@@ -28,67 +28,45 @@ optional true
2828
longname
2929
description is the keyword used to turn on the netcdf variable shuffle filter when the deflate option is also set. The shuffle filter has the effect of storing the first byte of all of a variable's values in a chunk contiguously, followed by all the second bytes, etc. This can be an optimization for compression with certain types of data.
3030

31-
block options
32-
name chunk_record
33-
type record chunking chunk_time chunk_face chunk_z chunk_y chunk_x
34-
reader urword
35-
optional true
36-
longname netcdf export chunking record
37-
description netcdf export chunking record
38-
39-
block options
40-
name chunking
41-
type keyword
42-
in_record true
43-
reader urword
44-
optional false
45-
longname keyword when defining chunking parameters
46-
description is a keyword for providing netcdf export chunk sizes. Chunking can dramatically impact data access times and optimal chunking is highly dependent on access patterns (timeseries vs spatial, for example). It can also significantly impact compressibility of the data. A valid input record specifies chunk\_time and either chunk\_face (MESH) or chunk\_z, chunk\_y, and chunk\_x (STRUCTURED).
47-
4831
block options
4932
name chunk_time
5033
type integer
51-
in_record true
5234
reader urword
53-
optional false
35+
optional true
5436
longname chunking parameter for the time dimension
55-
description is the keyword used to provide a netcdf export time dimension chunk size.
37+
description is the keyword used to provide a data chunk size for the time dimension in a NETCDF\_MESH2D or NETCDF\_STRUCTURED output file. Must be used in combination with the the chunk\_face parameter (NETCDF\_MESH2D) or the chunk\_z, chunk\_y, and chunk\_x parameter set (NETCDF\_STRUCTURED) to have an effect.
5638

5739
block options
5840
name chunk_face
5941
type integer
60-
in_record true
6142
reader urword
6243
optional true
6344
longname chunking parameter for the mesh face dimension
64-
description is the keyword used to provide a mesh face dimension chunk size.
45+
description is the keyword used to provide a data chunk size for the face dimension in a NETCDF\_MESH2D output file. Must be used in combination with the the chunk\_time parameter to have an effect.
6546

6647
block options
6748
name chunk_z
6849
type integer
69-
in_record true
7050
reader urword
7151
optional true
7252
longname chunking parameter for structured z
73-
description is the keyword used to provide a structured grid z dimensions chunk size.
53+
description is the keyword used to provide a data chunk size for the z dimension in a NETCDF\_STRUCTURED output file. Must be used in combination with the the chunk\_time, chunk\_x and chunk\_y parameter set to have an effect.
7454

7555
block options
7656
name chunk_y
7757
type integer
78-
in_record true
7958
reader urword
8059
optional true
8160
longname chunking parameter for structured y
82-
description is the keyword used to provide a structured grid y dimensions chunk size.
61+
description is the keyword used to provide a data chunk size for the y dimension in a NETCDF\_STRUCTURED output file. Must be used in combination with the the chunk\_time, chunk\_x and chunk\_z parameter set to have an effect.
8362

8463
block options
8564
name chunk_x
8665
type integer
87-
in_record true
8866
reader urword
8967
optional true
9068
longname chunking parameter for structured x
91-
description is the keyword used to provide a structured grid x dimensions chunk size.
69+
description is the keyword used to provide a data chunk size for the x dimension in a NETCDF\_STRUCTURED output file. Must be used in combination with the the chunk\_time, chunk\_y and chunk\_z parameter set to have an effect.
9270

9371
block options
9472
name modflow6_attr_off
@@ -107,12 +85,12 @@ type integer
10785
optional true
10886
reader urword
10987
longname number of cells in layer
110-
description is the number of cells in a in a projected plane layer.
88+
description is the number of cells in a projected plane layer.
11189

11290
# --------------------- utl ncf griddata ---------------------
11391

11492
block griddata
115-
name lat
93+
name latitude
11694
type double precision
11795
shape (ncpl)
11896
optional true
@@ -121,7 +99,7 @@ longname cell center latitude
12199
description cell center latitude.
122100

123101
block griddata
124-
name lon
102+
name longitude
125103
type double precision
126104
shape (ncpl)
127105
optional true

0 commit comments

Comments
 (0)