[develop] feat: predefined grid support for smoke/dust component (#1190)

* Adds support for SRW predefined grids to the smoke/dust component. NA grids currently fail in prep tasks due to incorrect sources for ICs (need GFS not RAP/HRRR). Currently supported grids: CONUS 13km, CONUS 3km, CONUS 25km. Note that predefined grid data is currently staged on Hera.
* Smoke/dust task runs in parallel.
* Adds unit tests.
* Adds support for T1 platforms.
* Scripts refactored to use an initialize-run-finalize strategy. Scripts have been broken into modules more reminiscent of a library.

---------

Co-authored-by: Chan-hoo.Jeon <chan-hoo.jeon@noaa.gov>
Co-authored-by: Chan-hoo Jeon <Chan-hoo.Jeon@gaea65.ncrc.gov>
Co-authored-by: michael.lueken <Michael.Lueken@noaa.gov>
Co-authored-by: Chan-Hoo.Jeon-NOAA <60152248+chan-hoo@users.noreply.github.com>

.github/workflows/python_tests.yaml

@@ -24,21 +24,22 @@ jobs:
       - name: Install Micromamba and srw_app environment
         uses: mamba-org/setup-micromamba@v1
         with:
-          environment-file: environment.yml
-          environment-name: srw_app
+          environment-file: sd_environment.yml
+          environment-name: srw_sd
           init-shell: bash
           cache-downloads: true
           cache-environment: true
 
       - name: Lint the python code
         run: |
-          micromamba activate srw_app
+          micromamba activate srw_sd
           export PYTHONPATH=$(pwd)/ush
           pylint --ignore-imports=yes tests/test_python/
           pylint ush/create_*.py
           pylint ush/generate_FV3LAM_wflow.py
           pylint ush/set_fv3nml*.py
           pylint ush/update_input_nml.py
+          pylint ush/smoke_dust
 
       - name: Checkout externals
         run: |
@@ -47,15 +48,16 @@ jobs:
       - name: Run python unittests
         run: |
           # exclude test_retrieve_data that is tested in functional test
-          micromamba activate srw_app
+          micromamba activate srw_sd
           export UNIT_TEST=True
           export PYTHONPATH=$(pwd)/ush
           python -m unittest tests/test_python/*.py
+          export PYTHONPATH=${PYTHONPATH}:$(pwd)/tests
+          python -m pytest tests/test_python/test_smoke_dust
 
       - name: Run python functional tests
         run: |
-          micromamba activate srw_app
+          micromamba activate srw_sd
           export CI=true
           export PYTHONPATH=${PWD}/ush
           python3 -m unittest tests/test_python/test_retrieve_data.py
-

.gitignore

@@ -27,3 +27,4 @@ conda_loc
 
 *.swp
 __pycache__
+.idea

.pylintrc

@@ -25,7 +25,7 @@ clear-cache-post-run=no
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may
 # run arbitrary code.
-extension-pkg-allow-list=
+extension-pkg-allow-list=mpi4py,netCDF4
 
 # A comma-separated list of package or module names from where C extensions may
 # be loaded. Extensions are loading into the active Python interpreter and may

doc/TechDocs/ush/modules.rst

@@ -29,9 +29,5 @@ ush
    set_leadhrs
    set_predef_grid_params
    setup
-   smoke_dust_add_smoke
-   smoke_dust_fire_emiss_tools
-   smoke_dust_generate_fire_emissions
-   smoke_dust_hwp_tools
-   smoke_dust_interp_tools
+   smoke_dust
    update_input_nml

doc/TechDocs/ush/smoke_dust.core.regrid.rst

@@ -0,0 +1,26 @@
+smoke\_dust.core.regrid package
+===============================
+
+.. automodule:: smoke_dust.core.regrid
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Submodules
+----------
+
+smoke\_dust.core.regrid.common module
+-------------------------------------
+
+.. automodule:: smoke_dust.core.regrid.common
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+smoke\_dust.core.regrid.processor module
+----------------------------------------
+
+.. automodule:: smoke_dust.core.regrid.processor
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/smoke_dust.core.rst

@@ -0,0 +1,58 @@
+smoke\_dust.core package
+========================
+
+.. automodule:: smoke_dust.core
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   smoke_dust.core.regrid
+
+Submodules
+----------
+
+smoke\_dust.core.common module
+------------------------------
+
+.. automodule:: smoke_dust.core.common
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+smoke\_dust.core.context module
+-------------------------------
+
+.. automodule:: smoke_dust.core.context
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+smoke\_dust.core.cycle module
+-----------------------------
+
+.. automodule:: smoke_dust.core.cycle
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+smoke\_dust.core.preprocessor module
+------------------------------------
+
+.. automodule:: smoke_dust.core.preprocessor
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+smoke\_dust.core.variable module
+--------------------------------
+
+.. automodule:: smoke_dust.core.variable
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/TechDocs/ush/smoke_dust.rst

@@ -0,0 +1,34 @@
+smoke\_dust package
+===================
+
+.. automodule:: smoke_dust
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Subpackages
+-----------
+
+.. toctree::
+   :maxdepth: 4
+
+   smoke_dust.core
+
+Submodules
+----------
+
+smoke\_dust.add\_smoke module
+-----------------------------
+
+.. automodule:: smoke_dust.add_smoke
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+smoke\_dust.generate\_emissions module
+--------------------------------------
+
+.. automodule:: smoke_dust.generate_emissions
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc/UsersGuide/BuildingRunningTesting/SRW-SD.rst

@@ -1,8 +1,8 @@
 .. _srw-sd:
 
-=====================================
+==================================
 SRW Smoke & Dust (SRW-SD) Features
-=====================================
+==================================
 
 .. attention::
 
@@ -15,14 +15,14 @@ This chapter provides instructions for running a simple, example six-hour foreca
    Although this chapter is the primary documentation resource for running the SRW-SD configuration, users may need to refer to :numref:`Chapter %s <BuildSRW>` and :numref:`Chapter %s <RunSRW>` for additional information on building and running the SRW App, respectively. 
 
 Quick Start Guide (SRW-SD)
-=============================
+==========================
 
 .. attention::
 
    These instructions should work smoothly on Hera and Orion/Hercules, but users on other systems may need to make additional adjustments.
 
 Download the Code
--------------------
+-----------------
 
 Clone the |branch| branch of the authoritative SRW App repository:
 
@@ -32,7 +32,7 @@ Clone the |branch| branch of the authoritative SRW App repository:
    cd ufs-srweather-app/sorc
 
 Checkout Externals
----------------------
+------------------
 
 Users must run the ``checkout_externals`` script to collect (or "check out") the individual components of the SRW App (AQM version) from their respective GitHub repositories. 
 
@@ -41,7 +41,7 @@ Users must run the ``checkout_externals`` script to collect (or "check out") the
    ./manage_externals/checkout_externals -e Externals_smoke_dust.cfg
 
 Build the SRW App
-------------------
+-----------------
 
 .. code-block:: console
 
@@ -54,7 +54,7 @@ Building the SRW App with SRW-SD on other machines, including other :srw-wiki:`L
 If SRW-SD builds correctly, users should see the standard executables listed in :numref:`Table %s <ExecDescription>` in the ``ufs-srweather-app/exec`` directory.
 
 Load the |wflow_env| Environment
---------------------------------------------
+--------------------------------
 
 Load the workflow environment:
 
@@ -70,7 +70,7 @@ where ``<machine>`` is ``hera``, ``orion``, or ``hercules``. The workflow should
 .. _srw-sd-config:
 
 Configure an Experiment
----------------------------
+-----------------------
 
 Users will need to configure their experiment by setting parameters in the ``config.yaml`` file. To start, users can copy a default experiment setting into ``config.yaml``:
 
@@ -115,7 +115,7 @@ When using the basic ``config.smoke_dust.yaml`` experiment, the usual pre-proces
 .. _srw-sd-more-tasks:
 
 Additional SRW-SD Tasks
---------------------------
+-----------------------
 
 .. COMMENT: Add workflow diagram in the future. 
 
@@ -164,7 +164,7 @@ The Python scripts listed in :numref:`Table %s <sd-scripts>` are used to perform
      - Regridding utilities using `esmpy <https://earthsystemmodeling.org/esmpy/>`_ that interpolate data from the RAVE observational grid to the RRFS grid.
 
 Generate the Workflow
-------------------------
+---------------------
 
 Generate the workflow:
 
@@ -197,7 +197,7 @@ Users may check experiment status from the experiment directory with either of t
 .. _srw-sd-success:
 
 Experiment Output
---------------------
+-----------------
 
 The workflow run is complete when all tasks display a "SUCCEEDED" message. If everything goes smoothly, users will eventually see a workflow status table similar to the following: 
 

doc/conf.py

@@ -36,18 +36,22 @@
 
 numfig = True
 
-nitpick_ignore = [('py:class', 'obj'),
-                  ('py:class', 'yaml.dumper.Dumper'),
-                   ('py:class', 'xml.etree.ElementTree'),
-                   ('py:class', 'Basemap'),
-                   ('py:class', 'pandas.DataFrame'),
-                   ('py:class', 'numpy.ndarray'),
-                   ('py:class', 'pandas.Index'),
-                   ('py:class', 'xarray.DataArray'),
-                   ('py:class', 'netCDF4.Dataset'),
-                   ('py:class', 'ESMF.Field'),
-                   ('py:class', 'ESMF.Grid'),
-                   ]
+nitpick_ignore = [
+    ('py:class', 'obj'),
+    ('py:class', 'yaml.dumper.Dumper'),
+    ('py:class', 'xml.etree.ElementTree'),
+    ('py:class', 'Basemap'),
+    ('py:class', 'numpy.ma.MaskedArray'),
+    ('py:class', 'esmpy.Field'),
+    ('py:class', 'pandas.DataFrame'),
+    ('py:class', 'netCDF4.Variable'),
+    ('py:class', 'pandas.DatetimeIndex'),
+    ('py:class', 'netCDF4.Dataset'),
+    ('py:class', 'numpy.ndarray'),
+    ('py:class', 'pydantic.BaseModel'),
+    ('py:class', 'esmpy.Grid'),
+    ('py:class', 'esmpy.StaggerLoc'),
+]
 
 # -- General configuration ---------------------------------------------------
 
@@ -264,20 +268,24 @@ def setup(app):
 
 # -- Options for autodoc extension ---------------------------------------
 
-autodoc_mock_imports = ["f90nml",
-                        "cartopy",
-                        "mpl_toolkits.basemap",
-                        "fill_jinja_template",
-                        "matplotlib",
-                        "numpy",
-                        "uwtools",
-                        "mpl_toolkits",
-                        "metplus",
-                        "netCDF4",
-                        "pandas",
-                        "xarray",
-                        "ESMF",
-   ]
+autodoc_mock_imports = [
+    "f90nml",
+    "cartopy",
+    "mpl_toolkits.basemap",
+    "fill_jinja_template",
+    "matplotlib",
+    "numpy",
+    "uwtools",
+    "mpl_toolkits",
+    "metplus",
+    "esmpy",
+    "netCDF4",
+    "pandas",
+    "xarray",
+    "mpi4py",
+    "pydantic",
+    "typer",
+]
 
 logger = logging.getLogger(__name__)