From 13d25cfc3614de978bfd4b7f273d1f13cb820878 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 18 Jan 2024 22:29:43 -0500 Subject: [PATCH 1/4] Add `POSTAMBLE_CMD` into preamble postamble (#2235) This PR adds the ability to run a command at the end of jobs via the preamble's postamble function. A new command can be set via `POSTAMBLE_CMD` and will be invoked at the end of jobs. Users can add the command to the top of an env file to have every job run it or it can be placed within a job if-block in the env file to run for just that job. Resolves #1145 --- env/HERA.env | 2 ++ ush/preamble.sh | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/env/HERA.env b/env/HERA.env index 284d54820e..fb156645f8 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -18,6 +18,8 @@ export npe_node_max=40 export launcher="srun -l --export=ALL" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" +#export POSTAMBLE_CMD='report-mem' + # Configure MPI environment #export I_MPI_ADJUST_ALLREDUCE=5 #export MPI_BUFS_PER_PROC=2048 diff --git a/ush/preamble.sh b/ush/preamble.sh index be64684aa8..d10d7541c8 100644 --- a/ush/preamble.sh +++ b/ush/preamble.sh @@ -16,6 +16,8 @@ # TRACE (YES/NO): Whether to echo every command (set -x) [default: "YES"] # STRICT (YES/NO): Whether to exit immediately on error or undefined variable # (set -eu) [default: "YES"] +# POSTAMBLE_CMD (empty/set): A command to run at the end of the job +# [default: empty] # ####### set +x @@ -70,6 +72,24 @@ postamble() { start_time="${2}" rc="${3}" + # Execute postamble command + # + # Commands can be added to the postamble by appending them to $POSTAMBLE_CMD: + # POSTAMBLE_CMD="new_thing; ${POSTAMBLE_CMD:-}" # (before existing commands) + # POSTAMBLE_CMD="${POSTAMBLE_CMD:-}; new_thing" # (after existing commands) + # + # Always use this form so previous POSTAMBLE_CMD are not overwritten. This should + # only be used for commands that execute conditionally (i.e. on certain machines + # or jobs). Global changes should just be added to this function. + # These commands will be called when EACH SCRIPT terminates, so be mindful. Please + # consult with global-workflow CMs about permanent changes to $POSTAMBLE_CMD or + # this postamble function. + # + + if [[ -v 'POSTAMBLE_CMD' ]]; then + ${POSTAMBLE_CMD} + fi + # Calculate the elapsed time end_time=$(date +%s) end_time_human=$(date -d@"${end_time}" -u +%H:%M:%S) From 491928712ae1dabac62750450c9cad8f538c2a3e Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 18 Jan 2024 22:31:27 -0500 Subject: [PATCH 2/4] GOCART Emission updates for GEFSv13 and GDAS (#2201) This PR addresses several things needed for more recent simulations using GOCART2G as well as preparing for the GEFSv13 30 year run. The main updates are: - Update CEDS to use monthly emission files instead of daily. This will drastically reduce the number of files needed. No science change - Update biogenic MEGAN inputs to use a climatology rather than the offline biogenic previously used. This is needed to deal with simulations where the current dataset is not available. - Update volcanic SO4 emissions to use degassing emissions only. This is to support the need for more recent simulations where data is not available. --- parm/ufs/gocart/ExtData.other | 36 ++++++++++++++--------------- parm/ufs/gocart/SU2G_instance_SU.rc | 4 ++-- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 789576305e..c58b3e9ae8 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -54,20 +54,20 @@ pSO2_OCS NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # --------------- # # VOCs - OFFLINE MEGAN BIOG -OC_ISOPRENE NA N Y %y4-%m2-%d2t12:00:00 none none isoprene ExtData/nexus/MEGAN_OFFLINE_BVOC/v2019-10/%y4/MEGAN.OFFLINE.BIOVOC.%y4.emis.%y4%m2%d2.nc -OC_LIMO NA N Y %y4-%m2-%d2t12:00:00 none none limo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2019-10/%y4/MEGAN.OFFLINE.BIOVOC.%y4.emis.%y4%m2%d2.nc -OC_MTPA NA N Y %y4-%m2-%d2t12:00:00 none none mtpa ExtData/nexus/MEGAN_OFFLINE_BVOC/v2019-10/%y4/MEGAN.OFFLINE.BIOVOC.%y4.emis.%y4%m2%d2.nc -OC_MTPO NA N Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2019-10/%y4/MEGAN.OFFLINE.BIOVOC.%y4.emis.%y4%m2%d2.nc +OC_ISOPRENE NA N Y %y4-%m2-%d2t12:00:00 none none isoprene ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc +OC_LIMO NA N Y %y4-%m2-%d2t12:00:00 none none limo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc +OC_MTPA NA N Y %y4-%m2-%d2t12:00:00 none none mtpa ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc +OC_MTPO NA N Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc # Biofuel Source -- Included in AeroCom anthropogenic emissions OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null @@ -88,11 +88,11 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null @@ -133,11 +133,11 @@ BRC_AVIATION_CRS NA Y Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null pSOA_BIOB_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # # ======= Nitrate Sources ======== -# EMI_NH3_AG 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_ag ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +# EMI_NH3_AG 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_ag ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EMI_NH3_EN 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_en /dev/null -# EMI_NH3_IN 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_in ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc -# EMI_NH3_RE 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_re ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc -# EMI_NH3_TR 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_tr ExtData/nexus/CEDS/v2019/%y4/CEDS.2019.emis.%y4%m2%d2.nc +# EMI_NH3_IN 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_in ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +# EMI_NH3_RE 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_re ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +# EMI_NH3_TR 'kg m-2 s-1' N Y %y4-%m2-%d2T12:00:00 none none NH3_tr ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EMI_NH3_OC 'kg m-2 s-1' Y Y %y4-%m2-%d2T12:00:00 none none emiss_ocn ExtData/PIESA/sfc/GEIA.emis_NH3.ocean.x576_y361.t12.20080715_12z.nc4 # # -------------------------------------------------------------- diff --git a/parm/ufs/gocart/SU2G_instance_SU.rc b/parm/ufs/gocart/SU2G_instance_SU.rc index e365827760..79484b3068 100644 --- a/parm/ufs/gocart/SU2G_instance_SU.rc +++ b/parm/ufs/gocart/SU2G_instance_SU.rc @@ -7,8 +7,8 @@ aerosol_monochromatic_optics_file: ExtData/monochromatic/optics_SU.v1_3.nc nbins: 4 -# Volcanic pointwise sources -volcano_srcfilen: ExtData/nexus/VOLCANO/v2021-09/%y4/%m2/so2_volcanic_emissions_Carns.%y4%m2%d2.rc +# Volcanic pointwise sources | degassing only | replace with path to historical volcanic emissions for scientific runs +volcano_srcfilen: ExtData/nexus/VOLCANO/v2021-09/so2_volcanic_emissions_CARN_v202005.degassing_only.rc # Heights [m] of LTO, CDS and CRS aviation emissions layers aviation_vertical_layers: 0.0 100.0 9.0e3 10.0e3 From d4c55d1011f8b0385d25b62ac04710837ed8413e Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Fri, 19 Jan 2024 15:43:37 -0500 Subject: [PATCH 3/4] Update typing hint for WCOSS version of python (#2238) The typing hint `typing.List` was deprecated with python 3.9 in favor of using the primitive `list[str]`, but the functional version of python on WCOSS2 is <3.9, causing `setup_xml.py` to fail there. This replaces `list[str]` as a typing hint with the deprecated form until the supported version on WCOSS2 is >=3.9. --- workflow/rocoto/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 8a40e645d0..d16a009eff 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -4,6 +4,7 @@ from applications.applications import AppConfig import rocoto.rocoto as rocoto from wxflow import Template, TemplateConstants, to_timedelta +from typing import List __all__ = ['Tasks'] @@ -124,7 +125,7 @@ def _template_to_rocoto_cycstring(self, template: str, subs_dict: dict = {}) -> rocoto_conversion_dict.get) @staticmethod - def _get_forecast_hours(cdump, config) -> list[str]: + def _get_forecast_hours(cdump, config) -> List[str]: fhmin = config['FHMIN'] fhmax = config['FHMAX'] fhout = config['FHOUT'] From f4d187f4e45fe89583d18987d68a883490827104 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Mon, 22 Jan 2024 12:28:28 -0500 Subject: [PATCH 4/4] Converts obsproc to obsprep in prepoceanobs config file (#2236) Converts obsproc to obsprep in prepoceanobs config file as a compliment to mutually depended GDASApp PR NOAA-EMC/GDASApp#858 The motivations are explained in refs NOAA-EMC/GDASApp#857 --- parm/config/gfs/config.prepoceanobs | 2 +- sorc/gdas.cd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.prepoceanobs b/parm/config/gfs/config.prepoceanobs index d7c4e37bb9..56fc349ce2 100644 --- a/parm/config/gfs/config.prepoceanobs +++ b/parm/config/gfs/config.prepoceanobs @@ -7,7 +7,7 @@ echo "BEGIN: config.prepoceanobs" export OCNOBS2IODAEXEC=${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda.x export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config -export OBSPROC_YAML=@OBSPROC_YAML@ +export OBSPREP_YAML=@OBSPREP_YAML@ export OBS_LIST=@SOCA_OBS_LIST@ [[ -n "${OBS_LIST}" ]] || export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml export OBS_YAML=${OBS_LIST} diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b79c0a2c36..6044d09962 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b79c0a2c36cb632212e6991f784190310a2e0e41 +Subproject commit 6044d099626f568a1a671ea86398e2aedfaf72f4