Add the mean EnKF soil increment to the deterministic member (#3295)

Add mean EnKF soil increment to the deterministic member. Includes new regriding script and executable (in GDASApp) to perform masked bi-linear interpolation of the soil increments. Resolves [2773](#2773) --------- Co-authored-by: Cory Martin <cory.r.martin@noaa.gov> Co-authored-by: Rahul Mahajan <aerorahul@users.noreply.github.com>
NOAA-EMC · Feb 27, 2025 · 25778ff · 25778ff
1 parent 8245df7
commit 25778ff
Show file tree

Hide file tree

Showing 28 changed files with 350 additions and 37 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -180,6 +180,7 @@ syndat_qctropcy.sh @JiayiPeng-NOAA
 tropcy_relocate.sh @JiayiPeng-NOAA 
 tropcy_relocate_extrkr.sh @JiayiPeng-NOAA 
 wave_*.sh @JessicaMeixner-NOAA @sbanihash
+regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA
 
 # ush/python
 ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA

diff --git a/ci/cases/yamls/atmsoilDA_defaults_ci.yaml b/ci/cases/yamls/atmsoilDA_defaults_ci.yaml
@@ -0,0 +1,6 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+base:
+  DO_GSISOILDA: "YES"
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  DO_TEST_MODE: "NO"
diff --git a/env/GAEAC5.env b/env/GAEAC5.env
@@ -175,6 +175,15 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "eobs")
 

diff --git a/env/GAEAC6.env b/env/GAEAC6.env
@@ -175,6 +175,15 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "eobs")
 

diff --git a/env/HERA.env b/env/HERA.env
@@ -183,6 +183,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     export NTHREADS_CYCLE=${threads_per_task:-14}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then 
+        ntasks_regrid=36 
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -248,6 +256,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then 
+        ntasks_regrid=36 
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -186,6 +186,14 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
  ;;
  "eobs")
 
@@ -263,6 +271,13 @@ case ${step} in
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "epos")
 

diff --git a/env/JET.env b/env/JET.env
@@ -156,6 +156,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -221,6 +229,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

diff --git a/env/ORION.env b/env/ORION.env
@@ -184,6 +184,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -252,6 +260,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -175,6 +175,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export OMP_PLACES=cores
@@ -260,6 +268,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

diff --git a/jobs/JGLOBAL_ATMOS_SFCANL b/jobs/JGLOBAL_ATMOS_SFCANL
@@ -24,6 +24,9 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
   COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
 
+RUN="enkfgdas" MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+  COMIN_ATMOS_ENKF_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL
+
 mkdir -p "${COMOUT_ATMOS_RESTART}"
 
 

diff --git a/jobs/rocoto/esfc.sh b/jobs/rocoto/esfc.sh
@@ -4,7 +4,10 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
+# For DO_GSISOILDA=YES need to switch to ufsda modules 
+# until g-w issue 3390 is resolved.
 . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#. ${HOMEgfs}/ush/load_ufsda_modules.sh
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 

diff --git a/jobs/rocoto/sfcanl.sh b/jobs/rocoto/sfcanl.sh
@@ -4,7 +4,10 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
+# For DO_GSISOILDA=YES need to switch to ufsda modules 
+# until g-w issue 3390 is resolved.
 . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#. ${HOMEgfs}/ush/load_ufsda_modules.sh
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 

diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb
@@ -21,7 +21,11 @@ export aero_diagb_weight=0.9
 export aero_staticb_rescaling_factor=2.0
 export aero_diagb_n_halo=4
 export aero_diagb_n_neighbors=16
-export aero_diagb_smooth_horiz_iter=200
+aero_diagb_smooth_horiz_iter=0
 export aero_diagb_smooth_vert_iter=0
+if [[ "${CASE_ANL}" == "C384" ]]; then
+  aero_diagb_smooth_horiz_iter=200
+fi
+export aero_diagb_smooth_horiz_iter
 
 echo "END: config.aeroanlgenb"
diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal
@@ -48,10 +48,10 @@ export OZINFO=${FIXgfs}/gsi/global_ozinfo.txt
 export SATINFO=${FIXgfs}/gsi/global_satinfo.txt
 export OBERROR=${FIXgfs}/gsi/prepobs_errtable.global
 
-if [[ ${GSI_SOILANAL} = "YES" ]]; then
+if [[ ${DO_GSISOILDA} = "YES" ]]; then
     export hofx_2m_sfcfile=".true."
     export reducedgrid=".false." # not possible for sfc analysis, Jeff Whitaker says it's not useful anyway
-    export paranc=".false." # temporary until sfc io coded for parance (PR being prepared by T. Gichamo)
+    # NOTE: convinfo here will be over-written by date-specific files below.
     export CONVINFO=${FIXgfs}/gsi/global_convinfo_2mObs.txt
     export ANAVINFO=${FIXgfs}/gsi/global_anavinfo_soilanal.l127.txt
 fi

diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
@@ -435,7 +435,7 @@ else
   export restart_interval_gdas="6"
 fi
 
-export GSI_SOILANAL=@GSI_SOILANAL@
+export DO_GSISOILDA=@DO_GSISOILDA@
 
 # turned on nsst in anal and/or fcst steps, and turn off rtgsst
 export DONST="YES"

diff --git a/parm/config/gfs/config.esfc b/parm/config/gfs/config.esfc
@@ -21,10 +21,17 @@ if [[ "${DO_JEDIATMENS}" == "YES" ]]; then
    export DONST="NO"
 fi
 
+if [[ "${RUN/enkf}" == "gfs" ]]; then  
+    echo "turning off gsi soilda for gfs run"
+    DO_GSISOILDA="NO"
+fi
+
 # set up soil analysis
-if [[ ${GSI_SOILANAL} = "YES" ]]; then
-    export DO_LNDINC=".true."
-    export LND_SOI_FILE="lnd_incr"
+if [[ ${DO_GSISOILDA} == "YES" ]]; then
+    export GCYCLE_DO_SOILINCR=".true."
+    export GCYCLE_INTERP_LANDINCR=".false."
+    export LSOIL_INCR=2
+    export REGRID_EXEC="${HOMEgfs}/sorc/gdas.cd/build/bin/regridStates.x"
 fi
 
 echo "END: config.esfc"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
@@ -718,12 +718,12 @@ case ${step} in
         ntasks_gfs=825
         threads_per_task=5
         ;;
-      "C384")
+      "C384" | "C192")
         ntasks_gdas=160
         ntasks_gfs=160
         threads_per_task=10
         ;;
-      "C192" | "C96" | "C48")
+      "C96" | "C48")
         ntasks_gdas=84
         ntasks_gfs=84
         threads_per_task=5
@@ -1274,7 +1274,7 @@ case ${step} in
     ;;
 
   "esfc")
-    walltime="00:25:00"
+    walltime="01:00:00"
     ntasks=80
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))

diff --git a/parm/config/gfs/config.sfcanl b/parm/config/gfs/config.sfcanl
@@ -13,4 +13,16 @@ if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then
    export DONST="NO"
 fi
 
+if [[ "${RUN/enkf}" == "gfs" ]]; then  
+    echo "turning off gsi soilda for gfs run"
+    DO_GSISOILDA="NO"
+fi
+
+if [[ "${DO_GSISOILDA}" == "YES" ]]; then
+    export GCYCLE_DO_SOILINCR=".true."
+    export GCYCLE_INTERP_LANDINCR=".false."
+    export LSOIL_INCR=2
+    export REGRID_EXEC=${REGRID_EXEC:-"${HOMEgfs}/sorc/gdas.cd/build/bin/regridStates.x"}
+fi
+
 echo "END: config.sfcanl"
diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml
@@ -18,7 +18,7 @@ base:
   FHMAX_GFS: 120
   FHMAX_HF_GFS: 0
   FCST_BREAKPOINTS: ""
-  GSI_SOILANAL: "NO"
+  DO_GSISOILDA: "NO"
   EUPD_CYC: "gdas"
   FHMAX_ENKF_GFS: 12
   DOHYBVAR_OCN: "NO"