Add the mean EnKF soil increment to the deterministic member

.github/CODEOWNERS

@@ -181,6 +181,7 @@ syndat_qctropcy.sh @JiayiPeng-NOAA
 tropcy_relocate.sh @JiayiPeng-NOAA 
 tropcy_relocate_extrkr.sh @JiayiPeng-NOAA 
 wave_*.sh @JessicaMeixner-NOAA @sbanihash
+regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA
 
 # ush/python
 ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA

env/GAEAC5.env

@@ -175,6 +175,15 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "eobs")
 

env/GAEAC6.env

@@ -175,6 +175,15 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "eobs")
 

env/HERA.env

@@ -183,6 +183,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     export NTHREADS_CYCLE=${threads_per_task:-14}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then 
+        ntasks_regrid=36 
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -248,6 +256,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then 
+        ntasks_regrid=36 
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

env/HERCULES.env

@@ -186,6 +186,14 @@ case ${step} in
     export NTHREADS_CYCLE=${threads_per_task:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
  ;;
  "eobs")
 
@@ -262,6 +270,13 @@ case ${step} in
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
  ;;
  "epos")
 

env/JET.env

@@ -156,6 +156,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -221,6 +229,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

env/ORION.env

@@ -184,6 +184,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export MKL_NUM_THREADS=4
@@ -250,6 +258,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

env/WCOSS2.env

@@ -175,6 +175,14 @@ elif [[ "${step}" = "sfcanl" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default}"
 
+    # REGRID requires multiple of 6 tasks
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "eobs" ]]; then
 
     export OMP_PLACES=cores
@@ -260,6 +268,13 @@ elif [[ "${step}" = "esfc" ]]; then
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
     export APRUN_CYCLE="${APRUN_default} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}"
 
+    ntdiv6=$((ntasks/6))
+    ntasks_regrid=$((ntdiv6*6))
+    if [[ ${ntasks_regrid} -gt 36 ]]; then
+        ntasks_regrid=36
+    fi
+    export APRUN_REGRID="${launcher} -n ${ntasks_regrid} "
+
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}

jobs/JGLOBAL_ATMOS_SFCANL

@@ -24,6 +24,9 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
   COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
 
+RUN="enkfgdas" MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+  COMIN_ATMOS_ENKF_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL
+
 mkdir -p "${COMOUT_ATMOS_RESTART}"
 
 

jobs/rocoto/esfc.sh

@@ -4,7 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
+# CSD for DO_GSISOILDA, need ufsda modules (temporary, until merged)
 . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#. ${HOMEgfs}/ush/load_ufsda_modules.sh
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 

jobs/rocoto/sfcanl.sh

@@ -4,7 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
+# CSD for DO_GSISOILDA, need ufsda modules (temporary, until merged)
 . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+#. ${HOMEgfs}/ush/load_ufsda_modules.sh
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 

parm/config/gfs/config.anal

@@ -48,10 +48,10 @@ export OZINFO=${FIXgfs}/gsi/global_ozinfo.txt
 export SATINFO=${FIXgfs}/gsi/global_satinfo.txt
 export OBERROR=${FIXgfs}/gsi/prepobs_errtable.global
 
-if [[ ${GSI_SOILANAL} = "YES" ]]; then
+if [[ ${DO_GSISOILDA} = "YES" ]]; then
     export hofx_2m_sfcfile=".true."
     export reducedgrid=".false." # not possible for sfc analysis, Jeff Whitaker says it's not useful anyway
-    export paranc=".false." # temporary until sfc io coded for parance (PR being prepared by T. Gichamo)
+    # NOTE: convinfo here will be over-written by date-specific files below.
     export CONVINFO=${FIXgfs}/gsi/global_convinfo_2mObs.txt
     export ANAVINFO=${FIXgfs}/gsi/global_anavinfo_soilanal.l127.txt
 fi

parm/config/gfs/config.base

@@ -435,7 +435,7 @@ else
   export restart_interval_gdas="6"
 fi
 
-export GSI_SOILANAL=@GSI_SOILANAL@
+export DO_GSISOILDA=@DO_GSISOILDA@
 
 # turned on nsst in anal and/or fcst steps, and turn off rtgsst
 export DONST="YES"

parm/config/gfs/config.esfc

@@ -21,10 +21,20 @@ if [[ "${DO_JEDIATMENS}" == "YES" ]]; then
    export DONST="NO"
 fi
 
+_RUN=${RUN:-"gfs"}
+_RUN=${RUN/enkf/}
+
+if [[ "${_RUN}" == "gfs" ]]; then  
+    echo "turning off gsi soilda for gfs run"
+    DO_GSISOILDA="NO"
+fi
+
 # set up soil analysis
-if [[ ${GSI_SOILANAL} = "YES" ]]; then
-    export DO_LNDINC=".true."
-    export LND_SOI_FILE="lnd_incr"
+if [[ ${DO_GSISOILDA} == "YES" ]]; then
+     export GCYCLE_DO_SOILINCR=".true."
+     export GCYCLE_INTERP_LANDINCR=".false."
+     export LSOIL_INCR=2
+    export REGRID_EXEC=${REGRID_EXEC:-"${HOMEgfs}/sorc/gdas.cd/build/bin/regridStates.x"}
 fi
 
 echo "END: config.esfc"

parm/config/gfs/config.resources

@@ -679,12 +679,12 @@ case ${step} in
         ntasks_gfs=825
         threads_per_task=5
         ;;
-      "C384")
+      "C384" | "C192")
         ntasks_gdas=160
         ntasks_gfs=160
         threads_per_task=10
         ;;
-      "C192" | "C96" | "C48")
+      "C96" | "C48")
         ntasks_gdas=84
         ntasks_gfs=84
         threads_per_task=5
@@ -1234,7 +1234,7 @@ case ${step} in
     ;;
 
   "esfc")
-    walltime="00:25:00"
+    walltime="01:00:00"
     ntasks=80
     threads_per_task=1
     tasks_per_node=$(( max_tasks_per_node / threads_per_task ))

parm/config/gfs/config.sfcanl

@@ -13,4 +13,19 @@ if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then
    export DONST="NO"
 fi
 
+_RUN=${RUN:-"gfs"}
+_RUN=${RUN/enkf/}
+
+if [[ "${_RUN}" == "gfs" ]]; then  
+    echo "turning off gsi soilda for gfs run"
+    DO_GSISOILDA="NO"
+fi
+
+if [[ "${DO_GSISOILDA}" == "YES" ]]; then
+    export GCYCLE_DO_SOILINCR=".true."
+    export GCYCLE_INTERP_LANDINCR=".false."
+    export LSOIL_INCR=2
+    export REGRID_EXEC=${REGRID_EXEC:-"${HOMEgfs}/sorc/gdas.cd/build/bin/regridStates.x"}
+fi
+
 echo "END: config.sfcanl"

parm/config/gfs/yaml/defaults.yaml

@@ -19,7 +19,7 @@ base:
   FHMAX_HF_GFS: 0
   FCST_BREAKPOINTS: ""
   DO_VRFY_OCEANDA: "NO"
-  GSI_SOILANAL: "NO"
+  DO_GSISOILDA: "NO"
   EUPD_CYC: "gdas"
   FHMAX_ENKF_GFS: 12
   DOHYBVAR_OCN: "NO"