diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6a9cf9d47c..a6e45f6e6c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -180,6 +180,7 @@ syndat_qctropcy.sh @JiayiPeng-NOAA tropcy_relocate.sh @JiayiPeng-NOAA tropcy_relocate_extrkr.sh @JiayiPeng-NOAA wave_*.sh @JessicaMeixner-NOAA @sbanihash +regrid_gsiSfcIncr_to_tile.sh @ClaraDraper-NOAA # ush/python ush/python/pygfs/jedi/__init__.py @aerorahul @DavidNew-NOAA diff --git a/ci/cases/yamls/atmsoilDA_defaults_ci.yaml b/ci/cases/yamls/atmsoilDA_defaults_ci.yaml new file mode 100644 index 0000000000..8ace896632 --- /dev/null +++ b/ci/cases/yamls/atmsoilDA_defaults_ci.yaml @@ -0,0 +1,6 @@ +defaults: + !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml +base: + DO_GSISOILDA: "YES" + ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} + DO_TEST_MODE: "NO" diff --git a/env/GAEAC5.env b/env/GAEAC5.env index 8af8003dff..f05d04fd09 100755 --- a/env/GAEAC5.env +++ b/env/GAEAC5.env @@ -175,6 +175,15 @@ case ${step} in export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + + # REGRID requires multiple of 6 tasks + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + ;; "eobs") diff --git a/env/GAEAC6.env b/env/GAEAC6.env index 90150948e3..5f5540c55a 100755 --- a/env/GAEAC6.env +++ b/env/GAEAC6.env @@ -175,6 +175,15 @@ case ${step} in export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + + # REGRID requires multiple of 6 tasks + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + ;; "eobs") diff --git a/env/HERA.env b/env/HERA.env index 68f64617e1..30a45b8e5c 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -183,6 +183,14 @@ elif [[ "${step}" = "sfcanl" ]]; then export NTHREADS_CYCLE=${threads_per_task:-14} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + # REGRID requires multiple of 6 tasks + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "eobs" ]]; then export MKL_NUM_THREADS=4 @@ -248,6 +256,13 @@ elif [[ "${step}" = "esfc" ]]; then [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index b565881a73..5e6ca72f08 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -186,6 +186,14 @@ case ${step} in export NTHREADS_CYCLE=${threads_per_task:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + + # REGRID requires multiple of 6 tasks + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " ;; "eobs") @@ -263,6 +271,13 @@ case ${step} in [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + ;; "epos") diff --git a/env/JET.env b/env/JET.env index 7bfd912062..3b6373d763 100755 --- a/env/JET.env +++ b/env/JET.env @@ -156,6 +156,14 @@ elif [[ "${step}" = "sfcanl" ]]; then [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default}" + # REGRID requires multiple of 6 tasks + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "eobs" ]]; then export MKL_NUM_THREADS=4 @@ -221,6 +229,13 @@ elif [[ "${step}" = "esfc" ]]; then [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default}" + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} diff --git a/env/ORION.env b/env/ORION.env index 64f3f3d555..435844fab8 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -184,6 +184,14 @@ elif [[ "${step}" = "sfcanl" ]]; then [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + # REGRID requires multiple of 6 tasks + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "eobs" ]]; then export MKL_NUM_THREADS=4 @@ -252,6 +260,13 @@ elif [[ "${step}" = "esfc" ]]; then [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index fe6acf88fb..a6be4464e2 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -175,6 +175,14 @@ elif [[ "${step}" = "sfcanl" ]]; then [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default}" + # REGRID requires multiple of 6 tasks + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "eobs" ]]; then export OMP_PLACES=cores @@ -260,6 +268,13 @@ elif [[ "${step}" = "esfc" ]]; then [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} export APRUN_CYCLE="${APRUN_default} -ppn ${tasks_per_node_cycle} --cpu-bind depth --depth ${NTHREADS_CYCLE}" + ntdiv6=$((ntasks/6)) + ntasks_regrid=$((ntdiv6*6)) + if [[ ${ntasks_regrid} -gt 36 ]]; then + ntasks_regrid=36 + fi + export APRUN_REGRID="${launcher} -n ${ntasks_regrid} " + elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} diff --git a/jobs/JGLOBAL_ATMOS_SFCANL b/jobs/JGLOBAL_ATMOS_SFCANL index 2822b1e94b..ed8e6aebe2 100755 --- a/jobs/JGLOBAL_ATMOS_SFCANL +++ b/jobs/JGLOBAL_ATMOS_SFCANL @@ -24,6 +24,9 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL +RUN="enkfgdas" MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_ATMOS_ENKF_ANALYSIS_STAT:COM_ATMOS_ANALYSIS_TMPL + mkdir -p "${COMOUT_ATMOS_RESTART}" diff --git a/jobs/rocoto/esfc.sh b/jobs/rocoto/esfc.sh index 85f44151c9..b8f9393113 100755 --- a/jobs/rocoto/esfc.sh +++ b/jobs/rocoto/esfc.sh @@ -4,7 +4,10 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules +# For DO_GSISOILDA=YES need to switch to ufsda modules +# until g-w issue 3390 is resolved. . ${HOMEgfs}/ush/load_fv3gfs_modules.sh +#. ${HOMEgfs}/ush/load_ufsda_modules.sh status=$? [[ ${status} -ne 0 ]] && exit ${status} diff --git a/jobs/rocoto/sfcanl.sh b/jobs/rocoto/sfcanl.sh index 44f93ee0c3..c9459a9d8d 100755 --- a/jobs/rocoto/sfcanl.sh +++ b/jobs/rocoto/sfcanl.sh @@ -4,7 +4,10 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules +# For DO_GSISOILDA=YES need to switch to ufsda modules +# until g-w issue 3390 is resolved. . ${HOMEgfs}/ush/load_fv3gfs_modules.sh +#. ${HOMEgfs}/ush/load_ufsda_modules.sh status=$? [[ ${status} -ne 0 ]] && exit ${status} diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb index 075b4be90b..c117aff481 100644 --- a/parm/config/gfs/config.aeroanlgenb +++ b/parm/config/gfs/config.aeroanlgenb @@ -21,7 +21,11 @@ export aero_diagb_weight=0.9 export aero_staticb_rescaling_factor=2.0 export aero_diagb_n_halo=4 export aero_diagb_n_neighbors=16 -export aero_diagb_smooth_horiz_iter=200 +aero_diagb_smooth_horiz_iter=0 export aero_diagb_smooth_vert_iter=0 +if [[ "${CASE_ANL}" == "C384" ]]; then + aero_diagb_smooth_horiz_iter=200 +fi +export aero_diagb_smooth_horiz_iter echo "END: config.aeroanlgenb" diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal index 123bd6decd..b66ba0c800 100644 --- a/parm/config/gfs/config.anal +++ b/parm/config/gfs/config.anal @@ -48,10 +48,10 @@ export OZINFO=${FIXgfs}/gsi/global_ozinfo.txt export SATINFO=${FIXgfs}/gsi/global_satinfo.txt export OBERROR=${FIXgfs}/gsi/prepobs_errtable.global -if [[ ${GSI_SOILANAL} = "YES" ]]; then +if [[ ${DO_GSISOILDA} = "YES" ]]; then export hofx_2m_sfcfile=".true." export reducedgrid=".false." # not possible for sfc analysis, Jeff Whitaker says it's not useful anyway - export paranc=".false." # temporary until sfc io coded for parance (PR being prepared by T. Gichamo) + # NOTE: convinfo here will be over-written by date-specific files below. export CONVINFO=${FIXgfs}/gsi/global_convinfo_2mObs.txt export ANAVINFO=${FIXgfs}/gsi/global_anavinfo_soilanal.l127.txt fi diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 586029f785..d1bf45d238 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -435,7 +435,7 @@ else export restart_interval_gdas="6" fi -export GSI_SOILANAL=@GSI_SOILANAL@ +export DO_GSISOILDA=@DO_GSISOILDA@ # turned on nsst in anal and/or fcst steps, and turn off rtgsst export DONST="YES" diff --git a/parm/config/gfs/config.esfc b/parm/config/gfs/config.esfc index 684dea4ee3..f2c8414be4 100644 --- a/parm/config/gfs/config.esfc +++ b/parm/config/gfs/config.esfc @@ -21,10 +21,17 @@ if [[ "${DO_JEDIATMENS}" == "YES" ]]; then export DONST="NO" fi +if [[ "${RUN/enkf}" == "gfs" ]]; then + echo "turning off gsi soilda for gfs run" + DO_GSISOILDA="NO" +fi + # set up soil analysis -if [[ ${GSI_SOILANAL} = "YES" ]]; then - export DO_LNDINC=".true." - export LND_SOI_FILE="lnd_incr" +if [[ ${DO_GSISOILDA} == "YES" ]]; then + export GCYCLE_DO_SOILINCR=".true." + export GCYCLE_INTERP_LANDINCR=".false." + export LSOIL_INCR=2 + export REGRID_EXEC="${HOMEgfs}/sorc/gdas.cd/build/bin/regridStates.x" fi echo "END: config.esfc" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 95d6ad117d..613eaec3d2 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -718,12 +718,12 @@ case ${step} in ntasks_gfs=825 threads_per_task=5 ;; - "C384") + "C384" | "C192") ntasks_gdas=160 ntasks_gfs=160 threads_per_task=10 ;; - "C192" | "C96" | "C48") + "C96" | "C48") ntasks_gdas=84 ntasks_gfs=84 threads_per_task=5 @@ -1274,7 +1274,7 @@ case ${step} in ;; "esfc") - walltime="00:25:00" + walltime="01:00:00" ntasks=80 threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) diff --git a/parm/config/gfs/config.sfcanl b/parm/config/gfs/config.sfcanl index e2fde8992a..e235154f2d 100644 --- a/parm/config/gfs/config.sfcanl +++ b/parm/config/gfs/config.sfcanl @@ -13,4 +13,16 @@ if [[ "${DO_JEDIATMVAR}" == "YES" ]]; then export DONST="NO" fi +if [[ "${RUN/enkf}" == "gfs" ]]; then + echo "turning off gsi soilda for gfs run" + DO_GSISOILDA="NO" +fi + +if [[ "${DO_GSISOILDA}" == "YES" ]]; then + export GCYCLE_DO_SOILINCR=".true." + export GCYCLE_INTERP_LANDINCR=".false." + export LSOIL_INCR=2 + export REGRID_EXEC=${REGRID_EXEC:-"${HOMEgfs}/sorc/gdas.cd/build/bin/regridStates.x"} +fi + echo "END: config.sfcanl" diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 91ed9e0eb0..6f8cb840dd 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -18,7 +18,7 @@ base: FHMAX_GFS: 120 FHMAX_HF_GFS: 0 FCST_BREAKPOINTS: "" - GSI_SOILANAL: "NO" + DO_GSISOILDA: "NO" EUPD_CYC: "gdas" FHMAX_ENKF_GFS: 12 DOHYBVAR_OCN: "NO" diff --git a/scripts/exgdas_enkf_sfc.sh b/scripts/exgdas_enkf_sfc.sh index 1944325317..bfe559a27a 100755 --- a/scripts/exgdas_enkf_sfc.sh +++ b/scripts/exgdas_enkf_sfc.sh @@ -24,7 +24,7 @@ pwd=$(pwd) # Base variables DONST=${DONST:-"NO"} -GSI_SOILANAL=${GSI_SOILANAL:-"NO"} +DO_GSISOILDA=${DO_GSISOILDA:-"NO"} DOSFCANL_ENKF=${DOSFCANL_ENKF:-"YES"} export CASE=${CASE:-384} ntiles=${ntiles:-6} @@ -58,6 +58,7 @@ DOIAU=${DOIAU_ENKF:-"NO"} # Global_cycle stuff CYCLESH=${CYCLESH:-${USHgfs}/global_cycle.sh} +REGRIDSH=${REGRIDSH:-"${USHgfs}/regrid_gsiSfcIncr_to_tile.sh"} export CYCLEXEC=${CYCLEXEC:-${EXECgfs}/global_cycle} APRUN_CYCLE=${APRUN_CYCLE:-${APRUN:-""}} NTHREADS_CYCLE=${NTHREADS_CYCLE:-${NTHREADS:-1}} @@ -115,6 +116,23 @@ else export NST_FILE="NULL" fi +# regrid the surface increment files +if [[ ${DO_GSISOILDA} = "YES" ]]; then + + export CASE_IN=${CASE_ENS} + export CASE_OUT=${CASE_ENS} + export OCNRES_OUT=${OCNRES} + export NMEM_REGRID=${NMEM_ENS} + if [[ "${DOIAU}" == "YES" ]]; then + export LFHR=3 # match BDATE + else # DOSFCANL_ENKF + export LFHR=6 # PDYcyc + fi + + "${REGRIDSH}" + +fi + export APRUNCY=${APRUN_CYCLE:-$APRUN_ESFC} export OMP_NUM_THREADS_CY=${NTHREADS_CYCLE:-$NTHREADS_ESFC} export MAX_TASKS_CY=$NMEM_ENS @@ -144,7 +162,7 @@ if [ $DOIAU = "YES" ]; then COMIN_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + COMIN_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ COMIN_SNOW_ANALYSIS_MEM:COM_SNOW_ANALYSIS_TMPL @@ -163,11 +181,11 @@ if [ $DOIAU = "YES" ]; then ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc" "${DATA}/fngrid.${cmem}" ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}" - if [[ ${GSI_SOILANAL} = "YES" ]]; then - FHR=6 - ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ - "${DATA}/lnd_incr.${cmem}" + if [[ ${DO_GSISOILDA} = "YES" ]]; then + ${NCP} "${COMIN_ATMOS_ANALYSIS_MEM}/sfci00${LFHR}.tile${n}.nc" \ + "${DATA}/soil_xainc.${cmem}" fi + done # ensembles CDATE="${PDY}${cyc}" ${CYCLESH} @@ -186,25 +204,16 @@ if [ $DOIAU = "YES" ]; then MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ COM_ATMOS_RESTART_MEM:COM_ATMOS_RESTART_TMPL - MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ - COM_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL - [[ ${TILE_NUM} -eq 1 ]] && mkdir -p "${COM_ATMOS_RESTART_MEM}" cpfs "${DATA}/fnbgso.${cmem}" "${COM_ATMOS_RESTART_MEM}/${bPDY}.${bcyc}0000.sfcanl_data.tile${n}.nc" - - if [[ ${GSI_SOILANAL} = "YES" ]]; then - FHR=6 - ${NCP} "${COM_ATMOS_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ - "${DATA}/lnd_incr.${cmem}" - fi done # ensembles done fi -if [ $DOSFCANL_ENKF = "YES" ]; then +if [[ "${DOSFCANL_ENKF}" == "YES" ]]; then for n in $(seq 1 $ntiles); do export TILE_NUM=$n @@ -216,8 +225,8 @@ if [ $DOSFCANL_ENKF = "YES" ]; then smem=$((smem - NMEM_ENS_MAX)) fi gmemchar="mem"$(printf %03i "$smem") - cmem=$(printf %03i $imem) - memchar="mem$cmem" + cmem=$(printf %03i ${imem}) + memchar="mem${cmem}" RUN="${GDUMP_ENS}" MEMDIR=${gmemchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ COMIN_SNOW_ANALYSIS_MEM:COM_SNOW_ANALYSIS_TMPL @@ -225,6 +234,9 @@ if [ $DOSFCANL_ENKF = "YES" ]; then RUN="${GDUMP_ENS}" MEMDIR=${gmemchar} YMD=${gPDY} HH=${gcyc} declare_from_tmpl \ COMIN_ATMOS_RESTART_MEM_PREV:COM_ATMOS_RESTART_TMPL + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMIN_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + # determine where the input snow restart files come from if [[ "${DO_JEDISNOWDA:-}" == "YES" ]]; then sfcdata_dir="${COMIN_SNOW_ANALYSIS_MEM}" @@ -238,6 +250,10 @@ if [ $DOSFCANL_ENKF = "YES" ]; then ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}_grid.tile${n}.nc" "${DATA}/fngrid.${cmem}" ${NCP} "${FIXgfs}/orog/${CASE}/${CASE}.mx${OCNRES}_oro_data.tile${n}.nc" "${DATA}/fnorog.${cmem}" + if [[ ${DO_GSISOILDA} = "YES" ]]; then + ${NCP} "${COMIN_ATMOS_ANALYSIS_MEM}/sfci00${LFHR}.tile${n}.nc" \ + "${DATA}/soil_xainc.${cmem}" + fi done CDATE="${PDY}${cyc}" ${CYCLESH} @@ -272,4 +288,4 @@ fi cd "${pwd}" || exit 1 -exit ${err} +exit "${err}" diff --git a/scripts/exgdas_enkf_update.sh b/scripts/exgdas_enkf_update.sh index 9517544b1a..4d75b182b0 100755 --- a/scripts/exgdas_enkf_update.sh +++ b/scripts/exgdas_enkf_update.sh @@ -91,7 +91,7 @@ else mem_offset=0 fi INCREMENTS_TO_ZERO=${INCREMENTS_TO_ZERO:-"'NONE'"} -GSI_SOILANAL=${GSI_SOILANAL:-"NO"} +DO_GSISOILDA=${DO_GSISOILDA:-"NO"} ################################################################################ @@ -213,7 +213,7 @@ for imem in $(seq 1 $NMEM_ENS); do for FHR in $nfhrs; do ${NLN} "${COMIN_ATMOS_HISTORY_MEM_PREV}/${GPREFIX}atmf00${FHR}${ENKF_SUFFIX}.nc" \ "sfg_${PDY}${cyc}_fhr0${FHR}_${memchar}" - if [ $GSI_SOILANAL = "YES" ]; then + if [[ "${DO_GSISOILDA}" = "YES" ]]; then ${NLN} "${COMIN_ATMOS_HISTORY_MEM_PREV}/${GPREFIX}sfcf00${FHR}${ENKF_SUFFIX}.nc" \ "bfg_${PDY}${cyc}_fhr0${FHR}_${memchar}" fi @@ -238,7 +238,7 @@ for imem in $(seq 1 $NMEM_ENS); do "incr_${PDY}${cyc}_fhr0${FHR}_${memchar}" fi fi - if [ $GSI_SOILANAL = "YES" ]; then + if [[ "${DO_GSISOILDA}" = "YES" ]]; then ${NLN} "${COMOUT_ATMOS_ANALYSIS_MEM}/${APREFIX}sfci00${FHR}.nc" \ "sfcincr_${PDY}${cyc}_fhr0${FHR}_${memchar}" fi @@ -254,6 +254,12 @@ for FHR in $nfhrs; do ${NLN} "${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}sfcf00${FHR}.ensmean.nc" \ "sfgsfc_${PDY}${cyc}_fhr0${FHR}_ensmean" fi + if [[ "${DO_GSISOILDA}" = "YES" ]]; then + ${NLN} "${COMIN_ATMOS_HISTORY_STAT_PREV}/${GPREFIX}sfcf00${FHR}.ensmean.nc" \ + "bfg_${PDY}${cyc}_fhr0${FHR}_ensmean" + ${NLN} "${COMIN_ATMOS_ANALYSIS_STAT}/${APREFIX}sfci00${FHR}.nc" \ + "sfcincr_${PDY}${cyc}_fhr0${FHR}_ensmean" + fi done if [[ $USE_CFP = "YES" ]]; then diff --git a/scripts/exglobal_atmos_sfcanl.sh b/scripts/exglobal_atmos_sfcanl.sh index cbc43b0979..95b2ae4796 100755 --- a/scripts/exglobal_atmos_sfcanl.sh +++ b/scripts/exglobal_atmos_sfcanl.sh @@ -28,6 +28,7 @@ cd "${DATA}" || exit 99 # Dependent Scripts and Executables CYCLESH=${CYCLESH:-${USHgfs}/global_cycle.sh} +REGRIDSH=${REGRIDSH:-"${USHgfs}/regrid_gsiSfcIncr_to_tile.sh"} export CYCLEXEC=${CYCLEXEC:-${EXECgfs}/global_cycle} NTHREADS_CYCLE=${NTHREADS_CYCLE:-24} APRUN_CYCLE=${APRUN_CYCLE:-${APRUN:-""}} @@ -111,19 +112,46 @@ fi # Collect the dates in the window to update surface restarts gcycle_dates=("${PDY}${cyc}") # Always update surface restarts at middle of window +soilinc_fhrs=("${assim_freq}") # increment file at middle of window +LFHR=${assim_freq} if [[ "${DOIAU:-}" == "YES" ]]; then # Update surface restarts at beginning of window half_window=$(( assim_freq / 2 )) + soilinc_fhrs+=("${half_window}") + LFHR=-1 BDATE=$(date --utc -d "${PDY} ${cyc} - ${half_window} hours" +%Y%m%d%H) gcycle_dates+=("${BDATE}") fi +# if doing GSI soil anaysis, copy increment file and re-grid it to native model resolution +if [[ "${DO_GSISOILDA}" = "YES" ]]; then + + export COMIN_SOIL_ANALYSIS_MEM="${COMIN_ATMOS_ENKF_ANALYSIS_STAT}" + export COMOUT_ATMOS_ANALYSIS_MEM="${COMIN_ATMOS_ANALYSIS}" + export CASE_IN="${CASE_ENS}" + export CASE_OUT="${CASE}" + export OCNRES_OUT="${OCNRES}" + export LFHR + + "${REGRIDSH}" + +fi + # Loop over the dates in the window to update the surface restarts -for gcycle_date in "${gcycle_dates[@]}"; do +for hr in "${!gcycle_dates[@]}"; do + gcycle_date=${gcycle_dates[hr]} + FHR=${soilinc_fhrs[hr]} echo "Updating surface restarts for ${gcycle_date} ..." datestr="${gcycle_date:0:8}.${gcycle_date:8:2}0000" + if [[ "${DO_GSISOILDA}" = "YES" ]]; then + for (( nn=1; nn <= ntiles; nn++ )); do + ${NCP} "${COMIN_ATMOS_ANALYSIS}/sfci00${FHR}.tile${nn}.nc" \ + "${DATA}/soil_xainc.00${nn}" + done + fi + # Copy inputs from COMIN to DATA for (( nn=1; nn <= ntiles; nn++ )); do ${NCP} "${sfcdata_dir}/${datestr}.sfc_data.tile${nn}.nc" "${DATA}/fnbgsi.00${nn}" diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 7b90ef70f1..f91938b871 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 7b90ef70f1b55cffc14b0c9d9acd0ebd5c5d7201 +Subproject commit f91938b8714c9274c18f91e4d2a712473176a224 diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index a57f5660cc..dc451c7754 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -369,7 +369,8 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then "gdasapp_land_ensrecenter.x" "bufr2ioda.x" "calcfIMS.exe" - "apply_incr.exe") + "apply_incr.exe" + "regridStates.x") for gdasexe in "${JEDI_EXE[@]}"; do [[ -s "${gdasexe}" ]] && rm -f "${gdasexe}" ${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/${gdasexe}" . diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 5b2bfa39a5..0f8cb9faa3 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 5b2bfa39a5b1fbe6bb1d247102679885685c9336 +Subproject commit 0f8cb9faa33d0985e0ca6250e6842d821a9f3cde diff --git a/ush/regrid_gsiSfcIncr_to_tile.sh b/ush/regrid_gsiSfcIncr_to_tile.sh new file mode 100755 index 0000000000..0e28c77c51 --- /dev/null +++ b/ush/regrid_gsiSfcIncr_to_tile.sh @@ -0,0 +1,125 @@ +#! /usr/bin/env bash + +source "${USHgfs}/preamble.sh" + +#------------------------------------------------------------------------------------------------- +# Script to regrid surface increment from GSI grid +# to fv3 tiles. +# Clara Draper, Dec 2024 +#------------------------------------------------------------------------------------------------- + +# temporary files on hera, until g-w issue 3392 is resolved. +TMP_FIX_FILES=/scratch2/BMC/gsienkf/Clara.Draper/gw_new_fix_files/ + + +export PGMOUT=${PGMOUT:-${pgmout:-'&1'}} +export PGMERR=${PGMERR:-${pgmerr:-'&2'}} +export REDOUT=${REDOUT:-'1>'} +export REDERR=${REDERR:-'2>'} + +export PGM=${REGRID_EXEC} +export pgm=${PGM} + +NMEM_REGRID=${NMEM_REGRID:-1} +CASE_IN=${CASE_IN:-${CASE_ENS}} +LFHR=${LFHR:-6} + +# get resolutions +LONB_CASE_IN=$((4*${CASE_IN:1})) +LATB_CASE_IN=$((2*${CASE_IN:1})) + +ntiles=6 + +APREFIX_ENS="enkfgdas.t${cyc}z." + +LSOIL_INCR=${LSOIL_INCR:-2} + +n_vars=$(( LSOIL_INCR*2 )) + +soil_incr_vars="" +for vi in $( seq 1 "${LSOIL_INCR}" ); do + soil_incr_vars=${soil_incr_vars}'"soilt'${vi}'_inc"', +done +for vi in $( seq 1 "${LSOIL_INCR}" ); do + soil_incr_vars=${soil_incr_vars}'"slc'${vi}'_inc"', +done + +cat << EOF > regrid.nml + &config + n_vars=${n_vars}, + variable_list=${soil_incr_vars} + missing_value=0., + / + &input + gridtype="gau_inc", + ires=${LONB_CASE_IN}, + jres=${LATB_CASE_IN}, + fname="enkfgdas.sfci.nc", + dir="./", + fname_coord="gaussian_scrip.nc", + dir_coord="./" +/ + + &output + gridtype="fv3_rst", + ires=${CASE_OUT:1}, + jres=${CASE_OUT:1}, + fname="sfci", + dir="./", + fname_mask="vegetation_type" + dir_mask="./" + dir_coord="./", + / +EOF + +# input, fixed files +${NCP} "${TMP_FIX_FILES}/gaussian.${LONB_CASE_IN}.${LATB_CASE_IN}.nc" \ + "${DATA}/gaussian_scrip.nc" + +# output, fixed files +${NCP} "${FIXorog}/${CASE_OUT}/${CASE_OUT}_mosaic.nc" \ + "${DATA}/${CASE_OUT}_mosaic.nc" + +for n in $(seq 1 "${ntiles}"); do + ${NCP} "${FIXorog}/${CASE_OUT}/sfc/${CASE_OUT}.mx${OCNRES_OUT}.vegetation_type.tile${n}.nc" \ + "${DATA}/vegetation_type.tile${n}.nc" + ${NCP} "${FIXorog}/${CASE_OUT}/${CASE_OUT}_grid.tile${n}.nc" \ + "${DATA}/${CASE_OUT}_grid.tile${n}.nc" +done + +if (( LFHR >= 0 )); then + soilinc_fhrs=("${LFHR}") +else # construct restart times for deterministic member + soilinc_fhrs=("${assim_freq}") # increment file at middle of window + if [[ "${DOIAU:-}" == "YES" ]]; then # Update surface restarts at beginning of window + half_window=$(( assim_freq / 2 )) + soilinc_fhrs+=("${half_window}") + fi +fi + +for imem in $(seq 1 "${NMEM_REGRID}"); do + if (( NMEM_REGRID > 1 )); then + cmem=$(printf %03i "${imem}") + memchar="mem${cmem}" + + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMOUT_ATMOS_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + + MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \ + COMIN_SOIL_ANALYSIS_MEM:COM_ATMOS_ANALYSIS_TMPL + fi + + for FHR in "${soilinc_fhrs[@]}"; do + ${NCP} "${COMIN_SOIL_ANALYSIS_MEM}/${APREFIX_ENS}sfci00${FHR}.nc" \ + "${DATA}/enkfgdas.sfci.nc" + + ${APRUN_REGRID} "${REGRID_EXEC}" "${REDOUT}${PGMOUT}" "${REDERR}${PGMERR}" + + for n in $(seq 1 "${ntiles}"); do + cpfs "${DATA}/sfci.tile${n}.nc" "${COMOUT_ATMOS_ANALYSIS_MEM}/sfci00${FHR}.tile${n}.nc" + done + done +done + +exit 0 + diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 0dd2ce44a4..ae36d2ffbc 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -48,6 +48,7 @@ def _get_run_options(self, conf: Configuration) -> Dict[str, Any]: run_options[run]['do_jediatmens'] = base.get('DO_JEDIATMENS', False) run_options[run]['do_jediocnvar'] = base.get('DO_JEDIOCNVAR', False) run_options[run]['do_jedisnowda'] = base.get('DO_JEDISNOWDA', False) + run_options[run]['do_gsisoilda'] = base.get('DO_GSISOILDA', False) run_options[run]['do_mergensst'] = base.get('DO_MERGENSST', False) return run_options diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index f540ee2167..614bda37e2 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -252,6 +252,10 @@ def sfcanl(self): if self.options['do_jedisnowda']: dep_dict = {'type': 'task', 'name': f'{self.run}_snowanl'} deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_gsisoilda']: + dep_dict = {'type': 'task', 'name': 'enkfgdas_eupd'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.options['do_jedisnowda'] or self.options['do_gsisoilda']: dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) else: dependencies = rocoto.create_dependency(dep=deps)