Merge branch 'NOAA-EMC:develop' into IASI-NG

ci/Jenkinsfile

@@ -138,8 +138,14 @@ pipeline {
                                             }
                                         }
                                         try {
-                                            sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}")
-                                            gist_url=sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}", returnStdout: true).trim()
+                                            sh(script: """
+                                              source ${HOMEgfs}/workflow/gw_setup.sh
+                                              ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_BUILD_${env.CHANGE_ID}
+                                            """)
+                                            gist_url=sh(script: """
+                                              source ${HOMEgfs}/workflow/gw_setup.sh
+                                              ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_BUILD_${env.CHANGE_ID}
+                                            """, returnStdout: true).trim()
                                             sh(script:  """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Build **FAILED** on **${Machine}** in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
                                         } catch (Exception error_comment) {
                                             echo "Failed to comment on PR: ${error_comment.getMessage()}"
@@ -158,7 +164,10 @@ pipeline {
                                 }
                             }
                             // Get a list of CI cases to run
-                            CI_CASES = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
+                            CI_CASES = sh(script: """
+                              source ${HOMEgfs}/workflow/gw_setup.sh
+                              ${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}
+                            """, returnStdout: true).trim().split()
                             echo "Cases to run: ${CI_CASES}"
                         }
                     }
@@ -179,7 +188,10 @@ pipeline {
                                 script {
                                         env.RUNTESTS = "${CUSTOM_WORKSPACE}/RUNTESTS"
                                         try {
-                                              error_output = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml", returnStdout: true).trim()
+                                              error_output = sh(script: """
+                                                source ${HOMEgfs}/workflow/gw_setup.sh
+                                                ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh create_experiment ${HOMEgfs}/ci/cases/pr/${caseName}.yaml
+                                              """, returnStdout: true).trim()
                                         } catch (Exception error_create) {
                                             sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "${Case} **FAILED** to create experiment on ${Machine} in BUILD# ${env.BUILD_NUMBER}\n with the error:\n\\`\\`\\`\n${error_output}\\`\\`\\`" """)
                                             error("Case ${caseName} failed to create experiment directory")
@@ -194,10 +206,19 @@ pipeline {
                                                 def error_file = "${CUSTOM_WORKSPACE}/RUNTESTS/${pslot}_error.logs"
                                                 sh(script: " rm -f ${error_file}")
                                                 try {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'")
-                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}")
+                                                    sh(script: """
+                                                      source ${HOMEgfs}/workflow/gw_setup.sh
+                                                      ${HOMEgfs}/ci/scripts/run-check_ci.sh ${CUSTOM_WORKSPACE} ${pslot} 'global-workflow'
+                                                    """)
+                                                    sh(script: """
+                                                      source ${HOMEgfs}/workflow/gw_setup.sh
+                                                      ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cleanup_experiment ${CUSTOM_WORKSPACE}/RUNTESTS/EXPDIR/${pslot}
+                                                    """)
                                                 } catch (Exception error_experment) {
-                                                    sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}")
+                                                    sh(script: """
+                                                      source ${HOMEgfs}/workflow/gw_setup.sh
+                                                      ${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_batch_jobs ${pslot}
+                                                    """)
                                                     ws(CUSTOM_WORKSPACE) {
                                                         def error_logs = ""
                                                         def error_logs_message = ""
@@ -217,9 +238,15 @@ pipeline {
                                                                 }
                                                             }
                                                             try {
-                                                                gist_url = sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}", returnStdout: true).trim()
+                                                                gist_url = sh(script: """
+                                                                  source ${HOMEgfs}/workflow/gw_setup.sh
+                                                                  ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --gist PR_${env.CHANGE_ID}
+                                                                """, returnStdout: true).trim()
                                                                 sh(script: """${GH} pr comment ${env.CHANGE_ID} --repo ${repo_url} --body "Experiment ${caseName} **FAILED** on ${Machine} in Build# ${env.BUILD_NUMBER} with error logs:\n\\`\\`\\`\n${error_logs_message}\\`\\`\\`\n\nFollow link here to view the contents of the above file(s): [(link)](${gist_url})" """)
-                                                                sh(script: "${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}")
+                                                                sh(script: """
+                                                                  source ${HOMEgfs}/workflow/gw_setup.sh
+                                                                  ${HOMEgfs}/ci/scripts/utils/publish_logs.py --file ${error_logs} --repo PR_${env.CHANGE_ID}
+                                                                """)
                                                             } catch (Exception error_comment) {
                                                                 echo "Failed to comment on PR: ${error_comment.getMessage()}"
                                                             }

ci/cases/yamls/gefs_defaults_ci.yaml

@@ -4,7 +4,6 @@ base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
   SFS_POST: "NO"
   FHOUT_GFS: 6
-stage_ic:
   USE_OCN_ENS_PERTURB_FILES: "NO"
   USE_ATM_ENS_PERTURB_FILES: "NO"
 ocn:

ci/cases/yamls/sfs_defaults.yaml

@@ -7,7 +7,7 @@ base:
   DO_BUFRSND: "NO"
   DO_GEMPAK: "NO"
   DO_AWIPS: "NO"
-  KEEPDATA: "YES"
+  KEEPDATA: "NO"
   DO_EXTRACTVARS: "NO"
   FHMAX_GFS: 2976
   FHMAX_HF_GFS: 0
@@ -17,6 +17,8 @@ base:
   FHOUT_ICE_GFS: 24
   FCST_BREAKPOINTS: ""
   REPLAY_ICS: "NO"
+  USE_OCN_ENS_PERTURB_FILES: "YES"
+  USE_ATM_ENS_PERTURB_FILES: "YES"
   HPSSARCH: "NO"
   LOCALARCH: "NO"
   SFS_POST: "YES"
@@ -26,8 +28,5 @@ fcst:
   MONO: "mono"
   reforecast: "YES"
   FHZER: 24
-stage_ic:
-  USE_OCN_ENS_PERTURB_FILES: "YES"
-  USE_ATM_ENS_PERTURB_FILES: "YES"
 ocn:
   MOM6_INTERP_ICS: "YES"

ci/scripts/utils/launch_java_agent.sh

@@ -65,14 +65,14 @@ controller_url="https://jenkins.epic.oarcloud.noaa.gov"
 controller_user=${controller_user:-"terry.mcguinness"}
 controller_user_auth_token="jenkins_token"
 
-HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
+HOMEGFS_="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
 host=$(hostname)
 
 #########################################################################
 #  Set up runtime environment varibles for accounts on supproted machines
 #########################################################################
 
-source "${HOMEgfs}/ush/detect_machine.sh"
+source "${HOMEGFS_}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
   hera | orion | hercules | wcoss2 | gaea)
     echo "Launch Jenkins Java Controler on ${MACHINE_ID}";;
@@ -84,10 +84,10 @@ esac
 LOG=lanuched_agent-$(date +%Y%m%d%M).log
 rm -f "${LOG}"
 
-source "${HOMEgfs}/ush/module-setup.sh"
-module use "${HOMEgfs}/modulefiles"
+source "${HOMEGFS_}/ush/module-setup.sh"
+module use "${HOMEGFS_}/modulefiles"
 module load "module_gwsetup.${MACHINE_ID}"
-source "${HOMEgfs}/ci/platforms/config.${MACHINE_ID}"
+source "${HOMEGFS_}/ci/platforms/config.${MACHINE_ID}"
 
 JAVA_HOME="${JENKINS_AGENT_LANUCH_DIR}/JAVA/jdk-17.0.10"
 if [[ ! -d "${JAVA_HOME}" ]]; then
@@ -102,9 +102,10 @@ JAVA="${JAVA_HOME}/bin/java"
 echo "JAVA VERSION: "
 ${JAVA} -version
 
-export GH="${HOME}/bin/gh"
-[[ -f "${GH}" ]] || echo "gh is not installed in ${HOME}/bin"
+GH=$(command -v gh || echo "${HOME}/bin/gh")
+[[ -f "${GH}" ]] || ( echo "ERROR: GitHub CLI (gh) not found. (exiting with error)"; exit 1 )
 ${GH} --version
+export GH
 
 check_mark=$("${GH}" auth status -t 2>&1 | grep "Token:" | awk '{print $1}') || true
 if [[ "${check_mark}" != "✓" ]]; then

env/WCOSS2.env

@@ -123,7 +123,7 @@ elif [[ "${step}" = "marineanlchkpt" ]]; then
 elif [[ "${step}" = "ocnanalecen" ]]; then
 
     export NTHREADS_OCNANALECEN=${NTHREADSmax}
-    export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+    export APRUN_OCNANALECEN="${APRUN_default}"
 
 elif [[ "${step}" = "marineanlletkf" ]]; then
 

parm/archive/enkf_restartb_grp.yaml.j2

@@ -38,5 +38,5 @@ enkf_restartb_grp:
         - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res"
         - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc"
         {% endfor %}
-        {% endfor %}
         {% endif %}
+        {% endfor %}

parm/config/gefs/config.base

@@ -227,8 +227,13 @@ export SDATE_GFS=@SDATE_GFS@
 export REPLAY_ICS=@REPLAY_ICS@
 if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
     export OFFSET_START_HOUR=$(( assim_freq / 2 ))
+    echo "WARNING: Replay ICs require perturbation files, ignoring any previous settings"
+    export USE_OCN_ENS_PERTURB_FILES="YES"
+    export USE_ATM_ENS_PERTURB_FILES="YES"
 else
     export OFFSET_START_HOUR=0
+    export USE_OCN_ENS_PERTURB_FILES=@USE_OCN_ENS_PERTURB_FILES@
+    export USE_ATM_ENS_PERTURB_FILES=@USE_ATM_ENS_PERTURB_FILES@
 fi
 
 # GFS output and frequency

parm/config/gefs/config.efcs

@@ -57,7 +57,7 @@ if (( OCNRES <= 100 )); then
     export EPBL_LSCALE="500.E3,1000.E3,2000.E3,2000.E3,2000.E3"
 fi
 
-if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
+if [[ "${USE_OCN_ENS_PERTURB_FILES:-NO}" == "YES" ]]; then
     export ODA_INCUPD="True"
     export ODA_TEMPINC_VAR='t_pert'
     export ODA_SALTINC_VAR='s_pert'

parm/config/gefs/config.fcst

@@ -192,14 +192,14 @@ case ${imp_physics} in
 
         export hord_mt_nh_nonmono=5
         export hord_xx_nh_nonmono=5
+        export hord_dp_nh_nonmono=-5
         export vtdm4_nh_nonmono=0.02
         export nord=2
         if [[ "${TYPE}" == "nh"* ]]; then
             export dddmp=0.1
         else
             export dddmp=0.
 	    fi
-        export dddmp=0.1
         export d4_bg=0.12
         ;;
     11) # GFDL
@@ -216,6 +216,7 @@ case ${imp_physics} in
 
         export hord_mt_nh_nonmono=5
         export hord_xx_nh_nonmono=5
+        export hord_dp_nh_nonmono=-5
         export vtdm4_nh_nonmono=0.02
         export nord=2
         export d4_bg=0.12

parm/config/gefs/config.nsst

@@ -12,10 +12,6 @@ export NST_MODEL=2
 
 # nstf_name(2) : NST_SPINUP : 0 = OFF, 1 = ON,
 export NST_SPINUP=0
-cdate="${PDY}${cyc}"
-if (( cdate < 2017072000 )); then
-    export NST_SPINUP=1
-fi
 
 # nstf_name(3) : NST_RESV (Reserved, NSST Analysis) : 0 = OFF, 1 = ON
 export NST_RESV=0

parm/config/gefs/config.resources

@@ -219,7 +219,7 @@ case ${step} in
 
     case "${CASE}" in
       "C48" | "C96" | "C192")
-        declare -x "walltime"="03:00:00"
+        declare -x "walltime"="04:00:00"
         ;;
       "C384" | "C768" | "C1152")
         declare -x "walltime"="06:00:00"

parm/config/gefs/config.stage_ic

@@ -32,8 +32,4 @@ if [[ -z "${ICSDIR}" ]] ; then
 
 fi
 
-#use of perturbations files for ensembles
-export USE_OCN_ENS_PERTURB_FILES=@USE_OCN_ENS_PERTURB_FILES@
-export USE_ATM_ENS_PERTURB_FILES=@USE_ATM_ENS_PERTURB_FILES@
-
 echo "END: config.stage_ic"

parm/config/gefs/config.ufs

@@ -99,8 +99,8 @@ case "${fv3_res}" in
         export DELTIM=600
         export layout_x=2
         export layout_y=2
-        export layout_x_gfs=4
-        export layout_y_gfs=4
+        export layout_x_gfs=6
+        export layout_y_gfs=8
         export nthreads_fv3=1
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1

parm/config/gefs/config.wave

@@ -16,28 +16,10 @@ export RUNwave="${RUN}wave"
 export RUNRSTwave="gdas"
 
 #grid dependent variable defaults
-export waveGRDN='1'                   # grid number for ww3_multi
-export waveGRDG='10'                  # grid group for ww3_multi
-export USE_WAV_RMP='NO'               # YES/NO rmp grid remapping pre-processed coefficients
-export waveMULTIGRID='.false.'        # .true./.false. for multi or shel
 export MESH_WAV="mesh.${waveGRD}.nc"  # Mesh grid for wave model for CMEPS
-export waveesmfGRD=' '                # input grid for multigrid
 
 #Grid dependent variables for various grids
 case "${waveGRD}" in
-  "gnh_10m;aoc_9km;gsh_15m")
-    #GFSv16 settings:
-    export waveGRDN='1 2 3'
-    export waveGRDG='10 20 30'
-    export USE_WAV_RMP='YES'
-    export waveMULTIGRID='.true.'
-    export IOSRV='3'
-    export MESH_WAV=' '
-    export waveesmfGRD='glox_10m'
-    export waveuoutpGRD='points'
-    export waveinterpGRD='glo_15mxt at_10m ep_10m wc_10m glo_30m'
-    export wavepostGRD='gnh_10m aoc_9km gsh_15m'
-    ;;
   "gwes_30m")
     #Grid used for P8
     export waveinterpGRD=''
@@ -157,7 +139,7 @@ export WW3CURIENS='F'
 export GOFILETYPE=1     # GOFILETYPE=1 one gridded file per output step
 export POFILETYPE=1     # POFILETYPE=1 one point file per output step
 
-# Parameters for ww3_multi/shel.inp
+# Parameters for ww3_shel.inp
 # Unified output T or F
 export FUNIPNT='T'
 # Output server type (see ww3_shel/multi.inp in WW3 repo)

parm/config/gefs/yaml/defaults.yaml

@@ -14,22 +14,20 @@ base:
   FHOUT_HF_GFS: 1
   FCST_BREAKPOINTS: "48"
   REPLAY_ICS: "NO"
-  USE_OCN_PERTURB_FILES: "false"
   FHOUT_GFS: 6
   FHOUT_OCN_GFS: 6
   FHOUT_ICE_GFS: 6
   HPSSARCH: "NO"
   LOCALARCH: "NO"
   SFS_POST: "NO"
+  USE_OCN_ENS_PERTURB_FILES: "NO"
+  USE_ATM_ENS_PERTURB_FILES: "NO"
   DO_TEST_MODE: "NO"
 fcst:
   reforecast: "NO"
   FHZER: 6
   TYPE: "nh"
   MONO: "non-mono"
-stage_ic:
-  USE_OCN_ENS_PERTURB_FILES: "NO"
-  USE_ATM_ENS_PERTURB_FILES: "NO"
 ocn:
   MOM6_INTERP_ICS: "NO"
 # config.aero has just a system-specific path to add.

parm/config/gfs/config.fcst

@@ -203,6 +203,7 @@ case ${imp_physics} in
 
         export hord_mt_nh_nonmono=5
         export hord_xx_nh_nonmono=5
+        export hord_dp_nh_nonmono=-5
         export vtdm4_nh_nonmono=0.02
         export nord=2
         export dddmp=0.1
@@ -236,6 +237,7 @@ case ${imp_physics} in
 
         export hord_mt_nh_nonmono=5
         export hord_xx_nh_nonmono=5
+        export hord_dp_nh_nonmono=-5
         export vtdm4_nh_nonmono=0.02
         export nord=2
         export d4_bg=0.12

parm/config/gfs/config.wave

@@ -16,28 +16,10 @@ export RUNwave="${RUN}wave"
 export RUNRSTwave="gdas"
 
 #grid dependent variable defaults
-export waveGRDN='1'                   # grid number for ww3_multi
-export waveGRDG='10'                  # grid group for ww3_multi
-export USE_WAV_RMP='NO'               # YES/NO rmp grid remapping pre-processed coefficients
-export waveMULTIGRID='.false.'        # .true./.false. for multi or shel
 export MESH_WAV="mesh.${waveGRD}.nc"  # Mesh grid for wave model for CMEPS
-export waveesmfGRD=' '                # input grid for multigrid
 
 #Grid dependent variables for various grids
 case "${waveGRD}" in
-  "gnh_10m;aoc_9km;gsh_15m")
-    #GFSv16 settings:
-    export waveGRDN='1 2 3'
-    export waveGRDG='10 20 30'
-    export USE_WAV_RMP='YES'
-    export waveMULTIGRID='.true.'
-    export IOSRV='3'
-    export MESH_WAV=' '
-    export waveesmfGRD='glox_10m'
-    export waveuoutpGRD='points'
-    export waveinterpGRD='glo_15mxt at_10m ep_10m wc_10m glo_30m'
-    export wavepostGRD='gnh_10m aoc_9km gsh_15m'
-    ;;
   "gwes_30m")
     #Grid used for P8
     export waveinterpGRD=''
@@ -187,7 +169,7 @@ export WW3CURIENS='F'
 export GOFILETYPE=1     # GOFILETYPE=1 one gridded file per output step
 export POFILETYPE=1     # POFILETYPE=1 one point file per output step
 
-# Parameters for ww3_multi/shel.inp
+# Parameters for ww3_shel.inp
 # Unified output T or F
 export FUNIPNT='T'
 # Output server type (see ww3_shel/multi.inp in WW3 repo)

parm/wmo/grib2_awpgfs000.003

@@ -104,4 +104,3 @@
 &GRIBIDS DESC=' R H      925 mb ',WMOHEAD='YRPA92 KWBC',PDTN= 0 ,PDT=  1 1 2 0 81 0 0 1 0 100 0 92500 255 0 0 /
 &GRIBIDS DESC=' U GRD    925 mb ',WMOHEAD='YUPA92 KWBC',EXTRACT=.true.,PDTN= 0 ,PDT=  2 2 2 0 81 0 0 1 0 100 0 92500 255 0 0 /
 &GRIBIDS DESC=' V GRD    925 mb ',WMOHEAD='YVPA92 KWBC',EXTRACT=.true.,PDTN= 0 ,PDT=  2 3 2 0 81 0 0 1 0 100 0 92500 255 0 0 /
-&GRIBIDS DESC=' 5WAVH    500 mb ',WMOHEAD='YHPA50 KWBC',PDTN= 0 ,PDT=  3 193 2 0 81 0 0 1 0 100 0 50000 255 0 0 /

parm/wmo/grib2_awpgfs006.003

@@ -106,4 +106,3 @@
 &GRIBIDS DESC=' R H      925 mb ',WMOHEAD='YRPB92 KWBC',PDTN= 0 ,PDT=  1 1 2 0 96 0 0 1 6 100 0 92500 255 0 0 /
 &GRIBIDS DESC=' U GRD    925 mb ',WMOHEAD='YUPB92 KWBC',EXTRACT=.true.,PDTN= 0 ,PDT=  2 2 2 0 96 0 0 1 6 100 0 92500 255 0 0 /
 &GRIBIDS DESC=' V GRD    925 mb ',WMOHEAD='YVPB92 KWBC',EXTRACT=.true.,PDTN= 0 ,PDT=  2 3 2 0 96 0 0 1 6 100 0 92500 255 0 0 /
-&GRIBIDS DESC=' 5WAVH    500 mb ',WMOHEAD='YHPB50 KWBC',PDTN= 0 ,PDT=  3 193 2 0 96 0 0 1 6 100 0 50000 255 0 0 /

parm/wmo/grib2_awpgfs012.003

@@ -106,4 +106,3 @@
 &GRIBIDS DESC=' R H      925 mb ',WMOHEAD='YRPC92 KWBC',PDTN= 0 ,PDT=  1 1 2 0 96 0 0 1 12 100 0 92500 255 0 0 /
 &GRIBIDS DESC=' U GRD    925 mb ',WMOHEAD='YUPC92 KWBC',EXTRACT=.true.,PDTN= 0 ,PDT=  2 2 2 0 96 0 0 1 12 100 0 92500 255 0 0 /
 &GRIBIDS DESC=' V GRD    925 mb ',WMOHEAD='YVPC92 KWBC',EXTRACT=.true.,PDTN= 0 ,PDT=  2 3 2 0 96 0 0 1 12 100 0 92500 255 0 0 /
-&GRIBIDS DESC=' 5WAVH    500 mb ',WMOHEAD='YHPC50 KWBC',PDTN= 0 ,PDT=  3 193 2 0 96 0 0 1 12 100 0 50000 255 0 0 /