Splitting up the archive task

.github/CODEOWNERS

@@ -76,7 +76,8 @@ scripts/exgdas_atmos_gempak_gif_ncdc.sh @GwenChen-NOAA
 scripts/exgdas_atmos_nawips.sh @GwenChen-NOAA
 scripts/exgdas_atmos_verfozn.sh @EdwardSafford-NOAA
 scripts/exgdas_atmos_verfrad.sh @EdwardSafford-NOAA
-scripts/exgdas_enkf_earc.py @DavidHuber-NOAA
+scripts/exgdas_enkf_earc_vrfy.py @DavidHuber-NOAA
+scripts/exgdas_enkf_earc_tars.py @DavidHuber-NOAA
 scripts/exgdas_enkf_ecen.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA
 scripts/exgdas_enkf_post.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA
 scripts/exgdas_enkf_select_obs.sh @CoryMartin-NOAA @RussTreadon-NOAA @CatherineThomas-NOAA

docs/source/jobs.rst

@@ -18,7 +18,7 @@ An experimental run is different from operations in the following ways:
 
 * Addition steps in experimental mode:
 
-  - archive (arch)
+  - archive (arch_vrfy & arch_tars)
 
   - cleanup (cleanup)
 
@@ -39,7 +39,9 @@ Jobs in the GFS Configuration
 | analdiag          | Creates netCDF diagnostic files containing observation values, innovation (O-F), error, quality control, as well as   |
 |                   | other analysis-related quantities (cnvstat, radstat, ozstat files).                                                   |
 +-------------------+-----------------------------------------------------------------------------------------------------------------------+
-| arch              | Archives select files from the deterministic model and cleans up older data.                                          |
+|arch_tars          | Optional archive job that backs up the COM data structure.                                                            |
++-------------------+-----------------------------------------------------------------------------------------------------------------------+
+|arch_vrfy          | Archives select files from the deterministic model and cleans up older data.                                          |
 +-------------------+-----------------------------------------------------------------------------------------------------------------------+
 | earcN/eamn        | Archival script for EnKF: 1) Write select EnKF output to HPSS; 2) Copy select files to online archive; 3) Clean up    |
 |                   | EnKF temporary run directories; 4) Remove "old" EnKF files from rotating directory.                                   |

jobs/JGDAS_ENKF_ARCHIVE → jobs/JGDAS_ENKF_ARCHIVE_TARS

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "earc" -c "base earc"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_tars" -c "base earc_tars"
 
 
 ##############################################
@@ -17,7 +17,7 @@ MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
 # Run archive script
 ###############################################################
 
-"${SCRgfs}/exgdas_enkf_earc.py"
+"${SCRgfs}/exgdas_enkf_earc_tars.py"
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"
 

jobs/JGDAS_ENKF_ARCHIVE_VRFY

@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "earc_vrfy" -c "base earc_vrfy"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_TOP
+MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+  COMIN_ATMOS_ANALYSIS_ENSSTAT:COM_ATMOS_ANALYSIS_TMPL \
+  COMIN_ATMOS_HISTORY_ENSSTAT:COM_ATMOS_HISTORY_TMPL \
+  COMIN_SNOW_ANALYSIS_ENSSTAT:COM_SNOW_ANALYSIS_TMPL
+
+###############################################################
+# Run archive script
+###############################################################
+
+"${SCRgfs}/exgdas_enkf_earc_vrfy.py"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+###############################################################
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/JGLOBAL_ARCHIVE → jobs/JGLOBAL_ARCHIVE_TARS

@@ -1,48 +1,49 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "arch" -c "base arch wave"
-source "${USHgfs}/wave_domain_grid.sh" 
+source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_tars" -c "base arch_tars wave"
+source "${USHgfs}/wave_domain_grid.sh"
+
 ##############################################
 # Set variables used in the script
 ##############################################
 YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
-	COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
-	COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
-	COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \
-	COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \
-	COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
-	COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \
-	COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \
-	COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \
-	COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \
-	COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \
-	COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \
-	COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
-	COMIN_MED_RESTART:COM_MED_RESTART_TMPL \
-	COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
-	COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \
-	COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \
-	COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \
-	COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \
-	COMIN_OBS:COM_OBS_TMPL \
-	COMIN_TOP:COM_TOP_TMPL \
-	COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \
-	COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \
-	COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \
-	COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \
-	COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
-        COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \
-        COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \
-	COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \
-	COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \
-	COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \
-	COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \
-	COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \
-	COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \
-	COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \
-	COMIN_CONF:COM_CONF_TMPL \
-	COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL
+    COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+    COMIN_ATMOS_BUFR:COM_ATMOS_BUFR_TMPL \
+    COMIN_ATMOS_GEMPAK:COM_ATMOS_GEMPAK_TMPL \
+    COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \
+    COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
+    COMIN_ATMOS_INPUT:COM_ATMOS_INPUT_TMPL \
+    COMIN_ATMOS_MASTER:COM_ATMOS_MASTER_TMPL \
+    COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL \
+    COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \
+    COMIN_ATMOS_WMO:COM_ATMOS_WMO_TMPL \
+    COMIN_CHEM_HISTORY:COM_CHEM_HISTORY_TMPL \
+    COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+    COMIN_MED_RESTART:COM_MED_RESTART_TMPL \
+    COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
+    COMIN_ICE_HISTORY:COM_ICE_HISTORY_TMPL \
+    COMIN_ICE_INPUT:COM_ICE_INPUT_TMPL \
+    COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \
+    COMIN_ICE_GRIB:COM_ICE_GRIB_TMPL \
+    COMIN_OBS:COM_OBS_TMPL \
+    COMIN_TOP:COM_TOP_TMPL \
+    COMIN_OCEAN_HISTORY:COM_OCEAN_HISTORY_TMPL \
+    COMIN_OCEAN_RESTART:COM_OCEAN_RESTART_TMPL \
+    COMIN_OCEAN_GRIB:COM_OCEAN_GRIB_TMPL \
+    COMIN_OCEAN_NETCDF:COM_OCEAN_NETCDF_TMPL \
+    COMIN_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
+    COMIN_OCEAN_BMATRIX:COM_OCEAN_BMATRIX_TMPL \
+    COMIN_ICE_BMATRIX:COM_ICE_BMATRIX_TMPL \
+    COMIN_WAVE_GRID:COM_WAVE_GRID_TMPL \
+    COMIN_WAVE_HISTORY:COM_WAVE_HISTORY_TMPL \
+    COMIN_WAVE_STATION:COM_WAVE_STATION_TMPL \
+    COMIN_WAVE_RESTART:COM_WAVE_RESTART_TMPL \
+    COMIN_ATMOS_OZNMON:COM_ATMOS_OZNMON_TMPL \
+    COMIN_ATMOS_RADMON:COM_ATMOS_RADMON_TMPL \
+    COMIN_ATMOS_MINMON:COM_ATMOS_MINMON_TMPL \
+    COMIN_CONF:COM_CONF_TMPL \
+    COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL
 
 for grid in "0p25" "0p50" "1p00"; do
     YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \
@@ -70,7 +71,7 @@ fi
 # Run archive script
 ###############################################################
 
-${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive.py}
+${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive_tars.py}
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"
 

jobs/JGLOBAL_ARCHIVE_VRFY

@@ -0,0 +1,50 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "arch_vrfy" -c "base arch_vrfy wave"
+
+
+##############################################
+# Set variables used in the script
+##############################################
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+    COMIN_ATMOS_GENESIS:COM_ATMOS_GENESIS_TMPL \
+    COMIN_ATMOS_HISTORY:COM_ATMOS_HISTORY_TMPL \
+    COMIN_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL \
+    COMIN_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+    COMIN_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL \
+    COMIN_OBS:COM_OBS_TMPL \
+    COMOUT_ATMOS_TRACK:COM_ATMOS_TRACK_TMPL
+
+for grid in "0p25" "0p50" "1p00"; do
+    YMD=${PDY} HH=${cyc} GRID=${grid} declare_from_tmpl -rx \
+       "COMIN_ATMOS_GRIB_${grid}:COM_ATMOS_GRIB_GRID_TMPL"
+done
+
+###############################################################
+# Run archive script
+###############################################################
+
+${GLOBALARCHIVESH:-${SCRgfs}/exglobal_archive_vrfy.py}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || (echo "${DATAROOT} does not exist. ABORT!"; exit 1)
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/rocoto/arch.sh → jobs/rocoto/arch_tars.sh

@@ -13,12 +13,12 @@ status=$?
 PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
 export PYTHONPATH
 
-export job="arch"
+export job="arch_tars"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-"${HOMEgfs}"/jobs/JGLOBAL_ARCHIVE
+"${HOMEgfs}"/jobs/JGLOBAL_ARCHIVE_TARS
 status=$?
 
 exit "${status}"

jobs/rocoto/arch_vrfy.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+. "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
+export PYTHONPATH
+
+export job="arch_vrfy"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGLOBAL_ARCHIVE_VRFY
+status=$?
+
+exit "${status}"

jobs/rocoto/earc.sh → jobs/rocoto/earc_tars.sh

@@ -13,12 +13,12 @@ status=$?
 PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
 export PYTHONPATH
 
-export job="earc"
+export job="earc_tars"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-"${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE"
+"${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE_TARS"
 status=$?
 
 exit "${status}"

jobs/rocoto/earc_vrfy.sh

@@ -0,0 +1,24 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+. "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+###############################################################
+# setup python path for workflow utilities and tasks
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/ush/python"
+export PYTHONPATH
+
+export job="earc_vrfy"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGDAS_ENKF_ARCHIVE_VRFY"
+status=$?
+
+exit "${status}"

parm/archive/gfsa.yaml.j2

@@ -6,7 +6,7 @@ gfsa:
         # Logs
         # TODO explicitly name all logs to include
         {% for log in glob("logs/" ~ cycle_YMDH ~ "/gfs*.log") %}
-        {% if not "gfs_arch.log" in log %}
+        {% if not "gfs_arch_tars.log" in log %}
         - "{{ log }}"
         {% endif %}
         {% endfor %}

parm/config/gefs/config.arch → parm/config/gefs/config.arch_tars

@@ -1,15 +1,15 @@
 #! /usr/bin/env bash
 
-########## config.arch ##########
+########## config.arch_tars ##########
 # Archive specific
 
-echo "BEGIN: config.arch"
+echo "BEGIN: config.arch_tars"
 
 # Get task specific resources
-. "${EXPDIR}/config.resources" arch
+. "${EXPDIR}/config.resources" arch_tars
 
 export ARCH_GAUSSIAN="YES"
 export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS}
 export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS}
 
-echo "END: config.arch"
+echo "END: config.arch_tars"

parm/config/gfs/config.arch → parm/config/gefs/config.arch_vrfy

@@ -1,15 +1,15 @@
 #! /usr/bin/env bash
 
-########## config.arch ##########
+########## config.arch_vrfy ##########
 # Archive specific
 
-echo "BEGIN: config.arch"
+echo "BEGIN: config.arch_vrfy"
 
 # Get task specific resources
-. "${EXPDIR}/config.resources" arch
+. "${EXPDIR}/config.resources" arch_vrfy
 
 export ARCH_GAUSSIAN="YES"
 export ARCH_GAUSSIAN_FHMAX=${FHMAX_GFS}
 export ARCH_GAUSSIAN_FHINC=${FHOUT_GFS}
 
-echo "END: config.arch"
+echo "END: config.arch_vrfy"

parm/config/gefs/config.base

@@ -334,11 +334,16 @@ export DO_METP="NO"         # Run METPLUS jobs - set METPLUS settings in config.
 export DO_FIT2OBS="NO"      # Run fit to observations package
 
 # Archiving options
+export VRFYARCH="@VRFYARCH@"        # save verification data locally
 export HPSSARCH="@HPSSARCH@"        # save data to HPSS archive
 export LOCALARCH="@LOCALARCH@"        # save data to local archive
 if [[ ${HPSSARCH} = "YES" ]] && [[ ${LOCALARCH} = "YES" ]]; then
    echo "Both HPSS and local archiving selected.  Please choose one or the other."
    exit 3
+elif [[ ${HPSSARCH} = "YES" ]] || [[ ${LOCALARCH} = "YES" ]]; then
+   export DO_ARCHTAR="YES"
+else
+   export DO_ARCHTAR="NO"
 fi
 export ARCH_CYC=00           # Archive data at this cycle for warm start and/or forecast-only capabilities
 export ARCH_WARMICFREQ=4     # Archive frequency in days for warm start capability
@@ -348,8 +353,6 @@ export ARCH_EXPDIR_FREQ=0    # How often to archive the EXPDIR in hours or 0 for
 export ARCH_HASHES='YES'     # Archive the hashes of the GW and submodules and 'git status' for each; requires ARCH_EXPDIR
 export ARCH_DIFFS='NO'       # Archive the output of 'git diff' for the GW; requires ARCH_EXPDIR
 
-export DELETE_COM_IN_ARCHIVE_JOB="YES"   # NO=retain ROTDIR.  YES default in arch.sh and earc.sh.
-
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}