diff --git a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml index 4147249a4c..21aece76a5 100644 --- a/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml @@ -8,12 +8,13 @@ arguments: resdetatmos: 384 resdetocean: 0.25 nens: 0 - interval: 6 + interval: 0 start: warm comroot: {{ 'RUNTESTS' | getenv }}/COMROOT expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021063018 edate: 2021070306 #icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2 - icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/ yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marine3dvar.yaml diff --git a/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml b/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml index f0e0b42c28..57680ffd02 100644 --- a/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml +++ b/ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml @@ -15,5 +15,6 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021063018 edate: 2021070306 - icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + #icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/ + icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/hybrid-test/ yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marinehyb.yaml diff --git a/ci/cases/gfsv17/marine3dvar.yaml b/ci/cases/gfsv17/marine3dvar.yaml index abf86f0aa8..da4d4fb599 100644 --- a/ci/cases/gfsv17/marine3dvar.yaml +++ b/ci/cases/gfsv17/marine3dvar.yaml @@ -17,7 +17,12 @@ base: ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} DO_TEST_MODE: "YES" +prepoceanobs: + use_exp_obs: "YES" + dmpdir_exp: /scratch1/NCEPDEV/da/common/ + marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca + SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml SOCA_NINNER: 100 diff --git a/ci/cases/gfsv17/marinehyb.yaml b/ci/cases/gfsv17/marinehyb.yaml index ed62e607a8..72515088e8 100644 --- a/ci/cases/gfsv17/marinehyb.yaml +++ b/ci/cases/gfsv17/marinehyb.yaml @@ -16,6 +16,11 @@ base: FHMAX_GFS: 240 ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }} +prepoceanobs: + use_exp_obs: "YES" + dmpdir_exp: /scratch1/NCEPDEV/da/common/ + marineanl: SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca - SOCA_NINNER: 20 # revert to ~100 after the memory leak is fixed + SOCA_ANL_GEOM: {{ HOMEgfs }}/fix/gdas/soca/720x540x75/soca + SOCA_NINNER: 100 diff --git a/parm/config/gfs/config.marineanl b/parm/config/gfs/config.marineanl index b0fbbedb54..e3dcc4529e 100644 --- a/parm/config/gfs/config.marineanl +++ b/parm/config/gfs/config.marineanl @@ -11,6 +11,7 @@ export JCB_ALGO_YAML_VAR=@JCB_ALGO_YAML_VAR@ export MARINE_OBS_YAML_DIR="${PARMgfs}/gdas/soca/obs/config" export MARINE_OBS_LIST_YAML=@SOCA_OBS_LIST@ export SOCA_INPUT_FIX_DIR=@SOCA_INPUT_FIX_DIR@ +export SOCA_ANL_GEOM=@SOCA_ANL_GEOM@ export SOCA_NINNER=@SOCA_NINNER@ export DOMAIN_STACK_SIZE=116640000 #TODO: Make the stack size resolution dependent export SOCA_ENS_BKG_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/soca_ens_bkg_stage.yaml.j2" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index aac06ff03c..50c88d1a5c 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -504,7 +504,7 @@ case ${step} in walltime="00:10:00" ntasks=1 threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} memory="24GB" ;; @@ -512,7 +512,7 @@ case ${step} in walltime="00:10:00" ntasks=1 threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} memory="48GB" ;; @@ -520,7 +520,10 @@ case ${step} in npes=16 ntasks=16 case ${OCNRES} in - "025") ntasks=480;; + "025") + ntasks=480 + memory="256GB" + ;; "050") ntasks=16;; "100") ntasks=16;; "500") ntasks=16;; @@ -532,7 +535,7 @@ case ${step} in walltime="00:30:00" threads_per_task=1 export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; "marineanlvar") @@ -540,7 +543,7 @@ case ${step} in case ${OCNRES} in "025") ntasks=480 - memory="96GB" + memory="256GB" ;; "050") ntasks=16 @@ -559,10 +562,10 @@ case ${step} in exit 4 esac - walltime="00:15:00" + walltime="00:30:00" threads_per_task=1 export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; "ocnanalecen") @@ -570,7 +573,7 @@ case ${step} in case ${OCNRES} in "025") ntasks=40 - memory="96GB" + memory="256GB" ;; "050") ntasks=16 @@ -590,9 +593,9 @@ case ${step} in esac walltime="00:10:00" - threads_per_task=1 export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; "marineanlletkf") @@ -600,7 +603,7 @@ case ${step} in case ${OCNRES} in "025") ntasks=480 - memory="96GB" + memory="256GB" ;; "050") ntasks=16 @@ -619,10 +622,10 @@ case ${step} in exit 4 esac - walltime="00:10:00" - threads_per_task=1 + walltime="00:30:00" export is_exclusive=True - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / 2 )) ;; @@ -630,7 +633,7 @@ case ${step} in walltime="00:10:00" ntasks=1 threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} case ${OCNRES} in "025") memory="128GB" @@ -654,7 +657,7 @@ case ${step} in walltime="00:30:00" ntasks=${max_tasks_per_node} threads_per_task=1 - tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + tasks_per_node=${max_tasks_per_node} ;; "ocnanalvrfy") diff --git a/parm/config/gfs/yaml/defaults.yaml b/parm/config/gfs/yaml/defaults.yaml index 55f4b03f50..9018142502 100644 --- a/parm/config/gfs/yaml/defaults.yaml +++ b/parm/config/gfs/yaml/defaults.yaml @@ -54,6 +54,7 @@ snowanl: marineanl: SOCA_INPUT_FIX_DIR: "${FIXgfs}/gdas/soca/72x35x25/soca" + SOCA_ANL_GEOM: "${FIXgfs}/gdas/soca/72x35x25/soca" SOCA_OBS_LIST: "${PARMgfs}/gdas/soca/obs/obs_list.yaml" # TODO: This is also repeated in oceanprepobs SOCA_NINNER: 100 JCB_ALGO_YAML_VAR: "${PARMgfs}/gdas/soca/marine-jcb-3dfgat.yaml.j2" diff --git a/parm/gdas/soca_bmat_jedi_config.yaml.j2 b/parm/gdas/soca_bmat_jedi_config.yaml.j2 index 4e476d3117..9517626b98 100644 --- a/parm/gdas/soca_bmat_jedi_config.yaml.j2 +++ b/parm/gdas/soca_bmat_jedi_config.yaml.j2 @@ -8,35 +8,42 @@ soca_diagb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_diagb.x' mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_diagb +soca_chgres: + rundir: '{{ DATA }}' + exe_src: '{{ EXECgfs }}/gdas.x' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jedi_args: ['soca', 'convertstate'] + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_chgres soca_parameters_diffusion_vt: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_parameters_diffusion_vt soca_setcorscales: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_setcorscales.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_setcorscales soca_parameters_diffusion_hz: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_soca_error_covariance_toolbox.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_parameters_diffusion_hz soca_ensb: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_ens_handler.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' jcb_algo: soca_ensb soca_ensweights: rundir: '{{ DATA }}' exe_src: '{{ EXECgfs }}/gdas_socahybridweights.x' - mpi_cmd: '{{ APRUN_MARINEBMAT }}' - jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' - jcb_algo: soca_ensweights + mpi_cmd: '{{ APRUN_MARINEBMAT }}' + jcb_base_yaml: '{{ PARMgfs }}/gdas/soca/marine-jcb-base.yaml' + jcb_algo: soca_ensweights diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 15113adfdb..2f12db42f2 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 15113adfdbf2500ec2d5099fc9b62b21fbfcc9b8 +Subproject commit 2f12db42f2c2c3bfcfe5fb84ae839496549c0eb2 diff --git a/ush/python/pygfs/task/marine_analysis.py b/ush/python/pygfs/task/marine_analysis.py index dd48c9f80f..bd61d8bc5a 100644 --- a/ush/python/pygfs/task/marine_analysis.py +++ b/ush/python/pygfs/task/marine_analysis.py @@ -179,9 +179,13 @@ def _prep_scratch_dir(self: Task) -> None: soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_list).sync() - # prepare the MOM6 input.nml + # prepare the deterministic MOM6 input.nml mdau.prep_input_nml(self.task_config) + # prepare the input.nml for the analysis geometry + mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml", + simple_geom=True, mom_input="./anl_geom/MOM_input") + # stage the soca utility yamls (gridgen, fields and ufo mapping yamls) logger.info(f"Staging SOCA utility yaml files from {self.task_config.PARMsoca}") soca_utility_list = parse_j2yaml(self.task_config.MARINE_UTILITY_YAML_TMPL, self.task_config) @@ -206,7 +210,7 @@ def _prep_variational_yaml(self: Task) -> None: envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs envconfig_jcb['NMEM_ENS'] = self.task_config.NMEM_ENS envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion' - if self.task_config.NMEM_ENS >= 3: + if self.task_config.NMEM_ENS >= 2: envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion' envconfig_jcb['DATA'] = self.task_config.DATA envconfig_jcb['OPREFIX'] = self.task_config.OPREFIX @@ -364,7 +368,7 @@ def list_all_files(dir_in, dir_out, wc='*', fh_list=[]): post_file_list = [] # Make a copy the IAU increment - post_file_list.append([os.path.join(anl_dir, 'inc.nc'), + post_file_list.append([os.path.join(anl_dir, 'ocn.inc.nc'), os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocninc.nc')]) domains = ['ocn', 'ice'] diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 8e2b84a673..e64c1527c3 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -74,7 +74,7 @@ def __init__(self, config): # Create dictionary of Jedi objects expected_keys = ['gridgen', 'soca_diagb', 'soca_parameters_diffusion_vt', 'soca_setcorscales', - 'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights'] + 'soca_parameters_diffusion_hz', 'soca_ensb', 'soca_ensweights', 'soca_chgres'] self.jedi_dict = Jedi.get_jedi_dict(self.task_config.JEDI_CONFIG_YAML, self.task_config, expected_keys) @logit(logger) @@ -105,9 +105,13 @@ def initialize(self: Task) -> None: soca_fix_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_list).sync() - # prepare the MOM6 input.nml + # prepare the deterministic MOM6 input.nml mdau.prep_input_nml(self.task_config) + # prepare the input.nml for the analysis geometry + mdau.prep_input_nml(self.task_config, output_nml="./anl_geom/mom_input.nml", + simple_geom=True, mom_input="./anl_geom/MOM_input") + # stage backgrounds # TODO(G): Check ocean backgrounds dates for consistency bkg_list = parse_j2yaml(self.task_config.MARINE_DET_STAGE_BKG_YAML_TMPL, self.task_config) @@ -127,6 +131,7 @@ def initialize(self: Task) -> None: # initialize JEDI applications self.jedi_dict['gridgen'].initialize(self.task_config) self.jedi_dict['soca_diagb'].initialize(self.task_config) + self.jedi_dict['soca_chgres'].initialize(self.task_config) self.jedi_dict['soca_parameters_diffusion_vt'].initialize(self.task_config) self.jedi_dict['soca_setcorscales'].initialize(self.task_config) self.jedi_dict['soca_parameters_diffusion_hz'].initialize(self.task_config) @@ -163,11 +168,15 @@ def execute(self) -> None: None """ + # soca grid generation self.jedi_dict['gridgen'].execute() # variance partitioning self.jedi_dict['soca_diagb'].execute() + # Interpolate f009 bkg to analysis geometry + self.jedi_dict['soca_chgres'].execute() + # horizontal diffusion self.jedi_dict['soca_setcorscales'].execute() self.jedi_dict['soca_parameters_diffusion_hz'].execute() diff --git a/ush/python/pygfs/utils/marine_da_utils.py b/ush/python/pygfs/utils/marine_da_utils.py index 50d9d84e86..748b5ff9c9 100644 --- a/ush/python/pygfs/utils/marine_da_utils.py +++ b/ush/python/pygfs/utils/marine_da_utils.py @@ -45,7 +45,10 @@ def link_executable(task_config: AttrDict, exe_name: str) -> None: @logit(logger) -def prep_input_nml(task_config: AttrDict) -> None: +def prep_input_nml(task_config: AttrDict, + output_nml: str = "mom_input.nml", + simple_geom: bool = False, + mom_input: str = "MOM_input") -> None: """Prepare the mom_input.nml file """ # stage input.nml.j2 @@ -56,9 +59,11 @@ def prep_input_nml(task_config: AttrDict) -> None: # swap date and stacksize date_init = [int(s) for s in task_config.MARINE_WINDOW_END.strftime('%Y,%m,%d,%H,%M,%S').split(',')] input_nml_config = {'domain_stack_size': task_config.DOMAIN_STACK_SIZE, - 'date_init': date_init} + 'date_init': date_init, + 'simple_geom': simple_geom, + 'mom_input': mom_input} jinja_input_nml = jinja.Jinja(mom_input_nml_tmpl, input_nml_config) - jinja_input_nml.save('mom_input.nml') + jinja_input_nml.save(output_nml) @logit(logger)