From 298c95cdfa68d8c1d93fb9762dbadb93785a9961 Mon Sep 17 00:00:00 2001 From: Ingrid Carlson Date: Fri, 19 Jul 2024 14:35:42 -0600 Subject: [PATCH 1/8] Create cupid workflow Created template.cupid file and added a workflow to run it. Added module load conda to run successfully for. Added a load conda command to the config_machines.xml file for derecho. Updated the scratch path on casper. --- machines/casper/config_machines.xml | 4 ++-- machines/config_workflow.xml | 13 +++++++++++++ machines/derecho/config_machines.xml | 1 + machines/template.cupid | 10 ++++++++++ 4 files changed, 26 insertions(+), 2 deletions(-) create mode 100755 machines/template.cupid diff --git a/machines/casper/config_machines.xml b/machines/casper/config_machines.xml index afd46f00..799b9f34 100644 --- a/machines/casper/config_machines.xml +++ b/machines/casper/config_machines.xml @@ -3,7 +3,7 @@ LINUX nvhpc,intel openmpi - /glade/scratch/$USER + /glade/derecho/scratch/$USER $ENV{CESMDATAROOT}/inputdata /glade/p/cgd/tss/CTSM_datm_forcing_data $CIME_OUTPUT_ROOT/archive/$CASE @@ -108,7 +108,7 @@ /glade/u/apps/dav/modulefiles/default/compilers:/glade/u/apps/dav/modulefiles/default/idep 256M - /glade/scratch/$USER + /glade/derecho/scratch/$USER /glade/p/cesmdata/cseg $ENV{NETCDF} diff --git a/machines/config_workflow.xml b/machines/config_workflow.xml index fe10bbd2..7753f0f0 100644 --- a/machines/config_workflow.xml +++ b/machines/config_workflow.xml @@ -68,6 +68,19 @@ + + + + case.st_archive + 1 + + 1 + 1 + 0:20:00 + + + + diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 12446f37..93f376fe 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -41,6 +41,7 @@ cesmdev/1.0 ncarenv/23.09 + conda/latest craype diff --git a/machines/template.cupid b/machines/template.cupid new file mode 100755 index 00000000..fd2b30f4 --- /dev/null +++ b/machines/template.cupid @@ -0,0 +1,10 @@ +#!/bin/bash + +# Batch system directives +{{ batchdirectives }} + +module load conda +conda activate cupid-dev +./generate_cupid_config_file.py + +cupid-run From 4834b4ce1ad3e8e9de8edb46cf77a041dd01b75e Mon Sep 17 00:00:00 2001 From: Ingrid Carlson Date: Wed, 24 Jul 2024 11:11:32 -0600 Subject: [PATCH 2/8] Get rid of code that loads conda and add code that will ensure the commands in the template are run with the correct environment set up. Add module load nco to config_machines.xml file --- machines/derecho/config_machines.xml | 3 ++- machines/template.cupid | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index 93f376fe..e0b91114 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -41,7 +41,8 @@ cesmdev/1.0 ncarenv/23.09 - conda/latest + conda/latest + nco craype diff --git a/machines/template.cupid b/machines/template.cupid index fd2b30f4..fc0d244b 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -3,7 +3,7 @@ # Batch system directives {{ batchdirectives }} -module load conda +source .env_mach_specific.sh conda activate cupid-dev ./generate_cupid_config_file.py From 95ae795cbe5a5aad0c50e9765859088b1944494c Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Sat, 7 Sep 2024 09:51:42 -0600 Subject: [PATCH 3/8] Move generate_cupid_config_file into CUPiD template.cupid assumes CUPiD is in the CESM source tree (tools/CUPiD) and runs the generate script from there. Also cleaned up some white space --- machines/derecho/config_machines.xml | 4 ++-- machines/template.cupid | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/machines/derecho/config_machines.xml b/machines/derecho/config_machines.xml index e0b91114..a5af18dc 100644 --- a/machines/derecho/config_machines.xml +++ b/machines/derecho/config_machines.xml @@ -41,8 +41,8 @@ cesmdev/1.0 ncarenv/23.09 - conda/latest - nco + conda/latest + nco craype diff --git a/machines/template.cupid b/machines/template.cupid index fc0d244b..fd1da83e 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -5,6 +5,6 @@ source .env_mach_specific.sh conda activate cupid-dev -./generate_cupid_config_file.py +{{ srcroot }}/tools/CUPiD/cesm_scripts/generate_cupid_config_file.py -cupid-run +cupid-run From cb6b713ac4f676af4702273120a1214c8c3691fe Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Sun, 8 Sep 2024 12:38:38 -0600 Subject: [PATCH 4/8] Update template.cupid call to generate config generate_cupid_config_file.py now expects some commandline arguments, so the cupid template as been modified accordingly. --- machines/template.cupid | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/machines/template.cupid b/machines/template.cupid index fd1da83e..144f2dfa 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -5,6 +5,9 @@ source .env_mach_specific.sh conda activate cupid-dev -{{ srcroot }}/tools/CUPiD/cesm_scripts/generate_cupid_config_file.py +{{ srcroot }}/tools/CUPiD/cesm_scripts/generate_cupid_config_file.py \ + --case {{ CASE }} \ + --cesm-output-dir {{ DOUT_S_ROOT }} \ + --cupid-root {{ SRCROOT }}/tools/CUPiD cupid-run From bbfb13330c65d9a14801eca4885780043f302802 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Mon, 9 Sep 2024 17:14:09 -0600 Subject: [PATCH 5/8] Changed API for CUPiD script --- machines/template.cupid | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index 144f2dfa..b0b464cf 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -5,9 +5,7 @@ source .env_mach_specific.sh conda activate cupid-dev -{{ srcroot }}/tools/CUPiD/cesm_scripts/generate_cupid_config_file.py \ - --case {{ CASE }} \ - --cesm-output-dir {{ DOUT_S_ROOT }} \ - --cupid-root {{ SRCROOT }}/tools/CUPiD +{{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ + --cesm-root {{ SRCROOT }} cupid-run From 48d78fb3f0cf4806e1fcc4afb1afd4027f955408 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Wed, 18 Dec 2024 10:38:56 -0700 Subject: [PATCH 6/8] Updates for CUPiD API changes --- machines/template.cupid | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index b0b464cf..ed4a4a6f 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -4,8 +4,9 @@ {{ batchdirectives }} source .env_mach_specific.sh -conda activate cupid-dev +conda activate cupid-infrastructure {{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ --cesm-root {{ SRCROOT }} -cupid-run +cupid-timeseries +cupid-diagnostics From 4b2ac3b0dd4d7284a0f7458c1830ac0386d47fb8 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 19 Dec 2024 15:55:20 -0700 Subject: [PATCH 7/8] Add a few local variables to template.cupid These will eventually be populated from an env_?.xml file, but for now starting a list of variables we want the user to be able to set. Also added two minor updates: 1. unset PYTHONPATH to ensure we are only using conda-installed python packages 2. The API for the script to generate the ADF config file has been changed - we now point to the directory containing a CUPiD config file rather than the CUPiD example --- machines/config_workflow.xml | 1 + machines/template.cupid | 45 ++++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/machines/config_workflow.xml b/machines/config_workflow.xml index 7753f0f0..7f6f2fc3 100644 --- a/machines/config_workflow.xml +++ b/machines/config_workflow.xml @@ -71,6 +71,7 @@ + false case.st_archive 1 diff --git a/machines/template.cupid b/machines/template.cupid index ed4a4a6f..3ac1ae4d 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -1,12 +1,47 @@ -#!/bin/bash +#!/bin/bash -e # Batch system directives {{ batchdirectives }} +# Set environment for CESM source .env_mach_specific.sh + +# Use cupid-infrastructure environment for running these scripts +# Note: on derecho, the cesmdev module creates a python conflict +# by setting $PYTHONPATH; since this is conda-based we +# want an empty PYTHONPATH environment variable +MACH=`./xmlquery --value MACH` +unset PYTHONPATH conda activate cupid-infrastructure -{{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ - --cesm-root {{ SRCROOT }} -cupid-timeseries -cupid-diagnostics +# Set variables that should eventually come from environment file +CUPID_EXAMPLE=key_metrics +RUN_CUPID_ANALYSIS=TRUE +RUN_CUPID_TIMESERIES=TRUE + +if [ "${RUN_CUPID_ANALYSIS}" == "TRUE" ]; then + # 1. Generate CUPiD config file + {{ srcroot }}/tools/CUPiD/helper_scripts/generate_cupid_config_for_cesm_case.py \ + --cesm-root {{ SRCROOT }} + + # 2. Generate ADF config file + {{ srcroot }}/tools/CUPiD/helper_scripts/generate_adf_config_file.py \ + --cesm-root {{ SRCROOT }} \ + --cupid-config-loc ${PWD} \ + --adf-template {{ SRCROOT }}/tools/CUPiD/externals/ADF/config_amwg_default_plots.yaml \ + --out-file adf_config.yml + + # 3. Generate timeseries files and run ADF + if [ "${RUN_CUPID_TIMESERIES}" == "TRUE" ]; then + cupid-timeseries + fi + conda deactivate + conda activate cupid-analysis + {{ SRCROOT }}/tools/CUPiD/externals/ADF/run_adf_diag adf_config.yml + + # 4. Run CUPiD and build webpage + conda deactivate + conda activate cupid-infrastructure + cupid-diagnostics + cupid-webpage +fi From d3cdbd0bfc5dd9addda01449817a96d74c8b86f8 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Mon, 30 Dec 2024 15:03:52 -0700 Subject: [PATCH 8/8] Call cupid scripts directly Don't assume that (cupid-infrastructure) is pointing to the right CUPiD version --- machines/template.cupid | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/machines/template.cupid b/machines/template.cupid index 3ac1ae4d..8b1b94e2 100755 --- a/machines/template.cupid +++ b/machines/template.cupid @@ -33,7 +33,7 @@ if [ "${RUN_CUPID_ANALYSIS}" == "TRUE" ]; then # 3. Generate timeseries files and run ADF if [ "${RUN_CUPID_TIMESERIES}" == "TRUE" ]; then - cupid-timeseries + {{ srcroot }}/tools/CUPiD/cupid/run_timeseries.py fi conda deactivate conda activate cupid-analysis @@ -42,6 +42,6 @@ if [ "${RUN_CUPID_ANALYSIS}" == "TRUE" ]; then # 4. Run CUPiD and build webpage conda deactivate conda activate cupid-infrastructure - cupid-diagnostics - cupid-webpage + {{ srcroot }}/tools/CUPiD/cupid/run_diagnostics.py + {{ srcroot }}/tools/CUPiD/cupid/cupid_webpage.py fi