initial setup for panda testing of PDR2

examples/common_setup.sh

@@ -2,17 +2,12 @@
 
 EXAMPLES=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 
-# Use this for tests or things you're not quite sure of
-root_coll="u/${USER}/cm"
-# Use this for runs you want in the butler in the traditional place
-#root_coll="HSC/runs/RC2"
-#root_coll="2.2i/runs/test-med-1"
+root_coll="u/sierrav/panda_testing"
 
-export CM_PROD_DIR=`echo ${EXAMPLES} | sed 's/\/examples//'`
+export CM_PROD_DIR="${EXAMPLES}/.."
 export CM_PROD_URL="output/archive"
 export CM_BUTLER="${butler_repo}"
 export CM_SCRIPT_METHOD="slurm"
 
-
 latest_panda=$(ls -td /cvmfs/sw.lsst.eu/linux-x86_64/panda_env/v* | head -1)
 source $latest_panda/setup_panda_usdf.sh

examples/pdr2_setup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+EXAMPLES=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+source $EXAMPLES/common_setup.sh
+ticket_num="DM-42607"
+config="pdr2.yaml"
+config_name="pdr2"
+lsst_version="w_2024_03"
+p_name="${lsst_version}"
+c_name="${ticket_num}"
+fullname="${p_name}/${c_name}"
+db_path="output/cm_${p_name}_${c_name}.db"
+
+export CM_CONFIGS="src/lsst/cm/prod/configs/PDR2"
+export CM_DB="sqlite:///${db_path}"

examples/script_pdr2.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+EXAMPLES=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# Prompt for config/setup files
+# Each new setup will need its own elif here
+
+valid_config=false
+read -p "Which config would you like to use? (HSC-RC2, rc2_subset, pdr2) " CONFIG
+while [ $valid_config = false ]; do
+    if [ $CONFIG = 'HSC-RC2' ]
+    then
+	    source $EXAMPLES/hsc_weekly_setup.sh
+        valid_config=true
+    elif [ $CONFIG = 'rc2_subset' ]
+    then
+        source $EXAMPLES/hsc_rc2_subset_setup.sh
+        valid_config=true
+    elif [ $CONFIG = 'pdr2' ]
+    then
+	butler_repo='/sdf/group/rubin/repo/main'
+        source $EXAMPLES/pdr2_setup.sh
+        valid_config=true
+    else
+        read -p "That config was not valid. Please enter a valid config " CONFIG
+    fi
+done
+
+default=no
+read -p "This script will blow away the cm database and start a fresh one based on your setup file. Are you sure you want to do this? (y/N)" ANSWER
+ANSWER=${ANSWER:-$default}
+
+yeses="y Y yes Yes YES yEs yES yeS"
+blow_away=false
+for yes in $yeses; do
+    if [ $yes = $ANSWER ]
+    then
+	    blow_away=true
+	    break
+    else
+	    continue
+    fi
+done
+
+echo "Your environment variables have already been set up."
+if $blow_away
+then
+
+    echo "Enjoy your fresh cm database!"
+    \rm -rf $CM_PROD_URL/$fullname $db_path
+    \rm -rf submit/${root_coll}/${fullname}
+    butler remove-collections --confirm /repo/main "${root_coll}/${fullname}*"
+
+    mkdir -p output
+
+    cm create
+    cm parse --config-name ${config_name} --config-yaml ${config}
+    cm load-error-types --config-yaml examples/error_code_decisions.yaml
+    cm insert --production-name ${p_name}
+    cm insert --production-name ${p_name} --campaign-name ${c_name} --butler-repo ${butler_repo} --config-name ${config_name} --config-block campaign --lsst-version ${lsst_version} --root-coll ${root_coll}
+    cm daemon --fullname ${fullname} --max-running 0
+else
+    echo "We did not delete your cm database. You may proceed to your usual cm business without panic."
+fi

src/lsst/cm/prod/configs/PDR2/fitAffineWcsConfig.yaml

@@ -0,0 +1,10 @@
+description: Retarget wcsFitter to FitAffineWcsTask
+imports:
+  - $DRP_PIPE_DIR/pipelines/HSC/DRP-RC2.yaml
+tasks:
+  calibrate:
+    class: lsst.pipe.tasks.calibrate.CalibrateTask
+    config:
+    python: |
+      from lsst.meas.astrom import FitAffineWcsTask
+      config.astrometry.wcsFitter.retarget(FitAffineWcsTask)

src/lsst/cm/prod/configs/PDR2/pdr2.yaml

@@ -0,0 +1,151 @@
+prepare_step3:
+    class_name: lsst.cm.tools.db.script_handler.PrepareStep3Handler
+    data_id_file: "${CM_CONFIGS}/step3_veto_filter.yml"
+collect:
+    class_name: lsst.cm.tools.db.script_handler.CollectScriptHandler
+collect_step:
+    class_name: lsst.cm.tools.db.script_handler.CollectStepScriptHandler
+ancil:
+    class_name: lsst.cm.tools.db.script_handler.AncillaryScriptHandler
+    collections:
+        - HSC/calib
+        - HSC/masks
+        - HSC/fgcmcal/lut/PDR2/DM-39549
+        - refcats
+        - skymaps
+job:
+    class_name: lsst.cm.tools.db.job_handler.JobHandler
+workflow:
+    class_name: lsst.cm.tools.db.workflow_handler.WorkflowHandler
+    input_type: source
+    output_type: run
+    job_config: job
+rescue_workflow:
+    includes: ["workflow"]
+    rescue: True
+group:
+    class_name: lsst.cm.tools.db.group_handler.GroupHandler
+    input_type: source
+    output_type: run
+    workflow_config: workflow
+step:
+    class_name: lsst.cm.tools.db.step_handler.StepHandler
+    input_type: source
+    output_type: run
+    group_config: group
+    scripts:
+        - collect_step
+    templates:
+        coll_out: "u/sierrav/panda_testing/DM-42607/{step_name}"
+step1:
+    prerequisites: []
+    includes: ["step"]
+    pipeline_yaml: "${CM_PROD_DIR}/src/lsst/cm/prod/configs/PDR2/fitAffineWcsConfig.yaml#step1"
+    data_query_base: "instrument = 'HSC' and detector != 9 and detector < 140 and band in ('g', 'r', 'i', 'z', 'y')"
+    split_args:
+        dataset: raw
+        field: exposure
+        min_queries: 40
+step2a:
+    prerequisites: ["step1"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2a"
+    data_query_base: "instrument = 'HSC' and skymap = 'hsc_rings_v1'"
+step2b:
+    prerequisites: ["step2a"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2b"
+    data_query_base: "instrument = 'HSC' and skymap = 'hsc_rings_v1'"
+    split_args:
+        dataset: calexp
+        field: tract
+        min_queries: 30
+step2c:
+    prerequisites: ["step2b"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2c"
+    data_query_base: "instrument = 'HSC' and detector != 9 and detector < 140 and band in ('g', 'r', 'i', 'z', 'y')"
+step2d:
+    prerequisites: ["step2c"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2d"
+    data_query_base: >
+      instrument = 'HSC' and skymap = 'hsc_rings_v1' and detector != 9 and detector < 140
+      and band in ('g', 'r', 'i', 'z', 'y')
+    split_args:
+        dataset: raw
+        field: exposure
+        min_queries: 15
+step2e:
+    prerequisites: ["step2d"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2e"
+    data_query_base: "instrument = 'HSC' and detector != 9 and detector < 140 and band in ('g', 'r', 'i', 'z', 'y')"
+step3:
+    prerequisites: ["step2e"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step3"
+    data_query_base: "instrument = 'HSC' and skymap = 'hsc_rings_v1'"
+    scripts:
+        - prepare_step3
+        - collect_step
+step4:
+    prerequisites: ["step3"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step4"
+    data_query_base: >
+      instrument = 'HSC' and skymap = 'hsc_rings_v1' and detector != 9 and detector < 140
+      and band in ('g', 'r', 'i', 'z', 'y')
+    split_args:
+        dataset: raw
+        field: visit
+        min_queries: 40
+step5:
+    prerequisites: ["step4"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step5"
+    data_query_base: "instrument = 'HSC' and skymap = 'hsc_rings_v1'"
+    split_args:
+        dataset: isolated_star_cat
+        field: tract
+        min_queries: 90
+step6:
+    prerequisites: ["step4"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step6"
+    data_query_base: "instrument = 'HSC' and detector != 9 and detector < 140 and band in ('g', 'r', 'i', 'z', 'y')"
+    split_args:
+        dataset: raw
+        field: exposure
+        min_queries: 20
+step7:
+    prerequisites: ["step3"]
+    includes: ["step"]
+    pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step7"
+    data_query_base: "instrument = 'HSC'"
+campaign:
+    class_name: lsst.cm.tools.db.campaign_handler.CampaignHandler
+    root_coll: dummy
+    input_type: tagged
+    output_type: run
+    bps_yaml_template: "${CM_CONFIGS}/pdr2_template.yaml"
+    bps_script_template: "${CM_CONFIGS}/pdr2_bps_template.sh"
+    templates:
+        coll_source: HSC/raw/PDR2/all
+        coll_in: HSC/raw/PDR2/all
+        coll_out: u/sierrav/panda_testing/DM-42607
+        coll_ancil: u/sierrav/panda_testing/DM-42607/ancillary
+    scripts:
+        - ancil
+        - collect
+    steps:
+        - step1
+        - step2a
+        - step2b
+        - step2c
+        - step2d
+        - step2e
+        - step3
+        - step4
+        - step5
+        - step6
+        - step7

src/lsst/cm/prod/configs/PDR2/pdr2_bps_template.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env -S -i CM_PROD_DIR="${CM_PROD_DIR}" HOME="${HOME}" bash
+
+# The shebang lines above are needed b/c setup lsst_distrib in putting
+# the lsst python _after_ the virtual env python in the PATH, which
+# is causing errors
+
+# setup LSST env.
+export WEEKLY='{lsst_version}'
+source /cvmfs/sw.lsst.eu/linux-x86_64/lsst_distrib/${WEEKLY}/loadLSST.bash
+setup lsst_distrib
+
+# setup PanDA env.
+latest_panda=$(ls -td /cvmfs/sw.lsst.eu/linux-x86_64/panda_env/v* | head -1)
+setupScript=${latest_panda}/setup_panda_usdf.sh
+source $setupScript ${WEEKLY}
+
+env | grep PANDA
+
+# let's drop a panda_auth status here for kicks
+panda_auth status
+
+# Uncomment this for any custom bps setups
+#setup -j -r path/to/local/ctrl_bps
+#setup -j -r path/to/local/ctrl_bps_panda

src/lsst/cm/prod/configs/PDR2/pdr2_template.yaml

@@ -0,0 +1,21 @@
+includeConfigs:
+- ${CTRL_BPS_PANDA_DIR}/config/bps_usdf.yaml
+- ${DRP_PIPE_DIR}/bps/clustering/DRP-recalibrated.yaml
+- ${CM_PROD_DIR}/src/lsst/cm/prod/configs/PDR2/requestMemory.yaml
+numberOfRetries: 3
+
+fileDistributionEndPoint: "file:///sdf/data/rubin/panda_jobs/panda_cache/{operator}/panda_cache_box/{payloadFolder}/{uniqProcName}/"
+# This helps the database make an efficient query plan for faster qGraph builds.
+extraQgraphOptions: --dataset-query-constraint objectTable_tract
+custom_lsst_setup: ""
+
+# turn off the -o option in pipetask commands
+pipetaskOutput: ""
+
+executionButler:
+  requestMemory: 64000
+  queue: "SLAC_Rubin_Merge"
+  # turn off chaining in the execution butler because
+  # no output collection
+  command2: ""
+  command3: ""