mods to get testing to work on derecho

Externals.cfg

@@ -9,22 +9,22 @@ required = True
 local_path = components/cice
 
 [cime]
-tag = cime5.6.33
+tag = cime5.6.45
 protocol = git
 repo_url = https://github.com/ESMCI/cime
 required = True
 local_path = cime
 
 [cism]
-tag = release-cesm2.0.04
+tag = cism-release-cesm2.1.2_03
 protocol = git
 repo_url = https://github.com/ESCOMP/cism-wrapper
 externals = Externals_CISM.cfg
 required = True
 local_path = components/cism
 
 [clm]
-tag = release-clm5.0.27
+tag = release-clm5.0.35
 protocol = git
 repo_url = https://github.com/ESCOMP/ctsm
 externals = Externals_CLM.cfg

bld/Makefile.in

@@ -408,7 +408,7 @@ ifeq ($(FC_TYPE),intel)
   CPPDEF        += -DCPRINTEL $(NO_CONTIGUOUS_FLAG)
   CFLAGS        += -std=gnu99
   FC_FLAGS      := -ftz -convert big_endian -fp-model source -no-fma \
-                   -qno-opt-dynamic-align -assume realloc_lhs -xHost
+                   -qno-opt-dynamic-align -assume realloc_lhs
   FC_FLAGS_O3OPT:= $(FC_FLAGS)
   FC_FLAGS_NOOPT:= -ftz -convert big_endian -O0
   FREEFLAGS     := 

test/system/CAM_runcmnd.sh

@@ -41,6 +41,14 @@ if [ $run_mode = mpi ] || [ $run_mode = hybrid ]; then
  	    # cmnd="${cmnd} ddt --connect mpiexec_mpt -np $ntasks omplace -vv "
             ;;
 
+        # Derecho
+	        de*  )
+
+		            cmnd="${cmnd} mpiexec -n $2  "
+			                # cmnd="${cmnd} ddt --connect mpiexec_mpt -np $ntasks omplace -vv "
+					            ;;
+
+
         # hobart and leehill
         hob* | h[[:digit:]]* | le* | izu* | i[[:digit:]]* )
 

test/system/test_driver.sh

@@ -2,7 +2,7 @@
 #
 # test_driver.sh:  driver for the testing of CAM with standalone scripts
 #
-# usage on hobart, izumi, leehill, cheyenne
+# usage on hobart, izumi, leehill, cheyenne, derecho
 # ./test_driver.sh
 #
 # **more details in the CAM testing user's guide, accessible
@@ -323,6 +323,163 @@ cat > ${submit_script_cime} << EOF
 #PBS -j oe
 #PBS -l inception=login
 
+EOF
+
+##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
+    ;;
+
+    ##derecho
+    derecho* | dec* )
+    submit_script="`pwd -P`/test_driver_derecho_${cur_time}.sh"
+    submit_script_cb="`pwd -P`/test_driver_derecho_cb_${cur_time}.sh"
+    submit_script_cime="`pwd -P`/test_driver_derecho_cime_${cur_time}.sh"
+
+    if [ -z "$CAM_ACCOUNT" ]; then
+        echo "ERROR: Must set the environment variable CAM_ACCOUNT"
+        exit 2
+    fi
+
+    if [ -z "$CAM_BATCHQ" ]; then
+        export CAM_BATCHQ="main"
+    fi
+
+    # wallclock for run job
+    wallclock_limit="5:00:00"
+
+    if [ $gmake_j = 0 ]; then
+        gmake_j=128
+    fi
+
+    # run tests on 1 nodes using 128 tasks/node, 2 threads/task
+    CAM_TASKS=32
+    CAM_THREADS=2
+
+    # change parallel configuration on 2 nodes using 32 tasks, 1 threads/task
+    CAM_RESTART_TASKS=32
+    CAM_RESTART_THREADS=1
+
+    mach_workspace="/glade/derecho/scratch"
+
+##vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv writing to batch script vvvvvvvvvvvvvvvvvvv
+
+cat > ${submit_script_cb} << EOF
+#!/bin/sh
+#
+#PBS -N test_dr
+#PBS -q $CAM_BATCHQ
+#PBS -A $CAM_ACCOUNT
+#PBS -l walltime=2:00:00
+#PBS -l select=1:ncpus=128:mpiprocs=128
+#PBS -j oe
+
+export TMPDIR=/glade/derecho/scratch/$USER
+
+if [ -n "\$PBS_JOBID" ]; then    #batch job
+   export JOBID=\`echo \${PBS_JOBID} | cut -f1 -d'.'\`
+   initdir=`pwd -P`
+   interactive=false
+else
+   interactive=true
+fi
+
+export CAM_RBOPTIONS="build_only"
+
+## create_newcase looks for account number in ACCOUNT environment variable
+export ACCOUNT=$CAM_ACCOUNT
+
+# tasks and threads need to be set in the cb script because TCB_ccsm.sh uses
+# them to set the pe_layout
+export CAM_THREADS=$CAM_THREADS
+export CAM_TASKS=$CAM_TASKS
+
+source $LMOD_ROOT/lmod/init/sh
+
+##module load intel/19.0.5
+module load mkl
+##module list
+
+export INC_NETCDF=\${NCAR_ROOT_NETCDF}/include
+export LIB_NETCDF=\${NCAR_ROOT_NETCDF}/lib
+
+export CFG_STRING="-cc mpicc -fc mpif90 -fc_type intel -ldflags -mkl=cluster"
+export MAKE_CMD="gmake -j $gmake_j"
+export CCSM_MACH="derecho"
+export MACH_WORKSPACE="$mach_workspace"
+dataroot=${CESMDATAROOT}
+echo_arg="-e"
+input_file="tests_pretag_cheyenne"
+
+EOF
+
+#-------------------------------------------
+
+cat > ${submit_script} << EOF
+#!/bin/sh
+#
+#PBS -N test_dr
+#PBS -q $CAM_BATCHQ
+#PBS -A $CAM_ACCOUNT
+#PBS -l walltime=$wallclock_limit
+#PBS -l select=2:ncpus=128:mpiprocs=128:ompthreads=2
+#PBS -j oe
+
+export TMPDIR=/glade/derecho/scratch/$USER
+
+if [ -n "\$PBS_JOBID" ]; then    #batch job
+   export JOBID=\`echo \${PBS_JOBID} | cut -f1 -d'.'\`
+   initdir=`pwd -P`
+   interactive=false
+else
+   interactive=true
+fi
+
+export CAM_RBOPTIONS="run_only"
+ulimit -c unlimited
+
+##omp threads
+export OMP_STACKSIZE=256M
+export CAM_THREADS=$CAM_THREADS
+export CAM_RESTART_THREADS=$CAM_RESTART_THREADS
+
+##mpi tasks
+export CAM_TASKS=$CAM_TASKS
+export CAM_RESTART_TASKS=$CAM_RESTART_TASKS
+
+##derecho hacks to avoid MPI_LAUNCH_TIMEOUT
+MPI_IB_CONGESTED=1
+MPI_LAUNCH_TIMEOUT=40
+
+source $LMOD_ROOT/lmod/init/sh
+
+##module load intel/19.0.5
+##module load mkl
+##module list
+
+export CCSM_MACH="derecho"
+export MACH_WORKSPACE="$mach_workspace"
+export CPRNC_EXE=${CESMDATAROOT}/tools/cime/tools/cprnc/cprnc
+
+dataroot=${CESMDATAROOT}
+
+echo_arg="-e"
+
+input_file="tests_pretag_cheyenne"
+
+EOF
+
+#-------------------------------------------
+
+cat > ${submit_script_cime} << EOF
+#!/bin/bash
+#
+#PBS -N cime-tests
+#PBS -q $CAM_BATCHQ
+#PBS -A $CAM_ACCOUNT
+#PBS -l walltime=$wallclock_limit
+#PBS -l select=2:ncpus=128:mpiprocs=128:ompthreads=2
+#PBS -j oe
+
+
 EOF
 
 ##^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ writing to batch script ^^^^^^^^^^^^^^^^^^^
@@ -936,7 +1093,7 @@ if [ -n "${submit_script_cb}" ]; then
 
     case $hostname in
         # cheyenne
-        chey* | r* )
+        chey* | r* | derecho* | dec* )
             batch_queue_submit='qsub -V'
 	    ;;
         *)
@@ -990,6 +1147,9 @@ comp=""
 if [ "${hostname:0:4}" == "chey" ]; then
   cesm_test_mach="cheyenne"
 fi
+if [ "${hostname:0:4}" == "dere" ]; then
+  cesm_test_mach="derecho"
+fi
 if [ "${hostname:0:6}" == "hobart" ]; then
   cesm_test_mach="hobart"
 fi
@@ -1009,7 +1169,14 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then
   fi
 
   for cesm_test in ${cesm_test_suite}; do
-    testargs="--xml-category ${cesm_test} --xml-machine ${cesm_test_mach}"
+    # Force derecho to run the cheyenne testlist.
+    # After the transition to derecho is completed, this if statement can be removed and
+    # just the else needs to remain.
+    if [ "${cesm_test_mach}" == "derecho" ]; then
+      testargs="--xml-category ${cesm_test} --xml-machine cheyenne --mach ${cesm_test_mach} --retry 2"
+    else
+      testargs="--xml-category ${cesm_test} --xml-machine ${cesm_test_mach} --retry 2"
+    fi
 
     if [ -n "${use_existing}" ]; then
       test_id="${use_existing}"
@@ -1085,6 +1252,13 @@ if [ "${cesm_test_suite}" != "none" -a -n "${cesm_test_mach}" ]; then
       qsub -V ${submit_script_cime}
     fi
 
+    if [ "${hostname:0:2}" == "de" ]; then
+      echo "cd ${script_dir}" >> ${submit_script_cime}
+      echo './create_test' ${testargs} >> ${submit_script_cime}
+      chmod u+x ${submit_script_cime}
+      qsub -V ${submit_script_cime}
+    fi
+
     if [ "${hostname:0:6}" == "hobart" ]; then
       echo "cd ${script_dir}" >> ${submit_script_cime}
       echo './create_test' ${testargs} >> ${submit_script_cime}
@@ -1119,6 +1293,11 @@ if $run_cam_regression; then
 	qsub -V ${submit_script_cb}
 	;;
 
+    ##derecho
+    derecho* | dec* )
+	qsub -V ${submit_script_cb}
+	;;
+
     ##hobart
     hob* | h[[:digit:]]* )
 #        qsub ${submit_script}