Merge pull request #364 from sfiligoi/stampede3

Add Stampede3 platform

cgyro/src/cgyro_write_restart.F90

@@ -54,6 +54,7 @@ subroutine cgyro_write_restart_one
 
 #ifndef __INTEL_COMPILER
   integer :: rename
+  integer :: UNLINK
 #endif
 
   character(8)  :: sdate
@@ -63,6 +64,7 @@ subroutine cgyro_write_restart_one
   integer(KIND=8) :: count_rate, count_max
   real :: cp_dt
   integer :: j,ic0,statusfd
+  integer :: ierr
 
   ! use system_clock to be consistent with cgyro_kernel
   call system_clock(start_time,count_rate,count_max)
@@ -103,15 +105,15 @@ subroutine cgyro_write_restart_one
     ! Anything but restart_preservation_mode == 4
     ! User does not want high guarntees for the old file
     ! So, remove .old file, if it exists
-    call UNLINK(trim(path)//runfile_restart//".old")
+    ierr  = UNLINK(trim(path)//runfile_restart//".old")
     ! NOTE: We will not check if it succeeded... not important, may not even exist (yet)
   endif
 
   if ((i_proc == 0) .and. (restart_preservation_mode<2)) then 
     ! restart_preservation_mode == 1
     ! User wants to save disk space
     ! So, remove existing restart file, if it exists
-    call UNLINK(trim(path)//runfile_restart)
+    ierr = UNLINK(trim(path)//runfile_restart)
     ! NOTE: We will not check if it succeeded... not important, may not even exist (yet)
   endif
 

platform/build/make.inc.STAMPEDE3_ICX_IFX

@@ -0,0 +1,29 @@
+#----------------------------------------------------------
+# STAMPEDE3 (TACC) [Ice Lake nodes]
+#----------------------------------------------------------
+
+IDENTITY="TACC STAMPEDE3 ICX (ifx)"
+CORES_PER_NODE=80
+NUMAS_PER_NODE=2
+
+# Compilers and flags
+#FC     = mpiifx -gen-interfaces -module ${GACODE_ROOT}/modules -I${GACODE_ROOT}/module -I${TACC_FFTW3_INC}
+FC     = mpiifx -gen-interfaces -module ${GACODE_ROOT}/modules -I${GACODE_ROOT}/module -I${TACC_MKL_INC}/fftw
+F77    = ${FC}
+
+FOMP   = -qopenmp
+FMATH  = -real-size 64
+FOPT   = -xICELAKE-SERVER -Ofast
+FDEBUG = -eD -Ktrap=fp -m 1
+
+# System math libraries
+LMATH = -qmkl
+
+# NetCDF
+NETCDF_DIR=$TACC_NETCDF_DIR
+NETCDF=-L${NETCDF_DIR}/lib -lnetcdff -lnetcdf
+NETCDF_INC = ${NETCDF_DIR}/include
+
+# Archive 
+ARCH = ar cr
+

platform/build/make.inc.STAMPEDE3_SPR_IFX

@@ -0,0 +1,31 @@
+#----------------------------------------------------------
+# STAMPEDE2 (TACC) [Skylake nodes]
+#
+# - 2x24 cores * 2 hyperthreads
+#----------------------------------------------------------
+
+IDENTITY="TACC STAMPEDE3 SPR (ifx)"
+CORES_PER_NODE=112
+NUMAS_PER_NODE=16
+
+# Compilers and flags
+#FC     = mpiifx -gen-interfaces -module ${GACODE_ROOT}/modules -I${GACODE_ROOT}/module -I${TACC_FFTW3_INC}
+FC     = mpiifx -gen-interfaces -module ${GACODE_ROOT}/modules -I${GACODE_ROOT}/module -I${TACC_MKL_INC}/fftw
+F77    = ${FC}
+
+FOMP   = -qopenmp
+FMATH  = -real-size 64
+FOPT   = -xSAPPHIRERAPIDS -Ofast
+FDEBUG = -eD -Ktrap=fp -m 1
+
+# System math libraries
+LMATH = -qmkl
+
+# NetCDF
+NETCDF_DIR=$TACC_NETCDF_DIR
+NETCDF=-L${NETCDF_DIR}/lib -lnetcdff -lnetcdf
+NETCDF_INC = ${NETCDF_DIR}/include
+
+# Archive 
+ARCH = ar cr
+

platform/env/env.STAMPEDE3_ICX_IFX

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -n "$SSH_TTY" ] ; then
+   echo "Setting up $GACODE_PLATFORM environment for gacode"
+fi
+
+module load python
+module load netcdf

platform/env/env.STAMPEDE3_SPR_IFX

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [ -n "$SSH_TTY" ] ; then
+   echo "Setting up $GACODE_PLATFORM environment for gacode"
+fi
+
+module load python
+module load netcdf

platform/exec/exec.STAMPEDE3_ICX_IFX

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash 
+#
+# FUNCTION:
+#  Parallel execution script
+#---------------------------------------------------
+
+simdir=${1}
+nmpi=${2}
+exec=${3}
+nomp=${4}
+numa=${5}
+mpinuma=${6}
+nidle=${7}
+
+# nmpi = MPI tasks
+# nomp = OpenMP threads per MPI task
+# numa = NUMAs active per node
+# mpinuma = MPI tasks per active NUMA 
+
+. $GACODE_ROOT/shared/bin/gacode_mpi_tool
+
+cd $simdir
+
+if [ "$nomp" -lt 2 ]; then
+  echo "WARNING: This mode should only be used with nomp>=2, detected nomp=$nomp"
+fi
+
+export OMP_NUM_THREADS=$nomp
+export OMP_STACKSIZE=32M
+echo ibrun $exec
+ibrun $exec

platform/exec/exec.STAMPEDE3_SPR_IFX

@@ -0,0 +1,31 @@
+#! /usr/bin/env bash 
+#
+# FUNCTION:
+#  Parallel execution script
+#---------------------------------------------------
+
+simdir=${1}
+nmpi=${2}
+exec=${3}
+nomp=${4}
+numa=${5}
+mpinuma=${6}
+nidle=${7}
+
+# nmpi = MPI tasks
+# nomp = OpenMP threads per MPI task
+# numa = NUMAs active per node
+# mpinuma = MPI tasks per active NUMA 
+
+. $GACODE_ROOT/shared/bin/gacode_mpi_tool
+
+cd $simdir
+
+if [ "$nomp" -lt 2 ]; then
+  echo "WARNING: This mode should only be used with nomp>=2, detected nomp=$nomp"
+fi
+
+export OMP_NUM_THREADS=$nomp
+export OMP_STACKSIZE=32M
+echo ibrun $exec
+ibrun $exec

platform/qsub/qsub.STAMPEDE3_ICX_IFX

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+echo "-queue: icx"
+
+# Default queue
+if [ "$QUEUE" == "null_queue" ] ; then
+   QUEUE=spr
+fi
+
+bfile=$SIMDIR/batch.src
+
+echo "#!/bin/bash -l" > $bfile
+echo "#SBATCH -J $LOCDIR" >> $bfile
+#echo "#SBATCH -A $REPO" >> $bfile
+echo "#SBATCH -o $SIMDIR/batch.out" >> $bfile
+echo "#SBATCH -e $SIMDIR/batch.err" >> $bfile
+echo "#SBATCH -p $QUEUE" >> $bfile
+echo "#SBATCH -t $WALLTIME" >> $bfile
+echo "#SBATCH -N $nodes" >> $bfile
+echo "#SBATCH -n $nmpi" >> $bfile
+echo "$CODE -e $LOCDIR -n $nmpi -nomp $nomp -numa $numa -mpinuma $mpinuma -p $SIMROOT" >> $bfile 

platform/qsub/qsub.STAMPEDE3_SPR_IFX

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+echo "-queue: spr"
+
+# Default queue
+if [ "$QUEUE" == "null_queue" ] ; then
+   QUEUE=spr
+fi
+
+bfile=$SIMDIR/batch.src
+
+echo "#!/bin/bash -l" > $bfile
+echo "#SBATCH -J $LOCDIR" >> $bfile
+#echo "#SBATCH -A $REPO" >> $bfile
+echo "#SBATCH -o $SIMDIR/batch.out" >> $bfile
+echo "#SBATCH -e $SIMDIR/batch.err" >> $bfile
+echo "#SBATCH -p $QUEUE" >> $bfile
+echo "#SBATCH -t $WALLTIME" >> $bfile
+echo "#SBATCH -N $nodes" >> $bfile
+echo "#SBATCH -n $nmpi" >> $bfile
+echo "$CODE -e $LOCDIR -n $nmpi -nomp $nomp -numa $numa -mpinuma $mpinuma -p $SIMROOT" >> $bfile