From 9d562fb7b6e4ec24593d459a7cf1f7b7ba984277 Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Wed, 27 Nov 2024 08:48:18 -0800 Subject: [PATCH] Add xgyro (#425) * Add basic xgyro framework * Add full CGYRO simulation logic in, and fix xgyro paths * Add xgyro python wrapper * Write errors on stderr * Print sub-simulation info into out.xgyro.info * Fix xgyro dir creation from template, and add xgyro to top Makefile * Add regression support for xgyro * Add xgyro reg02 and reg03 * Add support for MPI_RANK_ORDER=2, add reg test --- Makefile | 3 + f2py/pygacode/gacodeinput.py | 14 +- shared/bin/gacode_reg_do | 31 +- shared/bin/gacode_reg_do_restart | 51 +- shared/bin/gacode_setup | 1 + xgyro/Makefile | 27 + xgyro/bin/xgyro | 328 ++ xgyro/bin/xgyro_parse.py | 79 + xgyro/src/Makefile | 36 + xgyro/src/xgyro.f90 | 127 + xgyro/src/xgyro_globals.f90 | 35 + xgyro/src/xgyro_io.f90 | 57 + xgyro/src/xgyro_mpi_setup.f90 | 102 + xgyro/src/xgyro_read_input.f90 | 110 + xgyro/tools/input/reg01/input.xgyro | 5 + xgyro/tools/input/reg01/reg01/input.cgyro | 42 + xgyro/tools/input/reg01/reg01/out.cgyro.prec | 4 + .../tools/input/reg01/reg01/out.cgyro.prec.1 | 8 + .../tools/input/reg01/reg01/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg01/reg02/input.cgyro | 43 + xgyro/tools/input/reg01/reg02/out.cgyro.prec | 4 + .../tools/input/reg01/reg02/out.cgyro.prec.1 | 8 + .../tools/input/reg01/reg02/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg01/reg03/input.cgyro | 46 + xgyro/tools/input/reg01/reg03/out.cgyro.prec | 4 + .../tools/input/reg01/reg03/out.cgyro.prec.1 | 8 + .../tools/input/reg01/reg03/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg01/reg04/input.cgyro | 35 + xgyro/tools/input/reg01/reg04/input.gacode | 2219 +++++++ .../tools/input/reg01/reg04/input.gacode.geo | 5110 +++++++++++++++++ xgyro/tools/input/reg01/reg04/out.cgyro.prec | 4 + .../tools/input/reg01/reg04/out.cgyro.prec.1 | 8 + .../tools/input/reg01/reg04/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg02/input.xgyro | 7 + xgyro/tools/input/reg02/reg04/input.cgyro | 35 + xgyro/tools/input/reg02/reg04/input.gacode | 2219 +++++++ .../tools/input/reg02/reg04/input.gacode.geo | 5110 +++++++++++++++++ xgyro/tools/input/reg02/reg04/out.cgyro.prec | 4 + .../tools/input/reg02/reg04/out.cgyro.prec.1 | 8 + .../tools/input/reg02/reg04/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg02/reg05/input.cgyro | 49 + xgyro/tools/input/reg02/reg05/out.cgyro.prec | 4 + .../tools/input/reg02/reg05/out.cgyro.prec.1 | 8 + .../tools/input/reg02/reg05/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg02/reg17/input.cgyro | 52 + xgyro/tools/input/reg02/reg17/out.cgyro.prec | 4 + .../tools/input/reg02/reg17/out.cgyro.prec.1 | 8 + .../tools/input/reg02/reg17/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg03/input.xgyro | 6 + xgyro/tools/input/reg03/reg06/input.cgyro | 51 + xgyro/tools/input/reg03/reg06/out.cgyro.prec | 4 + .../tools/input/reg03/reg06/out.cgyro.prec.1 | 8 + .../tools/input/reg03/reg06/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg03/reg07/input.cgyro | 49 + xgyro/tools/input/reg03/reg07/out.cgyro.prec | 4 + .../tools/input/reg03/reg07/out.cgyro.prec.1 | 8 + .../tools/input/reg03/reg07/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg04/input.xgyro | 8 + xgyro/tools/input/reg04/reg10/input.cgyro | 45 + xgyro/tools/input/reg04/reg10/out.cgyro.prec | 4 + .../tools/input/reg04/reg10/out.cgyro.prec.1 | 8 + .../tools/input/reg04/reg10/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg04/reg11/input.cgyro | 42 + xgyro/tools/input/reg04/reg11/out.cgyro.prec | 8 + .../tools/input/reg04/reg11/out.cgyro.prec.1 | 16 + .../tools/input/reg04/reg11/out.cgyro.prec.2 | 24 + xgyro/tools/input/reg04/reg19/input.cgyro | 55 + xgyro/tools/input/reg04/reg19/out.cgyro.prec | 4 + .../tools/input/reg04/reg19/out.cgyro.prec.1 | 8 + .../tools/input/reg04/reg19/out.cgyro.prec.2 | 12 + xgyro/tools/input/reg_list | 4 + xgyro/tools/input/simdir_list | 6 + 72 files changed, 16529 insertions(+), 22 deletions(-) create mode 100644 xgyro/Makefile create mode 100755 xgyro/bin/xgyro create mode 100644 xgyro/bin/xgyro_parse.py create mode 100644 xgyro/src/Makefile create mode 100644 xgyro/src/xgyro.f90 create mode 100644 xgyro/src/xgyro_globals.f90 create mode 100644 xgyro/src/xgyro_io.f90 create mode 100644 xgyro/src/xgyro_mpi_setup.f90 create mode 100644 xgyro/src/xgyro_read_input.f90 create mode 100644 xgyro/tools/input/reg01/input.xgyro create mode 100644 xgyro/tools/input/reg01/reg01/input.cgyro create mode 100644 xgyro/tools/input/reg01/reg01/out.cgyro.prec create mode 100644 xgyro/tools/input/reg01/reg01/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg01/reg01/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg01/reg02/input.cgyro create mode 100644 xgyro/tools/input/reg01/reg02/out.cgyro.prec create mode 100644 xgyro/tools/input/reg01/reg02/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg01/reg02/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg01/reg03/input.cgyro create mode 100644 xgyro/tools/input/reg01/reg03/out.cgyro.prec create mode 100644 xgyro/tools/input/reg01/reg03/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg01/reg03/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg01/reg04/input.cgyro create mode 100644 xgyro/tools/input/reg01/reg04/input.gacode create mode 100644 xgyro/tools/input/reg01/reg04/input.gacode.geo create mode 100644 xgyro/tools/input/reg01/reg04/out.cgyro.prec create mode 100644 xgyro/tools/input/reg01/reg04/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg01/reg04/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg02/input.xgyro create mode 100644 xgyro/tools/input/reg02/reg04/input.cgyro create mode 100644 xgyro/tools/input/reg02/reg04/input.gacode create mode 100644 xgyro/tools/input/reg02/reg04/input.gacode.geo create mode 100644 xgyro/tools/input/reg02/reg04/out.cgyro.prec create mode 100644 xgyro/tools/input/reg02/reg04/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg02/reg04/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg02/reg05/input.cgyro create mode 100644 xgyro/tools/input/reg02/reg05/out.cgyro.prec create mode 100644 xgyro/tools/input/reg02/reg05/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg02/reg05/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg02/reg17/input.cgyro create mode 100644 xgyro/tools/input/reg02/reg17/out.cgyro.prec create mode 100644 xgyro/tools/input/reg02/reg17/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg02/reg17/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg03/input.xgyro create mode 100644 xgyro/tools/input/reg03/reg06/input.cgyro create mode 100644 xgyro/tools/input/reg03/reg06/out.cgyro.prec create mode 100644 xgyro/tools/input/reg03/reg06/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg03/reg06/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg03/reg07/input.cgyro create mode 100644 xgyro/tools/input/reg03/reg07/out.cgyro.prec create mode 100644 xgyro/tools/input/reg03/reg07/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg03/reg07/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg04/input.xgyro create mode 100644 xgyro/tools/input/reg04/reg10/input.cgyro create mode 100644 xgyro/tools/input/reg04/reg10/out.cgyro.prec create mode 100644 xgyro/tools/input/reg04/reg10/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg04/reg10/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg04/reg11/input.cgyro create mode 100644 xgyro/tools/input/reg04/reg11/out.cgyro.prec create mode 100644 xgyro/tools/input/reg04/reg11/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg04/reg11/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg04/reg19/input.cgyro create mode 100644 xgyro/tools/input/reg04/reg19/out.cgyro.prec create mode 100644 xgyro/tools/input/reg04/reg19/out.cgyro.prec.1 create mode 100644 xgyro/tools/input/reg04/reg19/out.cgyro.prec.2 create mode 100644 xgyro/tools/input/reg_list create mode 100644 xgyro/tools/input/simdir_list diff --git a/Makefile b/Makefile index 47970cc8f..d626ae187 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ all: cd vgen ; make cd tglf ; make cd cgyro ; make + cd xgyro ; make cd gyro ; make cd tgyro ; make cd profiles_gen ; make @@ -21,6 +22,7 @@ clean: cd vgen ; make clean cd tglf ; make clean cd cgyro ; make clean + cd xgyro ; make clean cd gyro ; make clean cd tgyro ; make clean cd profiles_gen ; make clean @@ -42,6 +44,7 @@ distclean: cd neo ; make clean cd tglf ; make clean cd cgyro ; make clean + cd xgyro ; make clean cd gyro ; make clean cd tgyro ; make clean cd profiles_gen ; make clean diff --git a/f2py/pygacode/gacodeinput.py b/f2py/pygacode/gacodeinput.py index da64f5570..cfdb83396 100644 --- a/f2py/pygacode/gacodeinput.py +++ b/f2py/pygacode/gacodeinput.py @@ -36,6 +36,11 @@ def add(self,param,default,n=1): self.data_dict[p]=default self.data_orderlist.append(p) + def rename(self,org_param,new_param): + self.data_dict[new_param] = self.data_dict[org_param] + del self.data_dict[org_param] + self.data_orderlist[self.data_orderlist.index(org_param)] = new_param + def dep(self,param,default): self.dep_dict[param]=default self.dep_orderlist.append(param) @@ -56,7 +61,7 @@ def printmsg(self): def set_extension(self,text): self.extension = text - def read_input(self,inputfile): + def read_input(self,inputfile,write_ext=True): # 1. read user input file with open(inputfile,'r') as fin: for line in fin.readlines(): @@ -88,8 +93,11 @@ def read_input(self,inputfile): self.error=1 self.error_msg=self.error_msg+"ERROR: (gacodeinput) Bogus parameter "+x+'\n' - if self.error == 0: - with open(inputfile+self.extension,'w') as f: + if write_ext and (self.error == 0): + self.write_parsed(inputfile+self.extension) + + def write_parsed(self,outfile): + with open(outfile,'w') as f: for x in self.data_orderlist: f.write(self.data_dict[x]+' '+x+'\n') diff --git a/shared/bin/gacode_reg_do b/shared/bin/gacode_reg_do index 76a11f125..adf24f3be 100755 --- a/shared/bin/gacode_reg_do +++ b/shared/bin/gacode_reg_do @@ -8,6 +8,7 @@ # # EXAMPLE: # gacode_reg_do 2 1 0 neo 1e-5 +# gacode_reg_do 2 1 0 xgyro/cgyro 1e-5 #---------------------------------------------------- n=$# @@ -23,10 +24,14 @@ fi n_proc=${1} n_omp=${2} reset=${3} -code=${4} +code_raw=${4} tol=${5} scase=${6} +# if a / is present in code_raw string, use the part after / to signify another code +code=`python3 -c "import sys; print(sys.argv[1].split('/')[0])" ${code_raw}` +code2=`python3 -c "import sys; carr=sys.argv[1].split('/'); print(carr[1] if (len(carr)==2) else '')" ${code_raw}` + echo "REGRESSION TESTING: $code" testdir=$PWD/${code}_regression_test @@ -44,7 +49,11 @@ then compdir=$GACODE_ADD_ROOT/$code/tools/input fi -precfile=out.$code.prec +if [ "x${code2}" == "x" ]; then + precfile=out.${code}.prec +else + precfile=out.${code2}.prec +fi if [ "$scase" == "" ] then @@ -59,13 +68,27 @@ cd $testdir for sim in $list do $code -g $sim -p $testdir > out.$sim - rm -f $sim/$precfile + if [ "x${code2}" == "x" ]; then + rm -f $sim/$precfile + else + # this is actually a bundled test, check subdirs + for dname in `(cd $sim && ls -p |grep / | sed 's#/##')`; do + rm -f $sim/$dname/$precfile + done + fi if [ $n_omp -eq 1 ] && [ $n_proc -eq 1 ] ; then $code -e $sim -p $testdir > out.$sim else $code -e $sim -n $n_proc -nomp $n_omp -p $testdir > out.$sim fi - gacode_reg $sim $compdir $precfile $tol + if [ "x${code2}" == "x" ]; then + gacode_reg $sim $compdir $precfile $tol + else + # this is actually a bundled test, check subdirs + for dname in `(cd $sim && ls -p |grep / | sed 's#/##')`; do + gacode_reg $sim/$dname $compdir $precfile $tol + done + fi if [ $reset -eq 1 ] then # Overwrite regression data with current data diff --git a/shared/bin/gacode_reg_do_restart b/shared/bin/gacode_reg_do_restart index a78df9943..f0f9c1425 100755 --- a/shared/bin/gacode_reg_do_restart +++ b/shared/bin/gacode_reg_do_restart @@ -9,6 +9,7 @@ # # EXAMPLE: # gacode_reg_do_restart 2 1 0 cgyro 1e-6 reg01 +# gacode_reg_do_restart 2 1 0 xgyro/cgyro 1e-6 reg01 #---------------------------------------------------- n=$# @@ -24,10 +25,14 @@ fi n_proc=${1} n_omp=${2} reset=${3} -code=${4} +code_raw=${4} tol=${5} scase=${6} +# if a / is present in code_raw string, use the part after / to signify another code +code=`python3 -c "import sys; print(sys.argv[1].split('/')[0])" ${code_raw}` +code2=`python3 -c "import sys; carr=sys.argv[1].split('/'); print(carr[1] if (len(carr)==2) else '')" ${code_raw}` + echo "REGRESSION TESTING: $code" testdir=$PWD/${code}_regression_test @@ -45,8 +50,13 @@ then compdir=$GACODE_ADD_ROOT/$code/tools/input fi -precfile=out.$code.prec -infile=input.$code +if [ "x${code2}" == "x" ]; then + precfile=out.${code}.prec + infile=input.${code} +else + precfile=out.${code2}.prec + infile=input.${code2} +fi if [ "$scase" == "" ] then @@ -61,33 +71,44 @@ cd $testdir for sim in $list do $code -g $sim -p $testdir > out.$sim - rm -f $sim/$precfile - # patch input file to have frequent checkpointing - mv $sim/$infile $sim/$infile.org - grep -v RESTART_STEP $sim/$infile.org > $sim/$infile - echo "RESTART_STEP=1" >> $sim/$infile + if [ "x${code2}" == "x" ]; then + dnames="$sim" + else + dnames=`(ls -p -d ${sim}/* |grep '/$' |sed 's/.$//' )` + fi + for dname in $dnames; do + rm -f ${dname}/$precfile + # patch input file to have frequent checkpointing + mv ${dname}/$infile ${dname}/$infile.org + grep -v RESTART_STEP ${dname}/$infile.org > ${dname}/$infile + echo "RESTART_STEP=1" >> ${dname}/$infile + done # now run it 3 times for ((i=0; $i<3; i=$i+1)) do if [ $i -ne 0 ] then - cp $sim/$precfile $sim/$precfile.$oi + for dname in $dnames; do + cp ${dname}/$precfile ${dname}/$precfile.$oi + done fi if [ $n_omp -eq 1 ] && [ $n_proc -eq 1 ] ; then $code -e $sim -p $testdir > out.$sim else $code -e $sim -n $n_proc -nomp $n_omp -p $testdir > out.$sim fi - cp $sim/$precfile $sim/$precfile.$i - if [ $i -eq 0 ] - then + for dname in $dnames; do + cp ${dname}/$precfile ${dname}/$precfile.$i + if [ $i -eq 0 ] + then # reuse the non-checkpointed result from original regression test myprecfile=$precfile - else + else # use the additional iterations else myprecfile=$precfile.$i - fi - gacode_reg $sim $compdir $myprecfile $tol + fi + gacode_reg $dname $compdir $myprecfile $tol + done oi=$i done if [ $reset -eq 1 ] diff --git a/shared/bin/gacode_setup b/shared/bin/gacode_setup index 37c7b51f6..3afa61637 100755 --- a/shared/bin/gacode_setup +++ b/shared/bin/gacode_setup @@ -6,6 +6,7 @@ export PATH=$GACODE_ROOT/tgyro/bin:${PATH} export PATH=$GACODE_ROOT/gyro/bin:${PATH} export PATH=$GACODE_ROOT/cgyro/bin:${PATH} +export PATH=$GACODE_ROOT/xgyro/bin:${PATH} export PATH=$GACODE_ROOT/neo/bin:${PATH} export PATH=$GACODE_ROOT/vgen/bin:${PATH} export PATH=$GACODE_ROOT/tglf/bin:${PATH} diff --git a/xgyro/Makefile b/xgyro/Makefile new file mode 100644 index 000000000..e3ff01a8d --- /dev/null +++ b/xgyro/Makefile @@ -0,0 +1,27 @@ +#---------------------------- +# Toplevel makefile for xgyro +#---------------------------- + +include ${GACODE_ROOT}/platform/build/make.inc.${GACODE_PLATFORM} + +export EXTRA_LIBS = \ + ${GACODE_ROOT}/cgyro/src/cgyro_lib.a \ + ${GACODE_ROOT}/f2py/expro/expro_lib.a \ + ${GACODE_ROOT}/f2py/geo/geo_lib.a \ + ${GACODE_ROOT}/shared/math/math_lib.a + +all: + gacode_getversion > .VERSION + cd ${GACODE_ROOT}/cgyro && make + cd src && make + +clean: + cd ${GACODE_ROOT}/modules && rm -f xgyro*.mod + cd src && make clean + +deepclean: + cd ${GACODE_ROOT}/cgyro && make deepclean + cd ${GACODE_ROOT}/modules && rm -f xgyro*.mod + cd src && make clean + +.IGNORE: diff --git a/xgyro/bin/xgyro b/xgyro/bin/xgyro new file mode 100755 index 000000000..6a2f68679 --- /dev/null +++ b/xgyro/bin/xgyro @@ -0,0 +1,328 @@ +#!/bin/bash +#============================================================= +# xgyro +# +# PURPOSE: +# Top-level script controlling execution of XGYRO. +#============================================================= + +#============================================================= +# Syntax validity check and help message +# +n=$# + +if [ $n -eq 0 ] +then + echo + echo "Usage: xgyro [options]" + echo + echo " -p " + echo " Set optional path to simulation directory." + echo " The default path is \$XGYRO_DIR/sim." + echo + echo "TEST MODE" + echo + echo " -t " + echo " Sanity test for INPUT data in ." + echo + echo "EXECUTE MODE" + echo + echo " -e " + echo " Use input data in ." + echo + echo " -n " + echo " Number of MPI tasks." + echo + echo " -nomp " + echo " Number of OpenMP threads per MPI task." + echo + echo " -numa " + echo " NUMAs active per node." + echo + echo " -mpinuma " + echo " MPI tasks per active NUMA." + echo + echo "REGRESSION TESTING" + echo + echo " -r" + echo " Run full regression suite." + echo + echo " -rs " + echo " Run specified regression case." + echo + echo " -rt" + echo " Enable restart regression testing." + echo + echo " -reset" + echo " Reset regression data." + echo + echo "TEMPLATE GENERATION" + echo + echo " -g" + echo " List available simulation templates." + echo + echo " -g