diff --git a/config/cesm/config_inputdata.xml b/config/cesm/config_inputdata.xml
index cf0d8f7c82a..3188ec7d679 100644
--- a/config/cesm/config_inputdata.xml
+++ b/config/cesm/config_inputdata.xml
@@ -12,7 +12,15 @@
https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata
+
+ wget
+ ftp://ftp.cgd.ucar.edu/cesm/inputdata
+ anonymous
+ user@example.edu
+ ../inputdata_checksum.dat
+
+
diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml
index 41387371dc9..4b5af1d0327 100644
--- a/config/cesm/machines/config_compilers.xml
+++ b/config/cesm/machines/config_compilers.xml
@@ -656,50 +656,29 @@ using a fortran linker.
-
+
+ $ENV{NETCDF_C_ROOT}
+ $ENV{NETCDF_FORTRAN_ROOT}
+ $ENV{PARALLEL_NETCDF_ROOT}
--host=Linux
- -xCORE-AVX2
+ -march=core-avx2
- -xCORE-AVX2
+ -march=core-avx2
-
- -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf
-
-
- -DHAVE_PAPI -DHAVE_SLASHPROC
-
- -mkl
+ -Wl,-rpath,$ENV{NETCDF_FORTRAN_ROOT}/lib -Wl,-rpath,$ENV{NETCDF_C_ROOT}/lib -Wl,-rpath,$ENV{PARALLEL_NETCDF_ROOT}/lib
-
- FALSE
-
-
-
-
- --host=Linux
-
-
- -xMIC-AVX512
-
-
- -xMIC-AVX512
-
- -L$(NETCDF_DIR) -lnetcdff -Wl,--as-needed,-L$(NETCDF_DIR)/lib -lnetcdff -lnetcdf
+ -L$(NETCDF_FORTRAN_ROOT)/lib -lnetcdff -L$ENV{NETCDF_C_ROOT}/lib -lnetcdf -L$(PARALLEL_NETCDF_ROOT)/lib -lpnetcdf
+ -qmkl=cluster
-
- -DHAVE_SLASHPROC
+
+ -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY
-
- -mkl -lmemkind -zmuldefs
-
-
- FALSE
diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml
index c46a5534391..0f928102e5e 100644
--- a/config/cesm/machines/config_machines.xml
+++ b/config/cesm/machines/config_machines.xml
@@ -1904,8 +1904,10 @@ This allows using a different mpirun command to launch unit tests
PrgEnv-intel
intel intel/2023.1.0
+ /global/cfs/cdirs/ccsm1/sw/perlmutter/modules/intel/2023.1.0/
+
-
+
+
craype craype/2.7.23
cray-libsci/23.09.1.1
-
- cray-mpich/8.1.25
-
cray-netcdf-hdf5parallel
cray-hdf5-parallel
@@ -1933,10 +1933,13 @@ This allows using a different mpirun command to launch unit tests
cray-hdf5/1.12.2.7
cray-netcdf/4.9.0.7
-
- cray-hdf5-parallel/1.12.2.7
- cray-netcdf-hdf5parallel/4.9.0.7
- cray-parallel-netcdf/1.12.3.7
+
+ cray-mpich/8.1.25
+ /global/cfs/cdirs/ccsm1/sw/perlmutter/modules/cray-mpich/8.1.25-dr5sgoi/intel/2023.1.0/
+ netcdf-c/4.9.2
+ netcdf-fortran/4.6.1
+ parallel-netcdf/1.12.3
+ parallelio/2.6.2
cmake/3.24.3
@@ -1944,9 +1947,16 @@ This allows using a different mpirun command to launch unit tests
256M
-
+
+ -1
+
diff --git a/config/xml_schemas/config_compilers_v2.xsd b/config/xml_schemas/config_compilers_v2.xsd
index c1b289303cd..3b3820b18d2 100644
--- a/config/xml_schemas/config_compilers_v2.xsd
+++ b/config/xml_schemas/config_compilers_v2.xsd
@@ -180,6 +180,8 @@
+
+
diff --git a/scripts/lib/CIME/Servers/__init__.py b/scripts/lib/CIME/Servers/__init__.py
index 8d22604875c..5cd14c0f89d 100644
--- a/scripts/lib/CIME/Servers/__init__.py
+++ b/scripts/lib/CIME/Servers/__init__.py
@@ -1,5 +1,6 @@
-#pylint: disable=import-error
+# pylint: disable=import-error
from distutils.spawn import find_executable
+
has_gftp = find_executable("globus-url-copy")
has_svn = find_executable("svn")
has_wget = find_executable("wget")
diff --git a/scripts/lib/CIME/Servers/ftp.py b/scripts/lib/CIME/Servers/ftp.py
index 21aa5207692..209525a2bac 100644
--- a/scripts/lib/CIME/Servers/ftp.py
+++ b/scripts/lib/CIME/Servers/ftp.py
@@ -4,34 +4,69 @@
# pylint: disable=super-init-not-called
from CIME.XML.standard_module_setup import *
from CIME.Servers.generic_server import GenericServer
+from CIME.utils import Timeout
from ftplib import FTP as FTPpy
from ftplib import all_errors as all_ftp_errors
+import socket
logger = logging.getLogger(__name__)
# I think that multiple inheritence would be useful here, but I couldnt make it work
# in a py2/3 compatible way.
class FTP(GenericServer):
- def __init__(self, address, user='', passwd=''):
- ftp_server, root_address = address.split('/', 1)
- logger.info("server address {} root path {}".format(ftp_server, root_address))
- self.ftp = FTPpy(ftp_server)
+ def __init__(self, address, user="", passwd="", server=None):
if not user:
- user = ''
+ user = ""
if not passwd:
- passwd = ''
-
+ passwd = ""
+ expect(server, " Must call via ftp_login function")
+ root_address = address.split("/", 1)[1]
+ self.ftp = server
self._ftp_server = address
-
stat = self.ftp.login(user, passwd)
logger.debug("login stat {}".format(stat))
if "Login successful" not in stat:
- logging.warning("FAIL: Could not login to ftp server {}\n error {}".format(ftp_server, stat))
+ logging.warning(
+ "FAIL: Could not login to ftp server {}\n error {}".format(
+ address, stat
+ )
+ )
return None
- stat = self.ftp.cwd(root_address)
- logger.debug("cwd {} stat {}".format(root_address,stat))
+ try:
+ stat = self.ftp.cwd(root_address)
+ except all_ftp_errors as err:
+ logging.warning("ftplib returned error {}".format(err))
+ return None
+
+ logger.debug("cwd {} stat {}".format(root_address, stat))
if "Directory successfully changed" not in stat:
- logging.warning("FAIL: Could not cd to server root directory {}\n error {}".format(root_address, stat))
+ logging.warning(
+ "FAIL: Could not cd to server root directory {}\n error {}".format(
+ root_address, stat
+ )
+ )
+ return None
+
+ @classmethod
+ def ftp_login(cls, address, user="", passwd=""):
+ ftp_server, root_address = address.split("/", 1)
+ logger.info("server address {} root path {}".format(ftp_server, root_address))
+ try:
+ with Timeout(60):
+ ftp = FTPpy(ftp_server)
+
+ except socket.error as e:
+ logger.warning("ftp login timeout! {} ".format(e))
return None
+ except RuntimeError:
+ logger.warning("ftp login timeout!")
+ return None
+ result = None
+ try:
+ result = cls(address, user=user, passwd=passwd, server=ftp)
+ except all_ftp_errors as e:
+ logger.warning("ftp error: {}".format(e))
+
+ return result
def fileexists(self, rel_path):
try:
@@ -42,20 +77,29 @@ def fileexists(self, rel_path):
if rel_path not in stat:
if not stat or not stat[0].startswith(rel_path):
- logging.warning("FAIL: File {} not found.\nerror {}".format(rel_path, stat))
+ logging.warning(
+ "FAIL: File {} not found.\nerror {}".format(rel_path, stat)
+ )
return False
return True
def getfile(self, rel_path, full_path):
try:
- stat = self.ftp.retrbinary('RETR {}'.format(rel_path), open(full_path, "wb").write)
+ stat = self.ftp.retrbinary(
+ "RETR {}".format(rel_path), open(full_path, "wb").write
+ )
except all_ftp_errors:
+ if os.path.isfile(full_path):
+ os.remove(full_path)
logger.warning("ERROR from ftp server, trying next server")
return False
- if (stat != '226 Transfer complete.'):
- logging.warning("FAIL: Failed to retreve file '{}' from FTP repo '{}' stat={}\n".
- format(rel_path, self._ftp_server, stat))
+ if stat != "226 Transfer complete.":
+ logging.warning(
+ "FAIL: Failed to retreve file '{}' from FTP repo '{}' stat={}\n".format(
+ rel_path, self._ftp_server, stat
+ )
+ )
return False
return True
@@ -67,4 +111,4 @@ def getdirectory(self, rel_path, full_path):
return False
for _file in stat:
- self.getfile(_file, full_path+os.sep+os.path.basename(_file))
+ self.getfile(_file, full_path + os.sep + os.path.basename(_file))
diff --git a/scripts/lib/CIME/Servers/generic_server.py b/scripts/lib/CIME/Servers/generic_server.py
index 6cebb46bba5..537df181324 100644
--- a/scripts/lib/CIME/Servers/generic_server.py
+++ b/scripts/lib/CIME/Servers/generic_server.py
@@ -7,17 +7,21 @@
from CIME.XML.standard_module_setup import *
from socket import _GLOBAL_DEFAULT_TIMEOUT
+
logger = logging.getLogger(__name__)
+
class GenericServer(object):
- def __init__(self, host=' ',user=' ', passwd=' ', acct=' ', timeout=_GLOBAL_DEFAULT_TIMEOUT):
+ def __init__(
+ self, host=" ", user=" ", passwd=" ", acct=" ", timeout=_GLOBAL_DEFAULT_TIMEOUT
+ ):
raise NotImplementedError
def fileexists(self, rel_path):
- ''' Returns True if rel_path exists on server '''
+ """Returns True if rel_path exists on server"""
raise NotImplementedError
def getfile(self, rel_path, full_path):
- ''' Get file from rel_path on server and place in location full_path on client
- fail if full_path already exists on client, return True if successful '''
+ """Get file from rel_path on server and place in location full_path on client
+ fail if full_path already exists on client, return True if successful"""
raise NotImplementedError
diff --git a/scripts/lib/CIME/Servers/gftp.py b/scripts/lib/CIME/Servers/gftp.py
index b99c9f3049f..f23943b583a 100644
--- a/scripts/lib/CIME/Servers/gftp.py
+++ b/scripts/lib/CIME/Servers/gftp.py
@@ -8,31 +8,52 @@
logger = logging.getLogger(__name__)
+
class GridFTP(GenericServer):
- def __init__(self, address, user='', passwd=''):
+ def __init__(self, address, user="", passwd=""):
self._root_address = address
def fileexists(self, rel_path):
- stat,out,err = run_cmd("globus-url-copy -list {}".format(os.path.join(self._root_address, os.path.dirname(rel_path))+os.sep))
+ stat, out, err = run_cmd(
+ "globus-url-copy -list {}".format(
+ os.path.join(self._root_address, os.path.dirname(rel_path)) + os.sep
+ )
+ )
if stat or os.path.basename(rel_path) not in out:
- logging.warning("FAIL: File {} not found.\nstat={} error={}".format(rel_path, stat, err))
+ logging.warning(
+ "FAIL: File {} not found.\nstat={} error={}".format(rel_path, stat, err)
+ )
return False
return True
def getfile(self, rel_path, full_path):
- stat, _,err = run_cmd("globus-url-copy -v {} file://{}".format(os.path.join(self._root_address, rel_path), full_path))
+ stat, _, err = run_cmd(
+ "globus-url-copy -v {} file://{}".format(
+ os.path.join(self._root_address, rel_path), full_path
+ )
+ )
- if (stat != 0):
- logging.warning("FAIL: GridFTP repo '{}' does not have file '{}' error={}\n".
- format(self._root_address,rel_path, err))
+ if stat != 0:
+ logging.warning(
+ "FAIL: GridFTP repo '{}' does not have file '{}' error={}\n".format(
+ self._root_address, rel_path, err
+ )
+ )
return False
return True
def getdirectory(self, rel_path, full_path):
- stat, _,err = run_cmd("globus-url-copy -v -r {}{} file://{}{}".format(os.path.join(self._root_address, rel_path), os.sep, full_path, os.sep))
+ stat, _, err = run_cmd(
+ "globus-url-copy -v -r {}{} file://{}{}".format(
+ os.path.join(self._root_address, rel_path), os.sep, full_path, os.sep
+ )
+ )
- if (stat != 0):
- logging.warning("FAIL: GridFTP repo '{}' does not have directory '{}' error={}\n".
- format(self._root_address,rel_path, err))
+ if stat != 0:
+ logging.warning(
+ "FAIL: GridFTP repo '{}' does not have directory '{}' error={}\n".format(
+ self._root_address, rel_path, err
+ )
+ )
return False
return True
diff --git a/scripts/lib/CIME/Servers/svn.py b/scripts/lib/CIME/Servers/svn.py
index 3b8dd18ea83..7e06ab310d4 100644
--- a/scripts/lib/CIME/Servers/svn.py
+++ b/scripts/lib/CIME/Servers/svn.py
@@ -7,9 +7,10 @@
logger = logging.getLogger(__name__)
+
class SVN(GenericServer):
- def __init__(self, address, user='', passwd=''):
- self._args = ''
+ def __init__(self, address, user="", passwd=""):
+ self._args = ""
if user:
self._args += "--username {}".format(user)
if passwd:
@@ -17,20 +18,35 @@ def __init__(self, address, user='', passwd=''):
self._svn_loc = address
- err = run_cmd("svn --non-interactive --trust-server-cert {} ls {}".format(self._args, address))[0]
+ err = run_cmd(
+ "svn --non-interactive --trust-server-cert {} ls {}".format(
+ self._args, address
+ )
+ )[0]
if err != 0:
logging.warning(
-"""
+ """
Could not connect to svn repo '{0}'
This is most likely either a credential, proxy, or network issue .
-To check connection and store your credential run 'svn ls {0}' and permanently store your password""".format(address))
+To check connection and store your credential run 'svn ls {0}' and permanently store your password""".format(
+ address
+ )
+ )
return None
def fileexists(self, rel_path):
full_url = os.path.join(self._svn_loc, rel_path)
- stat, out, err = run_cmd("svn --non-interactive --trust-server-cert {} ls {}".format(self._args, full_url))
- if (stat != 0):
- logging.warning("FAIL: SVN repo '{}' does not have file '{}'\nReason:{}\n{}\n".format(self._svn_loc, full_url, out, err))
+ stat, out, err = run_cmd(
+ "svn --non-interactive --trust-server-cert {} ls {}".format(
+ self._args, full_url
+ )
+ )
+ if stat != 0:
+ logging.warning(
+ "FAIL: SVN repo '{}' does not have file '{}'\nReason:{}\n{}\n".format(
+ self._svn_loc, full_url, out, err
+ )
+ )
return False
return True
@@ -38,10 +54,17 @@ def getfile(self, rel_path, full_path):
if not rel_path:
return False
full_url = os.path.join(self._svn_loc, rel_path)
- stat, output, errput = \
- run_cmd("svn --non-interactive --trust-server-cert {} export {} {}".format(self._args, full_url, full_path))
- if (stat != 0):
- logging.warning("svn export failed with output: {} and errput {}\n".format(output, errput))
+ stat, output, errput = run_cmd(
+ "svn --non-interactive --trust-server-cert {} export {} {}".format(
+ self._args, full_url, full_path
+ )
+ )
+ if stat != 0:
+ logging.warning(
+ "svn export failed with output: {} and errput {}\n".format(
+ output, errput
+ )
+ )
return False
else:
logging.info("SUCCESS\n")
@@ -49,10 +72,17 @@ def getfile(self, rel_path, full_path):
def getdirectory(self, rel_path, full_path):
full_url = os.path.join(self._svn_loc, rel_path)
- stat, output, errput = \
- run_cmd("svn --non-interactive --trust-server-cert {} export --force {} {}".format(self._args, full_url, full_path))
- if (stat != 0):
- logging.warning("svn export failed with output: {} and errput {}\n".format(output, errput))
+ stat, output, errput = run_cmd(
+ "svn --non-interactive --trust-server-cert {} export --force {} {}".format(
+ self._args, full_url, full_path
+ )
+ )
+ if stat != 0:
+ logging.warning(
+ "svn export failed with output: {} and errput {}\n".format(
+ output, errput
+ )
+ )
return False
else:
logging.info("SUCCESS\n")
diff --git a/scripts/lib/CIME/Servers/wget.py b/scripts/lib/CIME/Servers/wget.py
index eb685791884..56517cffa7b 100644
--- a/scripts/lib/CIME/Servers/wget.py
+++ b/scripts/lib/CIME/Servers/wget.py
@@ -4,35 +4,71 @@
# pylint: disable=super-init-not-called
from CIME.XML.standard_module_setup import *
from CIME.Servers.generic_server import GenericServer
+
logger = logging.getLogger(__name__)
+
class WGET(GenericServer):
- def __init__(self, address, user='', passwd=''):
- self._args = ''
+ def __init__(self, address, user="", passwd=""):
+ self._args = "--no-check-certificate "
if user:
self._args += "--user {} ".format(user)
if passwd:
self._args += "--password {} ".format(passwd)
self._server_loc = address
- err = run_cmd("wget {} --spider {}".format(self._args, address))[0]
- expect(err == 0,"Could not connect to repo '{0}'\nThis is most likely either a proxy, or network issue .")
+ @classmethod
+ def wget_login(cls, address, user="", passwd=""):
+ args = "--no-check-certificate "
+ if user:
+ args += "--user {} ".format(user)
+ if passwd:
+ args += "--password {} ".format(passwd)
+ try:
+ err = run_cmd("wget {} --spider {}".format(args, address), timeout=60)[0]
+ except:
+ logger.warning(
+ "Could not connect to repo '{0}'\nThis is most likely either a proxy, or network issue .(location 1)".format(
+ address
+ )
+ )
+ return None
+
+ if err and not "storage.neonscience.org" in address:
+ logger.warning(
+ "Could not connect to repo '{0}'\nThis is most likely either a proxy, or network issue .(location 2)".format(
+ address
+ )
+ )
+ return None
+
+ return cls(address, user=user, passwd=passwd)
def fileexists(self, rel_path):
full_url = os.path.join(self._server_loc, rel_path)
stat, out, err = run_cmd("wget {} --spider {}".format(self._args, full_url))
- if (stat != 0):
- logging.warning("FAIL: Repo '{}' does not have file '{}'\nReason:{}\n{}\n".format(self._server_loc, full_url, out.encode('utf-8'), err.encode('utf-8')))
+
+ if stat != 0:
+ logging.warning(
+ "FAIL: Repo '{}' does not have file '{}'\nReason:{}\n{}\n".format(
+ self._server_loc, full_url, out, err
+ )
+ )
return False
return True
def getfile(self, rel_path, full_path):
full_url = os.path.join(self._server_loc, rel_path)
- stat, output, errput = \
- run_cmd("wget {} {} -nc --output-document {}".format(self._args, full_url, full_path))
- if (stat != 0):
- logging.warning("wget failed with output: {} and errput {}\n".format(output, errput))
+ stat, output, errput = run_cmd(
+ "wget {} {} -nc --output-document {}".format(
+ self._args, full_url, full_path
+ )
+ )
+ if stat != 0:
+ logging.warning(
+ "wget failed with output: {} and errput {}\n".format(output, errput)
+ )
# wget puts an empty file if it fails.
try:
os.remove(full_path)
@@ -45,12 +81,16 @@ def getfile(self, rel_path, full_path):
def getdirectory(self, rel_path, full_path):
full_url = os.path.join(self._server_loc, rel_path)
- stat, output, errput = \
- run_cmd("wget {} {} -r -N --no-directories ".format(self._args, full_url+os.sep), from_dir=full_path)
+ stat, output, errput = run_cmd(
+ "wget {} {} -r -N --no-directories ".format(self._args, full_url + os.sep),
+ from_dir=full_path,
+ )
logger.debug(output)
logger.debug(errput)
- if (stat != 0):
- logging.warning("wget failed with output: {} and errput {}\n".format(output, errput))
+ if stat != 0:
+ logging.warning(
+ "wget failed with output: {} and errput {}\n".format(output, errput)
+ )
# wget puts an empty file if it fails.
try:
os.remove(full_path)
diff --git a/src/externals/pio2/src/gptl/GPTLget_memusage.c b/src/externals/pio2/src/gptl/GPTLget_memusage.c
index 4b0d138b2b6..2cea42236f1 100644
--- a/src/externals/pio2/src/gptl/GPTLget_memusage.c
+++ b/src/externals/pio2/src/gptl/GPTLget_memusage.c
@@ -100,7 +100,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
*/
pid = (int) getpid ();
- if (pid > 999999) {
+ if (pid > 9999999) {
fprintf (stderr, "get_memusage: pid %d is too large\n", pid);
return -1;
}
diff --git a/src/share/timing/GPTLget_memusage.c b/src/share/timing/GPTLget_memusage.c
index 83f41e5e603..3e7ccb95a73 100644
--- a/src/share/timing/GPTLget_memusage.c
+++ b/src/share/timing/GPTLget_memusage.c
@@ -116,7 +116,7 @@ int GPTLget_memusage (int *size, int *rss, int *share, int *text, int *datastack
*/
pid = (int) getpid ();
- if (pid > 999999) {
+ if (pid > 9999999) {
fprintf (stderr, "get_memusage: pid %d is too large\n", pid);
return -1;
}