From 1e75eb0c3db32142e669b25255df11338d1940b4 Mon Sep 17 00:00:00 2001 From: Andy Aschwanden Date: Tue, 7 Nov 2023 19:46:49 -0800 Subject: [PATCH] More work on Systems --- README.md | 5 ++ hindcasts/hindcast.py | 38 +++++++++----- hpc-systems/chinook.toml | 60 +++++++++++++++++++++ hpc-systems/debug.toml | 30 +++++++++++ hpc-systems/pleiades.toml | 59 +++++++++++++++++++++ pism_ragis/systems.py | 108 +++++++++++++++++++++++++++----------- pyproject.toml | 7 ++- tests/data/debug.txt | 30 +++++++++++ tests/test_systems.py | 42 ++++++++++++--- 9 files changed, 326 insertions(+), 53 deletions(-) create mode 100644 hpc-systems/chinook.toml create mode 100644 hpc-systems/debug.toml create mode 100644 hpc-systems/pleiades.toml create mode 100644 tests/data/debug.txt diff --git a/README.md b/README.md index 3e837e9..3d9fcf9 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,8 @@ ### Synopsis The stability of the Greenland Ice Sheet in a warming climate is a critical societal concern. Predicting Greenland's contribution to sea level remains a challenge as historical simulations of the past decades show limited agreement with observations. In this project, we develop a data assimilation framework that combines sparse observations and the ice sheet model PISM to produce a reanalysis of the state of the Greenland Ice Sheet from 1980 to 2020 using probabilistic filtering methods. + + +### + +This repository contains scripts and functions to generate and analyze hindcasts performed with the [![Parallel Ice Sheet Model (PISM)]](https://pism.io) diff --git a/hindcasts/hindcast.py b/hindcasts/hindcast.py index 37dc973..3ed798f 100755 --- a/hindcasts/hindcast.py +++ b/hindcasts/hindcast.py @@ -1,11 +1,23 @@ -#!/usr/bin/env python -# Copyright (C) 2019-23 Andy Aschwanden - -# Historical simulations for -# "A reanalyis of the Greenland Ice Sheet" +# Copyright (C) 2023 Andy Aschwanden +# +# This file is part of pism-ragis. +# +# PISM-RAGIS is free software; you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation; either version 3 of the License, or (at your option) any later +# version. +# +# PISM-RAGIS is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more +# details. +# +# You should have received a copy of the GNU General Public License +# along with PISM; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA """ -Perform hindcasts of the Greenland Ice Sheet +Generate scrips to hindcasts of the Greenland Ice Sheet using the Parallel Ice Sheet Model (PISM) """ import inspect @@ -21,7 +33,7 @@ import xarray as xr -def current_script_directory(): +def current_script_directory() -> str: """ Return the current directory """ @@ -34,6 +46,7 @@ def current_script_directory(): sys.path.append(join(script_directory, "../pism_ragis")) import computing # pylint: disable=C0413 +from systems import Systems # pylint: disable=C0413 grid_choices = [ 18000, @@ -53,6 +66,9 @@ def current_script_directory(): 150, ] +available_systems = Systems() +available_systems.default_path = "../hpc-systems" + if __name__ == "__main__": # set up the option parser parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) @@ -169,7 +185,7 @@ def current_script_directory(): "-s", "--system", dest="system", - choices=computing.list_systems(), + choices=available_systems.list_systems(), help="computer system to use.", default="debug", ) @@ -264,7 +280,7 @@ def current_script_directory(): osize = options.osize queue = options.queue walltime = options.walltime - system = options.system + system = available_systems[options.system] spatial_ts = options.spatial_ts test_climate_models = options.test_climate_models @@ -277,9 +293,7 @@ def current_script_directory(): stress_balance = options.stress_balance version = options.version - ensemble_file = options.ensemble_file - domain = options.domain pism_exec = computing.generate_domain(domain) @@ -358,7 +372,7 @@ def current_script_directory(): mkdir -p $each done\n\n """ - if system != "debug": + if system["machine"] != "debug": cmd = f"""lfs setstripe -c -1 {dirs["output"]}""" sub.call(shlex.split(cmd)) cmd = f"""lfs setstripe -c -1 {dirs["spatial_tmp"]}""" diff --git a/hpc-systems/chinook.toml b/hpc-systems/chinook.toml new file mode 100644 index 0000000..fe0316e --- /dev/null +++ b/hpc-systems/chinook.toml @@ -0,0 +1,60 @@ +machine = "chinook" + +[MPI] + +mpido = "mpirun -np {cores} -machinefile ./nodes_$SLURM_JOBID" + +[scheduler] + +name = "SLRUM" +submit = "sbatch" +job_id = "SLURM_JOBID" + +[filesystem] + +work_dir = "SLURM_SUBMIT_DIR" + +[partitions] + +default = "new" + +[partitions.old] + +name = "old-chinook" +cores_per_node = 24 +queues = ["t1standard", "t1small", "t2standard", "t2small"] + +[partitions.new] + +name = "new-chinook" +cores_per_node = 40 +queues = ["t1standard", "t1small", "t2standard", "t2small"] + +[job] + +header = """#!/bin/sh +#SBATCH --partition={queue} +#SBATCH --ntasks={cores} +#SBATCH --tasks-per-node={ppn} +#SBATCH --time={walltime} +#SBATCH --mail-type=BEGIN +#SBATCH --mail-type=END +#SBATCH --mail-type=FAIL +#SBATCH --output=pism.%j + +module list + +umask 007 + +cd $SLURM_SUBMIT_DIR + +# Generate a list of compute node hostnames reserved for this job, +# this ./nodes file is necessary for slurm to spawn mpi processes +# across multiple compute nodes +srun -l /bin/hostname | sort -n | awk '{{print $2}}' > ./nodes_$SLURM_JOBID + +ulimit -l unlimited +ulimit -s unlimited +ulimit + +""" diff --git a/hpc-systems/debug.toml b/hpc-systems/debug.toml new file mode 100644 index 0000000..abedd73 --- /dev/null +++ b/hpc-systems/debug.toml @@ -0,0 +1,30 @@ +machine = "debug" + +[MPI] + +mpido = "mpirun -np {cores}" + +[scheduler] + +name = "shell" +submit = "sh" +job_id = "" + +[filesystem] + +work_dir = "PWD" + +[partitions] + +default = "debug" + +[partitions.debug] + +name = "debug" + +[job] + +header = """ + + +""" diff --git a/hpc-systems/pleiades.toml b/hpc-systems/pleiades.toml new file mode 100644 index 0000000..aa28092 --- /dev/null +++ b/hpc-systems/pleiades.toml @@ -0,0 +1,59 @@ +machine = "pleiades" + +[partitions] + +default = "sandy_bridge" + +[partitions.broadwell] + +name = "bro" +cores_per_node = 28 +queues = ["debug", "normal", "long"] + +[partitions.haswell] + +name = "has" +cores_per_node = 24 +queues = ["debug", "normal", "long"] + +[partitions.ivy_bridge] + +name = "ivy" +cores_per_node = 20 +queues = ["debug", "normal", "long"] + +[partitions.sandy_bridge] + +name = "san" +cores_per_node = 16 +queues = ["debug", "normal", "long"] + +[MPI] + +mpido = "mpiexec -n {cores}" + +[scheduler] + +name = "QSUB" +submit = "qusb" +job_id = "PBS_JOBID" + +[filesystem] + +work_dir = "PBS_O_WORKDIR" + +[job] + +header = """#PBS -S /bin/bash +#PBS -N cfd +#PBS -l walltime={walltime} +#PBS -m e +#PBS -W group_list={gid} +#PBS -q {queue} +#PBS -lselect={nodes}:ncpus={ppn}:mpiprocs={ppn}:model={partition} +#PBS -j oe + +module list + +cd $PBS_O_WORKDIR +""" diff --git a/pism_ragis/systems.py b/pism_ragis/systems.py index 3f9e75b..7dbecda 100644 --- a/pism_ragis/systems.py +++ b/pism_ragis/systems.py @@ -22,7 +22,7 @@ import math from pathlib import Path -from typing import Union +from typing import Any, Iterator, Union import toml @@ -33,17 +33,43 @@ class System: """ def __init__(self, d: Union[dict, Path, str]): + self._values = {} if isinstance(d, dict): for key, value in d.items(): - setattr(self, key, value) - elif isinstance(d, Path): + self._values[key] = value + elif isinstance(d, (Path, str)): for key, value in toml.load(d).items(): - setattr(self, key, value) - elif isinstance(d, str): - self._values = toml.loads(d) + self._values[key] = value else: print(f"{d} not recognized") + def __getitem__(self, name) -> Any: + return self._values[name] + + def __setitem__(self, key, value): + setattr(self, key, value) + + def __iter__(self) -> Iterator: + return iter(self._values) + + def keys(self): + """ + Return keys + """ + return self._values.keys() + + def items(self): + """ + Return items + """ + return self._values.items() + + def values(self): + """ + Return values + """ + return self._values.values() + def make_batch_header( self, partition: str = "chinook_new", @@ -51,7 +77,7 @@ def make_batch_header( walltime: str = "8:00:00", n_cores: int = 40, gid: Union[None, str] = None, - ): + ) -> str: """ Create a batch header from system and kwargs """ @@ -84,36 +110,36 @@ def make_batch_header( m_str = "\n".join(list(lines)) return m_str - def list_partitions(self): + def list_partitions(self) -> list: """ List all partitions """ return [ values["name"] - for key, values in self.partitions.items() # type: ignore[attr-defined] # pylint: disable=E1101 + for key, values in self["partitions"].items() if key != "default" ] - def list_queues(self, partition: Union[None, str] = None): + def list_queues(self, partition: Union[None, str] = None) -> list: """ List available queues for partition. If no partition is given return default partition """ if not partition: - p = self.partitions["default"] # type: ignore[attr-defined] # pylint: disable=E1101 + p = self["partitions"]["default"] else: p = partition partition = p.split("_")[-1] - return self.partitions[partition]["queues"] # type: ignore[attr-defined] # pylint: disable=E1101 + return self["partitions"][partition]["queues"] - def to_dict(self): + def to_dict(self) -> dict: """ Returns self as dictionary """ - return self.__dict__ + return self._values - def __repr__(self): + def __repr__(self) -> str: repr_str = "" repr_str += toml.dumps(self.to_dict()) @@ -133,8 +159,8 @@ class Systems: """ def __init__(self): - self._default_path: Path = Path("tests/data") - self.add_systems_from_path(self._default_path) + self._default_path: Path = Path("hpc-systems") + self.add_from_path(self._default_path) @property def default_path(self): @@ -146,14 +172,15 @@ def default_path(self): @default_path.setter def default_path(self, value): self._default_path = value + self.add_from_path(self._default_path) - def __getitem__(self, name): + def __getitem__(self, name) -> Any: return self._values[name] def __setitem__(self, key, value): setattr(self, key, value) - def __iter__(self): + def __iter__(self) -> Iterator: return iter(self._values) def keys(self): @@ -174,16 +201,34 @@ def values(self): """ return self._values.values() - def add_system(self, system): + def __repr__(self) -> str: + return self.dump() + + def __len__(self) -> int: + return len(self.values()) + + def list_systems(self) -> list: """ - Add a system from a System class + Return name of machines as list """ - system.to_dict() + return list(self.keys()) - def add_systems_from_path(self, path): + def add_system(self, system: System): """ + Add a system + """ + machine = system["machine"] + if machine not in self.keys(): + self._values[machine] = system + return None + else: + msg = f"{machine} already exists" + print(msg) + return msg - Add systems from a pathlib.Path. + def add_from_path(self, path: Union[Path, str]): + """ + Add systems from a pathlib.Path or str. Use glob to add all files with suffix `toml`. """ @@ -195,10 +240,16 @@ def add_systems_from_path(self, path): sys[machine] = System(s) self._values = sys - def __len__(self): - return len(self.values()) + def add_system_from_file(self, path: Union[Path, str]): + """ + Add a system from a pathlib.Path or str. - def dump(self): + """ + s = toml.load(path) + machine = s["machine"] + self._values[machine] = System(s) + + def dump(self) -> str: """ Dump class to string """ @@ -208,6 +259,3 @@ def dump(self): repr_str += toml.dumps(s.to_dict()) repr_str += "\n" return repr_str - - def __repr__(self): - return self.dump() diff --git a/pyproject.toml b/pyproject.toml index 9b81356..900da0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "pism-ragis" -version = "0.1.0" +version = "0.2.0" maintainers = [{name = "Andy Aschwanden", email = "andy.aschwanden@gmail.com"}] description = """Home of NASA ROSES project "A Reanalysis of the Greenland Ice Sheet""" readme = "README.md" @@ -30,11 +30,10 @@ build-backend = "setuptools.build_meta" [tool.setuptools] py-modules = ["pism_ragis"] - [tool.poetry] name = "pism-ragis" -version = "0.1.0" -description = "" +version = "0.2.0" +description = """Home of NASA ROSES project "A Reanalysis of the Greenland Ice Sheet""" authors = ["Andy Aschwanden "] readme = "README.md" diff --git a/tests/data/debug.txt b/tests/data/debug.txt new file mode 100644 index 0000000..83d8ec3 --- /dev/null +++ b/tests/data/debug.txt @@ -0,0 +1,30 @@ +machine = "debugger" + +[MPI] + +mpido = "mpirun -np {cores}" + +[scheduler] + +name = "shell" +submit = "sh" +job_id = "" + +[filesystem] + +work_dir = "PWD" + +[partitions] + +default = "debug" + +[partitions.debug] + +name = "debug" + +[job] + +header = """ + + +""" diff --git a/tests/test_systems.py b/tests/test_systems.py index 707c1bb..2c67a2f 100644 --- a/tests/test_systems.py +++ b/tests/test_systems.py @@ -19,7 +19,6 @@ """ Tests for Systems class -We need a way to register machine. Maybe the machine list should be its own repo? """ from pathlib import Path @@ -30,7 +29,7 @@ @pytest.fixture(name="machine_file") -def fixture_machine_file(): +def fixture_machine_file() -> Path: """ Return Path to toml file """ @@ -38,7 +37,7 @@ def fixture_machine_file(): @pytest.fixture(name="machine_dict") -def fixture_machine_dict(): +def fixture_machine_dict() -> dict: """ Return system dict """ @@ -81,7 +80,7 @@ def test_system_from_dict(machine_dict): Test creating a System from a dictionary """ s = System(machine_dict) - assert s.machine == "chinook" # type: ignore[attr-defined] # pylint: disable=E1101 + assert s["machine"] == "chinook" def test_system_from_file(machine_file): @@ -89,7 +88,7 @@ def test_system_from_file(machine_file): Test creating a System from a toml file """ s = System(machine_file) - assert s.machine == "chinook" # type: ignore[attr-defined] # pylint: disable=E1101 + assert s["machine"] == "chinook" def test_system_list_queues(system): @@ -124,11 +123,40 @@ def test_systems_len(systems): """ Test len of Systems """ - assert len(systems) == 2 + assert len(systems) == 3 def test_systems_default_path(systems): """ Test default path """ - assert systems.default_path == Path("tests/data") + assert systems.default_path == Path("hpc-systems") + + +def test_systems_from_pathlib_path(): + """ + Test adding systems from a pathlib path + """ + systems = Systems() + systems.add_from_path(Path("tests/data")) + assert len(systems) == 2 + + +def test_systems_from_str_path(): + """ + Test adding systems from a str path + """ + systems = Systems() + systems.add_from_path("tests/data") + assert len(systems) == 2 + + +def test_systems_add_system(systems): + """ + Test adding a system + Test checking if system exists + """ + system = System("tests/data/debug.txt") + systems.add_system(system) + assert len(systems) == 4 + assert systems.add_system(system) == "debugger already exists"