From 2ea609b494af56c3cc8b80b97022b3bc5751bfdd Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 14 Dec 2023 13:45:51 -0500 Subject: [PATCH 1/4] validate YAML file using schema before constructing module. --- CHANGES.rst | 3 +++ tests/test_modules.py | 6 ------ xclim/core/indicator.py | 11 +++++++++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index f69c45d54..937e64934 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,9 @@ Changelog ========= +* Validate YAML indicators description before trying to build module. (:issue:`1523`) + + v0.47.0 (2023-12-01) -------------------- Contributors to this version: Juliette Lavoie (:user:`juliettelavoie`), Pascal Bourgault (:user:`aulemahal`), Trevor James Smith (:user:`Zeitsperre`), David Huard (:user:`huard`), Éric Dupuis (:user:`coxipi`). diff --git a/tests/test_modules.py b/tests/test_modules.py index d25f93c17..b6b3abb47 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -61,12 +61,6 @@ def test_custom_indices(open_dataset): # Use the example data used in the Extending Xclim notebook for testing. example_path = Path(__file__).parent.parent / "docs" / "notebooks" / "example" - schema = yamale.make_schema( - Path(__file__).parent.parent / "xclim" / "data" / "schema.yml" - ) - data = yamale.make_data(example_path / "example.yml") - yamale.validate(schema, data) - pr = open_dataset("ERA5/daily_surface_cancities_1990-1993.nc").pr # This tests load_module with a python file that is _not_ on the PATH diff --git a/xclim/core/indicator.py b/xclim/core/indicator.py index 36cb63771..df1d68b3c 100644 --- a/xclim/core/indicator.py +++ b/xclim/core/indicator.py @@ -85,8 +85,8 @@ In the following, the section under `` is referred to as `data`. When creating indicators from a dictionary, with :py:meth:`Indicator.from_dict`, the input dict must follow the same structure of `data`. -The resulting yaml file can be validated using the provided schema (in xclim/data/schema.yml) -and the YAMALE tool :cite:p:`lopker_yamale_2022`. See the "Extending xclim" notebook for more info. +When a module is built from a yaml file, the yaml is first validated against the schema (see xclim/data/schema.yml) +using the YAMALE library (:cite:p:`lopker_yamale_2022`). See the "Extending xclim" notebook for more info. Inputs ~~~~~~ @@ -115,6 +115,7 @@ import numpy as np import xarray +import yamale from xarray import DataArray, Dataset from yaml import safe_load @@ -1716,6 +1717,12 @@ def build_indicator_module_from_yaml( # noqa: C901 with ymlpath.open(encoding=encoding) as f: yml = safe_load(f) + # Read schema + schema = yamale.make_schema(Path(__file__).parent.parent / "data" / "schema.yml") + + # Validate - a YamaleError will be raised if the module does not comply with the schema. + yamale.validate(schema, yamale.make_data(ymlpath)) + # Load values from top-level in yml. # Priority of arguments differ. module_name = name or yml.get("module", filepath.stem) From 9b1e2dd7a08701a1581fd00ba18c5d70b5a6a004 Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 14 Dec 2023 14:45:27 -0500 Subject: [PATCH 2/4] Add note in the docs about schema validation --- docs/notebooks/extendxclim.ipynb | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/docs/notebooks/extendxclim.ipynb b/docs/notebooks/extendxclim.ipynb index d4e9a4c04..35e8a2597 100644 --- a/docs/notebooks/extendxclim.ipynb +++ b/docs/notebooks/extendxclim.ipynb @@ -397,26 +397,15 @@ "\n", "\n", "#### Validation of the YAML file\n", - "Using [yamale](https://github.com/23andMe/Yamale), it is possible to check if the YAML file is valid. `xclim` ships with a schema (in `xclim/data/schema.yml`) file. The file can be located with:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from importlib.resources import path\n", "\n", - "with path(\"xclim.data\", \"schema.yml\") as f:\n", - " print(f)" + "Using [yamale](https://github.com/23andMe/Yamale), it is possible to check if the YAML file is valid. `xclim` ships with a schema (in `xclim/data/schema.yml`) file. " ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "And the validation can be executed either in a python session:" + "The validation can be executed in a python session:" ] }, { @@ -425,6 +414,7 @@ "metadata": {}, "outputs": [], "source": [ + "from importlib.resources import path\n", "import yamale\n", "\n", "with path(\"xclim.data\", \"schema.yml\") as f:\n", @@ -437,13 +427,15 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "No errors means it passed. The validation can also be run through the command line with:\n", + "Or the validation can alternatively be run from the command line with:\n", "\n", "```bash\n", "yamale -s path/to/schema.yml path/to/module.yml\n", "```\n", "\n", - "#### Loading the module and computating of the indices." + "Note that xclim builds indicators from a yaml file, as shown in the next example, it validates it first. \n", + "\n", + "#### Loading the module and computing indicators." ] }, { From 8b1885aad0c9907fc99adcde89ee22e5122fc6d0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:51:30 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/notebooks/extendxclim.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/notebooks/extendxclim.ipynb b/docs/notebooks/extendxclim.ipynb index 35e8a2597..ac5af394d 100644 --- a/docs/notebooks/extendxclim.ipynb +++ b/docs/notebooks/extendxclim.ipynb @@ -415,6 +415,7 @@ "outputs": [], "source": [ "from importlib.resources import path\n", + "\n", "import yamale\n", "\n", "with path(\"xclim.data\", \"schema.yml\") as f:\n", From ad7096550b8a8cdede3c1191cdde4169d865ef0f Mon Sep 17 00:00:00 2001 From: David Huard Date: Thu, 14 Dec 2023 17:02:51 -0500 Subject: [PATCH 4/4] Read YML file with encoding for validation --- xclim/core/indicator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xclim/core/indicator.py b/xclim/core/indicator.py index df1d68b3c..6590ab9a4 100644 --- a/xclim/core/indicator.py +++ b/xclim/core/indicator.py @@ -1721,7 +1721,9 @@ def build_indicator_module_from_yaml( # noqa: C901 schema = yamale.make_schema(Path(__file__).parent.parent / "data" / "schema.yml") # Validate - a YamaleError will be raised if the module does not comply with the schema. - yamale.validate(schema, yamale.make_data(ymlpath)) + yamale.validate( + schema, yamale.make_data(content=ymlpath.read_text(encoding=encoding)) + ) # Load values from top-level in yml. # Priority of arguments differ.