diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a017960..1e0fc22 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6032e17..0a38260 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -58,7 +58,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install dandischema run: python -m pip install . diff --git a/.github/workflows/test-dandi-cli.yml b/.github/workflows/test-dandi-cli.yml index 98cfc82..4605cd9 100644 --- a/.github/workflows/test-dandi-cli.yml +++ b/.github/workflows/test-dandi-cli.yml @@ -24,11 +24,10 @@ jobs: - macos-latest python: # Use the only Python which is ATM also used by dandi-api - # - 3.7 - # - 3.8 # - 3.9 # - '3.10' - '3.11' + # - '3.12' version: - master - release @@ -36,11 +35,11 @@ jobs: - normal include: - os: ubuntu-latest - python: 3.8 + python: 3.9 mode: dandi-devel version: master - os: ubuntu-latest - python: 3.8 + python: 3.9 mode: dandi-devel version: release exclude: diff --git a/.github/workflows/test-nonetwork.yml b/.github/workflows/test-nonetwork.yml index 9c50916..2f2302b 100644 --- a/.github/workflows/test-nonetwork.yml +++ b/.github/workflows/test-nonetwork.yml @@ -17,9 +17,8 @@ jobs: os: - windows-2019 - ubuntu-latest - - macos-12 + - macos-latest python: - - 3.8 - 3.9 - '3.10' - '3.11' diff --git a/.github/workflows/test-schema.yml b/.github/workflows/test-schema.yml index ff36f39..c4c1c68 100644 --- a/.github/workflows/test-schema.yml +++ b/.github/workflows/test-schema.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install dandischema run: python -m pip install . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fcc744f..cdf236a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,9 +18,8 @@ jobs: os: - windows-2019 - ubuntu-latest - - macos-12 + - macos-latest python: - - 3.8 - 3.9 - '3.10' - '3.11' diff --git a/.github/workflows/typing.yml b/.github/workflows/typing.yml index c01c56c..dd8a09e 100644 --- a/.github/workflows/typing.yml +++ b/.github/workflows/typing.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install dependencies run: | diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 36ab1b7..eca0871 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,14 +2,14 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/PyCQA/isort diff --git a/dandischema/metadata.py b/dandischema/metadata.py index 309707e..10b591b 100644 --- a/dandischema/metadata.py +++ b/dandischema/metadata.py @@ -380,13 +380,18 @@ def _add_asset_to_stats(assetmeta: Dict[str, Any], stats: _stats_type) -> None: stats["tissuesample"].append(sample) stats["dataStandard"] = stats.get("dataStandard", []) + + def add_if_missing(standard: dict) -> None: + if standard not in stats["dataStandard"]: + stats["dataStandard"].append(standard) + if "nwb" in assetmeta["encodingFormat"]: - if models.nwb_standard not in stats["dataStandard"]: - stats["dataStandard"].append(models.nwb_standard) + add_if_missing(models.nwb_standard) # TODO: RF assumption that any .json implies BIDS if set(Path(assetmeta["path"]).suffixes).intersection((".json", ".nii")): - if models.bids_standard not in stats["dataStandard"]: - stats["dataStandard"].append(models.bids_standard) + add_if_missing(models.bids_standard) + if Path(assetmeta["path"]).suffixes == [".ome", ".zarr"]: + add_if_missing(models.ome_ngff_standard) # TODO?: move/bind such helpers as .from_metadata or alike within diff --git a/dandischema/models.py b/dandischema/models.py index f328f9b..365aa69 100644 --- a/dandischema/models.py +++ b/dandischema/models.py @@ -4,7 +4,18 @@ from enum import Enum import os import re -from typing import Any, Dict, List, Literal, Optional, Sequence, Type, TypeVar, Union +from typing import ( + Annotated, + Any, + Dict, + List, + Literal, + Optional, + Sequence, + Type, + TypeVar, + Union, +) from warnings import warn from pydantic import ( @@ -23,9 +34,6 @@ ) from pydantic.json_schema import JsonSchemaValue from pydantic_core import CoreSchema -from typing_extensions import ( - Annotated, # TODO: import from `typing` when Python 3.8 support is dropped -) from zarr_checksum.checksum import InvalidZarrChecksum, ZarrDirectoryDigest from .consts import DANDI_SCHEMA_VERSION @@ -803,6 +811,11 @@ class StandardsType(BaseType): identifier="RRID:SCR_016124", ).model_dump(mode="json", exclude_none=True) +ome_ngff_standard = StandardsType( + name="OME/NGFF Standard", + identifier="DOI:10.25504/FAIRsharing.9af712", +).model_dump(mode="json", exclude_none=True) + class ContactPoint(DandiBaseModel): email: Optional[EmailStr] = Field( diff --git a/dandischema/tests/test_metadata.py b/dandischema/tests/test_metadata.py index 61ac2d7..25399f1 100644 --- a/dandischema/tests/test_metadata.py +++ b/dandischema/tests/test_metadata.py @@ -542,7 +542,7 @@ def test_aggregation_bids() -> None: { "id": "dandiasset:6668d37f-e842-4b73-8c20-082a1dd0d31a", "path": "sub-MITU01/ses-20210703h01m05s04/microscopy/sub-MITU01_" - "run-1_sample-163_stain-YO_chunk-5_spim.h5", + "run-1_sample-163_stain-YO_chunk-5_spim.ome.zarr", "access": [ {"status": "dandi:OpenAccess", "schemaKey": "AccessRequirements"} ], @@ -565,7 +565,7 @@ def test_aggregation_bids() -> None: "contentSize": 38474544973, "dateModified": "2021-07-22T23:59:16.060551-04:00", "schemaVersion": "0.4.4", - "encodingFormat": "application/x-hdf5", + "encodingFormat": "application/x-zarr", "wasGeneratedBy": [ { "id": "urn:uuid:aef77d59-7a7f-4320-9d4b-9b03f3e25e54", @@ -588,7 +588,7 @@ def test_aggregation_bids() -> None: { "id": "dandiasset:84dd580f-8d4a-43f8-bda3-6fb53fb5d3a2", "path": "sub-MITU01/ses-20210703h16m32s10/microscopy/sub-MITU01_" - "ses-20210703h16m32s10_run-1_sample-162_stain-LEC_chunk-5_spim.h5", + "ses-20210703h16m32s10_run-1_sample-162_stain-LEC_chunk-5_spim.ome.zarr", "access": [ {"status": "dandi:OpenAccess", "schemaKey": "AccessRequirements"} ], @@ -611,7 +611,7 @@ def test_aggregation_bids() -> None: "contentSize": 61774316916, "dateModified": "2021-10-01T18:28:16.038990-04:00", "schemaVersion": "0.6.0", - "encodingFormat": "application/x-hdf5", + "encodingFormat": "application/x-zarr", "wasGeneratedBy": [ { "id": "urn:uuid:8f69a248-0e6a-4fa1-8369-ae1cc63d59d8", @@ -634,3 +634,7 @@ def test_aggregation_bids() -> None: ] summary = aggregate_assets_summary(data) assert summary["numberOfSamples"] == 2 + assert ( + sum(_.get("name", "").startswith("OME/NGFF") for _ in summary["dataStandard"]) + == 1 + ) # only a single entry so we do not duplicate them diff --git a/dandischema/types.py b/dandischema/types.py index 1f2eb48..555793e 100644 --- a/dandischema/types.py +++ b/dandischema/types.py @@ -1,12 +1,11 @@ # This file is for defining types that extend existing types through the use of # `typing.Annotated`. -from typing import Type +from typing import Annotated, Type from pydantic import ByteSize, GetCoreSchemaHandler, GetJsonSchemaHandler from pydantic.json_schema import JsonSchemaValue from pydantic_core import CoreSchema, core_schema -from typing_extensions import Annotated class _ByteSizeJsonSchemaAnnotation: diff --git a/setup.cfg b/setup.cfg index 9ccdf35..dcaab7b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,6 @@ classifiers = License :: OSI Approved :: Apache Software License Operating System :: OS Independent Programming Language :: Python - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 @@ -27,12 +26,11 @@ project_urls = Source Code = https://github.com/dandi/dandischema [options] -python_requires = >=3.8 +python_requires = >=3.9 install_requires = jsonschema[format] pydantic[email] ~= 2.4 requests - typing_extensions; python_version < "3.9" zarr_checksum zip_safe = False packages = find_namespace: