Merge branch 'master' into remove-anyOf-customization

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.9'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/release.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Install dandischema
         run: python -m pip install .

.github/workflows/test-dandi-cli.yml

@@ -24,23 +24,22 @@ jobs:
           - macos-latest
         python:
           # Use the only Python which is ATM also used by dandi-api
-          # - 3.7
-          # - 3.8
           # - 3.9
           # - '3.10'
           - '3.11'
+          # - '3.12'
         version:
           - master
           - release
         mode:
           - normal
         include:
           - os: ubuntu-latest
-            python: 3.8
+            python: 3.9
             mode: dandi-devel
             version: master
           - os: ubuntu-latest
-            python: 3.8
+            python: 3.9
             mode: dandi-devel
             version: release
         exclude:

.github/workflows/test-nonetwork.yml

@@ -17,9 +17,8 @@ jobs:
         os:
           - windows-2019
           - ubuntu-latest
-          - macos-12
+          - macos-latest
         python:
-          - 3.8
           - 3.9
           - '3.10'
           - '3.11'

.github/workflows/test-schema.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install dandischema
         run: python -m pip install .

.github/workflows/test.yml

@@ -18,9 +18,8 @@ jobs:
         os:
           - windows-2019
           - ubuntu-latest
-          - macos-12
+          - macos-latest
         python:
-          - 3.8
           - 3.9
           - '3.10'
           - '3.11'

.github/workflows/typing.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install dependencies
         run: |

.pre-commit-config.yaml

@@ -2,14 +2,14 @@
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
     -   id: trailing-whitespace
     -   id: end-of-file-fixer
     -   id: check-yaml
     -   id: check-added-large-files
 -   repo: https://github.com/psf/black
-    rev: 24.8.0
+    rev: 24.10.0
     hooks:
     -   id: black
 -   repo: https://github.com/PyCQA/isort

dandischema/metadata.py

@@ -380,13 +380,18 @@ def _add_asset_to_stats(assetmeta: Dict[str, Any], stats: _stats_type) -> None:
                 stats["tissuesample"].append(sample)
 
     stats["dataStandard"] = stats.get("dataStandard", [])
+
+    def add_if_missing(standard: dict) -> None:
+        if standard not in stats["dataStandard"]:
+            stats["dataStandard"].append(standard)
+
     if "nwb" in assetmeta["encodingFormat"]:
-        if models.nwb_standard not in stats["dataStandard"]:
-            stats["dataStandard"].append(models.nwb_standard)
+        add_if_missing(models.nwb_standard)
     # TODO: RF assumption that any .json implies BIDS
     if set(Path(assetmeta["path"]).suffixes).intersection((".json", ".nii")):
-        if models.bids_standard not in stats["dataStandard"]:
-            stats["dataStandard"].append(models.bids_standard)
+        add_if_missing(models.bids_standard)
+    if Path(assetmeta["path"]).suffixes == [".ome", ".zarr"]:
+        add_if_missing(models.ome_ngff_standard)
 
 
 # TODO?: move/bind such helpers as .from_metadata or alike within

dandischema/models.py

@@ -4,7 +4,18 @@
 from enum import Enum
 import os
 import re
-from typing import Any, Dict, List, Literal, Optional, Sequence, Type, TypeVar, Union
+from typing import (
+    Annotated,
+    Any,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Sequence,
+    Type,
+    TypeVar,
+    Union,
+)
 from warnings import warn
 
 from pydantic import (
@@ -23,9 +34,6 @@
 )
 from pydantic.json_schema import JsonSchemaValue
 from pydantic_core import CoreSchema
-from typing_extensions import (
-    Annotated,  # TODO: import from `typing` when Python 3.8 support is dropped
-)
 from zarr_checksum.checksum import InvalidZarrChecksum, ZarrDirectoryDigest
 
 from .consts import DANDI_SCHEMA_VERSION
@@ -803,6 +811,11 @@ class StandardsType(BaseType):
     identifier="RRID:SCR_016124",
 ).model_dump(mode="json", exclude_none=True)
 
+ome_ngff_standard = StandardsType(
+    name="OME/NGFF Standard",
+    identifier="DOI:10.25504/FAIRsharing.9af712",
+).model_dump(mode="json", exclude_none=True)
+
 
 class ContactPoint(DandiBaseModel):
     email: Optional[EmailStr] = Field(

dandischema/tests/test_metadata.py

@@ -542,7 +542,7 @@ def test_aggregation_bids() -> None:
         {
             "id": "dandiasset:6668d37f-e842-4b73-8c20-082a1dd0d31a",
             "path": "sub-MITU01/ses-20210703h01m05s04/microscopy/sub-MITU01_"
-            "run-1_sample-163_stain-YO_chunk-5_spim.h5",
+            "run-1_sample-163_stain-YO_chunk-5_spim.ome.zarr",
             "access": [
                 {"status": "dandi:OpenAccess", "schemaKey": "AccessRequirements"}
             ],
@@ -565,7 +565,7 @@ def test_aggregation_bids() -> None:
             "contentSize": 38474544973,
             "dateModified": "2021-07-22T23:59:16.060551-04:00",
             "schemaVersion": "0.4.4",
-            "encodingFormat": "application/x-hdf5",
+            "encodingFormat": "application/x-zarr",
             "wasGeneratedBy": [
                 {
                     "id": "urn:uuid:aef77d59-7a7f-4320-9d4b-9b03f3e25e54",
@@ -588,7 +588,7 @@ def test_aggregation_bids() -> None:
         {
             "id": "dandiasset:84dd580f-8d4a-43f8-bda3-6fb53fb5d3a2",
             "path": "sub-MITU01/ses-20210703h16m32s10/microscopy/sub-MITU01_"
-            "ses-20210703h16m32s10_run-1_sample-162_stain-LEC_chunk-5_spim.h5",
+            "ses-20210703h16m32s10_run-1_sample-162_stain-LEC_chunk-5_spim.ome.zarr",
             "access": [
                 {"status": "dandi:OpenAccess", "schemaKey": "AccessRequirements"}
             ],
@@ -611,7 +611,7 @@ def test_aggregation_bids() -> None:
             "contentSize": 61774316916,
             "dateModified": "2021-10-01T18:28:16.038990-04:00",
             "schemaVersion": "0.6.0",
-            "encodingFormat": "application/x-hdf5",
+            "encodingFormat": "application/x-zarr",
             "wasGeneratedBy": [
                 {
                     "id": "urn:uuid:8f69a248-0e6a-4fa1-8369-ae1cc63d59d8",
@@ -634,3 +634,7 @@ def test_aggregation_bids() -> None:
     ]
     summary = aggregate_assets_summary(data)
     assert summary["numberOfSamples"] == 2
+    assert (
+        sum(_.get("name", "").startswith("OME/NGFF") for _ in summary["dataStandard"])
+        == 1
+    )  # only a single entry so we do not duplicate them

dandischema/types.py

@@ -1,12 +1,11 @@
 # This file is for defining types that extend existing types through the use of
 # `typing.Annotated`.
 
-from typing import Type
+from typing import Annotated, Type
 
 from pydantic import ByteSize, GetCoreSchemaHandler, GetJsonSchemaHandler
 from pydantic.json_schema import JsonSchemaValue
 from pydantic_core import CoreSchema, core_schema
-from typing_extensions import Annotated
 
 
 class _ByteSizeJsonSchemaAnnotation:

setup.cfg

@@ -12,7 +12,6 @@ classifiers =
     License :: OSI Approved :: Apache Software License
     Operating System :: OS Independent
     Programming Language :: Python
-    Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
@@ -27,12 +26,11 @@ project_urls =
     Source Code = https://github.com/dandi/dandischema
 
 [options]
-python_requires = >=3.8
+python_requires = >=3.9
 install_requires =
     jsonschema[format]
     pydantic[email] ~= 2.4
     requests
-    typing_extensions; python_version < "3.9"
     zarr_checksum
 zip_safe = False
 packages = find_namespace: