From 5ffd6eca33082d79f2358ccbee75afaff187ffdd Mon Sep 17 00:00:00 2001 From: Gorodn Watts Date: Fri, 27 Dec 2024 17:33:19 -0700 Subject: [PATCH 1/8] Add tests for endpoint dict --- .vscode/settings.json | 1 + tests/test_config.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.vscode/settings.json b/.vscode/settings.json index 2c83d14e..169856c4 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -52,6 +52,7 @@ "ncols", "ndarray", "NOQA", + "notreallyatoken", "nqueries", "ntuples", "numpy", diff --git a/tests/test_config.py b/tests/test_config.py index e24c1672..43d9f5c8 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -52,6 +52,21 @@ def test_config_read(tempdir): Configuration.read(config_path="invalid.yaml") +@patch('servicex.configuration.tempfile.gettempdir', return_value="./mytemp") +def test_config_endpoint_dict(tempdir): + os.environ['UserName'] = "p_higgs" + c = Configuration.read(config_path="tests/example_config.yaml") + endpoints = c.endpoint_dict() + assert len(endpoints) == 3 + assert "servicex-uc-af" in endpoints + + # Make sure we get back what we expect + ep = endpoints["servicex-uc-af"] + assert ep.endpoint == "https://servicex.af.uchicago.edu" + assert ep.name == "servicex-uc-af" + assert ep.token == "notreallyatoken" + + @patch('servicex.configuration.tempfile.gettempdir', return_value="./mytemp") def test_default_cache_path(tempdir): From f4b06f2e3b6121f44214ea752986c02cacb67745 Mon Sep 17 00:00:00 2001 From: Gorodn Watts Date: Sat, 28 Dec 2024 11:31:55 -0700 Subject: [PATCH 2/8] Add code to register a regular old endpoint by code. --- servicex/configuration.py | 24 +++++++++++++++++++++++- tests/conftest.py | 6 ++++++ tests/test_config.py | 23 ++++++++++++++++++++++- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/servicex/configuration.py b/servicex/configuration.py index 11de65bf..2739e8a6 100644 --- a/servicex/configuration.py +++ b/servicex/configuration.py @@ -41,6 +41,9 @@ class Endpoint(BaseModel): token: Optional[str] = "" +g_registered_endpoints: List[Endpoint] = [] + + class Configuration(BaseModel): api_endpoints: List[Endpoint] default_endpoint: Optional[str] = Field(alias="default-endpoint", default=None) @@ -101,7 +104,9 @@ def read(cls, config_path: Optional[str] = None): yaml_config = cls._add_from_path(walk_up_tree=True) if yaml_config: - return Configuration.model_validate(yaml_config) + r = Configuration.model_validate(yaml_config) + r.api_endpoints += g_registered_endpoints + return r else: path_extra = f"in {config_path}" if config_path else "" raise NameError( @@ -144,3 +149,20 @@ def _add_from_path(cls, path: Optional[Path] = None, walk_up_tree: bool = False) dir = dir.parent return config + + @classmethod + def register_endpoint(cls, ep: Endpoint): + '''Store this endpoint registration + + Args: + ep (Endpoint): The endpoint to store + ''' + global g_registered_endpoints + g_registered_endpoints.append(ep) + + @classmethod + def clear_registered_endpoints(cls): + '''Clear the list of registered endpoints. + ''' + global g_registered_endpoints + g_registered_endpoints = [] diff --git a/tests/conftest.py b/tests/conftest.py index c7ae6d61..630f698e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -218,3 +218,9 @@ def codegen_list(): 'python': 'http://servicex-code-gen-python:8000', 'uproot': 'http://servicex-code-gen-uproot:8000', 'uproot-raw': 'http://servicex-code-gen-uproot-raw:8000'} + + +@fixture(autouse=True) +def clear_registered_endpoints(): + from servicex.configuration import Configuration + Configuration.clear_registered_endpoints() diff --git a/tests/test_config.py b/tests/test_config.py index 43d9f5c8..ca518dda 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -29,7 +29,7 @@ from unittest.mock import patch import pytest -from servicex.configuration import Configuration +from servicex.configuration import Configuration, Endpoint @patch('servicex.configuration.tempfile.gettempdir', return_value="./mytemp") @@ -81,3 +81,24 @@ def test_default_cache_path(tempdir): c = Configuration.read(config_path="tests/example_config_no_cache_path.yaml") assert c.cache_path == "mytemp/servicex_p_higgs" del os.environ['USER'] + + +@patch("servicex.configuration.tempfile.gettempdir", return_value="./mytemp") +def test_config_register(tempdir): + Configuration.register_endpoint( + Endpoint( + endpoint="https://servicex.cern.ch", + name="servicex-cern", + token="notreallyatoken2", + ) + ) + + os.environ["UserName"] = "p_higgs" + c = Configuration.read(config_path="tests/example_config.yaml") + endpoints = c.endpoint_dict() + assert len(endpoints) == 4 + assert "servicex-cern" in endpoints + + # Make sure we get back what we expect + ep = endpoints["servicex-cern"] + assert ep.endpoint == "https://servicex.cern.ch" From 802fb423d3e31b0b781e9f0cc92d422d48bc3783 Mon Sep 17 00:00:00 2001 From: Gorodn Watts Date: Sat, 28 Dec 2024 23:57:57 -0500 Subject: [PATCH 3/8] TODO: Remove end points? --- servicex/servicex_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 9c1f1cb0..b25a18db 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -236,6 +236,7 @@ def __init__(self, backend=None, url=None, config_path=None): will search in local directory and up in enclosing directories """ self.config = Configuration.read(config_path) + # TODO: Remove this as an instance var (no reason to carry it around?). self.endpoints = self.config.endpoint_dict() if not url and not backend: From ed9f9f2ebc192f0d4e8a4731a3053c7884224390 Mon Sep 17 00:00:00 2001 From: Gorodn Watts Date: Tue, 31 Dec 2024 17:53:02 -0500 Subject: [PATCH 4/8] Use a protocol --- .vscode/settings.json | 1 - servicex/configuration.py | 7 ++++++- servicex/protocols.py | 31 +++++++++++++++++++++++++++++++ tests/test_config.py | 21 +++++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 servicex/protocols.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 169856c4..e701e373 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -102,7 +102,6 @@ ], "python.analysis.typeCheckingMode": "basic", "python.testing.pytestArgs": [ - "--cov=servicex", "tests" ], "python.testing.unittestEnabled": false, diff --git a/servicex/configuration.py b/servicex/configuration.py index 2739e8a6..2936ae39 100644 --- a/servicex/configuration.py +++ b/servicex/configuration.py @@ -39,6 +39,8 @@ class Endpoint(BaseModel): endpoint: str name: str token: Optional[str] = "" + # TODO: don't know how to use ServiceXAdapterProtocol here as pydantic can't handle it + adapter: Optional[object] = None g_registered_endpoints: List[Endpoint] = [] @@ -155,8 +157,11 @@ def register_endpoint(cls, ep: Endpoint): '''Store this endpoint registration Args: - ep (Endpoint): The endpoint to store + ep: Endpoint object to register ''' + # TODO: This requires exposing Endpoint + # There is no check in this setup that the adaptor is a valid ServiceXAdapterProtocol + # because I couldn't figure out how to make pydantic handle a protocol object. global g_registered_endpoints g_registered_endpoints.append(ep) diff --git a/servicex/protocols.py b/servicex/protocols.py new file mode 100644 index 00000000..b4af6c2a --- /dev/null +++ b/servicex/protocols.py @@ -0,0 +1,31 @@ +from typing import List, Protocol + +from servicex.models import CachedDataset, TransformRequest, TransformStatus + + +class ServiceXAdapterProtocol(Protocol): + async def get_transforms(self) -> List[TransformStatus]: + ... + + def get_code_generators(self): + ... + + async def get_datasets( + self, did_finder=None, show_deleted=False + ) -> List[CachedDataset]: + ... + + async def get_dataset(self, dataset_id=None) -> CachedDataset: + ... + + async def delete_dataset(self, dataset_id=None) -> bool: + ... + + async def delete_transform(self, transform_id=None): + ... + + async def submit_transform(self, transform_request: TransformRequest) -> str: + ... + + async def get_transform_status(self, request_id: str) -> TransformStatus: + ... diff --git a/tests/test_config.py b/tests/test_config.py index ca518dda..d0a47fdf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -102,3 +102,24 @@ def test_config_register(tempdir): # Make sure we get back what we expect ep = endpoints["servicex-cern"] assert ep.endpoint == "https://servicex.cern.ch" + + +@patch("servicex.configuration.tempfile.gettempdir", return_value="./mytemp") +def test_config_register_adaptor(tempdir): + 'Make sure we can do this with an adaptor' + class MyAdaptor: + pass + + Configuration.register_endpoint( + Endpoint( + name="my-adaptor", + adapter=MyAdaptor, + endpoint="", + ) + ) + + os.environ["UserName"] = "p_higgs" + c = Configuration.read(config_path="tests/example_config.yaml") + endpoints = c.endpoint_dict() + assert len(endpoints) == 4 + assert "my-adaptor" in endpoints From 899308fdb1b0dfec55e615cebdcccc4fc5cff925 Mon Sep 17 00:00:00 2001 From: Gorodn Watts Date: Wed, 1 Jan 2025 02:22:08 -0500 Subject: [PATCH 5/8] Correctly create the SX Adaptor --- servicex/servicex_client.py | 9 ++++++--- tests/test_servicex_client.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index b25a18db..6d99253e 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -254,9 +254,12 @@ def __init__(self, backend=None, url=None, config_path=None): elif backend: if backend not in self.endpoints: raise ValueError(f"Backend {backend} not defined in .servicex file") - self.servicex = ServiceXAdapter( - self.endpoints[backend].endpoint, - refresh_token=self.endpoints[backend].token, + ep = self.endpoints[backend] + self.servicex = ( + ep.adapter if ep.adapter is not None else ServiceXAdapter( + self.endpoints[backend].endpoint, + refresh_token=self.endpoints[backend].token, + ) ) self.query_cache = QueryCache(self.config) diff --git a/tests/test_servicex_client.py b/tests/test_servicex_client.py index e3c0ef70..b348d156 100644 --- a/tests/test_servicex_client.py +++ b/tests/test_servicex_client.py @@ -32,6 +32,7 @@ from servicex.query_cache import QueryCache from servicex.servicex_adapter import ServiceXAdapter from servicex.servicex_client import ServiceXClient +from servicex.configuration import Configuration, Endpoint @fixture @@ -79,3 +80,20 @@ def test_delete_transform(mock_cache, servicex_adaptor): sx = ServiceXClient(config_path="tests/example_config.yaml") sx.delete_transform("123-45-6789") servicex_adaptor.delete_transform.assert_called_once_with("123-45-6789") + + +def test_adaptor_created(mock_cache): + class my_adaptor: + pass + + my_backend = my_adaptor() + Configuration.register_endpoint( + Endpoint( + name="my-backend", + adapter=my_backend, + endpoint="", + ) + ) + + sx = ServiceXClient(config_path="tests/example_config.yaml", backend="my-backend") + assert sx.servicex == my_backend From bee8aea9baa661d9b78dac1f229a67efa1dc8cbd Mon Sep 17 00:00:00 2001 From: Gorodn Watts Date: Wed, 1 Jan 2025 21:49:06 -0500 Subject: [PATCH 6/8] Add code for minio, and a protocol --- servicex/protocols.py | 23 +++++++++++++++++++++- servicex/query_core.py | 9 +++++++-- servicex/servicex_client.py | 4 +++- tests/conftest.py | 4 +++- tests/test_dataset.py | 36 +++++++++++++++++++++++++--------- tests/test_servicex_dataset.py | 12 +++++++++++- 6 files changed, 73 insertions(+), 15 deletions(-) diff --git a/servicex/protocols.py b/servicex/protocols.py index b4af6c2a..d8675448 100644 --- a/servicex/protocols.py +++ b/servicex/protocols.py @@ -1,6 +1,7 @@ +from pathlib import Path from typing import List, Protocol -from servicex.models import CachedDataset, TransformRequest, TransformStatus +from servicex.models import CachedDataset, TransformRequest, TransformStatus, ResultFile class ServiceXAdapterProtocol(Protocol): @@ -29,3 +30,23 @@ async def submit_transform(self, transform_request: TransformRequest) -> str: async def get_transform_status(self, request_id: str) -> TransformStatus: ... + + +class MinioAdapterProtocol(Protocol): + async def list_bucket(self) -> List[ResultFile]: + ... + + async def download_file( + self, object_name: str, local_dir: str, shorten_filename: bool = False) -> Path: + ... + + async def get_signed_url(self, object_name: str) -> str: + ... + + @classmethod + def for_transform(cls, transform: TransformStatus) -> 'MinioAdapterProtocol': + ... + + @classmethod + def hash_path(cls, file_name: str) -> str: + ... diff --git a/servicex/query_core.py b/servicex/query_core.py index d58841c3..e7b4dbee 100644 --- a/servicex/query_core.py +++ b/servicex/query_core.py @@ -32,7 +32,7 @@ from abc import ABC from asyncio import Task, CancelledError import logging -from typing import List, Optional, Union +from typing import Callable, List, Optional, Union from servicex.expandable_progress import ExpandableProgress from rich.logging import RichHandler @@ -51,6 +51,7 @@ ) from servicex.query_cache import QueryCache from servicex.servicex_adapter import ServiceXAdapter +from servicex.protocols import MinioAdapterProtocol from make_it_sync import make_sync @@ -66,6 +67,7 @@ class ServiceXException(Exception): class Query: + def __init__( self, dataset_identifier: DID, @@ -74,6 +76,7 @@ def __init__( sx_adapter: ServiceXAdapter, config: Configuration, query_cache: Optional[QueryCache], + minio_generator: Callable[[TransformRequest], MinioAdapterProtocol], servicex_polling_interval: int = 5, minio_polling_interval: int = 5, result_format: ResultFormat = ResultFormat.parquet, @@ -116,6 +119,8 @@ def __init__( self.files_completed = None self._return_qastle = True + self.minio_generator = minio_generator + self.request_id = None self.ignore_cache = ignore_cache self.fail_if_incomplete = fail_if_incomplete @@ -483,7 +488,7 @@ async def retrieve_current_transform_status(self): # status. This includes the minio host and credentials. We use the # transform id as the bucket. if not self.minio: - self.minio = MinioAdapter.for_transform(self.current_status) + self.minio = self.minio_generator(self.current_status) async def download_files( self, diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 6d99253e..f91bc00b 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -40,6 +40,7 @@ ) from servicex.types import DID from servicex.dataset_group import DatasetGroup +from servicex.minio_adapter import MinioAdapter from make_it_sync import make_sync from servicex.databinder_models import ServiceXSpec, General, Sample @@ -375,12 +376,13 @@ def generic_query( dataset_identifier=dataset_identifier, sx_adapter=self.servicex, title=title, + minio_generator=MinioAdapter.for_transform, codegen=real_codegen, config=self.config, query_cache=self.query_cache, result_format=result_format, ignore_cache=ignore_cache, query_string_generator=query, - fail_if_incomplete=fail_if_incomplete + fail_if_incomplete=fail_if_incomplete, ) return qobj diff --git a/tests/conftest.py b/tests/conftest.py index 630f698e..2021559a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -37,6 +37,7 @@ TransformStatus, TransformedResults, ) +from servicex.minio_adapter import MinioAdapter from servicex.dataset_identifier import FileListDataset from servicex.minio_adapter import MinioAdapter @@ -72,7 +73,8 @@ def python_dataset(dummy_parquet_file): result_format=ResultFormat.parquet, sx_adapter=None, # type: ignore config=None, # type: ignore - query_cache=None # type: ignore + query_cache=None, # type: ignore + minio_generator=MinioAdapter.for_transform, ) # type: ignore def foo(): diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 31e3a098..e5a97117 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -49,9 +49,15 @@ async def test_as_signed_urls_happy(transformed_result): # Test when display_progress is True and provided_progress is None did = FileListDataset("/foo/bar/baz.root") - dataset = Query(dataset_identifier=did, codegen="uproot", - title="", config=None, - sx_adapter=None, query_cache=None) + dataset = Query( + dataset_identifier=did, + codegen="uproot", + title="", + config=None, + sx_adapter=None, + query_cache=None, + minio_generator=MinioAdapter.for_transform, + ) dataset.submit_and_download = AsyncMock() dataset.submit_and_download.return_value = transformed_result @@ -63,9 +69,15 @@ async def test_as_signed_urls_happy(transformed_result): async def test_as_signed_urls_happy_dataset_group(transformed_result): # Test when display_progress is True and provided_progress is None did = FileListDataset("/foo/bar/baz.root") - dataset = Query(dataset_identifier=did, codegen="uproot", - title="", config=None, - sx_adapter=None, query_cache=None) + dataset = Query( + dataset_identifier=did, + codegen="uproot", + title="", + config=None, + sx_adapter=None, + query_cache=None, + minio_generator=MinioAdapter.for_transform, + ) dataset.submit_and_download = AsyncMock() dataset.submit_and_download.return_value = transformed_result @@ -77,9 +89,15 @@ async def test_as_signed_urls_happy_dataset_group(transformed_result): @pytest.mark.asyncio async def test_as_files_happy(transformed_result): did = FileListDataset("/foo/bar/baz.root") - dataset = Query(dataset_identifier=did, codegen="uproot", - title="", config=None, - sx_adapter=None, query_cache=None) + dataset = Query( + dataset_identifier=did, + codegen="uproot", + title="", + config=None, + sx_adapter=None, + query_cache=None, + minio_generator=MinioAdapter.for_transform, + ) dataset.submit_and_download = AsyncMock() dataset.submit_and_download.return_value = transformed_result diff --git a/tests/test_servicex_dataset.py b/tests/test_servicex_dataset.py index 6acfaf69..3b2b6206 100644 --- a/tests/test_servicex_dataset.py +++ b/tests/test_servicex_dataset.py @@ -42,6 +42,7 @@ from servicex.query_core import ServiceXException, Query from servicex.servicex_client import ServiceXClient from servicex.uproot_raw.uproot_raw import UprootRawQuery +from servicex.minio_adapter import MinioAdapter transform_status = TransformStatus( **{ @@ -222,6 +223,7 @@ async def test_submit(mocker): sx_adapter=servicex, query_cache=mock_cache, config=Configuration(api_endpoints=[]), + minio_generator=MinioAdapter.for_transform, ) datasource.query_string_generator = FuncADLQuery_Uproot().FromTree("nominal") with ExpandableProgress(display_progress=False) as progress: @@ -263,6 +265,7 @@ async def test_submit_partial_success(mocker): sx_adapter=servicex, query_cache=mock_cache, config=Configuration(api_endpoints=[]), + minio_generator=MinioAdapter.for_transform, ) datasource.query_string_generator = FuncADLQuery_Uproot().FromTree("nominal") with ExpandableProgress(display_progress=False) as progress: @@ -303,6 +306,7 @@ async def test_use_of_cache(mocker): sx_adapter=servicex, query_cache=cache, config=config, + minio_generator=MinioAdapter.for_transform, ) datasource.query_string_generator = FuncADLQuery_Uproot().FromTree("nominal") datasource.result_format = ResultFormat.parquet @@ -325,6 +329,7 @@ async def test_use_of_cache(mocker): sx_adapter=servicex2, query_cache=cache, config=config, + minio_generator=MinioAdapter.for_transform, ) datasource2.query_string_generator = FuncADLQuery_Uproot().FromTree("nominal") datasource2.result_format = ResultFormat.parquet @@ -387,6 +392,7 @@ async def test_submit_cancel(mocker): sx_adapter=servicex, query_cache=mock_cache, config=Configuration(api_endpoints=[]), + minio_generator=MinioAdapter.for_transform, ) datasource.query_string_generator = FuncADLQuery_Uproot().FromTree("nominal") with ExpandableProgress(display_progress=False) as progress: @@ -425,6 +431,7 @@ async def test_submit_fatal(mocker): sx_adapter=servicex, query_cache=mock_cache, config=Configuration(api_endpoints=[]), + minio_generator=MinioAdapter.for_transform, ) datasource.query_string_generator = FuncADLQuery_Uproot().FromTree("nominal") with ExpandableProgress(display_progress=False) as progress: @@ -532,6 +539,7 @@ def test_transform_request(): sx_adapter=servicex, query_cache=None, config=Configuration(api_endpoints=[]), + minio_generator=MinioAdapter.for_transform, ) datasource.query_string_generator = (FuncADLQuery_Uproot() .FromTree("nominal") @@ -579,6 +587,7 @@ async def test_use_of_ignore_cache(mocker, servicex): sx_adapter=servicex, query_cache=cache, config=config, + minio_generator=MinioAdapter.for_transform, ) datasource_without_ignore_cache.query_string_generator = \ FuncADLQuery_Uproot().FromTree("nominal") @@ -592,7 +601,8 @@ async def test_use_of_ignore_cache(mocker, servicex): sx_adapter=servicex, query_cache=cache, config=config, - ignore_cache=True + ignore_cache=True, + minio_generator=MinioAdapter.for_transform, ) datasource_with_ignore_cache.query_string_generator = \ FuncADLQuery_Uproot().FromTree("nominal") From 7b5eef5cee9071cc9faeac783350ecffc477f297 Mon Sep 17 00:00:00 2001 From: Gorodn Watts Date: Wed, 1 Jan 2025 22:05:29 -0500 Subject: [PATCH 7/8] A test to make sure minio adaptor is properly created! --- servicex/configuration.py | 7 ++++++- servicex/servicex_client.py | 2 ++ tests/test_servicex_client.py | 21 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/servicex/configuration.py b/servicex/configuration.py index 2936ae39..3d176988 100644 --- a/servicex/configuration.py +++ b/servicex/configuration.py @@ -28,9 +28,12 @@ import os import tempfile from pathlib import Path, PurePath -from typing import List, Optional, Dict +from typing import Callable, List, Optional, Dict from pydantic import BaseModel, Field, AliasChoices, model_validator +# TODO: allow including this, but current import loop +# from servicex.models import TransformResult +from servicex.protocols import MinioAdapterProtocol import yaml @@ -41,6 +44,8 @@ class Endpoint(BaseModel): token: Optional[str] = "" # TODO: don't know how to use ServiceXAdapterProtocol here as pydantic can't handle it adapter: Optional[object] = None + # TODO: TransformResult causes an import loop, so call it object for now. + minio: Optional[Callable[[object], MinioAdapterProtocol]] = None g_registered_endpoints: List[Endpoint] = [] diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index f91bc00b..408efd3c 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -252,6 +252,7 @@ def __init__(self, backend=None, url=None, config_path=None): if url: self.servicex = ServiceXAdapter(url) + self.minio_generator = MinioAdapter.for_transform elif backend: if backend not in self.endpoints: raise ValueError(f"Backend {backend} not defined in .servicex file") @@ -262,6 +263,7 @@ def __init__(self, backend=None, url=None, config_path=None): refresh_token=self.endpoints[backend].token, ) ) + self.minio_generator = MinioAdapter.for_transform if ep.minio is None else ep.minio self.query_cache = QueryCache(self.config) self.code_generators = set(self.get_code_generators(backend).keys()) diff --git a/tests/test_servicex_client.py b/tests/test_servicex_client.py index b348d156..6f46f54c 100644 --- a/tests/test_servicex_client.py +++ b/tests/test_servicex_client.py @@ -97,3 +97,24 @@ class my_adaptor: sx = ServiceXClient(config_path="tests/example_config.yaml", backend="my-backend") assert sx.servicex == my_backend + + +def test_minio_created(mock_cache): + class my_minio: + def __init__(self, x): + self.x = x + pass + + def my_ctor_func(x): + return my_minio(x) + + Configuration.register_endpoint( + Endpoint( + name="my-backend", + endpoint="fork-it-over", + minio=my_ctor_func, + ) + ) + + sx = ServiceXClient(config_path="tests/example_config.yaml", backend="my-backend") + assert sx.minio_generator == my_ctor_func From cc943af90f5b934d682c714f1a14717c49d16027 Mon Sep 17 00:00:00 2001 From: Gordon Watts Date: Fri, 3 Jan 2025 16:58:55 -0500 Subject: [PATCH 8/8] Missed a query call - tested and fixed. --- servicex/servicex_client.py | 5 +++-- tests/test_servicex_client.py | 20 +++++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 408efd3c..ac008350 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -371,14 +371,15 @@ def generic_query( if real_codegen not in self.code_generators: raise NameError( f"{codegen} code generator not supported by serviceX " - f"deployment at {self.servicex.url}" + f"deployment at {self.servicex.url}. Supported codegens are " + f"[{', '.join(self.code_generators)}]" ) qobj = Query( dataset_identifier=dataset_identifier, sx_adapter=self.servicex, title=title, - minio_generator=MinioAdapter.for_transform, + minio_generator=self.minio_generator, codegen=real_codegen, config=self.config, query_cache=self.query_cache, diff --git a/tests/test_servicex_client.py b/tests/test_servicex_client.py index 6f46f54c..6fe8d2ac 100644 --- a/tests/test_servicex_client.py +++ b/tests/test_servicex_client.py @@ -33,6 +33,7 @@ from servicex.servicex_adapter import ServiceXAdapter from servicex.servicex_client import ServiceXClient from servicex.configuration import Configuration, Endpoint +from servicex.query_core import Query @fixture @@ -55,7 +56,7 @@ def mock_cache(mocker): } } cache_mock.return_value = mock_cache - return cache_mock + return mock_cache def test_get_datasets(mock_cache, servicex_adaptor): @@ -99,7 +100,7 @@ class my_adaptor: assert sx.servicex == my_backend -def test_minio_created(mock_cache): +def test_minio_created_and_used(mock_cache, mocker): class my_minio: def __init__(self, x): self.x = x @@ -107,7 +108,7 @@ def __init__(self, x): def my_ctor_func(x): return my_minio(x) - + Configuration.register_endpoint( Endpoint( name="my-backend", @@ -116,5 +117,18 @@ def my_ctor_func(x): ) ) + # Mock the Query class + mock_query = mocker.patch('servicex.servicex_client.Query', autospec=True) + + # Make sure externally accessible property is correct. sx = ServiceXClient(config_path="tests/example_config.yaml", backend="my-backend") assert sx.minio_generator == my_ctor_func + + # Call generic_query and verify the minio_generator argument + sx.generic_query( + dataset_identifier="some-did", + query="some-query", + codegen="ROOT" + ) + mock_query.assert_called_once() + assert mock_query.call_args[1]['minio_generator'] == my_ctor_func