From 372c2470fc61137e36b373ba1ff7e6e13760960c Mon Sep 17 00:00:00 2001 From: Baptiste O'Jeanson Date: Tue, 1 Aug 2023 16:03:58 +0200 Subject: [PATCH] WIP --- edge_orchestrator/config/.active_config | 1 - ...ker_classification_with_1_fake_camera.json | 2 +- ...er_classification_with_2_fake_cameras.json | 27 +++- .../application/api_routes.py | 144 ++++++++++-------- .../edge_orchestrator/application/config.py | 74 +++++---- .../no_active_configuration_exception.py | 19 +++ .../edge_orchestrator/application/server.py | 7 + .../binary_storage/binary_storage_factory.py | 2 +- .../filesystem_binary_storage.py | 10 +- .../infrastructure/filesystem_helpers.py | 5 +- .../filesystem_metadata_storage.py | 10 +- .../metadata_storage_factory.py | 2 +- 12 files changed, 190 insertions(+), 113 deletions(-) create mode 100644 edge_orchestrator/edge_orchestrator/application/no_active_configuration_exception.py diff --git a/edge_orchestrator/config/.active_config b/edge_orchestrator/config/.active_config index fbcf12d5..e69de29b 100644 --- a/edge_orchestrator/config/.active_config +++ b/edge_orchestrator/config/.active_config @@ -1 +0,0 @@ -toto diff --git a/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json b/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json index a7acc468..5baac647 100644 --- a/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json +++ b/edge_orchestrator/config/station_configs/marker_classification_with_1_fake_camera.json @@ -2,7 +2,7 @@ "binary_storage": { "type": "filesystem", "params": { - "src_directory_path": "/tmp/binary_storage" + "src_directory": "/tmp/vio/edge_orchestrator/data/storage" } }, "metadata_storage": { diff --git a/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json b/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json index 29981b48..f4e3dd24 100644 --- a/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json +++ b/edge_orchestrator/config/station_configs/marker_classification_with_2_fake_cameras.json @@ -1,6 +1,21 @@ { + "binary_storage": { + "type": "filesystem", + "params": { + "src_directory": "/tmp/vio/edge_orchestrator/data/storage" + } + }, + "metadata_storage": { + "type": "filesystem" + }, + "model_forward": { + "type": "fake" + }, + "telemetry_sink": { + "type": "fake" + }, "cameras": { - "camera_id3": { + "camera_#1": { "type": "fake", "source": "marker_images", "position": "back", @@ -14,11 +29,13 @@ "camera_rule": { "name": "expected_label_rule", "parameters": { - "expected_label": ["OK"] + "expected_label": [ + "OK" + ] } } }, - "camera_id4": { + "camera_#4": { "type": "fake", "source": "marker_images", "position": "back", @@ -32,7 +49,9 @@ "camera_rule": { "name": "expected_label_rule", "parameters": { - "expected_label": ["OK"] + "expected_label": [ + "OK" + ] } } } diff --git a/edge_orchestrator/edge_orchestrator/application/api_routes.py b/edge_orchestrator/edge_orchestrator/application/api_routes.py index b585ce38..ddb8209b 100644 --- a/edge_orchestrator/edge_orchestrator/application/api_routes.py +++ b/edge_orchestrator/edge_orchestrator/application/api_routes.py @@ -1,15 +1,26 @@ +from http import HTTPStatus + from fastapi import APIRouter, Depends +from starlette.responses import JSONResponse, Response from typing_extensions import Annotated +from edge_orchestrator import logger from edge_orchestrator.application.config import ( Settings, get_settings, get_metadata_storage, + get_station_config, + get_binary_storage, + get_inventory, ) -from edge_orchestrator.domain.ports.metadata_storage import MetadataStorage -from infrastructure.metadata_storage.metadata_storage_factory import ( - MetadataStorageFactory, +from edge_orchestrator.application.dto.station_config_dto import StationConfigDto +from edge_orchestrator.application.no_active_configuration_exception import ( + NoActiveConfigurationException, ) +from edge_orchestrator.domain.ports.binary_storage import BinaryStorage +from edge_orchestrator.domain.ports.inventory import Inventory +from edge_orchestrator.domain.ports.metadata_storage import MetadataStorage +from edge_orchestrator.domain.ports.station_config import StationConfig api_router = APIRouter() @@ -31,68 +42,75 @@ def read_all( @api_router.get("/items/{item_id}") def get_item( item_id: str, - metadata_storage: MetadataStorage = Depends( - MetadataStorageFactory.get_metadata_storage - ), + metadata_storage: Annotated[MetadataStorage, Depends(get_metadata_storage)], ): return metadata_storage.get_item_metadata(item_id) -# @api_router.get("/items/{item_id}/binaries/{camera_id}") -# def get_item_binary( -# item_id: str, -# camera_id: str, -# binary_storage: BinaryStorage = Depends(BinaryStorageFactory.get_binary_storage), -# ): -# content_binary = binary_storage.get_item_binary(item_id, camera_id) -# return Response( -# content=content_binary, status_code=HTTPStatus.OK, media_type="image/jpeg" -# ) -# -# -# @api_router.get("/items/{item_id}/binaries") -# def get_item_binaries( -# item_id: str, -# binary_storage: BinaryStorage = Depends(BinaryStorageFactory.get_binary_storage), -# ): -# return binary_storage.get_item_binaries(item_id) -# -# -# @api_router.get("/items/{item_id}/state") -# def get_item_state( -# item_id: str, -# metadata_storage: MetadataStorage = Depends( -# MetadataStorageFactory.get_metadata_storage -# ), -# ): -# return metadata_storage.get_item_state(item_id) - - -# @api_router.get("/inventory") -# def get_inventory(inventory: Inventory = Depends(get_inventory)): -# return inventory -# -# -# @api_router.get("/configs") -# def get_all_configs(station_config: StationConfig = Depends(get_station_config)): -# station_config.load() -# return station_config.all_configs -# -# -# @api_router.get("/configs/active") -# def get_active_config(station_config: StationConfig = Depends(get_station_config)): -# return station_config.active_config -# +@api_router.get("/items/{item_id}/binaries/{camera_id}") +def get_item_binary( + item_id: str, + camera_id: str, + binary_storage: BinaryStorage = Depends(get_binary_storage), +): + content_binary = binary_storage.get_item_binary(item_id, camera_id) + return Response( + content=content_binary, status_code=HTTPStatus.OK, media_type="image/jpeg" + ) + + # -# @api_router.post("/configs/active") -# def set_station_config( -# station_config_dto: StationConfigDto, -# station_config: StationConfig = Depends(get_station_config), -# ): -# if station_config_dto.config_name == "": -# return JSONResponse( -# status_code=403, -# content={"message": "No configuration selected!"}, -# ) -# station_config.set_station_config(station_config_dto.config_name) -# return station_config.active_config +@api_router.get("/items/{item_id}/binaries") +def get_item_binaries( + item_id: str, + binary_storage: BinaryStorage = Depends(get_binary_storage), +): + return binary_storage.get_item_binaries(item_id) + + +@api_router.get("/items/{item_id}/state") +def get_item_state( + item_id: str, + metadata_storage: MetadataStorage = Depends(get_metadata_storage), +): + return metadata_storage.get_item_state(item_id) + + +@api_router.get("/inventory") +def get_inventory(inventory: Inventory = Depends(get_inventory)): + return inventory + + +@api_router.get("/configs") +def get_all_configs(station_config: StationConfig = Depends(get_station_config)): + station_config.load() + return station_config.all_configs + + +@api_router.get("/configs/active") +def get_active_config(station_config: StationConfig = Depends(get_station_config)): + if station_config.active_config is None: + raise NoActiveConfigurationException("no_active_configuration") + return station_config.active_config + + +@api_router.post("/configs/active") +def set_station_config( + station_config_dto: StationConfigDto, + station_config: StationConfig = Depends(get_station_config), +): + if station_config_dto.config_name == "": + return JSONResponse( + status_code=403, + content={"message": "No configuration selected!"}, + ) + try: + station_config.set_station_config(station_config_dto.config_name) + except KeyError: + err_msg = ( + f"config_name '{station_config_dto.config_name}' is unknown. " + f"Valid configs are: {list(station_config.all_configs.keys())}" + ) + logger.error(err_msg) + raise NoActiveConfigurationException(err_msg) + return station_config.active_config diff --git a/edge_orchestrator/edge_orchestrator/application/config.py b/edge_orchestrator/edge_orchestrator/application/config.py index 462f9064..44e922d8 100644 --- a/edge_orchestrator/edge_orchestrator/application/config.py +++ b/edge_orchestrator/edge_orchestrator/application/config.py @@ -4,7 +4,16 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from edge_orchestrator import logger +from edge_orchestrator.application.no_active_configuration_exception import ( + NoActiveConfigurationException, +) from edge_orchestrator.domain.models.edge_station import EdgeStation +from edge_orchestrator.domain.ports.binary_storage import BinaryStorage +from edge_orchestrator.domain.ports.inventory import Inventory +from edge_orchestrator.domain.ports.metadata_storage import MetadataStorage +from edge_orchestrator.domain.ports.model_forward import ModelForward +from edge_orchestrator.domain.ports.station_config import StationConfig +from edge_orchestrator.domain.ports.telemetry_sink import TelemetrySink from edge_orchestrator.domain.use_cases.supervisor import Supervisor from edge_orchestrator.domain.use_cases.uploader import Uploader from edge_orchestrator.infrastructure.binary_storage.binary_storage_factory import ( @@ -39,7 +48,7 @@ class Settings(BaseSettings): @lru_cache() -def get_settings(): +def get_settings() -> Settings: return Settings() @@ -68,72 +77,79 @@ def get_active_config_name(active_config_path: Path) -> str: with open(active_config_path, "r") as active_config: return active_config.read().strip() else: - return "no_active_config" + return "no_active_configuration" @lru_cache() -def get_inventory(): +def get_inventory() -> Inventory: return JsonInventory(get_settings().inventory_path) @lru_cache() -def get_station_config(): +def get_station_config(setting_station_config_from_file: bool = True) -> StationConfig: settings = get_settings() - active_config_name = get_active_config_name(settings.active_config_path) station_config = JsonStationConfig( settings.station_configs_folder, get_inventory(), settings.data_folder ) + if setting_station_config_from_file: + station_config = set_station_config_from_file( + station_config, settings.active_config_path + ) + return station_config + + +def set_station_config_from_file( + station_config: StationConfig, active_config_path: Path +) -> StationConfig: + active_config_name = get_active_config_name(active_config_path) try: station_config.set_station_config(active_config_name) except KeyError: - logger.error( - f"config_name '{active_config_name}' is unknown.\n" + err_msg = ( + f"config_name '{active_config_name}' is unknown. " f"Valid configs are: {list(station_config.all_configs.keys())}" ) + logger.error(err_msg) + raise NoActiveConfigurationException(err_msg) return station_config -def get_active_config(): - station_config = get_station_config() - return station_config.active_config - - @lru_cache() -def get_binary_storage(): - active_config = get_active_config() +def get_binary_storage() -> BinaryStorage: + station_config = get_station_config() return BinaryStorageFactory.get_binary_storage( - active_config["binary_storage"].get("type"), - **active_config["binary_storage"].get("params", {}), + station_config.active_config["binary_storage"].get("type"), + **station_config.active_config["binary_storage"].get("params", {}), ) @lru_cache() -def get_metadata_storage(): - active_config = get_active_config() +def get_metadata_storage() -> MetadataStorage: + station_config = get_station_config() return MetadataStorageFactory.get_metadata_storage( - active_config["metadata_storage"]["type"], - **active_config["metadata_storage"].get("params", {}), + station_config.active_config["metadata_storage"]["type"], + **station_config.active_config["metadata_storage"].get("params", {}), ) @lru_cache() -def get_edge_station(): +def get_edge_station() -> EdgeStation: return EdgeStation(get_station_config()) @lru_cache() -def get_model_forward(): - active_config = get_active_config() +def get_model_forward() -> ModelForward: + station_config = get_station_config() return ModelForwardFactory.get_model_forward( - active_config["model_forward"]["type"], - **active_config["model_forward"].get("params", {}), + station_config.active_config["model_forward"]["type"], + **station_config.active_config["model_forward"].get("params", {}), ) @lru_cache() -def get_telemetry_sink(): - active_config = get_active_config() +def get_telemetry_sink() -> TelemetrySink: + station_config = get_station_config() return TelemetrySinkFactory.get_telemetry_sink( - active_config["telemetry_sink"]["type"], - **active_config["telemetry_sink"].get("params", {}), + station_config.active_config["telemetry_sink"]["type"], + **station_config.active_config["telemetry_sink"].get("params", {}), ) diff --git a/edge_orchestrator/edge_orchestrator/application/no_active_configuration_exception.py b/edge_orchestrator/edge_orchestrator/application/no_active_configuration_exception.py new file mode 100644 index 00000000..e80fd49c --- /dev/null +++ b/edge_orchestrator/edge_orchestrator/application/no_active_configuration_exception.py @@ -0,0 +1,19 @@ +from fastapi import Request +from fastapi.responses import JSONResponse + + +class NoActiveConfigurationException(Exception): + def __init__(self, name: str): + self.name = name + + +async def no_active_configuration_exception_handler( + request: Request, exc: NoActiveConfigurationException +): + return JSONResponse( + status_code=403, + content={ + "message": f"No active configuration selected: {exc.name}! " + "Set the active station configuration before triggering the inspection or the upload." + }, + ) diff --git a/edge_orchestrator/edge_orchestrator/application/server.py b/edge_orchestrator/edge_orchestrator/application/server.py index f901b314..4df118dc 100644 --- a/edge_orchestrator/edge_orchestrator/application/server.py +++ b/edge_orchestrator/edge_orchestrator/application/server.py @@ -2,6 +2,10 @@ from starlette.middleware.cors import CORSMiddleware from edge_orchestrator.application.api_routes import api_router +from edge_orchestrator.application.no_active_configuration_exception import ( + NoActiveConfigurationException, + no_active_configuration_exception_handler, +) from edge_orchestrator.application.trigger_routes import trigger_router @@ -9,6 +13,9 @@ def server() -> FastAPI: app = FastAPI() app.include_router(api_router) app.include_router(trigger_router) + app.add_exception_handler( + NoActiveConfigurationException, no_active_configuration_exception_handler + ) app.add_middleware( CORSMiddleware, allow_origins=["*"], diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/binary_storage_factory.py b/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/binary_storage_factory.py index 22c8ef99..cf90d5a1 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/binary_storage_factory.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/binary_storage_factory.py @@ -32,7 +32,7 @@ def get_binary_storage( **binary_storage_config: Optional[Dict[str, Any]], ) -> BinaryStorage: if not binary_storage_config: - binary_storage_config["src_directory_path"] = get_tmp_path() + binary_storage_config["src_directory"] = get_tmp_path() try: return AVAILABLE_BINARY_STORAGES[binary_storage_type]( **binary_storage_config diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/filesystem_binary_storage.py b/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/filesystem_binary_storage.py index f2e3f11d..0b0c02bf 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/filesystem_binary_storage.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/binary_storage/filesystem_binary_storage.py @@ -7,11 +7,11 @@ class FileSystemBinaryStorage(BinaryStorage): - def __init__(self, src_directory_path: Optional[Path] = None): - if src_directory_path is None: - src_directory_path = get_tmp_path() - - self.folder = src_directory_path + def __init__(self, src_directory: Optional[str] = None): + if src_directory is None: + self.folder = get_tmp_path() + else: + self.folder = Path(src_directory) def save_item_binaries(self, item: Item): path = self.folder / item.id diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py b/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py index 8f497ff1..1e0472be 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/filesystem_helpers.py @@ -1,8 +1,7 @@ -import uuid from pathlib import Path def get_tmp_path() -> Path: - tmp_path = Path(f"/tmp/{uuid.uuid4()}/vio/edge_orchestrator/data/storage") - tmp_path.mkdir(parents=True) + tmp_path = Path(f"/tmp/vio/edge_orchestrator/data/storage") + tmp_path.mkdir(parents=True, exist_ok=True) return tmp_path diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/filesystem_metadata_storage.py b/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/filesystem_metadata_storage.py index cf1b2f4d..1926371f 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/filesystem_metadata_storage.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/filesystem_metadata_storage.py @@ -8,11 +8,11 @@ class FileSystemMetadataStorage(MetadataStorage): - def __init__(self, src_directory_path: Optional[Path] = None): - if src_directory_path is None: - src_directory_path = get_tmp_path() - - self.folder = src_directory_path + def __init__(self, src_directory: Optional[str] = None): + if src_directory is None: + self.folder = get_tmp_path() + else: + self.folder = Path(src_directory) def save_item_metadata(self, item: Item): (self.folder / item.id).mkdir(parents=True, exist_ok=True) diff --git a/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/metadata_storage_factory.py b/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/metadata_storage_factory.py index 96f987cd..c2fae268 100644 --- a/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/metadata_storage_factory.py +++ b/edge_orchestrator/edge_orchestrator/infrastructure/metadata_storage/metadata_storage_factory.py @@ -36,7 +36,7 @@ def get_metadata_storage( **metadata_storage_config: Optional[Dict[str, Any]], ) -> MetadataStorage: if not metadata_storage_type: - metadata_storage_type["src_directory_path"] = get_tmp_path() + metadata_storage_type["src_directory"] = get_tmp_path() try: # return AVAILABLE_METADATA_STORAGES[metadata_storage_type]() return AVAILABLE_METADATA_STORAGES[metadata_storage_type](