Use GSON config

openworkload · Dec 22, 2024 · 16ca10a · 16ca10a
1 parent fedc6cb
commit 16ca10a
Show file tree

Hide file tree

Showing 7 changed files with 76 additions and 20 deletions.
diff --git a/config/cloud-gate.json b/config/cloud-gate.json
@@ -0,0 +1,15 @@
+{
+  "base": {
+    "cache_expire": 7200000,
+    "cache_dir": "~/.cache/swm"
+  },
+
+  "azure": {
+    "storage": {
+       "container_name": ""
+    }
+  },
+
+  "openstack": {
+  }
+}
diff --git a/swmcloudgate/cache.py b/swmcloudgate/cache.py
@@ -1,3 +1,4 @@
+import os
 import atexit
 import pickle  # nosec B403
 import logging
@@ -33,12 +34,12 @@ def __del__(self) -> None:
 
     @property
     def expire(self) -> int:
-        return self._settings.cache_expire
+        return self._settings.base.cache_expire
 
     def fetch_and_update(self, key: list[str]) -> list[BaseModel] | None:
         LOG.debug(f"Try to fetch {self._data_kind} from in-memory cache by key: {key}")
         for timestamp, cached_key, cached_value in self._data[:]:
-            if timestamp >= datetime.now() - timedelta(seconds=self._settings.cache_expire):
+            if timestamp >= datetime.now() - timedelta(seconds=self._settings.base.cache_expire):
                 if key == cached_key:
                     return cached_value
         return None
@@ -51,7 +52,7 @@ def update(self, key: list[str], value: list[BaseModel]) -> tuple[int, int]:
         data: list[tuple[datetime, list[str], list[BaseModel]]] = []
         found = False
         now = datetime.now()
-        fresh_timestamp = now - timedelta(seconds=self._settings.cache_expire)
+        fresh_timestamp = now - timedelta(seconds=self._settings.base.cache_expire)
         for timestamp, cached_key, cached_value in self._data:
             if cached_key == key:
                 if timestamp < fresh_timestamp:
@@ -78,9 +79,11 @@ def update(self, key: list[str], value: list[BaseModel]) -> tuple[int, int]:
         return changed, deleted
 
     def _load_from_filesystem(self) -> tuple[list[tuple[datetime, list[str], list[BaseModel]]], Path]:
-        LOG.debug(f"Load cache data from directory: {self._settings.cache_dir}")
+        LOG.debug(f"Load cache data from directory: {self._settings.base.cache_dir}")
         data: list[tuple[datetime, list[str], list[BaseModel]]] = []
-        cache_file_path = Path(f"{self._settings.cache_dir}/cloud-gate-{self._data_provider}-{self._data_kind}.dat")
+        cache_file_path = Path(
+            f"{self._settings.base.cache_dir}/cloud-gate-{self._data_provider}-{self._data_kind}.dat"
+        )
         if cache_file_path.exists():
             data = self._read(cache_file_path)
         else:
@@ -109,8 +112,12 @@ def _write(self, file_path: Path, data: list[tuple[datetime, list[str], list[Bas
 
 
 @lru_cache(maxsize=64)
-def data_cache(data_kind: str, data_provider: str, cache_dir: str = "") -> Cache:
-    settings = config.get_settings(cache_dir) if cache_dir else config.get_settings()
+def data_cache(
+    data_kind: str,
+    data_provider: str,
+    config_file: Path = Path(os.path.expanduser("~/.swm/cloud-gate.json")),
+) -> Cache:
+    settings = config.get_settings(config_file)
     return Cache(data_kind, data_provider, settings)
 
 

diff --git a/swmcloudgate/config.py b/swmcloudgate/config.py
@@ -1,19 +1,37 @@
 import os
+import json
+from pathlib import Path
 from functools import lru_cache
 
-from pydantic import BaseSettings
+from pydantic import Field, BaseModel, ConfigDict, BaseSettings
 
-USER_DIR = os.path.expanduser("~")
 
+class AzureStorage(BaseModel):
+    account: str
+    token: str
 
-class Settings(BaseSettings):
-    cache_expire: int = 14 * 24 * 3600
-    cache_dir: str
 
-    class Config:
-        env_file = f"{USER_DIR}/.swm/cloud-gate.conf"
+class AzureSettings(BaseModel):
+    storage: AzureStorage
+
+
+class OpenStackSettings(BaseModel):
+    pass
+
+
+class BaseConfig(BaseModel):
+    cache_expire: int = Field(14 * 24 * 3600)
+    cache_dir: str = Field(os.path.expanduser("~/.cache/swm"))
+
+
+class Settings(BaseSettings):
+    base: BaseConfig
+    azure: AzureSettings | None = None
+    openstack: OpenStackSettings | None = None
 
 
 @lru_cache()
-def get_settings(cache_dir: str = f"{USER_DIR}/.cache/swm") -> Settings:
-    return Settings(cache_dir=cache_dir)
+def get_settings(config_file: Path) -> Settings:
+    with open(config_file, "r") as file:
+        config_data = json.load(file)
+    return Settings(**config_data)
diff --git a/swmcloudgate/routers/openstack/converters.py b/swmcloudgate/routers/openstack/converters.py
@@ -47,7 +47,15 @@ def convert_to_partition(stack: typing.Dict[str, typing.Any]) -> PartInfo:
 
 
 def convert_to_flavor(data: NodeSize) -> Flavor:
-    return Flavor(id=data.id, name=data.name, cpus=data.vcpus, mem=data.ram, storage=data.disk, price=data.price)
+    return Flavor(
+        id=data.id,
+        name=data.name,
+        cpus=data.vcpus,
+        gpus=0,
+        mem=data.ram,
+        storage=data.disk,
+        price=data.price,
+    )
 
 
 def convert_to_image(image: NodeImage) -> ImageInfo:

diff --git a/test/test_azure.py b/test/test_azure.py
@@ -59,6 +59,7 @@ async def test_list_flavors(self):
                 "flavors": [
                     {
                         "cpus": 2,
+                        "gpus": 0,
                         "id": "9348abe1-2a12-4ba7-9942-920a58fa887f",
                         "mem": 1073,
                         "name": "flavor1",
@@ -67,6 +68,7 @@ async def test_list_flavors(self):
                     },
                     {
                         "cpus": 8,
+                        "gpus": 0,
                         "id": "5acfa3a8-991b-4e5e-822b-3fadbfc93f9a",
                         "mem": 2147,
                         "name": "flavor2",

diff --git a/test/test_cache.py b/test/test_cache.py
@@ -1,8 +1,9 @@
+import json
 import pickle
 import unittest
 from pathlib import Path
 from datetime import datetime, timedelta
-from tempfile import TemporaryDirectory
+from tempfile import NamedTemporaryFile, TemporaryDirectory
 
 from swmcloudgate import cache
 from swmcloudgate.routers.models import BaseModel
@@ -12,7 +13,11 @@ class TestCache(unittest.TestCase):
     def setUp(self):
         self._cache_dir = Path(TemporaryDirectory().name)
         self._cache_dir.mkdir(parents=True, exist_ok=True)
-        self._cache = cache.data_cache("test_data_kind", "test", str(self._cache_dir))
+        self._config_file = NamedTemporaryFile()
+        data = {"base": {"cache_dir": self._cache_dir.as_posix()}}
+        with open(self._config_file.name, "w") as json_file:
+            json.dump(data, json_file, indent=4)
+        self._cache = cache.data_cache("test_data_kind", "test", Path(self._config_file.name))
 
     def test_fetch_and_update(self):
         self.assertIsNone(self._cache.fetch_and_update(["key1", "key2"]))

diff --git a/test/test_openstack.py b/test/test_openstack.py
@@ -52,9 +52,10 @@ async def test_list_flavors(self):
             data,
             {
                 "flavors": [
-                    {"cpus": 2, "id": "p1", "mem": 1073741824, "name": "flavor1", "price": 3.0, "storage": 12884901888},
+                    {"cpus": 2, "id": "p1", "gpus": 0, "mem": 1073741824, "name": "flavor1", "price": 3.0, "storage": 12884901888},
                     {
                         "cpus": 8,
+                        "gpus": 0,
                         "id": "p2",
                         "mem": 2147483648,
                         "name": "flavor2",