diff --git a/README.md b/README.md index bd40cc52..5be7bade 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,12 @@ pip ```bash cd ./app/Entirety + pip install -e git+https://jugit.fz-juelich.de/iek-10/public/ict-platform/fiware-applications/jsonschemaparser@v0.6.2#egg=jsonschemaparser pip install -r requirements.txt ``` +> **Note:** The jsonschemaparser is a package from a repository. +> It might cause conflicts with other libs. Therefore, we install it separately. +> Please ignore the relevant ERROR message. pre-commit @@ -94,7 +98,7 @@ provide following settings in your env file. For a full list of settings see [settings](./docs/SETTINGS.md). -## User and permissions model +## User and permissions model The user and permissions model of _Entirety_ is described in the [user model documentation](./docs/USERMODEL.md). @@ -118,14 +122,14 @@ See [changelog](./docs/CHANGELOG.md) for detailed overview of changes. ## Further project information - National 5G Energy Hub National 5G Energy Hub ## Acknowledgments -We gratefully acknowledge the financial support of the Federal Ministry
-for Economic Affairs and Climate Action (BMWK), promotional references +We gratefully acknowledge the financial support of the Federal Ministry
+for Economic Affairs and Climate Action (BMWK), promotional references 03EN1030B and 03ET1561B. - BMWK BMWK diff --git a/app/Entirety/.env.EXAMPLE b/app/Entirety/.env.EXAMPLE index 4933f616..ec192de4 100644 --- a/app/Entirety/.env.EXAMPLE +++ b/app/Entirety/.env.EXAMPLE @@ -28,7 +28,7 @@ LOKI_SRC_HOST=entirety LOKI_TIMEZONE=Europe/Berlin LOCAL_AUTH=True -LOGIN_URL=/ +LOGIN_URL=/accounts/login LOGIN_REDIRECT_URL=/ LOGOUT_REDIRECT_URL=/ # OIDC diff --git a/app/Entirety/entirety/asgi.py b/app/Entirety/entirety/asgi.py index 6ff89c56..a2a13c87 100644 --- a/app/Entirety/entirety/asgi.py +++ b/app/Entirety/entirety/asgi.py @@ -10,8 +10,6 @@ import os from django.core.asgi import get_asgi_application -from pydantic_settings import SetUp -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "entirety.settings.Settings") -SetUp().configure() +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "entirety.settings") application = get_asgi_application() diff --git a/app/Entirety/entirety/settings.py b/app/Entirety/entirety/settings.py index 963d752d..32ff29ae 100644 --- a/app/Entirety/entirety/settings.py +++ b/app/Entirety/entirety/settings.py @@ -2,105 +2,103 @@ import logging.config as LOG from pathlib import Path -from typing import List, Any, Optional, Sequence, Union +from typing import List, Any, Optional, Sequence, Union, Dict, ClassVar from mimetypes import add_type import django_loki import dj_database_url -from pydantic import BaseSettings, Field, AnyUrl, validator, DirectoryPath -from pydantic_settings import PydanticSettings -from pydantic_settings.database import DatabaseDsn -from pydantic_settings.settings import ( - DatabaseSettings, - PydanticSettings, - TemplateBackendModel, +from pydantic_settings import BaseSettings, SettingsConfigDict +from pydantic import ( + Field, + AnyUrl, + validator, + DirectoryPath, + field_validator, + PostgresDsn, ) +from pydjantic import BaseDBConfig, to_django from utils.generators import generate_secret_key from django.contrib.messages import constants as messages -__version__ = "0.4.0" +__version__ = "1.1.0" +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR: DirectoryPath = Path(__file__).resolve().parent.parent -class PostgresSettings(BaseSettings): - DATABASE_USER = Field(env="DATABASE_USER", default="postgres") - DATABASE_PASSWORD = Field(env="DATABASE_PASSWORD", default="postgrespw") - DATABASE_HOST = Field(env="DATABASE_HOST", default="localhost") - DATABASE_PORT = Field(env="DATABASE_PORT", default="5432") +# MIME type issue +add_type("text/css", ".css", False) - class Config: - case_sensitive = False - env_file = ".env" - env_file_encoding = "utf-8" - -class Databases(DatabaseSettings): - - ps = PostgresSettings() - default: DatabaseDsn = Field( - default=f"postgres://{ps.DATABASE_USER}:{ps.DATABASE_PASSWORD}@{ps.DATABASE_HOST}:{ps.DATABASE_PORT}/postgres" +class PostgresDB(BaseSettings): + model_config = SettingsConfigDict( + extra="ignore", + case_sensitive=False, + env_file=".env", + env_file_encoding="utf-8", + env_prefix="DATABASE_", ) + ENGINE: str = "django.db.backends.postgresql" + HOST: str = Field(default="localhost", alias="DATABASE_HOST") + # TODO may need to add a new variable + NAME: str = Field(default="postgres", alias="DATABASE_NAME") + PASSWORD: str = Field(default="postgrespw", alias="DATABASE_PASSWORD") + PORT: int = Field(default=5432, alias="DATABASE_PORT") + USER: str = Field(default="postgres", alias="DATABASE_USER") + OPTIONS: dict = Field(default={}, alias="DATABASE_OPTIONS") + # TODO need to check + CONN_MAX_AGE: int = Field(default=0, alias="DATABASE_CONN_MAX_AGE") - @validator("*") - def format_database_settings(cls, v): - if isinstance(v, PostgresSettings): - return {} - else: - return super(Databases, cls).format_database_settings(v) - class Config: - case_sensitive = False - env_file = ".env" - env_file_encoding = "utf-8" +class Databases(BaseDBConfig): + default: PostgresDB = PostgresDB() class LokiSettings(BaseSettings): - LOKI_ENABLE: bool = Field(default=False, env="LOKI_ENABLE") - LOKI_LEVEL: str = Field(default="INFO", env="LOKI_LEVEL") - LOKI_PORT: int = Field(default=3100, env="LOKI_PORT") - LOKI_TIMEOUT: float = Field(default=0.5, env="LOKI_TIMEOUT") - LOKI_PROTOCOL: str = Field(default="http", env="LOKI_PROTOCOL") - LOKI_SRC_HOST: str = Field(default="entirety", env="LOKI_SRC_HOST") - LOKI_TIMEZONE: str = Field(default="Europe/Berlin", env="LOKI_TIMEZONE") - LOKI_HOST: str = Field(default="localhost", env="LOKI_HOST") - - class Config: - case_sensitive = False - env_file = ".env" - env_file_encoding = "utf-8" + model_config = SettingsConfigDict( + extra="ignore", case_sensitive=False, env_file=".env", env_file_encoding="utf-8" + ) + LOKI_ENABLE: bool = Field(default=False, alias="LOKI_ENABLE") + LOKI_LEVEL: str = Field(default="INFO", alias="LOKI_LEVEL") + LOKI_PORT: int = Field(default=3100, alias="LOKI_PORT") + LOKI_TIMEOUT: float = Field(default=0.5, alias="LOKI_TIMEOUT") + LOKI_PROTOCOL: str = Field(default="http", alias="LOKI_PROTOCOL") + LOKI_SRC_HOST: str = Field(default="entirety", alias="LOKI_SRC_HOST") + LOKI_TIMEZONE: str = Field(default="Europe/Berlin", alias="LOKI_TIMEZONE") + LOKI_HOST: str = Field(default="localhost", alias="LOKI_HOST") class AuthenticationSettings(BaseSettings): - LOCAL_AUTH = Field(default=True, env="LOCAL_AUTH") - - class Config: - case_sensitive = False - env_file = ".env" - env_file_encoding = "utf-8" + model_config = SettingsConfigDict( + extra="ignore", case_sensitive=False, env_file=".env", env_file_encoding="utf-8" + ) + LOCAL_AUTH: bool = Field(default=True, alias="LOCAL_AUTH") class AppLoadSettings(BaseSettings): - ENTITIES_LOAD: bool = Field(default=True, env="ENTITIES_LOAD") - DEVICES_LOAD: bool = Field(default=True, env="DEVICES_LOAD") - NOTIFICATIONS_LOAD: bool = Field(default=True, env="NOTIFICATIONS_LOAD") - SEMANTICS_LOAD: bool = Field(default=True, env="SEMANTICS_LOAD") + model_config = SettingsConfigDict( + extra="ignore", case_sensitive=False, env_file=".env", env_file_encoding="utf-8" + ) - class Config: - case_sensitive = False - env_file = ".env" - env_file_encoding = "utf-8" + ENTITIES_LOAD: bool = Field(default=True, alias="ENTITIES_LOAD") + DEVICES_LOAD: bool = Field(default=True, alias="DEVICES_LOAD") + NOTIFICATIONS_LOAD: bool = Field(default=True, alias="NOTIFICATIONS_LOAD") + SEMANTICS_LOAD: bool = Field(default=True, alias="SEMANTICS_LOAD") -class Settings(PydanticSettings): - add_type("text/css", ".css", True) +class Settings(BaseSettings): + model_config = SettingsConfigDict( + extra="ignore", + case_sensitive=False, + env_file=".env", + env_file_encoding="utf-8", + # ignored_types=["ClassVar"] + ) __auth = AuthenticationSettings() - LOCAL_AUTH = __auth.LOCAL_AUTH - LOKI = LokiSettings() - APP_LOAD = AppLoadSettings() - - # Build paths inside the project like this: BASE_DIR / 'subdir'. - BASE_DIR: DirectoryPath = Path(__file__).resolve().parent.parent + LOCAL_AUTH: bool = __auth.LOCAL_AUTH + LOKI: LokiSettings = LokiSettings() + APP_LOAD: AppLoadSettings = AppLoadSettings() - VERSION = __version__ + VERSION: str = __version__ # Application definition INSTALLED_APPS: List[str] = [ @@ -122,10 +120,10 @@ class Settings(PydanticSettings): "users", "smartdatamodels", ] + # TODO how to define constant variable + CRISPY_ALLOWED_TEMPLATE_PACKS: str = "bootstrap5" - CRISPY_ALLOWED_TEMPLATE_PACKS = "bootstrap5" - - CRISPY_TEMPLATE_PACK = "bootstrap5" + CRISPY_TEMPLATE_PACK: str = "bootstrap5" MIDDLEWARE: List[str] = [ "corsheaders.middleware.CorsMiddleware", @@ -138,9 +136,7 @@ class Settings(PydanticSettings): "django.middleware.clickjacking.XFrameOptionsMiddleware", ] - CORS_ORIGIN_ALLOW_ALL = True - - MESSAGE_TAGS = { + MESSAGE_TAGS: dict = { messages.DEBUG: "alert-info", messages.INFO: "alert-info", messages.SUCCESS: "alert-success", @@ -148,9 +144,9 @@ class Settings(PydanticSettings): messages.ERROR: "alert-danger", } - ROOT_URLCONF = "entirety.urls" - FORM_RENDERER = "django.forms.renderers.TemplatesSetting" - TEMPLATES: List[TemplateBackendModel] = [ + ROOT_URLCONF: str = "entirety.urls" + FORM_RENDERER: str = "django.forms.renderers.TemplatesSetting" + TEMPLATES: List[Dict] = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", "DIRS": ["templates"], @@ -166,7 +162,7 @@ class Settings(PydanticSettings): }, ] - WSGI_APPLICATION = "entirety.wsgi.application" + WSGI_APPLICATION: str = "entirety.wsgi.application" # Password validation # https://docs.djangoproject.com/en/4.0/ref/settings/#auth-password-validators @@ -186,16 +182,16 @@ class Settings(PydanticSettings): }, ] - AUTH_USER_MODEL = "users.User" + AUTH_USER_MODEL: str = "users.User" - USE_I18N = True + USE_I18N: bool = True - USE_TZ = True + USE_TZ: bool = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/4.0/howto/static-files/ - STATIC_URL = "static/" + STATIC_URL: str = "static/" STATICFILES_DIRS: List[DirectoryPath] = [ os.path.join(BASE_DIR, "static"), @@ -207,16 +203,17 @@ class Settings(PydanticSettings): "compressor.finders.CompressorFinder", ] - COMPRESS_PRECOMPILERS = (("text/x-scss", "django_libsass.SassCompiler"),) + COMPRESS_PRECOMPILERS: tuple = (("text/x-scss", "django_libsass.SassCompiler"),) # Default primary key field type # https://docs.djangoproject.com/en/4.0/ref/settings/#default-auto-field - DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" + DEFAULT_AUTO_FIELD: str = "django.db.models.BigAutoField" LOGGING_CONFIG: Union[str, None] = None - LOGGERS = { + # TODO more structured annotation + LOGGERS: dict = { "projects.views": { "propagate": False, "level": "INFO", @@ -247,6 +244,9 @@ class Settings(PydanticSettings): }, } + LOGGER: ClassVar + HANDLER: ClassVar + if LOKI.LOKI_ENABLE is True: for LOGGER in LOGGERS: LOGGERS[LOGGER]["handlers"] = ["loki"] @@ -271,7 +271,7 @@ class Settings(PydanticSettings): } } - LOGGING = { + LOGGING: dict = { "version": 1, "disable_existing_loggers": True, "formatters": { @@ -290,12 +290,13 @@ class Settings(PydanticSettings): # Media location # https://docs.djangoproject.com/en/4.0/howto/static-files/#serving-files # -uploaded-by-a-user-during-development - MEDIA_URL = "/media/" + MEDIA_URL: str = "/media/" # Settings provided by environment - SECRET_KEY: str = Field(default=generate_secret_key(), env="DJANGO_SECRET_KEY") + SECRET_KEY: str = Field(default=generate_secret_key(), alias="DJANGO_SECRET_KEY") - @validator("SECRET_KEY") + @field_validator("SECRET_KEY") + @classmethod def secret_key_not_empty(cls, v) -> str: v_cleaned = v.strip() if not v_cleaned: @@ -305,93 +306,95 @@ def secret_key_not_empty(cls, v) -> str: return v_cleaned # SECURITY WARNING: don't run with debug turned on in production! - DEBUG: bool = Field(default=False, env="DJANGO_DEBUG") + DEBUG: bool = Field(default=False, alias="DJANGO_DEBUG") - ALLOWED_HOSTS: List = Field(default=["*"], env="ALLOWED_HOSTS") + ALLOWED_HOSTS: List = Field(default=["*"], alias="ALLOWED_HOSTS") - CB_URL: AnyUrl = Field(default="http://localhost:1026", env="CB_URL") - MQTT_BASE_TOPIC: str = Field(default="/Entirety", env="MQTT_BASE_TOPIC") + CB_URL: AnyUrl = Field(default="http://localhost:1026", alias="CB_URL") + MQTT_BASE_TOPIC: str = Field(default="/Entirety", alias="MQTT_BASE_TOPIC") - QL_URL: AnyUrl = Field(default="http://localhost:8668", env="QL_URL") + QL_URL: AnyUrl = Field(default="http://localhost:8668", alias="QL_URL") - IOTA_URL: AnyUrl = Field(default="http://localhost:4041", env="IOTA_URL") + IOTA_URL: AnyUrl = Field(default="http://localhost:4041", alias="IOTA_URL") # CSRF - CSRF_TRUSTED_ORIGINS: list = Field(default=[], env="CSRF_TRUSTED_ORIGINS ") + CSRF_TRUSTED_ORIGINS: list = Field(default=[], alias="CSRF_TRUSTED_ORIGINS ") # Database # https://docs.djangoproject.com/en/4.0/ref/settings/#databases - DATABASES: Databases = Field({}) + DATABASES: Databases = Databases() - LOGIN_REDIRECT_URL: str = Field(default="/", env="LOGIN_REDIRECT_URL") - LOGIN_URL: str = Field(default="/", env="LOGIN_URL") - LOGOUT_REDIRECT_URL: str = Field(default="/", env="LOGOUT_REDIRECT_URL") + LOGIN_REDIRECT_URL: str = Field(default="/", alias="LOGIN_REDIRECT_URL") + LOGIN_URL: str = Field(default="/accounts/login", alias="LOGIN_URL") + LOGOUT_REDIRECT_URL: str = Field(default="/", alias="LOGOUT_REDIRECT_URL") if not __auth.LOCAL_AUTH: INSTALLED_APPS.append("mozilla_django_oidc") MIDDLEWARE.append("mozilla_django_oidc.middleware.SessionRefresh") AUTHENTICATION_BACKENDS: Sequence[str] = ("entirety.oidc.CustomOIDCAB",) - OIDC_LOGIN_URL: str = Field(default="/oidc/authenticate", env="OIDC_LOGIN_URL") + OIDC_LOGIN_URL: str = Field( + default="/oidc/authenticate", alias="OIDC_LOGIN_URL" + ) LOGIN_URL = OIDC_LOGIN_URL OIDC_LOGIN_REDIRECT_URL: str = Field( - default="/oidc/callback/", env="OIDC_LOGIN_REDIRECT_URL" + default="/oidc/callback/", alias="OIDC_LOGIN_REDIRECT_URL" ) LOGIN_REDIRECT_URL = OIDC_LOGIN_REDIRECT_URL OIDC_LOGOUT_REDIRECT_URL: str = Field( - default="/", env="OIDC_LOGOUT_REDIRECT_URL" + default="/", alias="OIDC_LOGOUT_REDIRECT_URL" ) LOGOUT_REDIRECT_URL = OIDC_LOGOUT_REDIRECT_URL - OIDC_RP_SIGN_ALGO: str = Field(default="RS256", env="OIDC_RP_SIGN_ALGO") - OIDC_OP_JWKS_ENDPOINT: str = Field(env="OIDC_OP_JWKS_ENDPOINT") + OIDC_RP_SIGN_ALGO: str = Field(default="RS256", alias="OIDC_RP_SIGN_ALGO") + OIDC_OP_JWKS_ENDPOINT: str = Field(alias="OIDC_OP_JWKS_ENDPOINT") - OIDC_RP_CLIENT_ID: str = Field(env="OIDC_RP_CLIENT_ID") - OIDC_RP_CLIENT_SECRET: str = Field(env="OIDC_RP_CLIENT_SECRET") + OIDC_RP_CLIENT_ID: str = Field(alias="OIDC_RP_CLIENT_ID") + OIDC_RP_CLIENT_SECRET: str = Field(alias="OIDC_RP_CLIENT_SECRET") OIDC_OP_AUTHORIZATION_ENDPOINT: str = Field( - env="OIDC_OP_AUTHORIZATION_ENDPOINT" + alias="OIDC_OP_AUTHORIZATION_ENDPOINT" ) - OIDC_OP_TOKEN_ENDPOINT: str = Field(env="OIDC_OP_TOKEN_ENDPOINT") - OIDC_OP_USER_ENDPOINT: str = Field(env="OIDC_OP_USER_ENDPOINT") - OIDC_OP_LOGOUT_ENDPOINT: str = Field(env="OIDC_OP_LOGOUT_ENDPOINT") + OIDC_OP_TOKEN_ENDPOINT: str = Field(alias="OIDC_OP_TOKEN_ENDPOINT") + OIDC_OP_USER_ENDPOINT: str = Field(alias="OIDC_OP_USER_ENDPOINT") + OIDC_OP_LOGOUT_ENDPOINT: str = Field(alias="OIDC_OP_LOGOUT_ENDPOINT") OIDC_OP_LOGOUT_URL_METHOD: str = Field( - default="users.views.provider_logout", env="OIDC_OP_LOGOUT_URL_METHOD" + default="users.views.provider_logout", alias="OIDC_OP_LOGOUT_URL_METHOD" ) - OIDC_STORE_ID_TOKEN: bool = Field(default=True, env="OIDC_STORE_ID_TOKEN") + OIDC_STORE_ID_TOKEN: bool = Field(default=True, alias="OIDC_STORE_ID_TOKEN") OIDC_SUPER_ADMIN_ROLE: str = Field( - default="super_admin", env="OIDC_SUPER_ADMIN_ROLE" + default="super_admin", alias="OIDC_SUPER_ADMIN_ROLE" ) OIDC_SERVER_ADMIN_ROLE: str = Field( - default="server_admin", env="OIDC_SERVER_ADMIN_ROLE" + default="server_admin", alias="OIDC_SERVER_ADMIN_ROLE" ) OIDC_PROJECT_ADMIN_ROLE: str = Field( - default="project_admin", env="OIDC_PROJECT_ADMIN_ROLE" + default="project_admin", alias="OIDC_PROJECT_ADMIN_ROLE" ) - OIDC_USER_ROLE: str = Field(default="user", env="OIDC_USER_ROLE") + OIDC_USER_ROLE: str = Field(default="user", alias="OIDC_USER_ROLE") OIDC_TOKEN_ROLE_PATH: str = Field( - default="$.entirety.roles", env="OIDC_TOKEN_ROLE_PATH" + default="$.entirety.roles", alias="OIDC_TOKEN_ROLE_PATH" ) # Internationalization # https://docs.djangoproject.com/en/4.0/topics/i18n/ - LANGUAGE_CODE: str = Field(default="en-us", env="LANGUAGE_CODE") + LANGUAGE_CODE: str = Field(default="en-us", alias="LANGUAGE_CODE") STATIC_ROOT: DirectoryPath = Field( - default=os.path.join(BASE_DIR, "cache/"), env="STATIC_ROOT" + default=os.path.join(BASE_DIR, "cache/"), alias="STATIC_ROOT" ) MEDIA_ROOT: DirectoryPath = Field( - default=os.path.join(BASE_DIR, "media/"), env="MEDIA_ROOT" + default=os.path.join(BASE_DIR, "media/"), alias="MEDIA_ROOT" ) - TIME_ZONE: str = Field(default="Europe/Berlin", env="TIME_ZONE") + TIME_ZONE: str = Field(default="Europe/Berlin", alias="TIME_ZONE") - COMPRESS_ENABLED: bool = Field(default=not DEBUG, env="COMPRESS_ENABLED") + COMPRESS_ENABLED: bool = Field(default=not DEBUG, alias="COMPRESS_ENABLED") - DJANGO_TABLES2_TEMPLATE = "django_tables2/bootstrap4.html" + DJANGO_TABLES2_TEMPLATE: str = "django_tables2/bootstrap4.html" if APP_LOAD.ENTITIES_LOAD is True: INSTALLED_APPS.append("entities") @@ -403,7 +406,5 @@ def secret_key_not_empty(cls, v) -> str: INSTALLED_APPS.append("semantics") - class Config: - case_sensitive = False - env_file = ".env" - env_file_encoding = "utf-8" + +to_django(settings=Settings()) diff --git a/app/Entirety/entirety/wsgi.py b/app/Entirety/entirety/wsgi.py index f1b1bb37..72afaf9f 100644 --- a/app/Entirety/entirety/wsgi.py +++ b/app/Entirety/entirety/wsgi.py @@ -10,10 +10,8 @@ import os from django.core.wsgi import get_wsgi_application -from pydantic_settings import SetUp -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "entirety.settings.Settings") +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "entirety.settings") -SetUp().configure() application = get_wsgi_application() diff --git a/app/Entirety/entities/requests.py b/app/Entirety/entities/requests.py index fb55513a..da375277 100644 --- a/app/Entirety/entities/requests.py +++ b/app/Entirety/entities/requests.py @@ -1,6 +1,7 @@ import json from enum import Enum - +import pydantic +from pydantic import ConfigDict import requests from django.conf import settings from django.core.exceptions import ValidationError @@ -23,6 +24,17 @@ class AttributeTypes(Enum): NUMBER = "Number" +class EntityTableItem(pydantic.BaseModel): + """ + Temporary class to store entity data for the table + """ + + model_config = ConfigDict(extra="allow") + id: str + type: str + attrs: int + + def get_entities_list(self, id_pattern, type_pattern, project): data = [] with ContextBrokerClient( @@ -35,8 +47,12 @@ def get_entities_list(self, id_pattern, type_pattern, project): for entity in cb_client.get_entity_list( id_pattern=id_pattern, type_pattern=type_pattern ): - entity_to_add = entity.copy() - entity_to_add.attrs = len(entity.dict()) - 2 + entity_to_add = EntityTableItem( + id=entity.id, + type=entity.type, + attrs=len(entity.model_dump(exclude={"id", "type"})), + **entity.model_dump(exclude={"id", "type"}) + ) data.append(entity_to_add) except requests.RequestException as err: raise err diff --git a/app/Entirety/entities/views.py b/app/Entirety/entities/views.py index 124f2929..4fdfee82 100644 --- a/app/Entirety/entities/views.py +++ b/app/Entirety/entities/views.py @@ -391,7 +391,7 @@ def post(self, request, *args, **kwargs): basic_info.fields["type"].widget.attrs["readonly"] = True attributes_form_set = formset_factory(AttributeForm, max_num=0) attributes = attributes_form_set(request.POST, prefix="attr") - context = super(Update, self).get_context_data(**kwargs) + context = self.get_context_data(**kwargs) if context["view_only"] is True: raise PermissionError context["basic_info"] = basic_info diff --git a/app/Entirety/manage.py b/app/Entirety/manage.py index a098e11e..db1ec071 100644 --- a/app/Entirety/manage.py +++ b/app/Entirety/manage.py @@ -3,14 +3,10 @@ import os import sys -from pydantic_settings import SetUp - def main(): """Run administrative tasks.""" - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "entirety.settings.Settings") - SetUp().configure() - + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "entirety.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: diff --git a/app/Entirety/requirements.txt b/app/Entirety/requirements.txt index bc45a787..4e3e6205 100644 --- a/app/Entirety/requirements.txt +++ b/app/Entirety/requirements.txt @@ -1,22 +1,18 @@ backports.zoneinfo==0.2.1;python_version<"3.9" -dj_database_url==1.0.0 +dj_database_url~=2.1.0 Django~=4.1.2 django_crispy_forms==1.14.0 django-jsonforms==1.1.2 -django-jsonform==2.17.2 django_loki==0.1.4 django_resized==1.0.2 django_tables2==2.4.1 django-cors-headers~=3.14.0 -django-pydantic-settings==0.6.3 django-compressor~=4.1 django-libsass==0.9 -filip~=0.2.5 -jsonpath_ng==1.6.1 +filip~=0.5.0 +pydjantic~=1.1.0 mozilla_django_oidc==2.0.0 -pydantic>=1.10.13,<2.0.0 crispy-bootstrap5==0.7 pre-commit==2.20.0 psycopg2==2.9.4 Pillow==10.3.0 --e git+https://jugit.fz-juelich.de/iek-10/public/ict-platform/fiware-applications/jsonschemaparser@v0.4.2#egg=jsonschemaparser diff --git a/app/Entirety/smartdatamodels/examples/sensor.json b/app/Entirety/smartdatamodels/examples/sensor.json new file mode 100644 index 00000000..f79ff3aa --- /dev/null +++ b/app/Entirety/smartdatamodels/examples/sensor.json @@ -0,0 +1,35 @@ +{ + "$id": "https://n5geh.github.io/n5geh.test-datamodel.io/Sensor.json", + "type": "object", + "$schema": "http://json-schema.org/schema#", + "required": [ + "id", + "type" + ], + "properties": { + "id": { + "type": "string" + }, + "type": { + "enum": [ + "Device" + ], + "type": "string" + }, + "modelName": { + "type": "string", + "description": "Sensor model name." + }, + "manufacturer": { + "type": "string", + "description": "Sensor manufacturer name." + }, + "documentation": { + "type": "string", + "format": "uri", + "description": "A link to documentation." + } + }, + "description": "This entity captures the static properties of common iot sensor", + "$schemaVersion": "0.0.3" +} diff --git a/app/Entirety/smartdatamodels/examples/temperatureSensor.json b/app/Entirety/smartdatamodels/examples/temperatureSensor.json new file mode 100644 index 00000000..90fdf096 --- /dev/null +++ b/app/Entirety/smartdatamodels/examples/temperatureSensor.json @@ -0,0 +1,28 @@ +{ + "$id": "https://n5geh.github.io/n5geh.test-datamodel.io/TemperatureSensor.json", + "type": "object", + "allOf": [ + { + "$ref": "https://n5geh.github.io/n5geh.test-datamodel.io/Sensor.json" + }, + { + "required": [ + "temperature" + ], + "properties": { + "type": { + "enum": [ + "TemperatureSensor" + ], + "type": "string" + }, + "temperature": { + "type": "number" + } + } + } + ], + "$schema": "http://json-schema.org/schema#", + "description": "This entity captures the static properties of common temperature sensor", + "$schemaVersion": "0.0.3" +} diff --git a/app/Entirety/utils/parser.py b/app/Entirety/utils/parser.py index eebf3f20..26e98d3e 100644 --- a/app/Entirety/utils/parser.py +++ b/app/Entirety/utils/parser.py @@ -3,11 +3,15 @@ import tempfile import json from jsonschemaparser import JsonSchemaParser +from jsonschemaparser.models import NormalizedModel from smartdatamodels.models import SmartDataModel from entities.requests import AttributeTypes import os import glob import uuid +from pydantic import AwareDatetime +from pydantic_core import PydanticUndefinedType + MANDATORY_ENTITY_FIELDS: List[str] = ["id", "type"] @@ -51,38 +55,47 @@ def parser(schema_name): ) # load json schema with JsonSchemaParser() as schema_parser: - parsed_schema = schema_parser.parse_schema(schema=path) + model_class_pydantic = ( + schema_parser.__getitem__( + schema_parser.parse_schema(schema=path, model_class=NormalizedModel) + ) + .dict() + .get("pydantic_class") + ) delete_json_files_in_temp() - return parsed_schema + return model_class_pydantic.model_fields, data_model def parse_entity(schema_name): - parsed_schema = parser(schema_name) - # clean up the temporary json files - data_model = parsed_schema.datamodel - entity_json = extract_id_and_type(data_model) - for key, value in data_model.__fields__.items(): + model, name = parser(schema_name) + entity_json = extract_id_and_type(model, name) + for key, value in model.items(): # check for id and type if key not in MANDATORY_ENTITY_FIELDS: - entity_json[key] = {"type": type_mapping(value), "value": value.default} + entity_json[key] = { + "type": type_mapping(value.annotation), + "value": ( + None + if isinstance(value.default, PydanticUndefinedType) + else value.default + ), + } return entity_json def parse_device(schema_name): - parsed_schema = parser(schema_name) - # clean up the temporary json files - data_model = parsed_schema.datamodel - entity_json = extract_id_and_type(data_model) + model, name = parser(schema_name) + entity_json = extract_id_and_type(model, name) device_json = { "entity_name": entity_json["id"], "entity_type": entity_json["type"], "device_id": None, + "attributes": [], } - for key, value in data_model.__fields__.items(): - device_json["attributes"] = [] + for key, value in model.items(): if key not in MANDATORY_ENTITY_FIELDS: device_json["attributes"].append( - {"type": type_mapping(value), "name": key, "object_id": None} + {"type": type_mapping(value.annotation), "name": key, "object_id": None} ) return device_json @@ -100,26 +113,29 @@ def parse_device(schema_name): # return entity_json -def extract_id_and_type(model): +def extract_id_and_type(model, name): # TODO generation of ID and Type is not robust entity_json = {} - for key, value in model.__fields__.items(): + for key, value in model.items(): if key == "id": unique_id = str(uuid.uuid4())[:4] # Truncate to the first 4 characters - entity_json["id"] = f"{model.__name__}:{unique_id}" + entity_json["id"] = f"{name}:{unique_id}" elif key == "type": - entity_json["type"] = model.__name__ + entity_json["type"] = name return entity_json def type_mapping(value): - if value.type_ == (datetime.datetime or Optional[datetime.datetime]): + if ( + value == (datetime.datetime or Optional[datetime.datetime]) + or value == Optional[AwareDatetime] + ): return AttributeTypes.DATETIME.value - elif value.type_ == (str or Optional[str]): + elif value == (str or Optional[str]): return AttributeTypes.STRING.value - elif value.type_ == (int or Optional[int]): + elif value == (int or Optional[int]): return AttributeTypes.NUMBER.value - elif value.type_ == (float or Optional[float]): + elif value == (float or Optional[float]): return AttributeTypes.FLOAT.value else: return AttributeTypes.STRING.value diff --git a/docker/Dockerfile b/docker/Dockerfile index 231c17d1..c496ccb4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,9 +20,8 @@ EXPOSE 8000 # Get pip to download and install requirements: RUN python -m pip install --upgrade pip RUN pip install uwsgi +RUN pip install -e git+https://jugit.fz-juelich.de/iek-10/public/ict-platform/fiware-applications/jsonschemaparser@v0.6.2#egg=jsonschemaparser RUN pip install --no-cache-dir -r ./requirements.txt -RUN pip uninstall -y pydantic -RUN pip install pydantic[dotenv]==1.7.2 # Runner script here RUN chmod +x ./runner.sh