From 57689a3e90074fdbae98cde4d94a56d53a1774f6 Mon Sep 17 00:00:00 2001 From: Hugh Greenberg Date: Thu, 17 Oct 2024 13:18:04 -0600 Subject: [PATCH 1/3] Added initial sqlalchemy support --- dsi/backends/sqlalchemy.py | 43 ++++++++++ dsi/backends/tests/test_sqlalchemy.py | 110 ++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 154 insertions(+) create mode 100644 dsi/backends/sqlalchemy.py create mode 100644 dsi/backends/tests/test_sqlalchemy.py diff --git a/dsi/backends/sqlalchemy.py b/dsi/backends/sqlalchemy.py new file mode 100644 index 00000000..22645012 --- /dev/null +++ b/dsi/backends/sqlalchemy.py @@ -0,0 +1,43 @@ +from typing import List +from typing import Optional +from sqlalchemy import ForeignKey +from sqlalchemy import String +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column +from sqlalchemy.orm import relationship +from sqlalchemy import create_engine +from sqlalchemy.orm import Session +import csv +import json +import re +import yaml +import toml + +from dsi.backends.filesystem import Filesystem + +class SqlAlchemy(Filesystem): + filename = "sqlite:///fs.db" + engine = None + + def __init__(self, filename, base): + self.filename = filename + self.engine = create_engine(filename, echo=True) + base.metadata.create_all(self.engine) + + def put_artifacts(self, artifact_list): + with Session(self.engine) as session: + session.add_all(artifact_list) + session.commit() + + def query(self, stmt): + results = [] + with Session(self.engine) as session: + for obj in session.scalars(stmt): + results.append(obj) + + return results + + def close(self): + if self.engine: + self.engine.dispose() diff --git a/dsi/backends/tests/test_sqlalchemy.py b/dsi/backends/tests/test_sqlalchemy.py new file mode 100644 index 00000000..5ace3424 --- /dev/null +++ b/dsi/backends/tests/test_sqlalchemy.py @@ -0,0 +1,110 @@ +import git +from collections import OrderedDict + +from dsi.backends.sqlalchemy import SqlAlchemy +from typing import List +from typing import Optional +from sqlalchemy import ForeignKey +from sqlalchemy import String +from sqlalchemy.orm import DeclarativeBase +from sqlalchemy.orm import Mapped +from sqlalchemy.orm import mapped_column +from sqlalchemy.orm import relationship +from sqlalchemy import select +import os +import subprocess +import csv + +isVerbose = True + +class Base(DeclarativeBase): + pass + + +class Wildfire(Base): + __tablename__ = "wildfire" + + id: Mapped[int] = mapped_column(primary_key=True) + files: Mapped[List["File"]] = relationship( + back_populates="wildfire", cascade="all, delete-orphan" + ) + wind_speed: Mapped[float] + wdir: Mapped[int] + smois: Mapped[float] + fuels: Mapped[str] + ignition: Mapped[str] + safe_unsafe_ignition_pattern: Mapped[str] + safe_unsafe_fire_behavior: Mapped[str] + does_fire_meet_objectives: Mapped[str] + rationale_if_unsafe: Mapped[Optional[str]] + burned_area: Mapped[int] + def __repr__(self) -> str: + return f"Wildfire(id={self.id!r})" + +class File(Base): + __tablename__ = "file" + + id: Mapped[int] = mapped_column(primary_key=True) + wildfire_id: Mapped[int] = mapped_column(ForeignKey("wildfire.id")) + path: Mapped[str] + wildfire: Mapped["Wildfire"] = relationship(back_populates="files") + def __repr__(self) -> str: + return f"File(id={self.id!r}, artifact_id={self.wildfire_id!r}, path={self.path!r})" + +def get_git_root(path): + git_repo = git.Repo(path, search_parent_directories=True) + git_root = git_repo.git.rev_parse("--show-toplevel") + return (git_root) + + +def test_wildfire_data_sql_artifact(): + engine_path = "sqlite:///wildfire.db" + store = SqlAlchemy(engine_path, Base) + store.close() + + # No error implies success + assert True + +def test_wildfire_data_csv_artifact(): + csvpath = '/'.join([get_git_root('.'), 'examples/wildfire/wildfiredata.csv']) + engine_path = "sqlite:///wildfire.db" + store = SqlAlchemy(engine_path, Base) + print(csvpath) + with open(csvpath) as csv_file: + print(csvpath) + csv_reader = csv.reader(csv_file, delimiter=',') + header = next(csv_reader) + artifacts = [] + for row in csv_reader: + row_zipped = zip(header, row) + row_dict = dict(row_zipped) + wildfire_row = Wildfire( + wind_speed=row_dict['wind_speed'], + wdir=row_dict['wdir'], + smois=row_dict['smois'], + fuels=row_dict['fuels'], + ignition=row_dict['ignition'], + safe_unsafe_ignition_pattern=row_dict['safe_unsafe_ignition_pattern'], + safe_unsafe_fire_behavior=row_dict['safe_unsafe_fire_behavior'], + does_fire_meet_objectives=row_dict['does_fire_meet_objectives'], + rationale_if_unsafe=row_dict['rationale_if_unsafe'], + burned_area=row_dict['burned_area'], + files=[File(path=row_dict['FILE'])] + ) + print(row) + artifacts.append(wildfire_row) + store.put_artifacts(artifacts) + store.close() + + # No error implies success + assert True + +def test_wildfire_artifact_query(): + engine_path = "sqlite:///wildfire.db" + store = SqlAlchemy(engine_path, Base) + stmt = select(Wildfire).where(Wildfire.burned_area > 188000) + results = store.query(stmt) + print(results) + store.close() + # No error implies success + assert True diff --git a/requirements.txt b/requirements.txt index 62b553b5..54ec9b07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ pyarrow>=12.0.1 pydantic>=2.1.1 nbconvert>=7.13.0 gitpython>=3.0.0 +sqlalchemy>=2.0.35 From 64d0f9cddc9a4fe8d893476b91aa3fda7f4fc688 Mon Sep 17 00:00:00 2001 From: Jesus Pulido Date: Thu, 24 Oct 2024 14:14:19 -0600 Subject: [PATCH 2/3] Rename requirements.txt to requirements.sqlalchemy.txt --- requirements.txt => requirements.sqlalchemy.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename requirements.txt => requirements.sqlalchemy.txt (100%) diff --git a/requirements.txt b/requirements.sqlalchemy.txt similarity index 100% rename from requirements.txt rename to requirements.sqlalchemy.txt From 532e30386731501890ad1703c2e8ebd0bccd9826 Mon Sep 17 00:00:00 2001 From: Jesus Pulido Date: Tue, 29 Oct 2024 11:58:40 -0600 Subject: [PATCH 3/3] Readded requirements.txt to validate CI --- requirements.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 requirements.txt diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..62b553b5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +pandas>=2.0.2 +pyarrow>=12.0.1 +pydantic>=2.1.1 +nbconvert>=7.13.0 +gitpython>=3.0.0