Merge pull request #15 from UW-Macrostrat/update-database-library

Update database library

CHANGELOG.md

@@ -1,6 +1,16 @@
 # Changelog
 
-## [Unreleased]
+## [2.2.0] - 2024-01-04
+
+- Update application for Python 3.11
+- Update dependencies including SQLAlchemy 2
+
+## [2.0.0]
+
+- Switch dependency management to Poetry
+- Break into submodules
+
+## [Unknown]
 
 - New [notebooks](notebooks) showing
   [usage in Python](notebooks/Corelle-Basic-Usage.ipynb) and building towards

bin/test-docker

@@ -3,6 +3,12 @@
 
 set -e
 
+# Check if we're in a TTY
+TTY_FLAG=""
+if [ -t 1 ]; then
+  TTY_FLAG="-it"
+fi
+
 # Load dotenv if it exists
 if [ -f .env ]; then
   set -o allexport
@@ -44,7 +50,7 @@ docker run --rm \
 EOF
 
 # Run the tests
-docker run -t --rm --link corelle-db:database corelle /code/bin/run-tests $@
+docker run $TTY_FLAG --rm --link corelle-db:database corelle /code/bin/run-tests $@
 
 # Stop the database
 docker stop corelle-db

py-packages/client/pyproject.toml

@@ -3,18 +3,18 @@ authors = ["Daven Quinn <[email protected]>"]
 description = "An API server application for the Corelle plate-rotation system"
 license = "MIT"
 name = "corelle.client"
-version = "2.0.0"
+version = "2.1.0"
 
 packages = [{ include = "corelle" }]
 
 [tool.poetry.dependencies]
 
-geopandas = "^0.12.0"
+geopandas = "^0.14.1"
 numpy = "^1.23.4"
 numpy-quaternion = "^2022.4.2"
 pandas = "^1.5.1||^2.0.0"
-python = "^3.8"
-shapely = "^1.8.5.post1||^2.0.0"
+python = "^3.9"
+shapely = "^2.0.0"
 
 [tool.poetry.dev-dependencies]
 

py-packages/engine/corelle/engine/load_data.py

@@ -13,9 +13,10 @@
 from macrostrat.utils import working_directory
 from wget import download
 from contextlib import contextmanager
-from geoalchemy2 import func as gfunc
+from geoalchemy2 import functions as gfunc
 from geoalchemy2.shape import from_shape
 from shapely.geometry import shape, MultiPolygon
+from macrostrat.database import run_sql
 
 from .database import db
 from .query import get_sql
@@ -118,14 +119,13 @@ def import_plates(model_id, datafile, fields={}):
 
     # Run database transaction
     conn = connect()
-    trans = conn.begin()
     conn.execute(insert(__plate).values(plates))
     for poly in plate_polygons:
         conn.execute(insert(__plate_polygon).values(poly))
-    trans.commit()
+    conn.commit()
 
     # For faster updates, this materialized view could become an actual table
-    conn.execute("REFRESH MATERIALIZED VIEW corelle.plate_polygon_cache")
+    run_sql(conn, "REFRESH MATERIALIZED VIEW corelle.plate_polygon_cache")
 
 
 def create_feature(dataset, feature):
@@ -151,9 +151,8 @@ def import_features(name, features, overwrite=False):
         for i, feature in enumerate(src):
             vals.append(create_feature(name, feature))
 
-    trans = conn.begin()
     conn.execute(insert(__feature).values(vals))
-    trans.commit()
+    conn.commit()
 
     step1 = perf_counter()
     elapsed = step1 - start
@@ -164,9 +163,8 @@ def import_features(name, features, overwrite=False):
     )
 
     sql = get_sql("cache-feature-dataset")
-    trans = conn.begin()
-    conn.execute(sql, dataset_id=name)
-    trans.commit()
+    conn.execute(sql, dict(dataset_id=name))
+    conn.commit()
 
     elapsed = perf_counter() - step1
     echo(f"  cached transformed features in {elapsed:.2f} seconds")
@@ -250,7 +248,7 @@ def import_model(
     print("Importing model", name)
     conn = connect()
     q = text("SELECT count(*) FROM corelle.model WHERE name=:name")
-    res = conn.execute(q, name=name).scalar()
+    res = conn.execute(q, dict(name=name)).scalar()
     if res == 1 and not overwrite:
         print("Model has already been imported.")
         return
@@ -330,7 +328,7 @@ def load_basic_data():
             q = text(
                 "SELECT count(*) FROM (SELECT DISTINCT dataset_id FROM corelle.feature WHERE dataset_id=:name) AS a"
             )
-            res = connect().execute(q, name=dsn).scalar()
+            res = connect().execute(q, dict(name=dsn)).scalar()
             if res == 1:
                 continue
 

py-packages/engine/corelle/engine/rotate.py

@@ -89,7 +89,7 @@ def get_rotation(
                 break
     else:
         # Fall back to fetching the data ourselves
-        pairs = conn.execute(__sql, **params).fetchall()
+        pairs = conn.execute(__sql, params).fetchall()
     if len(pairs) == 0:
         return __cache(None)
     if verbose:
@@ -138,14 +138,14 @@ def get_rotation(
 
 def plates_for_model(model):
     sql = get_sql("plates-for-model")
-    for row in conn.execute(sql, model_name=model):
+    for row in conn.execute(sql, dict(model_name=model)):
         yield row[0]
 
 
 def get_plate_id(point, model, time):
     sql = get_sql("plate-for-point")
     return conn.execute(
-        sql, lon=point[0], lat=point[1], model_name=model, time=time
+        sql, dict(lon=point[0], lat=point[1], model_name=model, time=time)
     ).scalar()
 
 
@@ -170,7 +170,7 @@ def rotate_point(point, model, time):
 
 def plates_for_model(model):
     """Returns a list of plate IDs for a given model"""
-    return [k[0] for k in conn.execute(__model_plates_sql, model_name=model)]
+    return [k[0] for k in conn.execute(__model_plates_sql, dict(model_name=model))]
 
 
 def get_all_rotations(
@@ -191,14 +191,14 @@ def get_all_rotations(
     """
     if plates is None:
         if active_only:
-            res = conn.execute(__active_plates_sql, time=time, model_name=model)
+            res = conn.execute(__active_plates_sql, dict(time=time, model_name=model))
             plates = [p[0] for p in res]
         else:
             plates = plates_for_model(model)
 
     if rowset is None:
         _rowset = conn.execute(
-            __tstep_rotation_pairs, time=time, model_name=model
+            __tstep_rotation_pairs, dict(time=time, model_name=model)
         ).fetchall()
     else:
         _rowset = [
@@ -228,14 +228,16 @@ def get_rotation_series(model, *times, **kwargs):
 
     plate_ages = conn.execute(
         __plate_time_ranges,
-        model_name=model,
+        dict(model_name=model),
     ).fetchall()
 
     rowset = conn.execute(
         __model_rotation_pairs,
-        model_name=model,
-        early_age=float(max(times)),
-        late_age=float(min(times)),
+        dict(
+            model_name=model,
+            early_age=float(max(times)),
+            late_age=float(min(times)),
+        ),
     ).fetchall()
     for t in times:
         t = float(t)

py-packages/engine/corelle/tests/rotation_testing_functions.py

@@ -8,6 +8,7 @@
 from corelle.engine.database import db
 from geoalchemy2.shape import to_shape
 from geoalchemy2.elements import WKBElement
+from sqlalchemy import text
 from corelle.math import sph2cart, cart2sph
 
 identity_quaternion = Q.from_float_array([1, 0, 0, 0])
@@ -41,7 +42,7 @@ def rotate_point(point, quaternion, func="corelle.rotate_geometry"):
     sql = f"SELECT {func}(ST_SetSRID(ST_MakePoint(:lon, :lat), 4326), :quaternion)"
     # Get the result of the rotation as a WKBElement
     result = db.session.execute(
-        sql,
+        text(sql),
         params=dict(
             lon=point[0],
             lat=point[1],
@@ -60,7 +61,7 @@ def rotate_with_ob_tran(start_pos, lon_p, lat_p, lon_0):
     # Get the result of the rotation as a WKBElement
     with db.session_scope() as session:
         result = session.execute(
-            sql,
+            text(sql),
             params=dict(
                 x=start_pos[0],
                 y=start_pos[1],

py-packages/engine/corelle/tests/test_database_rotations.py

@@ -14,6 +14,7 @@
 from corelle.math import euler_to_quaternion, sph2cart, cart2sph
 from pytest import mark
 from corelle.engine.rotate import get_plate_id, get_rotation
+from sqlalchemy import text
 from .rotation_testing_functions import (
     rotation_functions,
     postgis_rotation_functions,
@@ -28,7 +29,7 @@ def test_postgis_noop_rotation():
     sql = "SELECT corelle.rotate_geometry(ST_GeomFromText('POINT(0 0)', 4326), :quaternion)"
     # Get the result of the rotation as a WKBElement
     result = db.session.execute(
-        sql, params=dict(quaternion=list(N.array(identity, dtype=float)))
+        text(sql), params=dict(quaternion=list(N.array(identity, dtype=float)))
     ).scalar()
     # Convert the WKBElement to a Shapely geometry
     geom = to_shape(WKBElement(result))
@@ -91,7 +92,7 @@ def test_postgis_euler_recovery(case):
     euler = (*case.pole, case.angle)
     q = euler_to_quaternion(euler)
     result = db.session.execute(
-        sql, params=dict(quaternion=[q.w, q.x, q.y, q.z])
+        text(sql), params=dict(quaternion=[q.w, q.x, q.y, q.z])
     ).scalar()
     if case.angle == 0:
         assert result is None
@@ -138,7 +139,7 @@ def test_postgis_geometry_rotation(func, geom):
         assert wkt.loads(geom).is_valid
 
         result = session.execute(
-            sql, params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
+            text(sql), params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
         ).scalar()
         geom = to_shape(WKBElement(result))
         print(geom.wkt)
@@ -152,7 +153,7 @@ def test_postgis_geometry_rotation_invalid(geom):
     with db.session_scope() as session:
         g0 = wkt.loads(geom)
         result = session.execute(
-            sql, params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
+            text(sql), params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
         ).scalar()
         assert result is not None
         geom = to_shape(WKBElement(result))
@@ -163,7 +164,7 @@ def _rotate_geom(geom, q, func="corelle.rotate_geometry"):
     sql = f"SELECT {func}(ST_GeomFromText(:geom, 4326), :quaternion)"
     with db.session_scope() as session:
         result = session.execute(
-            sql, params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
+            text(sql), params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
         ).scalar()
         assert result is not None
         return to_shape(WKBElement(result))
@@ -190,7 +191,7 @@ def test_inverse_rotation(geom, func):
         # Check validity of the input geometry
         g0 = wkt.loads(geom)
         result = session.execute(
-            sql, params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
+            text(sql), params=dict(geom=geom, quaternion=[q.w, q.x, q.y, q.z])
         ).scalar()
         assert result is not None
         geom = to_shape(WKBElement(result))
@@ -199,7 +200,7 @@ def test_inverse_rotation(geom, func):
         print(g0.wkt, geom.wkt)
 
         # Check that the geometry is the same as the original
-        assert g0.almost_equals(geom)
+        assert g0.equals_exact(geom, tolerance=1e-10)
 
 
 def make_quaternion(*arr):
@@ -275,7 +276,7 @@ def test_postgis_vector_rotation(q, direction):
     vx0 = Q.rotate_vectors(q, vx)
 
     result = db.session.execute(
-        "SELECT corelle.rotate_vector(:vector, :quaternion)::float[]",
+        text("SELECT corelle.rotate_vector(:vector, :quaternion)::float[]"),
         params=dict(quaternion=[q.w, q.x, q.y, q.z], vector=list(vx)),
     ).scalar()
     coords = list(result)