Merge pull request #12 from lsst-dm/tickets/DM-44099d

DM-44099: Fix query method output and update flex schema cache.

python/lsst/consdb/pqserver.py

@@ -526,23 +526,33 @@ def insert_flexible_metadata(
     table = instrument_tables.get_flexible_metadata_table(instrument, obs_type)
     schema = instrument_tables.flexible_metadata_schemas[instrument][obs_type]
 
-    with engine.connect() as conn:
-        value_dict = info["values"]
-        for key, value in value_dict.items():
+    value_dict = info["values"]
+    for key, value in value_dict.items():
+        if key not in schema:
+            # Refresh cached schema
+            schema = dict()
+            schema_table = instrument_tables.get_flexible_metadata_schema(instrument, obs_type)
+            schema_stmt = sqlalchemy.select(schema_table.c["key", "dtype", "doc", "unit", "ucd"])
+            with engine.connect() as conn:
+                for row in conn.execute(schema_stmt):
+                    schema[row[0]] = row[1:]
+            instrument_tables.flexible_metadata_schemas[instrument][obs_type] = schema
             if key not in schema:
                 raise BadValueException("key", key, list(schema.keys()))
 
-            # check value against dtype
-            dtype = schema[key][0]
-            if dtype == "bool" and not isinstance(value, bool):
-                raise BadValueException("bool value", value)
-            elif dtype == "int" and not isinstance(value, int):
-                raise BadValueException("int value", value)
-            elif dtype == "float" and not isinstance(value, float):
-                raise BadValueException("float value", value)
-            elif dtype == "str" and not isinstance(value, str):
-                raise BadValueException("str value", value)
+        # check value against dtype
+        dtype = schema[key][0]
+        if dtype == "bool" and not isinstance(value, bool):
+            raise BadValueException("bool value", value)
+        elif dtype == "int" and not isinstance(value, int):
+            raise BadValueException("int value", value)
+        elif dtype == "float" and not isinstance(value, float):
+            raise BadValueException("float value", value)
+        elif dtype == "str" and not isinstance(value, str):
+            raise BadValueException("str value", value)
 
+    with engine.connect() as conn:
+        for key, value in value_dict.items():
             value_str = str(value)
             stmt: sqlalchemy.sql.dml.Insert
             if request.args and request.args.get("u") == "1":
@@ -627,7 +637,7 @@ def insert(instrument: str) -> dict[str, Any] | tuple[dict[str, str], int]:
 
 
 @app.post("/consdb/query")
-def query() -> list[list[Any]] | tuple[dict[str, str], int]:
+def query() -> dict[str, Any] | tuple[dict[str, str], int]:
     """Query the ConsDB database.
 
     Parameters
@@ -637,9 +647,11 @@ def query() -> list[list[Any]] | tuple[dict[str, str], int]:
 
     Returns
     -------
-    json_dict: `list` [ `list` [ `Any` ] ]
+    json_dict: `dict` [ `str`, `Any` ]
         JSON response with 200 HTTP status on success.
-        Response is a list of rows of column values.
+        Response is a dict with a ``columns`` key with value being a list
+        of string column names and a ``data`` key with value being a list
+        of rows.
 
     Raises
     ------
@@ -651,17 +663,19 @@ def query() -> list[list[Any]] | tuple[dict[str, str], int]:
     with engine.connect() as conn:
         cursor = conn.exec_driver_sql(info["query"])
         first = True
-        result = []
+        result = {}
+        rows = []
         for row in cursor:
             if first:
-                result.append(list(row._fields))
+                result["columns"] = list(row._fields)
                 first = False
-            result.append(list(row))
+            rows.append(list(row))
+        result["data"] = rows
     return result
 
 
 @app.get("/consdb/schema/<instrument>/<table>")
-def schema(instrument: str, table: str) -> dict[str, list[str, str]]:
+def schema(instrument: str, table: str) -> dict[str, list[str]]:
     """Retrieve the descriptions of columns in a ConsDB table.
 
     Parameters
@@ -673,7 +687,7 @@ def schema(instrument: str, table: str) -> dict[str, list[str, str]]:
 
     Returns
     -------
-    json_dict: `dict` [ `str`, `list` [ `str`, `str` ] ]
+    json_dict: `dict` [ `str`, `list` [ `str` ] ]
         JSON response with 200 HTTP status on success.
         Response is a dict with column names as keys and lists of data type
         and documentation strings as values.

tests/test_pqserver.py

@@ -212,7 +212,7 @@ def test_flexible_metadata(client):
     response = client.post("/consdb/query", json={"query": "SELECT * FROM exposure ORDER BY day_obs;"})
     _assert_http_status(response, 200)
     result = response.json
-    assert len(result) == 3
-    assert "exposure_id" in result[0]
-    assert 20240321 in result[1]
-    assert "AT_O_20240327_000002" in result[2]
+    assert len(result) == 2
+    assert "exposure_id" in result["columns"]
+    assert 20240321 in result["data"][0]
+    assert "AT_O_20240327_000002" in result["data"][1]