Merge pull request #35 from lsst-dm/tickets/DM-45816

DM-45816 Yet even more unit tests +947-9

Dockerfile.pytest

@@ -1,12 +1,18 @@
-ARG OBS_LSST_VERSION=w_2024_21
+ARG OBS_LSST_VERSION=w_2024_32
 FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION}
+USER root
+
+# testing.postgresql looks for pg in /usr/local...
+RUN yum install -y postgresql-server postgresql && rmdir /usr/local/bin && ln -s /usr/bin /usr/local
+
 USER lsst
-RUN source loadLSST.bash && mamba install aiokafka httpx
-RUN source loadLSST.bash && pip install kafkit aiokafka httpx pytest-asyncio
+RUN source loadLSST.bash && mamba install -y aiokafka httpx
+RUN source loadLSST.bash && pip install kafkit aiokafka httpx pytest-asyncio testing.postgresql lsst-felis
 
 WORKDIR /home/lsst/
+
 COPY --chown=lsst .  ./consdb/
 WORKDIR /home/lsst/consdb/
 RUN source /opt/lsst/software/stack/loadLSST.bash && pip install -e .
 
-ENTRYPOINT [ "/bin/bash", "-c", "source /opt/lsst/software/stack/loadLSST.bash; setup obs_lsst; pytest ."  ]
+ENTRYPOINT [ "/bin/bash", "-c", "source /opt/lsst/software/stack/loadLSST.bash; setup obs_lsst; setup felis; pytest ."  ]

python/lsst/consdb/hinfo.py

@@ -398,6 +398,24 @@ def process_resource(resource: ResourcePath, instrument_dict: dict, update: bool
         conn.commit()
 
 
+def process_local_path(path: str) -> None:
+    """Processes all yaml files in the specified path recursively.
+
+    Parameters
+    -----------
+    path : `str`
+        Path to directory that contains yaml files.
+    """
+    if os.path.isdir(path):
+        for _, _, files in os.walk(path):
+            for file in files:
+                if file.endswith(".yaml"):
+                    process_resource(ResourcePath(file), get_instrument_dict(instrument))
+    # If a yaml file is provided on the command line, process it.
+    elif os.path.isfile(path) and path.endswith(".yaml"):
+        process_resource(ResourcePath(path), get_instrument_dict(instrument))
+
+
 def process_date(day_obs: str, instrument_dict: dict, update: bool = False) -> None:
     """Process all headers from a given observation day (as YYYY-MM-DD).
 
@@ -527,7 +545,7 @@ def get_instrument_dict(instrument: str) -> dict:
             ),
         }
     else:
-        raise ValueError("Unrecognized instrument: {instrument}")
+        raise ValueError(f"Unrecognized instrument: {instrument}")
 
     return instrument_dict
 
@@ -578,5 +596,10 @@ async def main() -> None:
 
 
 if __name__ == "__main__":
+    import sys
+
     engine = setup_postgres()
-    asyncio.run(main())
+    if len(sys.argv) > 1:
+        process_local_path(sys.argv[1])
+    else:
+        asyncio.run(main())

python/lsst/consdb/pqserver.py

@@ -105,7 +105,7 @@ def _missing_(cls, value):
     docs_url=f"{path_prefix}/docs",
     redoc_url=f"{path_prefix}/redoc",
 )
-engine = setup_postgres()
+engine = None
 logger = setup_logging(__name__)
 
 ########################
@@ -123,6 +123,7 @@ def __init__(self):
         self.schemas = dict()
         self.flexible_metadata_schemas = dict()
         self.obs_id_column = dict()
+        self.timestamp_columns = dict()
         for instrument in self.instrument_list:
             md = sqlalchemy.MetaData(schema=f"cdb_{instrument}")
             md.reflect(engine)
@@ -131,11 +132,21 @@ def __init__(self):
             self.obs_id_column[instrument] = dict()
             self.flexible_metadata_schemas[instrument] = dict()
             for table in md.tables:
+                # Find all timestamp columns in the table
+                self.timestamp_columns[table] = set(
+                    [
+                        column.name
+                        for column in md.tables[table].columns
+                        if isinstance(column.type, sqlalchemy.DateTime)
+                    ]
+                )
+
                 for col_name in ObsIdColname:
                     col_name = col_name.value
                     if col_name in md.tables[table].columns:
                         self.obs_id_column[instrument][table] = col_name
                         break
+
             for obs_type in ObsTypeEnum:
                 obs_type = obs_type.value
                 table_name = f"cdb_{instrument}.{obs_type}_flexdata"
@@ -150,6 +161,26 @@ def __init__(self):
                             schema[row[0]] = row[1:]
                     self.flexible_metadata_schemas[instrument][obs_type] = schema
 
+    def get_timestamp_columns(self, table: str) -> set[str]:
+        """Returns a set containing all timestamp columns.
+
+        Given the instrument and table name, returns a set
+        of strings listing all the columns in the table that
+        are a timestamp.
+
+        Parameters
+        ----------
+        table : `str`
+            The name of the table, e.g., "cdb_latiss.exposure".
+
+        Returns
+        -------
+        `set[str]`
+            The names of all timestamp columns in the table.
+        """
+        columns = self.timestamp_columns[table]
+        return columns
+
     def refresh_flexible_metadata_schema(self, instrument: str, obs_type: str):
         schema = dict()
         schema_table = self.get_flexible_metadata_schema(instrument, obs_type)
@@ -271,7 +302,7 @@ def compute_wide_view_name(self, instrument: str, obs_type: str) -> str:
 
         Returns
         -------
-        view_nae: `str`
+        view_name: `str`
             Name of the appropriate wide view.
         """
         instrument = instrument.lower()
@@ -287,7 +318,7 @@ def compute_wide_view_name(self, instrument: str, obs_type: str) -> str:
         return view_name
 
 
-instrument_tables = InstrumentTables()
+instrument_tables = None
 
 
 ##################
@@ -467,15 +498,15 @@ class AddKeyRequestModel(BaseModel):
     )
 
     @field_validator("unit")
-    def validate_unit(v):
+    def validate_unit(v: str):
         try:
             _ = astropy.units.Unit(v)
         except ValueError:
             raise ValueError(f"'{v}' is a not a valid unit.")
         return v
 
     @field_validator("ucd")
-    def validate_ucd(v):
+    def validate_ucd(v: str):
         if not astropy.io.votable.ucd.check_ucd(v):
             raise ValueError(f"'{v}' is not a valid IVOA UCD.")
         return v
@@ -723,7 +754,6 @@ class InsertMultipleRequestModel(BaseModel):
 class InsertMultipleResponseModel(BaseModel):
     message: str = Field(title="Human-readable response message")
     instrument: str = Field(title="Instrument name (e.g., ``LATISS``)")
-    obs_type: ObsTypeEnum = Field(title="The observation type (e.g., ``exposure``)")
     obs_id: ObservationIdType | list[ObservationIdType] = Field(title="Observation ID")
     table: str = Field(title="Table name")
 
@@ -757,13 +787,21 @@ def insert_multiple(
         table_name = schema + table_name
     table_obj = instrument_tables.schemas[instrument_l].tables[table_name]
     table_name = f"cdb_{instrument_l}." + table.lower()
-    table = instrument_tables.schemas[instrument_l].tables[table_name]
     obs_id_colname = instrument_tables.obs_id_column[instrument_l][table_name]
 
+    timestamp_columns = instrument_tables.get_timestamp_columns(table_name)
+
     with engine.connect() as conn:
         for obs_id, valdict in data.obs_dict.items():
             valdict[obs_id_colname] = obs_id
 
+            # Convert timestamps in the input from string to datetime
+            for column in timestamp_columns:
+                if column in valdict and valdict[column] is not None:
+                    timestamp = valdict[column]
+                    timestamp = astropy.time.Time(timestamp, format="isot", scale="tai")
+                    valdict[column] = timestamp.to_datetime()
+
             stmt: sqlalchemy.sql.dml.Insert
             stmt = sqlalchemy.dialects.postgresql.insert(table_obj).values(valdict)
             if u != 0:
@@ -775,7 +813,7 @@ def insert_multiple(
 
     return InsertMultipleResponseModel(
         message="Data inserted",
-        table=table_name,
+        table=table,
         instrument=instrument,
         obs_id=data.obs_dict.keys(),
     )
@@ -936,3 +974,8 @@ def schema(
     if table not in schema.tables:
         raise BadValueException("table", table, list(schema.tables.keys()))
     return {c.name: [str(c.type), c.doc] for c in schema.tables[table].columns}
+
+
+if __name__ == "__main__":
+    engine = setup_postgres()
+    instrument_tables = InstrumentTables()

tests/lsstcomcamsim/ccdexposure.ecsv

@@ -0,0 +1,19 @@
+# %ECSV 1.0
+# ---
+# datatype:
+# - {name: ccdexposure_id, datatype: int64}
+# - {name: exposure_id, datatype: int64}
+# - {name: detector, datatype: int64}
+# - {name: s_region, datatype: string, subtype: json}
+# schema: astropy-2.0
+ccdexposure_id exposure_id detector s_region
+730865976064 7024040300451 0 null
+730865976065 7024040300451 1 null
+730865976066 7024040300451 2 null
+730865976067 7024040300451 3 null
+730865976068 7024040300451 4 null
+730865976069 7024040300451 5 null
+730865976070 7024040300451 6 null
+730865976071 7024040300451 7 null
+730865976072 7024040300451 8 null
+

tests/lsstcomcamsim/ccdvisit1_quicklook.ecsv

@@ -0,0 +1,46 @@
+# %ECSV 1.0
+# ---
+# datatype:
+# - {name: ccdvisit_id, datatype: int64}
+# - {name: s_ra, datatype: string, subtype: json}
+# - {name: s_dec, datatype: string, subtype: json}
+# - {name: zenith_distance, datatype: float64}
+# - {name: photometric_calib, datatype: string, subtype: json}
+# - {name: psf_sigma, datatype: float64}
+# - {name: sky_bg, datatype: float64}
+# - {name: sky_noise, datatype: float64}
+# - {name: astrom_offset_mean, datatype: float64}
+# - {name: astrom_offset_std, datatype: float64}
+# - {name: n_psf_star, datatype: int64}
+# - {name: psf_star_delta_e1_median, datatype: float64}
+# - {name: psf_star_delta_e2_median, datatype: float64}
+# - {name: psf_star_delta_e1_scatter, datatype: float64}
+# - {name: psf_star_delta_e2_scatter, datatype: float64}
+# - {name: psf_star_delta_size_median, datatype: float64}
+# - {name: psf_star_delta_size_scatter, datatype: float64}
+# - {name: psf_star_scaled_delta_size_scatter, datatype: float64}
+# - {name: psf_trace_radius_delta, datatype: float64}
+# - {name: max_dist_to_nearest_psf, datatype: float64}
+# - {name: zero_point, datatype: float64}
+# - {name: seeing_zenith_500nm, datatype: string, subtype: json}
+# - {name: mean_var, datatype: float64}
+# - {name: eff_time, datatype: string, subtype: json}
+# - {name: eff_time_psf_sigma_scale, datatype: string, subtype: json}
+# - {name: eff_time_sky_bg_scale, datatype: string, subtype: json}
+# - {name: eff_time_zero_point_scale, datatype: string, subtype: json}
+# - {name: psf_area, datatype: string, subtype: json}
+# - {name: psf_ixx, datatype: string, subtype: json}
+# - {name: psf_ixy, datatype: string, subtype: json}
+# - {name: psf_iyy, datatype: string, subtype: json}
+# schema: astropy-2.0
+ccdvisit_id s_ra s_dec zenith_distance photometric_calib psf_sigma sky_bg sky_noise astrom_offset_mean astrom_offset_std n_psf_star psf_star_delta_e1_median psf_star_delta_e2_median psf_star_delta_e1_scatter psf_star_delta_e2_scatter psf_star_delta_size_median psf_star_delta_size_scatter psf_star_scaled_delta_size_scatter psf_trace_radius_delta max_dist_to_nearest_psf zero_point seeing_zenith_500nm mean_var eff_time eff_time_psf_sigma_scale eff_time_sky_bg_scale eff_time_zero_point_scale psf_area psf_ixx psf_ixy psf_iyy
+730865976064 null null 19.908431369156446 null 1.4783571153492203 325.756928877905 18.8443699448574 0.008262017001223145 0.004718156295271102 217 0.00016299915786729263 0.00011438961203464899 0.011962956264587885 0.010951664970417795 -0.0016228945823550944 0.014533274780719559 0.009831288919328727 0.012445912191268338 514.4628010947106 31.38873096483804 null 353.2532882820847 null null null null null null null null
+730865976065 null null 20.101420893447923 null 1.4775241728998028 326.24286852031946 16.168533669485356 0.008345244097560052 0.004644486211030871 205 0.0007096202011189533 -6.279953687492439e-05 0.010025114280863591 0.010869131022067498 -0.00013780353456638306 0.011419562132594574 0.00773608179738651 0.01826433405243688 495.26432725169224 31.391624077205588 null 261.2827813341214 null null null null null null null null
+730865976066 null null 20.295261471307356 null 1.481360383220493 327.9174807704985 17.117237088100463 0.008037184779008407 0.004509928499067784 240 0.00020827345612088252 -0.0009301515235772186 0.012141335021302824 0.01204704147640473 -0.0001849443492182301 0.013509237523051394 0.009123403884200907 0.010163010271619921 575.3210439920313 31.397368599773657 null 291.87838481228005 null null null null null null null null
+730865976067 null null 20.04385574654343 null 1.4819077958347195 327.42115081846714 18.195719319686763 0.007306209156368198 0.003911773454604958 203 -0.0005030365146762635 -0.0002584813719447203 0.010691458148750244 0.009808981209256286 -0.0017265064325687751 0.011395387470475023 0.007700107967144595 0.01040220001654757 462.0034884059603 31.395384340242202 null 329.06359432923387 null null null null null null null null
+730865976068 null null 20.235612462876873 null 1.4796918868486453 330.20515786483884 16.721598318052813 0.006869899714417714 0.004140340915062614 228 0.00043941160380314096 2.30499724320982e-05 0.010656812118126873 0.00926258473422664 -0.001182418450328404 0.012192983396004468 0.00824411396790928 0.009473662935317151 453.98200950766716 31.403766055859773 null 279.144652089946 null null null null null null null null
+730865976069 null null 20.428235446662057 null 1.4808431755577847 327.98930598422885 18.01940772728306 0.007426047832899482 0.00407317264847736 219 0.0007112101408741738 0.00025940136765663716 0.008677658245206814 0.011400476261642561 -0.0007948797192933732 0.012432583745027833 0.008408350955801825 0.01922813661572187 562.6383678700041 31.398043978205017 null 323.94810586398717 null null null null null null null null
+730865976070 null null 20.181037254777436 null 1.4786767957703044 324.92875876277685 18.04389147144221 0.007065422075437092 0.0040108591151109966 204 -0.0005667716476274514 -0.000498213881041874 0.01082692511868196 0.011470010573759007 0.0006602196068982158 0.01102648496164427 0.0074530341952774885 0.011156733720228518 501.3851614050777 31.387959747786027 null 324.84230133614454 null null null null null null null null
+730865976071 null null 20.37155386161841 null 1.4825542676900207 332.9820567816496 17.435919459383122 0.007608150512381772 0.0040053085159124395 209 0.0008352543379669189 -0.0007314137363722759 0.009076289733708769 0.008944283164177712 0.0011724778113302303 0.012661660598416875 0.008543480730366567 0.02383131519393955 531.2477601301455 31.415341164852908 null 303.0631709313133 null null null null null null null null
+730865976072 null null 20.562948229915577 null 1.481181714112671 316.3169114049524 18.965357862481312 0.006607995409089556 0.0038569409113254414 177 0.0003617365886151129 -9.827991873108399e-05 0.00876221677148559 0.010159400087041156 -0.000808027201299022 0.011083649183102768 0.007473748468841092 0.018365489162744053 730.3504366105043 31.359705721805007 null 357.5558361986811 null null null null null null null null
+