DM-43722 EFD Transform Implementation

.github/workflows/build.yaml

@@ -31,3 +31,10 @@ jobs:
         image: ${{ github.repository }}-pq
         github_token: ${{ secrets.GITHUB_TOKEN }}
         dockerfile: Dockerfile.pqserver
+
+    - name: Build efdtransform
+      uses: lsst-sqre/build-and-push-to-ghcr@v1
+      with:
+        image: ${{ github.repository }}-efdtransform
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        dockerfile: Dockerfile.efdtransform

.pre-commit-config.yaml

@@ -15,7 +15,7 @@ repos:
         # supported by your project here, or alternatively use
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
-        language_version: python3.12
+        language_version: python3.11
   - repo: https://github.com/pycqa/isort
     rev: 5.13.2
     hooks:

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "editor.formatOnSave": true,
+    "[python]": {
+        "editor.tabSize": 4,
+        "editor.rulers": [
+            79,
+            110
+        ],
+    },
+    "python.analysis.extraPaths": [
+        "./python",
+        "./python",
+        "./python/lsst"
+    ]
+}

Dockerfile.efdtransform

@@ -0,0 +1,46 @@
+# Define the LSST version
+ARG OBS_LSST_VERSION=w_2024_47
+FROM lsstsqre/centos:7-stack-lsst_distrib-${OBS_LSST_VERSION}
+
+# Set user
+USER lsst
+
+# Install dependencies
+RUN source loadLSST.bash && mamba install -y aiokafka httpx && \
+    pip install \
+    kafkit==0.2.1 \
+    lsst_efd_client==0.12.0
+
+# Copy the Python module
+COPY --chown=lsst:lsst python /opt/lsst/software/stack/python
+
+# Create and populate the data directory
+RUN mkdir -p /opt/lsst/software/stack/data
+COPY --chown=lsst:lsst tmp/efd_transform/*.db /opt/lsst/software/stack/data/
+
+# Environment variables
+ENV CONFIG_FILE="/opt/lsst/software/stack/python/lsst/consdb/efd_transform/config_LATISS.yml" \
+    INSTRUMENT="LATISS" \
+    BUTLER_REPO="s3://rubin-summit-users/butler.yaml" \
+    S3_ENDPOINT_URL="https://s3dfrgw.slac.stanford.edu/" \
+    LSST_RESOURCES_S3_PROFILE_embargo="https://sdfembs3.sdf.slac.stanford.edu" \
+    PGUSER="rubin" \
+    CONSDB_URL="sqlite:////opt/lsst/software/stack/data/test.db" \
+    TIMEDELTA="5" \
+    LOG_FILE="/opt/lsst/software/stack/data/transform.log" \
+    PYTHONPATH="/opt/lsst/software/stack/python:$PYTHONPATH"
+
+# Run the Python script
+CMD ["bash", "-c", "source loadLSST.bash; setup lsst_distrib; python /opt/lsst/software/stack/python/lsst/consdb/efd_transform/transform_efd.py -c \"$CONFIG_FILE\" -i \"$INSTRUMENT\" -r \"$BUTLER_REPO\" -d \"$CONSDB_URL\" -E \"$EFD\" -t \"$TIMEDELTA\" -l \"$LOG_FILE\""]
+
+
+# Example:
+# docker run --rm -it \
+#     --volume $PWD/data:/opt/lsst/software/stack/data \
+#     -e CONFIG_FILE=/opt/lsst/software/stack/lsst/consdb/efd_transform/config_LATISS.yml \
+#     -e INSTRUMENT=LATISS \
+#     -e BUTLER_REPO=s3://rubin-summit-users/butler.yaml \
+#     -e EFD=usdf_efd \
+#     -e CONSDB_URL=sqlite:////opt/lsst/software/stack/data/test.db \
+#     -e TIMEDELTA=5 \
+#     consdb/efd_transform:latest

pyproject.toml

@@ -44,4 +44,4 @@ asyncio_mode = "auto"
 test = ["pytest"]
 dev = [
     "documenteer[guide] < 2",
-]
+]

python/lsst/consdb/efd_transform/__init__.py

@@ -0,0 +1,39 @@
+"""Provides a structured framework for processing and transforming data from the (EFD).
+
+Overview:
+---------
+This module offers tools for accessing, transforming, and managing EFD data, supporting
+workflows and integration with the LSST ecosystem. It includes capabilities for
+data retrieval, transformation pipelines, configuration management, and schema
+generation.
+
+Components:
+-----------
+- **Data Access**: The `dao` subpackage provides Data Access Objects (DAOs) to interact
+  with specific database tables such as `exposure_efd` and `visit_efd`.
+- **Data Transformation**: Includes utilities for applying structured transformations
+  to EFD data, including summarization and restructuring.
+- **Configuration Handling**: Supports loading and validating instrument configurations
+  through YAML files for flexible setup and operation.
+- **Schema Generation**: Automates the creation of database schemas based on predefined
+  instrument configurations.
+- **Queue Management**: Implements tools for task scheduling and queue-based workflows.
+
+Submodules:
+-----------
+- `dao`: Contains DAOs for database interactions.
+- `config_model`: Defines models for validating YAML configurations.
+- `generate_schema`: Includes schema generation utilities.
+- `summary`: Provides tools for summarizing EFD data.
+- `transform`: Manages transformation pipelines.
+- `transform_efd`: Contains specialized transformation methods.
+- `queue_manager`: Handles task queue management.
+
+Configuration Files:
+--------------------
+The module uses YAML files for instrument-specific configurations:
+- `config_LATISS.yaml`: Configuration for LATISS.
+- `config_LSSTComCam.yaml`: Configuration for LSSTComCam.
+- `config_LSSTComCamSim.yaml`: Configuration for LSSTComCamSim.
+
+"""