Merge pull request #22 from lsst-dm/tickets/DM-38395

Tickets/dm 38395: creating a test suite that successfully runs within github workflow

.github/workflows/lint.yaml

@@ -8,4 +8,4 @@ on:
 
 jobs:
   call-workflow:
-    uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main
+    uses: lsst/rubin_workflows/.github/workflows/lint.yaml@main

.github/workflows/test.yml

@@ -0,0 +1,47 @@
+name: test
+on:
+  push:
+    branches:
+    - main
+    tags:
+    - v*
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the repo
+      uses: actions/checkout@v3
+    - name: Setup python 3.10
+      uses: actions/setup-python@v2
+      with:
+          python-version: "3.10"
+    #- name: docker build
+    #  run: |
+    #    docker build . -f Dockerfile
+    - name: Install dependencies
+      run: |
+       python -m ensurepip
+       python -m pip install lsst-daf-butler
+    #   pip install pytest-flake8
+    #   pip install flake8
+    #   python -m pip install pytest
+    #- name: install lsstinstall
+    #  run: |
+    #   cd ./tests
+    #   curl -OL https://ls.st/lsstinstall 
+    #   chmod u+x lsstinstall 
+    #   ./lsstinstall -T "w_2023_19"
+    #   source loadLSST.sh
+    #   # mamba activate lsst-scipipe-6.0.0
+    #   eups distrib install -t "w_2023_19" daf_butler
+    #   setup -t w_2023_19 daf_butler
+    #   # eups distrib install -t "w_2023_19" lsst_distrib
+    #   python test_move_embargo_args.py
+    - name: Test with pytest
+      run: |
+       # mamba activate lsst-scipipe-6.0.0
+       cd ./tests
+       python test_move_embargo_args.py
+       # pytest -s test_move_embargo_args.py --log-cli-level 11

.gitignore

@@ -8,6 +8,7 @@ config.log
 *.pyc
 *_wrap.cc
 *Lib.py
+.ipynb_checkpoints
 
 # Built by sconsUtils
 version.py

Dockerfile

@@ -0,0 +1,26 @@
+# Dockerfile
+COPY ./src /opt/lsst/transfer_embargo
+WORKDIR /opt/lsst/transfer_embargo
+
+FROM python:3.9
+
+# ADD move_embargo_scratch.py .
+
+# RUN setup lsst_distrib -t w_2023_19
+
+
+
+FROM lsstsqre/newinstall:latest
+USER lsst
+RUN source loadLSST.bash && mamba install rucio-clients
+RUN source loadLSST.bash && eups distrib install -t "w_2023_21" obs_lsst
+
+# RUN pip install -r requirements.txt
+
+CMD ["python", "-m", "ensurepip"]
+CMD ["python", "-m", "pip", "install", "lsst-daf-butler"]
+# this is from the test.yml file
+#python -m ensurepip
+#python -m pip install lsst-daf-butler
+
+# CMD ["python", "./move_embargo_scratch.py"]

src/move_embargo_args.py

@@ -1,96 +1,141 @@
 import argparse
+
 import astropy.time
 from lsst.daf.butler import Butler, Timespan
-
-# remove_collection clears the collection from scratch_butler if set to True.
-REMOVE_COLLECTION = False
-
-# transfers data from embargo to scratch butler when set to True.
-TRANSFER = False
+from lsst.daf.butler.cli.cliLog import CliLog
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='Transferring data from embargo butler to another butler')
+    parser = argparse.ArgumentParser(
+        description="Transferring data from embargo butler to another butler"
+    )
 
     # at least one arg in dataId needed for 'where' clause.
     parser.add_argument(
-        "-f", "--fromrepo", type=str,
-        required=True, default='/repo/embargo',
-        help="Butler Repository path from which data is transferred. Input str. Default = '/repo/embargo'")
+        "fromrepo",
+        type=str,
+        nargs="?",
+        default="/repo/embargo",
+        help="Butler Repository path from which data is transferred. Input str. Default = '/repo/embargo'",
+    )
     parser.add_argument(
-        "-t", "--torepo", type=str, default='/home/j/jarugula/scratch',
-        required=True, help="Repository to which data is transferred. Input str")
-    parser.add_argument("--embargohours", type=float, required=True, default=80.0,
-                        help="Embargo time period in hours. Input float")
-    parser.add_argument("--instrument", type=str, required=True, default='LATISS',
-                        help="Instrument. Input str")
-    parser.add_argument("--datasettype", type=str, required=False, default='raw',
-                        help="Dataset type. Input str")
-    parser.add_argument("--collections", type=str, required=False, default='LATISS/raw/all',
-                        help="Data Collections. Input str")
-    parser.add_argument("--nowtime", type=str, required=False, default='now',
-                        help="Now time in (ISO, TAI timescale). If left blank it will \
-                        use astropy.time.Time.now.")
-
+        "torepo",
+        type=str,
+        help="Repository to which data is transferred. Input str",
+    )
+    parser.add_argument(
+        "instrument",
+        type=str,
+        nargs="?",
+        default="LATISS",
+        help="Instrument. Input str",
+    )
+    parser.add_argument(
+        "--embargohours",
+        type=float,
+        required=False,
+        default=80.0,
+        help="Embargo time period in hours. Input float",
+    )
+    parser.add_argument(
+        "--datasettype",
+        type=str,
+        required=False,
+        default="raw",
+        help="Dataset type. Input str",
+    )
+    parser.add_argument(
+        "--collections",
+        type=str,
+        required=False,
+        default="LATISS/raw/all",
+        help="Data Collections. Input str",
+    )
+    parser.add_argument(
+        "--nowtime",
+        type=str,
+        required=False,
+        default="now",
+        help="Now time in (ISO, TAI timescale). If left blank it will \
+                        use astropy.time.Time.now.",
+    )
+    parser.add_argument(
+        "--move",
+        type=str,
+        required=False,
+        default="False",
+        help="Copies if False, deletes original if True",
+    )
+    parser.add_argument(
+        "--log",
+        type=str,
+        required=False,
+        default="False",
+        help="No logging if False, longlog if True",
+    )
     return parser.parse_args()
 
 
 if __name__ == "__main__":
     namespace = parse_args()
-    # Define embargo and scratch butler
-    butler = Butler(namespace.fromrepo)
+    # Define embargo and destination butler
+    # If move is true, then you'll need write
+    # permissions from the fromrepo (embargo)
+    butler = Butler(namespace.fromrepo, writeable=namespace.move)
+    print("temp from path:", namespace.fromrepo)
+    print("temp to path:", namespace.torepo)
     registry = butler.registry
     dest = Butler(namespace.torepo, writeable=True)
     scratch_registry = dest.registry
     datasetType = namespace.datasettype
     collections = namespace.collections
-
+    move = namespace.move
     # Dataset to move
-    dataId = {'instrument': namespace.instrument}
-
+    dataId = {"instrument": namespace.instrument}
     # Define embargo period
-    embargo_period = astropy.time.TimeDelta(namespace.embargohours*3600., format='sec')
-    if namespace.nowtime != 'now':
-        now = astropy.time.Time(namespace.nowtime, scale='tai', format='iso')
+    embargo_period = astropy.time.TimeDelta(
+        namespace.embargohours * 3600.0, format="sec"
+    )
+    if namespace.nowtime != "now":
+        now = astropy.time.Time(namespace.nowtime, scale="tai", format="iso")
     else:
-        now = astropy.time.Time.now()
+        now = astropy.time.Time.now().tai
     timespan_embargo = Timespan(now - embargo_period, now)
-
     # The Dimensions query
     # If (now - embargo period, now) does not overlap
     # with observation time interval: move
     # Else: don't move
     # Save data Ids of these observations into a list
-    after_embargo = [dt.id for dt in
-                     registry.queryDimensionRecords('exposure', dataId=dataId, datasets=datasetType,
-                                                    collections=collections,
-                                                    where="NOT exposure.timespan OVERLAPS\
+    after_embargo = [
+        dt.id
+        for dt in registry.queryDimensionRecords(
+            "exposure",
+            dataId=dataId,
+            datasets=datasetType,
+            collections=collections,
+            where="NOT exposure.timespan OVERLAPS\
                                                     timespan_embargo",
-                                                    bind={"timespan_embargo": timespan_embargo})]
-    # after_embargo = []
-    # for i, dt in enumerate(registry.queryDimensionRecords('exposure',
-    #                          dataId=dataId,
-    #                          datasets=datasetType,
-    #                          collections=collections,
-    #                          where="NOT exposure.timespan\
-    #                          OVERLAPS\
-    #                          timespan_embargo",
-    #                          bind={"timespan_embargo": timespan_embargo})):
-    #     end_time = dt.timespan.end
-    #     if now - end_time >= embargo_period:
-    #        after_embargo.append(dt.id)
-
+            bind={"timespan_embargo": timespan_embargo},
+        )
+    ]
     # Query the DataIds after embargo period
-    datasetRefs = registry.queryDatasets(datasetType, dataId=dataId, collections=collections,
-                                         where="exposure.id IN (exposure_ids)",
-                                         bind={"exposure_ids": after_embargo})
-
-    # Copy the Dataset after embargo period from
-    # embargo butler to scratch butler.
-    if TRANSFER:
-        dest.transfer_from(butler, source_refs=datasetRefs, transfer='copy',
-                           skip_missing=True, register_dataset_types=True,
-                           transfer_dimensions=True)
-    # Remove collection from scratch butler
-    if REMOVE_COLLECTION:
-        dest.pruneCollection(collections, purge=True, unstore=True)
+    datasetRefs = registry.queryDatasets(
+        datasetType,
+        dataId=dataId,
+        collections=collections,
+        where="exposure.id IN (exposure_ids)",
+        bind={"exposure_ids": after_embargo},
+    ).expanded()
+    if namespace.log == "True":
+        cli_log = CliLog.initLog(longlog=True)
+        CliLog.setLogLevels([(None, "DEBUG")])
+    out = dest.transfer_from(
+        butler,
+        source_refs=datasetRefs,
+        transfer="copy",
+        skip_missing=True,
+        register_dataset_types=True,
+        transfer_dimensions=True,
+    )
+    if move == "True":
+        butler.pruneDatasets(refs=datasetRefs, unstore=True, purge=True)