MTH5 spectrograms prep (2 of several)

.github/workflows/tests.yml

@@ -37,9 +37,8 @@ jobs:
         echo $CONDA_PREFIX
         conda install -c conda-forge pytest pytest-cov certifi">=2017.4.17" pandoc
         pip install -r requirements-dev.txt
-        # pip install git+https://github.com/kujaku11/mt_metadata.git@main
-        # pip install git+https://github.com/kujaku11/mth5.git@master
-        # pip install git+https://github.com/MTgeophysics/mtpy-v2.git@main
+        pip install git+https://github.com/kujaku11/mt_metadata.git@features
+        pip install git+https://github.com/kujaku11/mth5.git@features
         pip install git+https://github.com/MTgeophysics/mtpy-v2.git@main
         pip uninstall aurora -y
 
@@ -48,32 +47,26 @@ jobs:
         echo $CONDA_PREFIX
         pip install -e .
         echo "Install complete"
-        echo "Uninstalling mth5"
-        pip uninstall mth5 -y   
-        echo "Uninstalling mt_metadata"             
-        pip uninstall mt_metadata -y
-        pip install git+https://github.com/kujaku11/mt_metadata.git@fix_issue_235
-        pip install git+https://github.com/kujaku11/mth5.git@fix_issue_271
         conda list
         pip freeze
 
-#    - name: Install Jupyter and dependencies
-#      run: |
-#        pip install jupyter
-#        pip install ipykernel
-#        python -m ipykernel install --user --name aurora-test
-#        # Install any other dependencies you need
-#
-#    - name: Execute Jupyter Notebooks
-#      run: |
-#        jupyter nbconvert --to notebook --execute docs/examples/dataset_definition.ipynb
-#        jupyter nbconvert --to notebook --execute docs/examples/operate_aurora.ipynb
-#        jupyter nbconvert --to notebook --execute docs/tutorials/pkd_units_check.ipynb
-#        jupyter nbconvert --to notebook --execute docs/tutorials/pole_zero_fitting/lemi_pole_zero_fitting_example.ipynb
-#        jupyter nbconvert --to notebook --execute docs/tutorials/processing_configuration.ipynb
-#        jupyter nbconvert --to notebook --execute docs/tutorials/process_cas04_multiple_station.ipynb
-#        jupyter nbconvert --to notebook --execute docs/tutorials/synthetic_data_processing.ipynb
-#        # Replace "notebook.ipynb" with your notebook's filename
+    - name: Install Jupyter and dependencies
+      run: |
+        pip install jupyter
+        pip install ipykernel
+        python -m ipykernel install --user --name aurora-test
+        # Install any other dependencies you need
+
+    - name: Execute Jupyter Notebooks
+      run: |
+        jupyter nbconvert --to notebook --execute docs/examples/dataset_definition.ipynb
+        jupyter nbconvert --to notebook --execute docs/examples/operate_aurora.ipynb
+        jupyter nbconvert --to notebook --execute docs/tutorials/pkd_units_check.ipynb
+        jupyter nbconvert --to notebook --execute docs/tutorials/pole_zero_fitting/lemi_pole_zero_fitting_example.ipynb
+        jupyter nbconvert --to notebook --execute docs/tutorials/processing_configuration.ipynb
+        jupyter nbconvert --to notebook --execute docs/tutorials/process_cas04_multiple_station.ipynb
+        jupyter nbconvert --to notebook --execute docs/tutorials/synthetic_data_processing.ipynb
+        # Replace "notebook.ipynb" with your notebook's filename
 
 #    - name: Commit changes (if any)
 #      run: |

aurora/pipelines/fourier_coefficients.py

@@ -63,6 +63,7 @@
 
 import mth5.mth5
 import pathlib
+import xarray as xr
 
 from aurora.pipelines.time_series_helpers import calibrate_stft_obj
 from aurora.pipelines.time_series_helpers import prototype_decimate
@@ -88,6 +89,7 @@ def fc_decimations_creator(
     time_period: Optional[TimePeriod] = None,
 ) -> List[FCDecimation]:
     """
+    TODO: move this to mt_metadata / replace with mt_metadata method once moved.
 
     Creates mt_metadata FCDecimation objects that parameterize Fourier coefficient decimation levels.
 
@@ -232,34 +234,79 @@ def add_fcs_to_mth5(m: MTH5, fc_decimations: Optional[Union[str, list]] = None)
             # If timing corrections were needed they could go here, right before STFT
 
             for i_dec_level, fc_decimation in enumerate(fc_decimations):
-                if i_dec_level != 0:
+                try:
+                    assert i_dec_level == fc_decimation.time_series_decimation.level
+                except:
+                    msg = "decimation level has unexpected value"
+                    logger.warning(msg)
+
+                if (
+                    i_dec_level != 0
+                ):  # TODO: take this number from fc_decimation.time_series_decimation.level
                     # Apply decimation
-                    run_xrds = prototype_decimate(fc_decimation, run_xrds)
-
-                # check if this decimation level yields a valid spectrogram
-                if not fc_decimation.is_valid_for_time_series_length(
-                    run_xrds.time.shape[0]
-                ):
-                    logger.info(
-                        f"Decimation Level {i_dec_level} invalid, TS of {run_xrds.time.shape[0]} samples too short"
-                    )
-                    continue
-
-                stft_obj = run_ts_to_stft_scipy(fc_decimation, run_xrds)
-                stft_obj = calibrate_stft_obj(stft_obj, run_obj)
-
-                # Pack FCs into h5 and update metadata
-                fc_decimation_group: FCDecimationGroup = fc_group.add_decimation_level(
-                    f"{i_dec_level}", decimation_level_metadata=fc_decimation
+                    ts_decimation = fc_decimation.time_series_decimation
+                    run_xrds = prototype_decimate(
+                        ts_decimation, run_xrds
+                    )  # TODO: replace this with mth5 decimation
+
+                _add_spectrogram_to_mth5(
+                    fc_decimation=fc_decimation,
+                    run_obj=run_obj,
+                    run_xrds=run_xrds,
+                    fc_group=fc_group,
                 )
-                fc_decimation_group.from_xarray(
-                    stft_obj, fc_decimation_group.metadata.sample_rate_decimation
-                )
-                fc_decimation_group.update_metadata()
-                fc_group.update_metadata()
+
     return
 
 
+def _add_spectrogram_to_mth5(
+    fc_decimation: FCDecimation,
+    run_obj: mth5.groups.RunGroup,
+    run_xrds: xr.Dataset,
+    fc_group: mth5.groups.FCGroup,
+) -> None:
+    """
+
+    This function has been factored out of add_fcs_to_mth5.
+    This is the most atomic level of adding FCs and will be useful as standalone method.
+
+    Parameters
+    ----------
+    fc_decimation : FCDecimation
+        Metadata about how the decimation level is to be processed
+
+    run_xrds : xarray.core.dataset.Dataset
+        Time series to be converted to a spectrogram and stored in MTH5.
+
+    Returns
+    -------
+    run_xrds : xarray.core.dataset.Dataset
+        pre-whitened time series
+
+    """
+
+    # check if this decimation level yields a valid spectrogram
+    if not fc_decimation.is_valid_for_time_series_length(run_xrds.time.shape[0]):
+        logger.info(
+            f"Decimation Level {fc_decimation.time_series_decimation.level} invalid, TS of {run_xrds.time.shape[0]} samples too short"
+        )
+        return
+
+    stft_obj = run_ts_to_stft_scipy(fc_decimation, run_xrds)
+    stft_obj = calibrate_stft_obj(stft_obj, run_obj)
+
+    # Pack FCs into h5 and update metadata
+    fc_decimation_group: FCDecimationGroup = fc_group.add_decimation_level(
+        f"{fc_decimation.time_series_decimation.level}",
+        decimation_level_metadata=fc_decimation,
+    )
+    fc_decimation_group.from_xarray(
+        stft_obj, fc_decimation_group.metadata.time_series_decimation.sample_rate
+    )
+    fc_decimation_group.update_metadata()
+    fc_group.update_metadata()
+
+
 def get_degenerate_fc_decimation(sample_rate: float) -> list:
     """
 
@@ -289,6 +336,7 @@ def get_degenerate_fc_decimation(sample_rate: float) -> list:
     return output
 
 
+# TODO: Delete after mth5 issue #271 is closed and merged.
 @path_or_mth5_object
 def read_back_fcs(m: Union[MTH5, pathlib.Path, str], mode: str = "r") -> None:
     """

aurora/pipelines/time_series_helpers.py

@@ -157,6 +157,7 @@ def run_ts_to_stft_scipy(
     run_xrds_orig: xr.Dataset,
 ) -> xr.Dataset:
     """
+    TODO: Replace with mth5 run_ts_to_stft_scipy method
     Converts a runts object into a time series of Fourier coefficients.
     This method uses scipy.signal.spectrogram.
 
@@ -177,7 +178,9 @@ def run_ts_to_stft_scipy(
         Time series of Fourier coefficients
     """
     run_xrds = apply_prewhitening(decimation_obj, run_xrds_orig)
-    windowing_scheme = window_scheme_from_decimation(decimation_obj)
+    windowing_scheme = window_scheme_from_decimation(
+        decimation_obj
+    )  # TODO: deprecate in favor of stft.window.taper
 
     stft_obj = xr.Dataset()
     for channel_id in run_xrds.data_vars:

tests/synthetic/test_decimation_methods.py

@@ -49,16 +49,16 @@ def test_decimation_methods_agree():
         if dec_level_id == 0:
             run_obj = mth5_obj.get_run(station_id, run_id, survey=None)
             run_ts = run_obj.to_runts(start=None, end=None)
-            run_xrts = run_ts.dataset
-            decimated_ts[dec_level_id]["run_xrts"] = run_xrts
+            run_xrds = run_ts.dataset
+            decimated_ts[dec_level_id]["run_xrds"] = run_xrds
             current_sample_rate = run_obj.metadata.sample_rate
 
         if dec_level_id > 0:
-            run_xrts = decimated_ts[dec_level_id - 1]["run_xrts"]
+            run_xrds = decimated_ts[dec_level_id - 1]["run_xrds"]
             target_sample_rate = current_sample_rate / (dec_config.decimation.factor)
 
-            decimated_1 = prototype_decimate(dec_config.decimation, run_xrts)
-            decimated_2 = run_xrts.sps_filters.decimate(
+            decimated_1 = prototype_decimate(dec_config.decimation, run_xrds)
+            decimated_2 = run_xrds.sps_filters.decimate(
                 target_sample_rate=target_sample_rate
             )
 
@@ -67,7 +67,7 @@ def test_decimation_methods_agree():
             assert np.isclose(difference.to_array(), 0).all()
 
             logger.info("prototype decimate aurora method agrees with mth5 decimate")
-            decimated_ts[dec_level_id]["run_xrts"] = decimated_1
+            decimated_ts[dec_level_id]["run_xrds"] = decimated_1
             current_sample_rate = target_sample_rate
     return
 

tests/synthetic/test_fourier_coefficients.py

@@ -3,7 +3,8 @@
 from aurora.config.config_creator import ConfigCreator
 from aurora.pipelines.fourier_coefficients import add_fcs_to_mth5
 from aurora.pipelines.fourier_coefficients import fc_decimations_creator
-from aurora.pipelines.fourier_coefficients import read_back_fcs
+
+# from aurora.pipelines.fourier_coefficients import read_back_fcs
 from aurora.pipelines.process_mth5 import process_mth5
 from aurora.test_utils.synthetic.make_processing_configs import (
     create_test_run_config,
@@ -13,6 +14,7 @@
 from mth5.data.make_mth5_from_asc import create_test2_h5
 from mth5.data.make_mth5_from_asc import create_test3_h5
 from mth5.data.make_mth5_from_asc import create_test12rr_h5
+from mth5.timeseries.spectre.helpers import read_back_fcs
 
 # from mtpy-v2
 from mtpy.processing import RunSummary, KernelDataset
@@ -128,8 +130,13 @@ def test_123(self):
         return tfc
 
     def test_fc_decimations_creator(self):
-        """"""
-        cfgs = fc_decimations_creator(1.0)
+        """
+        # TODO: Move this into mt_metadata
+        Returns
+        -------
+
+        """
+        cfgs = fc_decimations_creator(initial_sample_rate=1.0)
 
         # test time period must of of type
         with self.assertRaises(NotImplementedError):