Add metadata optional arg

.github/workflows/python-publish.yml

@@ -0,0 +1,36 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

ipfx/bin/run_x_to_nwb_conversion.py

@@ -1,5 +1,6 @@
 #!/bin/env python
 
+import yaml
 import os
 import argparse
 import logging
@@ -9,7 +10,9 @@
 from ipfx.x_to_nwb.DatConverter import DatConverter
 
 
-def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False, outputFeedbackChannel=False, multipleGroupsPerFile=False, compression=True):
+def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False,
+            outputFeedbackChannel=False, multipleGroupsPerFile=False, compression=True,
+            metafile=None):
     """
     Convert the given file to a NeuroDataWithoutBorders file using pynwb
 
@@ -25,6 +28,7 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False
     :param multipleGroupsPerFile: Write all Groups in the DAT file into one NWB
                                   file. By default we create one NWB per Group (ignored for ABF files).
     :param compression: Toggle compression for HDF5 datasets
+    :param metafile: The path to the metadata YAML file (optional)
 
     :return: path of the created NWB file
     """
@@ -53,16 +57,26 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False
         else:
             raise ValueError(f"The output file {outFile} does already exist.")
 
+    # Load metadata from YAML file
+    if metafile is None:
+        metadata = None
+    else:
+        print(metafile)
+        with open(metafile) as f:
+            metadata = yaml.safe_load(f)
+
     if ext == ".abf":
         if outputMetadata:
             ABFConverter.outputMetadata(inFileOrFolder)
         else:
-            ABFConverter(inFileOrFolder, outFile, outputFeedbackChannel=outputFeedbackChannel, compression=compression)
+            ABFConverter(inFileOrFolder, outFile, outputFeedbackChannel=outputFeedbackChannel, compression=compression,
+                         metadata=metadata)
     elif ext == ".dat":
         if outputMetadata:
             DatConverter.outputMetadata(inFileOrFolder)
         else:
-            DatConverter(inFileOrFolder, outFile, multipleGroupsPerFile=multipleGroupsPerFile, compression=compression)
+            DatConverter(inFileOrFolder, outFile, multipleGroupsPerFile=multipleGroupsPerFile, compression=compression,
+                         metadata=metadata)
 
     else:
         raise ValueError(f"The extension {ext} is currently not supported.")
@@ -99,6 +113,8 @@ def main():
                         help="Output ADC data to the NWB file which stems from stimulus feedback channels.")
     abf_group.add_argument("--realDataChannel", type=str, action="append",
                         help=f"Define additional channels which hold non-feedback channel data. The default is {ABFConverter.adcNamesWithRealData}.")
+    abf_group.add_argument('--metafile', default=None, type=str,
+                           help='The path to the metadata YAML file.')
 
     dat_group.add_argument("--multipleGroupsPerFile", action="store_true", default=False,
                            help="Write all Groups from a DAT file into a single NWB file. By default we create one NWB file per Group.")
@@ -131,7 +147,8 @@ def main():
                 outputMetadata=args.outputMetadata,
                 outputFeedbackChannel=args.outputFeedbackChannel,
                 multipleGroupsPerFile=args.multipleGroupsPerFile,
-                compression=args.compression)
+                compression=args.compression,
+                metafile=args.metafile)
 
 
 if __name__ == "__main__":

ipfx/x_to_nwb/ABFConverter.py

@@ -15,9 +15,12 @@
 import pyabf
 
 from pynwb.device import Device
+from pynwb.file import Subject
 from pynwb import NWBHDF5IO, NWBFile
 from pynwb.icephys import IntracellularElectrode
 
+from ndx_dandi_icephys import DandiIcephysMetadata
+
 from ipfx.x_to_nwb.conversion_utils import PLACEHOLDER, V_CLAMP_MODE, I_CLAMP_MODE, I0_CLAMP_MODE, \
      parseUnit, getStimulusSeriesClass, getAcquiredSeriesClass, createSeriesName, convertDataset, \
      getPackageInfo, createCycleID
@@ -30,7 +33,7 @@ class ABFConverter:
     protocolStorageDir = None
     adcNamesWithRealData = ["IN 0", "IN 1", "IN 2", "IN 3"]
 
-    def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True):
+    def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True, metadata=None):
         """
         Convert the given ABF file to NWB
 
@@ -39,6 +42,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T
         outFile               -- target filepath (must not exist)
         outputFeedbackChannel -- Output ADC data from feedback channels as well (useful for debugging only)
         compression           -- Toggle compression for HDF5 datasets
+        metadata              -- Metadata dictionary with user-defined values for some nwb fields
         """
 
         inFiles = []
@@ -52,6 +56,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T
 
         self.outputFeedbackChannel = outputFeedbackChannel
         self.compression = compression
+        self.metadata = metadata
 
         self._settings = self._getJSONFiles(inFileOrFolder)
 
@@ -72,6 +77,18 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T
 
         nwbFile = self._createFile()
 
+        # If Subject information is present in metadata
+        if self.metadata is not None:
+            if 'Subject' in self.metadata:
+                nwbFile.subject = self._createSubject()
+            if 'lab_meta_data' in self.metadata:
+                nwbFile.add_lab_meta_data(
+                    DandiIcephysMetadata(
+                        cell_id=self.metadata['lab_meta_data'].get('cell_id', None),
+                        tissue_sample_id=self.metadata['lab_meta_data'].get('tissue_sample_id', None),
+                    )
+                )
+
         device = self._createDevice()
         nwbFile.add_device(device)
 
@@ -306,23 +323,26 @@ def getFileComments(abfs):
         if len(session_description) == 0:
             session_description = PLACEHOLDER
 
-        identifier = sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest()
-        session_start_time = self.refabf.abfDateTime
         creatorName = self.refabf._stringsIndexed.uCreatorName
         creatorVersion = formatVersion(self.refabf.creatorVersion)
-        experiment_description = (f"{creatorName} v{creatorVersion}")
-        source_script_file_name = "run_x_to_nwb_conversion.py"
-        source_script = json.dumps(getPackageInfo(), sort_keys=True, indent=4)
-        session_id = PLACEHOLDER
-
-        return NWBFile(session_description=session_description,
-                       identifier=identifier,
-                       session_start_time=session_start_time,
-                       experimenter=None,
-                       experiment_description=experiment_description,
-                       session_id=session_id,
-                       source_script_file_name=source_script_file_name,
-                       source_script=source_script)
+        nwbfile_kwargs = dict(
+            session_description=session_description,
+            identifier=sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest(),
+            session_start_time=self.refabf.abfDateTime,
+            experimenter=None,
+            experiment_description="{} v{}".format(creatorName, creatorVersion),
+            source_script_file_name="run_x_to_nwb_conversion.py",
+            source_script=json.dumps(getPackageInfo(), sort_keys=True, indent=4),
+            session_id=PLACEHOLDER
+        )
+
+        if self.metadata and 'NWBFile' in self.metadata:
+            nwbfile_kwargs.update(self.metadata['NWBFile'])
+
+        # Create nwbfile with initial metadata
+        nwbfile = NWBFile(**nwbfile_kwargs)
+
+        return nwbfile
 
     def _createDevice(self):
         """
@@ -334,6 +354,12 @@ def _createDevice(self):
 
         return Device(f"{digitizer} with {telegraph}")
 
+    def _createSubject(self):
+        """
+        Create a pynwb Subject object from the metadata contents.
+        """
+        return Subject(**self.metadata['Subject'])
+
     def _createElectrodes(self, device):
         """
         Create pynwb ic_electrodes objects from the ABF file contents.
@@ -362,7 +388,6 @@ def _createStimulusSeries(self, electrodes):
         counter = 0
 
         for file_index, abf in enumerate(self.abfs):
-
             stimulus_description = ABFConverter._getProtocolName(abf.protocol)
             scale_factor = self._getScaleFactor(abf, stimulus_description)
 
@@ -490,7 +515,7 @@ def _getAmplifierSettings(self, abf, clampMode, adcName):
                     d["whole_cell_capacitance_comp"] = np.nan
                     d["whole_cell_series_resistance_comp"] = np.nan
 
-            elif clampMode in (I_CLAMP_MODE, I0_CLAMP_MODE):
+            elif clampMode in (I_CLAMP_MODE,):
                 if settings["GetHoldingEnable"]:
                     d["bias_current"] = settings["GetHolding"]
                 else:
@@ -593,7 +618,7 @@ def _createAcquiredSeries(self, electrodes):
                                                       whole_cell_capacitance_comp=settings["whole_cell_capacitance_comp"],  # noqa: E501
                                                       whole_cell_series_resistance_comp=settings["whole_cell_series_resistance_comp"])  # noqa: E501
 
-                    elif clampMode in (I_CLAMP_MODE, I0_CLAMP_MODE):
+                    elif clampMode is I_CLAMP_MODE:
                         acquistion_data = seriesClass(name=name,
                                                       data=data,
                                                       sweep_number=np.uint64(cycle_id),
@@ -609,6 +634,20 @@ def _createAcquiredSeries(self, electrodes):
                                                       bridge_balance=settings["bridge_balance"],
                                                       stimulus_description=stimulus_description,
                                                       capacitance_compensation=settings["capacitance_compensation"])
+                    elif clampMode is I0_CLAMP_MODE:
+                        acquistion_data = seriesClass(
+                            name=name,
+                            data=data,
+                            sweep_number=np.uint64(cycle_id),
+                            unit=unit,
+                            electrode=electrode,
+                            gain=gain,
+                            resolution=resolution,
+                            conversion=conversion,
+                            starting_time=starting_time,
+                            rate=rate,
+                            description=description,
+                        )
                     else:
                         raise ValueError(f"Unsupported clamp mode {clampMode}.")
 

ipfx/x_to_nwb/DatConverter.py

@@ -10,6 +10,7 @@
 from pynwb.device import Device
 from pynwb import NWBHDF5IO, NWBFile
 from pynwb.icephys import IntracellularElectrode
+from pynwb.file import Subject
 
 from ipfx.x_to_nwb.hr_bundle import Bundle
 from ipfx.x_to_nwb.hr_stimsetgenerator import StimSetGenerator
@@ -22,7 +23,7 @@
 
 class DatConverter:
 
-    def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=True):
+    def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=True, metadata=None):
         """
         Convert DAT files, created by PatchMaster, to NWB v2 files.
 
@@ -33,6 +34,7 @@ def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=Tru
         multipleGroupsPerFile: switch determining if multiple DAT groups per
                                file are created or not
         compression: Toggle compression for HDF5 datasets
+        metadata: Metadata dictionary with user-defined values for some nwb fields
 
         Returns
         -------
@@ -44,6 +46,7 @@ def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=Tru
 
         self.bundle = Bundle(inFile)
         self.compression = compression
+        self.metadata = metadata
 
         self._check()
 
@@ -63,6 +66,10 @@ def generateList(multipleGroupsPerFile, pul):
 
             nwbFile = self._createFile()
 
+            # If Subject information is present in metadata, add a Subject object
+            if self.metadata is not None and 'Subject' in self.metadata:
+                nwbFile.subject = Subject(**self.metadata['Subject'])
+
             device = self._createDevice()
             nwbFile.add_device(device)
 
@@ -298,7 +305,7 @@ def _isValidAmplifierState(ampState):
     def _getAmplifierState(bundle, series, trace):
         """
         Different PatchMaster versions create different DAT file layouts. This function tries
-        to accomodate that as it returns the correct object.
+        to accommodate that as it returns the correct object.
 
         Parameters
         ----------
@@ -401,14 +408,19 @@ def _createFile(self):
         source_script = json.dumps(getPackageInfo(), sort_keys=True, indent=4)
         session_id = PLACEHOLDER
 
-        return NWBFile(session_description=session_description,
-                       identifier=identifier,
-                       session_start_time=self.session_start_time,
-                       experimenter=None,
-                       experiment_description=experiment_description,
-                       session_id=session_id,
-                       source_script=source_script,
-                       source_script_file_name=source_script_file_name)
+        nwbfile_kwargs = dict(session_description=session_description,
+                              identifier=identifier,
+                              session_start_time=self.session_start_time,
+                              experimenter=None,
+                              experiment_description=experiment_description,
+                              session_id=session_id,
+                              source_script=source_script,
+                              source_script_file_name=source_script_file_name)
+
+        if self.metadata is not None and 'NWBFile' in self.metadata:
+            nwbfile_kwargs.update(self.metadata['NWBFile'])
+
+        return NWBFile(**nwbfile_kwargs)
 
     def _createDevice(self):
         """

ipfx/x_to_nwb/conversion_utils.py

@@ -23,7 +23,6 @@
 I_CLAMP_MODE = 1
 I0_CLAMP_MODE = 2
 
-
 # TODO Use the pint package if doing that manually gets too involved
 def parseUnit(unitString):
     """

requirements.txt

@@ -14,3 +14,4 @@ pynwb==1.0.2
 six
 watchdog
 pg8000
+pyyaml

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = python2.7, python3.6
+envlist = python3.6
 
 [testenv]