diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..3bfabfc1 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,36 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/ipfx/bin/run_x_to_nwb_conversion.py b/ipfx/bin/run_x_to_nwb_conversion.py index b56e09c5..91947868 100755 --- a/ipfx/bin/run_x_to_nwb_conversion.py +++ b/ipfx/bin/run_x_to_nwb_conversion.py @@ -1,5 +1,6 @@ #!/bin/env python +import yaml import os import argparse import logging @@ -9,7 +10,9 @@ from ipfx.x_to_nwb.DatConverter import DatConverter -def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False, outputFeedbackChannel=False, multipleGroupsPerFile=False, compression=True): +def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False, + outputFeedbackChannel=False, multipleGroupsPerFile=False, compression=True, + metafile=None): """ Convert the given file to a NeuroDataWithoutBorders file using pynwb @@ -25,6 +28,7 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False :param multipleGroupsPerFile: Write all Groups in the DAT file into one NWB file. By default we create one NWB per Group (ignored for ABF files). :param compression: Toggle compression for HDF5 datasets + :param metafile: The path to the metadata YAML file (optional) :return: path of the created NWB file """ @@ -53,16 +57,26 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False else: raise ValueError(f"The output file {outFile} does already exist.") + # Load metadata from YAML file + if metafile is None: + metadata = None + else: + print(metafile) + with open(metafile) as f: + metadata = yaml.safe_load(f) + if ext == ".abf": if outputMetadata: ABFConverter.outputMetadata(inFileOrFolder) else: - ABFConverter(inFileOrFolder, outFile, outputFeedbackChannel=outputFeedbackChannel, compression=compression) + ABFConverter(inFileOrFolder, outFile, outputFeedbackChannel=outputFeedbackChannel, compression=compression, + metadata=metadata) elif ext == ".dat": if outputMetadata: DatConverter.outputMetadata(inFileOrFolder) else: - DatConverter(inFileOrFolder, outFile, multipleGroupsPerFile=multipleGroupsPerFile, compression=compression) + DatConverter(inFileOrFolder, outFile, multipleGroupsPerFile=multipleGroupsPerFile, compression=compression, + metadata=metadata) else: raise ValueError(f"The extension {ext} is currently not supported.") @@ -99,6 +113,8 @@ def main(): help="Output ADC data to the NWB file which stems from stimulus feedback channels.") abf_group.add_argument("--realDataChannel", type=str, action="append", help=f"Define additional channels which hold non-feedback channel data. The default is {ABFConverter.adcNamesWithRealData}.") + abf_group.add_argument('--metafile', default=None, type=str, + help='The path to the metadata YAML file.') dat_group.add_argument("--multipleGroupsPerFile", action="store_true", default=False, help="Write all Groups from a DAT file into a single NWB file. By default we create one NWB file per Group.") @@ -131,7 +147,8 @@ def main(): outputMetadata=args.outputMetadata, outputFeedbackChannel=args.outputFeedbackChannel, multipleGroupsPerFile=args.multipleGroupsPerFile, - compression=args.compression) + compression=args.compression, + metafile=args.metafile) if __name__ == "__main__": diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 907ad4da..5e76a5cd 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -15,9 +15,12 @@ import pyabf from pynwb.device import Device +from pynwb.file import Subject from pynwb import NWBHDF5IO, NWBFile from pynwb.icephys import IntracellularElectrode +from ndx_dandi_icephys import DandiIcephysMetadata + from ipfx.x_to_nwb.conversion_utils import PLACEHOLDER, V_CLAMP_MODE, I_CLAMP_MODE, I0_CLAMP_MODE, \ parseUnit, getStimulusSeriesClass, getAcquiredSeriesClass, createSeriesName, convertDataset, \ getPackageInfo, createCycleID @@ -30,7 +33,7 @@ class ABFConverter: protocolStorageDir = None adcNamesWithRealData = ["IN 0", "IN 1", "IN 2", "IN 3"] - def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True): + def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True, metadata=None): """ Convert the given ABF file to NWB @@ -39,6 +42,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T outFile -- target filepath (must not exist) outputFeedbackChannel -- Output ADC data from feedback channels as well (useful for debugging only) compression -- Toggle compression for HDF5 datasets + metadata -- Metadata dictionary with user-defined values for some nwb fields """ inFiles = [] @@ -52,6 +56,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T self.outputFeedbackChannel = outputFeedbackChannel self.compression = compression + self.metadata = metadata self._settings = self._getJSONFiles(inFileOrFolder) @@ -72,6 +77,18 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T nwbFile = self._createFile() + # If Subject information is present in metadata + if self.metadata is not None: + if 'Subject' in self.metadata: + nwbFile.subject = self._createSubject() + if 'lab_meta_data' in self.metadata: + nwbFile.add_lab_meta_data( + DandiIcephysMetadata( + cell_id=self.metadata['lab_meta_data'].get('cell_id', None), + tissue_sample_id=self.metadata['lab_meta_data'].get('tissue_sample_id', None), + ) + ) + device = self._createDevice() nwbFile.add_device(device) @@ -306,23 +323,26 @@ def getFileComments(abfs): if len(session_description) == 0: session_description = PLACEHOLDER - identifier = sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest() - session_start_time = self.refabf.abfDateTime creatorName = self.refabf._stringsIndexed.uCreatorName creatorVersion = formatVersion(self.refabf.creatorVersion) - experiment_description = (f"{creatorName} v{creatorVersion}") - source_script_file_name = "run_x_to_nwb_conversion.py" - source_script = json.dumps(getPackageInfo(), sort_keys=True, indent=4) - session_id = PLACEHOLDER - - return NWBFile(session_description=session_description, - identifier=identifier, - session_start_time=session_start_time, - experimenter=None, - experiment_description=experiment_description, - session_id=session_id, - source_script_file_name=source_script_file_name, - source_script=source_script) + nwbfile_kwargs = dict( + session_description=session_description, + identifier=sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest(), + session_start_time=self.refabf.abfDateTime, + experimenter=None, + experiment_description="{} v{}".format(creatorName, creatorVersion), + source_script_file_name="run_x_to_nwb_conversion.py", + source_script=json.dumps(getPackageInfo(), sort_keys=True, indent=4), + session_id=PLACEHOLDER + ) + + if self.metadata and 'NWBFile' in self.metadata: + nwbfile_kwargs.update(self.metadata['NWBFile']) + + # Create nwbfile with initial metadata + nwbfile = NWBFile(**nwbfile_kwargs) + + return nwbfile def _createDevice(self): """ @@ -334,6 +354,12 @@ def _createDevice(self): return Device(f"{digitizer} with {telegraph}") + def _createSubject(self): + """ + Create a pynwb Subject object from the metadata contents. + """ + return Subject(**self.metadata['Subject']) + def _createElectrodes(self, device): """ Create pynwb ic_electrodes objects from the ABF file contents. @@ -362,7 +388,6 @@ def _createStimulusSeries(self, electrodes): counter = 0 for file_index, abf in enumerate(self.abfs): - stimulus_description = ABFConverter._getProtocolName(abf.protocol) scale_factor = self._getScaleFactor(abf, stimulus_description) @@ -490,7 +515,7 @@ def _getAmplifierSettings(self, abf, clampMode, adcName): d["whole_cell_capacitance_comp"] = np.nan d["whole_cell_series_resistance_comp"] = np.nan - elif clampMode in (I_CLAMP_MODE, I0_CLAMP_MODE): + elif clampMode in (I_CLAMP_MODE,): if settings["GetHoldingEnable"]: d["bias_current"] = settings["GetHolding"] else: @@ -593,7 +618,7 @@ def _createAcquiredSeries(self, electrodes): whole_cell_capacitance_comp=settings["whole_cell_capacitance_comp"], # noqa: E501 whole_cell_series_resistance_comp=settings["whole_cell_series_resistance_comp"]) # noqa: E501 - elif clampMode in (I_CLAMP_MODE, I0_CLAMP_MODE): + elif clampMode is I_CLAMP_MODE: acquistion_data = seriesClass(name=name, data=data, sweep_number=np.uint64(cycle_id), @@ -609,6 +634,20 @@ def _createAcquiredSeries(self, electrodes): bridge_balance=settings["bridge_balance"], stimulus_description=stimulus_description, capacitance_compensation=settings["capacitance_compensation"]) + elif clampMode is I0_CLAMP_MODE: + acquistion_data = seriesClass( + name=name, + data=data, + sweep_number=np.uint64(cycle_id), + unit=unit, + electrode=electrode, + gain=gain, + resolution=resolution, + conversion=conversion, + starting_time=starting_time, + rate=rate, + description=description, + ) else: raise ValueError(f"Unsupported clamp mode {clampMode}.") diff --git a/ipfx/x_to_nwb/DatConverter.py b/ipfx/x_to_nwb/DatConverter.py index 4bd3fe22..8490bbb0 100644 --- a/ipfx/x_to_nwb/DatConverter.py +++ b/ipfx/x_to_nwb/DatConverter.py @@ -10,6 +10,7 @@ from pynwb.device import Device from pynwb import NWBHDF5IO, NWBFile from pynwb.icephys import IntracellularElectrode +from pynwb.file import Subject from ipfx.x_to_nwb.hr_bundle import Bundle from ipfx.x_to_nwb.hr_stimsetgenerator import StimSetGenerator @@ -22,7 +23,7 @@ class DatConverter: - def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=True): + def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=True, metadata=None): """ Convert DAT files, created by PatchMaster, to NWB v2 files. @@ -33,6 +34,7 @@ def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=Tru multipleGroupsPerFile: switch determining if multiple DAT groups per file are created or not compression: Toggle compression for HDF5 datasets + metadata: Metadata dictionary with user-defined values for some nwb fields Returns ------- @@ -44,6 +46,7 @@ def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=Tru self.bundle = Bundle(inFile) self.compression = compression + self.metadata = metadata self._check() @@ -63,6 +66,10 @@ def generateList(multipleGroupsPerFile, pul): nwbFile = self._createFile() + # If Subject information is present in metadata, add a Subject object + if self.metadata is not None and 'Subject' in self.metadata: + nwbFile.subject = Subject(**self.metadata['Subject']) + device = self._createDevice() nwbFile.add_device(device) @@ -298,7 +305,7 @@ def _isValidAmplifierState(ampState): def _getAmplifierState(bundle, series, trace): """ Different PatchMaster versions create different DAT file layouts. This function tries - to accomodate that as it returns the correct object. + to accommodate that as it returns the correct object. Parameters ---------- @@ -401,14 +408,19 @@ def _createFile(self): source_script = json.dumps(getPackageInfo(), sort_keys=True, indent=4) session_id = PLACEHOLDER - return NWBFile(session_description=session_description, - identifier=identifier, - session_start_time=self.session_start_time, - experimenter=None, - experiment_description=experiment_description, - session_id=session_id, - source_script=source_script, - source_script_file_name=source_script_file_name) + nwbfile_kwargs = dict(session_description=session_description, + identifier=identifier, + session_start_time=self.session_start_time, + experimenter=None, + experiment_description=experiment_description, + session_id=session_id, + source_script=source_script, + source_script_file_name=source_script_file_name) + + if self.metadata is not None and 'NWBFile' in self.metadata: + nwbfile_kwargs.update(self.metadata['NWBFile']) + + return NWBFile(**nwbfile_kwargs) def _createDevice(self): """ diff --git a/ipfx/x_to_nwb/conversion_utils.py b/ipfx/x_to_nwb/conversion_utils.py index d631b222..e93a11e2 100644 --- a/ipfx/x_to_nwb/conversion_utils.py +++ b/ipfx/x_to_nwb/conversion_utils.py @@ -23,7 +23,6 @@ I_CLAMP_MODE = 1 I0_CLAMP_MODE = 2 - # TODO Use the pint package if doing that manually gets too involved def parseUnit(unitString): """ diff --git a/requirements.txt b/requirements.txt index 3c34ad8b..87b99c86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ pynwb==1.0.2 six watchdog pg8000 +pyyaml diff --git a/tox.ini b/tox.ini index 7d674a67..04cc7717 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = python2.7, python3.6 +envlist = python3.6 [testenv]