From f5265e63a8f84385288dc7222ecd983b1d9af5a6 Mon Sep 17 00:00:00 2001 From: luiztauffer Date: Wed, 12 Feb 2020 14:38:26 +0700 Subject: [PATCH 01/18] uses IZeroClampSeries --- ipfx/x_to_nwb/ABFConverter.py | 23 +++++++++++++++++++++-- ipfx/x_to_nwb/conversion_utils.py | 14 +++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 3bb3ed07..cb6c05a2 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -18,7 +18,7 @@ from pynwb import NWBHDF5IO, NWBFile from pynwb.icephys import IntracellularElectrode -from ipfx.x_to_nwb.conversion_utils import PLACEHOLDER, V_CLAMP_MODE, I_CLAMP_MODE, \ +from ipfx.x_to_nwb.conversion_utils import PLACEHOLDER, V_CLAMP_MODE, I_CLAMP_MODE, I0_CLAMP_MODE, \ parseUnit, getStimulusSeriesClass, getAcquiredSeriesClass, createSeriesName, convertDataset, \ getPackageInfo, createCycleID @@ -362,7 +362,6 @@ def _createStimulusSeries(self, electrodes): counter = 0 for file_index, abf in enumerate(self.abfs): - stimulus_description = ABFConverter._getProtocolName(abf.protocol) scale_factor = self._getScaleFactor(abf, stimulus_description) @@ -519,6 +518,10 @@ def _getAmplifierSettings(self, abf, clampMode, adcName): d["bias_current"] = np.nan d["bridge_balance"] = np.nan d["capacitance_compensation"] = np.nan + elif clampMode == I0_CLAMP_MODE: + d["bias_current"] = np.nan + d["bridge_balance"] = np.nan + d["capacitance_compensation"] = np.nan else: warnings.warn("Unsupported clamp mode {clampMode}") @@ -608,6 +611,22 @@ def _createAcquiredSeries(self, electrodes): bridge_balance=settings["bridge_balance"], stimulus_description=stimulus_description, capacitance_compensation=settings["capacitance_compensation"]) + elif clampMode == I0_CLAMP_MODE: + acquistion_data = seriesClass(name=name, + data=data, + sweep_number=np.uint64(cycle_id), + unit=unit, + electrode=electrode, + gain=gain, + resolution=resolution, + conversion=conversion, + starting_time=starting_time, + rate=rate, + description=description, + bias_current=settings["bias_current"], + bridge_balance=settings["bridge_balance"], + stimulus_description=stimulus_description, + capacitance_compensation=settings["capacitance_compensation"]) else: raise ValueError(f"Unsupported clamp mode {clampMode}.") diff --git a/ipfx/x_to_nwb/conversion_utils.py b/ipfx/x_to_nwb/conversion_utils.py index dbca40eb..84c89afa 100644 --- a/ipfx/x_to_nwb/conversion_utils.py +++ b/ipfx/x_to_nwb/conversion_utils.py @@ -10,8 +10,8 @@ import numpy as np -from pynwb.icephys import CurrentClampStimulusSeries, VoltageClampStimulusSeries -from pynwb.icephys import CurrentClampSeries, VoltageClampSeries +from pynwb.icephys import (CurrentClampStimulusSeries, VoltageClampStimulusSeries, + CurrentClampSeries, VoltageClampSeries, IZeroClampSeries) try: from pynwb.form.backends.hdf5.h5_utils import H5DataIO @@ -21,7 +21,7 @@ PLACEHOLDER = "PLACEHOLDER" V_CLAMP_MODE = 0 I_CLAMP_MODE = 1 - +I0_CLAMP_MODE = 2 # TODO Use the pint package if doing that manually gets too involved def parseUnit(unitString): @@ -31,6 +31,8 @@ def parseUnit(unitString): if unitString == "pA": return 1e-12, "A" + elif unitString == "nA": + return 1e-9, "A" elif unitString == "A": return 1.0, "A" elif unitString == "mV": @@ -50,6 +52,8 @@ def getStimulusSeriesClass(clampMode): return VoltageClampStimulusSeries elif clampMode == I_CLAMP_MODE: return CurrentClampStimulusSeries + elif clampMode == I0_CLAMP_MODE: + return CurrentClampStimulusSeries else: raise ValueError(f"Unsupported clamp mode {clampMode}.") @@ -63,6 +67,8 @@ def getAcquiredSeriesClass(clampMode): return VoltageClampSeries elif clampMode == I_CLAMP_MODE: return CurrentClampSeries + elif clampMode == I0_CLAMP_MODE: + return IZeroClampSeries else: raise ValueError(f"Unsupported clamp mode {clampMode}.") @@ -188,5 +194,7 @@ def clampModeToString(clampMode): return "I_CLAMP_MODE" elif clampMode == V_CLAMP_MODE: return "V_CLAMP_MODE" + elif clampMode == I0_CLAMP_MODE: + return "I0_CLAMP_MODE" else: raise ValueError(f"Unknown clampMode {clampMode}") From 3c9098a8190661613d4dadac398d092fd817882b Mon Sep 17 00:00:00 2001 From: luiztauffer Date: Wed, 12 Feb 2020 16:05:57 +0700 Subject: [PATCH 02/18] fills settings values to np.nan, for I0_CLAMP_MODE, if there is a settings json file to be read --- ipfx/x_to_nwb/ABFConverter.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index cb6c05a2..37042419 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -503,6 +503,10 @@ def _getAmplifierSettings(self, abf, clampMode, adcName): d["capacitance_compensation"] = settings["GetNeutralizationCap"] else: d["capacitance_compensation"] = np.nan + elif clampMode == I0_CLAMP_MODE: + d["bias_current"] = np.nan + d["bridge_balance"] = np.nan + d["capacitance_compensation"] = np.nan else: warnings.warn("Unsupported clamp mode {clampMode}") else: From 0f4373b4c2888a7506d39c06dafbeceda98a4c80 Mon Sep 17 00:00:00 2001 From: luiztauffer Date: Wed, 19 Feb 2020 14:51:11 +0700 Subject: [PATCH 03/18] IZeroClampSeries does not require a CurrentClampSeries stimulus --- ipfx/x_to_nwb/ABFConverter.py | 29 +++++++++++++++-------------- ipfx/x_to_nwb/conversion_utils.py | 2 +- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 37042419..6a68cfb8 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -391,20 +391,21 @@ def _createStimulusSeries(self, electrodes): seriesClass = getStimulusSeriesClass(self._getClampMode(abf, channel)) - stimulus = seriesClass(name=name, - data=data, - sweep_number=np.uint64(cycle_id), - unit=unit, - electrode=electrode, - gain=gain, - resolution=resolution, - conversion=conversion, - starting_time=starting_time, - rate=rate, - description=description, - stimulus_description=stimulus_description) - - series.append(stimulus) + if seriesClass is not None: + stimulus = seriesClass(name=name, + data=data, + sweep_number=np.uint64(cycle_id), + unit=unit, + electrode=electrode, + gain=gain, + resolution=resolution, + conversion=conversion, + starting_time=starting_time, + rate=rate, + description=description, + stimulus_description=stimulus_description) + + series.append(stimulus) return series diff --git a/ipfx/x_to_nwb/conversion_utils.py b/ipfx/x_to_nwb/conversion_utils.py index 84c89afa..a8eecd34 100644 --- a/ipfx/x_to_nwb/conversion_utils.py +++ b/ipfx/x_to_nwb/conversion_utils.py @@ -53,7 +53,7 @@ def getStimulusSeriesClass(clampMode): elif clampMode == I_CLAMP_MODE: return CurrentClampStimulusSeries elif clampMode == I0_CLAMP_MODE: - return CurrentClampStimulusSeries + return None else: raise ValueError(f"Unsupported clamp mode {clampMode}.") From 16b136f9c6f2b432fadd8cc363d812c3033f1669 Mon Sep 17 00:00:00 2001 From: luiztauffer Date: Wed, 19 Feb 2020 15:44:15 +0700 Subject: [PATCH 04/18] - metafile passed as optional argument to ABFConverter - pyyaml added in requirements.txt --- ipfx/bin/run_x_to_nwb_conversion.py | 21 ++++++++++++++++++--- ipfx/x_to_nwb/ABFConverter.py | 3 ++- requirements.txt | 1 + 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/ipfx/bin/run_x_to_nwb_conversion.py b/ipfx/bin/run_x_to_nwb_conversion.py index b56e09c5..74b26825 100755 --- a/ipfx/bin/run_x_to_nwb_conversion.py +++ b/ipfx/bin/run_x_to_nwb_conversion.py @@ -1,5 +1,6 @@ #!/bin/env python +import yaml import os import argparse import logging @@ -9,7 +10,9 @@ from ipfx.x_to_nwb.DatConverter import DatConverter -def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False, outputFeedbackChannel=False, multipleGroupsPerFile=False, compression=True): +def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False, + outputFeedbackChannel=False, multipleGroupsPerFile=False, compression=True, + metafile=None): """ Convert the given file to a NeuroDataWithoutBorders file using pynwb @@ -25,6 +28,7 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False :param multipleGroupsPerFile: Write all Groups in the DAT file into one NWB file. By default we create one NWB per Group (ignored for ABF files). :param compression: Toggle compression for HDF5 datasets + :param metafile: The path to the metadata YAML file (optional) :return: path of the created NWB file """ @@ -53,11 +57,19 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False else: raise ValueError(f"The output file {outFile} does already exist.") + # Load metadata from YAML file + if metafile is None: + metadata = {} + else: + with open(metafile) as f: + metadata = yaml.safe_load(f) + if ext == ".abf": if outputMetadata: ABFConverter.outputMetadata(inFileOrFolder) else: - ABFConverter(inFileOrFolder, outFile, outputFeedbackChannel=outputFeedbackChannel, compression=compression) + ABFConverter(inFileOrFolder, outFile, outputFeedbackChannel=outputFeedbackChannel, compression=compression, + metadata=metadata) elif ext == ".dat": if outputMetadata: DatConverter.outputMetadata(inFileOrFolder) @@ -99,6 +111,8 @@ def main(): help="Output ADC data to the NWB file which stems from stimulus feedback channels.") abf_group.add_argument("--realDataChannel", type=str, action="append", help=f"Define additional channels which hold non-feedback channel data. The default is {ABFConverter.adcNamesWithRealData}.") + abf_group.add_argument('--metafile', default=None, type=argparse.FileType('r'), + help='The path to the metadata YAML file.') dat_group.add_argument("--multipleGroupsPerFile", action="store_true", default=False, help="Write all Groups from a DAT file into a single NWB file. By default we create one NWB file per Group.") @@ -131,7 +145,8 @@ def main(): outputMetadata=args.outputMetadata, outputFeedbackChannel=args.outputFeedbackChannel, multipleGroupsPerFile=args.multipleGroupsPerFile, - compression=args.compression) + compression=args.compression, + metafile=args.metafile) if __name__ == "__main__": diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 6a68cfb8..bce45ce0 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -30,7 +30,7 @@ class ABFConverter: protocolStorageDir = None adcNamesWithRealData = ["IN 0", "IN 1", "IN 2", "IN 3"] - def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True): + def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True, metadata={}): """ Convert the given ABF file to NWB @@ -39,6 +39,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T outFile -- target filepath (must not exist) outputFeedbackChannel -- Output ADC data from feedback channels as well (useful for debugging only) compression -- Toggle compression for HDF5 datasets + metadata -- Metadata dictionary with user-defined values for some nwb fields """ inFiles = [] diff --git a/requirements.txt b/requirements.txt index c3caf9ed..b64f1afc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,3 +14,4 @@ pynwb==1.0.2 six watchdog pg8000 +pyyaml From 760539faf816453103313cfded4ef85c5a6c2a37 Mon Sep 17 00:00:00 2001 From: luiztauffer Date: Wed, 19 Feb 2020 16:01:31 +0700 Subject: [PATCH 05/18] use (optional) metadata information when creating nwbfile --- ipfx/x_to_nwb/ABFConverter.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index bce45ce0..8ad8ac9e 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -71,7 +71,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T self.totalSeriesCount = self._getMaxTimeSeriesCount() - nwbFile = self._createFile() + nwbFile = self._createFile(metadata=metadata) device = self._createDevice() nwbFile.add_device(device) @@ -279,7 +279,7 @@ def getCount(abf): return sum(map(getCount, self.abfs)) - def _createFile(self): + def _createFile(self, metadata={}): """ Create a pynwb NWBFile object from the ABF file contents. """ @@ -307,23 +307,23 @@ def getFileComments(abfs): if len(session_description) == 0: session_description = PLACEHOLDER - identifier = sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest() - session_start_time = self.refabf.abfDateTime + meta_ini = {} + meta_ini['identifier'] = sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest() + meta_ini['session_start_time'] = self.refabf.abfDateTime creatorName = self.refabf._stringsIndexed.uCreatorName creatorVersion = formatVersion(self.refabf.creatorVersion) - experiment_description = (f"{creatorName} v{creatorVersion}") - source_script_file_name = "run_x_to_nwb_conversion.py" - source_script = json.dumps(getPackageInfo(), sort_keys=True, indent=4) - session_id = PLACEHOLDER - - return NWBFile(session_description=session_description, - identifier=identifier, - session_start_time=session_start_time, - experimenter=None, - experiment_description=experiment_description, - session_id=session_id, - source_script_file_name=source_script_file_name, - source_script=source_script) + meta_ini['experiment_description'] = (f"{creatorName} v{creatorVersion}") + meta_ini['source_script_file_name'] = "run_x_to_nwb_conversion.py" + meta_ini['source_script'] = json.dumps(getPackageInfo(), sort_keys=True, indent=4) + meta_ini['session_id'] = PLACEHOLDER + + # Overwrite default values with user-passed values + meta_ini.update(metadata['NWBFile']) + + # Create nwbfile with initial metadata + nwbfile = NWBFile(**meta_ini) + + return nwbfile def _createDevice(self): """ From 7a6dd3eef62f7fc1869f76c9c3c430221202c074 Mon Sep 17 00:00:00 2001 From: luiztauffer Date: Wed, 19 Feb 2020 16:22:39 +0700 Subject: [PATCH 06/18] add Subject to nwbfile if present in metadata WARNING: there is a bug on the required version of pynwb (1.0.2) that won't allow passing the field 'date_of_birth' for the Subject object. Either ther version should be updated to the most recent, or 'date_of_birth' can't be used --- ipfx/x_to_nwb/ABFConverter.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 8ad8ac9e..cf728e6e 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -15,6 +15,7 @@ import pyabf from pynwb.device import Device +from pynwb.file import Subject from pynwb import NWBHDF5IO, NWBFile from pynwb.icephys import IntracellularElectrode @@ -73,6 +74,11 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T nwbFile = self._createFile(metadata=metadata) + # If Subject information is present in metadata + if 'Subject' in metadata: + subject = self._createSubject(metadata['Subject']) + nwbFile.subject = subject + device = self._createDevice() nwbFile.add_device(device) @@ -335,6 +341,22 @@ def _createDevice(self): return Device(f"{digitizer} with {telegraph}") + def _createSubject(self, metadata): + """ + Create a pynwb Subject object from the metadata contents. + """ + subject = Subject( + age=metadata['age'], + subject_id=metadata['subject_id'], + species=metadata['species'], + description=metadata['description'], + genotype=metadata['genotype'], + # date_of_birth=metadata['date_of_birth'], + weight=metadata['weight'], + sex=metadata['sex'] + ) + return subject + def _createElectrodes(self, device): """ Create pynwb ic_electrodes objects from the ABF file contents. From 2ea372b3c8c667ec2c170043ab298816dc73203e Mon Sep 17 00:00:00 2001 From: luiztauffer Date: Wed, 19 Feb 2020 18:53:19 +0700 Subject: [PATCH 07/18] - Loads LabMetaData (extension module) information to created nwbfile file - adds ndx-labmetadata-abf extension to requirements --- ipfx/x_to_nwb/ABFConverter.py | 17 ++++++++++++++++- requirements.txt | 1 + 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index cf728e6e..7fb232ac 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -9,6 +9,7 @@ import glob import warnings import logging +import copy import numpy as np @@ -323,12 +324,26 @@ def getFileComments(abfs): meta_ini['source_script'] = json.dumps(getPackageInfo(), sort_keys=True, indent=4) meta_ini['session_id'] = PLACEHOLDER + meta_ini_user = copy.deepcopy(metadata['NWBFile']) + meta_lab_meta_data = meta_ini_user.pop('lab_meta_data', None) + # Overwrite default values with user-passed values - meta_ini.update(metadata['NWBFile']) + meta_ini.update(meta_ini_user) # Create nwbfile with initial metadata nwbfile = NWBFile(**meta_ini) + # Creates LabMetaData container + if meta_lab_meta_data is not None: + from ndx_labmetadata_abf import LabMetaData_ext + + lab_metadata = LabMetaData_ext( + name=meta_lab_meta_data['name'], + cell_id=meta_lab_meta_data['cell_id'], + tissue_sample_id=meta_lab_meta_data['tissue_sample_id'], + ) + nwbfile.add_lab_meta_data(lab_metadata) + return nwbfile def _createDevice(self): diff --git a/requirements.txt b/requirements.txt index b64f1afc..c85c2e1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ six watchdog pg8000 pyyaml +ndx-labmetadata-abf From d68053798cbe70615f32735c44319880569fee50 Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Wed, 19 Feb 2020 09:48:10 -0800 Subject: [PATCH 08/18] clean up ABFConverter --- ipfx/x_to_nwb/ABFConverter.py | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 7fb232ac..2981e4d4 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -9,7 +9,6 @@ import glob import warnings import logging -import copy import numpy as np @@ -314,35 +313,27 @@ def getFileComments(abfs): if len(session_description) == 0: session_description = PLACEHOLDER - meta_ini = {} - meta_ini['identifier'] = sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest() - meta_ini['session_start_time'] = self.refabf.abfDateTime creatorName = self.refabf._stringsIndexed.uCreatorName creatorVersion = formatVersion(self.refabf.creatorVersion) - meta_ini['experiment_description'] = (f"{creatorName} v{creatorVersion}") - meta_ini['source_script_file_name'] = "run_x_to_nwb_conversion.py" - meta_ini['source_script'] = json.dumps(getPackageInfo(), sort_keys=True, indent=4) - meta_ini['session_id'] = PLACEHOLDER - - meta_ini_user = copy.deepcopy(metadata['NWBFile']) - meta_lab_meta_data = meta_ini_user.pop('lab_meta_data', None) + meta_ini = dict( + identifier=sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest(), + session_start_time=self.refabf.abfDateTime, + experiment_description="{} v{}".format(creatorName, creatorVersion), + source_script_file_name="run_x_to_nwb_conversion.py", + source_script=json.dumps(getPackageInfo(), sort_keys=True, indent=4), + session_id=PLACEHOLDER + ) # Overwrite default values with user-passed values - meta_ini.update(meta_ini_user) + meta_ini.update(metadata['NWBFile']) # Create nwbfile with initial metadata nwbfile = NWBFile(**meta_ini) # Creates LabMetaData container - if meta_lab_meta_data is not None: + if 'lab_meta_data' in metadata: from ndx_labmetadata_abf import LabMetaData_ext - - lab_metadata = LabMetaData_ext( - name=meta_lab_meta_data['name'], - cell_id=meta_lab_meta_data['cell_id'], - tissue_sample_id=meta_lab_meta_data['tissue_sample_id'], - ) - nwbfile.add_lab_meta_data(lab_metadata) + nwbfile.add_lab_meta_data(LabMetaData_ext(**metadata['lab_meta_data'])) return nwbfile From 44ba66288c86987b20e98ecf5a3ec16ca989cd4e Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Wed, 19 Feb 2020 09:56:00 -0800 Subject: [PATCH 09/18] clean up _createSubject, passing dict through to constructor --- ipfx/x_to_nwb/ABFConverter.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 2981e4d4..0153718a 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -76,7 +76,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T # If Subject information is present in metadata if 'Subject' in metadata: - subject = self._createSubject(metadata['Subject']) + subject = self._createSubject(**metadata['Subject']) nwbFile.subject = subject device = self._createDevice() @@ -347,21 +347,12 @@ def _createDevice(self): return Device(f"{digitizer} with {telegraph}") - def _createSubject(self, metadata): + def _createSubject(self, **subject_fields): """ Create a pynwb Subject object from the metadata contents. """ - subject = Subject( - age=metadata['age'], - subject_id=metadata['subject_id'], - species=metadata['species'], - description=metadata['description'], - genotype=metadata['genotype'], - # date_of_birth=metadata['date_of_birth'], - weight=metadata['weight'], - sex=metadata['sex'] - ) - return subject + subject_fields.pop('date_of_birth') # backwards compatibility + return Subject(**subject_fields) def _createElectrodes(self, device): """ From bb6edb0bcb214068a07eaf49ae3fc04842968ea2 Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Wed, 19 Feb 2020 09:59:23 -0800 Subject: [PATCH 10/18] fill in missing fields of NWBFile constructor --- ipfx/x_to_nwb/ABFConverter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 0153718a..64ae69f2 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -316,8 +316,10 @@ def getFileComments(abfs): creatorName = self.refabf._stringsIndexed.uCreatorName creatorVersion = formatVersion(self.refabf.creatorVersion) meta_ini = dict( + session_description=session_description, identifier=sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest(), session_start_time=self.refabf.abfDateTime, + experimenter=None, experiment_description="{} v{}".format(creatorName, creatorVersion), source_script_file_name="run_x_to_nwb_conversion.py", source_script=json.dumps(getPackageInfo(), sort_keys=True, indent=4), From a9e9a5e04540fc807d777dd9d75a837581122a8c Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Wed, 19 Feb 2020 10:18:33 -0800 Subject: [PATCH 11/18] check if metadata contains NWBFile --- ipfx/x_to_nwb/ABFConverter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 64ae69f2..5cb66240 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -327,7 +327,8 @@ def getFileComments(abfs): ) # Overwrite default values with user-passed values - meta_ini.update(metadata['NWBFile']) + if 'NWBFile' in metadata: + meta_ini.update(metadata['NWBFile']) # Create nwbfile with initial metadata nwbfile = NWBFile(**meta_ini) From 00d7e7f23a84efb69414302c57ba34e56ecea061 Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Thu, 5 Mar 2020 21:48:29 +1300 Subject: [PATCH 12/18] * add check for subject date_of_birth * change name of extension --- ipfx/x_to_nwb/ABFConverter.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 5cb66240..149d1331 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -335,8 +335,8 @@ def getFileComments(abfs): # Creates LabMetaData container if 'lab_meta_data' in metadata: - from ndx_labmetadata_abf import LabMetaData_ext - nwbfile.add_lab_meta_data(LabMetaData_ext(**metadata['lab_meta_data'])) + from ndx_dandi_icephys import DandiIcephysMetadata + nwbfile.add_lab_meta_data(DandiIcephysMetadata(**metadata['lab_meta_data'])) return nwbfile @@ -354,7 +354,9 @@ def _createSubject(self, **subject_fields): """ Create a pynwb Subject object from the metadata contents. """ - subject_fields.pop('date_of_birth') # backwards compatibility + if 'date_of_birth' in subject_fields: + subject_fields.pop('date_of_birth') # backwards compatibility + warnings.warn('date_of_birth removed from Subject info. Must update pynwb version to accept this field.') return Subject(**subject_fields) def _createElectrodes(self, device): From e5bf72503388c1022422100e82caebbb77994379 Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Thu, 5 Mar 2020 22:14:18 +1300 Subject: [PATCH 13/18] change metadata default to None --- ipfx/x_to_nwb/ABFConverter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 149d1331..fb6c172a 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -285,7 +285,7 @@ def getCount(abf): return sum(map(getCount, self.abfs)) - def _createFile(self, metadata={}): + def _createFile(self, metadata=None): """ Create a pynwb NWBFile object from the ABF file contents. """ @@ -327,14 +327,14 @@ def getFileComments(abfs): ) # Overwrite default values with user-passed values - if 'NWBFile' in metadata: + if metadata and 'NWBFile' in metadata: meta_ini.update(metadata['NWBFile']) # Create nwbfile with initial metadata nwbfile = NWBFile(**meta_ini) # Creates LabMetaData container - if 'lab_meta_data' in metadata: + if metadata and 'lab_meta_data' in metadata: from ndx_dandi_icephys import DandiIcephysMetadata nwbfile.add_lab_meta_data(DandiIcephysMetadata(**metadata['lab_meta_data'])) From 9dd1827b34facd086812da3dcd589aa974ef029b Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Fri, 10 Apr 2020 22:21:47 -0400 Subject: [PATCH 14/18] add optional metadata arg --- ipfx/bin/run_x_to_nwb_conversion.py | 8 ++++--- ipfx/x_to_nwb/ABFConverter.py | 33 +++++++++++------------------ ipfx/x_to_nwb/DatConverter.py | 32 +++++++++++++++++++--------- tox.ini | 2 +- 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/ipfx/bin/run_x_to_nwb_conversion.py b/ipfx/bin/run_x_to_nwb_conversion.py index 74b26825..91947868 100755 --- a/ipfx/bin/run_x_to_nwb_conversion.py +++ b/ipfx/bin/run_x_to_nwb_conversion.py @@ -59,8 +59,9 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False # Load metadata from YAML file if metafile is None: - metadata = {} + metadata = None else: + print(metafile) with open(metafile) as f: metadata = yaml.safe_load(f) @@ -74,7 +75,8 @@ def convert(inFileOrFolder, overwrite=False, fileType=None, outputMetadata=False if outputMetadata: DatConverter.outputMetadata(inFileOrFolder) else: - DatConverter(inFileOrFolder, outFile, multipleGroupsPerFile=multipleGroupsPerFile, compression=compression) + DatConverter(inFileOrFolder, outFile, multipleGroupsPerFile=multipleGroupsPerFile, compression=compression, + metadata=metadata) else: raise ValueError(f"The extension {ext} is currently not supported.") @@ -111,7 +113,7 @@ def main(): help="Output ADC data to the NWB file which stems from stimulus feedback channels.") abf_group.add_argument("--realDataChannel", type=str, action="append", help=f"Define additional channels which hold non-feedback channel data. The default is {ABFConverter.adcNamesWithRealData}.") - abf_group.add_argument('--metafile', default=None, type=argparse.FileType('r'), + abf_group.add_argument('--metafile', default=None, type=str, help='The path to the metadata YAML file.') dat_group.add_argument("--multipleGroupsPerFile", action="store_true", default=False, diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 21de357f..551bcccb 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -31,7 +31,7 @@ class ABFConverter: protocolStorageDir = None adcNamesWithRealData = ["IN 0", "IN 1", "IN 2", "IN 3"] - def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True, metadata={}): + def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=True, metadata=None): """ Convert the given ABF file to NWB @@ -54,6 +54,7 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T self.outputFeedbackChannel = outputFeedbackChannel self.compression = compression + self.metadata = metadata self._settings = self._getJSONFiles(inFileOrFolder) @@ -72,12 +73,11 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T self.totalSeriesCount = self._getMaxTimeSeriesCount() - nwbFile = self._createFile(metadata=metadata) + nwbFile = self._createFile() # If Subject information is present in metadata - if 'Subject' in metadata: - subject = self._createSubject(**metadata['Subject']) - nwbFile.subject = subject + if self.metadata is not None and 'Subject' in self.metadata: + nwbFile.subject = self._createSubject() device = self._createDevice() nwbFile.add_device(device) @@ -285,7 +285,7 @@ def getCount(abf): return sum(map(getCount, self.abfs)) - def _createFile(self, metadata=None): + def _createFile(self): """ Create a pynwb NWBFile object from the ABF file contents. """ @@ -315,7 +315,7 @@ def getFileComments(abfs): creatorName = self.refabf._stringsIndexed.uCreatorName creatorVersion = formatVersion(self.refabf.creatorVersion) - meta_ini = dict( + nwbfile_kwargs = dict( session_description=session_description, identifier=sha256(" ".join([abf.fileGUID for abf in self.abfs]).encode()).hexdigest(), session_start_time=self.refabf.abfDateTime, @@ -326,17 +326,11 @@ def getFileComments(abfs): session_id=PLACEHOLDER ) - # Overwrite default values with user-passed values - if metadata and 'NWBFile' in metadata: - meta_ini.update(metadata['NWBFile']) + if self.metadata and 'NWBFile' in self.metadata: + nwbfile_kwargs.update(self.metadata['NWBFile']) # Create nwbfile with initial metadata - nwbfile = NWBFile(**meta_ini) - - # Creates LabMetaData container - if metadata and 'lab_meta_data' in metadata: - from ndx_dandi_icephys import DandiIcephysMetadata - nwbfile.add_lab_meta_data(DandiIcephysMetadata(**metadata['lab_meta_data'])) + nwbfile = NWBFile(**nwbfile_kwargs) return nwbfile @@ -350,14 +344,11 @@ def _createDevice(self): return Device(f"{digitizer} with {telegraph}") - def _createSubject(self, **subject_fields): + def _createSubject(self): """ Create a pynwb Subject object from the metadata contents. """ - if 'date_of_birth' in subject_fields: - subject_fields.pop('date_of_birth') # backwards compatibility - warnings.warn('date_of_birth removed from Subject info. Must update pynwb version to accept this field.') - return Subject(**subject_fields) + return Subject(**self.metadata['Subject']) def _createElectrodes(self, device): """ diff --git a/ipfx/x_to_nwb/DatConverter.py b/ipfx/x_to_nwb/DatConverter.py index 4bd3fe22..8490bbb0 100644 --- a/ipfx/x_to_nwb/DatConverter.py +++ b/ipfx/x_to_nwb/DatConverter.py @@ -10,6 +10,7 @@ from pynwb.device import Device from pynwb import NWBHDF5IO, NWBFile from pynwb.icephys import IntracellularElectrode +from pynwb.file import Subject from ipfx.x_to_nwb.hr_bundle import Bundle from ipfx.x_to_nwb.hr_stimsetgenerator import StimSetGenerator @@ -22,7 +23,7 @@ class DatConverter: - def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=True): + def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=True, metadata=None): """ Convert DAT files, created by PatchMaster, to NWB v2 files. @@ -33,6 +34,7 @@ def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=Tru multipleGroupsPerFile: switch determining if multiple DAT groups per file are created or not compression: Toggle compression for HDF5 datasets + metadata: Metadata dictionary with user-defined values for some nwb fields Returns ------- @@ -44,6 +46,7 @@ def __init__(self, inFile, outFile, multipleGroupsPerFile=False, compression=Tru self.bundle = Bundle(inFile) self.compression = compression + self.metadata = metadata self._check() @@ -63,6 +66,10 @@ def generateList(multipleGroupsPerFile, pul): nwbFile = self._createFile() + # If Subject information is present in metadata, add a Subject object + if self.metadata is not None and 'Subject' in self.metadata: + nwbFile.subject = Subject(**self.metadata['Subject']) + device = self._createDevice() nwbFile.add_device(device) @@ -298,7 +305,7 @@ def _isValidAmplifierState(ampState): def _getAmplifierState(bundle, series, trace): """ Different PatchMaster versions create different DAT file layouts. This function tries - to accomodate that as it returns the correct object. + to accommodate that as it returns the correct object. Parameters ---------- @@ -401,14 +408,19 @@ def _createFile(self): source_script = json.dumps(getPackageInfo(), sort_keys=True, indent=4) session_id = PLACEHOLDER - return NWBFile(session_description=session_description, - identifier=identifier, - session_start_time=self.session_start_time, - experimenter=None, - experiment_description=experiment_description, - session_id=session_id, - source_script=source_script, - source_script_file_name=source_script_file_name) + nwbfile_kwargs = dict(session_description=session_description, + identifier=identifier, + session_start_time=self.session_start_time, + experimenter=None, + experiment_description=experiment_description, + session_id=session_id, + source_script=source_script, + source_script_file_name=source_script_file_name) + + if self.metadata is not None and 'NWBFile' in self.metadata: + nwbfile_kwargs.update(self.metadata['NWBFile']) + + return NWBFile(**nwbfile_kwargs) def _createDevice(self): """ diff --git a/tox.ini b/tox.ini index 7d674a67..04cc7717 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = python2.7, python3.6 +envlist = python3.6 [testenv] From 1e5bb1a3da6c10d1ac49c675a27abd40cd4f12b0 Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Fri, 10 Apr 2020 22:25:19 -0400 Subject: [PATCH 15/18] * remove old I0 logic * remove dandiset extension dep --- ipfx/x_to_nwb/ABFConverter.py | 20 -------------------- requirements.txt | 1 - 2 files changed, 21 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index 551bcccb..d613bb55 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -520,10 +520,6 @@ def _getAmplifierSettings(self, abf, clampMode, adcName): d["capacitance_compensation"] = settings["GetNeutralizationCap"] else: d["capacitance_compensation"] = np.nan - elif clampMode == I0_CLAMP_MODE: - d["bias_current"] = np.nan - d["bridge_balance"] = np.nan - d["capacitance_compensation"] = np.nan else: warnings.warn("Unsupported clamp mode {clampMode}") else: @@ -632,22 +628,6 @@ def _createAcquiredSeries(self, electrodes): bridge_balance=settings["bridge_balance"], stimulus_description=stimulus_description, capacitance_compensation=settings["capacitance_compensation"]) - elif clampMode == I0_CLAMP_MODE: - acquistion_data = seriesClass(name=name, - data=data, - sweep_number=np.uint64(cycle_id), - unit=unit, - electrode=electrode, - gain=gain, - resolution=resolution, - conversion=conversion, - starting_time=starting_time, - rate=rate, - description=description, - bias_current=settings["bias_current"], - bridge_balance=settings["bridge_balance"], - stimulus_description=stimulus_description, - capacitance_compensation=settings["capacitance_compensation"]) else: raise ValueError(f"Unsupported clamp mode {clampMode}.") diff --git a/requirements.txt b/requirements.txt index fb4091f2..87b99c86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,4 +15,3 @@ six watchdog pg8000 pyyaml -ndx-labmetadata-abf From aadd7d2eb3e869f5a8f10e71a9281129ec769d1a Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Fri, 10 Apr 2020 22:27:11 -0400 Subject: [PATCH 16/18] * remove old I0 logic --- ipfx/x_to_nwb/ABFConverter.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index d613bb55..ad31d40d 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -535,10 +535,6 @@ def _getAmplifierSettings(self, abf, clampMode, adcName): d["bias_current"] = np.nan d["bridge_balance"] = np.nan d["capacitance_compensation"] = np.nan - elif clampMode == I0_CLAMP_MODE: - d["bias_current"] = np.nan - d["bridge_balance"] = np.nan - d["capacitance_compensation"] = np.nan else: warnings.warn("Unsupported clamp mode {clampMode}") From 3ce72999bca511dd70e5e35b1ddbee6e8b1fc7a4 Mon Sep 17 00:00:00 2001 From: "!git for-each-ref --format='%(refname:short)' `git symbolic-ref HEAD`" Date: Wed, 24 Mar 2021 20:20:59 -0400 Subject: [PATCH 17/18] * added DANDI metadata * fixed inputs to IZeroClamp --- ipfx/x_to_nwb/ABFConverter.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/ipfx/x_to_nwb/ABFConverter.py b/ipfx/x_to_nwb/ABFConverter.py index ad31d40d..5e76a5cd 100644 --- a/ipfx/x_to_nwb/ABFConverter.py +++ b/ipfx/x_to_nwb/ABFConverter.py @@ -19,6 +19,8 @@ from pynwb import NWBHDF5IO, NWBFile from pynwb.icephys import IntracellularElectrode +from ndx_dandi_icephys import DandiIcephysMetadata + from ipfx.x_to_nwb.conversion_utils import PLACEHOLDER, V_CLAMP_MODE, I_CLAMP_MODE, I0_CLAMP_MODE, \ parseUnit, getStimulusSeriesClass, getAcquiredSeriesClass, createSeriesName, convertDataset, \ getPackageInfo, createCycleID @@ -76,8 +78,16 @@ def __init__(self, inFileOrFolder, outFile, outputFeedbackChannel, compression=T nwbFile = self._createFile() # If Subject information is present in metadata - if self.metadata is not None and 'Subject' in self.metadata: - nwbFile.subject = self._createSubject() + if self.metadata is not None: + if 'Subject' in self.metadata: + nwbFile.subject = self._createSubject() + if 'lab_meta_data' in self.metadata: + nwbFile.add_lab_meta_data( + DandiIcephysMetadata( + cell_id=self.metadata['lab_meta_data'].get('cell_id', None), + tissue_sample_id=self.metadata['lab_meta_data'].get('tissue_sample_id', None), + ) + ) device = self._createDevice() nwbFile.add_device(device) @@ -505,7 +515,7 @@ def _getAmplifierSettings(self, abf, clampMode, adcName): d["whole_cell_capacitance_comp"] = np.nan d["whole_cell_series_resistance_comp"] = np.nan - elif clampMode in (I_CLAMP_MODE, I0_CLAMP_MODE): + elif clampMode in (I_CLAMP_MODE,): if settings["GetHoldingEnable"]: d["bias_current"] = settings["GetHolding"] else: @@ -608,7 +618,7 @@ def _createAcquiredSeries(self, electrodes): whole_cell_capacitance_comp=settings["whole_cell_capacitance_comp"], # noqa: E501 whole_cell_series_resistance_comp=settings["whole_cell_series_resistance_comp"]) # noqa: E501 - elif clampMode in (I_CLAMP_MODE, I0_CLAMP_MODE): + elif clampMode is I_CLAMP_MODE: acquistion_data = seriesClass(name=name, data=data, sweep_number=np.uint64(cycle_id), @@ -624,6 +634,20 @@ def _createAcquiredSeries(self, electrodes): bridge_balance=settings["bridge_balance"], stimulus_description=stimulus_description, capacitance_compensation=settings["capacitance_compensation"]) + elif clampMode is I0_CLAMP_MODE: + acquistion_data = seriesClass( + name=name, + data=data, + sweep_number=np.uint64(cycle_id), + unit=unit, + electrode=electrode, + gain=gain, + resolution=resolution, + conversion=conversion, + starting_time=starting_time, + rate=rate, + description=description, + ) else: raise ValueError(f"Unsupported clamp mode {clampMode}.") From c7b45a01bbe4f98152359388e6cf7a45adf80f85 Mon Sep 17 00:00:00 2001 From: luiz Date: Mon, 2 Aug 2021 10:18:02 +0200 Subject: [PATCH 18/18] Merge branch 'master' into add_metadata --- .github/workflows/python-publish.yml | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 00000000..3bfabfc1 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,36 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }}