updated metadata plugin reader

lanl · Jan 13, 2025 · 3d1e25f · 3d1e25f
1 parent bac716e
commit 3d1e25f
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 9 deletions.
diff --git a/dsi/plugins/file_reader.py b/dsi/plugins/file_reader.py
@@ -591,7 +591,7 @@ class MetadataReader1(FileReader):
     def __init__(self, filenames, target_table_prefix = None, **kwargs):
         '''
         `filenames`: one metadata json file or a list of metadata json files to be ingested
-        `target_table_prefix`: prefix to be added to every table created to differentiate between other metadata json file sources
+        `target_table_prefix`: prefix to be added to every table created to differentiate between other metadata file sources
         '''
         super().__init__(filenames, **kwargs)
         if isinstance(filenames, str):
@@ -632,10 +632,10 @@ def add_rows(self) -> None:
                         json_data[col_name] = [str(col_data)]
                     else:
                         json_data[col_name] = [col_data]
-
+
+            filename = filename[filename.rfind("/") + 1:]
+            filename = filename[:filename.rfind(".")]
             if self.target_table_prefix is not None:
                 filename = self.target_table_prefix + "__" + filename
             self.metadata_file_data[filename] = json_data
-            json_data.clear()
-
         self.set_schema_2(self.metadata_file_data)
diff --git a/examples/coreterminal.py b/examples/coreterminal.py
@@ -5,12 +5,13 @@
 
 a=Terminal(debug_flag=False)
 
-a.load_module('plugin','Bueno','reader', filenames=['data/bueno1.data', 'data/bueno2.data'])
-a.load_module('plugin','Hostname','reader')
+# a.load_module('plugin','Bueno','reader', filenames=['data/bueno1.data', 'data/bueno2.data'])
+# a.load_module('plugin','Hostname','reader')
 
-a.load_module('plugin', 'Schema', 'reader', filename="data/example_schema.json", target_table_prefix = "student")
-a.load_module('plugin', 'YAML1', 'reader', filenames=["data/student_test1.yml", "data/student_test2.yml"], target_table_prefix = "student")
-a.load_module('plugin', 'TOML1', 'reader', filenames=["data/results.toml", "data/results1.toml"], target_table_prefix = "results")
+# a.load_module('plugin', 'Schema', 'reader', filename="data/example_schema.json", target_table_prefix = "student")
+# a.load_module('plugin', 'YAML1', 'reader', filenames=["data/student_test1.yml", "data/student_test2.yml"], target_table_prefix = "student")
+# a.load_module('plugin', 'TOML1', 'reader', filenames=["data/results.toml", "data/results1.toml"], target_table_prefix = "results")
+a.load_module('plugin', 'MetadataReader1', 'reader', filenames=["data/metadata.json"])
 
 # a.load_module('plugin', "Table_Plot", "writer", table_name = "student__physics", filename = "student__physics")
 # a.load_module('plugin', 'ER_Diagram', 'writer', filename = 'er_diagram.pdf')#, target_table_prefix = "physics")