Merge pull request #119 from lanl/avoid_yaml_toml_dups

avoid duplicate sql statements

dsi/backends/sqlite.py

@@ -644,6 +644,8 @@ def tomlToSqlite(self, filenames, db_name, deleteSql=True):
 
         `deleteSql`: flag to delete temp SQL file that creates the database. Default is True, but change to False for testing or comparing outputs
         """
+
+        sql_statements = []
         if isinstance(filenames, str):
             filenames = [filenames]
 
@@ -667,10 +669,16 @@ def tomlToSqlite(self, filenames, db_name, deleteSql=True):
                         else:
                             createStmt += f"{key} {data_types[type(val)]}, "
                             insertUnitStmt+= "NULL, "
-
-                    sql_file.write(createStmt[:-2] + ");\n\n")
-                    sql_file.write(createUnitStmt[:-2] + ");\n\n")
-                    sql_file.write(insertUnitStmt[:-2] + ");\n\n")
+
+                    if createStmt not in sql_statements:
+                        sql_statements.append(createStmt)
+                        sql_file.write(createStmt[:-2] + ");\n\n")
+                    if createUnitStmt not in sql_statements:
+                        sql_statements.append(createUnitStmt)
+                        sql_file.write(createUnitStmt[:-2] + ");\n\n")
+                    if insertUnitStmt not in sql_statements:
+                        sql_statements.append(insertUnitStmt)
+                        sql_file.write(insertUnitStmt[:-2] + ");\n\n")
 
                 insertStmt = f"INSERT INTO {tableName} VALUES( "
                 for val in tableData.values():
@@ -681,7 +689,9 @@ def tomlToSqlite(self, filenames, db_name, deleteSql=True):
                     else:
                         insertStmt+= f"{val}, "
 
-                sql_file.write(insertStmt[:-2] + ");\n\n")
+                if insertStmt not in sql_statements:
+                    sql_statements.append(insertStmt)
+                    sql_file.write(insertStmt[:-2] + ");\n\n")
 
         subprocess.run(["sqlite3", db_name+".db"], stdin= open(db_name+".sql", "r"))