Merge pull request #117 from lanl/testing_CI

Created CI files for backend and plugin files on the open

.github/workflows/test_file_reader.yml

@@ -0,0 +1,36 @@
+name: file_reader test
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+
+
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pyyaml
+        pip install toml
+        pip install .  
+    - name: Test reader
+      run: |
+        pip install pytest
+        pytest dsi/plugins/tests/test_file_reader.py

.github/workflows/test_file_writer.yml

@@ -0,0 +1,35 @@
+name: file_writer test
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pyyaml
+        pip install toml
+        pip install .  
+    - name: Test reader
+      run: |
+        pip install pytest
+        pytest dsi/plugins/tests/test_file_writer.py

.github/workflows/test_plugin.yml

@@ -0,0 +1,38 @@
+name: test_plugin.py test
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        python -m pip install opencv-python
+        pip install pyyaml
+        pip install toml
+        pip install .
+        pip install graphviz
+        sudo apt-get install graphviz
+    - name: Test reader
+      run: |
+        pip install pytest
+        pytest dsi/tests/test_plugin.py

.github/workflows/test_sqlite.yml

@@ -0,0 +1,35 @@
+name: sqlite.py test
+
+on:
+  push:
+    branches:
+    - main
+  pull_request:
+    branches:
+    - main
+
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.10']
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install pyyaml
+        pip install toml
+        pip install .
+    - name: Test reader
+      run: |
+        pip install pytest
+        pytest dsi/backends/tests/test_sqlite.py

dsi/backends/sqlite.py

@@ -576,8 +576,8 @@ def yamlToSqlite(self, filename, db_name, deleteSql=True):
                     for key, val in table['columns'].items():
                         createUnitStmt+= f"{key} VARCHAR, "
                         if data_types[type(val)] == "VARCHAR" and self.check_type(val[:val.find(" ")]) in [" INT", " FLOAT"]:
-                            createStmt += f"{key}{self.check_type(val[:val.find(" ")])}, "
-                            insertUnitStmt+= f"'{val[val.find(" ")+1:]}', "
+                            createStmt += f"{key}{self.check_type(val[:val.find(' ')])}, "
+                            insertUnitStmt+= f"'{val[val.find(' ')+1:]}', "
                         else:
                             createStmt += f"{key} {data_types[type(val)]}, "
                             insertUnitStmt+= "NULL, "
@@ -589,7 +589,7 @@ def yamlToSqlite(self, filename, db_name, deleteSql=True):
                 insertStmt = f"INSERT INTO {tableName} VALUES( "
                 for val in table['columns'].values():
                     if data_types[type(val)] == "VARCHAR" and self.check_type(val[:val.find(" ")]) in [" INT", " FLOAT"]:
-                        insertStmt+= f"{val[:val.find(" ")]}, "
+                        insertStmt+= f"{val[:val.find(' ')]}, "
                     elif data_types[type(val)] == "VARCHAR":
                         insertStmt+= f"'{val}', "
                     else:

dsi/backends/tests/test_sqlite.py

@@ -23,7 +23,7 @@ def test_wildfire_data_sql_artifact():
     assert True
 
 def test_wildfire_data_csv_artifact():
-    csvpath = '/'.join([get_git_root('.'), 'dsi/data/wildfiredata.csv'])
+    csvpath = '/'.join([get_git_root('.'), 'examples/data/wildfiredata.csv'])
     dbpath = "wildfire.db"
     store = Sqlite(dbpath)
     store.put_artifacts_csv(csvpath, "simulation", isVerbose=isVerbose)
@@ -60,7 +60,7 @@ def test_jsondata_artifact_put():
    assert True
 
 def test_yosemite_data_csv_artifact():
-    csvpath = '/'.join([get_git_root('.'), 'dsi/data/yosemite5.csv'])
+    csvpath = '/'.join([get_git_root('.'), 'examples/data/yosemite5.csv'])
     dbpath = "yosemite.db"
     store = Sqlite(dbpath)
     store.put_artifacts_csv(csvpath, "vision", isVerbose=isVerbose)
@@ -87,8 +87,8 @@ def test_artifact_query():
 
 def test_yaml_reader():
     reader = Sqlite("yaml-test.db")
-    reader.yamlToSqlite("../../../examples/data/schema.yml", "yaml-test", deleteSql=False)
-    subprocess.run(["diff", "../../../examples/data/compare-schema.sql", "yaml-test.sql"], stdout=open("compare_sql.txt", "w"))
+    reader.yamlToSqlite("examples/data/schema.yml", "yaml-test", deleteSql=False)
+    subprocess.run(["diff", "examples/data/compare-schema.sql", "yaml-test.sql"], stdout=open("compare_sql.txt", "w"))
     file_size = os.path.getsize("compare_sql.txt")
     os.remove("compare_sql.txt")
     os.remove("yaml-test.sql")
@@ -98,8 +98,8 @@ def test_yaml_reader():
 
 def test_toml_reader():
     reader = Sqlite("toml-test.db")
-    reader.tomlToSqlite("../../../examples/data/schema.toml", "toml-test", deleteSql=False)
-    subprocess.run(["diff", "../../../examples/data/compare-schema.sql", "toml-test.sql"], stdout=open("compare_sql.txt", "w"))
+    reader.tomlToSqlite("examples/data/schema.toml", "toml-test", deleteSql=False)
+    subprocess.run(["diff", "examples/data/compare-schema.sql", "toml-test.sql"], stdout=open("compare_sql.txt", "w"))
     file_size = os.path.getsize("compare_sql.txt")
     os.remove("compare_sql.txt")
     os.remove("toml-test.sql")

dsi/plugins/file_writer.py

@@ -36,7 +36,7 @@ def __init__(self, filenames, **kwargs):
 
     def export_erd(self, dbname, fname):
         """
-        Function that outputs a dot file for the given database.
+        Function that outputs a ER diagram for the given database.
 
         `dbname`: database to create an ER diagram for
 

dsi/plugins/tests/test_file_writer.py

@@ -12,6 +12,6 @@ def get_git_root(path):
 
 def test_csv_plugin_type():
     path = '/'.join([get_git_root('.'), 'examples/data', 'wildfiredata.sqlite_db'])
-    back = Sqlite(filenames=path)
+    back = Sqlite(filename=path)
 
     #assert type(plug.output_collector) == OrderedDict

dsi/tests/test_plugin.py

@@ -1,23 +1,17 @@
 from dsi.plugins import file_writer as fw
 import cv2
-import sqlite3
 import numpy as np
+import subprocess
 import os
 
 def test_export_db_erd():
-
-    connection = sqlite3.connect("test.db")
-    cursor = connection.cursor()
-    cursor.execute("CREATE TABLE IF NOT EXISTS example (id INTEGER, name TEXT, age INTEGER)")
-    cursor.execute("INSERT INTO example VALUES (1, 'alice', 20)")
-    cursor.execute("INSERT INTO example VALUES (2, 'bob', 30)")
-    cursor.execute("INSERT INTO example VALUES (3, 'eve', 40)")
-    connection.commit()
-    connection.close()
-
-    er_image = cv2.imread("test1.png") 
-    pixel_mean = np.mean(er_image)
-
+    subprocess.run(["sqlite3", "erd_test.db"], stdin= open("examples/data/erd_test.sql", "r"))
+    fw.ER_Diagram("erd_test.db").export_erd("erd_test.db", "erd_test_output")
+    os.remove("erd_test.db")
+
+    er_image = cv2.imread("erd_test_output.png")
     assert er_image is not None #check if image generated at all
-    os.remove("test1.png")
+
+    pixel_mean = np.mean(er_image)
+    os.remove("erd_test_output.png")
     assert pixel_mean != 255 #check if image is all white pixels (no diagram generated)

examples/data/erd_test.sql

@@ -0,0 +1,39 @@
+-- Create the `publishers` table
+CREATE TABLE publishers (
+    publisher_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    address TEXT
+);
+
+-- Create the `authors` table
+CREATE TABLE authors (
+    author_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    name TEXT NOT NULL,
+    birth_date DATE
+);
+
+-- Create the `books` table
+CREATE TABLE books (
+    book_id INTEGER PRIMARY KEY AUTOINCREMENT,
+    title TEXT NOT NULL,
+    publish_date DATE,
+    publisher_id INTEGER,
+    author_id INTEGER,
+    FOREIGN KEY (publisher_id) REFERENCES publishers(publisher_id),
+    FOREIGN KEY (author_id) REFERENCES authors(author_id)
+);
+
+-- Insert some sample data into `publishers`
+INSERT INTO publishers (name, address) VALUES
+('Penguin Random House', 'New York, NY'),
+('HarperCollins', 'New York, NY');
+
+-- Insert some sample data into `authors`
+INSERT INTO authors (name, birth_date) VALUES
+('J.K. Rowling', '1965-07-31'),
+('George R.R. Martin', '1948-09-20');
+
+-- Insert some sample data into `books`
+INSERT INTO books (title, publish_date, publisher_id, author_id) VALUES
+('Harry Potter and the Philosophers Stone', '1997-06-26', 1, 1),
+('A Game of Thrones', '1996-08-06', 2, 2);