Organize

.github/workflows/python-app.yml

@@ -0,0 +1,33 @@
+name: mlex_latent_explorer
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.9
+        cache: 'pip'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install .
+        pip install .[dev]
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest

.gitignore

@@ -3,6 +3,7 @@ __pycache__/
 *.py[cod]
 *$py.class
 test.py
+**cache**
 
 # output dir
 results/

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.9
+LABEL maintainer="THE MLEXCHANGE TEAM"
+
+RUN ls
+
+RUN pip3 install --upgrade pip &&\
+    pip3 install -r .
+
+WORKDIR /app/work
+ENV HOME /app/work
+COPY src src
+
+CMD ["bash"]
+CMD python3 src/frontend.py

README.md

@@ -11,6 +11,35 @@ To run this demo:
 - clone this repo and then run `docker-compose up` in the main folder.  
 - Go to `http://0.0.0.0:8070/` in a brower.
 
+## Developer Setup
+If you are developing this library, there are a few things to note.
+
+1. Install development dependencies:
+
+```
+pip install .
+pip install ".[dev]"
+```
+
+2. Install pre-commit
+This step will setup the pre-commit package. After this, commits will get run against flake8, black, isort.
+
+```
+pre-commit install
+```
+
+3. (Optional) If you want to check what pre-commit would do before commiting, you can run:
+
+```
+pre-commit run --all-files
+```
+
+4. To run test cases:
+
+```
+python -m pytest
+```
+
 
 ## Copyright
 MLExchange Copyright (c) 2023, The Regents of the University of California,

pyproject.toml

@@ -0,0 +1,52 @@
+[project]
+name = "mlex_latent_explorer"
+version = "0.0.1"
+authors = [
+    { name="Runbo Jiang", email="[email protected]"}
+]
+description = "A web app to visualize latent vectors in 2D or 3D, supporting PCA and UMAP for dimension reduction."
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+]
+
+dependencies = [
+    "dash==2.9.3",
+    "dash-core-components==2.0.0",
+    "dash-bootstrap-components==1.0.2",
+    "dash-html-components==2.0.0",
+    "dash-iconify==0.1.2",
+    "plotly==5.14.1",
+    "scikit-learn==1.3.0",
+    "dash-uploader==0.6.0",
+    "requests==2.26.0",
+    "pyarrow==11.0.0",
+    "diskcache==5.6.3",
+    "dash_component_editor@git+https://github.com/mlexchange/mlex_dash_component_editor",
+    "mlex_file_manager@git+https://github.com/mlexchange/mlex_file_manager",
+    "Pillow",
+    "pandas",
+    "numpy",
+    "python-dotenv",
+    "prefect-client==2.14.21"
+]
+
+[project.optional-dependencies]
+# These are required for developing the package (running the tests, building
+# the documentation) but not necessarily required for _using_ it.
+dev = [
+    "dash[testing]",
+    "flake8",
+    "pre-commit",
+    "pytest-mock",
+    "tiled[all]",
+]
+
+[project.urls]
+Homepage = "https://github.com/mlexchange/mlex_latent_explorer"
+Issues = "https://github.com/mlexchange/mlex_latent_explorer/issues/"
+
+[tool.isort]
+profile = "black"

src/frontend.py

@@ -46,14 +46,14 @@
 FLOW_TYPE = os.getenv("FLOW_TYPE", "podman")
 
 # Slurm
-PARTITIONS_CPU = json.loads(os.getenv("PARTITIONS_CPU", []))
-RESERVATIONS_CPU = json.loads(os.getenv("RESERVATIONS_CPU", []))
+PARTITIONS_CPU = json.loads(os.getenv("PARTITIONS_CPU", "[]"))
+RESERVATIONS_CPU = json.loads(os.getenv("RESERVATIONS_CPU", "[]"))
 MAX_TIME_CPU = os.getenv("MAX_TIME_CPU", "1:00:00")
-PARTITIONS_GPU = json.loads(os.getenv("PARTITIONS_CPU", []))
-RESERVATIONS_GPU = json.loads(os.getenv("RESERVATIONS_CPU", []))
+PARTITIONS_GPU = json.loads(os.getenv("PARTITIONS_CPU", "[]"))
+RESERVATIONS_GPU = json.loads(os.getenv("RESERVATIONS_CPU", "[]"))
 MAX_TIME_GPU = os.getenv("MAX_TIME_CPU", "1:00:00")
 SUBMISSION_SSH_KEY = os.getenv("SUBMISSION_SSH_KEY", "")
-FORWARD_PORTS = json.loads(os.getenv("FORWARD_PORTS", []))
+FORWARD_PORTS = json.loads(os.getenv("FORWARD_PORTS", "[]"))
 
 # Mlex content api
 CONTENT_API_URL = os.getenv("CONTENT_API_URL", "http://localhost:8000/api/v0/models")

src/test/layout_test.py

@@ -0,0 +1,10 @@
+from src.frontend import show_clustering_gui_layouts
+
+
+def test_clustering_layout():
+    layout = show_clustering_gui_layouts("KMeans")
+    assert layout is not None
+    layout = show_clustering_gui_layouts("DBSCAN")
+    assert layout is not None
+    layout = show_clustering_gui_layouts("HDBSCAN")
+    assert layout is not None