Merge pull request #22 from swelborn/helm

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "backend/agent/thirdparty/podman-hpc-py"]
 	path = backend/agent/thirdparty/podman-hpc-py
 	url = https://github.com/cjh1/podman-hpc-py.git
+[submodule "helm/spin-helm"]
+	path = helm/spin-helm
+	url = https://github.com/NERSC/spin-helm.git

backend/agent/agent/config.py

@@ -1,6 +1,6 @@
 from typing import Any
 
-from pydantic import Field, NatsDsn
+from pydantic import Field, NatsDsn, WebsocketUrl
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
@@ -9,7 +9,9 @@ class Settings(BaseSettings):
     LOCAL: bool = True
     DOCKER_COMPATIBILITY_MODE: bool = False
     MOUNTS: list[dict[str, Any]] = []
-    NATS_SERVER_URL: NatsDsn = Field(default="nats://localhost:4222")
-    NATS_SERVER_URL_IN_CONTAINER: NatsDsn = Field(default="nats://nats1:4222")
+    NATS_SERVER_URL: NatsDsn | WebsocketUrl = Field(default="nats://localhost:4222")
+    NATS_SERVER_URL_IN_CONTAINER: NatsDsn | WebsocketUrl = Field(
+        default="nats://nats1:4222"
+    )
 
 cfg = Settings()  # type: ignore

backend/app/app/core/config.py

@@ -9,6 +9,7 @@
     HttpUrl,
     NatsDsn,
     PostgresDsn,
+    WebsocketUrl,
     computed_field,
     model_validator,
 )
@@ -119,7 +120,7 @@ def _enforce_non_default_secrets(self) -> Self:
         return self
 
     # NATS
-    NATS_SERVER_URL: NatsDsn = Field(default="nats://nats1:4222")
+    NATS_SERVER_URL: NatsDsn | WebsocketUrl = Field(default="nats://nats1:4222")
 
 
 settings = Settings()  # type: ignore

backend/app/pyproject.toml

@@ -28,6 +28,7 @@ sentry-sdk = {extras = ["fastapi"], version = "^1.40.6"}
 pyjwt = "^2.8.0"
 nats-py = "^2.8.0"
 core = {path = "../core", develop = true}
+aiohttp = "^3.10.5"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.4.3"

backend/operators/operators/config.py

@@ -1,10 +1,9 @@
-
-from pydantic import Field, NatsDsn
+from pydantic import Field, NatsDsn, WebsocketUrl
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file=None)
-    NATS_SERVER_URL: NatsDsn = Field(default="nats://localhost:4222")
+    NATS_SERVER_URL: NatsDsn | WebsocketUrl = Field(default="nats://localhost:4222")
 
 cfg = Settings()

backend/operators/pyproject.toml

@@ -14,6 +14,7 @@ pyzmq = "^26.1.0"
 networkx = "^3.3"
 numpy = "^1.26"
 pydantic-settings = "^2.4.0"
+aiohttp = "^3.10.5"
 
 
 [build-system]

backend/orchestrator/orchestrator/config.py

@@ -1,10 +1,10 @@
-from pydantic import Field, NatsDsn
+from pydantic import Field, NatsDsn, WebsocketUrl
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file=None)
-    NATS_SERVER_URL: NatsDsn = Field(default="nats://localhost:4222")
+    NATS_SERVER_URL: NatsDsn | WebsocketUrl = Field(default="nats://localhost:4222")
 
 
 cfg = Settings()

backend/orchestrator/pyproject.toml

@@ -7,11 +7,12 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = "^3.10"
-nats-py = { git = "https://github.com/nats-io/nats.py.git", branch = "services" }
 pydantic = "^2.8.2"
 requests = "^2.32.3"
 core = {path = "../core", develop = true}
 pydantic-settings = "^2.4.0"
+aiohttp = "^3.10.5"
+nats-py = "^2.8.0"
 
 
 [build-system]

helm/nats-values.yaml

@@ -0,0 +1,60 @@
+config:
+  cluster:
+    enabled: true
+
+  jetstream:
+    enabled: true
+
+  # Uncomment for external access through a websocket (outside NERSC)
+  websocket:
+    enabled: true
+    port: 8080
+    ingress:
+      enabled: true
+      hosts:
+        # - <your domain cname>
+        - ingress.interactem.production.svc.spin.nersc.org
+      path: /
+      pathType: Exact
+      className: nginx
+      tlsSecretName: tls-cert
+      name: ingress
+      merge:
+        metadata:
+          annotations:
+            nginx.ingress.kubernetes.io/whitelist-source-range: >-
+              128.55.206.0/24, 128.55.205.0/24, 128.55.64.0/18
+            # this is to allow the ACME challenge to pass through the ingress given
+            # the above whitelist. TODO: Not sure if it works.
+            nginx.ingress.kubernetes.io/server-snippet: |
+                location /.well-known/acme-challenge/ {
+                    allow all;
+                }
+
+natsBox:
+  container:
+    image:
+      tag: 0.14.4-nonroot
+
+
+extraResources:
+  - apiVersion: v1
+    kind: Service
+    metadata:
+      name: nats-external
+      namespace: interactem
+      labels:
+        app.kubernetes.io/component: nats
+        app.kubernetes.io/instance: nats
+        app.kubernetes.io/managed-by: Helm
+        app.kubernetes.io/name: nats
+    spec:
+      type: LoadBalancer
+      selector:
+        app.kubernetes.io/component: nats
+        app.kubernetes.io/instance: nats
+        app.kubernetes.io/name: nats
+      ports:
+        - name: nats
+          port: 5672 # exposed port
+          targetPort: 4222 # internal port

helm/scripts/create_kubeconfig.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+kubectl -n interactem create secret generic kubeconfig --from-file=kubeconfig=${HOME}/.kube/config

helm/scripts/deploy.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+export SPIN_GID=99894
+export SPIN_UID=99894
+
+cd ../spin-helm/nats/kustomize
+
+helm upgrade --install -n interactem nats nats/nats --version 1.2.2 -f ../../../nats-values.yaml --post-renderer ./kustomize.sh
+
+cd ../../..
+helm upgrade --install -n interactem -f ./tls-acme-values.yaml acmecron ./spin-helm/tls-acme

helm/tls-acme-values.yaml

@@ -0,0 +1,21 @@
+replicaCount: 1
+
+podSecurityContext:
+  fsGroup: 99894 # NERSC GID for the project
+
+securityContext:
+  runAsUser: 99894 # NERSC UID
+
+service:
+  port: 8080
+
+ingress:
+  name: ingress
+  enabled: false
+
+# concatenate the list .ingress.hosts.host into ":" separated string
+domain: ingress.interactem.production.svc.spin.nersc.org
+
+email: [email protected] # email address to be used for the ACME account
+
+cluster: production.svc.spin.nersc.org