Merge pull request #17 from lsst-sqre/tickets/DM-39043

DM-39043: Add Kafka producer for Slack messages and migrate to a client-server architecture

.github/workflows/ci.yaml

@@ -1,19 +1,21 @@
 name: CI
 
 "on":
+  merge_group: {}
+  pull_request: {}
   push:
     branches-ignore:
       # These should always correspond to pull requests, so ignore them for
       # the push trigger and let them be triggered by the pull_request
       # trigger, avoiding running the workflow twice.  This is a minor
       # optimization so there's no need to ensure this is comprehensive.
       - "dependabot/**"
+      - "gh-readonly-queue/**"
       - "renovate/**"
       - "tickets/**"
       - "u/**"
     tags:
       - "*"
-  pull_request: {}
 
 jobs:
   lint:
@@ -26,19 +28,38 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Run pre-commit
         uses: pre-commit/[email protected]
 
-  test:
+  test-client:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python:
+          - "3.11"
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run tox
+        uses: lsst-sqre/run-tox@v1
+        with:
+          python-version: ${{ matrix.python }}
+          tox-envs: "typing"
+          cache-key-prefix: "client"
+          working-directory: "./client"
+
+  test-server:
 
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
         python:
-          - "3.10"
+          - "3.11"
 
     steps:
       - uses: actions/checkout@v3
@@ -48,6 +69,8 @@ jobs:
         with:
           python-version: ${{ matrix.python }}
           tox-envs: "py,coverage-report,typing"
+          cache-key-prefix: "server"
+          working-directory: "./server"
 
   docs:
     runs-on: ubuntu-latest
@@ -68,9 +91,10 @@ jobs:
       - name: Run tox
         uses: lsst-sqre/run-tox@v1
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           tox-envs: "docs"
           cache-key-prefix: "docs"
+          working-directory: "./server"
 
       # Only attempt documentation uploads for long-lived branches, tagged
       # releases, and pull requests from ticket branches.  This avoids version
@@ -84,13 +108,14 @@ jobs:
           username: ${{ secrets.LTD_USERNAME }}
           password: ${{ secrets.LTD_PASSWORD }}
         if: >
-          github.event_name != 'pull_request'
-          || startsWith(github.head_ref, 'tickets/')
+          github.event_name != 'merge_group'
+          && (github.event_name != 'pull_request'
+            || startsWith(github.head_ref, 'tickets/'))
 
   build:
 
     runs-on: ubuntu-latest
-    needs: [lint, test]
+    needs: [lint, test-server]
 
     # Only do Docker builds of tagged releases and pull requests from ticket
     # branches.  This will still trigger on pull requests from untrusted
@@ -125,11 +150,29 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
           context: .
           push: true
           tags: |
             ghcr.io/lsst-sqre/squarebot:${{ steps.vars.outputs.tag }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+  pypi:
+
+    runs-on: ubuntu-latest
+    needs: [lint, test-client, docs]
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0 # full history for setuptools_scm
+
+      - name: Build and publish
+        uses: lsst-sqre/build-and-publish-to-pypi@v1
+        with:
+          pypi-token: ${{ secrets.PYPI_SQRE_ADMIN }}
+          python-version: "3.11"
+          upload: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
+          working-directory: "./client"

.pre-commit-config.yaml

@@ -7,13 +7,13 @@ repos:
       - id: check-toml
 
   - repo: https://github.com/PyCQA/isort
-    rev: 5.11.4
+    rev: 5.12.0
     hooks:
       - id: isort
         additional_dependencies: [toml]
 
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 23.3.0
     hooks:
       - id: black
 

.prettierignore

@@ -0,0 +1 @@
+changelog.d/_template.md

CHANGELOG.md

@@ -1,10 +1,22 @@
 # Change log
 
-## Unreleased (2023-01-27)
+<!-- scriv-insert-here -->
 
-SQuaRE Bot (`@squarebot` on Slack) is the next iteration of the SQuaRE Bot Jr. and is intended to finally become the successor to the original Hubot-based `#sqrbot`.
-SQuaRE Bot is a migration of the original SQuaRE Bot Jr. code base from aiohttp into FastAPI.
-With this migration, SQuaRE Bot now takes advangate of Pydantic modelling for end-to-end type checking from the payloads received by the HTTP handlers to the messages published to Kafka.
+<a id='changelog-0.7.0'></a>
+
+## 0.7.0 (2023-05-19)
+
+### Backwards-incompatible changes
+
+- sqrbot-jr is now Squarebot. It's docker image is now published at `ghcr.io/lsst-sqre/squarebot`. The app is also deployed through a Helm chart in [lsst-sqre/phalanx](https://github.com/lsst-sqre/phalanx), rather than as a Kustomize manifest. See _New features_ for more details.
+- Avro schemas for messages
+- Slack interaction events are currently unsupported; only Slack messages are published to Kafka.
+
+### New features
+
+- Squarebot is rebuilt for the modern SQuaRE app architecture: FastAPI, deployed with Helm through Phalanx.
+- Squarebot uses Pydantic for modelling Avro-encoded Kafka messages. This allows for end-to-end type checking from the HTTP handlers to the messages published to Kafka.
+- The `lsst-sqre/squarebot` repository is now a monorepo that contains the Squarebot service and a client library (`rubin-squarebot` on PyPI). The client library contains Pydantic models for the Avro-encoded messages published to Kafka by Squarebot. See [SQR-075](https://sqr-075.lsst.io) for details on the monorepo architecture and [SQR-076](https://sqr-076.lsst.io) background on how Pydantic is used for Kafka message modelling.
 
 ## 0.6.0 (2019-12-02)
 

Dockerfile

@@ -14,7 +14,7 @@
 #   - Runs a non-root user.
 #   - Sets up the entrypoint and port.
 
-FROM python:3.10.7-slim-bullseye as base-image
+FROM python:3.11.3-slim-bullseye as base-image
 
 # Update system packages
 COPY scripts/install-base-packages.sh .
@@ -35,7 +35,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 RUN pip install --upgrade --no-cache-dir pip setuptools wheel
 
 # Install the app's Python runtime dependencies
-COPY requirements/main.txt ./requirements.txt
+COPY server/requirements/main.txt ./requirements.txt
 RUN pip install --quiet --no-cache-dir -r requirements.txt
 
 FROM dependencies-image AS install-image
@@ -45,7 +45,8 @@ ENV PATH="/opt/venv/bin:$PATH"
 
 COPY . /workdir
 WORKDIR /workdir
-RUN pip install --no-cache-dir .
+RUN pip install --no-cache-dir ./client
+RUN pip install --no-cache-dir ./server
 
 FROM base-image AS runtime-image
 

Makefile

@@ -1,27 +1,24 @@
 .PHONY: update-deps
 update-deps:
 	pip install --upgrade pip-tools pip setuptools
-	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/main.txt requirements/main.in
-	pip-compile --upgrade --build-isolation --generate-hashes --output-file requirements/dev.txt requirements/dev.in
-
-# Useful for testing against a Git version of Safir.
-.PHONY: update-deps-no-hashes
-update-deps-no-hashes:
-	pip install --upgrade pip-tools pip setuptools
-	pip-compile --upgrade --build-isolation --allow-unsafe --output-file requirements/main.txt requirements/main.in
-	pip-compile --upgrade --build-isolation --allow-unsafe --output-file requirements/dev.txt requirements/dev.in
+	# pip-compile --upgrade --build-isolation --generate-hashes --output-file server/requirements/main.hashed.txt server/requirements/main.in
+	# pip-compile --upgrade --build-isolation --generate-hashes --output-file server/requirements/dev.hashed.txt server/requirements/dev.in
+	pip-compile --upgrade --build-isolation --allow-unsafe --output-file server/requirements/main.txt server/requirements/main.in
+	pip-compile --upgrade --build-isolation --allow-unsafe --output-file server/requirements/dev.txt server/requirements/dev.in
 
 .PHONY: init
 init:
-	pip install --editable .
-	pip install --upgrade -r requirements/main.txt -r requirements/dev.txt
-	rm -rf .tox
-	pip install --upgrade pre-commit tox
+	pip install --editable "./client[dev]"
+	pip install --editable ./server
+	pip install --upgrade -r server/requirements/main.txt -r server/requirements/dev.txt
+	rm -rf ./server/.tox
+	rm -rf ./client/.tox
+	pip install --upgrade pre-commit tox scriv
 	pre-commit install
 
 .PHONY: update
 update: update-deps init
 
 .PHONY: run
 run:
-	tox -e=run
+	cd server && tox run -e=run

README.md

@@ -0,0 +1,46 @@
+# Squarebot
+
+Squarebot is the nexus service for ChatOps and event-driven bots for Rubin Observatory.
+As a Slack app, Squarebot receives events from Slack and publishes them into specific _Square Events_ Kafka topics in the [Roundtable Kubernetes cluster](https://phalanx.lsst.io/environments/roundtable-prod/index.html).
+Other applications on Roundtable can consume these events and act on them, such as by posting messages back to Slack or by performing some other background automation.
+
+```mermaid
+flowchart LR
+    subgraph Slack
+    message[Slack message]
+    interaction[Slack interaction]
+    end
+    subgraph sqrbot ["Squarebot"]
+    eventapi["/slack/event"]
+    interactionapi["/slack/interaction"]
+    message -->|HTTP POST| eventapi
+    interaction -->|HTTP POST| interactionapi
+    end
+    subgraph kafka ["Kafka Topics"]
+    topicmention["lsst.square-events.squarebot.slack.app.mention"]
+    topicchannel["lsst.square-events.squarebot.slack.message.channel"]
+    topicgroup["lsst.square-events.squarebot.slack.message.group"]
+    topicim["lsst.square-events.squarebot.slack.message.im"]
+    topicmpim["lsst.square-events.squarebot.slack.message.mpim"]
+    topicinteraction["lsst.square-events.squarebot.slack.interaction"]
+    end
+    eventapi --> topicmention
+    eventapi --> topicchannel
+    eventapi --> topicgroup
+    eventapi --> topicim
+    eventapi --> topicmpim
+    interactionapi --> topicinteraction
+    subgraph backends ["Backends"]
+    backend1["Backend 1"]
+    backend2["Backend 2"]
+    end
+    topicchannel --> backend1
+    topicgroup --> backend1
+    topicmention --> backend2
+    topicinteraction --> backend2
+```
+
+Slack integration is implemented at the time.
+We plan to add support for other event sources, such as GitHub, in the future.
+
+Squarebot is built on top of [FastAPI](https://fastapi.tiangolo.com/), a modern Python web framework, with Rubin/SQuaRE libraries [Safir](https://safir.lsst.io) (FastAPI application libraries) and [Kafkit](https://kafkit.lsst.io) (Kafka integration).

changelog.d/_template.md

@@ -0,0 +1,7 @@
+<!-- Delete the sections that don't apply -->
+{%- for cat in config.categories %}
+
+### {{ cat }}
+
+-
+{%- endfor %}

client/LICENSE

@@ -0,0 +1 @@
+../LICENSE

client/README.md

@@ -0,0 +1 @@
+../README.md