Merge pull request #3 from openradx/procrastinate

Integrate Procrastinate in example project

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.12"]
-        poetry-version: ["1.7.1"]
+        poetry-version: ["1.8.3"]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 15
@@ -29,12 +29,26 @@ jobs:
         run: poetry install --with dev
       - name: Configure environment
         run: poetry run invoke init-workspace
-      - name: Setup Playwright
-        run: poetry run playwright install --with-deps
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Build and cache Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          target: development
+          load: true
+          tags: example_project_dev:latest
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Start Docker containers
+        run: poetry run invoke compose-up --no-build
       - name: Run linting
         # https://github.com/actions/runner/issues/241#issuecomment-745902718
         shell: 'script -q -e -c "bash {0}"'
         run: poetry run invoke lint
       - name: Run tests
         shell: 'script -q -e -c "bash {0}"'
         run: poetry run invoke test --cov
+      - name: Stop Docker containers
+        if: ${{ always() }}
+        run: poetry run invoke compose-down

.vscode/settings.json

@@ -27,7 +27,7 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[dockercompose]": {
-    "editor.defaultFormatter": "ms-azuretools.vscode-docker"
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[dockerfile]": {
     "editor.defaultFormatter": "ms-azuretools.vscode-docker"

Dockerfile

@@ -0,0 +1,95 @@
+FROM python:3.12-bullseye as python-base
+
+# python
+# ENV variables are also available in the later build stages
+ENV PYTHONUNBUFFERED=1 \
+    # prevents python creating .pyc files
+    PYTHONDONTWRITEBYTECODE=1 \
+    \
+    # pip
+    PIP_NO_CACHE_DIR=off \
+    PIP_DISABLE_PIP_VERSION_CHECK=on \
+    PIP_DEFAULT_TIMEOUT=100 \
+    \
+    # poetry
+    # https://python-poetry.org/docs/#installing-with-the-official-installer
+    # https://python-poetry.org/docs/configuration/#using-environment-variables
+    POETRY_VERSION=1.8.3 \
+    # make poetry install to this location
+    POETRY_HOME="/opt/poetry" \
+    # make poetry create the virtual environment in the project's root
+    # it gets named `.venv`
+    POETRY_VIRTUALENVS_IN_PROJECT=true \
+    # do not ask any interactive question
+    POETRY_NO_INTERACTION=1 \
+    \
+    # paths
+    # this is where our requirements + virtual environment will live
+    PYSETUP_PATH="/opt/pysetup" \
+    VENV_PATH="/opt/pysetup/.venv" \
+    # needed for adit-radis-shared to be found
+    PYTHONPATH="/app"
+
+# prepend poetry and venv to path
+ENV PATH="$POETRY_HOME/bin:$VENV_PATH/bin:$PATH"
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+    # deps for db management commands
+    postgresql-client
+
+
+# `builder-base` stage is used to build deps + create our virtual environment
+FROM python-base as builder-base
+RUN apt-get update \
+    && apt-get install --no-install-recommends -y \
+    # deps for installing poetry
+    curl \
+    # deps for building python deps
+    build-essential
+
+# install poetry - respects $POETRY_VERSION & $POETRY_HOME
+RUN curl -sSL https://install.python-poetry.org | python3 -
+
+# copy project requirement files here to ensure they will be cached.
+WORKDIR $PYSETUP_PATH
+COPY poetry.lock pyproject.toml ./
+
+# install runtime deps - uses $POETRY_VIRTUALENVS_IN_PROJECT internally
+RUN poetry install --without dev
+
+
+# `development` image is used during development / testing
+FROM python-base as development
+WORKDIR $PYSETUP_PATH
+
+# copy in our built poetry + venv
+COPY --from=builder-base $POETRY_HOME $POETRY_HOME
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+
+# quicker install as runtime deps are already installed
+RUN poetry install
+
+# Install requirements for end-to-end testing
+RUN playwright install --with-deps chromium
+
+# Required folders for web service
+RUN mkdir -p /var/www/web/logs \
+    /var/www/web/static \
+    /var/www/web/ssl
+
+# will become mountpoint of our code
+WORKDIR /app
+
+
+# `production` image used for runtime
+FROM python-base as production
+COPY --from=builder-base $PYSETUP_PATH $PYSETUP_PATH
+COPY . /app/
+
+# Required folders for web service
+RUN mkdir -p /var/www/web/logs \
+    /var/www/web/static \
+    /var/www/web/ssl
+
+WORKDIR /app

TODO.md

@@ -20,3 +20,4 @@
 - Use django-environ as in the docs (see shared)
 - Make package.json more minimal (as in shared)
 - Get rid of static vendor stuff (its in common app now)
+- No need to do collectstatics in development container

adit_radis_shared/common/management/base/celery_worker.py → adit_radis_shared/common/management/base/procrastinate_worker.py

@@ -1,18 +1,15 @@
 import logging
 import shlex
-import socket
 import subprocess
-from typing import Literal
 
 from .server_command import ServerCommand
 
 logger = logging.getLogger(__name__)
 
 
-class CeleryWorkerCommand(ServerCommand):
-    project: Literal["adit", "radis"]
-    help = "Starts a Celery worker"
-    server_name = "Celery worker"
+class ProcrastinateServerCommand(ServerCommand):
+    help = "Starts a Procrastinate worker"
+    server_name = "Procrastinate worker"
     worker_process: subprocess.Popen | None
 
     def __init__(self, *args, **kwargs):
@@ -22,37 +19,38 @@ def __init__(self, *args, **kwargs):
     def add_arguments(self, parser):
         super().add_arguments(parser)
 
-        # https://docs.celeryproject.org/en/stable/reference/cli.html
         parser.add_argument(
-            "-Q",
-            "--queue",
-            required=True,
-            help="The celery queue.",
+            "-q",
+            "--queues",
+            help="Comma-separated names of the queues to listen to (empty string for all queues)",
         )
         parser.add_argument(
             "-l",
             "--loglevel",
-            default="INFO",
+            default="warning",
+            choices=["warning", "info", "debug"],
             help="Logging level.",
         )
         parser.add_argument(
             "-c",
             "--concurrency",
             type=int,
-            default=0,
+            default=1,
             help="Number of child processes processing the queue (defaults to number of CPUs).",
         )
 
     def run_server(self, **options):
-        queue = options["queue"]
-        loglevel = options["loglevel"]
-        hostname = f"worker_{queue}_{socket.gethostname()}"
+        cmd = "./manage.py procrastinate"
 
-        cmd = f"celery -A {self.project} worker -Q {queue} -l {loglevel} -n {hostname}"
+        # https://procrastinate.readthedocs.io/en/stable/howto/basics/command_line.html
+        if options["loglevel"] == "debug":
+            cmd += " -v 1"
+
+        cmd += " worker --delete-jobs=always"
 
         concurrency = options["concurrency"]
-        if concurrency >= 1:
-            cmd += f" -c {concurrency}"
+        if concurrency > 1:
+            cmd += f" --concurrency {concurrency}"
 
         self.worker_process = subprocess.Popen(shlex.split(cmd))
         self.worker_process.wait()

adit_radis_shared/common/views.py

@@ -1,6 +1,5 @@
 from typing import Any, Callable
 
-from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import (
     LoginRequiredMixin,
@@ -109,15 +108,3 @@ def test_func(self):
     @classmethod
     def as_url(cls):
         return re_path(rf"^{cls.url_prefix}/(?P<path>.*)$", cls.as_view())  # type: ignore
-
-
-class FlowerProxyView(AdminProxyView):
-    upstream = f"http://{settings.FLOWER_HOST}:{settings.FLOWER_PORT}"  # type: ignore
-    url_prefix = "flower"
-    rewrite = ((rf"^/{url_prefix}$", rf"/{url_prefix}/"),)
-
-    @classmethod
-    def as_url(cls):
-        # Flower needs a bit different setup then the other proxy views as flower
-        # uses a prefix itself (see docker compose service)
-        return re_path(rf"^(?P<path>{cls.url_prefix}.*)$", cls.as_view())

adit_radis_shared/conftest.py

@@ -1,7 +1,9 @@
 import time
-from typing import Callable
+from typing import Callable, Generator
 
 import pytest
+from django.db import connection
+from django_test_migrations.migrator import Migrator
 from playwright.sync_api import Locator, Page, Response
 
 from adit_radis_shared.accounts.factories import UserFactory
@@ -61,3 +63,48 @@ def _create_and_login_user(server_url: str):
         return user
 
     return _create_and_login_user
+
+
+@pytest.fixture
+def migrator(migrator: Migrator) -> Generator[Migrator, None, None]:
+    yield migrator
+
+    # We have to manually cleanup the Procrastinate tables, functions and types
+    # as otherwise the reset of django_test_migrations will fail
+    # See https://github.com/procrastinate-org/procrastinate/issues/1090
+    with connection.cursor() as cursor:
+        cursor.execute("""
+        DO $$ 
+        DECLARE
+            prefix text := 'procrastinate';
+        BEGIN
+            -- Drop tables
+            EXECUTE (
+                SELECT string_agg('DROP TABLE IF EXISTS ' || quote_ident(tablename)
+                || ' CASCADE;', ' ')
+                FROM pg_tables
+                WHERE tablename LIKE prefix || '%'
+            );
+
+            -- Drop functions
+            EXECUTE (
+                SELECT string_agg(
+                    'DROP FUNCTION IF EXISTS ' || quote_ident(n.nspname) || '.'
+                    || quote_ident(p.proname) || '('
+                    || pg_catalog.pg_get_function_identity_arguments(p.oid) || ') CASCADE;',
+                    ' '
+                )
+                FROM pg_proc p
+                LEFT JOIN pg_namespace n ON n.oid = p.pronamespace
+                WHERE p.proname LIKE prefix || '%'
+            );
+
+            -- Drop types
+            EXECUTE (
+                SELECT string_agg('DROP TYPE IF EXISTS ' || quote_ident(typname)
+                || ' CASCADE;', ' ')
+                FROM pg_type
+                WHERE typname LIKE prefix || '%'
+            );
+        END $$;
+        """)