Adjusted publish tests and workflow

.github/workflows/pipeline.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
@@ -27,54 +27,131 @@ jobs:
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
           poetry install --no-interaction --no-root
-      - name: pylint
+      - name: Run pylint and black
         run: |
           source .venv/bin/activate
           pylint build tests
-      - name: black
+          black --check .
+
+  run-build-image-tests:
+    needs: code-quality
+    runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.10.13", "3.11.6", "3.12.0"]
+        os_variant: ["bookworm", "slim-bookworm"]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v3          
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.6.1
+          virtualenvs-in-project: true
+      - name: Load cached venv
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}            
+      - name: Install dependencies
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: |
+          poetry install --no-interaction --no-root
+      - name: Run tests for image builds with pytest
+        env:
+          PYTHON_VERSION: ${{ matrix.python_version }}
+          OS_VARIANT: ${{ matrix.os_variant }}
+          POETRY_VERSION: ${{ matrix.poetry_version }}
         run: |
           source .venv/bin/activate
-          black --check .
+          pytest tests/build_image --cov --cov-report=xml:build_image_coverage_report.xml
+      - name: Upload coverage report to artifactory
+        uses: actions/upload-artifact@v3
+        with:
+          name: build-image-coverage-report-${{ matrix.python_version }}-${{ matrix.os_variant }}-${{ matrix.poetry_version }}
+          path: build_image_coverage_report.xml
+          if-no-files-found: error
+          retention-days: 1          
 
-  run-tests:
+run-publish-image-tests:
     needs: code-quality
     runs-on: ubuntu-20.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python_version: ["3.10.13", "3.11.6", "3.12.0"]
+        os_variant: ["bookworm", "slim-bookworm"]
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@v3
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: actions/checkout@v4
+      - name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v3          
       - name: Setup Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: 3.11
       - name: Install Poetry
         uses: snok/install-poetry@v1
         with:
-          version: 1.4.1
+          version: 1.6.1
           virtualenvs-in-project: true
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v3
         with:
           path: .venv
-          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}            
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
           poetry install --no-interaction --no-root
-      - name: Run all tests with pytest
+      - name: Run tests for image publishing with pytest
+        env:
+          PYTHON_VERSION: ${{ matrix.python_version }}
+          OS_VARIANT: ${{ matrix.os_variant }}
+          POETRY_VERSION: ${{ matrix.poetry_version }}
         run: |
           source .venv/bin/activate
-          pytest --cov
+          pytest tests/publish_image --cov --cov-report=xml:publish_image_coverage_report.xml
+      - name: Upload coverage report to artifactory
+        uses: actions/upload-artifact@v3
+        with:
+          name: publish-image-coverage-report-${{ matrix.python_version }}-${{ matrix.os_variant }}-${{ matrix.poetry_version }}
+          path: publish_image_coverage_report.xml
+          if-no-files-found: error
+          retention-days: 1
+
+  upload-test-coverage-reports:
+    needs:
+      - run-build-image-tests
+      - run-publish-image-tests
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - name: Download coverage reports from artifactory
+        uses: actions/download-artifact@v3
+      - name: Compile the relevant reports
+        run: |
+          find . -name "*.xml" -exec cp {} . \;
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
+          files: ./build_image_coverage_report.xml,./publish_image_coverage_report.xml
           fail_ci_if_error: true
           token: ${{ secrets.CODECOV_TOKEN }}
-
+        
   publish-all-images:
-    needs: run-tests
+    needs:
+      - upload-test-coverage-reports      
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-20.04
     steps:

build/publish.py

@@ -1,10 +1,9 @@
 import click
-import docker
-from docker.client import DockerClient
-from build.constants import (
-    TARGET_ARCHITECTURES,
-)
-from build.images import UvicornPoetryImage
+from build.constants import PLATFORMS, APPLICATION_SERVER_PORT
+from build.utils import get_context, get_image_reference
+from pathlib import Path
+from python_on_whales import DockerClient, Builder
+from os import getenv
 
 
 @click.command()
@@ -19,59 +18,73 @@
     help="Docker Hub password",
 )
 @click.option(
-    "--version-tag", envvar="GIT_TAG_NAME", required=True, help="Version Tag"
+    "--version-tag", envvar="GIT_TAG_NAME", required=True, help="Version tag"
+)
+@click.option(
+    "--python-version",
+    envvar="PYTHON_VERSION",
+    required=True,
+    help="Python version",
+)
+@click.option(
+    "--os-variant",
+    envvar="OS_VARIANT",
+    required=True,
+    help="Operating system variant",
 )
 @click.option("--registry", envvar="REGISTRY", help="Docker registry")
 def main(
     docker_hub_username: str,
     docker_hub_password: str,
     version_tag: str,
+    python_version: str,
+    os_variant: str,
     registry: str,
 ) -> None:
-    docker_client: DockerClient = docker.from_env()
+    github_ref_name: str = getenv("GITHUB_REF_NAME")
+    context: Path = get_context()
+    image_reference: str = get_image_reference(
+        registry, version_tag, python_version, os_variant
+    )
+    cache_scope: str = f"{python_version}-{os_variant}"
 
-    for target_architecture in TARGET_ARCHITECTURES:
-        new_uvicorn_gunicorn_poetry_image: UvicornPoetryImage = (
-            UvicornPoetryImage(docker_client, target_architecture, version_tag)
+    if github_ref_name:
+        cache_to: str = (
+            f"type=gha,mode=max,scope={github_ref_name}-{cache_scope}"
         )
+        cache_from: str = f"type=gha,scope={github_ref_name}-{cache_scope}"
+    else:
+        cache_to = f"type=local,mode=max,dest=/tmp,scope={cache_scope}"
+        cache_from = f"type=local,src=/tmp,scope={cache_scope}"
 
-        # Delete old existing images
-        for old_image in docker_client.images.list(
-            new_uvicorn_gunicorn_poetry_image.image_name
-        ):
-            for tag in old_image.tags:
-                docker_client.images.remove(tag, force=True)
-
-        new_uvicorn_gunicorn_poetry_image.build()
-
-        # https://docs.docker.com/engine/reference/commandline/push/
-        # https://docs.docker.com/engine/reference/commandline/tag/
-        # https://docs.docker.com/engine/reference/commandline/image_tag/
-        if docker_hub_username and docker_hub_password:
-            login_kwargs: dict = {
-                "username": docker_hub_username,
-                "password": docker_hub_password,
-            }
-            if registry:
-                login_kwargs["registry"] = registry
+    docker_client: DockerClient = DockerClient()
+    builder: Builder = docker_client.buildx.create(
+        driver="docker-container", driver_options=dict(network="host")
+    )
 
-            docker_client.login(**login_kwargs)
+    docker_client.login(
+        server=registry,
+        username=docker_hub_username,
+        password=docker_hub_password,
+    )
 
-        if registry:
-            repository: str = (
-                f"{registry}/{new_uvicorn_gunicorn_poetry_image.image_name}"
-            )
-        else:
-            repository: str = new_uvicorn_gunicorn_poetry_image.image_name
+    docker_client.buildx.build(
+        context_path=context,
+        build_args={
+            "BASE_IMAGE": f"pfeiffermax/python-poetry:1.8.0-poetry1.7.1-python{python_version}-{os_variant}",
+            "APPLICATION_SERVER_PORT": APPLICATION_SERVER_PORT,
+        },
+        tags=image_reference,
+        platforms=PLATFORMS,
+        builder=builder,
+        cache_to=cache_to,
+        cache_from=cache_from,
+        push=True,
+    )
 
-        for line in docker_client.images.push(
-            repository,
-            tag=new_uvicorn_gunicorn_poetry_image.image_tag,
-            stream=True,
-            decode=True,
-        ):
-            print(line)
-    docker_client.close()
+    # Cleanup
+    docker_client.buildx.stop(builder)
+    docker_client.buildx.remove(builder)
 
 
 if __name__ == "__main__":

tests/build_image/conftest.py

@@ -1,5 +1,4 @@
 from os import getenv
-from time import sleep
 
 import pytest
 from python_on_whales import Builder, DockerClient
@@ -16,16 +15,6 @@
 )
 
 
-@pytest.fixture(scope="session")
-def python_version() -> str:
-    return getenv("PYTHON_VERSION")
-
-
-@pytest.fixture(scope="session")
-def os_variant() -> str:
-    return getenv("OS_VARIANT")
-
-
 @pytest.fixture(scope="session")
 def cache_settings(python_version: str, os_variant: str) -> tuple:
     github_ref_name: str = getenv("GITHUB_REF_NAME")

tests/conftest.py

@@ -1,5 +1,5 @@
 from random import randrange
-
+from os import getenv
 import pytest
 from python_on_whales import Builder, DockerClient
 from semver import VersionInfo
@@ -27,3 +27,13 @@ def image_version() -> str:
     )
     version_string: str = str(version)
     return version_string
+
+
+@pytest.fixture(scope="session")
+def python_version() -> str:
+    return getenv("PYTHON_VERSION")
+
+
+@pytest.fixture(scope="session")
+def os_variant() -> str:
+    return getenv("OS_VARIANT")