From 1b33c5c9cab7e045d4d55821e91ab2590907e0c6 Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 21:32:45 +0200 Subject: [PATCH 01/10] docker: Add dockerfile and github workflow to build a docker image for Amelie --- .github/workflows/build_docker_image.yml | 47 ++++++++++++++++++++ .github/workflows/ia_gitlab_ci.yml | 13 ------ .github/workflows/ia_gitlab_ci_retrieve.yml | 32 -------------- Dockerfile | 48 +++++++++++++++++++++ 4 files changed, 95 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/build_docker_image.yml delete mode 100644 .github/workflows/ia_gitlab_ci.yml delete mode 100644 .github/workflows/ia_gitlab_ci_retrieve.yml create mode 100644 Dockerfile diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml new file mode 100644 index 0000000..2000b7d --- /dev/null +++ b/.github/workflows/build_docker_image.yml @@ -0,0 +1,47 @@ +name: Build docker image and run tests + +on: [push, pull_request] + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + attestations: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: push + uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true diff --git a/.github/workflows/ia_gitlab_ci.yml b/.github/workflows/ia_gitlab_ci.yml deleted file mode 100644 index 2d0fbe2..0000000 --- a/.github/workflows/ia_gitlab_ci.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: Trigger CI job on Inter-Actief GitLab -on: [push, pull_request] -env: - GITLAB_TOKEN: ${{ secrets.IA_GITLAB_CI_TOKEN }} - GITHUB_REF: ${{ github.ref }} -jobs: - test: - name: Run Tests - runs-on: ubuntu-latest - steps: - - name: Trigger CI job on Inter-Actief GitLab - shell: bash - run: curl -X POST --fail -F token=$GITLAB_TOKEN -F ref="refs/heads/main" -F "variables[AMELIE_BRANCH_REF]=$GITHUB_REF" https://gitlab.ia.utwente.nl/api/v4/projects/153/trigger/pipeline diff --git a/.github/workflows/ia_gitlab_ci_retrieve.yml b/.github/workflows/ia_gitlab_ci_retrieve.yml deleted file mode 100644 index f54e133..0000000 --- a/.github/workflows/ia_gitlab_ci_retrieve.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Retrieve CI job status from Inter-Actief GitLab -on: - workflow_dispatch: - inputs: - result: - description: 'Result of pipeline' - required: true - type: choice - options: - - success - - failed - - canceled - details_url: - description: 'URL to Job details on the IA GitLab' - required: true - job_id: - description: 'The ID of the Job on the IA GitLab' - required: true -jobs: - test: - name: Retrieve CI job status - runs-on: ubuntu-latest - steps: - - name: Retrieve CI job status from Inter-Actief GitLab - shell: bash - run: | - echo "Hello there." >> $GITHUB_STEP_SUMMARY - echo "The result of job ${{ inputs.job_id }} was `${{ inputs.result }}`." >> $GITHUB_STEP_SUMMARY - echo "See more info at [${{ inputs.details_url }}](${{ inputs.details_url }})." >> $GITHUB_STEP_SUMMARY - - name: Fail if the result is failed - if: ${{ inputs.result == 'failed' }} - run: exit 1 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..f70c5e5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,48 @@ +# Build the amelie docker image based on Debian 11 (Bullseye) +FROM debian:bullseye + +# Install required debian packages for amelie +RUN apt-get update -y && apt-get upgrade -y && apt-get install -y apt-utils git net-tools python3 python3-pip mariadb-client libmariadb-dev xmlsec1 libssl-dev libldap-dev libsasl2-dev libjpeg-dev zlib1g-dev gettext + +# Enable nl_NL and en_US locales and rebuild locale +RUN apt-get update && \ + apt-get install -y locales && \ + sed -i -e 's/# nl_NL.UTF-8 UTF-8/nl_NL.UTF-8 UTF-8/' /etc/locale.gen && \ + sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ + dpkg-reconfigure --frontend=noninteractive locales + +# Make directories for amelie +RUN mkdir -p /amelie /amelie/static /media /photo_upload /config /var/log /var/run + +# Copy amelie sources +COPY . /amelie + +# Set /amelie as startup working directory +WORKDIR /amelie + +# Switch to root user +USER root + +# Install python requirements +RUN pip3 install -r requirements.txt + +# Install default configuration file +RUN cp "/amelie/amelie/settings/local.py.default" "/amelie/amelie/settings/local.py" + +# Correct permissions on directories +RUN chown -R 1000:1000 /amelie /media /photo_upload /config /var/log + +# Switch back to a local user +USER 1000:1000 + +# Check if Django can run +RUN python3 manage.py check + +# Expose volumes +VOLUME ["/amelie/static", "/media", "/photo_upload", "/config"] + +# Expose the web port +EXPOSE 80 + +# Start the website +CMD ["/amelie/scripts/start_web.sh"] From 453eff4a8a7b6e26c77e638753c8a0661c6727c8 Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 21:39:48 +0200 Subject: [PATCH 02/10] docker: Allow docker build action to write the IDToken --- .github/workflows/build_docker_image.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index 2000b7d..7d9308b 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -13,6 +13,7 @@ jobs: contents: read packages: write attestations: write + id-token: write steps: - name: Checkout repository uses: actions/checkout@v4 From 8729a0191669f30c01e1cea664ee849e695ca42d Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 21:49:33 +0200 Subject: [PATCH 03/10] docker: Remove attestation from docker build action for now --- .github/workflows/build_docker_image.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index 7d9308b..f16f4b0 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -12,8 +12,6 @@ jobs: permissions: contents: read packages: write - attestations: write - id-token: write steps: - name: Checkout repository uses: actions/checkout@v4 @@ -39,10 +37,3 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}} - subject-digest: ${{ steps.push.outputs.digest }} - push-to-registry: true From b3a65ae348df7aecc050db04755521f48d8e229c Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 22:13:29 +0200 Subject: [PATCH 04/10] docker: Add run tests step to build action --- .github/workflows/build_docker_image.yml | 22 +++++++++++++++++ amelie/settings/{gitlab.py => tests.py} | 0 amelie/settings/travis.py | 19 --------------- scripts/run_tests.sh | 31 ++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 19 deletions(-) rename amelie/settings/{gitlab.py => tests.py} (100%) delete mode 100644 amelie/settings/travis.py create mode 100644 scripts/run_tests.sh diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index f16f4b0..6e3dcd2 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -12,6 +12,18 @@ jobs: permissions: contents: read packages: write + + services: + mariadb: + image: mariadb:10.2 + env: + MARIADB_USER: amelie_test + MARIADB_PASSWORD: amelie_test + MYSQL_DATABASE: amelie_test + MYSQL_ROOT_PASSWORD: amelie_test + ports: ['3306:3306'] + options: --health-cmd="mysqladmin ping" --health-interval=5s --health-timeout=2s --health-retries=3 + steps: - name: Checkout repository uses: actions/checkout@v4 @@ -37,3 +49,13 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: Verify MariaDB connection + run: | + while ! mysqladmin ping -h"127.0.0.1" -P"3306" --silent; do + sleep 1 + done + + - name: Run Amelie tests + run: | + docker run --rm --entrypoint "/amelie/scripts/run_tests.sh" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.push.outputs.digest }} diff --git a/amelie/settings/gitlab.py b/amelie/settings/tests.py similarity index 100% rename from amelie/settings/gitlab.py rename to amelie/settings/tests.py diff --git a/amelie/settings/travis.py b/amelie/settings/travis.py deleted file mode 100644 index fadb9b0..0000000 --- a/amelie/settings/travis.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import absolute_import -# Settings used for running tests in Travis - -# Load default settings -# noinspection PyUnresolvedReferences -from amelie.settings.generic import * - -# Database -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'HOST': '127.0.0.1', - 'NAME': 'amelie_test', - 'USER': 'travis', - 'PASSWORD': '', - } -} - -SECRET_KEY = 'Cr38bXqNzU45MQx030enbqTyRZufMcywcRZJygkFKxnx5lu5Iq' diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh new file mode 100644 index 0000000..933a102 --- /dev/null +++ b/scripts/run_tests.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# Print some debugging information about the environment +# ------------------------------------------------------ +# Python version +python -V +# Path to the python binary being used +which python +# Pip version +pip -V +# Path to the pip binary being used +which pip +# Installed pip package list +pip freeze + +# Configure Django and run the tests +# ---------------------------------- +# Copy the test settings to local.py +cp ./amelie/settings/tests.py ./amelie/settings/local.py + +# Run Django initial checks +python manage.py check + +# Make sure staticfiles are collected into the static volume +python3 manage.py collectstatic --noinput + +# Make sure database is migrated +python3 manage.py migrate + +# Run Django tests +python manage.py test --keepdb From d4d04f7e4e66456065238909ccd83aee16ee44dc Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 22:17:50 +0200 Subject: [PATCH 05/10] docker: Update step versions and change docker image name --- .github/workflows/build_docker_image.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index 6e3dcd2..64e5830 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -29,7 +29,7 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} @@ -37,13 +37,13 @@ jobs: - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and push Docker image id: push - uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4 + uses: docker/build-push-action@v5 with: context: . push: true @@ -58,4 +58,4 @@ jobs: - name: Run Amelie tests run: | - docker run --rm --entrypoint "/amelie/scripts/run_tests.sh" ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.push.outputs.digest }} + docker run --rm --entrypoint "/amelie/scripts/run_tests.sh" ${{ env.IMAGE_NAME }}@${{ steps.push.outputs.digest }} From 2fd96bc719b292afed6ac1f7dc7b12400d1dea7d Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 22:25:21 +0200 Subject: [PATCH 06/10] docker: Change docker image name --- .github/workflows/build_docker_image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index 64e5830..a663121 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -58,4 +58,4 @@ jobs: - name: Run Amelie tests run: | - docker run --rm --entrypoint "/amelie/scripts/run_tests.sh" ${{ env.IMAGE_NAME }}@${{ steps.push.outputs.digest }} + docker run --rm --entrypoint "/amelie/scripts/run_tests.sh" ghcr.io/inter-actief/amelie@${{ steps.push.outputs.digest }} From bc82b97efefc0132aba0077e928a620d7fa78952 Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 22:29:18 +0200 Subject: [PATCH 07/10] docker: Make test script executable --- scripts/run_tests.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/run_tests.sh diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh old mode 100644 new mode 100755 From f466c456375d3a44a6950ca052191e8483d7d941 Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 22:34:29 +0200 Subject: [PATCH 08/10] docker: Fix python commands and test mysql hostname --- amelie/settings/tests.py | 2 +- scripts/run_tests.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/amelie/settings/tests.py b/amelie/settings/tests.py index 555cfb4..64d53de 100644 --- a/amelie/settings/tests.py +++ b/amelie/settings/tests.py @@ -9,7 +9,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'mariadb', + 'HOST': '127.0.0.1', 'NAME': 'amelie_test', 'USER': 'amelie_test', 'PASSWORD': 'amelie_test', diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh index 933a102..19cb633 100755 --- a/scripts/run_tests.sh +++ b/scripts/run_tests.sh @@ -3,7 +3,7 @@ # Print some debugging information about the environment # ------------------------------------------------------ # Python version -python -V +python3 -V # Path to the python binary being used which python # Pip version @@ -19,7 +19,7 @@ pip freeze cp ./amelie/settings/tests.py ./amelie/settings/local.py # Run Django initial checks -python manage.py check +python3 manage.py check # Make sure staticfiles are collected into the static volume python3 manage.py collectstatic --noinput @@ -28,4 +28,4 @@ python3 manage.py collectstatic --noinput python3 manage.py migrate # Run Django tests -python manage.py test --keepdb +python3 manage.py test --keepdb From 880278645cc5183b08ad845330e8d36d68bdaf82 Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 22:41:17 +0200 Subject: [PATCH 09/10] docker: Change test mysql host --- amelie/settings/tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amelie/settings/tests.py b/amelie/settings/tests.py index 64d53de..456a066 100644 --- a/amelie/settings/tests.py +++ b/amelie/settings/tests.py @@ -9,7 +9,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'HOST': '127.0.0.1', + 'HOST': '172.17.0.1', 'NAME': 'amelie_test', 'USER': 'amelie_test', 'PASSWORD': 'amelie_test', From 5c113ee1367abc2a2e73b04938e1a4b913183aac Mon Sep 17 00:00:00 2001 From: Kevin Alberts Date: Mon, 6 May 2024 22:46:22 +0200 Subject: [PATCH 10/10] docker: Fix database settings for tests --- amelie/settings/tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/amelie/settings/tests.py b/amelie/settings/tests.py index 456a066..55fdfd3 100644 --- a/amelie/settings/tests.py +++ b/amelie/settings/tests.py @@ -8,11 +8,15 @@ # Database DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.mysql', + 'ENGINE': 'amelie.tools.utf8mb4_mysql_backend', 'HOST': '172.17.0.1', 'NAME': 'amelie_test', 'USER': 'amelie_test', 'PASSWORD': 'amelie_test', + 'OPTIONS': {'charset': 'utf8mb4'}, + 'TEST': { + 'NAME': 'amelie_test', + } } }