Skip to content

Commit

Permalink
feat: pre-build foundation
Browse files Browse the repository at this point in the history
  • Loading branch information
adamblake committed Dec 17, 2024
1 parent a04c8ec commit c22ffaa
Show file tree
Hide file tree
Showing 2 changed files with 233 additions and 123 deletions.
202 changes: 202 additions & 0 deletions .github/workflows/build-test-publish.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,202 @@
---
name: Build, Test, and Publish an Image

on:
workflow_call:
inputs:
runs-on:
description: The platform to run the action on
type: string
required: false
default: ubuntu-latest
free-disk-space:
description: Whether to free disk space before building
type: boolean
required: false
default: false
image:
description: The image to build
type: string
required: true
target:
description: The target layer to build
type: string
required: false
build-args:
description: Build arguments to pass to the image
type: string
required: false
publish:
description: Whether to publish the image to remote registries
type: boolean
required: false
default: false
tags:
description: Tags to add to the image
type: string
required: false
labels:
description: Labels to add to the image
type: string
required: false
no-cache:
description: Whether to disable the build cache
type: boolean
required: false
default: false
cache-from:
description: Docker images that might be able to contribute cached layers
type: string
required: false

outputs:
image:
description: The image name
value: ${{ jobs.docker.outputs.image }}
digest:
description: The image digest
value: ${{ jobs.docker.outputs.digest }}

jobs:
docker:
runs-on: ${{ inputs.runs-on }}

outputs:
image: ${{ inputs.image }}
digest: ${{ steps.build.outputs.digest }}

services:
registry:
image: registry:2
ports:
- 5000:5000

steps:
# https://github.com/jlumbroso/free-disk-space
- name: Free Disk Space (Ubuntu)
if: inputs.free-disk-space
uses: jlumbroso/free-disk-space@main
with:
dotnet: ${{ inputs.free-disk-space }}
haskell: false
large-packages: ${{ inputs.free-disk-space }}
swap-storage: ${{ inputs.free-disk-space }}

# https://github.com/actions/checkout
- name: Checkout
uses: actions/checkout@v4

# https://github.com/docker/setup-qemu-action
- name: Set up QEMU
uses: docker/setup-qemu-action@v3

# https://github.com/docker/setup-buildx-action
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
# network=host driver-opt needed to push to local registry
driver-opts: network=host

# https://github.com/docker/login-action
- name: Login to GitHub Container Registry
if: inputs.publish
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

# https://github.com/docker/login-action
- name: Login to Docker Registry
if: inputs.publish
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}

# https://github.com/docker/build-push-action
- name: Build for linux/arm64
id: build-arm64
uses: docker/build-push-action@v5
with:
target: ${{ inputs.target }}
platforms: linux/arm64
build-args: ${{ inputs.build-args }}
secrets: "github_token=${{ secrets.GITHUB_TOKEN }}"
tags: localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:arm64
cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ inputs.image }}
cache-to: type=registry,ref=localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:buildcache-arm64,mode=max
no-cache: ${{ inputs.no-cache }}
push: true

- name: Test for linux/arm64
shell: bash
run: |
docker run --rm --platform=linux/arm64 \
--mount=type=bind,source="./tests/test-packages.sh",target=/tmp/test-packages.sh \
--mount=type=bind,source="./tests/packages.txt",target=/tmp/packages.txt \
--mount=type=bind,source="./tests/${{ inputs.image }}.sh",target=/tmp/test.sh \
localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:arm64 \
bash /tmp/test.sh
# https://github.com/docker/build-push-action
- name: Build for linux/amd64
id: build-amd64
uses: docker/build-push-action@v5
with:
target: ${{ inputs.target }}
platforms: linux/amd64
build-args: ${{ inputs.build-args }}
secrets: "github_token=${{ secrets.GITHUB_TOKEN }}"
tags: localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:amd64
cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ inputs.image }}
cache-to: type=registry,ref=localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:buildcache-amd64,mode=max
no-cache: ${{ inputs.no-cache }}
push: true

- name: Test for linux/amd64
shell: bash
run: |
docker run --rm --platform=linux/amd64 \
--mount=type=bind,source="./tests/test-packages.sh",target=/tmp/test-packages.sh \
--mount=type=bind,source="./tests/packages.txt",target=/tmp/packages.txt \
--mount=type=bind,source="./tests/${{ inputs.image }}.sh",target=/tmp/test.sh \
localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:amd64 \
bash /tmp/test.sh
# https://github.com/docker/metadata-action
- name: Image Metadata
id: meta
if: inputs.publish
uses: docker/metadata-action@v5
with:
images: |
${{ github.repository_owner }}/${{ inputs.image }}
ghcr.io/${{ github.repository_owner }}/${{ inputs.image }}
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=schedule,pattern={{date 'YYYY-MM-DD'}}
type=sha
${{ inputs.tags }}
labels: |
org.opencontainers.image.title=${{ inputs.image }}
org.opencontainers.image.url=https://github.com/${{ github.repository_owner }}/docker-stacks/pkgs/container/${{ inputs.image }}
${{ inputs.labels }}
# https://github.com/docker/build-push-action
- name: Build and push multi-arch
id: build
if: inputs.publish
uses: docker/build-push-action@v6
with:
target: ${{ inputs.target }}
platforms: linux/amd64,linux/arm64
build-args: ${{ inputs.build-args }}
secrets: "github_token=${{ secrets.GITHUB_TOKEN }}"
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: |
type=registry,ref=localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:buildcache-amd64
type=registry,ref=localhost:5000/${{ github.repository_owner }}/${{ inputs.image }}:buildcache-arm64
cache-to: type=inline
push: true
154 changes: 31 additions & 123 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -26,149 +26,57 @@ concurrency:
cancel-in-progress: true

jobs:
foundation:
uses: ./.github/workflows/build-test-publish.yml
secrets: inherit
with:
image: foundation
target: base
no-cache: ${{ github.event_name == 'schedule' }}
labels: |
org.opencontainers.image.title=Foundational Image for CourseKata Notebook Images
org.opencontainers.image.description=System dependencies for CourseKata notebook images.
build-args: |
PIXI_ENV=default
publish: true

docker:
runs-on: ubuntu-latest
needs: [foundation]
uses: ./.github/workflows/build-test-publish.yml
with:
image: ${{ matrix.image }}
no-cache: ${{ github.event_name == 'schedule' }}
labels: |
org.opencontainers.image.title=${{ matrix.title }}
org.opencontainers.image.description=${{ matrix.description }}
build-args: |
PIXI_ENV=${{ matrix.pixi_env }}
publish: true
free-disk-space: ${{ matrix.free_disk_space }}
cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/foundation
strategy:
matrix:
include:
- image: base-r-notebook
- image: base-r-notebook-test
pixi_env: base-r
title: Base R Notebook
description: Jupyter Lab, Python, and R, and that's it.
free_disk_space: false
free_more_disk_space: false

- image: essentials-notebook
- image: essentials-notebook-test
pixi_env: essentials
title: Essentials Notebook
description: CourseKata essentials - everything used in the books.
free_disk_space: true
free_more_disk_space: false

- image: r-notebook
- image: r-notebook-test
pixi_env: r
title: R Notebook
description: CourseKata essentials and other R packages for teaching and learning data science.
free_disk_space: true
free_more_disk_space: false

- image: datascience-notebook
- image: datascience-notebook-test
pixi_env: datascience
title: Data Science Notebook
description: R and Python packages for teaching and learning data science.
free_disk_space: true
free_more_disk_space: true

services:
registry:
image: registry:2
ports:
- 5000:5000

steps:
- name: Free Disk Space (Ubuntu)
if: matrix.free_disk_space
uses: jlumbroso/free-disk-space@main
with:
dotnet: ${{ matrix.free_more_disk_space }}
haskell: false
large-packages: ${{ matrix.free_more_disk_space }}
swap-storage: ${{ matrix.free_more_disk_space }}

- name: Checkout
uses: actions/checkout@v4

- name: Set up QEMU
uses: docker/setup-qemu-action@v3

- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver-opts: network=host

- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

- name: Login to Docker Registry
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_TOKEN }}

- name: Build for linux/arm64
uses: docker/build-push-action@v5
with:
platforms: linux/arm64
build-args: PIXI_ENV=${{ matrix.pixi_env }}
secrets: "github_token=${{ secrets.GITHUB_TOKEN }}"
tags: localhost:5000/${{ github.repository_owner }}/${{ matrix.image }}:arm64
cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
cache-to: type=registry,ref=localhost:5000/${{ github.repository_owner }}/${{ matrix.image }}:buildcache-arm64,mode=max
no-cache: ${{ github.event_name == 'schedule' }}
push: true

- name: Test for linux/arm64
shell: bash
run: |
docker run --rm --platform=linux/arm64 \
--mount=type=bind,source="./tests/test-packages.sh",target=/tmp/test-packages.sh \
--mount=type=bind,source="./tests/packages.txt",target=/tmp/packages.txt \
--mount=type=bind,source="./tests/${{ matrix.image }}.sh",target=/tmp/test.sh \
localhost:5000/${{ github.repository_owner }}/${{ matrix.image }}:arm64 \
bash /tmp/test.sh
- name: Build for linux/amd64
uses: docker/build-push-action@v5
with:
platforms: linux/amd64
build-args: PIXI_ENV=${{ matrix.pixi_env }}
secrets: "github_token=${{ secrets.GITHUB_TOKEN }}"
tags: localhost:5000/${{ github.repository_owner }}/${{ matrix.image }}:amd64
cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
cache-to: type=registry,ref=localhost:5000/${{ github.repository_owner }}/${{ matrix.image }}:buildcache-amd64,mode=max
no-cache: ${{ github.event_name == 'schedule' }}
push: true

- name: Test for linux/amd64
shell: bash
run: |
docker run --rm --platform=linux/amd64 \
--mount=type=bind,source="./tests/test-packages.sh",target=/tmp/test-packages.sh \
--mount=type=bind,source="./tests/packages.txt",target=/tmp/packages.txt \
--mount=type=bind,source="./tests/${{ matrix.image }}.sh",target=/tmp/test.sh \
localhost:5000/${{ github.repository_owner }}/${{ matrix.image }}:amd64 \
bash /tmp/test.sh
- name: Image Metadata
id: meta
uses: docker/metadata-action@v5
with:
images: |
${{ github.repository_owner }}/${{ matrix.image }}
ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
tags: |
type=raw,value=latest,enable={{is_default_branch}}
type=schedule,pattern={{date 'YYYY-MM-DD'}}
type=sha
labels: |
org.opencontainers.image.title=${{ matrix.title }}
org.opencontainers.image.description=${{ matrix.description }}
org.opencontainers.image.url=https://github.com/${{ github.repository_owner }}/docker-stacks/pkgs/container/${{ matrix.image }}
- name: Build and push multi-arch
uses: docker/build-push-action@v5
with:
platforms: linux/amd64,linux/arm64
build-args: PIXI_ENV=${{ matrix.pixi_env }}
secrets: "github_token=${{ secrets.GITHUB_TOKEN }}"
tags: |
ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}-test:latest
ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}-test:${{ steps.meta.outputs.tags }}
cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}
cache-to: type=registry,ref=ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:buildcache,mode=max
no-cache: ${{ github.event_name == 'schedule' }}
push: true

0 comments on commit c22ffaa

Please sign in to comment.