From d523bc49c3cab425628eeab372fce6b37174144f Mon Sep 17 00:00:00 2001 From: Will Winder Date: Thu, 20 Apr 2023 13:34:22 -0400 Subject: [PATCH] docker: data dir & user permissions, install ca-certificates, better data dir location, more. (#58) Improvements to the docker container and associated utilities. --- .github/workflows/goreleaser.yml | 8 +++++++- .goreleaser.yaml | 4 ++++ Dockerfile | 33 ++++++++++++++++++++++---------- docker/docker-entrypoint.sh | 14 ++++++++++++++ docs/Docker.md | 25 +++++++++++++++++++----- go.sum | 3 --- 6 files changed, 68 insertions(+), 19 deletions(-) create mode 100755 docker/docker-entrypoint.sh diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml index a59956b0..49858b58 100644 --- a/.github/workflows/goreleaser.yml +++ b/.github/workflows/goreleaser.yml @@ -30,11 +30,17 @@ jobs: - name: Setup Docker Buildx (docker multi-arch dependency) uses: docker/setup-buildx-action@v2 + - name: dockerhub-login + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - uses: goreleaser/goreleaser-action@v4 with: distribution: goreleaser version: latest - args: release --clean --skip-docker + args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml index d922b1dd..1b3f9284 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -42,6 +42,8 @@ dockers: - --label=org.opencontainers.image.created={{ .Date }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.licenses=MIT + extra_files: + - docker/docker-entrypoint.sh - use: buildx goos: linux goarch: arm64 @@ -55,6 +57,8 @@ dockers: - --label=org.opencontainers.image.created={{ .Date }} - --label=org.opencontainers.image.revision={{ .FullCommit }} - --label=org.opencontainers.image.licenses=MIT + extra_files: + - docker/docker-entrypoint.sh # automatically select amd64/arm64 when requesting "algorand/conduit" docker_manifests: diff --git a/Dockerfile b/Dockerfile index 9d3e0e2e..24cac785 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,27 @@ -# This dockerfile is used by goreleaser +# Build this Dockerfile with goreleaser. +# The binary must be present at /conduit FROM debian:bullseye-slim -RUN useradd conduit -RUN mkdir -p /conduit/data && \ - chown -R conduit.conduit /conduit +# Hard code UID/GID to 999 for consistency in advanced deployments. +# Install ca-certificates to enable using infra providers. +# Install gosu for fancy data directory management. +RUN groupadd --gid=999 --system algorand && \ + useradd --uid=999 --no-log-init --create-home --system --gid algorand algorand && \ + mkdir -p /data && \ + chown -R algorand.algorand /data && \ + apt-get update && \ + apt-get install -y gosu ca-certificates && \ + update-ca-certificates && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -# binary is passed into the build -COPY conduit /conduit/conduit +COPY conduit /usr/local/bin/conduit +COPY docker/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh -USER conduit -WORKDIR /conduit -ENTRYPOINT ["./conduit"] -CMD ["-d", "data"] +ENV CONDUIT_DATA_DIR /data +WORKDIR /data +# Note: docker-entrypoint.sh calls 'conduit'. Similar entrypoint scripts +# accept the binary as the first argument in order to surface a suite of +# tools (i.e. algod, goal, algocfg, ...). Maybe this will change in the +# future, but for now this approach seemed simpler. +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100755 index 00000000..da0e4a88 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -e + +# To allow mounting the data directory we need to change permissions +# to our algorand user. The script is initially run as the root user +# in order to change permissions; afterwards, the script is re-launched +# as the algorand user. +if [ "$(id -u)" = '0' ]; then + chown -R algorand:algorand $CONDUIT_DATA_DIR + exec gosu algorand "$0" "$@" +fi + +# always run the conduit command +exec conduit "$@" diff --git a/docs/Docker.md b/docs/Docker.md index 363490a3..0d1ffd16 100644 --- a/docs/Docker.md +++ b/docs/Docker.md @@ -1,5 +1,3 @@ -**This container is a work in progress and not yet deployed to docker hub.** - # Docker Image Algorand's Conduit data pipeline packaged for docker. @@ -42,8 +40,25 @@ docker run algorand/conduit init --importer algod --processors filter_processor ## Run with conduit.yml -With `conduit.yml` in your current working directory, -launch the container: +With `conduit.yml` in your current working directory, it can be mounted directly to `/data/conduit.yml`. This is good for testing and some deployments which override the starting round. For a more complete deployment see the next section which explains how to mount the entire data directory. + +Mount `conduit.yml` with the following command: +``` +docker run -it -v $(pwd)/conduit.yml:/data/conduit.yml algorand/conduit ``` -docker run -it -v $(pwd)/conduit.yml:/conduit/data/conduit.yml algorand/conduit + +# Mounting the Data Directory + +For production deployments, you should consider mounting the entire data directory. This way you can persist state across images during an upgrade, or for backups. The data directory is located at `/data`. When mounting a data directory, it must contain the `conduit.yml` file. + ``` +docker run -it -v $(pwd)/local_data_dir:/data algorand/conduit +``` + +## Volume Permissions + +The container executes in the context of the `algorand` user with UID=999 and GID=999 which is handled differently depending on your operating system or deployment platform. During startup the container temporarily runs as root in order to modify the permissions of `/data`. It then changes to the `algorand` user. This can sometimes cause problems, for example if your deployment platform doesn't allow containers to run as the root user. + +### Use specific UID and GID + +If you do not want the container to start as the root user you can specify a UID and GID. On the host system, ensure the directory being mounted uses UID=999 and GID=999. If the directory already has these permissions you may override the default user with `-u 999:999`. diff --git a/go.sum b/go.sum index 637cec5c..17079d9d 100644 --- a/go.sum +++ b/go.sum @@ -84,7 +84,6 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/algorand/avm-abi v0.1.1/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g= github.com/algorand/avm-abi v0.2.0 h1:bkjsG+BOEcxUcnGSALLosmltE0JZdg+ZisXKx0UDX2k= github.com/algorand/avm-abi v0.2.0/go.mod h1:+CgwM46dithy850bpTeHh9MC99zpn2Snirb3QTl2O/g= -github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230228201805-5b8c99b1412c h1:KAX6gb3+DLCTBcVhjDtuhcdbCeKnwIYKdj5Dv2JA/nI= github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230228201805-5b8c99b1412c/go.mod h1:Nt3EHpP8AznLs0/EFfhr0/xsVf5ucnvjNeRygGgbUzM= github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230324200319-055c8d2b174a h1:fv15GJlyepaaP517PeiJuPX0Q1Wmr17T8uZzevep/TU= github.com/algorand/go-algorand-sdk/v2 v2.0.0-20230324200319-055c8d2b174a/go.mod h1:Nt3EHpP8AznLs0/EFfhr0/xsVf5ucnvjNeRygGgbUzM= @@ -92,8 +91,6 @@ github.com/algorand/go-codec v1.1.8 h1:XDSreeeZY8gMst6Edz4RBkl08/DGMJOeHYkoXL2B7 github.com/algorand/go-codec v1.1.8/go.mod h1:XhzVs6VVyWMLu6cApb9/192gBjGRVGm5cX5j203Heg4= github.com/algorand/go-codec/codec v1.1.8 h1:lsFuhcOH2LiEhpBH3BVUUkdevVmwCRyvb7FCAAPeY6U= github.com/algorand/go-codec/codec v1.1.8/go.mod h1:tQ3zAJ6ijTps6V+wp8KsGDnPC2uhHVC7ANyrtkIY0bA= -github.com/algorand/indexer v0.0.0-20230306212826-146c4d38c5b4 h1:BLzw/1gSbntKblR4ywXdSSxTM/GeKhdkchXNtKUUnzs= -github.com/algorand/indexer v0.0.0-20230306212826-146c4d38c5b4/go.mod h1:ULZ8Qt539rs+FNkSYdoe9HuZ/z1cRAFsWCysylz0nDg= github.com/algorand/indexer v0.0.0-20230315150109-cf0074cfd4ed h1:aZ5FURJNLUmyayj10ahbVuPJtFQ6YBdp0mP3zJz7yyY= github.com/algorand/indexer v0.0.0-20230315150109-cf0074cfd4ed/go.mod h1:ULZ8Qt539rs+FNkSYdoe9HuZ/z1cRAFsWCysylz0nDg= github.com/algorand/oapi-codegen v1.12.0-algorand.0 h1:W9PvED+wAJc+9EeXPONnA+0zE9UhynEqoDs4OgAxKhk=