Skip to content

Commit

Permalink
Docker: make available as ARM64 (#538)
Browse files Browse the repository at this point in the history
* CI: set up arm64 build

* fee_estimates: clean up usage
  • Loading branch information
DeviaVir authored Dec 3, 2024
1 parent 4f5c82f commit 247771c
Show file tree
Hide file tree
Showing 7 changed files with 78 additions and 114 deletions.
26 changes: 19 additions & 7 deletions .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,17 +1,26 @@
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_TLS_CERTDIR: ""
DOCKER_BUILDKIT: 1
CI_DISPOSABLE_ENVIRONMENT: "true"
BULLSEYE_IMAGE: debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c
IMAGE_BASE: blockstream/esplora-base
IMAGE: blockstream/esplora
DOCKERHUB_ESPLORA_URL: "https://hub.docker.com/v2/repositories/blockstream/esplora/tags/"

stages:
- build
- plan
- deploy

default:
image: blockstream/gcloud-docker-tf:1.3.8
image: docker:27
services:
- name: docker:27-dind
command: ["dockerd", "--host=tcp://0.0.0.0:2375"]
alias: "docker"
before_script:
- docker info
- docker buildx create
--driver=docker-container
--name=buildkit-builder
--use
--platform linux/amd64,linux/arm64
tags:
- cloud
retry:
Expand All @@ -21,8 +30,11 @@ default:
- unknown_failure
- stuck_or_timeout_failure

stages:
- build

include:
- gitlab/**.yaml
- "gitlab/**.yml"

## disables MR-triggered pipelines and allows only branch-triggered pipelines
workflow:
Expand Down
4 changes: 1 addition & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM blockstream/esplora-base:latest AS build

FROM debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c
FROM debian:bookworm-slim

COPY --from=build /srv/explorer /srv/explorer
COPY --from=build /srv/wally_wasm /srv/wally_wasm
Expand All @@ -20,9 +20,7 @@ WORKDIR /srv/explorer/source

SHELL ["/bin/bash", "-c"]

# required to run some scripts as root (needed for docker)
RUN source /root/.nvm/nvm.sh \
&& npm config set unsafe-perm true \
&& npm install && (cd prerender-server && npm run dist) \
&& DEST=/srv/explorer/static/bitcoin-mainnet \
npm run dist -- bitcoin-mainnet \
Expand Down
63 changes: 29 additions & 34 deletions Dockerfile.deps
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Build libwally wasm files. Used for client-side blinding verification on Elements-based chains

FROM greenaddress/wallycore@sha256:d63d222be12f6b2535e1548aa7f21cf649e2230d6c9e01bd518c23eb0bccd46f AS libwally-wasm
FROM blockstream/wallycore@sha256:62cc52bd3ad9176b55cb486482368f858a4debee248a15d2fada0a62eb074a05 AS libwally-wasm
ARG NO_LIQUID
ENV EXPORTED_FUNCTIONS="['_malloc', '_free', '_wally_init','_wally_asset_value_commitment','_wally_asset_generator_from_bytes']"
ENV EXTRA_EXPORTED_RUNTIME_METHODS="['getValue', 'ccall']"
Expand All @@ -12,7 +12,7 @@ RUN sh -c '[ -n "$NO_LIQUID" ] && mkdir -p /wally/wally_dist || ( \
&& git submodule sync --recursive && git submodule update --init --recursive \
&& ./tools/build_wasm.sh --enable-elements)'

FROM debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c
FROM debian:bookworm-slim
SHELL ["/bin/bash", "-c"]

ENV CORE_BUILD_DEPS="autoconf autotools-dev automake libtool pkg-config bsdmainutils build-essential"
Expand All @@ -23,42 +23,37 @@ RUN mkdir -p /srv/explorer \
&& apt-get -yqq install ${ESPLORA_BUILD_DEPS} tor ${CORE_BUILD_DEPS}


RUN git clone --quiet --depth 1 --single-branch --branch v0.39.0 https://github.com/nvm-sh/nvm.git /root/.nvm \
RUN git clone --quiet --depth 1 --single-branch --branch v0.40.1 https://github.com/nvm-sh/nvm.git /root/.nvm \
&& rm -rf /root/.nvm/.git \
&& source /root/.nvm/nvm.sh \
&& nvm install v17.1.0
&& nvm install v18.20.4

# Build core from sources until PR https://github.com/bitcoin/bitcoin/pull/23387 is merged
ENV CORE_PATCH=contrib/0001-add-support-to-save-fee-estimates-without-shutting-d.patch
ENV CORE_SRC=/root/bitcoin
COPY ${CORE_PATCH} /${CORE_PATCH}
RUN git clone --quiet --depth 1 --branch v24.1 --single-branch --recursive https://github.com/bitcoin/bitcoin.git ${CORE_SRC} \
&& (cd ${CORE_SRC} \
&& git checkout 3116ccd790e76de8f64d2ef9aa5a2641c15bbd8b \
&& git apply /${CORE_PATCH} \
&& (cd depends \
&& make HOST=x86_64-pc-linux-gnu NO_QT=1 -j $(nproc --all)) \
&& ./autogen.sh \
&& CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure --prefix=/srv/explorer/bitcoin --disable-man --disable-zmq --disable-qt --disable-gui-tests --disable-bench \
--enable-experimental-asm --without-utils --enable-util-cli --without-libs --with-daemon --disable-maintainer-mode \
--disable-glibc-back-compat --disable-ccache --disable-dependency-tracking --disable-tests --with-gui=no \
&& make -j $(nproc --all) \
&& make install -j $(nproc --all) \
&& strip /srv/explorer/bitcoin/bin/* \
&& rm -fr ${CORE_SRC} /${CORE_PATCH})
ARG TARGETARCH
ENV SHA256SUM_ELEMENTS_X86_64="d1c0919955a1d50f2f1518e2ee27aa21fd5dd45cd481635d59fa453ff776dede"
ENV SHA256SUM_ELEMENTS_ARM64="b6345394020486f7f8af111e2063f355571488cd8e8f384ed1082606c94e6c0d"
ENV SHA256SUM_BITCOINCORE_X86_64="2a6974c5486f528793c79d42694b5987401e4a43c97f62b1383abf35bcee44a8"
ENV SHA256SUM_BITCOINCORE_ARM64="cb35e250ae9d0328aa90e7aad0b877ed692597420a1092e8ab1a5dd756209722"

#ENV SHA256SUM_BITCOINCORE=59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16
#ENV VERSION_BITCOINCORE=22.0
#RUN curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-x86_64-linux-gnu.tar.gz" \
# && echo "${SHA256SUM_BITCOINCORE} bitcoin.tar.gz" | sha256sum --check \
# && tar xzf bitcoin.tar.gz -C /srv/explorer \
# && ln -s "/srv/explorer/bitcoin-${VERSION_BITCOINCORE}" /srv/explorer/bitcoin \
# && rm bitcoin.tar.gz
ENV VERSION_BITCOINCORE=27.0
RUN if [ "${TARGETARCH}" = "arm64" ]; then \
curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-aarch64-linux-gnu.tar.gz"; \
echo "${SHA256SUM_BITCOINCORE_ARM64} bitcoin.tar.gz" | sha256sum --check; \
else \
curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-x86_64-linux-gnu.tar.gz"; \
echo "${SHA256SUM_BITCOINCORE_X86_64} bitcoin.tar.gz" | sha256sum --check; \
fi \
&& tar xzf bitcoin.tar.gz -C /srv/explorer \
&& ln -s "/srv/explorer/bitcoin-${VERSION_BITCOINCORE}" /srv/explorer/bitcoin \
&& rm bitcoin.tar.gz

ENV SHA256SUM_ELEMENTS=67aca46c12b8a81c960798502f81463b18c54d3ebd5c0804b26f1a336d148042
ENV VERSION_ELEMENTS=22.1.1
RUN curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-x86_64-linux-gnu.tar.gz" \
&& echo "${SHA256SUM_ELEMENTS} elements.tar.gz" | sha256sum --check \
ENV VERSION_ELEMENTS=23.2.3
RUN if [ "${TARGETARCH}" = "arm64" ]; then \
curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-aarch64-linux-gnu.tar.gz"; \
echo "${SHA256SUM_ELEMENTS_ARM64} elements.tar.gz" | sha256sum --check; \
else \
curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-x86_64-linux-gnu.tar.gz"; \
echo "${SHA256SUM_ELEMENTS_X86_64} elements.tar.gz" | sha256sum --check; \
fi \
&& tar xzf elements.tar.gz -C /srv/explorer \
&& ln -s "/srv/explorer/elements-${VERSION_ELEMENTS}" /srv/explorer/liquid \
&& mv /srv/explorer/liquid/bin/{elementsd,liquidd} \
Expand Down Expand Up @@ -94,4 +89,4 @@ RUN apt-get --auto-remove remove -yqq --purge ${ESPLORA_BUILD_DEPS} manpages ${C
&& apt-get autoclean \
&& rm -rf /usr/share/doc* /usr/share/man /usr/share/postgresql/*/man /var/lib/apt/lists/* /var/cache/* /tmp/* /root/.cache /*.deb /root/.cargo

COPY --from=libwally-wasm /wally/wally_dist /srv/wally_wasm
COPY --from=libwally-wasm /wally/wally_dist /srv/wally_wasm
21 changes: 1 addition & 20 deletions contrib/nginx-sync.conf.in
Original file line number Diff line number Diff line change
Expand Up @@ -9,29 +9,10 @@ location = /{NGINX_PATH}_sync/mempool {
}
}

# Dump the current mempool and return it
location = /{NGINX_PATH}_sync/fee_estimates {
auth_basic "private";
auth_basic_user_file /srv/explorer/htpasswd;

content_by_lua_block {
os.execute("/usr/bin/cli savefeeestimates");
ngx.exec("/{NGINX_PATH}_sync/fee_estimates.dat");
}
}

# Return mempool.dat without dumping it first, used as an internal redirect
# from the location block above
location = /{NGINX_PATH}_sync/mempool.dat {
auth_basic "private";
auth_basic_user_file /srv/explorer/htpasswd;
alias {DAEMON_DIR}/mempool.dat;
}

# Return fee_estimates.dat without dumping it first, used as an internal redirect
# from the location block above
location = /{NGINX_PATH}_sync/fee_estimates.dat {
auth_basic "private";
auth_basic_user_file /srv/explorer/htpasswd;
alias {DAEMON_DIR}/fee_estimates.dat;
}
}
61 changes: 27 additions & 34 deletions gitlab/build.yaml → gitlab/build.yml
Original file line number Diff line number Diff line change
@@ -1,51 +1,44 @@
build_base:
stage: build
rules:
- if: $CI_COMMIT_BRANCH
changes:
paths:
- Dockerfile.deps
compare_to: master
script:
- docker buildx build
--platform linux/amd64,linux/arm64
--push
-f Dockerfile.deps
--build-arg BUILDKIT_INLINE_CACHE=1
--cache-from ${IMAGE_BASE}:latest
-t ${IMAGE_BASE}:$CI_COMMIT_SHA
-t ${IMAGE_BASE}:latest
.

test_docker_build_esplora:
extends: .build_setup
stage: build
script:
- docker pull ${IMAGE}:latest || true
- docker build
--network=host
- docker buildx build
--platform linux/amd64,linux/arm64
--build-arg BUILDKIT_INLINE_CACHE=1
--cache-from ${IMAGE}:latest
-f Dockerfile
-t ${IMAGE} .
- docker rmi ${IMAGE} || true

build_esplora:
extends: .build_setup
stage: build
when: manual
script:
- curl -s "${DOCKERHUB_ESPLORA_URL}" | grep -q "$CI_COMMIT_SHA" || (
sed -i "s#esplora-base:latest#esplora-base:${BASE_TAG}#" Dockerfile
&& docker pull ${IMAGE}:latest
&& docker build
--network=host
&& docker buildx build
--platform linux/amd64,linux/arm64
--push
--build-arg BUILDKIT_INLINE_CACHE=1
--build-arg FOOT_HTML='<!-- '"$CI_COMMIT_SHA"' -->'
--cache-from ${IMAGE}:latest
-t ${IMAGE}:latest
-t ${IMAGE}:$CI_COMMIT_SHA .
&& docker push ${IMAGE}:$CI_COMMIT_SHA)
-t ${IMAGE}:$CI_COMMIT_SHA .)
- if [ $CI_COMMIT_BRANCH == "master" ]; then docker pull ${IMAGE}:$CI_COMMIT_SHA; docker tag ${IMAGE}:$CI_COMMIT_SHA ${IMAGE}:latest; docker push ${IMAGE}:latest; fi

build_base:
extends: .build_setup
rules:
- if: $CI_COMMIT_BRANCH
changes:
paths:
- Dockerfile.deps
compare_to: master
script:
- docker pull ${BULLSEYE_IMAGE} || true
- docker pull ${IMAGE_BASE}:latest || true
- >
docker build
-f Dockerfile.deps
--network=host
--build-arg BUILDKIT_INLINE_CACHE=1
--cache-from ${IMAGE_BASE}:latest
-t ${IMAGE_BASE}:$CI_COMMIT_SHA
-t ${IMAGE_BASE}:latest
.
- docker push ${IMAGE_BASE}:$CI_COMMIT_SHA
- docker push ${IMAGE_BASE}:latest
14 changes: 0 additions & 14 deletions gitlab/functions.yaml

This file was deleted.

3 changes: 1 addition & 2 deletions run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -298,8 +298,7 @@ if [ -n "$SYNC_SOURCE" ]; then
sleep 2 # without this, the download below would occasionally start while the terminating bitcoind is still flushing its mempool.dat
# then fetch a recent mempool.dat,
curl -v -u sync:$SYNC_SECRET -o $DAEMON_DIR/mempool.dat $SYNC_SOURCE/mempool || true
curl -v -u sync:$SYNC_SECRET -o $DAEMON_DIR/fee_estimates.dat $SYNC_SOURCE/fee_estimates || true
ls -l $DAEMON_DIR/{mempool,fee_estimates}.dat || true
ls -l $DAEMON_DIR/{mempool}.dat || true
# and let the runit services take over
fi

Expand Down

0 comments on commit 247771c

Please sign in to comment.