diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 169333c1..b1c8875a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,17 +1,26 @@ variables: + DOCKER_HOST: tcp://docker:2375 + DOCKER_TLS_CERTDIR: "" + DOCKER_BUILDKIT: 1 CI_DISPOSABLE_ENVIRONMENT: "true" BULLSEYE_IMAGE: debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c IMAGE_BASE: blockstream/esplora-base IMAGE: blockstream/esplora DOCKERHUB_ESPLORA_URL: "https://hub.docker.com/v2/repositories/blockstream/esplora/tags/" -stages: - - build - - plan - - deploy - default: - image: blockstream/gcloud-docker-tf:1.3.8 + image: docker:27 + services: + - name: docker:27-dind + command: ["dockerd", "--host=tcp://0.0.0.0:2375"] + alias: "docker" + before_script: + - docker info + - docker buildx create + --driver=docker-container + --name=buildkit-builder + --use + --platform linux/amd64,linux/arm64 tags: - cloud retry: @@ -21,8 +30,11 @@ default: - unknown_failure - stuck_or_timeout_failure +stages: + - build + include: - - gitlab/**.yaml + - "gitlab/**.yml" ## disables MR-triggered pipelines and allows only branch-triggered pipelines workflow: diff --git a/Dockerfile b/Dockerfile index 76daab51..6f7b63a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM blockstream/esplora-base:latest AS build -FROM debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c +FROM debian:bookworm-slim COPY --from=build /srv/explorer /srv/explorer COPY --from=build /srv/wally_wasm /srv/wally_wasm @@ -20,9 +20,7 @@ WORKDIR /srv/explorer/source SHELL ["/bin/bash", "-c"] -# required to run some scripts as root (needed for docker) RUN source /root/.nvm/nvm.sh \ - && npm config set unsafe-perm true \ && npm install && (cd prerender-server && npm run dist) \ && DEST=/srv/explorer/static/bitcoin-mainnet \ npm run dist -- bitcoin-mainnet \ diff --git a/Dockerfile.deps b/Dockerfile.deps index 9732c9c0..b5ace6a7 100644 --- a/Dockerfile.deps +++ b/Dockerfile.deps @@ -1,6 +1,6 @@ # Build libwally wasm files. Used for client-side blinding verification on Elements-based chains -FROM greenaddress/wallycore@sha256:d63d222be12f6b2535e1548aa7f21cf649e2230d6c9e01bd518c23eb0bccd46f AS libwally-wasm +FROM blockstream/wallycore@sha256:62cc52bd3ad9176b55cb486482368f858a4debee248a15d2fada0a62eb074a05 AS libwally-wasm ARG NO_LIQUID ENV EXPORTED_FUNCTIONS="['_malloc', '_free', '_wally_init','_wally_asset_value_commitment','_wally_asset_generator_from_bytes']" ENV EXTRA_EXPORTED_RUNTIME_METHODS="['getValue', 'ccall']" @@ -12,7 +12,7 @@ RUN sh -c '[ -n "$NO_LIQUID" ] && mkdir -p /wally/wally_dist || ( \ && git submodule sync --recursive && git submodule update --init --recursive \ && ./tools/build_wasm.sh --enable-elements)' -FROM debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c +FROM debian:bookworm-slim SHELL ["/bin/bash", "-c"] ENV CORE_BUILD_DEPS="autoconf autotools-dev automake libtool pkg-config bsdmainutils build-essential" @@ -23,42 +23,37 @@ RUN mkdir -p /srv/explorer \ && apt-get -yqq install ${ESPLORA_BUILD_DEPS} tor ${CORE_BUILD_DEPS} -RUN git clone --quiet --depth 1 --single-branch --branch v0.39.0 https://github.com/nvm-sh/nvm.git /root/.nvm \ +RUN git clone --quiet --depth 1 --single-branch --branch v0.40.1 https://github.com/nvm-sh/nvm.git /root/.nvm \ && rm -rf /root/.nvm/.git \ && source /root/.nvm/nvm.sh \ - && nvm install v17.1.0 + && nvm install v18.20.4 -# Build core from sources until PR https://github.com/bitcoin/bitcoin/pull/23387 is merged -ENV CORE_PATCH=contrib/0001-add-support-to-save-fee-estimates-without-shutting-d.patch -ENV CORE_SRC=/root/bitcoin -COPY ${CORE_PATCH} /${CORE_PATCH} -RUN git clone --quiet --depth 1 --branch v24.1 --single-branch --recursive https://github.com/bitcoin/bitcoin.git ${CORE_SRC} \ - && (cd ${CORE_SRC} \ - && git checkout 3116ccd790e76de8f64d2ef9aa5a2641c15bbd8b \ - && git apply /${CORE_PATCH} \ - && (cd depends \ - && make HOST=x86_64-pc-linux-gnu NO_QT=1 -j $(nproc --all)) \ - && ./autogen.sh \ - && CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure --prefix=/srv/explorer/bitcoin --disable-man --disable-zmq --disable-qt --disable-gui-tests --disable-bench \ - --enable-experimental-asm --without-utils --enable-util-cli --without-libs --with-daemon --disable-maintainer-mode \ - --disable-glibc-back-compat --disable-ccache --disable-dependency-tracking --disable-tests --with-gui=no \ - && make -j $(nproc --all) \ - && make install -j $(nproc --all) \ - && strip /srv/explorer/bitcoin/bin/* \ - && rm -fr ${CORE_SRC} /${CORE_PATCH}) +ARG TARGETARCH +ENV SHA256SUM_ELEMENTS_X86_64="d1c0919955a1d50f2f1518e2ee27aa21fd5dd45cd481635d59fa453ff776dede" +ENV SHA256SUM_ELEMENTS_ARM64="b6345394020486f7f8af111e2063f355571488cd8e8f384ed1082606c94e6c0d" +ENV SHA256SUM_BITCOINCORE_X86_64="2a6974c5486f528793c79d42694b5987401e4a43c97f62b1383abf35bcee44a8" +ENV SHA256SUM_BITCOINCORE_ARM64="cb35e250ae9d0328aa90e7aad0b877ed692597420a1092e8ab1a5dd756209722" -#ENV SHA256SUM_BITCOINCORE=59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16 -#ENV VERSION_BITCOINCORE=22.0 -#RUN curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-x86_64-linux-gnu.tar.gz" \ -# && echo "${SHA256SUM_BITCOINCORE} bitcoin.tar.gz" | sha256sum --check \ -# && tar xzf bitcoin.tar.gz -C /srv/explorer \ -# && ln -s "/srv/explorer/bitcoin-${VERSION_BITCOINCORE}" /srv/explorer/bitcoin \ -# && rm bitcoin.tar.gz +ENV VERSION_BITCOINCORE=27.0 +RUN if [ "${TARGETARCH}" = "arm64" ]; then \ + curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-aarch64-linux-gnu.tar.gz"; \ + echo "${SHA256SUM_BITCOINCORE_ARM64} bitcoin.tar.gz" | sha256sum --check; \ + else \ + curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-x86_64-linux-gnu.tar.gz"; \ + echo "${SHA256SUM_BITCOINCORE_X86_64} bitcoin.tar.gz" | sha256sum --check; \ + fi \ + && tar xzf bitcoin.tar.gz -C /srv/explorer \ + && ln -s "/srv/explorer/bitcoin-${VERSION_BITCOINCORE}" /srv/explorer/bitcoin \ + && rm bitcoin.tar.gz -ENV SHA256SUM_ELEMENTS=67aca46c12b8a81c960798502f81463b18c54d3ebd5c0804b26f1a336d148042 -ENV VERSION_ELEMENTS=22.1.1 -RUN curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-x86_64-linux-gnu.tar.gz" \ - && echo "${SHA256SUM_ELEMENTS} elements.tar.gz" | sha256sum --check \ +ENV VERSION_ELEMENTS=23.2.3 +RUN if [ "${TARGETARCH}" = "arm64" ]; then \ + curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-aarch64-linux-gnu.tar.gz"; \ + echo "${SHA256SUM_ELEMENTS_ARM64} elements.tar.gz" | sha256sum --check; \ + else \ + curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-x86_64-linux-gnu.tar.gz"; \ + echo "${SHA256SUM_ELEMENTS_X86_64} elements.tar.gz" | sha256sum --check; \ + fi \ && tar xzf elements.tar.gz -C /srv/explorer \ && ln -s "/srv/explorer/elements-${VERSION_ELEMENTS}" /srv/explorer/liquid \ && mv /srv/explorer/liquid/bin/{elementsd,liquidd} \ @@ -94,4 +89,4 @@ RUN apt-get --auto-remove remove -yqq --purge ${ESPLORA_BUILD_DEPS} manpages ${C && apt-get autoclean \ && rm -rf /usr/share/doc* /usr/share/man /usr/share/postgresql/*/man /var/lib/apt/lists/* /var/cache/* /tmp/* /root/.cache /*.deb /root/.cargo -COPY --from=libwally-wasm /wally/wally_dist /srv/wally_wasm +COPY --from=libwally-wasm /wally/wally_dist /srv/wally_wasm \ No newline at end of file diff --git a/contrib/nginx-sync.conf.in b/contrib/nginx-sync.conf.in index 9e65423a..af243e45 100644 --- a/contrib/nginx-sync.conf.in +++ b/contrib/nginx-sync.conf.in @@ -9,29 +9,10 @@ location = /{NGINX_PATH}_sync/mempool { } } -# Dump the current mempool and return it -location = /{NGINX_PATH}_sync/fee_estimates { - auth_basic "private"; - auth_basic_user_file /srv/explorer/htpasswd; - - content_by_lua_block { - os.execute("/usr/bin/cli savefeeestimates"); - ngx.exec("/{NGINX_PATH}_sync/fee_estimates.dat"); - } -} - # Return mempool.dat without dumping it first, used as an internal redirect # from the location block above location = /{NGINX_PATH}_sync/mempool.dat { auth_basic "private"; auth_basic_user_file /srv/explorer/htpasswd; alias {DAEMON_DIR}/mempool.dat; -} - -# Return fee_estimates.dat without dumping it first, used as an internal redirect -# from the location block above -location = /{NGINX_PATH}_sync/fee_estimates.dat { - auth_basic "private"; - auth_basic_user_file /srv/explorer/htpasswd; - alias {DAEMON_DIR}/fee_estimates.dat; -} +} \ No newline at end of file diff --git a/gitlab/build.yaml b/gitlab/build.yml similarity index 63% rename from gitlab/build.yaml rename to gitlab/build.yml index 6b5f0105..6df71c86 100644 --- a/gitlab/build.yaml +++ b/gitlab/build.yml @@ -1,51 +1,44 @@ +build_base: + stage: build + rules: + - if: $CI_COMMIT_BRANCH + changes: + paths: + - Dockerfile.deps + compare_to: master + script: + - docker buildx build + --platform linux/amd64,linux/arm64 + --push + -f Dockerfile.deps + --build-arg BUILDKIT_INLINE_CACHE=1 + --cache-from ${IMAGE_BASE}:latest + -t ${IMAGE_BASE}:$CI_COMMIT_SHA + -t ${IMAGE_BASE}:latest + . + test_docker_build_esplora: - extends: .build_setup + stage: build script: - - docker pull ${IMAGE}:latest || true - - docker build - --network=host + - docker buildx build + --platform linux/amd64,linux/arm64 --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from ${IMAGE}:latest -f Dockerfile -t ${IMAGE} . - - docker rmi ${IMAGE} || true build_esplora: - extends: .build_setup + stage: build when: manual script: - curl -s "${DOCKERHUB_ESPLORA_URL}" | grep -q "$CI_COMMIT_SHA" || ( sed -i "s#esplora-base:latest#esplora-base:${BASE_TAG}#" Dockerfile - && docker pull ${IMAGE}:latest - && docker build - --network=host + && docker buildx build + --platform linux/amd64,linux/arm64 + --push --build-arg BUILDKIT_INLINE_CACHE=1 --build-arg FOOT_HTML='' --cache-from ${IMAGE}:latest -t ${IMAGE}:latest - -t ${IMAGE}:$CI_COMMIT_SHA . - && docker push ${IMAGE}:$CI_COMMIT_SHA) + -t ${IMAGE}:$CI_COMMIT_SHA .) - if [ $CI_COMMIT_BRANCH == "master" ]; then docker pull ${IMAGE}:$CI_COMMIT_SHA; docker tag ${IMAGE}:$CI_COMMIT_SHA ${IMAGE}:latest; docker push ${IMAGE}:latest; fi - -build_base: - extends: .build_setup - rules: - - if: $CI_COMMIT_BRANCH - changes: - paths: - - Dockerfile.deps - compare_to: master - script: - - docker pull ${BULLSEYE_IMAGE} || true - - docker pull ${IMAGE_BASE}:latest || true - - > - docker build - -f Dockerfile.deps - --network=host - --build-arg BUILDKIT_INLINE_CACHE=1 - --cache-from ${IMAGE_BASE}:latest - -t ${IMAGE_BASE}:$CI_COMMIT_SHA - -t ${IMAGE_BASE}:latest - . - - docker push ${IMAGE_BASE}:$CI_COMMIT_SHA - - docker push ${IMAGE_BASE}:latest diff --git a/gitlab/functions.yaml b/gitlab/functions.yaml deleted file mode 100644 index 50ccdc9f..00000000 --- a/gitlab/functions.yaml +++ /dev/null @@ -1,14 +0,0 @@ -.dind_setup: - variables: - DOCKER_DRIVER: overlay2 - DOCKER_HOST: tcp://localhost:2375 - DOCKER_TLS_CERTDIR: "" - DOCKER_BUILDKIT: 1 - services: - - docker:20.10.12-dind - -.build_setup: - stage: build - extends: .dind_setup - rules: - - if: $CI_COMMIT_BRANCH diff --git a/run.sh b/run.sh index a8a57b98..cebacfdc 100755 --- a/run.sh +++ b/run.sh @@ -298,8 +298,7 @@ if [ -n "$SYNC_SOURCE" ]; then sleep 2 # without this, the download below would occasionally start while the terminating bitcoind is still flushing its mempool.dat # then fetch a recent mempool.dat, curl -v -u sync:$SYNC_SECRET -o $DAEMON_DIR/mempool.dat $SYNC_SOURCE/mempool || true - curl -v -u sync:$SYNC_SECRET -o $DAEMON_DIR/fee_estimates.dat $SYNC_SOURCE/fee_estimates || true - ls -l $DAEMON_DIR/{mempool,fee_estimates}.dat || true + ls -l $DAEMON_DIR/{mempool}.dat || true # and let the runit services take over fi