Docker: make available as ARM64 (#538)

* CI: set up arm64 build

* fee_estimates: clean up usage

Author: DeviaVir

.gitlab-ci.yml

@@ -1,17 +1,26 @@
 variables:
+  DOCKER_HOST: tcp://docker:2375
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_BUILDKIT: 1
   CI_DISPOSABLE_ENVIRONMENT: "true"
   BULLSEYE_IMAGE: debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c
   IMAGE_BASE: blockstream/esplora-base
   IMAGE: blockstream/esplora
   DOCKERHUB_ESPLORA_URL: "https://hub.docker.com/v2/repositories/blockstream/esplora/tags/"
 
-stages:
-  - build
-  - plan
-  - deploy
-
 default:
-  image: blockstream/gcloud-docker-tf:1.3.8
+  image: docker:27
+  services:
+    - name: docker:27-dind
+      command: ["dockerd", "--host=tcp://0.0.0.0:2375"]
+      alias: "docker"
+  before_script:
+    - docker info
+    - docker buildx create
+      --driver=docker-container
+      --name=buildkit-builder
+      --use
+      --platform linux/amd64,linux/arm64
   tags:
     - cloud
   retry:
@@ -21,8 +30,11 @@ default:
       - unknown_failure
       - stuck_or_timeout_failure
 
+stages:
+  - build
+
 include:
-  - gitlab/**.yaml
+  - "gitlab/**.yml"
 
 ## disables MR-triggered pipelines and allows only branch-triggered pipelines
 workflow:

Dockerfile

@@ -1,6 +1,6 @@
 FROM blockstream/esplora-base:latest AS build
 
-FROM debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c
+FROM debian:bookworm-slim
 
 COPY --from=build /srv/explorer /srv/explorer
 COPY --from=build /srv/wally_wasm /srv/wally_wasm
@@ -20,9 +20,7 @@ WORKDIR /srv/explorer/source
 
 SHELL ["/bin/bash", "-c"]
 
-# required to run some scripts as root (needed for docker)
 RUN source /root/.nvm/nvm.sh \
- && npm config set unsafe-perm true \
  && npm install && (cd prerender-server && npm run dist) \
  && DEST=/srv/explorer/static/bitcoin-mainnet \
     npm run dist -- bitcoin-mainnet \

Dockerfile.deps

@@ -1,6 +1,6 @@
 # Build libwally wasm files. Used for client-side blinding verification on Elements-based chains
 
-FROM greenaddress/wallycore@sha256:d63d222be12f6b2535e1548aa7f21cf649e2230d6c9e01bd518c23eb0bccd46f AS libwally-wasm
+FROM blockstream/wallycore@sha256:62cc52bd3ad9176b55cb486482368f858a4debee248a15d2fada0a62eb074a05 AS libwally-wasm
 ARG NO_LIQUID
 ENV EXPORTED_FUNCTIONS="['_malloc', '_free', '_wally_init','_wally_asset_value_commitment','_wally_asset_generator_from_bytes']"
 ENV EXTRA_EXPORTED_RUNTIME_METHODS="['getValue', 'ccall']"
@@ -12,7 +12,7 @@ RUN sh -c '[ -n "$NO_LIQUID" ] && mkdir -p /wally/wally_dist || ( \
     && git submodule sync --recursive && git submodule update --init --recursive \
     && ./tools/build_wasm.sh --enable-elements)'
 
-FROM debian:bullseye@sha256:4d6ab716de467aad58e91b1b720f0badd7478847ec7a18f66027d0f8a329a43c
+FROM debian:bookworm-slim
 SHELL ["/bin/bash", "-c"]
 
 ENV CORE_BUILD_DEPS="autoconf autotools-dev automake libtool pkg-config bsdmainutils build-essential"
@@ -23,42 +23,37 @@ RUN mkdir -p /srv/explorer \
  && apt-get -yqq install ${ESPLORA_BUILD_DEPS} tor ${CORE_BUILD_DEPS}
 
 
-RUN git clone --quiet --depth 1 --single-branch --branch v0.39.0 https://github.com/nvm-sh/nvm.git /root/.nvm \
+RUN git clone --quiet --depth 1 --single-branch --branch v0.40.1 https://github.com/nvm-sh/nvm.git /root/.nvm \
  && rm -rf /root/.nvm/.git \
  && source /root/.nvm/nvm.sh \
- && nvm install v17.1.0
+ && nvm install v18.20.4
 
-# Build core from sources until PR https://github.com/bitcoin/bitcoin/pull/23387 is merged
-ENV CORE_PATCH=contrib/0001-add-support-to-save-fee-estimates-without-shutting-d.patch
-ENV CORE_SRC=/root/bitcoin
-COPY ${CORE_PATCH} /${CORE_PATCH}
-RUN git clone --quiet --depth 1 --branch v24.1 --single-branch --recursive https://github.com/bitcoin/bitcoin.git ${CORE_SRC} \
- && (cd ${CORE_SRC} \
- && git checkout 3116ccd790e76de8f64d2ef9aa5a2641c15bbd8b \
- && git apply /${CORE_PATCH} \
- && (cd depends \
- && make HOST=x86_64-pc-linux-gnu NO_QT=1 -j $(nproc --all)) \
- && ./autogen.sh \
- && CONFIG_SITE=$PWD/depends/x86_64-pc-linux-gnu/share/config.site ./configure --prefix=/srv/explorer/bitcoin --disable-man --disable-zmq --disable-qt --disable-gui-tests --disable-bench \
-                --enable-experimental-asm --without-utils --enable-util-cli --without-libs --with-daemon --disable-maintainer-mode \
-                --disable-glibc-back-compat --disable-ccache --disable-dependency-tracking --disable-tests --with-gui=no \
- && make -j $(nproc --all) \
- && make install -j $(nproc --all) \
- && strip /srv/explorer/bitcoin/bin/* \
- && rm -fr ${CORE_SRC} /${CORE_PATCH})
+ARG TARGETARCH
+ENV SHA256SUM_ELEMENTS_X86_64="d1c0919955a1d50f2f1518e2ee27aa21fd5dd45cd481635d59fa453ff776dede"
+ENV SHA256SUM_ELEMENTS_ARM64="b6345394020486f7f8af111e2063f355571488cd8e8f384ed1082606c94e6c0d"
+ENV SHA256SUM_BITCOINCORE_X86_64="2a6974c5486f528793c79d42694b5987401e4a43c97f62b1383abf35bcee44a8"
+ENV SHA256SUM_BITCOINCORE_ARM64="cb35e250ae9d0328aa90e7aad0b877ed692597420a1092e8ab1a5dd756209722"
 
-#ENV SHA256SUM_BITCOINCORE=59ebd25dd82a51638b7a6bb914586201e67db67b919b2a1ff08925a7936d1b16
-#ENV VERSION_BITCOINCORE=22.0
-#RUN curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-x86_64-linux-gnu.tar.gz" \
-# && echo "${SHA256SUM_BITCOINCORE}  bitcoin.tar.gz" | sha256sum --check \
-# && tar xzf bitcoin.tar.gz -C /srv/explorer \
-# && ln -s "/srv/explorer/bitcoin-${VERSION_BITCOINCORE}" /srv/explorer/bitcoin \
-# && rm bitcoin.tar.gz
+ENV VERSION_BITCOINCORE=27.0
+RUN if [ "${TARGETARCH}" = "arm64" ]; then \
+   curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-aarch64-linux-gnu.tar.gz"; \
+   echo "${SHA256SUM_BITCOINCORE_ARM64}  bitcoin.tar.gz" | sha256sum --check; \
+   else \
+   curl -sL -o bitcoin.tar.gz "https://bitcoincore.org/bin/bitcoin-core-${VERSION_BITCOINCORE}/bitcoin-${VERSION_BITCOINCORE}-x86_64-linux-gnu.tar.gz"; \
+   echo "${SHA256SUM_BITCOINCORE_X86_64}  bitcoin.tar.gz" | sha256sum --check; \
+   fi \
+ && tar xzf bitcoin.tar.gz -C /srv/explorer \
+ && ln -s "/srv/explorer/bitcoin-${VERSION_BITCOINCORE}" /srv/explorer/bitcoin \
+ && rm bitcoin.tar.gz
 
-ENV SHA256SUM_ELEMENTS=67aca46c12b8a81c960798502f81463b18c54d3ebd5c0804b26f1a336d148042
-ENV VERSION_ELEMENTS=22.1.1
-RUN curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-x86_64-linux-gnu.tar.gz" \
- && echo "${SHA256SUM_ELEMENTS}  elements.tar.gz" | sha256sum --check \
+ENV VERSION_ELEMENTS=23.2.3
+RUN if [ "${TARGETARCH}" = "arm64" ]; then \
+   curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-aarch64-linux-gnu.tar.gz"; \
+   echo "${SHA256SUM_ELEMENTS_ARM64}  elements.tar.gz" | sha256sum --check; \
+   else \
+   curl -sL -o elements.tar.gz "https://github.com/ElementsProject/elements/releases/download/elements-${VERSION_ELEMENTS}/elements-${VERSION_ELEMENTS}-x86_64-linux-gnu.tar.gz"; \
+   echo "${SHA256SUM_ELEMENTS_X86_64}  elements.tar.gz" | sha256sum --check; \
+   fi \
  && tar xzf elements.tar.gz -C /srv/explorer \
  && ln -s "/srv/explorer/elements-${VERSION_ELEMENTS}" /srv/explorer/liquid \
  && mv /srv/explorer/liquid/bin/{elementsd,liquidd} \
@@ -94,4 +89,4 @@ RUN apt-get --auto-remove remove -yqq --purge ${ESPLORA_BUILD_DEPS} manpages ${C
  && apt-get autoclean \
  && rm -rf /usr/share/doc* /usr/share/man /usr/share/postgresql/*/man /var/lib/apt/lists/* /var/cache/* /tmp/* /root/.cache /*.deb /root/.cargo
 
-COPY --from=libwally-wasm /wally/wally_dist /srv/wally_wasm
+COPY --from=libwally-wasm /wally/wally_dist /srv/wally_wasm

contrib/nginx-sync.conf.in

@@ -9,29 +9,10 @@ location = /{NGINX_PATH}_sync/mempool {
     }
 }
 
-# Dump the current mempool and return it
-location = /{NGINX_PATH}_sync/fee_estimates {
-    auth_basic "private";
-    auth_basic_user_file /srv/explorer/htpasswd;
-
-    content_by_lua_block {
-        os.execute("/usr/bin/cli savefeeestimates");
-        ngx.exec("/{NGINX_PATH}_sync/fee_estimates.dat");
-    }
-}
-
 # Return mempool.dat without dumping it first, used as an internal redirect
 # from the location block above
 location = /{NGINX_PATH}_sync/mempool.dat {
     auth_basic "private";
     auth_basic_user_file /srv/explorer/htpasswd;
     alias {DAEMON_DIR}/mempool.dat;
-}
-
-# Return fee_estimates.dat without dumping it first, used as an internal redirect
-# from the location block above
-location = /{NGINX_PATH}_sync/fee_estimates.dat {
-    auth_basic "private";
-    auth_basic_user_file /srv/explorer/htpasswd;
-    alias {DAEMON_DIR}/fee_estimates.dat;
-}
+}

gitlab/build.yaml gitlab/build.yml

@@ -1,51 +1,44 @@
+build_base:
+  stage: build
+  rules:
+    - if: $CI_COMMIT_BRANCH
+      changes:
+        paths:
+          - Dockerfile.deps
+        compare_to: master
+  script:
+    - docker buildx build
+      --platform linux/amd64,linux/arm64
+      --push
+      -f Dockerfile.deps
+      --build-arg BUILDKIT_INLINE_CACHE=1
+      --cache-from ${IMAGE_BASE}:latest
+      -t ${IMAGE_BASE}:$CI_COMMIT_SHA
+      -t ${IMAGE_BASE}:latest
+      .
+
 test_docker_build_esplora:
-  extends: .build_setup
+  stage: build
   script:
-    - docker pull ${IMAGE}:latest || true
-    - docker build
-      --network=host
+    - docker buildx build
+      --platform linux/amd64,linux/arm64
       --build-arg BUILDKIT_INLINE_CACHE=1
       --cache-from ${IMAGE}:latest
       -f Dockerfile
       -t ${IMAGE} .
-    - docker rmi ${IMAGE} || true
 
 build_esplora:
-  extends: .build_setup
+  stage: build
   when: manual
   script:
     - curl -s "${DOCKERHUB_ESPLORA_URL}" | grep -q "$CI_COMMIT_SHA" || (
       sed -i "s#esplora-base:latest#esplora-base:${BASE_TAG}#" Dockerfile
-      && docker pull ${IMAGE}:latest
-      && docker build
-      --network=host
+      && docker buildx build
+      --platform linux/amd64,linux/arm64
+      --push
       --build-arg BUILDKIT_INLINE_CACHE=1
       --build-arg FOOT_HTML='<!-- '"$CI_COMMIT_SHA"' -->'
       --cache-from ${IMAGE}:latest
       -t ${IMAGE}:latest
-      -t ${IMAGE}:$CI_COMMIT_SHA .
-      && docker push ${IMAGE}:$CI_COMMIT_SHA)
+      -t ${IMAGE}:$CI_COMMIT_SHA .)
     - if [ $CI_COMMIT_BRANCH == "master" ]; then docker pull ${IMAGE}:$CI_COMMIT_SHA; docker tag ${IMAGE}:$CI_COMMIT_SHA ${IMAGE}:latest; docker push ${IMAGE}:latest; fi
-
-build_base:
-  extends: .build_setup
-  rules:
-    - if: $CI_COMMIT_BRANCH
-      changes:
-        paths:
-          - Dockerfile.deps
-        compare_to: master
-  script:
-    - docker pull ${BULLSEYE_IMAGE} || true
-    - docker pull ${IMAGE_BASE}:latest || true
-    - >
-      docker build
-      -f Dockerfile.deps
-      --network=host
-      --build-arg BUILDKIT_INLINE_CACHE=1
-      --cache-from ${IMAGE_BASE}:latest
-      -t ${IMAGE_BASE}:$CI_COMMIT_SHA
-      -t ${IMAGE_BASE}:latest
-      .
-    - docker push ${IMAGE_BASE}:$CI_COMMIT_SHA
-    - docker push ${IMAGE_BASE}:latest

gitlab/functions.yaml

@@ -298,8 +298,7 @@ if [ -n "$SYNC_SOURCE" ]; then
   sleep 2 # without this, the download below would occasionally start while the terminating bitcoind is still flushing its mempool.dat
   # then fetch a recent mempool.dat,
   curl -v -u sync:$SYNC_SECRET -o $DAEMON_DIR/mempool.dat $SYNC_SOURCE/mempool || true
-  curl -v -u sync:$SYNC_SECRET -o $DAEMON_DIR/fee_estimates.dat $SYNC_SOURCE/fee_estimates || true
-  ls -l $DAEMON_DIR/{mempool,fee_estimates}.dat || true
+  ls -l $DAEMON_DIR/{mempool}.dat || true
   # and let the runit services take over
 fi