Use layer caching in Docker builds in GHA (valhalla#5082)

Georepublic · Feb 6, 2025 · c7d90fc · c7d90fc
1 parent 868cb5d
commit c7d90fc
Show file tree

Hide file tree

Showing 4 changed files with 109 additions and 41 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -44,6 +44,7 @@ libvalhalla.la
 date_time_zonespec.csv
 mason/
 mason_packages/
+.DS_Store
 
 # scripts
 scripts/gdal-2.0.0/
@@ -60,9 +61,23 @@ CMakeLists.txt.user
 .idea
 /.tidytmp
 vcpkg*/
+overlay-ports-vcpkg/
+vcpkg-configuration.json
+run_route_scripts/
+test_requests/
 
-# documentation
-site/
-
-.DS_Store
+# docker
 Dockerfile
+
+# CI
+.github/
+.circleci/
+.pre-commit-config.yaml
+
+# docs etc
+site/
+docs/
+CONTRIBUTING.md
+NEWS 
+taginfo.json
+CHANGELOG.md
diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml
@@ -2,50 +2,101 @@ name: Publish Docker image
 on:
   push:
     branches:
-      - 'master'
+      - "master"
     tags:
-      - '*'
+      - "*"
     paths-ignore:
-      - '.gitignore'
-      - '**.md'
-      - 'test/'
+      - ".gitignore"
+      - "**.md"
+      - "test/"
 jobs:
   build_and_publish:
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-latest
+            platform: linux/amd64
+            tagsuffix: amd64
+          - os: ubuntu-24.04-arm
+            platform: linux/arm64
+            tagsuffix: arm64
+    outputs:
+      tag: ${{ steps.extract_tag.outputs.tag }}
+
     if: ${{ github.repository_owner == 'valhalla' }}
+    runs-on: ${{ matrix.os }}
+
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
         with:
-          submodules: 'recursive'
+          submodules: "recursive"
           fetch-depth: 0
+
       - name: Extract tag name
         if: startsWith(github.ref, 'refs/tags/')
         run: echo "##[set-output name=tag;]${GITHUB_REF#refs/tags/}"
         id: extract_tag
+
+      - name: Log in to GitHub Docker Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push latest
+        if: github.ref == 'refs/heads/master'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          provenance: false # needed to be able to converge into one image
+          platforms: ${{ matrix.platform }}
+          push: true
+          tags: ghcr.io/valhalla/valhalla:latest-${{ matrix.tagsuffix }}
+          cache-from: type=gha,scope=${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+
+      - name: Build and push tag
+        if: startsWith(github.ref, 'refs/tags/')
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          provenance: false # needed to be able to converge into one image
+          platforms: ${{ matrix.platform }}
+          push: true
+          tags: ghcr.io/valhalla/valhalla:${{ steps.extract_tag.outputs.tag }}-${{ matrix.tagsuffix }}
+          cache-from: type=gha,scope=${{ matrix.platform }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.platform }}
+
+  create-manifests:
+    runs-on: ubuntu-latest
+    needs: [build_and_publish]
+    steps:
       - name: Log in to GitHub Docker Registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build tag image
+
+      - name: Create and push tag manifest
         if: startsWith(github.ref, 'refs/tags/')
         run: |
-          sudo apt update --yes
-          sudo apt install --yes --quiet binfmt-support qemu-user-static qemu-system
-          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-          docker buildx create --name mybuilder
-          docker buildx use mybuilder
-          docker buildx inspect --bootstrap
-          docker buildx build --push --platform linux/amd64,linux/arm64 --tag ghcr.io/valhalla/valhalla:${{ steps.extract_tag.outputs.tag }} .
-      - name: Build latest image
+          docker manifest create \
+            ghcr.io/valhalla/valhalla:${{ needs.build_and_publish.outputs.tag }} \
+            --amend ghcr.io/valhalla/valhalla:${{ needs.build_and_publish.outputs.tag }}-amd64 \
+            --amend ghcr.io/valhalla/valhalla:${{ needs.build_and_publish.outputs.tag }}-arm64 
+          docker manifest push ghcr.io/valhalla/valhalla:${{ needs.build_and_publish.outputs.tag }}
+
+      - name: Create and push latest manifest
         if: github.ref == 'refs/heads/master'
         run: |
-          sudo apt update --yes
-          sudo apt install --yes --quiet binfmt-support qemu-user-static qemu-system
-          docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-          docker buildx create --name mybuilder
-          docker buildx use mybuilder
-          docker buildx inspect --bootstrap
-          docker buildx build --push --platform linux/amd64,linux/arm64 --tag ghcr.io/valhalla/valhalla:latest .
+          docker manifest create \
+            ghcr.io/valhalla/valhalla:latest \
+            --amend ghcr.io/valhalla/valhalla:latest-amd64 \
+            --amend ghcr.io/valhalla/valhalla:latest-arm64 
+          docker manifest push ghcr.io/valhalla/valhalla:latest
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -29,6 +29,7 @@
    * ADDED: Support `"access:conditional"` conditional restrictions like `"access:conditional"="no @ (Oct-May)"` [#5048](https://github.com/valhalla/valhalla/pull/5048)
    * CHANGED: Speed up pbf parsing by using libosmium [#5070](https://github.com/valhalla/valhalla/pull/5070)
    * ADDED: headings and correlated ll's in verbose matrix output [#5072](https://github.com/valhalla/valhalla/pull/5072)
+   * CHANGED: Faster Docker builds in CI [#5082](https://github.com/valhalla/valhalla/pull/5082) 
 
 ## Release Date: 2024-10-10 Valhalla 3.5.1
 * **Removed**

diff --git a/Dockerfile b/Dockerfile
@@ -8,8 +8,8 @@
 #  binaries that can target the target architecture. from there bob is your uncle maybe..
 
 ####################################################################
-FROM ubuntu:24.04 as builder
-MAINTAINER Kevin Kreiser <kevinkreiser@gmail.com>
+FROM ubuntu:24.04 AS builder
+LABEL org.opencontainers.image.authors="Kevin Kreiser <kevinkreiser@gmail.com>"
 
 ARG CONCURRENCY
 ARG ADDITIONAL_TARGETS
@@ -55,29 +55,30 @@ RUN rm -rf valhalla
 
 ####################################################################
 # copy the important stuff from the build stage to the runner image
-FROM ubuntu:24.04 as runner
-MAINTAINER Kevin Kreiser <kevinkreiser@gmail.com>
+FROM ubuntu:24.04 AS runner
+LABEL org.opencontainers.image.authors="Kevin Kreiser <kevinkreiser@gmail.com>"
 
 # basic paths
 ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH
 ENV LD_LIBRARY_PATH=/usr/local/lib:/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/lib32:/usr/lib32
 
 # github packaging niceties
-LABEL org.opencontainers.image.description = "Open Source Routing Engine for OpenStreetMap and Other Datasources"
-LABEL org.opencontainers.image.source = "https://github.com/valhalla/valhalla"
+LABEL org.opencontainers.image.description="Open Source Routing Engine for OpenStreetMap and Other Datasources"
+LABEL org.opencontainers.image.source="https://github.com/valhalla/valhalla"
+
+# we need to add back some runtime dependencies for binaries and scripts
+# install all the posix locales that we support
+RUN export DEBIAN_FRONTEND=noninteractive && apt update && \
+  apt install -y \
+  libcurl4 libczmq4 libluajit-5.1-2 libgdal34 \
+  libprotobuf-lite32 libsqlite3-0 libsqlite3-mod-spatialite libzmq5 zlib1g \
+  curl gdb locales parallel python3-minimal python-is-python3 python3-shapely python3-requests \
+  spatialite-bin unzip wget && rm -rf /var/lib/apt/lists/*
 
 # grab the builder stages artifacts
 COPY --from=builder /usr/local /usr/local
 COPY --from=builder /usr/local/lib/python3.12/dist-packages/valhalla/* /usr/local/lib/python3.12/dist-packages/valhalla/
 
-# we need to add back some runtime dependencies for binaries and scripts
-# install all the posix locales that we support
-RUN export DEBIAN_FRONTEND=noninteractive && apt update && \
-    apt install -y \
-      libcurl4 libczmq4 libluajit-5.1-2 libgdal34 \
-      libprotobuf-lite32 libsqlite3-0 libsqlite3-mod-spatialite libzmq5 zlib1g \
-      curl gdb locales parallel python3-minimal python-is-python3 python3-shapely python3-requests \
-      spatialite-bin unzip wget && rm -rf /var/lib/apt/lists/*
 RUN cat /usr/local/src/valhalla_locales | xargs -d '\n' -n1 locale-gen
 
 # python smoke test