Skip to content

Commit 0778617

Browse files
ArnavBalyanArnavBalyan
committed
Merge branch 'main' into arnavb/range-test-rewrite
2 parents 242c78e + 257dbc6 commit 0778617

File tree

400 files changed

+40521
-3736
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

400 files changed

+40521
-3736
lines changed

‎.github/workflows/docker_image.yml

+227-19
Original file line numberDiff line numberDiff line change
@@ -21,24 +21,29 @@ on:
2121
- main
2222
paths:
2323
- '.github/workflows/docker_image.yml'
24+
- '.github/workflows/util/install_spark_resources.sh'
2425
- 'dev/docker/Dockerfile.centos7-static-build'
26+
- 'dev/docker/Dockerfile.centos8-static-build'
2527
- 'dev/docker/Dockerfile.centos8-dynamic-build'
2628
- 'dev/docker/Dockerfile.centos8-dynamic-build-jdk11'
2729
- 'dev/docker/Dockerfile.centos8-dynamic-build-jdk17'
2830
schedule:
2931
- cron: '0 20 * * 0'
3032

33+
env:
34+
DOCKERHUB_REPO: apache/gluten
35+
3136
jobs:
3237
build-vcpkg-centos-7:
3338
if: ${{ startsWith(github.repository, 'apache/') }}
3439
runs-on: ubuntu-latest
3540

3641
steps:
3742
- name: Checkout repository
38-
uses: actions/checkout@v2
43+
uses: actions/checkout@v4
3944

4045
- name: Set up Docker Buildx
41-
uses: docker/setup-buildx-action@v1
46+
uses: docker/setup-buildx-action@v3
4247

4348
- name: Login to Docker Hub
4449
uses: docker/login-action@v2
@@ -47,51 +52,254 @@ jobs:
4752
password: ${{ secrets.DOCKERHUB_TOKEN }}
4853

4954
- name: Build and push Docker image
50-
uses: docker/build-push-action@v2
55+
uses: docker/build-push-action@v6
5156
with:
5257
context: .
5358
file: dev/docker/Dockerfile.centos7-static-build
5459
push: true
55-
tags: apache/gluten:vcpkg-centos-7
60+
tags: ${{ env.DOCKERHUB_REPO }}:vcpkg-centos-7
5661

57-
build-centos-8:
62+
build-vcpkg-centos-8:
5863
if: ${{ startsWith(github.repository, 'apache/') }}
59-
runs-on: ubuntu-latest
64+
runs-on: ${{ matrix.os }}
65+
strategy:
66+
matrix:
67+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
68+
69+
steps:
70+
- name: Checkout repository
71+
uses: actions/checkout@v2
72+
73+
- name: Docker meta
74+
id: meta
75+
uses: docker/metadata-action@v5
76+
with:
77+
images: ${{ env.DOCKERHUB_REPO }}
78+
tags: vcpkg-centos-8
79+
80+
- name: Set up Docker Buildx
81+
uses: docker/setup-buildx-action@v3
82+
83+
- name: Login to Docker Hub
84+
uses: docker/login-action@v2
85+
with:
86+
username: ${{ secrets.DOCKERHUB_USER }}
87+
password: ${{ secrets.DOCKERHUB_TOKEN }}
88+
89+
- name: Build and push by digest
90+
id: build
91+
uses: docker/build-push-action@v6
92+
with:
93+
context: .
94+
file: dev/docker/Dockerfile.centos8-static-build
95+
labels: ${{ steps.meta.outputs.labels }}
96+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
97+
98+
- name: Export digest
99+
run: |
100+
mkdir -p ${{ runner.temp }}/digests
101+
digest="${{ steps.build.outputs.digest }}"
102+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
103+
104+
- name: Upload digest
105+
uses: actions/upload-artifact@v4
106+
with:
107+
name: digests-vcpkg-centos-8-${{ matrix.os }}
108+
path: ${{ runner.temp }}/digests/*
109+
if-no-files-found: error
110+
retention-days: 1
111+
112+
build-centos-8-jdk8:
113+
runs-on: ${{ matrix.os }}
114+
strategy:
115+
matrix:
116+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
60117

61118
steps:
62119
- name: Checkout repository
63120
uses: actions/checkout@v2
64121

122+
- name: Docker meta
123+
id: meta
124+
uses: docker/metadata-action@v5
125+
with:
126+
images: ${{ env.DOCKERHUB_REPO }}
127+
tags: centos-8-jdk8
128+
65129
- name: Set up Docker Buildx
66-
uses: docker/setup-buildx-action@v1
130+
uses: docker/setup-buildx-action@v3
67131

68132
- name: Login to Docker Hub
69133
uses: docker/login-action@v2
70134
with:
71135
username: ${{ secrets.DOCKERHUB_USER }}
72136
password: ${{ secrets.DOCKERHUB_TOKEN }}
73137

74-
- name: Build and push Docker image Centos8
75-
uses: docker/build-push-action@v2
138+
- name: Build and push by digest
139+
id: build
140+
uses: docker/build-push-action@v6
76141
with:
77142
context: .
78143
file: dev/docker/Dockerfile.centos8-dynamic-build
79-
push: true
80-
tags: apache/gluten:centos-8 # JDK8 based
144+
labels: ${{ steps.meta.outputs.labels }}
145+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
146+
147+
- name: Export digest
148+
run: |
149+
mkdir -p ${{ runner.temp }}/digests
150+
digest="${{ steps.build.outputs.digest }}"
151+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
152+
153+
- name: Upload digest
154+
uses: actions/upload-artifact@v4
155+
with:
156+
name: digests-centos-8-jdk8-${{ matrix.os }}
157+
path: ${{ runner.temp }}/digests/*
158+
if-no-files-found: error
159+
retention-days: 1
160+
161+
build-centos-8-jdk11:
162+
if: ${{ startsWith(github.repository, 'apache/') }}
163+
runs-on: ${{ matrix.os }}
164+
strategy:
165+
matrix:
166+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
167+
168+
steps:
169+
- name: Checkout repository
170+
uses: actions/checkout@v2
81171

82-
- name: Build and push Docker image Centos8 + JDK11
83-
uses: docker/build-push-action@v2
172+
- name: Docker meta
173+
id: meta
174+
uses: docker/metadata-action@v5
175+
with:
176+
images: ${{ env.DOCKERHUB_REPO }}
177+
tags: centos-8-jdk11
178+
179+
- name: Set up Docker Buildx
180+
uses: docker/setup-buildx-action@v3
181+
182+
- name: Login to Docker Hub
183+
uses: docker/login-action@v2
184+
with:
185+
username: ${{ secrets.DOCKERHUB_USER }}
186+
password: ${{ secrets.DOCKERHUB_TOKEN }}
187+
188+
- name: Build and push by digest
189+
id: build
190+
uses: docker/build-push-action@v6
84191
with:
85192
context: .
86193
file: dev/docker/Dockerfile.centos8-dynamic-build-jdk11
87-
push: true
88-
tags: apache/gluten:centos-8-jdk11
194+
labels: ${{ steps.meta.outputs.labels }}
195+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
196+
197+
- name: Export digest
198+
run: |
199+
mkdir -p ${{ runner.temp }}/digests
200+
digest="${{ steps.build.outputs.digest }}"
201+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
202+
203+
- name: Upload digest
204+
uses: actions/upload-artifact@v4
205+
with:
206+
name: digests-centos-8-jdk11-${{ matrix.os }}
207+
path: ${{ runner.temp }}/digests/*
208+
if-no-files-found: error
209+
retention-days: 1
210+
211+
build-centos-8-jdk17:
212+
if: ${{ startsWith(github.repository, 'apache/') }}
213+
runs-on: ${{ matrix.os }}
214+
strategy:
215+
matrix:
216+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
217+
218+
steps:
219+
- name: Checkout repository
220+
uses: actions/checkout@v2
221+
222+
- name: Docker meta
223+
id: meta
224+
uses: docker/metadata-action@v5
225+
with:
226+
images: ${{ env.DOCKERHUB_REPO }}
227+
tags: centos-8-jdk17
228+
229+
- name: Set up Docker Buildx
230+
uses: docker/setup-buildx-action@v3
231+
232+
- name: Login to Docker Hub
233+
uses: docker/login-action@v2
234+
with:
235+
username: ${{ secrets.DOCKERHUB_USER }}
236+
password: ${{ secrets.DOCKERHUB_TOKEN }}
89237

90-
- name: Build and push Docker image Centos8 + JDK17
91-
uses: docker/build-push-action@v2
238+
- name: Build and push by digest
239+
id: build
240+
uses: docker/build-push-action@v6
92241
with:
93242
context: .
94243
file: dev/docker/Dockerfile.centos8-dynamic-build-jdk17
95-
push: true
96-
tags: apache/gluten:centos-8-jdk17
244+
labels: ${{ steps.meta.outputs.labels }}
245+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
246+
247+
- name: Export digest
248+
run: |
249+
mkdir -p ${{ runner.temp }}/digests
250+
digest="${{ steps.build.outputs.digest }}"
251+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
252+
253+
- name: Upload digest
254+
uses: actions/upload-artifact@v4
255+
with:
256+
name: digests-centos-8-jdk17-${{ matrix.os }}
257+
path: ${{ runner.temp }}/digests/*
258+
if-no-files-found: error
259+
retention-days: 1
260+
261+
merge:
262+
if: ${{ startsWith(github.repository, 'apache/') }}
263+
runs-on: ubuntu-latest
264+
strategy:
265+
matrix:
266+
digests: [ vcpkg-centos-8, centos-8-jdk8, centos-8-jdk11, centos-8-jdk17 ]
267+
needs:
268+
- build-vcpkg-centos-8
269+
- build-centos-8-jdk8
270+
- build-centos-8-jdk11
271+
- build-centos-8-jdk17
272+
steps:
273+
- name: Download digests
274+
uses: actions/download-artifact@v4
275+
with:
276+
path: ${{ runner.temp }}/digests
277+
pattern: digests-${{ matrix.digests }}-*
278+
merge-multiple: true
279+
280+
- name: Login to Docker Hub
281+
uses: docker/login-action@v2
282+
with:
283+
username: ${{ secrets.DOCKERHUB_USER }}
284+
password: ${{ secrets.DOCKERHUB_TOKEN }}
285+
286+
- name: Set up Docker Buildx
287+
uses: docker/setup-buildx-action@v3
288+
289+
- name: Docker meta
290+
id: meta
291+
uses: docker/metadata-action@v5
292+
with:
293+
images: ${{ env.DOCKERHUB_REPO }}
294+
tags: ${{ matrix.digests }}
295+
296+
- name: Create manifest list and push
297+
working-directory: ${{ runner.temp }}/digests
298+
run: |
299+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
300+
$(printf '${{ env.DOCKERHUB_REPO }}@sha256:%s ' *)
301+
302+
- name: Inspect image
303+
run: |
304+
docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{ steps.meta.outputs.version }}
97305

0 commit comments

Comments
 (0)