Skip to content

Commit 2b29adc

Browse files
ArnavBalyanArnavBalyan
committed
Merge branch 'main' into arnavb/double-timestamp
2 parents c3aa1f7 + 257dbc6 commit 2b29adc

File tree

60 files changed

+2163
-152
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+2163
-152
lines changed

‎.github/workflows/docker_image.yml

+226-19
Original file line numberDiff line numberDiff line change
@@ -23,23 +23,27 @@ on:
2323
- '.github/workflows/docker_image.yml'
2424
- '.github/workflows/util/install_spark_resources.sh'
2525
- 'dev/docker/Dockerfile.centos7-static-build'
26+
- 'dev/docker/Dockerfile.centos8-static-build'
2627
- 'dev/docker/Dockerfile.centos8-dynamic-build'
2728
- 'dev/docker/Dockerfile.centos8-dynamic-build-jdk11'
2829
- 'dev/docker/Dockerfile.centos8-dynamic-build-jdk17'
2930
schedule:
3031
- cron: '0 20 * * 0'
3132

33+
env:
34+
DOCKERHUB_REPO: apache/gluten
35+
3236
jobs:
3337
build-vcpkg-centos-7:
3438
if: ${{ startsWith(github.repository, 'apache/') }}
3539
runs-on: ubuntu-latest
3640

3741
steps:
3842
- name: Checkout repository
39-
uses: actions/checkout@v2
43+
uses: actions/checkout@v4
4044

4145
- name: Set up Docker Buildx
42-
uses: docker/setup-buildx-action@v1
46+
uses: docker/setup-buildx-action@v3
4347

4448
- name: Login to Docker Hub
4549
uses: docker/login-action@v2
@@ -48,51 +52,254 @@ jobs:
4852
password: ${{ secrets.DOCKERHUB_TOKEN }}
4953

5054
- name: Build and push Docker image
51-
uses: docker/build-push-action@v2
55+
uses: docker/build-push-action@v6
5256
with:
5357
context: .
5458
file: dev/docker/Dockerfile.centos7-static-build
5559
push: true
56-
tags: apache/gluten:vcpkg-centos-7
60+
tags: ${{ env.DOCKERHUB_REPO }}:vcpkg-centos-7
5761

58-
build-centos-8:
62+
build-vcpkg-centos-8:
5963
if: ${{ startsWith(github.repository, 'apache/') }}
60-
runs-on: ubuntu-latest
64+
runs-on: ${{ matrix.os }}
65+
strategy:
66+
matrix:
67+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
68+
69+
steps:
70+
- name: Checkout repository
71+
uses: actions/checkout@v2
72+
73+
- name: Docker meta
74+
id: meta
75+
uses: docker/metadata-action@v5
76+
with:
77+
images: ${{ env.DOCKERHUB_REPO }}
78+
tags: vcpkg-centos-8
79+
80+
- name: Set up Docker Buildx
81+
uses: docker/setup-buildx-action@v3
82+
83+
- name: Login to Docker Hub
84+
uses: docker/login-action@v2
85+
with:
86+
username: ${{ secrets.DOCKERHUB_USER }}
87+
password: ${{ secrets.DOCKERHUB_TOKEN }}
88+
89+
- name: Build and push by digest
90+
id: build
91+
uses: docker/build-push-action@v6
92+
with:
93+
context: .
94+
file: dev/docker/Dockerfile.centos8-static-build
95+
labels: ${{ steps.meta.outputs.labels }}
96+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
97+
98+
- name: Export digest
99+
run: |
100+
mkdir -p ${{ runner.temp }}/digests
101+
digest="${{ steps.build.outputs.digest }}"
102+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
103+
104+
- name: Upload digest
105+
uses: actions/upload-artifact@v4
106+
with:
107+
name: digests-vcpkg-centos-8-${{ matrix.os }}
108+
path: ${{ runner.temp }}/digests/*
109+
if-no-files-found: error
110+
retention-days: 1
111+
112+
build-centos-8-jdk8:
113+
runs-on: ${{ matrix.os }}
114+
strategy:
115+
matrix:
116+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
61117

62118
steps:
63119
- name: Checkout repository
64120
uses: actions/checkout@v2
65121

122+
- name: Docker meta
123+
id: meta
124+
uses: docker/metadata-action@v5
125+
with:
126+
images: ${{ env.DOCKERHUB_REPO }}
127+
tags: centos-8-jdk8
128+
66129
- name: Set up Docker Buildx
67-
uses: docker/setup-buildx-action@v1
130+
uses: docker/setup-buildx-action@v3
68131

69132
- name: Login to Docker Hub
70133
uses: docker/login-action@v2
71134
with:
72135
username: ${{ secrets.DOCKERHUB_USER }}
73136
password: ${{ secrets.DOCKERHUB_TOKEN }}
74137

75-
- name: Build and push Docker image Centos8
76-
uses: docker/build-push-action@v2
138+
- name: Build and push by digest
139+
id: build
140+
uses: docker/build-push-action@v6
77141
with:
78142
context: .
79143
file: dev/docker/Dockerfile.centos8-dynamic-build
80-
push: true
81-
tags: apache/gluten:centos-8 # JDK8 based
144+
labels: ${{ steps.meta.outputs.labels }}
145+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
146+
147+
- name: Export digest
148+
run: |
149+
mkdir -p ${{ runner.temp }}/digests
150+
digest="${{ steps.build.outputs.digest }}"
151+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
152+
153+
- name: Upload digest
154+
uses: actions/upload-artifact@v4
155+
with:
156+
name: digests-centos-8-jdk8-${{ matrix.os }}
157+
path: ${{ runner.temp }}/digests/*
158+
if-no-files-found: error
159+
retention-days: 1
160+
161+
build-centos-8-jdk11:
162+
if: ${{ startsWith(github.repository, 'apache/') }}
163+
runs-on: ${{ matrix.os }}
164+
strategy:
165+
matrix:
166+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
167+
168+
steps:
169+
- name: Checkout repository
170+
uses: actions/checkout@v2
82171

83-
- name: Build and push Docker image Centos8 + JDK11
84-
uses: docker/build-push-action@v2
172+
- name: Docker meta
173+
id: meta
174+
uses: docker/metadata-action@v5
175+
with:
176+
images: ${{ env.DOCKERHUB_REPO }}
177+
tags: centos-8-jdk11
178+
179+
- name: Set up Docker Buildx
180+
uses: docker/setup-buildx-action@v3
181+
182+
- name: Login to Docker Hub
183+
uses: docker/login-action@v2
184+
with:
185+
username: ${{ secrets.DOCKERHUB_USER }}
186+
password: ${{ secrets.DOCKERHUB_TOKEN }}
187+
188+
- name: Build and push by digest
189+
id: build
190+
uses: docker/build-push-action@v6
85191
with:
86192
context: .
87193
file: dev/docker/Dockerfile.centos8-dynamic-build-jdk11
88-
push: true
89-
tags: apache/gluten:centos-8-jdk11
194+
labels: ${{ steps.meta.outputs.labels }}
195+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
196+
197+
- name: Export digest
198+
run: |
199+
mkdir -p ${{ runner.temp }}/digests
200+
digest="${{ steps.build.outputs.digest }}"
201+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
202+
203+
- name: Upload digest
204+
uses: actions/upload-artifact@v4
205+
with:
206+
name: digests-centos-8-jdk11-${{ matrix.os }}
207+
path: ${{ runner.temp }}/digests/*
208+
if-no-files-found: error
209+
retention-days: 1
210+
211+
build-centos-8-jdk17:
212+
if: ${{ startsWith(github.repository, 'apache/') }}
213+
runs-on: ${{ matrix.os }}
214+
strategy:
215+
matrix:
216+
os: [ ubuntu-latest, ubuntu-24.04-arm ]
217+
218+
steps:
219+
- name: Checkout repository
220+
uses: actions/checkout@v2
221+
222+
- name: Docker meta
223+
id: meta
224+
uses: docker/metadata-action@v5
225+
with:
226+
images: ${{ env.DOCKERHUB_REPO }}
227+
tags: centos-8-jdk17
228+
229+
- name: Set up Docker Buildx
230+
uses: docker/setup-buildx-action@v3
231+
232+
- name: Login to Docker Hub
233+
uses: docker/login-action@v2
234+
with:
235+
username: ${{ secrets.DOCKERHUB_USER }}
236+
password: ${{ secrets.DOCKERHUB_TOKEN }}
90237

91-
- name: Build and push Docker image Centos8 + JDK17
92-
uses: docker/build-push-action@v2
238+
- name: Build and push by digest
239+
id: build
240+
uses: docker/build-push-action@v6
93241
with:
94242
context: .
95243
file: dev/docker/Dockerfile.centos8-dynamic-build-jdk17
96-
push: true
97-
tags: apache/gluten:centos-8-jdk17
244+
labels: ${{ steps.meta.outputs.labels }}
245+
outputs: type=image,"name=${{ env.DOCKERHUB_REPO }}",push-by-digest=true,push=true
246+
247+
- name: Export digest
248+
run: |
249+
mkdir -p ${{ runner.temp }}/digests
250+
digest="${{ steps.build.outputs.digest }}"
251+
touch "${{ runner.temp }}/digests/${digest#sha256:}"
252+
253+
- name: Upload digest
254+
uses: actions/upload-artifact@v4
255+
with:
256+
name: digests-centos-8-jdk17-${{ matrix.os }}
257+
path: ${{ runner.temp }}/digests/*
258+
if-no-files-found: error
259+
retention-days: 1
260+
261+
merge:
262+
if: ${{ startsWith(github.repository, 'apache/') }}
263+
runs-on: ubuntu-latest
264+
strategy:
265+
matrix:
266+
digests: [ vcpkg-centos-8, centos-8-jdk8, centos-8-jdk11, centos-8-jdk17 ]
267+
needs:
268+
- build-vcpkg-centos-8
269+
- build-centos-8-jdk8
270+
- build-centos-8-jdk11
271+
- build-centos-8-jdk17
272+
steps:
273+
- name: Download digests
274+
uses: actions/download-artifact@v4
275+
with:
276+
path: ${{ runner.temp }}/digests
277+
pattern: digests-${{ matrix.digests }}-*
278+
merge-multiple: true
279+
280+
- name: Login to Docker Hub
281+
uses: docker/login-action@v2
282+
with:
283+
username: ${{ secrets.DOCKERHUB_USER }}
284+
password: ${{ secrets.DOCKERHUB_TOKEN }}
285+
286+
- name: Set up Docker Buildx
287+
uses: docker/setup-buildx-action@v3
288+
289+
- name: Docker meta
290+
id: meta
291+
uses: docker/metadata-action@v5
292+
with:
293+
images: ${{ env.DOCKERHUB_REPO }}
294+
tags: ${{ matrix.digests }}
295+
296+
- name: Create manifest list and push
297+
working-directory: ${{ runner.temp }}/digests
298+
run: |
299+
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
300+
$(printf '${{ env.DOCKERHUB_REPO }}@sha256:%s ' *)
301+
302+
- name: Inspect image
303+
run: |
304+
docker buildx imagetools inspect ${{ env.DOCKERHUB_REPO }}:${{ steps.meta.outputs.version }}
98305

‎.github/workflows/util/install_spark_resources.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,9 @@ function install_spark() {
6868
echo "Skipping checksum because shasum is not installed." 1>&2
6969
fi
7070

71-
tar --strip-components=1 -xf "${local_binary}" spark-"${spark_version}"-bin-hadoop"${hadoop_version}"/jars/
72-
mkdir -p ${INSTALL_DIR}/shims/spark"${spark_version_short}"/spark_home/assembly/target/scala-"${scala_version}"
73-
mv jars ${INSTALL_DIR}/shims/spark"${spark_version_short}"/spark_home/assembly/target/scala-"${scala_version}"
71+
tar --strip-components=1 -xf "${local_binary}" spark-"${spark_version}"-bin-hadoop"${hadoop_version}""${scala_suffix_short}"/jars/
72+
mkdir -p ${INSTALL_DIR}/shims/spark"${spark_version_short}""${scala_suffix}"/spark_home/assembly/target/scala-"${scala_version}"
73+
mv jars ${INSTALL_DIR}/shims/spark"${spark_version_short}""${scala_suffix}"/spark_home/assembly/target/scala-"${scala_version}"
7474

7575
tar --strip-components=1 -xf "${local_source}" spark-"${spark_version}"/sql/core/src/test/resources/
7676
mkdir -p shims/spark"${spark_version_short}${scala_suffix}"/spark_home/

‎.github/workflows/velox_backend.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1015,7 +1015,7 @@ jobs:
10151015
cd $GITHUB_WORKSPACE/
10161016
export SPARK_SCALA_VERSION=2.13
10171017
$MVN_CMD clean test -Pspark-3.5 -Pscala-2.13 -Pbackends-velox -Pceleborn -Piceberg \
1018-
-Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark35-scala2.13/spark_home/" \
1018+
-Pdelta -Pspark-ut -DargLine="-Dspark.test.home=/opt/shims/spark35-scala-2.13/spark_home/" \
10191019
-DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags
10201020
- name: Upload test report
10211021
if: always()

0 commit comments

Comments
 (0)