Skip to content

Commit 315feff

Browse files
yaooqinnyaooqinn
authored
Merge branch 'main' into asf
2 parents 5083ab5 + 5170be1 commit 315feff

File tree

38 files changed

+1672
-98
lines changed

38 files changed

+1672
-98
lines changed

‎.github/workflows/util/install_spark_resources.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ function install_spark() {
6868
echo "Skipping checksum because shasum is not installed." 1>&2
6969
fi
7070

71-
tar --strip-components=1 -xf "${local_binary}" spark-"${spark_version}"-bin-hadoop"${hadoop_version}"/jars/
71+
tar --strip-components=1 -xf "${local_binary}" spark-"${spark_version}"-bin-hadoop"${hadoop_version}""${scala_suffix_short}"/jars/
7272
mkdir -p ${INSTALL_DIR}/shims/spark"${spark_version_short}"/spark_home/assembly/target/scala-"${scala_version}"
7373
mv jars ${INSTALL_DIR}/shims/spark"${spark_version_short}"/spark_home/assembly/target/scala-"${scala_version}"
7474

‎.github/workflows/velox_nightly.yml

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one or more
2+
# contributor license agreements. See the NOTICE file distributed with
3+
# this work for additional information regarding copyright ownership.
4+
# The ASF licenses this file to You under the Apache License, Version 2.0
5+
# (the "License"); you may not use this file except in compliance with
6+
# the License. You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
name: Velox backend nightly release
17+
18+
on:
19+
pull_request:
20+
paths:
21+
- '.github/workflows/velox_nightly.yml'
22+
schedule:
23+
- cron: '0 0 * * *'
24+
25+
env:
26+
ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
27+
CCACHE_DIR: "${{ github.workspace }}/.ccache"
28+
29+
concurrency:
30+
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
31+
cancel-in-progress: true
32+
33+
jobs:
34+
build-native-lib:
35+
runs-on: ubuntu-20.04
36+
steps:
37+
- uses: actions/checkout@v4
38+
- name: Get Ccache
39+
uses: actions/cache/restore@v4
40+
with:
41+
path: '${{ env.CCACHE_DIR }}'
42+
key: ccache-centos7-release-default-${{github.sha}}
43+
restore-keys: |
44+
ccache-centos7-release-default
45+
- name: Build Gluten velox third party
46+
run: |
47+
docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7 bash -c "
48+
df -a
49+
cd /work
50+
export CCACHE_DIR=/work/.ccache
51+
bash dev/ci-velox-buildstatic-centos-7.sh
52+
ccache -s
53+
mkdir -p /work/.m2/repository/org/apache/arrow/
54+
cp -r /root/.m2/repository/org/apache/arrow/* /work/.m2/repository/org/apache/arrow/
55+
"
56+
- name: Upload native libs
57+
uses: actions/upload-artifact@v4
58+
with:
59+
path: ./cpp/build/releases/
60+
name: velox-native-lib-${{github.sha}}
61+
retention-days: 1
62+
- name: Upload Artifact Arrow Jar
63+
uses: actions/upload-artifact@v4
64+
with:
65+
path: .m2/repository/org/apache/arrow/
66+
name: velox-arrow-jar-centos-7-${{github.sha}}
67+
68+
build-bundle-package-centos8:
69+
needs: build-native-lib
70+
runs-on: ubuntu-22.04
71+
container: centos:8
72+
steps:
73+
- uses: actions/checkout@v4
74+
- name: Download All Artifacts
75+
uses: actions/download-artifact@v4
76+
with:
77+
name: velox-native-lib-${{github.sha}}
78+
path: ./cpp/build/releases
79+
- name: Download All Arrow Jar Artifacts
80+
uses: actions/download-artifact@v4
81+
with:
82+
name: velox-arrow-jar-centos-7-${{github.sha}}
83+
path: /root/.m2/repository/org/apache/arrow/
84+
- name: Setup java and maven
85+
run: |
86+
sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* && \
87+
sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* && \
88+
yum update -y && yum install -y java-1.8.0-openjdk-devel wget && \
89+
wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \
90+
tar -xvf apache-maven-3.8.8-bin.tar.gz && \
91+
mv apache-maven-3.8.8 /usr/lib/maven
92+
- name: Get current date
93+
id: date
94+
run: echo "::set-output name=date::$(date +'%Y-%m-%d')"
95+
- name: Build for Spark 3.5
96+
run: |
97+
cd $GITHUB_WORKSPACE/ && \
98+
export MAVEN_HOME=/usr/lib/maven && \
99+
export PATH=${PATH}:${MAVEN_HOME}/bin && \
100+
mvn clean install -Pspark-3.5 -Pbackends-velox -Pceleborn -Puniffle -DskipTests -Dmaven.source.skip
101+
- name: Upload bundle package
102+
uses: actions/upload-artifact@v4
103+
with:
104+
name: nightly-gluten-velox-bundle-package-spark35-${{ steps.date.outputs.date }}
105+
path: package/target/gluten-velox-bundle-*.jar
106+
retention-days: 7
107+
108+

‎backends-clickhouse/pom.xml

+2-2
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,9 @@
130130
</exclusions>
131131
</dependency>
132132
<dependency>
133-
<groupId>io.glutenproject</groupId>
133+
<groupId>com.google.protobuf</groupId>
134134
<artifactId>protobuf-java</artifactId>
135-
<version>${custom.protobuf.version}</version>
135+
<version>${protobuf.version}</version>
136136
</dependency>
137137
<dependency>
138138
<groupId>org.apache.gluten</groupId>

‎backends-clickhouse/src/main/scala/org/apache/spark/sql/delta/ClickhouseSnapshot.scala

+5-3
Original file line numberDiff line numberDiff line change
@@ -84,9 +84,11 @@ case class FilterExprsAsKey(
8484
}
8585

8686
object ClickhouseSnapshot {
87+
protected def sparkSession: SparkSession = SparkSession.active
88+
8789
val deltaScanCache: Cache[FilterExprsAsKey, DeltaScan] = CacheBuilder.newBuilder
8890
.maximumSize(
89-
SparkSession.getActiveSession.get.conf
91+
sparkSession.conf
9092
.get(CHBackendSettings.GLUTEN_CLICKHOUSE_DELTA_SCAN_CACHE_SIZE, "10000")
9193
.toLong)
9294
.expireAfterAccess(7200L, TimeUnit.SECONDS)
@@ -95,7 +97,7 @@ object ClickhouseSnapshot {
9597

9698
val addFileToAddMTPCache: LoadingCache[AddFileAsKey, AddMergeTreeParts] = CacheBuilder.newBuilder
9799
.maximumSize(
98-
SparkSession.getActiveSession.get.conf
100+
sparkSession.conf
99101
.get(CHBackendSettings.GLUTEN_CLICKHOUSE_ADDFILES_TO_MTPS_CACHE_SIZE, "1000000")
100102
.toLong)
101103
.expireAfterAccess(7200L, TimeUnit.SECONDS)
@@ -109,7 +111,7 @@ object ClickhouseSnapshot {
109111

110112
val pathToAddMTPCache: Cache[String, AddMergeTreeParts] = CacheBuilder.newBuilder
111113
.maximumSize(
112-
SparkSession.getActiveSession.get.conf
114+
sparkSession.conf
113115
.get(CHBackendSettings.GLUTEN_CLICKHOUSE_TABLE_PATH_TO_MTPS_CACHE_SIZE, "1000000")
114116
.toLong)
115117
.expireAfterAccess(7200L, TimeUnit.SECONDS)

‎backends-velox/pom.xml

+90-2
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,27 @@
100100
<version>${iceberg.version}</version>
101101
<scope>provided</scope>
102102
</dependency>
103+
<dependency>
104+
<groupId>org.apache.iceberg</groupId>
105+
<artifactId>iceberg-spark-${sparkbundle.version}_${scala.binary.version}</artifactId>
106+
<version>${iceberg.version}</version>
107+
<type>test-jar</type>
108+
<scope>test</scope>
109+
</dependency>
110+
<dependency>
111+
<groupId>org.apache.iceberg</groupId>
112+
<artifactId>iceberg-hive-metastore</artifactId>
113+
<version>${iceberg.version}</version>
114+
<type>test-jar</type>
115+
<scope>test</scope>
116+
</dependency>
117+
<dependency>
118+
<groupId>org.apache.iceberg</groupId>
119+
<artifactId>iceberg-api</artifactId>
120+
<version>${iceberg.version}</version>
121+
<type>test-jar</type>
122+
<scope>test</scope>
123+
</dependency>
103124
</dependencies>
104125
</profile>
105126
<profile>
@@ -147,6 +168,73 @@
147168
</dependency>
148169
</dependencies>
149170
</profile>
171+
<profile>
172+
<id>spark-3.2</id>
173+
<build>
174+
<plugins>
175+
<plugin>
176+
<groupId>org.apache.maven.plugins</groupId>
177+
<artifactId>maven-compiler-plugin</artifactId>
178+
<configuration>
179+
<testExcludes>
180+
<testExclude>**/org/apache/gluten/spark34/**</testExclude>
181+
</testExcludes>
182+
</configuration>
183+
</plugin>
184+
</plugins>
185+
</build>
186+
</profile>
187+
<profile>
188+
<id>spark-3.3</id>
189+
<build>
190+
<plugins>
191+
<plugin>
192+
<groupId>org.apache.maven.plugins</groupId>
193+
<artifactId>maven-compiler-plugin</artifactId>
194+
<configuration>
195+
<testExcludes>
196+
<testExclude>**/org/apache/gluten/spark34/**</testExclude>
197+
</testExcludes>
198+
</configuration>
199+
</plugin>
200+
</plugins>
201+
</build>
202+
</profile>
203+
<profile>
204+
<id>spark-3.5</id>
205+
<build>
206+
<plugins>
207+
<plugin>
208+
<groupId>org.apache.maven.plugins</groupId>
209+
<artifactId>maven-compiler-plugin</artifactId>
210+
<configuration>
211+
<testExcludes>
212+
<testExclude>**/org/apache/gluten/spark34/**</testExclude>
213+
</testExcludes>
214+
</configuration>
215+
</plugin>
216+
</plugins>
217+
</build>
218+
</profile>
219+
<profile>
220+
<id>default</id>
221+
<activation>
222+
<activeByDefault>true</activeByDefault>
223+
</activation>
224+
<build>
225+
<plugins>
226+
<plugin>
227+
<groupId>org.apache.maven.plugins</groupId>
228+
<artifactId>maven-compiler-plugin</artifactId>
229+
<configuration>
230+
<testExcludes>
231+
<testExclude>**/org/apache/gluten/spark34/**</testExclude>
232+
</testExcludes>
233+
</configuration>
234+
</plugin>
235+
</plugins>
236+
</build>
237+
</profile>
150238
</profiles>
151239

152240
<dependencies>
@@ -157,9 +245,9 @@
157245
<scope>compile</scope>
158246
</dependency>
159247
<dependency>
160-
<groupId>io.glutenproject</groupId>
248+
<groupId>com.google.protobuf</groupId>
161249
<artifactId>protobuf-java</artifactId>
162-
<version>${custom.protobuf.version}</version>
250+
<version>${protobuf.version}</version>
163251
</dependency>
164252
<dependency>
165253
<groupId>org.apache.gluten</groupId>

0 commit comments

Comments
 (0)