Skip to content

Commit 8a256f0

Browse files
PHILO-HEPHILO-HE
authored
[GLUTEN-8266][VL] Use pre-installed resources for Spark/Celeborn (#8294)
* Initial * Change docker for celeborn test
1 parent b9aba24 commit 8a256f0

File tree

1 file changed

+8
-38
lines changed

1 file changed

+8
-38
lines changed

‎.github/workflows/velox_backend.yml

+8-38
Original file line numberDiff line numberDiff line change
@@ -546,7 +546,7 @@ jobs:
546546
spark: [ "spark-3.2" ]
547547
celeborn: [ "celeborn-0.5.2", "celeborn-0.4.2", "celeborn-0.3.2-incubating" ]
548548
runs-on: ubuntu-20.04
549-
container: ubuntu:22.04
549+
container: apache/gluten:centos-8
550550
steps:
551551
- uses: actions/checkout@v2
552552
- name: Download All Native Artifacts
@@ -559,15 +559,6 @@ jobs:
559559
with:
560560
name: arrow-jars-centos-7-${{github.sha}}
561561
path: /root/.m2/repository/org/apache/arrow/
562-
- name: Setup tzdata
563-
run: |
564-
apt-get update
565-
TZ="Etc/GMT" DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
566-
- name: Setup java and maven
567-
run: |
568-
apt-get update && apt-get install -y openjdk-8-jdk maven wget
569-
apt remove openjdk-11* -y
570-
echo "JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64" >> $GITHUB_ENV
571562
- name: Build for Spark ${{ matrix.spark }}
572563
run: |
573564
cd $GITHUB_WORKSPACE/
@@ -582,7 +573,6 @@ jobs:
582573
fi
583574
echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
584575
cd /opt && mkdir -p celeborn && \
585-
${WGET_CMD} https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \
586576
tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \
587577
mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
588578
bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g' > ./conf/celeborn-env.sh" && \
@@ -608,9 +598,8 @@ jobs:
608598
with:
609599
name: arrow-jars-centos-7-${{github.sha}}
610600
path: /root/.m2/repository/org/apache/arrow/
611-
- name: Prepare spark.test.home for Spark 3.2.2 (other tests)
601+
- name: Prepare
612602
run: |
613-
bash .github/workflows/util/install_spark_resources.sh 3.2
614603
dnf module -y install python39 && \
615604
alternatives --set python3 /usr/bin/python3.9 && \
616605
pip3 install setuptools && \
@@ -652,9 +641,6 @@ jobs:
652641
with:
653642
name: arrow-jars-centos-7-${{github.sha}}
654643
path: /root/.m2/repository/org/apache/arrow/
655-
- name: Prepare spark.test.home for Spark 3.2.2 (slow tests)
656-
run: |
657-
bash .github/workflows/util/install_spark_resources.sh 3.2
658644
- name: Build and run unit test for Spark 3.2.2 (slow tests)
659645
run: |
660646
cd $GITHUB_WORKSPACE/
@@ -683,9 +669,8 @@ jobs:
683669
with:
684670
name: arrow-jars-centos-7-${{github.sha}}
685671
path: /root/.m2/repository/org/apache/arrow/
686-
- name: Prepare spark.test.home for Spark 3.3.1 (other tests)
672+
- name: Prepare
687673
run: |
688-
bash .github/workflows/util/install_spark_resources.sh 3.3
689674
dnf module -y install python39 && \
690675
alternatives --set python3 /usr/bin/python3.9 && \
691676
pip3 install setuptools && \
@@ -728,9 +713,7 @@ jobs:
728713
with:
729714
name: arrow-jars-centos-7-${{github.sha}}
730715
path: /root/.m2/repository/org/apache/arrow/
731-
- name: Prepare spark.test.home for Spark 3.3.1 (slow tests)
732-
run: |
733-
bash .github/workflows/util/install_spark_resources.sh 3.3
716+
734717
- name: Build and Run unit test for Spark 3.3.1 (slow tests)
735718
run: |
736719
cd $GITHUB_WORKSPACE/
@@ -760,9 +743,8 @@ jobs:
760743
with:
761744
name: arrow-jars-centos-7-${{github.sha}}
762745
path: /root/.m2/repository/org/apache/arrow/
763-
- name: Prepare spark.test.home for Spark 3.4.3 (other tests)
746+
- name: Prepare
764747
run: |
765-
bash .github/workflows/util/install_spark_resources.sh 3.4
766748
dnf module -y install python39 && \
767749
alternatives --set python3 /usr/bin/python3.9 && \
768750
pip3 install setuptools && \
@@ -805,9 +787,6 @@ jobs:
805787
with:
806788
name: arrow-jars-centos-7-${{github.sha}}
807789
path: /root/.m2/repository/org/apache/arrow/
808-
- name: Prepare spark.test.home for Spark 3.4.3 (slow tests)
809-
run: |
810-
bash .github/workflows/util/install_spark_resources.sh 3.4
811790
- name: Build and Run unit test for Spark 3.4.3 (slow tests)
812791
run: |
813792
cd $GITHUB_WORKSPACE/
@@ -837,9 +816,8 @@ jobs:
837816
with:
838817
name: arrow-jars-centos-7-${{github.sha}}
839818
path: /root/.m2/repository/org/apache/arrow/
840-
- name: Prepare spark.test.home for Spark 3.5.2 (other tests)
819+
- name: Prepare
841820
run: |
842-
bash .github/workflows/util/install_spark_resources.sh 3.5
843821
dnf module -y install python39 && \
844822
alternatives --set python3 /usr/bin/python3.9 && \
845823
pip3 install setuptools && \
@@ -881,9 +859,8 @@ jobs:
881859
with:
882860
name: arrow-jars-centos-7-${{github.sha}}
883861
path: /root/.m2/repository/org/apache/arrow/
884-
- name: Prepare spark.test.home for Spark 3.5.2 (other tests)
862+
- name: Prepare
885863
run: |
886-
bash .github/workflows/util/install_spark_resources.sh 3.5-scala2.13
887864
dnf module -y install python39 && \
888865
alternatives --set python3 /usr/bin/python3.9 && \
889866
pip3 install setuptools && \
@@ -919,9 +896,6 @@ jobs:
919896
with:
920897
name: arrow-jars-centos-7-${{github.sha}}
921898
path: /root/.m2/repository/org/apache/arrow/
922-
- name: Prepare spark.test.home for Spark 3.5.2 (slow tests)
923-
run: |
924-
bash .github/workflows/util/install_spark_resources.sh 3.5
925899
- name: Build and Run unit test for Spark 3.5.2 (slow tests)
926900
run: |
927901
cd $GITHUB_WORKSPACE/
@@ -951,9 +925,8 @@ jobs:
951925
with:
952926
name: arrow-jars-centos-7-${{github.sha}}
953927
path: /root/.m2/repository/org/apache/arrow/
954-
- name: Prepare spark.test.home for Spark 3.5.2 (other tests)
928+
- name: Prepare
955929
run: |
956-
bash .github/workflows/util/install_spark_resources.sh 3.5
957930
dnf module -y install python39 && \
958931
alternatives --set python3 /usr/bin/python3.9 && \
959932
pip3 install setuptools && \
@@ -988,9 +961,6 @@ jobs:
988961
with:
989962
name: arrow-jars-centos-7-${{github.sha}}
990963
path: /root/.m2/repository/org/apache/arrow/
991-
- name: Prepare spark.test.home for Spark 3.5.2 (slow tests)
992-
run: |
993-
bash .github/workflows/util/install_spark_resources.sh 3.5
994964
- name: Build and Run unit test for Spark 3.5.2 (slow tests)
995965
run: |
996966
cd $GITHUB_WORKSPACE/

0 commit comments

Comments
 (0)