[GLUTEN-7458][VL] Upgrade to GCC-11 for centos-7/8 and ubuntu-20.04 (#7578)

Author: PHILO-HE

.github/workflows/build_bundle_package.yml

@@ -49,7 +49,7 @@ jobs:
           yum install sudo patch java-1.8.0-openjdk-devel -y && \
           cd $GITHUB_WORKSPACE/ep/build-velox/src && \
           ./get_velox.sh && \
-          source /opt/rh/devtoolset-9/enable && \
+          source /opt/rh/devtoolset-11/enable && \
           cd $GITHUB_WORKSPACE/ && \
           export NUM_THREADS=4
           ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=OFF  --build_benchmarks=OFF --enable_s3=OFF \

.github/workflows/velox_weekly.yml

@@ -55,12 +55,14 @@ jobs:
             -e 's/^#baseurl/baseurl/' \
             -e 's/mirror\.centos\.org/vault.centos.org/' \
             /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
+            yum install -y devtoolset-11
+            source /opt/rh/devtoolset-11/enable
           elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
             dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
             source /opt/rh/gcc-toolset-12/enable || exit 1
           else
-            dnf install -y --setopt=install_weak_deps=False gcc-toolset-9
-            source /opt/rh/gcc-toolset-9/enable || exit 1
+            dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
+            source /opt/rh/gcc-toolset-11/enable || exit 1
           fi
           yum install -y java-1.8.0-openjdk-devel patch wget git perl
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
@@ -87,6 +89,13 @@ jobs:
           # To avoid the prompt for region selection during installing tzdata.
           export DEBIAN_FRONTEND=noninteractive
           apt-get update && apt-get install -y sudo maven wget git
+          if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then
+            sudo apt install -y software-properties-common
+            sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+            sudo apt update && sudo apt install -y gcc-11 g++-11
+            sudo ln -sf /usr/bin/gcc-11 /usr/bin/gcc
+            sudo ln -sf /usr/bin/g++-11 /usr/bin/g++
+          fi
           sudo apt-get install -y openjdk-8-jdk
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh

dev/build_helper_functions.sh

@@ -165,7 +165,7 @@ function setup_linux {
       $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
       set +u
       export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
-      source /opt/rh/devtoolset-9/enable
+      source /opt/rh/devtoolset-11/enable
       set -u
       ;;
     *)
@@ -179,7 +179,7 @@ function setup_linux {
       $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
       set +u
       export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
-      source /opt/rh/devtoolset-9/enable
+      source /opt/rh/devtoolset-11/enable
       set -u
       ;;
     3) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
@@ -194,7 +194,7 @@ function setup_linux {
         $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
         set +u
         export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
-        source /opt/rh/devtoolset-9/enable
+        source /opt/rh/devtoolset-11/enable
         set -u
         ;;
     3.2) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;

dev/ci-velox-buildshared-centos-8.sh

@@ -2,6 +2,8 @@
 
 set -e
 
-source /opt/rh/gcc-toolset-9/enable
+# TODO: will be removed after docker is updated.
+dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
+source /opt/rh/gcc-toolset-11/enable
 ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --enable_ep_cache=OFF --build_tests=ON \
     --build_examples=ON --build_benchmarks=ON

dev/ci-velox-buildstatic-centos-7.sh

@@ -2,7 +2,9 @@
 
 set -e
 
-source /opt/rh/devtoolset-9/enable
+# TODO: will be removed after docker is updated.
+sudo yum install -y devtoolset-11
+source /opt/rh/devtoolset-11/enable
 export NUM_THREADS=4
 ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \
                            --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON

dev/docker/Dockerfile.centos7-static-build

@@ -11,7 +11,7 @@ RUN sed -i \
        -e 's/mirror\.centos\.org/vault.centos.org/' \
        /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
 
-RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel
+RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel ccache
 
 RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
 
@@ -26,7 +26,7 @@ RUN mkdir -p /var/cache/vcpkg
 ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite
 
 # Build arrow, then install the native libs to system paths and jar package to .m2/ directory.
-RUN cd /opt/gluten && source /opt/rh/devtoolset-9/enable && \
+RUN cd /opt/gluten && source /opt/rh/devtoolset-11/enable && \
     bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
                                     --enable_hdfs=ON --enable_abfs=ON  build_arrow && \
     rm -rf /opt/gluten

dev/docker/Dockerfile.centos8-dynamic-build

@@ -3,8 +3,8 @@ FROM centos:8
 RUN sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
 RUN sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
 
-RUN yum update -y && yum install -y epel-release sudo dnf
-RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-9
+RUN yum update -y && yum install -y epel-release sudo dnf ccache
+RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
 RUN echo "check_certificate = off" >> ~/.wgetrc
 
 RUN yum install -y java-1.8.0-openjdk-devel patch wget git perl
@@ -17,4 +17,4 @@ ENV PATH=${PATH}:/usr/lib/maven/bin
 
 RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
 
-RUN cd /opt/gluten && source /opt/rh/gcc-toolset-9/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten
+RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten

dev/docker/entrypoint.sh

@@ -1,4 +1,4 @@
 #! /bin/sh
 
-. /opt/rh/devtoolset-9/enable
+. /opt/rh/devtoolset-11/enable
 exec "$@"

dev/package-vcpkg.sh

@@ -11,10 +11,10 @@ ARCH=`uname -m`
 cd "$GLUTEN_DIR"
 if [ "$LINUX_OS" == "centos" ]; then
   if [ "$VERSION" == "8" ]; then
-    source /opt/rh/gcc-toolset-9/enable
+    source /opt/rh/gcc-toolset-11/enable
   elif [ "$VERSION" == "7" ]; then
     export MANPATH=""
-    source /opt/rh/devtoolset-9/enable
+    source /opt/rh/devtoolset-11/enable
   fi
 fi
 

dev/vcpkg/README.md

@@ -15,14 +15,14 @@ Please install build depends on your system to compile all libraries:
 sudo $GLUTEN_REPO/dev/vcpkg/setup-build-depends.sh
 ```
 
-For CentOS user, gcc 9 needs to be enabled manually before next step:
+For CentOS user, gcc 11 needs to be enabled manually before next step:
 
 ``` sh
 # CentOS 8
-source /opt/rh/gcc-toolset-9/enable
+source /opt/rh/gcc-toolset-11/enable
 
 # CentOS 7
-source /opt/rh/devtoolset-9/enable
+source /opt/rh/devtoolset-11/enable
 ```
 
 For unsupported linux distro, you can install the following packages from package manager.

dev/vcpkg/setup-build-depends.sh

@@ -26,18 +26,17 @@ install_maven_from_source() {
     fi
 }
 
-install_gcc9_from_source() {
+install_gcc11_from_source() {
      cur_gcc_version=$(gcc -dumpversion)
-     if [ "$(semver "$cur_gcc_version")" -lt "$(semver 9.0.0)" ]; then
-            gcc_version=gcc-9.4.0
+     if [ "$(semver "$cur_gcc_version")" -lt "$(semver 11.0.0)" ]; then
+            gcc_version=gcc-11.5.0
             gcc_install_dir=/usr/local/${gcc_version}
             cd /tmp
             if [ ! -d $gcc_version ]; then
                 wget https://ftp.gnu.org/gnu/gcc/${gcc_version}/${gcc_version}.tar.gz
                 tar -xvf ${gcc_version}.tar.gz
             fi
             cd ${gcc_version}
-            sed -i 's/ftp/https/g' contrib/download_prerequisites
             ./contrib/download_prerequisites
 
             mkdir gcc-build && cd gcc-build
@@ -68,7 +67,7 @@ install_centos_7() {
     yum -y install \
         wget curl tar zip unzip which patch sudo \
         ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
-        devtoolset-9 python3 pip dnf \
+        devtoolset-11 python3 pip dnf \
         bison \
         java-1.8.0-openjdk java-1.8.0-openjdk-devel
 
@@ -126,7 +125,7 @@ install_centos_8() {
     yum -y install \
         wget curl tar zip unzip git which sudo patch \
         cmake perl-IPC-Cmd autoconf automake libtool \
-        gcc-toolset-9-gcc gcc-toolset-9-gcc-c++ \
+        gcc-toolset-11 \
         flex bison python3 \
         java-1.8.0-openjdk java-1.8.0-openjdk-devel
 
@@ -135,60 +134,18 @@ install_centos_8() {
     install_maven_from_source
 }
 
-install_ubuntu_18.04() {
-    # Support for gcc-9 and g++-9
-    apt-get update && apt-get install -y software-properties-common
-    add-apt-repository -y ppa:ubuntu-toolchain-r/test
-
-    apt-get -y install \
-        wget curl tar zip unzip git \
-        build-essential ccache ninja-build pkg-config autoconf autoconf-archive libtool \
-        flex bison \
-        openjdk-8-jdk \
-        gcc-9 g++-9
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 900 --slave /usr/bin/g++ g++ /usr/bin/g++-9
-
-    # Install cmake 3.28.1 from source
-    apt remove -y --purge --auto-remove cmake
-    apt-get install -y libssl-dev
-
-    version=3.28
-    build=1
-    mkdir cmake_install
-    cd cmake_install
-    wget https://cmake.org/files/v$version/cmake-$version.$build.tar.gz
-    tar -xzvf cmake-$version.$build.tar.gz
-    cd cmake-$version.$build/
-
-    ./bootstrap
-    make -j$(nproc)
-    make install
-
-    cd ../../
-    rm -rf cmake_install
-    ln -fs /usr/local/bin/cmake /usr/bin/cmake
-
-    # Install automake 1.16
-    mkdir -p /tmp/automake
-    wget -O - http://ftp.gnu.org/gnu/automake/automake-1.16.5.tar.xz | tar -x --xz -C /tmp/automake --strip-components=1
-    cd /tmp/automake
-    ./configure
-    make install -j
-    cd
-    rm -rf /tmp/automake
-
-    # Fix aclocal search path
-    echo /usr/share/aclocal > /usr/local/share/aclocal/dirlist
-
-    install_maven_from_source
-}
-
 install_ubuntu_20.04() {
     apt-get update && apt-get -y install \
         wget curl tar zip unzip git \
         build-essential ccache cmake ninja-build pkg-config autoconf autoconf-archive libtool \
         flex bison \
         openjdk-8-jdk maven
+    # Overwrite gcc-9 installed by build-essential.
+    sudo apt install -y software-properties-common
+    sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+    sudo apt update && sudo apt install -y gcc-11 g++-11
+    sudo ln -sf /usr/bin/gcc-11 /usr/bin/gcc
+    sudo ln -sf /usr/bin/g++-11 /usr/bin/g++
 }
 
 install_ubuntu_22.04() { install_ubuntu_20.04; }
@@ -206,7 +163,7 @@ install_tencentos_3.2() {
     yum -y install \
         wget curl tar zip unzip git which \
         cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
-        gcc-toolset-9-gcc gcc-toolset-9-gcc-c++ \
+        gcc-toolset-11 \
         flex bison python3 \
         java-8-konajdk
 
@@ -229,7 +186,7 @@ install_debian_10() {
     apt update && apt-get -y install temurin-8-jdk
 
     install_maven_from_source
-    install_gcc9_from_source
+    install_gcc11_from_source
 }
 
 install_debian_11() {

ep/build-velox/src/get_velox.sh

@@ -69,6 +69,17 @@ function process_setup_ubuntu {
   # Do not install libunwind which can cause interruption when catching native exception.
   ensure_pattern_matched '\${SUDO} apt install -y libunwind-dev' scripts/setup-ubuntu.sh
   sed -i 's/${SUDO} apt install -y libunwind-dev//' scripts/setup-ubuntu.sh
+  # Overwrite gcc installed by build-essential.
+  ensure_pattern_matched '\${SUDO} pip3 install cmake==3.28.3' scripts/setup-ubuntu.sh
+  sed -i '/^  ${SUDO} pip3 install cmake==3.28.3/a\
+  \VERSION=`cat /etc/os-release | grep VERSION_ID`\
+  if [[ $VERSION =~ "20.04" ]]; then\
+    sudo apt install -y software-properties-common\
+    sudo add-apt-repository ppa:ubuntu-toolchain-r/test\
+    sudo apt update && sudo apt install -y gcc-11 g++-11\
+    sudo ln -sf /usr/bin/gcc-11 /usr/bin/gcc\
+    sudo ln -sf /usr/bin/g++-11 /usr/bin/g++\
+  fi' scripts/setup-ubuntu.sh
   ensure_pattern_matched 'ccache' scripts/setup-ubuntu.sh
   sed -i '/ccache/a\    *thrift* \\' scripts/setup-ubuntu.sh
   sed -i '/ccache/a\    libiberty-dev \\' scripts/setup-ubuntu.sh
@@ -121,10 +132,10 @@ function process_setup_alinux3 {
   sed -i "s/.*dnf_install epel-release/#&/" ${CURRENT_DIR}/setup-centos8.sh
   sed -i "s/.*run_and_time install_conda/#&/" ${CURRENT_DIR}/setup-centos8.sh
   sed -i "s/.*dnf config-manager --set-enabled powertools/#&/" ${CURRENT_DIR}/setup-centos8.sh
-  sed -i "s/gcc-toolset-9 //" ${CURRENT_DIR}/setup-centos8.sh
-  sed -i "s/.*source \/opt\/rh\/gcc-toolset-9\/enable/#&/" ${CURRENT_DIR}/setup-centos8.sh
-  sed -i 's|^export CC=/opt/rh/gcc-toolset-9/root/bin/gcc|# &|' ${CURRENT_DIR}/setup-centos8.sh
-  sed -i 's|^export CXX=/opt/rh/gcc-toolset-9/root/bin/g++|# &|' ${CURRENT_DIR}/setup-centos8.sh
+  sed -i "s/gcc-toolset-11 //" ${CURRENT_DIR}/setup-centos8.sh
+  sed -i "s/.*source \/opt\/rh\/gcc-toolset-11\/enable/#&/" ${CURRENT_DIR}/setup-centos8.sh
+  sed -i 's|^export CC=/opt/rh/gcc-toolset-11/root/bin/gcc|# &|' ${CURRENT_DIR}/setup-centos8.sh
+  sed -i 's|^export CXX=/opt/rh/gcc-toolset-11/root/bin/g++|# &|' ${CURRENT_DIR}/setup-centos8.sh
   sed -i 's/python39 python39-devel python39-pip //g' ${CURRENT_DIR}/setup-centos8.sh
   sed -i "s/.*pip.* install/#&/" ${CURRENT_DIR}/setup-centos8.sh
 }

ep/build-velox/src/setup-centos7.sh

@@ -157,7 +157,7 @@ function install_lzo {
 
 function install_boost {
   # Remove old version.
-  sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_* /opt/rh/devtoolset-9/root/usr/lib64/dyninst/libboost_*
+  sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_* /opt/rh/devtoolset-11/root/usr/lib64/dyninst/libboost_*
   sudo rm -rf /tmp/velox-deps/boost/ /usr/local/include/boost/ /usr/local/lib/cmake/Boost-1.72.0/
   cd "${DEPENDENCY_DIR}"
   wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost
@@ -239,8 +239,6 @@ $SUDO dnf makecache
 
 # dnf install dependency libraries
 dnf_install epel-release dnf-plugins-core # For ccache, ninja
-# PowerTools only works on CentOS8
-# dnf config-manager --set-enabled powertools
 dnf_install ccache wget which libevent-devel \
   yasm \
   openssl-devel libzstd-devel lz4-devel double-conversion-devel \
@@ -254,16 +252,10 @@ dnf_install autoconf automake libtool bison python3 python3-devel
 # Required for build flex
 dnf_install gettext-devel texinfo help2man
 
-# dnf_install conda
-
-# Activate gcc9; enable errors on unset variables afterwards.
-# GCC9 install via yum and devtoolset
-# dnf install gcc-toolset-9 only works on CentOS8
-
 $SUDO yum makecache
 yum_install centos-release-scl
-yum_install devtoolset-9
-source /opt/rh/devtoolset-9/enable || exit 1
+yum_install devtoolset-11
+source /opt/rh/devtoolset-11/enable || exit 1
 gcc --version
 set -u
 

ep/build-velox/src/setup-centos8.sh

@@ -38,8 +38,8 @@ export CXXFLAGS=$CFLAGS  # Used by boost.
 export CPPFLAGS=$CFLAGS  # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
-export CC=/opt/rh/gcc-toolset-9/root/bin/gcc
-export CXX=/opt/rh/gcc-toolset-9/root/bin/g++
+export CC=/opt/rh/gcc-toolset-11/root/bin/gcc
+export CXX=/opt/rh/gcc-toolset-11/root/bin/g++
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
 FB_OS_VERSION="v2024.05.20.00"
@@ -56,7 +56,7 @@ function install_build_prerequisites {
   dnf_install epel-release dnf-plugins-core # For ccache, ninja
   dnf config-manager --set-enabled powertools
   dnf update -y
-  dnf_install ninja-build curl ccache gcc-toolset-9 git wget which
+  dnf_install ninja-build curl ccache gcc-toolset-11 git wget which
   dnf_install yasm
   dnf_install autoconf automake python39 python39-devel python39-pip libtool
   pip3.9 install cmake==3.28.3
@@ -198,8 +198,8 @@ function install_velox_deps {
 
 (
   if [[ $# -ne 0 ]]; then
-    # Activate gcc9; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-9/enable || exit 1
+    # Activate gcc11; enable errors on unset variables afterwards.
+    source /opt/rh/gcc-toolset-11/enable || exit 1
     set -u
     for cmd in "$@"; do
       run_and_time "${cmd}"
@@ -212,8 +212,8 @@ function install_velox_deps {
     else
       echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
     fi
-    # Activate gcc9; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-9/enable || exit 1
+    # Activate gcc11; enable errors on unset variables afterwards.
+    source /opt/rh/gcc-toolset-11/enable || exit 1
     set -u
     install_velox_deps
     echo "All dependencies for Velox installed!"