refactor(build): Separate thrift and arrow install on Linux

This PR addresses two issues.
1. Address downloading reliability from archive.apache.org by switching to github.
2. Remove the bundling of thrift from arrow and make it a separate install.

This follows a PR where the bundled version of arrow still bundles thrift but has
changed download locations.

Author: czentgr

scripts/setup-centos9.sh

@@ -43,6 +43,8 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 FB_OS_VERSION="v2024.07.01.00"
 FMT_VERSION="10.1.1"
 BOOST_VERSION="boost-1.84.0"
+# Note: when updating arrow check if thrift needs an update as well.
+THRIFT_VERSION="v0.16.0"
 ARROW_VERSION="15.0.0"
 STEMMER_VERSION="2.2.0"
 DUCKDB_VERSION="v0.8.1"
@@ -191,8 +193,23 @@ function install_stemmer {
   )
 }
 
+function install_thrift {
+  wget_and_untar https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz thrift
+  (
+    cd ${DEPENDENCY_DIR}/thrift
+    ./bootstrap.sh
+    EXTRA_CXXFLAGS="-O3 -fPIC"
+    # Clang will generate warnings and they need to be suppressed, otherwise the build will fail.
+    if [[ ${USE_CLANG} != "false" ]]; then
+      EXTRA_CXXFLAGS="-O3 -fPIC -Wno-inconsistent-missing-override -Wno-unused-but-set-variable"
+    fi
+    ./configure --prefix=${INSTALL_PREFIX} --enable-tests=no --enable-tutorial=no --with-boost=${INSTALL_PREFIX} CXXFLAGS="${EXTRA_CXXFLAGS}"
+    make "-j${NPROC}" install
+  )
+}
+
 function install_arrow {
-  wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
+  wget_and_untar https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz arrow
   cmake_install_dir arrow/cpp \
     -DARROW_PARQUET=OFF \
     -DARROW_WITH_THRIFT=ON \
@@ -208,21 +225,14 @@ function install_arrow {
     -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
     -DCMAKE_BUILD_TYPE=Release \
     -DARROW_BUILD_STATIC=ON \
-    -DThrift_SOURCE=BUNDLED \
     -DBOOST_ROOT=${INSTALL_PREFIX}
-
-  (
-    # Install thrift.
-    cd ${DEPENDENCY_DIR}/arrow/cpp/_build/thrift_ep-prefix/src/thrift_ep-build
-    cmake --install ./ --prefix ${INSTALL_PREFIX}
-  )
 }
 
 function install_cuda {
   # See https://developer.nvidia.com/cuda-downloads
   dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
   local dashed="$(echo $1 | tr '.' '-')"
-  dnf install -y cuda-nvcc-$dashed cuda-cudart-devel-$dashed cuda-nvrtc-devel-$dashed cuda-driver-devel-$dashed 
+  dnf install -y cuda-nvcc-$dashed cuda-cudart-devel-$dashed cuda-nvrtc-devel-$dashed cuda-driver-devel-$dashed
 }
 
 function install_velox_deps {
@@ -242,6 +252,7 @@ function install_velox_deps {
   run_and_time install_fbthrift
   run_and_time install_duckdb
   run_and_time install_stemmer
+  run_and_time install_thrift
   run_and_time install_arrow
 }
 

scripts/setup-ubuntu.sh

@@ -75,6 +75,8 @@ function install_gcc11_if_needed {
 FB_OS_VERSION="v2024.07.01.00"
 FMT_VERSION="10.1.1"
 BOOST_VERSION="boost-1.84.0"
+# Note: when updating arrow check if thrift needs an update as well.
+THRIFT_VERSION="v0.16.0"
 ARROW_VERSION="15.0.0"
 STEMMER_VERSION="2.2.0"
 DUCKDB_VERSION="v0.8.1"
@@ -95,12 +97,12 @@ function install_build_prerequisites {
     git \
     pkg-config \
     wget
-  
+
   if [ ! -f ${PYTHON_VENV}/pyvenv.cfg ]; then
     echo "Creating Python Virtual Environment at ${PYTHON_VENV}"
     python3 -m venv ${PYTHON_VENV}
   fi
-  source ${PYTHON_VENV}/bin/activate;   
+  source ${PYTHON_VENV}/bin/activate;
   # Install to /usr/local to make it available to all users.
   ${SUDO} pip3 install cmake==3.28.3
 
@@ -244,8 +246,23 @@ function install_stemmer {
   )
 }
 
+function install_thrift {
+  wget_and_untar https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz thrift
+  (
+    cd ${DEPENDENCY_DIR}/thrift
+    ./bootstrap.sh
+    EXTRA_CXXFLAGS="-O3 -fPIC"
+    # Clang will generate warnings and they need to be suppressed, otherwise the build will fail.
+    if [[ ${USE_CLANG} != "false" ]]; then
+      EXTRA_CXXFLAGS="-O3 -fPIC -Wno-inconsistent-missing-override -Wno-unused-but-set-variable"
+    fi
+    ./configure --prefix=${INSTALL_PREFIX} --enable-tests=no --enable-tutorial=no --with-boost=${INSTALL_PREFIX} CXXFLAGS="${EXTRA_CXXFLAGS}"
+    make "-j${NPROC}" install
+  )
+}
+
 function install_arrow {
-  wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
+  wget_and_untar https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz arrow
   cmake_install_dir arrow/cpp \
     -DARROW_PARQUET=OFF \
     -DARROW_WITH_THRIFT=ON \
@@ -261,14 +278,7 @@ function install_arrow {
     -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
     -DCMAKE_BUILD_TYPE=Release \
     -DARROW_BUILD_STATIC=ON \
-    -DThrift_SOURCE=BUNDLED \
     -DBOOST_ROOT=${INSTALL_PREFIX}
-
-  (
-    # Install thrift.
-    cd ${DEPENDENCY_DIR}/arrow/cpp/_build/thrift_ep-prefix/src/thrift_ep-build
-    $SUDO cmake --install ./ --prefix ${INSTALL_PREFIX}
-  )
 }
 
 function install_cuda {
@@ -296,6 +306,7 @@ function install_velox_deps {
   run_and_time install_conda
   run_and_time install_duckdb
   run_and_time install_stemmer
+  run_and_time install_thrift
   run_and_time install_arrow
 }