refactor(build): Separate thrift and arrow install on Linux (#12034)

Summary:
This PR addresses two issues.
1. Address downloading reliability from archive.apache.org by switching to github.
2. Remove the bundling of thrift from arrow and make it a separate install.

This follows a PR where the bundled version of arrow still bundles thrift but has changed download locations.

Pull Request resolved: #12034

Reviewed By: kgpai

Differential Revision: D68174931

Pulled By: kagamiori

fbshipit-source-id: 539d860828ca70b0c0ef025580751bc5b8ccda68

Author: czentgr

scripts/setup-centos9.sh

@@ -43,6 +43,8 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 FB_OS_VERSION="v2024.07.01.00"
 FMT_VERSION="10.1.1"
 BOOST_VERSION="boost-1.84.0"
+THRIFT_VERSION="v0.16.0"
+# Note: when updating arrow check if thrift needs an update as well.
 ARROW_VERSION="15.0.0"
 STEMMER_VERSION="2.2.0"
 DUCKDB_VERSION="v0.8.1"
@@ -191,8 +193,23 @@ function install_stemmer {
   )
 }
 
+function install_thrift {
+  wget_and_untar https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz thrift
+  (
+    cd ${DEPENDENCY_DIR}/thrift
+    ./bootstrap.sh
+    EXTRA_CXXFLAGS="-O3 -fPIC"
+    # Clang will generate warnings and they need to be suppressed, otherwise the build will fail.
+    if [[ ${USE_CLANG} != "false" ]]; then
+      EXTRA_CXXFLAGS="-O3 -fPIC -Wno-inconsistent-missing-override -Wno-unused-but-set-variable"
+    fi
+    ./configure --prefix=${INSTALL_PREFIX} --enable-tests=no --enable-tutorial=no --with-boost=${INSTALL_PREFIX} CXXFLAGS="${EXTRA_CXXFLAGS}"
+    make "-j${NPROC}" install
+  )
+}
+
 function install_arrow {
-  wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
+  wget_and_untar https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz arrow
   cmake_install_dir arrow/cpp \
     -DARROW_PARQUET=OFF \
     -DARROW_WITH_THRIFT=ON \
@@ -208,21 +225,14 @@ function install_arrow {
     -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
     -DCMAKE_BUILD_TYPE=Release \
     -DARROW_BUILD_STATIC=ON \
-    -DThrift_SOURCE=BUNDLED \
     -DBOOST_ROOT=${INSTALL_PREFIX}
-
-  (
-    # Install thrift.
-    cd ${DEPENDENCY_DIR}/arrow/cpp/_build/thrift_ep-prefix/src/thrift_ep-build
-    cmake --install ./ --prefix ${INSTALL_PREFIX}
-  )
 }
 
 function install_cuda {
   # See https://developer.nvidia.com/cuda-downloads
   dnf config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel9/x86_64/cuda-rhel9.repo
   local dashed="$(echo $1 | tr '.' '-')"
-  dnf install -y cuda-nvcc-$dashed cuda-cudart-devel-$dashed cuda-nvrtc-devel-$dashed cuda-driver-devel-$dashed 
+  dnf install -y cuda-nvcc-$dashed cuda-cudart-devel-$dashed cuda-nvrtc-devel-$dashed cuda-driver-devel-$dashed
 }
 
 function install_velox_deps {
@@ -242,6 +252,7 @@ function install_velox_deps {
   run_and_time install_fbthrift
   run_and_time install_duckdb
   run_and_time install_stemmer
+  run_and_time install_thrift
   run_and_time install_arrow
 }
 

scripts/setup-ubuntu.sh

@@ -75,6 +75,8 @@ function install_gcc11_if_needed {
 FB_OS_VERSION="v2024.07.01.00"
 FMT_VERSION="10.1.1"
 BOOST_VERSION="boost-1.84.0"
+THRIFT_VERSION="v0.16.0"
+# Note: when updating arrow check if thrift needs an update as well.
 ARROW_VERSION="15.0.0"
 STEMMER_VERSION="2.2.0"
 DUCKDB_VERSION="v0.8.1"
@@ -94,13 +96,14 @@ function install_build_prerequisites {
     checkinstall \
     git \
     pkg-config \
+    libtool \
     wget
-  
+
   if [ ! -f ${PYTHON_VENV}/pyvenv.cfg ]; then
     echo "Creating Python Virtual Environment at ${PYTHON_VENV}"
     python3 -m venv ${PYTHON_VENV}
   fi
-  source ${PYTHON_VENV}/bin/activate;   
+  source ${PYTHON_VENV}/bin/activate;
   # Install to /usr/local to make it available to all users.
   ${SUDO} pip3 install cmake==3.28.3
 
@@ -244,8 +247,23 @@ function install_stemmer {
   )
 }
 
+function install_thrift {
+  wget_and_untar https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz thrift
+  (
+    cd ${DEPENDENCY_DIR}/thrift
+    ./bootstrap.sh
+    EXTRA_CXXFLAGS="-O3 -fPIC"
+    # Clang will generate warnings and they need to be suppressed, otherwise the build will fail.
+    if [[ ${USE_CLANG} != "false" ]]; then
+      EXTRA_CXXFLAGS="-O3 -fPIC -Wno-inconsistent-missing-override -Wno-unused-but-set-variable"
+    fi
+    ./configure --prefix=${INSTALL_PREFIX} --enable-tests=no --enable-tutorial=no --with-boost=${INSTALL_PREFIX} CXXFLAGS="${EXTRA_CXXFLAGS}"
+    make "-j${NPROC}" install
+  )
+}
+
 function install_arrow {
-  wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
+  wget_and_untar https://github.com/apache/arrow/archive/apache-arrow-${ARROW_VERSION}.tar.gz arrow
   cmake_install_dir arrow/cpp \
     -DARROW_PARQUET=OFF \
     -DARROW_WITH_THRIFT=ON \
@@ -261,14 +279,7 @@ function install_arrow {
     -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
     -DCMAKE_BUILD_TYPE=Release \
     -DARROW_BUILD_STATIC=ON \
-    -DThrift_SOURCE=BUNDLED \
     -DBOOST_ROOT=${INSTALL_PREFIX}
-
-  (
-    # Install thrift.
-    cd ${DEPENDENCY_DIR}/arrow/cpp/_build/thrift_ep-prefix/src/thrift_ep-build
-    $SUDO cmake --install ./ --prefix ${INSTALL_PREFIX}
-  )
 }
 
 function install_cuda {
@@ -296,6 +307,7 @@ function install_velox_deps {
   run_and_time install_conda
   run_and_time install_duckdb
   run_and_time install_stemmer
+  run_and_time install_thrift
   run_and_time install_arrow
 }