Merge branch 'main' into add-cuda-docker-images

Author: bdice

.github/workflows/build_pyvelox.yml

@@ -46,15 +46,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # TODO: https://github.com/facebookincubator/velox/issues/9014
-  if: false
   build_wheels:
     name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
+        os: [8-core-ubuntu]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -92,6 +90,7 @@ jobs:
 
       - run: mkdir -p ccache
       - name: "Restore ccache"
+        if: false 
         uses: actions/cache/restore@v3
         id: restore-cache
         with:
@@ -101,57 +100,48 @@ jobs:
             ccache-wheels-${{ matrix.os }}-
 
       - name: Install macOS dependencies
-        if: matrix.os == 'macos-11'
+        if: startsWith(matrix.os, 'macos')
         run: |
-          echo "OPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1/" >> $GITHUB_ENV
-          bash scripts/setup-macos.sh &&
-          bash scripts/setup-macos.sh install_folly
+          export INSTALL_PREFIX="$GITHUB_WORKSPACE/dependencies"
+          echo "CMAKE_PREFIX_PATH=$INSTALL_PREFIX" >> $GITHUB_ENV
+          bash scripts/setup-macos.sh 
 
       - name: "Create sdist"
-        if: matrix.os == 'ubuntu-22.04'
+        if: matrix.os == '8-core-ubuntu'
         env:
           BUILD_VERSION: "${{ inputs.version || steps.version.outputs.build_version }}"
         run: |
           python setup.py sdist --dist-dir wheelhouse
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@v2.23.0
         env:
           # required for preadv/pwritev
           MACOSX_DEPLOYMENT_TARGET: "11.0"
           CIBW_ARCHS: "x86_64"
-          # On PRs only build for Python 3.7
-          CIBW_BUILD: ${{ github.event_name == 'pull_request' && 'cp37-*' || 'cp3*' }}
+          # Only build for 3.12 for now
+          CIBW_BUILD: ${{ github.event_name == 'pull_request' && 'cp312-*' || 'cp312-*' }}
           CIBW_SKIP: "*musllinux* cp36-*"
-          CIBW_MANYLINUX_X86_64_IMAGE: "ghcr.io/facebookincubator/velox-dev:torcharrow-avx"
-          CIBW_BEFORE_ALL_LINUX: >
-            mkdir -p /output &&
-            cp -R /host${{ github.workspace }}/ccache /output/ccache &&
-            ccache -s
-          CIBW_ENVIRONMENT_PASS_LINUX: CCACHE_DIR BUILD_VERSION
+          CIBW_MANYLINUX_X86_64_IMAGE: "ghcr.io/facebookincubator/velox-dev:pyvelox"
+          CIBW_ENVIRONMENT_PASS_LINUX: "CCACHE_DIR BUILD_VERSION CMAKE_PREFIX_PATH"
           CIBW_TEST_EXTRAS: "tests"
           CIBW_TEST_COMMAND: "cd {project}/pyvelox && python -m unittest -v"
-          CIBW_TEST_SKIP: "*macos*"
-          CCACHE_DIR: "${{ matrix.os != 'macos-11' && '/output' || github.workspace }}/ccache"
+          CIBW_TEST_SKIP: "*"
+          # CCACHE_DIR: "${{ matrix.os != 'macos-11' && '/output' || github.workspace }}/ccache"
           BUILD_VERSION: "${{ inputs.version || steps.version.outputs.build_version }}"
         with:
           output-dir: wheelhouse
 
-      - name: "Move ccache to workspace"
-        if: matrix.os != 'macos-11'
-        run: |
-          mkdir -p ccache
-          cp -R ./wheelhouse/ccache/* ccache
-
       - name: "Save ccache"
+        if: false
         uses: actions/cache/save@v3
         id: cache
         with:
           path: "ccache"
           key: ccache-wheels-${{ matrix.os }}-${{ github.sha }}
 
       - name: "Rename wheel compatibility tag"
-        if: matrix.os == 'macos-11'
+        if: startsWith(matrix.os, 'macos')
         run: |
           brew install rename
           cd wheelhouse

.github/workflows/docker.yml

@@ -51,6 +51,9 @@ jobs:
           - name: Centos 9
             file: "scripts/centos.dockerfile"
             tags: "ghcr.io/facebookincubator/velox-dev:centos9"
+          - name: Pyvelox 
+            file: "scripts/pyvelox.dockerfile"
+            tags: "ghcr.io/facebookincubator/velox-dev:pyvelox"
           - name: Dev
             file: "scripts/ubuntu-22.04-cpp.dockerfile"
             args: ""

.github/workflows/linux-build-base.yml

@@ -39,6 +39,7 @@ jobs:
       GTest_SOURCE: BUNDLED
       simdjson_SOURCE: BUNDLED
       xsimd_SOURCE: BUNDLED
+      geos_SOURCE: BUNDLED
       CUDA_VERSION: "12.8"
       USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}"
     steps:
@@ -88,6 +89,7 @@ jobs:
             "-DVELOX_ENABLE_BENCHMARKS=ON"
             "-DVELOX_ENABLE_EXAMPLES=ON"
             "-DVELOX_ENABLE_ARROW=ON"
+            "-DVELOX_ENABLE_GEO=ON"
             "-DVELOX_ENABLE_PARQUET=ON"
             "-DVELOX_ENABLE_HDFS=ON"
             "-DVELOX_ENABLE_S3=ON"

.github/workflows/scheduled.yml

@@ -318,7 +318,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: cache_fuzzer
-          path: velox/_build/debug/velox/exec/tests/velox_cache_fuzzer_test
+          path: velox/_build/debug/velox/exec/fuzzer/velox_cache_fuzzer
           retention-days: "${{ env.RETENTION }}"
 
       - name: Upload table evolution fuzzer
@@ -751,6 +751,8 @@ jobs:
     container: ghcr.io/facebookincubator/velox-dev:centos9
     needs: compile
     timeout-minutes: 120
+    # Temporarily disable on PRs till flakiness is fixed #12167
+    if: ${{ github.event_name != 'pull_request' }}
     steps:
 
       - name: Download cache fuzzer
@@ -760,24 +762,24 @@ jobs:
 
       - name: Run Cache Fuzzer
         run: |
-          mkdir -p /tmp/cache_fuzzer_test/logs/
-          chmod -R 777 /tmp/cache_fuzzer_test
-          chmod +x velox_cache_fuzzer_test
-          ./velox_cache_fuzzer_test \
+          mkdir -p /tmp/cache_fuzzer/logs/
+          chmod -R 777 /tmp/cache_fuzzer
+          chmod +x velox_cache_fuzzer
+          ./velox_cache_fuzzer \
                 --seed ${RANDOM} \
                 --duration_sec $DURATION \
                 --minloglevel=0 \
                 --stderrthreshold=2 \
-                --log_dir=/tmp/cache_fuzzer_test/logs \
+                --log_dir=/tmp/cache_fuzzer/logs \
             && echo -e "\n\Cache fuzzer run finished successfully."
 
       - name: Archive Cache production artifacts
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-            name: cache-fuzzer-test-logs
+            name: cache-fuzzer-logs
             path: |
-              /tmp/cache_fuzzer_test
+              /tmp/cache_fuzzer
 
   table-evolution-fuzzer-run:
     name: Table Evolution Fuzzer

CMake/resolve_dependency_modules/README.md

@@ -41,6 +41,7 @@ by Velox. See details on bundling below.
 | DuckDB (testing)  | 0.8.1           | Yes      |
 | cpr (testing)     | 1.10.15         | Yes      |
 | arrow             | 15.0.0          | Yes      |
+| geos              | 3.13.0          | Yes      |
 
 # Bundled Dependency Management
 This module provides a dependency management system that allows us to automatically fetch and build dependencies from source if needed.

CMake/resolve_dependency_modules/geos.cmake

@@ -0,0 +1,31 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+include_guard(GLOBAL)
+
+# GEOS Configuration
+set(VELOX_GEOS_BUILD_VERSION 3.13.0)
+set(VELOX_GEOS_BUILD_SHA256_CHECKSUM
+    47ec83ff334d672b9e4426695f15da6e6368244214971fabf386ff8ef6df39e4)
+string(CONCAT VELOX_GEOS_SOURCE_URL "https://download.osgeo.org/geos/"
+              "geos-${VELOX_GEOS_BUILD_VERSION}.tar.bz2")
+
+velox_resolve_dependency_url(GEOS)
+
+FetchContent_Declare(
+  geos
+  URL ${VELOX_GEOS_SOURCE_URL}
+  URL_HASH ${VELOX_GEOS_BUILD_SHA256_CHECKSUM})
+set(BUILD_SHARED_LIBS ${VELOX_BUILD_SHARED})
+FetchContent_MakeAvailable(geos)
+unset(BUILD_SHARED_LIBS)

CMake/resolve_dependency_modules/gflags.cmake

@@ -43,7 +43,20 @@ set(GFLAGS_IS_SUBPROJECT ON)
 
 # Workaround for https://github.com/gflags/gflags/issues/277
 unset(BUILD_SHARED_LIBS)
+if(DEFINED CACHE{BUILD_SHARED_LIBS})
+  set(CACHED_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+  unset(BUILD_SHARED_LIBS CACHE)
+endif()
+
 FetchContent_MakeAvailable(gflags)
+
+# Workaround for https://github.com/gflags/gflags/issues/277
+if(DEFINED CACHED_BUILD_SHARED_LIBS)
+  set(BUILD_SHARED_LIBS
+      ${CACHED_BUILD_SHARED_LIBS}
+      CACHE BOOL "Restored after setting up gflags" FORCE)
+endif()
+
 # This causes find_package(gflags) in other dependencies to search in the build
 # directory and prevents the system gflags from being found when they don't use
 # the target directly (like folly).

CMake/resolve_dependency_modules/gtest.cmake

@@ -24,12 +24,12 @@ velox_resolve_dependency_url(GTEST)
 
 message(STATUS "Building gtest from source")
 FetchContent_Declare(
-  gtest
+  googletest
   URL ${VELOX_GTEST_SOURCE_URL}
   URL_HASH ${VELOX_GTEST_BUILD_SHA256_CHECKSUM}
   OVERRIDE_FIND_PACKAGE SYSTEM EXCLUDE_FROM_ALL)
 
-FetchContent_MakeAvailable(gtest)
+FetchContent_MakeAvailable(googletest)
 
 # Mask compilation warning in clang 16.
 target_compile_options(gtest PRIVATE -Wno-implicit-int-float-conversion)

CMakeLists.txt

@@ -79,6 +79,8 @@ option(
 option(VELOX_MONO_LIBRARY "Build single unified library." OFF)
 option(ENABLE_ALL_WARNINGS "Enable -Wall and -Wextra compiler warnings." ON)
 option(VELOX_BUILD_SHARED "Build Velox as shared libraries." OFF)
+option(VELOX_SKIP_WAVE_BRANCH_KERNEL_TEST "Disable Wave branch kernel test."
+       OFF)
 # While it's possible to build both in one go we currently want to build either
 # static or shared.
 cmake_dependent_option(
@@ -138,6 +140,7 @@ option(VELOX_ENABLE_ABFS "Build Abfs Connector" OFF)
 option(VELOX_ENABLE_HDFS "Build Hdfs Connector" OFF)
 option(VELOX_ENABLE_PARQUET "Enable Parquet support" ON)
 option(VELOX_ENABLE_ARROW "Enable Arrow support" OFF)
+option(VELOX_ENABLE_GEO "Enable Geospatial support" OFF)
 option(VELOX_ENABLE_REMOTE_FUNCTIONS "Enable remote function support" OFF)
 option(VELOX_ENABLE_CCACHE "Use ccache if installed." ON)
 
@@ -203,6 +206,7 @@ if(${VELOX_BUILD_PYTHON_PACKAGE})
   set(VELOX_ENABLE_AGGREGATES ON)
   set(VELOX_ENABLE_SPARK_FUNCTIONS ON)
   set(VELOX_BUILD_RUNNER OFF)
+  set(VELOX_BUILD_TEST_UTILS ON) # required for velox_exec_test_lib
 endif()
 
 if(${VELOX_ENABLE_DUCKDB})
@@ -493,7 +497,7 @@ if(${VELOX_BUILD_MINIMAL_WITH_DWIO}
 
   # Locate or build protobuf.
   velox_set_source(Protobuf)
-  velox_resolve_dependency(Protobuf CONFIG 3.21.7 REQUIRED)
+  velox_resolve_dependency(Protobuf 3.21.7 REQUIRED)
   include_directories(${Protobuf_INCLUDE_DIRS})
 endif()
 
@@ -594,7 +598,7 @@ if(NOT VELOX_DISABLE_GOOGLETEST)
   velox_set_source(GTest)
   velox_resolve_dependency(GTest)
   set(VELOX_GTEST_INCUDE_DIR
-      "${gtest_SOURCE_DIR}/googletest/include"
+      "${gtest_SOURCE_DIR}/include"
       PARENT_SCOPE)
 endif()
 
@@ -626,4 +630,9 @@ if(VELOX_ENABLE_ARROW)
   velox_resolve_dependency(Arrow)
 endif()
 
+if(VELOX_ENABLE_GEO)
+  velox_set_source(geos)
+  velox_resolve_dependency(geos)
+endif()
+
 add_subdirectory(velox)

CONTRIBUTING.md

@@ -287,7 +287,7 @@ with a benchmark.
    line arguments.
 
    ```
-# Test the new function in isolation. Use --only flag to restrict the set of functions
+   # Test the new function in isolation. Use --only flag to restrict the set of functions
    # and run for 60 seconds or longer.
    velox_expression_fuzzer_test --only <my-new-function-name> --duration_sec 60 --logtostderr=1 --enable_variadic_signatures --velox_fuzzer_enable_complex_types --velox_fuzzer_enable_decimal_type --lazy_vector_generation_ratio 0.2 --velox_fuzzer_enable_column_reuse --velox_fuzzer_enable_expression_reuse
 

scripts/pyvelox.dockerfile

@@ -0,0 +1,34 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# Build the test and build container for presto_cpp
+FROM quay.io/pypa/manylinux_2_28_x86_64:latest
+
+COPY scripts/setup-helper-functions.sh /
+COPY scripts/setup-manylinux.sh /
+
+# Build static folly to reduce wheel size (folly.so is ~120M)
+ENV  VELOX_BUILD_SHARED=OFF
+# The removal of the build dir has to happen in the same layer as the build
+# to minimize the image size. gh & jq are required for CI
+RUN mkdir build && ( cd build && bash /setup-manylinux.sh ) && rm -rf build && \
+        dnf install -y -q 'dnf-command(config-manager)' && \
+        dnf config-manager --add-repo 'https://cli.github.com/packages/rpm/gh-cli.repo' && \
+        dnf install -y -q gh jq && \
+        dnf clean all
+
+ENV CC=/opt/rh/gcc-toolset-12/root/bin/gcc \
+    CXX=/opt/rh/gcc-toolset-12/root/bin/g++
+
+ENTRYPOINT ["/bin/bash", "-c", "source /opt/rh/gcc-toolset-12/enable && exec \"$@\"", "--"]
+CMD ["/bin/bash"]

scripts/setup-centos9.sh

@@ -36,6 +36,7 @@ export CFLAGS=${CXXFLAGS//"-std=c++17"/} # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 VELOX_BUILD_SHARED=${VELOX_BUILD_SHARED:-"OFF"} #Build folly and gflags shared for use in libvelox.so.
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
+BUILD_GEOS="${BUILD_GEOS:-true}"
 USE_CLANG="${USE_CLANG:-false}"
 export INSTALL_PREFIX=${INSTALL_PREFIX:-"/usr/local"}
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
@@ -48,6 +49,7 @@ THRIFT_VERSION="v0.16.0"
 ARROW_VERSION="15.0.0"
 STEMMER_VERSION="2.2.0"
 DUCKDB_VERSION="v0.8.1"
+GEOS_VERSION="3.13.0"
 
 function dnf_install {
   dnf install -y -q --setopt=install_weak_deps=False "$@"
@@ -175,7 +177,7 @@ function install_mvfst {
 }
 
 function install_duckdb {
-  if $BUILD_DUCKDB ; then
+  if [[ "$BUILD_DUCKDB" == "true" ]]; then
     echo 'Building DuckDB'
     wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb
     cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
@@ -239,6 +241,13 @@ function install_cuda {
     cuda-nvrtc-devel-$dashed
 }
 
+function install_geos {
+  if [[ "$BUILD_GEOS" == "true" ]]; then
+    wget_and_untar https://github.com/libgeos/geos/archive/${GEOS_VERSION}.tar.gz geos
+    cmake_install_dir geos -DBUILD_TESTING=OFF
+  fi
+}
+
 function install_velox_deps {
   run_and_time install_velox_deps_from_dnf
   run_and_time install_conda
@@ -258,6 +267,7 @@ function install_velox_deps {
   run_and_time install_stemmer
   run_and_time install_thrift
   run_and_time install_arrow
+  run_and_time install_geos
 }
 
 (return 2> /dev/null) && return # If script was sourced, don't run commands.