Merge branch 'main' into fea/add-TableWriterBuilder-insertHandle-method

Author: mhaseeb123

.github/workflows/build_pyvelox.yml

@@ -32,10 +32,6 @@ on:
   #   - cron: '15 0 * * *'
   pull_request:
     paths:
-      - 'velox/**'
-      - '!velox/docs/**'
-      - 'third_party/**'
-      - 'pyvelox/**'
       - '.github/workflows/build_pyvelox.yml'
 
 permissions:
@@ -46,23 +42,22 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  # TODO: https://github.com/facebookincubator/velox/issues/9014
-  if: false
   build_wheels:
     name: Build wheels on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04]
+        os: [8-core-ubuntu, macos-13, macos-14]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           ref: ${{ inputs.ref || github.ref }}
           fetch-depth: 0
           submodules: recursive
+          persist-credentials: false
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: '3.10'
 
@@ -90,76 +85,87 @@ jobs:
           # NEXT_VERSION=$(echo $VERSION | awk -F. -v OFS=. '{$NF++ ; print}')
           echo "build_version=${VERSION}a${COMMITS_SINCE_TAG}" >> $GITHUB_OUTPUT
 
-      - run: mkdir -p ccache
       - name: "Restore ccache"
-        uses: actions/cache/restore@v3
+        # CCache doesn't hit in cibuildwheel
+        if: startsWith(matrix.os, 'macos')
+        uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
         id: restore-cache
         with:
           path: "ccache"
-          key: ccache-wheels-${{ matrix.os }}-${{ github.sha }}
-          restore-keys: |
-            ccache-wheels-${{ matrix.os }}-
+          key: ccache-wheels-${{ matrix.os }}
+
+      - name: "Restore macOS Dependencies"
+        if: startsWith(matrix.os, 'macos')
+        uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        id: restore-deps
+        with:
+          path: "dependencies"
+          key: dependencies-pyvelox-${{ matrix.os }}-${{ hashFiles('scripts/setup-macos.sh') }}
 
       - name: Install macOS dependencies
-        if: matrix.os == 'macos-11'
+        if: ${{ startsWith(matrix.os, 'macos') && steps.restore-deps.outputs.stash-hit != 'true' }}
+        env:
+          MACOSX_DEPLOYMENT_TARGET: "13.0"
         run: |
-          echo "OPENSSL_ROOT_DIR=/usr/local/opt/openssl@1.1/" >> $GITHUB_ENV
-          bash scripts/setup-macos.sh &&
-          bash scripts/setup-macos.sh install_folly
+          export INSTALL_PREFIX="$GITHUB_WORKSPACE/dependencies"
+          echo "CMAKE_PREFIX_PATH=$INSTALL_PREFIX" >> $GITHUB_ENV
+          echo "DYLD_LIBRARY_PATH=$INSTALL_PREFIX/lib:$DYLD_LIBRARY_PATH" >> $GITHUB_ENV
+          bash scripts/setup-macos.sh
+
+      - name: "Save macOS Dependencies"
+        if: ${{ startsWith(matrix.os, 'macos') && steps.restore-deps.outputs.stash-hit != 'true' }}
+        uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
+        with:
+          path: "dependencies"
+          key: dependencies-pyvelox-${{ matrix.os }}-${{ hashFiles('scripts/setup-macos.sh') }}
+          retention-days: 90
 
       - name: "Create sdist"
-        if: matrix.os == 'ubuntu-22.04'
+        if: matrix.os == '8-core-ubuntu'
         env:
           BUILD_VERSION: "${{ inputs.version || steps.version.outputs.build_version }}"
         run: |
           python setup.py sdist --dist-dir wheelhouse
 
       - name: Build wheels
-        uses: pypa/cibuildwheel@v2.22.0
+        uses: pypa/cibuildwheel@6cccd09a31908ffd175b012fb8bf4e1dbda3bc6c # v2
         env:
-          # required for preadv/pwritev
-          MACOSX_DEPLOYMENT_TARGET: "11.0"
-          CIBW_ARCHS: "x86_64"
-          # On PRs only build for Python 3.7
-          CIBW_BUILD: ${{ github.event_name == 'pull_request' && 'cp37-*' || 'cp3*' }}
+          MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-14' && '14' || '13' }}
+          CIBW_ARCHS: "native"
+          # Only build for 3.12 for now
+          CIBW_BUILD: ${{ github.event_name == 'pull_request' && 'cp312-*' || 'cp312-*' }}
           CIBW_SKIP: "*musllinux* cp36-*"
-          CIBW_MANYLINUX_X86_64_IMAGE: "ghcr.io/facebookincubator/velox-dev:torcharrow-avx"
-          CIBW_BEFORE_ALL_LINUX: >
-            mkdir -p /output &&
-            cp -R /host${{ github.workspace }}/ccache /output/ccache &&
-            ccache -s
-          CIBW_ENVIRONMENT_PASS_LINUX: CCACHE_DIR BUILD_VERSION
+          CIBW_MANYLINUX_X86_64_IMAGE: "ghcr.io/facebookincubator/velox-dev:pyvelox"
+          CIBW_ENVIRONMENT_PASS_LINUX:  "BUILD_VERSION"
+          # Don't waste time writing out to cache
+          CIBW_ENVIRONMENT_LINUX: "CCACHE_DISABLE=true"
           CIBW_TEST_EXTRAS: "tests"
           CIBW_TEST_COMMAND: "cd {project}/pyvelox && python -m unittest -v"
-          CIBW_TEST_SKIP: "*macos*"
-          CCACHE_DIR: "${{ matrix.os != 'macos-11' && '/output' || github.workspace }}/ccache"
+          CIBW_TEST_SKIP: "*"
+          CCACHE_DIR: "${{ github.workspace }}/ccache" 
           BUILD_VERSION: "${{ inputs.version || steps.version.outputs.build_version }}"
         with:
           output-dir: wheelhouse
 
-      - name: "Move ccache to workspace"
-        if: matrix.os != 'macos-11'
-        run: |
-          mkdir -p ccache
-          cp -R ./wheelhouse/ccache/* ccache
-
       - name: "Save ccache"
-        uses: actions/cache/save@v3
-        id: cache
+        # CCache doesn't hit in cibuildwheel
+        if: startsWith(matrix.os, 'macos')
+        uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
         with:
           path: "ccache"
-          key: ccache-wheels-${{ matrix.os }}-${{ github.sha }}
+          key: ccache-wheels-${{ matrix.os }}
 
       - name: "Rename wheel compatibility tag"
-        if: matrix.os == 'macos-11'
+        if: false #startsWith(matrix.os, 'macos')
         run: |
           brew install rename
           cd wheelhouse
           rename 's/11_0/10_15/g' *.whl
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
-          name: wheels
+          name: wheels-${{ matrix.os }}
+          retention-days: 5
           path: |
             ./wheelhouse/*.whl
             ./wheelhouse/*.tar.gz
@@ -170,19 +176,20 @@ jobs:
     needs: build_wheels
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/download-artifact@v4
+      - uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
         with:
-          name: wheels
+          pattern: wheels-*
+          merge-multiple: true
           path: ./wheelhouse
 
       - run: ls wheelhouse
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
         with:
           python-version: "3.10"
 
       - name: Publish a Python distribution to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.12.3
+        uses: pypa/gh-action-pypi-publish@76f52bc884231f62b9a034ebfe128415bbaabdfc # v1.12.4
         with:
           password: ${{ secrets.PYPI_API_TOKEN }}
           packages_dir: wheelhouse

.github/workflows/docker.yml

@@ -51,6 +51,9 @@ jobs:
           - name: Centos 9
             file: "scripts/centos.dockerfile"
             tags: "ghcr.io/facebookincubator/velox-dev:centos9"
+          - name: Pyvelox 
+            file: "scripts/pyvelox.dockerfile"
+            tags: "ghcr.io/facebookincubator/velox-dev:pyvelox"
           - name: Dev
             file: "scripts/ubuntu-22.04-cpp.dockerfile"
             args: ""

.github/workflows/linux-build-base.yml

@@ -39,7 +39,8 @@ jobs:
       GTest_SOURCE: BUNDLED
       simdjson_SOURCE: BUNDLED
       xsimd_SOURCE: BUNDLED
-      CUDA_VERSION: "12.4"
+      geos_SOURCE: BUNDLED
+      CUDA_VERSION: "12.8"
       USE_CLANG: "${{ inputs.use-clang && 'true' || 'false' }}"
     steps:
       - uses: actions/checkout@v4
@@ -88,6 +89,7 @@ jobs:
             "-DVELOX_ENABLE_BENCHMARKS=ON"
             "-DVELOX_ENABLE_EXAMPLES=ON"
             "-DVELOX_ENABLE_ARROW=ON"
+            "-DVELOX_ENABLE_GEO=ON"
             "-DVELOX_ENABLE_PARQUET=ON"
             "-DVELOX_ENABLE_HDFS=ON"
             "-DVELOX_ENABLE_S3=ON"

.github/workflows/scheduled.yml

@@ -89,7 +89,7 @@ jobs:
     name: Build
     # prevent errors when forks ff their main branch
     if: ${{ github.repository == 'facebookincubator/velox' }}
-    runs-on: 16-core-ubuntu
+    runs-on: 32-core-ubuntu
     container: ghcr.io/facebookincubator/velox-dev:centos9
     timeout-minutes: 120
     env:
@@ -318,7 +318,7 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: cache_fuzzer
-          path: velox/_build/debug/velox/exec/tests/velox_cache_fuzzer_test
+          path: velox/_build/debug/velox/exec/fuzzer/velox_cache_fuzzer
           retention-days: "${{ env.RETENTION }}"
 
       - name: Upload table evolution fuzzer
@@ -751,6 +751,8 @@ jobs:
     container: ghcr.io/facebookincubator/velox-dev:centos9
     needs: compile
     timeout-minutes: 120
+    # Temporarily disable on PRs till flakiness is fixed #12167
+    if: ${{ github.event_name != 'pull_request' }}
     steps:
 
       - name: Download cache fuzzer
@@ -760,24 +762,24 @@ jobs:
 
       - name: Run Cache Fuzzer
         run: |
-          mkdir -p /tmp/cache_fuzzer_test/logs/
-          chmod -R 777 /tmp/cache_fuzzer_test
-          chmod +x velox_cache_fuzzer_test
-          ./velox_cache_fuzzer_test \
+          mkdir -p /tmp/cache_fuzzer/logs/
+          chmod -R 777 /tmp/cache_fuzzer
+          chmod +x velox_cache_fuzzer
+          ./velox_cache_fuzzer \
                 --seed ${RANDOM} \
                 --duration_sec $DURATION \
                 --minloglevel=0 \
                 --stderrthreshold=2 \
-                --log_dir=/tmp/cache_fuzzer_test/logs \
+                --log_dir=/tmp/cache_fuzzer/logs \
             && echo -e "\n\Cache fuzzer run finished successfully."
 
       - name: Archive Cache production artifacts
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@v4
         with:
-            name: cache-fuzzer-test-logs
+            name: cache-fuzzer-logs
             path: |
-              /tmp/cache_fuzzer_test
+              /tmp/cache_fuzzer
 
   table-evolution-fuzzer-run:
     name: Table Evolution Fuzzer

CMake/resolve_dependency_modules/README.md

@@ -41,6 +41,7 @@ by Velox. See details on bundling below.
 | DuckDB (testing)  | 0.8.1           | Yes      |
 | cpr (testing)     | 1.10.15         | Yes      |
 | arrow             | 15.0.0          | Yes      |
+| geos              | 3.13.0          | Yes      |
 
 # Bundled Dependency Management
 This module provides a dependency management system that allows us to automatically fetch and build dependencies from source if needed.

CMake/resolve_dependency_modules/geos.cmake

@@ -0,0 +1,31 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+include_guard(GLOBAL)
+
+# GEOS Configuration
+set(VELOX_GEOS_BUILD_VERSION 3.13.0)
+set(VELOX_GEOS_BUILD_SHA256_CHECKSUM
+    47ec83ff334d672b9e4426695f15da6e6368244214971fabf386ff8ef6df39e4)
+string(CONCAT VELOX_GEOS_SOURCE_URL "https://download.osgeo.org/geos/"
+              "geos-${VELOX_GEOS_BUILD_VERSION}.tar.bz2")
+
+velox_resolve_dependency_url(GEOS)
+
+FetchContent_Declare(
+  geos
+  URL ${VELOX_GEOS_SOURCE_URL}
+  URL_HASH ${VELOX_GEOS_BUILD_SHA256_CHECKSUM})
+set(BUILD_SHARED_LIBS ${VELOX_BUILD_SHARED})
+FetchContent_MakeAvailable(geos)
+unset(BUILD_SHARED_LIBS)

CMake/resolve_dependency_modules/gflags.cmake

@@ -43,7 +43,20 @@ set(GFLAGS_IS_SUBPROJECT ON)
 
 # Workaround for https://github.com/gflags/gflags/issues/277
 unset(BUILD_SHARED_LIBS)
+if(DEFINED CACHE{BUILD_SHARED_LIBS})
+  set(CACHED_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+  unset(BUILD_SHARED_LIBS CACHE)
+endif()
+
 FetchContent_MakeAvailable(gflags)
+
+# Workaround for https://github.com/gflags/gflags/issues/277
+if(DEFINED CACHED_BUILD_SHARED_LIBS)
+  set(BUILD_SHARED_LIBS
+      ${CACHED_BUILD_SHARED_LIBS}
+      CACHE BOOL "Restored after setting up gflags" FORCE)
+endif()
+
 # This causes find_package(gflags) in other dependencies to search in the build
 # directory and prevents the system gflags from being found when they don't use
 # the target directly (like folly).

CMake/resolve_dependency_modules/gtest.cmake

@@ -24,12 +24,12 @@ velox_resolve_dependency_url(GTEST)
 
 message(STATUS "Building gtest from source")
 FetchContent_Declare(
-  gtest
+  googletest
   URL ${VELOX_GTEST_SOURCE_URL}
   URL_HASH ${VELOX_GTEST_BUILD_SHA256_CHECKSUM}
   OVERRIDE_FIND_PACKAGE SYSTEM EXCLUDE_FROM_ALL)
 
-FetchContent_MakeAvailable(gtest)
+FetchContent_MakeAvailable(googletest)
 
 # Mask compilation warning in clang 16.
 target_compile_options(gtest PRIVATE -Wno-implicit-int-float-conversion)