benchmark.yml

# Copyright (c) Facebook, Inc. and its affiliates.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: "Ubuntu Benchmark"

on:
  pull_request:
    paths:
      - '.github/workflows/benchmark.yml'
      - 'scripts/benchmark-requirements.txt'

  push:
    branches: [main]

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.sha }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  benchmark:
    if: github.repository == 'facebookincubator/velox'
    runs-on: 8-core-ubuntu-22.04
    env:
      CCACHE_DIR: "${{ github.workspace }}/ccache/"
      CCACHE_BASEDIR: "${{ github.workspace }}"
      BINARY_DIR: "${{ github.workspace }}/benchmarks/"
      CONTENDER_OUTPUT_PATH: "${{ github.workspace }}/benchmark-results/contender/"
      INSTALL_PREFIX: "${{ github.workspace }}/dependencies"
    steps:

      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          path: 'velox'
          submodules: 'recursive'
          persist-credentials: false

      - name: "Restore Dependencies"
        uses: actions/cache/restore@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
        id: restore-deps
        with:
          path: "${{ env.INSTALL_PREFIX }}"
          key: dependencies-benchmark-${{ hashFiles('velox/scripts/setup-ubuntu.sh') }}

      - name: "Install dependencies"
        if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }}
        run: |
            source velox/scripts/setup-ubuntu.sh 
            install_apt_deps 
            run_and_time install_fmt
            run_and_time install_protobuf
            run_and_time install_boost
            run_and_time install_folly
            run_and_time install_stemmer
            run_and_time install_thrift
            run_and_time install_arrow

      - name: "Save Dependencies"
        if: ${{ steps.restore-deps.outputs.cache-hit != 'true' }}
        uses: actions/cache/save@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
        with:
          path: "${{ env.INSTALL_PREFIX }}"
          key: dependencies-${{ hashFiles('velox/scripts/setup-ubuntu.sh') }}

      - name: "Restore ccache"
        uses: apache/infrastructure-actions/stash/restore@3354c1565d4b0e335b78a76aedd82153a9e144d4
        id: restore-cache
        with:
          path: "ccache"
          key: ccache-benchmark

      - name: Clear CCache Statistics
        run: |
          ccache -sz

      - name: Build Contender Benchmarks
        working-directory: velox
        run: |
            n_cores=$(nproc)
            make benchmarks-basic-build NUM_THREADS=$n_cores MAX_HIGH_MEM_JOBS=$n_cores MAX_LINK_JOBS=$n_cores
            mkdir -p  ${BINARY_DIR}/contender/
            cp -r --verbose _build/release/velox/benchmarks/basic/*  ${BINARY_DIR}/contender/

      - name: CCache after
        run: |
          ccache -vs

      - name: "Save ccache"
        uses: apache/infrastructure-actions/stash/save@3354c1565d4b0e335b78a76aedd82153a9e144d4
        with:
          path: "ccache"
          key: ccache-benchmark

      - name: "Install benchmark dependencies"
        run: |
          python3 -m pip install -r velox/scripts/benchmark-requirements.txt

      - name: "Run Benchmarks - Contender"
        working-directory: 'velox'
        run: |
            make benchmarks-basic-run \
                EXTRA_BENCHMARK_FLAGS="--binary_path ${BINARY_DIR}/contender/ --output_path ${CONTENDER_OUTPUT_PATH}"

      - name: "Upload result artifact"
        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
        with:
          path: "benchmark-results"
          name: "benchmark-results"
          retention-days: 5

  upload:
    runs-on: ubuntu-latest
    needs: benchmark
    # if: github.event_name == 'push'
    permissions:
      actions: read
      statuses: write
    steps:

    - name: Download artifacts
      uses: actions/download-artifact@cc203385981b70ca67e1cc392babf9cc229d5806 # v4.1.9
      with:
        merge-multiple: true
        path: /tmp/artifacts/

    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      with:
        path: velox
        persist-credentials: false

    - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
      with:
        python-version: '3.10'
        cache: 'pip'
        cache-dependency-path: "velox/scripts/*"

    - name: "Install dependencies"
      run: pip install -r velox/scripts/benchmark-requirements.txt

    - name: "Upload results"
      env:
        CONBENCH_URL: "https://velox-conbench.voltrondata.run/"
        CONBENCH_MACHINE_INFO_NAME: "GitHub-runner-8-core"
        CONBENCH_EMAIL: "${{ secrets.CONBENCH_EMAIL }}"
        CONBENCH_PASSWORD: "${{ secrets.CONBENCH_PASSWORD }}"
        CONBENCH_PROJECT_REPOSITORY: "${{ github.repository }}"
        CONBENCH_PROJECT_COMMIT: "${{ github.sha }}"
        RUN_ID: "GHA-${{ github.run_id }}-${{ github.run_attempt }}"
      run: |
        ./velox/scripts/benchmark-runner.py upload \
          --run_id "$RUN_ID" \
          --sha "$CONBENCH_PROJECT_COMMIT" \
          --pr_number 12605 \
          --output_dir "/tmp/artifacts/contender/"

    - name: "Check the status of the upload"
      # Status functions like failure() only work in `if:`
      if: failure()
      id: status
      run: echo "failed=true" >> $GITHUB_OUTPUT

    - name: "Create a GitHub Status on the contender commit (whether the upload was successful)"
      uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
      if: ${{ !cancelled() && steps.extract.conclusion != 'failure' }}
      with:
        script: |
          let url = 'https://github.com/${{github.repository}}/actions/runs/${{ github.run_id }}'
          let state = 'success'
          let description = 'Result upload succeeded!'

          if(${{ steps.status.outputs.failed == 'true' && true || false }}) {
            state = 'failure'
            description = 'Result upload failed!'
          }

          github.rest.repos.createCommitStatus({
            owner: context.repo.owner,
            repo: context.repo.repo,
            sha: context.sha
            state: state,
            target_url: url,
            description: description,
            context: 'Benchmark Result Upload'
          })

    - name: Create a GitHub Check benchmark report on the merged PR
      env:
        CONBENCH_URL: "https://velox-conbench.voltrondata.run/"
        GITHUB_APP_ID: "${{ secrets.GH_APP_ID }}"
        GITHUB_APP_PRIVATE_KEY: "${{ secrets.GH_APP_PRIVATE_KEY }}"
        COMMIT_MESSAGE: "${{ github.event.head_commit.message }}"
        CONTENDER_SHA: "${{ github.sha }}"
      run: |
        ./velox/scripts/benchmark-alert.py \
          --contender-sha "$CONTENDER_SHA" \
          --merge-commit-message "$COMMIT_MESSAGE" \
          --z-score-threshold 50