From 749a335b09e420747f9989c7be45c0afda716fba Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Mon, 17 Mar 2025 16:00:18 +0100 Subject: [PATCH 1/4] Update pr_criterion.yaml: use bare-metal runners --- .github/workflows/benchmark.yml | 59 +++++++++++++++++++++++++++++ .github/workflows/pr_criterion.yaml | 31 --------------- 2 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/benchmark.yml delete mode 100644 .github/workflows/pr_criterion.yaml diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000000..45ab6be15c --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,59 @@ +# This workflow runs a Criterion benchmark on a PR and compares the results against the base branch. +# It is triggered on a PR or a push to main. +# +# The workflow is gated on the presence of the "performance" label on the PR. +# +# The workflow runs on a self-hosted runner pool. We can't use the shared runners for this, +# because they are only permitted to run on the default branch to preserve resources. +# +# In the future, we might like to consider using bencher.dev or the framework used by otel-golang here. +on: + pull_request: + push: + branches: + - main +name: benchmark pull requests +jobs: + runBenchmark: + name: run benchmark + permissions: + pull-requests: write + + # If we're running on a PR, use ubuntu-latest - a shared runner. We can't use the self-hosted + # runners on arbitrary PRs, and we don't want to unleash that load on the pool anyway. + # If we're running on main, use the OTEL self-hosted runner pool. + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'self-hosted' }} + if: | + ${{ + github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'performance') || + github.event_name == 'push' + }} + env: + # For PRs, compare against the base branch + # For pushes to main, compare against the previous commit + #BRANCH_NAME: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.before }} + BRANCH_NAME: ${{ github.event.before }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 # Fetch current commit and its parent + - uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry + branchName: ${{ env.BRANCH_NAME }} + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry-appender-tracing + features: spec_unstable_logs_enabled + branchName: ${{ env.BRANCH_NAME }} + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry-sdk + features: rt-tokio,testing,metrics,logs,spec_unstable_metrics_views + branchName: ${{ env.BRANCH_NAME }} diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml deleted file mode 100644 index d86cebe482..0000000000 --- a/.github/workflows/pr_criterion.yaml +++ /dev/null @@ -1,31 +0,0 @@ -on: [pull_request] -name: benchmark pull requests -jobs: - runBenchmark: - name: run benchmark - permissions: - pull-requests: write - runs-on: ubuntu-latest - if: ${{ contains(github.event.pull_request.labels.*.name, 'performance') }} - steps: - - uses: actions/checkout@v4 - - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - uses: boa-dev/criterion-compare-action@v3 - with: - cwd: opentelemetry - branchName: ${{ github.base_ref }} - - uses: boa-dev/criterion-compare-action@v3 - with: - cwd: opentelemetry-appender-tracing - features: spec_unstable_logs_enabled - branchName: ${{ github.base_ref }} - - uses: boa-dev/criterion-compare-action@v3 - with: - cwd: opentelemetry-sdk - features: rt-tokio,testing,metrics,logs,spec_unstable_metrics_views - branchName: ${{ github.base_ref }} \ No newline at end of file From e78338934402cfe60300a6f5148f807786d5570b Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 19 Mar 2025 12:20:45 +0100 Subject: [PATCH 2/4] update comment on fetch-depth --- .github/workflows/pr_criterion.yaml | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/pr_criterion.yaml diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml new file mode 100644 index 0000000000..48901a8748 --- /dev/null +++ b/.github/workflows/pr_criterion.yaml @@ -0,0 +1,49 @@ +on: + pull_request: + push: + branches: + - main +name: benchmark pull requests +jobs: + runBenchmark: + name: run benchmark + permissions: + pull-requests: write + + # If we're running on a PR, use ubuntu-latest - a shared runner. We can't use the self-hosted + # runners on arbitrary PRs, and we don't want to unleash that load on the pool anyway. + # If we're running on main, use the OTEL self-hosted runner pool. + runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'self-hosted' }} + if: | + ${{ + github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'performance') || + github.event_name == 'push' + }} + env: + # For PRs, compare against the base branch + # For pushes to main, compare against the previous commit + BRANCH_NAME: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.before }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 10 # Fetch current commit and a few layers up, so that we have enough to diff against the previous commit + - uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry + branchName: ${{ env.BRANCH_NAME }} + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry-appender-tracing + features: spec_unstable_logs_enabled + branchName: ${{ env.BRANCH_NAME }} + - uses: boa-dev/criterion-compare-action@v3 + with: + cwd: opentelemetry-sdk + features: rt-tokio,testing,metrics,logs,spec_unstable_metrics_views + branchName: ${{ env.BRANCH_NAME }} From 551104d44fd7b1e8e28c4e1a9aec2448f0e0401a Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 19 Mar 2025 12:21:38 +0100 Subject: [PATCH 3/4] removed old benchmark --- .github/workflows/pr_criterion.yaml | 49 ----------------------------- 1 file changed, 49 deletions(-) delete mode 100644 .github/workflows/pr_criterion.yaml diff --git a/.github/workflows/pr_criterion.yaml b/.github/workflows/pr_criterion.yaml deleted file mode 100644 index 48901a8748..0000000000 --- a/.github/workflows/pr_criterion.yaml +++ /dev/null @@ -1,49 +0,0 @@ -on: - pull_request: - push: - branches: - - main -name: benchmark pull requests -jobs: - runBenchmark: - name: run benchmark - permissions: - pull-requests: write - - # If we're running on a PR, use ubuntu-latest - a shared runner. We can't use the self-hosted - # runners on arbitrary PRs, and we don't want to unleash that load on the pool anyway. - # If we're running on main, use the OTEL self-hosted runner pool. - runs-on: ${{ github.event_name == 'pull_request' && 'ubuntu-latest' || 'self-hosted' }} - if: | - ${{ - github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'performance') || - github.event_name == 'push' - }} - env: - # For PRs, compare against the base branch - # For pushes to main, compare against the previous commit - BRANCH_NAME: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.before }} - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 10 # Fetch current commit and a few layers up, so that we have enough to diff against the previous commit - - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - uses: boa-dev/criterion-compare-action@v3 - with: - cwd: opentelemetry - branchName: ${{ env.BRANCH_NAME }} - - uses: boa-dev/criterion-compare-action@v3 - with: - cwd: opentelemetry-appender-tracing - features: spec_unstable_logs_enabled - branchName: ${{ env.BRANCH_NAME }} - - uses: boa-dev/criterion-compare-action@v3 - with: - cwd: opentelemetry-sdk - features: rt-tokio,testing,metrics,logs,spec_unstable_metrics_views - branchName: ${{ env.BRANCH_NAME }} From 8d25e75317740654df47560f9a0f5ec081975425 Mon Sep 17 00:00:00 2001 From: Scott Gerring Date: Wed, 19 Mar 2025 12:24:44 +0100 Subject: [PATCH 4/4] missed the correct branch name val --- .github/workflows/benchmark.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 45ab6be15c..40d59c510e 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -29,10 +29,9 @@ jobs: github.event_name == 'push' }} env: - # For PRs, compare against the base branch + # For PRs, compare against the base branch - e.g., 'main'. # For pushes to main, compare against the previous commit - #BRANCH_NAME: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.before }} - BRANCH_NAME: ${{ github.event.before }} + BRANCH_NAME: ${{ github.event_name == 'pull_request' && github.base_ref || github.event.before }} steps: - uses: actions/checkout@v4 with: