From f6b942c0db06e8e7acea6756ebe2d64a143bc49f Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Wed, 25 Sep 2024 20:44:17 +0800 Subject: [PATCH 1/9] Performance history canary This PR adds a "canary" build to the performance regression CI of OpenJDK. The "canary" is a chosen revision of mmtk-core and mmtk-openjdk that is tested alongside each merged PR. The performance of the "canary" should not change unless there is an environment change. Spotting a change in the "canary" performance can help us identify environment changes that are unintended or otherwise unnoticed. TODO: Add a script to select the latest release tag as the canary instead of hard-coding a version. --- .github/workflows/perf-regression-ci.yml | 63 +++++++++++++++++------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 09cea5f6b6..2072b2c840 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -98,17 +98,18 @@ jobs: runs-on: [self-hosted, Linux, freq-scaling-off] timeout-minutes: 1440 steps: - - name: Checkout MMTk Core + # checkout latest versions + - name: Checkout MMTk Core (latest) uses: actions/checkout@v4 with: - path: mmtk-core - - name: Checkout OpenJDK Binding + path: latest/mmtk-core + - name: Checkout OpenJDK Binding (latest) uses: actions/checkout@v4 with: repository: mmtk/mmtk-openjdk - path: mmtk-openjdk - - name: Checkout OpenJDK - working-directory: mmtk-openjdk + path: latest/mmtk-openjdk + - name: Checkout OpenJDK (latest) + working-directory: latest/mmtk-openjdk run: | ./.github/scripts/ci-checkout.sh # checkout perf-kit @@ -120,21 +121,42 @@ jobs: path: ci-perf-kit token: ${{ secrets.CI_ACCESS_TOKEN }} submodules: true + # checkout canary versions. + # Currently using a release version. + # We need a script to find the latest stable version from a list of tags (`git tags -l`). + # Alternatively, we may use the latest commit in the last epoch once we stablize the idea of epoch. + - name: Checkout MMTk Core (canary) + uses: actions/checkout@v4 + with: + ref: "v0.27.0" + path: canary/mmtk-core + - name: Checkout OpenJDK Binding (canary) + uses: actions/checkout@v4 + with: + ref: "v0.27.0" + repository: mmtk/mmtk-openjdk + path: canary/mmtk-openjdk + - name: Checkout OpenJDK (canary) + working-directory: canary/mmtk-openjdk + run: | + ./.github/scripts/ci-checkout.sh # setup - - name: Overwrite MMTk core in openjdk binding - run: cp -r mmtk-core mmtk-openjdk/repos/ + - name: Setup directory structures + run: | + for BASE_DIR in . ./canary; do + pushd $BASE_DIR + # copy mmtk-core repo + cp -r mmtk-core mmtk-openjdk/repos/ + # replace dependency + ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path repos/mmtk-core + # cleanup previous build + rm -rf mmtk-openjdk/repos/openjdk/scratch + rm -rf mmtk-openjdk/repos/openjdk/build + popd + done - name: Setup Rust Toolchain + # This seems unused. run: echo "RUSTUP_TOOLCHAIN=`cat mmtk-core/rust-toolchain`" >> $GITHUB_ENV - # cleanup previosu build - - name: Cleanup previous build - run: | - rm -rf mmtk-openjdk/repos/openjdk/scratch - rm -rf mmtk-openjdk/repos/openjdk/build - - name: Setup - run: | - ./ci-perf-kit/scripts/history-run-setup.sh - sed -i 's/^mmtk[[:space:]]=/#ci:mmtk=/g' mmtk-openjdk/mmtk/Cargo.toml - sed -i 's/^#[[:space:]]mmtk/mmtk/g' mmtk-openjdk/mmtk/Cargo.toml - id: branch # we cannot use env vars in action input (the deploy step). So put the env var to this step's outputs. run: echo "branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT @@ -145,7 +167,10 @@ jobs: export RESULT_REPO_BRANCH=${{ env.RESULT_REPO_BRANCH }} export RESULT_REPO_ACCESS_TOKEN=${{ secrets.CI_ACCESS_TOKEN }} export FROM_DATE=2020-07-10 - ./ci-perf-kit/scripts/openjdk-history-run.sh ./mmtk-openjdk ./reports/${{ steps.branch.outputs.branch_name }} + ./ci-perf-kit/scripts/openjdk-history-run.sh \ + ./latest/mmtk-openjdk \ + ./canary/mmtk-openjdk \ + ./reports/${{ steps.branch.outputs.branch_name }} # deploy - name: Deploy to Github Page if: ${{ env.DEPLOY == 'true' }} From d7d55a1e16f2c569413eff056ea323aa472dc240 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 14 Oct 2024 17:47:51 +0800 Subject: [PATCH 2/9] Fix typo and update canary version --- .github/workflows/perf-regression-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 2072b2c840..7936c8efc5 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -128,12 +128,12 @@ jobs: - name: Checkout MMTk Core (canary) uses: actions/checkout@v4 with: - ref: "v0.27.0" + ref: "v0.28.0" path: canary/mmtk-core - name: Checkout OpenJDK Binding (canary) uses: actions/checkout@v4 with: - ref: "v0.27.0" + ref: "v0.28.0" repository: mmtk/mmtk-openjdk path: canary/mmtk-openjdk - name: Checkout OpenJDK (canary) @@ -143,7 +143,7 @@ jobs: # setup - name: Setup directory structures run: | - for BASE_DIR in . ./canary; do + for BASE_DIR in ./latest ./canary; do pushd $BASE_DIR # copy mmtk-core repo cp -r mmtk-core mmtk-openjdk/repos/ From 9d76f12bd809bd6a6a2c305dfea600595d9faf94 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 14 Oct 2024 17:59:57 +0800 Subject: [PATCH 3/9] Fix mmtk-core repo path --- .github/workflows/perf-regression-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 7936c8efc5..4969861996 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -148,7 +148,7 @@ jobs: # copy mmtk-core repo cp -r mmtk-core mmtk-openjdk/repos/ # replace dependency - ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path repos/mmtk-core + ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path ../repos/mmtk-core # cleanup previous build rm -rf mmtk-openjdk/repos/openjdk/scratch rm -rf mmtk-openjdk/repos/openjdk/build From 7ff02eda377118c9fb92a77fb7e4c2580af7ad3d Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Tue, 15 Oct 2024 12:07:58 +0800 Subject: [PATCH 4/9] Fix mmtk-core path again. --- .github/workflows/perf-regression-ci.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 4969861996..4e4d46a59a 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -145,10 +145,11 @@ jobs: run: | for BASE_DIR in ./latest ./canary; do pushd $BASE_DIR - # copy mmtk-core repo - cp -r mmtk-core mmtk-openjdk/repos/ # replace dependency - ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml --mmtk-core-path ../repos/mmtk-core + # Note that ci-replace-mmtk-dep.sh will apply `realpath()` to the `--mmtk-core-path` option. + # so we specify the relative path from the PWD to the mmtk-core repo. + ./mmtk-core/.github/scripts/ci-replace-mmtk-dep.sh mmtk-openjdk/mmtk/Cargo.toml \ + --mmtk-core-path mmtk-core # cleanup previous build rm -rf mmtk-openjdk/repos/openjdk/scratch rm -rf mmtk-openjdk/repos/openjdk/build From 3c29d9be1689a796dd362fd91a48435546d9fcca Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 18 Oct 2024 11:41:22 +0800 Subject: [PATCH 5/9] Not setting RUSTUP_TOOLCHAIN env var. The latest and the canary version may use different toolchain, and will be selected automatically when compiling them. --- .github/workflows/perf-regression-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 4e4d46a59a..ffc0cccf45 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -155,9 +155,6 @@ jobs: rm -rf mmtk-openjdk/repos/openjdk/build popd done - - name: Setup Rust Toolchain - # This seems unused. - run: echo "RUSTUP_TOOLCHAIN=`cat mmtk-core/rust-toolchain`" >> $GITHUB_ENV - id: branch # we cannot use env vars in action input (the deploy step). So put the env var to this step's outputs. run: echo "branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT From 4d3c5235a9217ce2cb008ef000d81e679fd44482 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 18 Oct 2024 14:42:35 +0800 Subject: [PATCH 6/9] Minor changes Extract canary version to an environment variable. Do not use secret when checking out ci-perf-kit because it is a public repo now. Add a name to the "branch" action. --- .github/workflows/perf-regression-ci.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index ffc0cccf45..842b914ff4 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -97,6 +97,12 @@ jobs: openjdk-perf-regression: runs-on: [self-hosted, Linux, freq-scaling-off] timeout-minutes: 1440 + env: + # This version will be used as the canary version, and will be used to checkout both + # `mmtk-core` and `mmtk-openjdk`. We choose one release version for this purpose. We may + # change to another release version if necessary, or introduce a mechanism to dynamically + # choose the canary in the future. + CANARY_VERSION: "v0.28.0" steps: # checkout latest versions - name: Checkout MMTk Core (latest) @@ -119,21 +125,17 @@ jobs: repository: mmtk/ci-perf-kit ref: "0.8.0" path: ci-perf-kit - token: ${{ secrets.CI_ACCESS_TOKEN }} submodules: true # checkout canary versions. - # Currently using a release version. - # We need a script to find the latest stable version from a list of tags (`git tags -l`). - # Alternatively, we may use the latest commit in the last epoch once we stablize the idea of epoch. - name: Checkout MMTk Core (canary) uses: actions/checkout@v4 with: - ref: "v0.28.0" + ref: $CANARY_VERSION path: canary/mmtk-core - name: Checkout OpenJDK Binding (canary) uses: actions/checkout@v4 with: - ref: "v0.28.0" + ref: $CANARY_VERSION repository: mmtk/mmtk-openjdk path: canary/mmtk-openjdk - name: Checkout OpenJDK (canary) @@ -155,7 +157,8 @@ jobs: rm -rf mmtk-openjdk/repos/openjdk/build popd done - - id: branch + - name: Setup branch name + id: branch # we cannot use env vars in action input (the deploy step). So put the env var to this step's outputs. run: echo "branch_name=$(echo ${GITHUB_REF#refs/heads/} | sed 's/\//_/g')" >> $GITHUB_OUTPUT # run From 569e357864d357bc5b42ae4d096e380f1e35277a Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Fri, 18 Oct 2024 15:07:52 +0800 Subject: [PATCH 7/9] Fix env var usage --- .github/workflows/perf-regression-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index 842b914ff4..b54792cec7 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -130,12 +130,12 @@ jobs: - name: Checkout MMTk Core (canary) uses: actions/checkout@v4 with: - ref: $CANARY_VERSION + ref: ${{ env.CANARY_VERSION }} path: canary/mmtk-core - name: Checkout OpenJDK Binding (canary) uses: actions/checkout@v4 with: - ref: $CANARY_VERSION + ref: ${{ env.CANARY_VERSION }} repository: mmtk/mmtk-openjdk path: canary/mmtk-openjdk - name: Checkout OpenJDK (canary) From e1adde4668af26aa39bbd0bd57207bc6961a2b45 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 21 Oct 2024 10:57:29 +0800 Subject: [PATCH 8/9] Add a comment to explain what a "canary" is. --- .github/workflows/perf-regression-ci.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index b54792cec7..d268841a41 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -99,9 +99,21 @@ jobs: timeout-minutes: 1440 env: # This version will be used as the canary version, and will be used to checkout both - # `mmtk-core` and `mmtk-openjdk`. We choose one release version for this purpose. We may - # change to another release version if necessary, or introduce a mechanism to dynamically - # choose the canary in the future. + # `mmtk-core` and `mmtk-openjdk`. + # + # A "canary" is a chosen version that is tested alongside each merged pull request. The + # performance of the canary should not change unless + # + # 1. There is an environment change, such as changes of the operating system, the hardware, + # the firmware, or the methodology we use for testing, or + # 2. There are non-deterministic factors (i.e. noises) affecting each execution. + # + # Running the canary alongside regular regression tests help us identify unnoticed environment + # changes and the level of noise. + # + # Currently, we choose one release version as the canary. We may change to another release + # version if necessary, or introduce a mechanism to dynamically choose the canary in the + # future. CANARY_VERSION: "v0.28.0" steps: # checkout latest versions From 1523bb4cd710426624ca84623fdd87c65ccef2a7 Mon Sep 17 00:00:00 2001 From: Kunshan Wang Date: Mon, 21 Oct 2024 11:21:12 +0800 Subject: [PATCH 9/9] Update comment and ci-perf-kit version --- .github/workflows/perf-regression-ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/perf-regression-ci.yml b/.github/workflows/perf-regression-ci.yml index d268841a41..7f0c92183a 100644 --- a/.github/workflows/perf-regression-ci.yml +++ b/.github/workflows/perf-regression-ci.yml @@ -111,9 +111,10 @@ jobs: # Running the canary alongside regular regression tests help us identify unnoticed environment # changes and the level of noise. # - # Currently, we choose one release version as the canary. We may change to another release - # version if necessary, or introduce a mechanism to dynamically choose the canary in the - # future. + # Currently, we choose one release version as the canary, and will use it until we can no + # longer run it for any reason, such as the toolchain for compiling that version is no longer + # available. Then we may change to another release version and mark the change of canary on + # the timeline, or introduce a mechanism to dynamically choose the canary version. CANARY_VERSION: "v0.28.0" steps: # checkout latest versions @@ -135,7 +136,7 @@ jobs: uses: actions/checkout@v4 with: repository: mmtk/ci-perf-kit - ref: "0.8.0" + ref: "0.8.1" path: ci-perf-kit submodules: true # checkout canary versions.