Merge branch 'main' into main

Author: alex4506

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,45 @@
-### Description
+## Description
+<!-- Please include a summary of the change, including which issue it fixes or what feature it adds. Include relevant motivation, context and documentation as appropriate. List dependencies that are required for this change, if any. -->
 
-### Test Plan
-<!-- Please provide us with clear details for verifying that your changes work. -->
+## Type of Change
+- [ ] New feature
+- [ ] Bug fix
+- [ ] Breaking change
+- [ ] Performance improvement
+- [ ] Refactoring
+- [ ] Dependency update
+- [ ] Documentation update
+- [ ] Tests
+
+## Which Components or Systems Does This Change Impact?
+- [ ] Validator Node
+- [ ] Full Node (API, Indexer, etc.)
+- [ ] Move/Aptos Virtual Machine
+- [ ] Aptos Framework
+- [ ] Aptos CLI/SDK
+- [ ] Developer Infrastructure
+- [ ] Other (specify)
+
+## How Has This Been Tested?
+<!--
+- Please ensure that the functionality introduced by this change is well tested and verified to work as expected.
+- Ensure tests cover both happy and unhappy paths.
+- List and link relevant tests.
+-->
+
+## Key Areas to Review
+<!--
+- Identify any critical parts of the code that require special attention or understanding. Explain why these parts are crucial to the functionality or architecture of the project.
+- Point out any areas where complex logic has been implemented. Provide a brief explanation of the logic and your approach to make it easier for reviewers to follow.
+- Highlight any areas where you are particularly concerned or unsure about the code's impact on the change. This can include potential performance or security issues, or compatibility with existing features.
+-->
+
+## Checklist
+- [ ] I have read and followed the [CONTRIBUTING](https://github.com/aptos-labs/aptos-core/blob/main/CONTRIBUTING.md) doc
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I identified and added all stakeholders and component owners affected by this change as reviewers
+- [ ] I tested both happy and unhappy path of the functionality
+- [ ] I have made corresponding changes to the documentation
+
+<!-- Thank you for your contribution! -->

.github/actions/file-change-determinator/action.yaml

@@ -1,6 +1,9 @@
 name: File Change Determinator
 description: Runs the file change determinator (to identify which files changed in a pull request)
 outputs:
+  move_compiler_v2_changes:
+    description: "Returns true if changes were detected that require the move compiler v2 tests to run"
+    value: ${{ !steps.move_compiler_v2_determinator.outputs.should_skip }}
   only_docs_changed:
     description: "Returns true if only docs files were changed in the pull request"
     value: ${{ steps.doc_change_determinator.outputs.should_skip }}
@@ -15,3 +18,18 @@ runs:
       with:
         skip_after_successful_duplicate: false # Don't skip if the action is a duplicate (this may cause false positives)
         paths_ignore: '["**/*.md"]'
+    # Run the move compiler v2 change determinator
+    - id: move_compiler_v2_determinator
+      continue-on-error: true # Avoid skipping any checks if this job fails (see: https://github.com/fkirc/skip-duplicate-actions/issues/301)
+      uses: fkirc/skip-duplicate-actions@v5
+      with:
+          skip_after_successful_duplicate: false # Don't skip if the action is a duplicate (this may cause false positives)
+          paths: '[
+                    "aptos-move/aptos-transactional-test-harness",
+                    "aptos-move/e2e-move-tests/**",
+                    "aptos-move/framework/**",
+                    "aptos-move/move-examples",
+                    "third_party/move/**",
+                    ".github/workflows/move-test-compiler-v2.yaml",
+                    ".github/actions/move-tests-compiler-v2/**"
+                  ]'

.github/actions/move-tests-compiler-v2/action.yaml

@@ -23,7 +23,7 @@ runs:
 
     # Run Aptos Move tests with compiler V2
     - name: Run Aptos Move tests with compiler V2
-      run: cargo nextest run --release -E 'not (test(test_smart_data_structures_gas))' --profile ci --locked -p e2e-move-tests -p aptos-framework --retries 3 --no-fail-fast
+      run: cargo nextest run --release --profile ci --locked -p e2e-move-tests -p aptos-framework --no-fail-fast
       shell: bash
       env:
         MOVE_COMPILER_V2: true

.github/actions/run-faucet-tests/action.yaml

@@ -9,7 +9,7 @@ inputs:
     description: "The docker image tag to use for the local testnet if NETWORK=custom."
     required: true
   GCP_DOCKER_ARTIFACT_REPO:
-    description: "The GCP Docker artifact repository."
+    description: "The GCP Docker artifact repository"
     required: true
 
 runs:
@@ -24,7 +24,7 @@ runs:
     - name: Run Redis server
       uses: shogo82148/actions-setup-redis@v1
       with:
-        redis-version: '6.x'
+        redis-version: "6.x"
 
     # Set up Rust for running the integration tests.
     - name: Set up Rust

.github/actions/run-local-testnet/action.yaml

@@ -37,7 +37,6 @@ runs:
 
       shell: bash
 
-
     # Install node + npm.
     - uses: actions/setup-node@v3
       with:

.github/actions/run-ts-sdk-e2e-tests/action.yaml

@@ -68,7 +68,6 @@ runs:
         timeout_minutes: 25
         command: cd ./ecosystem/typescript/sdk && pnpm run test:ci
 
-
     # Run the indexer TS SDK tests.
     - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
       name: ts-sdk-indexer-test

.github/actions/rust-unit-tests/action.yaml

@@ -4,6 +4,9 @@ inputs:
   GIT_CREDENTIALS:
     description: "Optional credentials to pass to git. Useful if you need to pull private repos for dependencies"
     required: false
+  BUILDKITE_ANALYTICS_TOKEN:
+    description: "The Buildkite API token"
+    required: true
 
 runs:
   using: composite
@@ -28,6 +31,11 @@ runs:
     - uses: taiki-e/install-action@v1.5.6
       with:
         tool: nextest
+    
+    # Install buildkite-test-collector
+    - name: Install buildkite-test-collector
+      run: cargo install buildkite-test-collector
+      shell: bash
 
     # Run a postgres database
     - name: Run postgres database
@@ -36,7 +44,10 @@ runs:
 
     # Run the rust unit tests
     - name: Run all unit tests
-      run: cargo nextest run --profile ci --cargo-profile ci --locked --workspace --exclude smoke-test --exclude aptos-testcases --retries 3 --no-fail-fast
+      run: |
+        NEXTEST_EXPERIMENTAL_LIBTEST_JSON=1 cargo nextest run --profile ci --cargo-profile ci --locked --workspace --exclude smoke-test --exclude aptos-testcases --retries 3 --no-fail-fast --message-format libtest-json > nextest_output.json
+        buildkite-test-collector < nextest_output.json || echo "Warning: buildkite-test-collector encountered an error"
+        rm nextest_output.json
       shell: bash
       env:
         INDEXER_DATABASE_URL: postgresql://postgres@localhost/postgres
@@ -47,3 +58,4 @@ runs:
         CVC5_EXE: /home/runner/bin/cvc5
         DOTNET_ROOT: /home/runner/.dotnet
         BOOGIE_EXE: /home/runner/.dotnet/tools/boogie
+        BUILDKITE_ANALYTICS_TOKEN: ${{ inputs.BUILDKITE_ANALYTICS_TOKEN }}

.github/workflows/aptos-node-release.yaml

@@ -32,7 +32,7 @@ jobs:
           aptos_node_cargo_toml: aptos-node/Cargo.toml
 
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@671dc9c9e0c2d73f07fa45a3eb0220e1622f0c5f # pin@v4
+        uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc # pin@v6.0.1
         with:
           add-paths: aptos-node
           title: "[aptos-node] update release version"

.github/workflows/check-protos.yaml

@@ -26,6 +26,13 @@ concurrency:
 jobs:
   check:
     runs-on: ubuntu-latest
+    if: | # Only run on each PR once an appropriate event occurs
+      (
+        github.event_name == 'workflow_dispatch' ||
+        github.event_name == 'push' ||
+        contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
+        github.event.pull_request.auto_merge != null
+      )
     steps:
       - uses: actions/checkout@v3
 

.github/workflows/cli-e2e-tests.yaml

@@ -57,7 +57,7 @@ jobs:
         with:
           max_attempts: 5
           timeout_minutes: 20
-          command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network devnet --image-repo-with-project ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-devnet
+          command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network devnet --image-repo-with-project ${{ vars.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-devnet
 
       # Run CLI tests against local testnet built from testnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
@@ -66,7 +66,7 @@ jobs:
         with:
           max_attempts: 5
           timeout_minutes: 20
-          command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network testnet --image-repo-with-project ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-testnet
+          command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network testnet --image-repo-with-project ${{ vars.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-testnet
 
       # Run CLI tests against local testnet built from mainnet branch.
       - uses: nick-fields/retry@7f8f3d9f0f62fe5925341be21c2e8314fd4f7c7c # pin@v2
@@ -75,7 +75,7 @@ jobs:
         with:
           max_attempts: 5
           timeout_minutes: 20
-          command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network mainnet --image-repo-with-project ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-mainnet
+          command: cd ./crates/aptos/e2e && poetry run python main.py -d --base-network mainnet --image-repo-with-project ${{ vars.GCP_DOCKER_ARTIFACT_REPO }} --test-cli-tag ${{ inputs.GIT_SHA }} --working-directory ${{ runner.temp }}/aptos-e2e-tests-mainnet
 
       - name: Print local testnet logs on failure
         if: ${{ !inputs.SKIP_JOB && failure() }}

.github/workflows/copy-images-to-dockerhub.yaml

@@ -58,8 +58,7 @@ jobs:
         env:
           FORCE_COLOR: 3 # Force color output as per https://github.com/google/zx#using-github-actions
           GIT_SHA: ${{ inputs.GIT_SHA || github.sha }} # If GIT_SHA is not provided, use the sha of the triggering branch
-          GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
-          GCP_DOCKER_ARTIFACT_REPO_US: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO_US }}
           AWS_ACCOUNT_ID: ${{ secrets.AWS_ECR_ACCOUNT_NUM }}
+          GCP_DOCKER_ARTIFACT_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}
           IMAGE_TAG_PREFIX: ${{ inputs.image_tag_prefix }}
         run: ./docker/release-images.mjs --wait-for-image-seconds=3600

.github/workflows/coverage.yaml

@@ -24,7 +24,7 @@ jobs:
   rust-unit-coverage:
     if: |
       contains(github.event.pull_request.labels.*.name, 'CICD:run-coverage') ||
-      ${{ github.event_name == 'schedule' && github.ref_name == 'main' }}
+      (github.event_name == 'schedule' && github.ref_name == 'main')
     # Note the tests run slowly due to instrutmentation. It takes CI 10 hrs
     timeout-minutes: 720
     runs-on: high-perf-docker
@@ -49,7 +49,7 @@ jobs:
   rust-smoke-coverage:
     if: |
       contains(github.event.pull_request.labels.*.name, 'CICD:run-coverage') ||
-      ${{ github.event_name == 'schedule' && github.ref_name == 'main' }}
+      (github.event_name == 'schedule' && github.ref_name == 'main')
     timeout-minutes: 240
     runs-on: high-perf-docker
     steps:
@@ -71,7 +71,10 @@ jobs:
           path: lcov_smoke.info
 
   upload-to-codecov:
-    if: (contains(github.event.pull_request.labels.*.name, 'CICD:run-coverage') || ${{ github.event_name }} == 'schedule') && !cancelled()
+    if: |
+      contains(github.event.pull_request.labels.*.name, 'CICD:run-coverage') ||
+      (github.event_name == 'schedule' && github.ref_name == 'main')
+      && !cancelled()
     needs: [ rust-unit-coverage, rust-smoke-coverage ]
     runs-on: ubuntu-latest
     steps:

.github/workflows/docker-build-test.yaml

@@ -52,8 +52,6 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
-  GCP_DOCKER_ARTIFACT_REPO_US: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO_US }}
   AWS_ECR_ACCOUNT_NUM: ${{ secrets.ENV_ECR_AWS_ACCOUNT_NUM }}
   # In case of pull_request events by default github actions merges main into the PR branch and then runs the tests etc
   # on the prospective merge result instead of only on the tip of the PR.

.github/workflows/docker-indexer-grpc-test.yaml

@@ -17,9 +17,9 @@ jobs:
       contents: read
       id-token: write
     env:
-      VALIDATOR_IMAGE_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/validator
-      FAUCET_IMAGE_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/faucet
-      INDEXER_GRPC_IMAGE_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/indexer-grpc
+      VALIDATOR_IMAGE_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}/validator
+      FAUCET_IMAGE_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}/faucet
+      INDEXER_GRPC_IMAGE_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}/indexer-grpc
       IMAGE_TAG: ${{ inputs.GIT_SHA || 'devnet' }} # hardcode to a known good build when not running on workflow_call
 
     steps:

.github/workflows/docker-update-images.yaml

@@ -30,7 +30,7 @@ jobs:
 
       - name: Create Pull Request
         if: ${{ steps.check_update.outputs.NEED_UPDATE == 'True' }}
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc # pin@v6.0.1
         with:
           token: ${{ secrets.REPO_TOKEN }}
           commit-message: "Update Docker images"

.github/workflows/faucet-tests-main.yaml

@@ -71,4 +71,4 @@ jobs:
         with:
           NETWORK: custom
           IMAGE_TAG: ${{ env.IMAGE_TAG }}
-          GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+          GCP_DOCKER_ARTIFACT_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}

.github/workflows/faucet-tests-prod.yaml

@@ -17,7 +17,7 @@ on:
 
 permissions:
   contents: read
-  id-token: write  # Required for GCP Workload Identity federation which we use to login into Google Artifact Registry
+  id-token: write # Required for GCP Workload Identity federation which we use to login into Google Artifact Registry
 
 jobs:
   # Note on the job-level `if` conditions:
@@ -51,7 +51,7 @@ jobs:
       - uses: ./.github/actions/run-faucet-tests
         with:
           NETWORK: devnet
-          GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+          GCP_DOCKER_ARTIFACT_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}
 
   # These tests ensure that the faucet works with the nodes running on testnet.
   run-tests-testnet:
@@ -74,4 +74,4 @@ jobs:
       - uses: ./.github/actions/run-faucet-tests
         with:
           NETWORK: testnet
-          GCP_DOCKER_ARTIFACT_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}
+          GCP_DOCKER_ARTIFACT_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}

.github/workflows/indexer-grpc-integration-tests.yaml

@@ -33,9 +33,9 @@ jobs:
     runs-on: high-perf-docker
     env:
       # spin up the local testnet using the latest devnet image
-      VALIDATOR_IMAGE_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/validator
-      FAUCET_IMAGE_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/faucet
-      INDEXER_GRPC_IMAGE_REPO: ${{ secrets.GCP_DOCKER_ARTIFACT_REPO }}/indexer-grpc
+      VALIDATOR_IMAGE_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}/validator
+      FAUCET_IMAGE_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}/faucet
+      INDEXER_GRPC_IMAGE_REPO: ${{ vars.GCP_DOCKER_ARTIFACT_REPO }}/indexer-grpc
       IMAGE_TAG: devnet
 
     steps:

.github/workflows/lint-test.yaml

@@ -118,9 +118,33 @@ jobs:
         if: needs.file_change_determinator.outputs.only_docs_changed != 'true'
         with:
           GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
+          BUILDKITE_ANALYTICS_TOKEN: ${{ secrets.BUILDKITE_ANALYTICS_TOKEN }}
       - run: echo "Skipping rust unit tests! Unrelated changes detected."
         if: needs.file_change_determinator.outputs.only_docs_changed == 'true'
 
+  # Run the cached packages build. This is a PR required job.
+  rust-build-cached-packages:
+    needs: file_change_determinator
+    if: | # Only run on each PR once an appropriate event occurs
+      (
+        github.event_name == 'workflow_dispatch' ||
+        github.event_name == 'push' ||
+        contains(github.event.pull_request.labels.*.name, 'CICD:run-e2e-tests') ||
+        github.event.pull_request.auto_merge != null
+      )
+    runs-on: high-perf-docker
+    steps:
+      - uses: actions/checkout@v4
+        if: needs.file_change_determinator.outputs.only_docs_changed != 'true'
+      - uses: aptos-labs/aptos-core/.github/actions/rust-setup@main
+        with:
+          GIT_CREDENTIALS: ${{ secrets.GIT_CREDENTIALS }}
+      - name: Run aptos cached packages build test
+        if: needs.file_change_determinator.outputs.only_docs_changed != 'true'
+        run: scripts/cargo_build_aptos_cached_packages.sh --check
+      - run: echo "Skipping cached packages test! Unrelated changes detected."
+        if: needs.file_change_determinator.outputs.only_docs_changed == 'true'
+
   # Run the consensus only unit tests
   rust-consensus-only-unit-test:
     runs-on: high-perf-docker