forest.yml

name: Integration tests
concurrency:
  group: '${{ github.workflow }}-${{ github.ref }}'
  cancel-in-progress: '${{ github.ref != ''refs/heads/main'' }}'
on:
  workflow_dispatch:
  merge_group:
  pull_request:
    branches:
      - main
    paths-ignore:
      - 'docs/**'
      - '.github/workflows/docs-*.yml'
  push:
    branches:
      - main
    paths-ignore:
      - 'docs/**'
      - '.github/workflows/docs-*.yml'

env:
  CI: 1
  CARGO_INCREMENTAL: 0
  CACHE_TIMEOUT_MINUTES: 5
  SCRIPT_TIMEOUT_MINUTES: 30
  AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
  AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
  RUSTC_WRAPPER: sccache
  CC: sccache clang
  CXX: sccache clang++
  FIL_PROOFS_PARAMETER_CACHE: /var/tmp/filecoin-proof-parameters
  SHELL_IMAGE: busybox
jobs:
  build-macos:
    name: Build MacOS
    runs-on: macos-latest
    steps:
      - name: Configure SCCache variables
        run: |
          # External PRs do not have access to 'vars' or 'secrets'.
          if [[ "$AWS_ACCESS_KEY_ID" != "" ]]; then
            echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV
            echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV
            echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV
          fi
          echo "SCCACHE_S3_KEY_PREFIX=macos" >> $GITHUB_ENV
      - name: Checkout Sources
        uses: actions/checkout@v4
      - name: Setup sccache
        uses: mozilla-actions/sccache-action@v0.0.7
        timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
        continue-on-error: true
      - uses: actions/setup-go@v5
        with:
          go-version-file: "go.work"
      - name: Cargo Check
        run: cargo check
  build-ubuntu:
    name: Build Ubuntu
    runs-on: ubuntu-24.04
    steps:
      - name: Configure SCCache variables
        run: |
          # External PRs do not have access to 'vars' or 'secrets'.
          if [[ "${{secrets.AWS_ACCESS_KEY_ID}}" != "" ]]; then
            echo "SCCACHE_ENDPOINT=${{ vars.SCCACHE_ENDPOINT}}" >> $GITHUB_ENV
            echo "SCCACHE_BUCKET=${{ vars.SCCACHE_BUCKET}}" >> $GITHUB_ENV
            echo "SCCACHE_REGION=${{ vars.SCCACHE_REGION}}" >> $GITHUB_ENV
          fi
      - run: lscpu
      - name: Show IP
        run: curl ifconfig.me
        continue-on-error: true
      - name: Checkout Sources
        uses: actions/checkout@v4
      - name: Setup sccache
        uses: mozilla-actions/sccache-action@v0.0.7
        timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
        continue-on-error: true
      - uses: actions/setup-go@v5
        with:
          go-version-file: "go.work"
      - name: Cargo Install
        env:
          # To minimize compile times: https://nnethercote.github.io/perf-book/build-configuration.html#minimizing-compile-times
          RUSTFLAGS: "-C linker=clang -C link-arg=-fuse-ld=lld"
        run: make install-slim-quick
      - uses: actions/upload-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: |
            ~/.cargo/bin/forest*
          if-no-files-found: error
  cargo-publish-dry-run:
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout Sources
        uses: actions/checkout@v4
      - name: Setup sccache
        uses: mozilla-actions/sccache-action@v0.0.7
        timeout-minutes: '${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }}'
        continue-on-error: true
      - name: Install Apt Dependencies
        run: |
          sudo make install-deps
      - uses: actions/setup-go@v5
        with:
          go-version-file: "go.work"
      - run: cargo publish --dry-run
  forest-cli-check:
    needs:
      - build-ubuntu
    name: Forest CLI checks
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: ${{ env.FIL_PROOFS_PARAMETER_CACHE }}
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - name: forest-cli check
        run: ./scripts/tests/forest_cli_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  calibnet-check:
    needs:
      - build-ubuntu
    name: Calibnet check
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - name: Other commands check
        run: ./scripts/tests/calibnet_other_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
        env:
          CC: clang
          CXX: clang++
  calibnet-stateless-mode-check:
    needs:
      - build-ubuntu
    name: Calibnet stateless mode check
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - name: Checkout Sources
        uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - run: ./scripts/tests/calibnet_stateless_mode_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  calibnet-stateless-rpc-check:
    needs:
      - build-ubuntu
    name: Calibnet stateless RPC check
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - name: Checkout Sources
        uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - run: ./scripts/tests/calibnet_stateless_rpc_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  state-migrations-check:
    needs:
      - build-ubuntu
    name: State migrations
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - name: Migration Regression Tests
        run: ./scripts/tests/calibnet_migration_regression_tests.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  calibnet-wallet-check:
    needs:
      - build-ubuntu
    name: Wallet tests
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - name: Wallet commands check
        env:
          CALIBNET_WALLET: '${{ secrets.CALIBNET_WALLET }}'
        run: |
          if [[ "$CALIBNET_WALLET" != "" ]]; then
            ./scripts/tests/calibnet_wallet_check.sh "$CALIBNET_WALLET"
          fi
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  calibnet-export-check:
    needs:
      - build-ubuntu
    name: Snapshot export checks
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - name: Snapshot export check
        run: ./scripts/tests/calibnet_export_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  calibnet-no-discovery-checks:
    needs:
      - build-ubuntu
    name: Calibnet no discovery checks
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - run: ./scripts/tests/calibnet_no_discovery_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  calibnet-kademlia-checks:
    needs:
      - build-ubuntu
    name: Calibnet kademlia checks
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - run: ./scripts/tests/calibnet_kademlia_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  calibnet-eth-mapping-check:
    needs:
      - build-ubuntu
    name: Calibnet eth mapping check
    runs-on: ubuntu-24.04
    steps:
      - run: lscpu
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - name: Checkout Sources
        uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - run: ./scripts/tests/calibnet_eth_mapping_check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  db-migration-checks:
    needs:
      - build-ubuntu
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/cache@v4
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-keys
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
          path: ~/.cargo/bin
      - name: Set permissions
        run: |
          chmod +x ~/.cargo/bin/forest*
      - name: Database migration checks
        run: ./scripts/tests/calibnet_db_migration.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
  local-devnet-check:
    name: Devnet checks
    runs-on: ubuntu-24.04
    needs:
      - build-ubuntu
    env:
      # We use a custom Dockerfile for CI to speed up the build process.
      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
    steps:
      - uses: actions/cache@v4
        id: cache
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-2k
      - name: Load cache into volume
        if: steps.cache.outputs.cache-hit == 'true'
        run: |
          docker volume create devnet_filecoin-proofs
          docker run --rm \
            -v devnet_filecoin-proofs:/proofs \
            -v $FIL_PROOFS_PARAMETER_CACHE:/cache \
            $SHELL_IMAGE \
            sh -c "cp /cache/* /proofs"
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
      - name: Devnet setup
        run: ./scripts/devnet/setup.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Devnet check
        run: ./scripts/devnet/check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Dump docker logs
        if: always()
        uses: jwalton/gh-docker-logs@v2
      - name: Prepare cache folder for uploading
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          docker run --rm \
            -v devnet_filecoin-proofs:/proofs \
            -v $FIL_PROOFS_PARAMETER_CACHE:/cache \
            $SHELL_IMAGE \
            sh -c "cp /proofs/* /cache"
          sudo chmod -R 755 $FIL_PROOFS_PARAMETER_CACHE
  local-devnet-curio-check:
    # Disabling this job as it is not providing any value until the curio setup is fixed. See: https://github.com/ChainSafe/forest/issues/5171
    if: false
    name: Devnet Curio checks
    runs-on: ubuntu-24.04
    needs:
      - build-ubuntu
    env:
      # We use a custom Dockerfile for CI to speed up the build process.
      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
    steps:
      - uses: actions/cache@v4
        id: cache
        with:
          path: '${{ env.FIL_PROOFS_PARAMETER_CACHE }}'
          key: proof-params-2k
      - name: Load cache into volume
        if: steps.cache.outputs.cache-hit == 'true'
        run: |
          docker volume create devnet_filecoin-proofs
          docker run --rm \
            -v devnet_filecoin-proofs:/proofs \
            -v $FIL_PROOFS_PARAMETER_CACHE:/cache \
            $SHELL_IMAGE \
            sh -c "cp /cache/* /proofs"
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
      - name: Devnet setup
        run: |
          cp ./scripts/devnet/setup.sh ./scripts/devnet-curio/setup.sh
          chmod +x ./scripts/devnet-curio/setup.sh
          ./scripts/devnet-curio/setup.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Devnet check
        run: |
          cp ./scripts/devnet/check.sh ./scripts/devnet-curio/check.sh
          chmod +x ./scripts/devnet-curio/check.sh
          ./scripts/devnet-curio/check.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Dump docker logs
        if: always()
        uses: jwalton/gh-docker-logs@v2
      - name: Prepare cache folder for uploading
        if: steps.cache.outputs.cache-hit != 'true'
        run: |
          docker run --rm \
            -v devnet_filecoin-proofs:/proofs \
            -v $FIL_PROOFS_PARAMETER_CACHE:/cache \
            $SHELL_IMAGE \
            sh -c "cp /proofs/* /cache"
          sudo chmod -R 755 $FIL_PROOFS_PARAMETER_CACHE
  calibnet-rpc-checks:
    needs:
      - build-ubuntu
    name: Calibnet RPC checks
    runs-on: ubuntu-24.04
    env:
      # We use a custom Dockerfile for CI to speed up the build process.
      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
    steps:
      # Remove some unnecessary software to free up space. This should free up around 15-20 GB.
      # This is required because of the limited space on the runner,
      # and disk space-hungry snapshots used in the setup.
      # This is taken from:
      # https://github.com/easimon/maximize-build-space/blob/fc881a613ad2a34aca9c9624518214ebc21dfc0c/action.yml#L121-L136
      # Using the action directly is not feasible as it does some more modifications that break the setup in our case.
      -   name: Remove unnecessary software
          run: |
              echo "Disk space before cleanup"
              df -h
              sudo rm -rf /usr/share/dotnet
              sudo rm -rf /usr/local/lib/android
              sudo rm -rf /opt/ghc
              sudo rm -rf /opt/hostedtoolcache/CodeQL

              echo "Disk space after cleanup"
              df -h

      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
      - name: Run api compare tests
        run: ./scripts/tests/api_compare/setup.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Dump docker logs
        if: always()
        uses: jwalton/gh-docker-logs@v2
      - name: Dump Docker usage
        if: always()
        run: |
          docker system df
          docker system df --verbose
          df -h
  bootstrap-checks-forest:
    needs:
      - build-ubuntu
    name: Bootstrap checks - Forest
    runs-on: ubuntu-24.04
    env:
      # We use a custom Dockerfile for CI to speed up the build process.
      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
      - name: Run bootstrap tests
        run: ./scripts/tests/bootstrapper/test_bootstrapper.sh forest
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Dump docker logs
        if: always()
        uses: jwalton/gh-docker-logs@v2
  bootstrap-checks-lotus:
    needs:
      - build-ubuntu
    name: Bootstrap checks - Lotus
    runs-on: ubuntu-24.04
    env:
      # We use a custom Dockerfile for CI to speed up the build process.
      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
      - name: Run bootstrap tests
        run: ./scripts/tests/bootstrapper/test_bootstrapper.sh lotus
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Dump docker logs
        if: always()
        uses: jwalton/gh-docker-logs@v2
  snapshot-parity-checks:
    needs:
      - build-ubuntu
    name: Calibnet snapshot parity checks
    runs-on: ubuntu-24.04
    # Run the job only on main to limit the runner cost.
    # The downside is that the offending commit can only be caught after being merged
    # and a manual revert is required.
    # Note that running this step on non-draft PR does not save the cost as it always re-run
    # all the jobs when a PR is moved out of draft. Also this parity test will be stale soon
    # once we start including the F3(fast finality) metadata in the Forest snapshots.
    if: github.ref == 'refs/heads/main'
    env:
      # We use a custom Dockerfile for CI to speed up the build process.
      FOREST_DOCKERFILE_OVERRIDE: scripts/devnet/forest_ci.dockerfile
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: 'forest-${{ runner.os }}'
      - name: Run snapshot parity tests
        run: ./scripts/tests/snapshot_parity/setup.sh
        timeout-minutes: '${{ fromJSON(env.SCRIPT_TIMEOUT_MINUTES) }}'
      - name: Dump docker logs
        if: always()
        uses: jwalton/gh-docker-logs@v2
  # Umbrella job to aggregate all integration tests and get their status
  integration-tests-status:
    needs:
      - build-macos
      - build-ubuntu
      - cargo-publish-dry-run
      - forest-cli-check
      - calibnet-check
      - calibnet-stateless-mode-check
      - calibnet-stateless-rpc-check
      - state-migrations-check
      - calibnet-wallet-check
      - calibnet-export-check
      - calibnet-no-discovery-checks
      - calibnet-kademlia-checks
      - calibnet-eth-mapping-check
      - db-migration-checks
      - local-devnet-check
      # - local-devnet-curio-check
      - calibnet-rpc-checks
      - bootstrap-checks-forest
      - bootstrap-checks-lotus
    name: Integration tests status
    runs-on: ubuntu-24.04
    steps:
      - run: echo "All integration tests passed."