diff --git a/.github/workflows/pr_stage_test.yml b/.github/workflows/pr_stage_test.yml
index 5dfc6cba6e..30a620cab1 100644
--- a/.github/workflows/pr_stage_test.yml
+++ b/.github/workflows/pr_stage_test.yml
@@ -63,55 +63,17 @@ jobs:
             coverage xml
             coverage report -m
         run: docker exec build-container bash -c "$SCRIPT"
-        
-  build_cu120:
-    runs-on: ubuntu-22.04
-    container:
-      image: pytorch/pytorch:2.5.1-cuda12.1-cudnn9-devel
-    strategy:
-      matrix:
-        python-version: [3.10.x]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Upgrade pip
-        run: pip install pip --upgrade
-      - name: Fetch GPG keys
-        run: |
-          apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
-          apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
-      - name: Install system dependencies
-        run: apt-get update && apt-get install -y git ffmpeg libturbojpeg
-      - name: Build MMEngine from source
-        run: pip install -e . -v
-      - name: Install unit tests dependencies
-        run: |
-          pip install -r requirements/tests.txt
-          pip install openmim
-          mim install mmcv
-      # Distributed related unit test may randomly error in PyTorch 1.13.0
-      - name: Run unittests and generate coverage report
-        run: |
-          coverage run --branch --source mmengine -m pytest tests/  --ignore tests/test_dist/
-          coverage xml
-          coverage report -m
-  build_cu117:
+
+build_cu117:
     runs-on: ubuntu-22.04
     env:
       CONTAINER: continuumio/miniconda3
     strategy:
       matrix:
         python-version: [3.10.16, 3.9]
-        torch: [2.0.0]
+        torch: [2.0.0, 2.5.1]
     steps:
       - uses: actions/checkout@v4
-      # - name: Set up Python ${{ matrix.python-version }}
-      #   uses: actions/setup-python@v4
-      #   with:
-      #     python-version: ${{ matrix.python-version }}
       - name: Start Docker Container
         run: |
           docker pull $CONTAINER