job_onnx_models_tests.yml

name: ONNX Models tests

on:
  workflow_call:
    inputs:
      runner:
        description: 'Machine on which the tests would run'
        type: string
        required: true
      container:
        description: 'JSON to be converted to the value of the "container" configuration for the job'
        type: string
        required: false
        default: '{"image": null}'

permissions: read-all

jobs:
  ONNX_Models_tests:
    name: ONNX Models tests
    timeout-minutes: 60
    runs-on: ${{ inputs.runner }}
    container: ${{ fromJSON(inputs.container) }}
    defaults:
      run:
        shell: bash
    env:
      DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
      OPENVINO_REPO: ${{ github.workspace }}/openvino
      INSTALL_DIR: ${{ github.workspace }}/install
      INSTALL_WHEELS_DIR: ${{ github.workspace }}/install/wheels
      INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests
      ONNX_MODELS_PATH: ${{ github.workspace }}/onnx_test_models
      # instead of using static MODELS_SHARE_PATH
      # choose one of the replicas dynamically instead
      # depending on GITHUB_RUN_NUMBER variable
      NUMBER_OF_REPLICAS: 2
      ONNX_MODEL_ZOO_SHA: "5faef4c33eba0395177850e1e31c4a6a9e634c82"
      USE_SYSTEM_CACHE: False # Using remote HuggingFace cache
    if: ${{ github.event_name != 'merge_group' }}
    steps:
      - name: Download OpenVINO artifacts (tests)
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: openvino_[tests]*
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true
          
      - name: Download OpenVINO artifacts (wheels)
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: openvino_[wheels]*
          path: ${{ env.INSTALL_WHEELS_DIR }}
          merge-multiple: true
        
      # Needed as ${{ github.workspace }} is not working correctly when using Docker
      - name: Setup Variables
        run: |
          echo "OPENVINO_REPO=$GITHUB_WORKSPACE/openvino" >> "$GITHUB_ENV"
          echo "INSTALL_DIR=$GITHUB_WORKSPACE/install" >> "$GITHUB_ENV"
          echo "INSTALL_TEST_DIR=$GITHUB_WORKSPACE/install/tests" >> "$GITHUB_ENV"
          echo "MODELS_SHARE_PATH=/mount/testdata$((GITHUB_RUN_NUMBER % NUMBER_OF_REPLICAS))" >> "$GITHUB_ENV"
          echo "LOGS_FOLDER=$GITHUB_WORKSPACE/onnx_models_tests_logs" >> "$GITHUB_ENV"
          echo "HF_HUB_CACHE=/mount/caches/huggingface" >> "$GITHUB_ENV"

      - name: Setup HuggingFace Cache Directory (Windows)
        if: runner.os == 'Windows'
        run: Add-Content -Path $env:GITHUB_ENV -Value "HF_HUB_CACHE=C:\\mount\\caches\\huggingface"

      - name: Extract OpenVINO packages and tests
        run: |
            pigz -dc openvino_tests.tar.gz | tar -xf - -C ${INSTALL_DIR}
        working-directory: ${{ env.INSTALL_DIR }}

      # Issue 148922
      # Can be a possible root cause for the bug
      # Uncomment only when there's a need
      # to update the models on the shared drive
      # - name: Fetch model_zoo_preprocess script
      #   uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      #   with:
      #     sparse-checkout: |
      #       src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh
      #     sparse-checkout-cone-mode: false
      #     path: 'openvino'

      # - name: Update Models
      #   run: bash ${OPENVINO_REPO}/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d ${MODELS_SHARE_PATH} -o -s "${{ env.ONNX_MODEL_ZOO_SHA }}"

      - name: Install OpenVINO Python wheels
        run: |
          # Install the core OV wheel
          python3 -m pip install ./openvino-*.whl
        working-directory: ${{ env.INSTALL_WHEELS_DIR }}

      - name: Install ONNX Models tests requirements
        run: python3 -m pip install -r ${INSTALL_TEST_DIR}/requirements_onnx

      - name: Install Python tests dependencies
        run: |
          # To enable pytest parallel features
          python3 -m pip install pytest-xdist[psutil] pytest-forked

      - name: Find ONNX models
        run: |
          find ${MODELS_SHARE_PATH} -depth -type f -name "*.onnx" -print

      - name: Collect ONNX Models tests
        run: |
          python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n 1 -k 'not _cuda' --model_zoo_xfail --collect-only

      - name: ONNX Models Tests
        run: |
          python3 -m pytest --backend="CPU" --model_zoo_dir="${MODELS_SHARE_PATH}" ${INSTALL_TEST_DIR}/onnx/tests/tests_python/test_zoo_models.py -v -n auto --forked -k 'not _cuda' --model_zoo_xfail

      - name: Upload logs from pytest
        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
        if: always()
        with:
          name: onnx_models_tests_logs
          path: ${{ env.LOGS_FOLDER }}
          if-no-files-found: 'ignore'