build_treon_reusable.yml

name: Build Treon Reusable Workflow

on:
  workflow_call:
    inputs:
      runs_on:
        required: true
        type: string
      python:
        required: true
        type: string
      container:
        required: false
        type: string
        default: null
      test_only_changed:
        required: false
        type: boolean
        default: false

permissions:
  contents: read

jobs:
  build_treon:
    runs-on: ${{ inputs.runs_on }}
    container:
      image: ${{ inputs.container }}
      options: --device /dev/dri:/dev/dri --group-add 109 --group-add 44
      volumes:
        - /dev/dri:/dev/dri
    steps:
      - name: Set env variables
        uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
        with:
          script: |
            const container = "${{ inputs.container }}";
            const runsOn = "${{ inputs.runs_on }}";
            const osName = !container ? "${{ inputs.runs_on }}" : container.replace(':', '-');
            const testDevice = runsOn === 'gpu' ? 'gpu' : 'cpu';
            const testReportDir = `${testDevice}-${osName}-${{ inputs.python }}`;
            core.exportVariable('OS_NAME', osName);
            core.exportVariable('TEST_DEVICE', testDevice);
            core.exportVariable('TEST_REPORT_DIR', testReportDir);
            core.exportVariable('GIT_CLONE_PROTECTION_ACTIVE', 'false');

      #### Installation/preparation ####
      #
      # These steps are also copied to convert_notebooks.yml
      - name: Checkout repository
        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6

      - name: Get changed files
        if: ${{ inputs.test_only_changed }}
        id: changed-files
        uses: tj-actions/changed-files@d6babd6899969df1a11d14c368283ea4436bca78 # v44.5.2
        with:
          files: |
            notebooks/*/**
            requirements.txt

      - name: List all changed files
        if: ${{ inputs.test_only_changed }}
        shell: bash
        run: |
          touch test_notebooks.txt
          changed_files="${{ steps.changed-files.outputs.all_changed_files }}"
          changed_files=$(echo $changed_files | tr '\\' '/')
          for file in $changed_files; do
            echo "$file was changed"
            echo $file >> test_notebooks.txt
          done

      - name: Dotenv Action
        id: dotenv
        uses: xom9ikk/dotenv@ac290ca23a42155a0cba1031d23afa46240116a9 # v2.3.0
        with:
          path: ./.github/workflows
      # Packages that notebooks need to run in plain os
      - name: Install required packages
        if: ${{ !inputs.container }}
        shell: bash
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
            sudo apt-get update -y
            sudo apt-get install libsndfile1 ffmpeg -y
          elif [ "$RUNNER_OS" == "macOS" ]; then
            brew install ffmpeg
          elif [ "$RUNNER_OS" == "Windows" ]; then
            choco install ffmpeg-full
          fi
      # Packages that notebooks need to run in docker container
      - name: Install required packages (container)
        if: ${{ inputs.container }}
        shell: bash
        env:
          DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input
          TZ: 'Europe/London' # to prevent tzdata from waiting user input
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
            apt-get update -y
            apt-get install git curl wget libsndfile1 libssl-dev unzip libsqlite3-dev libedit-dev libgl1 libgl1-mesa-glx libglib2.0-0 ffmpeg tk -y
            wget https://raw.githubusercontent.com/openvinotoolkit/openvino/master/scripts/install_dependencies/install_openvino_dependencies.sh
            chmod +x ./install_openvino_dependencies.sh
            ./install_openvino_dependencies.sh -c=core -c=dev -c=gpu -y
          fi

      - name: Install GPU Drivers
        if: ${{ inputs.container }}
        shell: bash
        run: |
          wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.15985.7/intel-igc-core_1.0.15985.7_amd64.deb
          wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.15985.7/intel-igc-opencl_1.0.15985.7_amd64.deb
          wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-level-zero-gpu-dbgsym_1.3.28454.6_amd64.ddeb
          wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-level-zero-gpu_1.3.28454.6_amd64.deb
          wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-opencl-icd-dbgsym_24.05.28454.6_amd64.ddeb
          wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/intel-opencl-icd_24.05.28454.6_amd64.deb
          wget https://github.com/intel/compute-runtime/releases/download/24.05.28454.6/libigdgmm12_22.3.11_amd64.deb
          dpkg -i *.deb

      - name: Set up Python
        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: '${{ inputs.python }}'
        env:
          AGENT_TOOLSDIRECTORY: ${{ inputs.container && '/opt/hostedtoolcache' || '' }}

      - name: Cache Pip Packages
        id: cachepip
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            pipcache
          key: ${{ env.PIP_CACHE_KEY }}-${{ env.OS_NAME }}-${{ inputs.python }}

      # Cache specific files to reduce downloads or prevent network issues
      - name: Cache Files
        id: cachefiles
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            # NOTE: when modifying cache paths, update FILES_CACHE_KEY in .env
            # and change cache paths in both treon.yml and convert_notebooks.yml
            case_00030.zip
            notebooks/ct-segmentation-quantize/kits19_frames_1
            notebooks/pytorch-post-training-quantization-nncf/output/tiny-imagenet-200.zip
            # omz cache location is set to this with test_replace
            notebooks/optical-character-recognition/open_model_zoo_cache
            notebooks/ct-scan-live-inference/kits19_frames_1
            notebooks/pytorch-quantization-aware-training/data/tiny-imagenet-200.zip
          key: ${{ env.FILES_CACHE_KEY }}

      # PaddleGAN stores cache in ppgan directory in CACHE_DIR
      - name: Set CACHE_DIR
        shell: bash
        run: |
          python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))'
          # replace backslashes with forward slashes for Windows paths
          python -c 'import os;print("CACHE_DIR={0}".format(os.path.expanduser(os.path.join("~", ".cache"))))' | sed -e 's/\\/\//g' >> $GITHUB_ENV

      # PaddleHub stores cache in directory pointed to by HUB_HOME environment variable
      - name: Set HUB_HOME
        shell: bash
        run: |
          echo HUB_HOME=${{ env.CACHE_DIR }}/.paddlehub >> $GITHUB_ENV

      # Cache PaddlePaddle cache directories to prevent CI failing due to network/download issues
      - name: Cache PaddlePaddle cache directories
        id: cacheusercache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            ${{ env.HUB_HOME }}
            ${{ env.CACHE_DIR }}/paddle
            ${{ env.CACHE_DIR }}/ppgan
          key: ${{ env.USER_CACHE_KEY }}-${{ env.OS_NAME }}

      - name: Cache openvino packages
        if: steps.cachepip.outputs.cache-hit != 'true'
        run: |
          python -m pip install --upgrade pip
          mkdir pipcache
          python -m pip install --cache-dir pipcache --no-deps openvino openvino-dev nncf
          cp -r pipcache pipcache_openvino
          # python -m pip uninstall -y openvino openvino-dev nncf

      # Download a small dataset to use for testing purposes in monai-kidney training notebook
      - name: Download CT files
        if: steps.cachefiles.outputs.cache-hit != 'true'
        run: |
          curl -O https://storage.openvinotoolkit.org/data/test_data/openvino_notebooks/kits19/case_00030.zip
      - name: Copy CT files
        run: |
          mkdir notebooks/ct-segmentation-quantize/kits19
          mkdir notebooks/ct-segmentation-quantize/kits19/kits19_frames
          unzip case_00030.zip
          cp -r case_00030 case_00001
          mv case_00030 notebooks/ct-segmentation-quantize/kits19/kits19_frames
          mv case_00001 notebooks/ct-segmentation-quantize/kits19/kits19_frames

      # Prevent test aborting by timeout for 'meter-reader' and 'paddle-ocr-webcam' notebooks
      - name: Download long loading models for 'meter-reader' and 'paddle-ocr-webcam' notebooks
        run: |
          mkdir notebooks/meter-reader/model
          curl -o notebooks/meter-reader/model/meter_det_model.tar.gz 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/meter-reader/meter_det_model.tar.gz'
          curl -o notebooks/meter-reader/model/meter_seg_model.tar.gz 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/meter-reader/meter_seg_model.tar.gz'
          mkdir notebooks/paddle-ocr-webcam/model
          curl -o notebooks/paddle-ocr-webcam/model/ch_PP-OCRv3_det_infer.tar 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/paddle-ocr/ch_PP-OCRv3_det_infer.tar'
          curl -o notebooks/paddle-ocr-webcam/model/ch_PP-OCRv3_rec_infer.tar 'https://storage.openvinotoolkit.org/repositories/openvino_notebooks/models/paddle-ocr/ch_PP-OCRv3_rec_infer.tar'

      - name: Install python dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r .ci/dev-requirements.txt --cache-dir pipcache
          python -m ipykernel install --user --name openvino_env

      # Cache OpenVINO packages (`mv` works cross-platform)
      - name: Make pipcache directory with OpenVINO packages
        if: steps.cachepip.outputs.cache-hit != 'true'
        run: |
          mv pipcache pipcache_full
          mv pipcache_openvino pipcache

      # Create list of installed pip packages that can be downloaded as artifacts
      # to verify the exact environment of a specific test run
      - name: pip freeze
        run: |
          python -m pip freeze
          python -m pip freeze > pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
      - name: Upload pip freeze artifact
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          name: pip-freeze-${{ env.TEST_DEVICE }}-${{ env.OS_NAME }}-${{ inputs.python }}
          path: pip-freeze-${{ env.TEST_DEVICE }}-${{ github.sha }}-${{ env.OS_NAME }}-${{ inputs.python }}.txt
      #### End installation/preparation

      - name: Check install
        run: |
          python check_install.py

      # Patch long running cells to run faster
      - name: Patch notebooks
        run: |
          python .ci/patch_notebooks.py . -td ${{ env.TEST_DEVICE }}

      # Test that JupyterLab runs without errors
      - name: Test Jupyterlab
        run: |
          jupyter lab notebooks --help

      # Main notebooks test. Verifies that all notebooks run without errors
      - name: Analysing with treon (Windows)
        if: runner.os == 'Windows'
        shell: bash
        run: |
          python .ci/validate_notebooks.py \
            --os ${{ env.OS_NAME }} \
            --python ${{ inputs.python }} \
            --device ${{ env.TEST_DEVICE }} \
            ${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
            ${{ env.TEST_DEVICE == 'gpu' && '--ignore_list .ci/heavy_win_gpu.txt' || '' }} \
            --report_dir test_report/${{ env.TEST_REPORT_DIR }} \
            --move_notebooks_dir c:/notebooks \
            --timeout 1200
      - name: Analysing with treon (Linux)
        if: runner.os == 'Linux'
        shell: bash
        run: |
          python .ci/validate_notebooks.py \
            --os ${{ env.OS_NAME }} \
            --python ${{ inputs.python }} \
            --device ${{ env.TEST_DEVICE }} \
            ${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
            ${{ env.TEST_DEVICE == 'gpu' && '--ignore_list .ci/heavy_ubuntu_gpu.txt' || '' }} \
            --report_dir test_report/${{ env.TEST_REPORT_DIR }} \
            --timeout 1200
      - name: Analysing with treon (MacOS)
        if: runner.os == 'MacOS'
        shell: bash
        run: |
          python .ci/validate_notebooks.py \
            --os ${{ env.OS_NAME }} \
            --python ${{ inputs.python }} \
            --device ${{ env.TEST_DEVICE }} \
            ${{ inputs.test_only_changed && '--test_list test_notebooks.txt' || '' }} \
            --report_dir test_report/${{ env.TEST_REPORT_DIR }} \
            --timeout 1200

      - name: Archive notebook test report
        if: always()
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          name: test_report-${{ env.TEST_REPORT_DIR }}
          path: test_report/

      # Show the cache after running the notebooks
      - name: Show cache
        if: runner.os != 'Windows'
        run: |
          ls -laR ${{ env.CACHE_DIR }}
          du -sh ${{ env.CACHE_DIR }}