[GHA] Introduce SDL pipeline

.github/dependency_review.yml

@@ -0,0 +1,16 @@
+fail-on-severity: 'low'
+allow-licenses:
+  - 'BSD-2-Clause'
+  - 'BSD-3-Clause'
+  - 'MIT'
+  - 'Apache-2.0'
+  - 'ISC'
+  - 'BlueOak-1.0.0'
+  - '0BSD'
+  - 'Python-2.0'
+fail-on-scopes:
+  - 'runtime'
+  - 'development'
+  - 'unknown'
+license-check: true
+vulnerability-check: true

.github/workflows/genai-tools.yml

@@ -70,12 +70,6 @@ jobs:
         uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Lint with flake8
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install flake8 pytest black
-          # stop the build if there are Python syntax errors or undefined names
-          python -m flake8 ${{ env.LLM_BENCH_PYPATH }} --config=${{ env.LLM_BENCH_PYPATH }}/setup.cfg
       - name: Download OpenVINO package
         uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
@@ -170,57 +164,9 @@ jobs:
           python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-llava  -pf ./tools/llm_bench/prompts/llava-1.5-7b.jsonl -ic 4
           rm -rf ./ov_models
 
-  wwb:
-    name: 'WWB tests'
-    defaults:
-      run:
-        shell: bash
-    runs-on: ubuntu-22.04-8-cores
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.11"]
-    needs: [ openvino_download ]
-    env:
-      OV_INSTALL_DIR: ${{ github.workspace }}/ov
-      SRC_DIR: ${{ github.workspace }}
-      WWB_PATH: ${{ github.workspace }}/tools/who_what_benchmark
-
-    steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-        with:
-          submodules: recursive
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Lint with flake8
-        run: |
-          python -m pip install --upgrade pip
-          python -m pip install flake8 pytest black
-          # stop the build if there are Python syntax errors or undefined names
-          python -m flake8 ${{ env.WWB_PATH }} --config=${{ env.WWB_PATH }}/setup.cfg
-      - name: Download OpenVINO package
-        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
-        with:
-          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
-          path: ${{ env.OV_INSTALL_DIR }}
-          merge-multiple: true
-      - name: Install dependencies
-        run: |
-          python -m pip install ${{ env.SRC_DIR }}/thirdparty/openvino_tokenizers -v ${{ needs.openvino_download.outputs.ov_wheel_source }}
-          python -m pip install ${{ env.SRC_DIR }} -v ${{ needs.openvino_download.outputs.ov_wheel_source }}
-          python -m pip install -r ${{ env.WWB_PATH }}/requirements.txt ${{ needs.openvino_download.outputs.ov_wheel_source }}
-          python -m pip install git+https://github.com/huggingface/optimum-intel.git@main
-        working-directory: ${{ env.OV_INSTALL_DIR }}
-      - name: WWB Tests
-        run: |
-          python -m pip install -v ${{ env.WWB_PATH }}
-          python -m pytest -v ${{ env.WWB_PATH }}/tests
-
   Overall_Status:
     name: ci/gha_overall_status_llm_bench
-    needs: [openvino_download, llm_bench, wwb]
+    needs: [openvino_download, llm_bench]
     if: ${{ always() }}
     runs-on: ubuntu-latest
     steps:

.github/workflows/linux.yml

@@ -335,8 +335,17 @@ jobs:
             cmd: 'tests/python_tests/test_kv_cache_eviction.py'
             timeout: 60
           - name: 'LLM & VLM'
-            cmd: 'tests/python_tests --ignore tests/python_tests/test_whisper_pipeline.py --ignore tests/python_tests/test_kv_cache_eviction.py --ignore tests/python_tests/test_whisper_pipeline_static.py'
-            timeout: 90
+            cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py'
+            timeout: 60
+          - name: 'Tokenizer tests'
+            cmd: 'tests/python_tests/test_tokenizer.py'
+            timeout: 60
+          - name: 'API tests'
+            cmd: 'tests/python_tests/test_continuous_batching.py tests/python_tests/test_generation_config.py tests/python_tests/test_sampling.py tests/python_tests/test_text_streamer.py'
+            timeout: 60
+          - name: 'WWB tests'
+            cmd: '/tools/who_what_benchmark/tests'
+            timeout: 60
     defaults:
       run:
         shell: bash
@@ -351,8 +360,6 @@ jobs:
       INSTALL_DIR: ${{ github.workspace }}/install
       SRC_DIR: ${{ github.workspace }}/src
       BUILD_DIR: ${{ github.workspace }}/build
-      TRANSFORMERS_CACHE: ${{ github.workspace }}/models  # Hugging Face transformers cache
-      HF_HOME: ${{ github.workspace }}/datasets           # Hugging Face datasets cache
 
     steps:
       - name: Clone openvino.genai

.github/workflows/mac.yml

@@ -346,10 +346,16 @@ jobs:
             timeout: 45
           - name: 'Cacheopt E2E'
             cmd: 'tests/python_tests/test_kv_cache_eviction.py'
-            timeout: 60 
+            timeout: 60
           - name: 'LLM & VLM'
-            cmd: 'tests/python_tests --ignore tests/python_tests/test_whisper_pipeline.py --ignore tests/python_tests/test_kv_cache_eviction.py --ignore tests/python_tests/test_whisper_pipeline_static.py'
-            timeout: 90
+            cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py'
+            timeout: 60
+          - name: 'Tokenizer tests'
+            cmd: 'tests/python_tests/test_tokenizer.py'
+            timeout: 60
+          - name: 'API tests'
+            cmd: 'tests/python_tests/test_continuous_batching.py tests/python_tests/test_generation_config.py tests/python_tests/test_sampling.py tests/python_tests/test_text_streamer.py'
+            timeout: 60
     defaults:
       run:
         shell: bash

.github/workflows/sdl.yml

@@ -0,0 +1,74 @@
+name: SDL tests
+on:
+  workflow_dispatch:
+  pull_request:
+  merge_group:
+  push:
+    branches:
+      - master
+      - 'releases/**'
+
+permissions: read-all
+
+concurrency:
+  # github.ref is not unique in post-commit
+  group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux-sdl
+  cancel-in-progress: true
+
+env:
+  PYTHON_VERSION: '3.11'
+
+jobs:
+  sdl_tests:
+    name: SDL tests
+    timeout-minutes: 10
+    defaults:
+      run:
+        shell: bash
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    steps:
+      - name: Clone sources and tests
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Setup Python ${{ env.PYTHON_VERSION }}
+        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: 'pip'
+
+      - name: Install Python tests dependencies
+        run: |
+          python3 -m pip install flake8 pytest black bandit
+
+      - name: Lint with flake8 (WWB)
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          python -m flake8 . --config=./setup.cfg
+        working-directory: ${{ github.workspace }}/tools/who_what_benchmark
+
+      - name: Lint with flake8 (LLM)
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          python -m flake8 . --config=./setup.cfg
+        working-directory: ${{ github.workspace }}/tools/llm_bench
+
+      - name: Bandit tests
+        run: python -m bandit --recursive --configfile bandit.yml .
+
+      - name: Run Trivy vulnerability scanner in fs mode
+        uses: aquasecurity/trivy-action@18f2510ee396bbf400402947b394f2dd8c87dbb0 # v0.29.0
+        with:
+          scan-type: 'fs'
+          scan-ref: '.'
+
+      - name: Dependency Review
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: actions/dependency-review-action@3b139cfc5fae8b618d3eae3675e383bb1769c019 # v4.5.0
+        with:
+          config-file: './.github/dependency_review.yml'
+

.github/workflows/windows.yml

@@ -381,8 +381,14 @@ jobs:
             cmd: 'tests/python_tests/test_kv_cache_eviction.py'
             timeout: 60
           - name: 'LLM & VLM'
-            cmd: 'tests/python_tests --ignore tests/python_tests/test_whisper_pipeline.py --ignore tests/python_tests/test_whisper_pipeline_static.py --ignore tests/python_tests/test_kv_cache_eviction.py'
-            timeout: 90
+            cmd: 'tests/python_tests/test_llm_pipeline.py tests/python_tests/test_llm_pipeline_static.py tests/python_tests/test_vlm_pipeline.py'
+            timeout: 60
+          - name: 'Tokenizer tests'
+            cmd: 'tests/python_tests/test_tokenizer.py'
+            timeout: 60
+          - name: 'API tests'
+            cmd: 'tests/python_tests/test_continuous_batching.py tests/python_tests/test_generation_config.py tests/python_tests/test_sampling.py tests/python_tests/test_text_streamer.py'
+            timeout: 60
     defaults:
       run:
         shell: pwsh