test

.github/workflows/test_api_cpu.yaml

@@ -53,7 +53,12 @@ jobs:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           PUSH_REPO_ID: optimum-benchmark/cpu
 
-      # - name: Run examples
+      # - if: ${{
+      #     (github.event_name == 'push') ||
+      #     (github.event_name == 'workflow_dispatch') ||
+      #     contains( github.event.pull_request.labels.*.name, 'examples')
+      #     }}
+      #   name: Run examples
       #   run: |
       #     pytest tests/test_examples.py -s -k "api and cpu"
       #   env:

.github/workflows/test_api_cuda.yaml

@@ -51,8 +51,14 @@ jobs:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
           PUSH_REPO_ID: optimum-benchmark/cuda
 
-      - name: Run examples
+      - if: ${{
+          (github.event_name == 'push') ||
+          (github.event_name == 'workflow_dispatch') ||
+          contains( github.event.pull_request.labels.*.name, 'examples')
+          }}
+        name: Run examples
         run: |
+          pip install -e .[testing,torchao,autoawq,auto-gptq]
           pytest tests/test_examples.py -x -s -k "api and cuda and pytorch"
         env:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}

.github/workflows/test_cli_cpu_ipex.yaml

@@ -32,9 +32,6 @@ jobs:
 
     runs-on: ubuntu-latest
 
-    container:
-      image: ghcr.io/huggingface/optimum-benchmark:latest-cpu
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -46,5 +43,10 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and ipex"
 
-      # - name: Run examples
+      # - if: ${{
+      #     (github.event_name == 'push') ||
+      #     (github.event_name == 'workflow_dispatch') ||
+      #     contains( github.event.pull_request.labels.*.name, 'examples')
+      #     }}
+      #   name: Run examples
       #   run: pytest tests/test_examples.py -s -k "cli and cpu and ipex"

.github/workflows/test_cli_cpu_llama_cpp.yaml

@@ -50,5 +50,10 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and llama_cpp"
 
-      - name: Run examples
+      - if: ${{
+          (github.event_name == 'push') ||
+          (github.event_name == 'workflow_dispatch') ||
+          contains( github.event.pull_request.labels.*.name, 'examples')
+          }}
+        name: Run examples
         run: pytest tests/test_examples.py -s -k "cli and cpu and llama_cpp"

.github/workflows/test_cli_cpu_onnxruntime.yaml

@@ -50,5 +50,10 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and onnxruntime"
 
-      - name: Run examples
+      - if: ${{
+          (github.event_name == 'push') ||
+          (github.event_name == 'workflow_dispatch') ||
+          contains( github.event.pull_request.labels.*.name, 'examples')
+          }}
+        name: Run examples
         run: pytest tests/test_examples.py -s -k "cli and cpu and onnxruntime"

.github/workflows/test_cli_cpu_openvino.yaml

@@ -32,9 +32,6 @@ jobs:
 
     runs-on: ubuntu-latest
 
-    container:
-      image: ghcr.io/huggingface/optimum-benchmark:latest-cpu
-
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -47,5 +44,10 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and openvino"
 
-      # - name: Run examples
+      # - if: ${{
+      #     (github.event_name == 'push') ||
+      #     (github.event_name == 'workflow_dispatch') ||
+      #     contains( github.event.pull_request.labels.*.name, 'examples')
+      #     }}
+      #   name: Run examples
       #   run: pytest tests/test_examples.py -s -k "cli and cpu and openvino"

.github/workflows/test_cli_cpu_py_txi.yaml

@@ -50,5 +50,10 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and py_txi"
 
-      # - name: Run examples
-      #   run: pytest tests/test_examples.py -s -k "cli and cpu and tgi"
+      # - if: ${{
+      #     (github.event_name == 'push') ||
+      #     (github.event_name == 'workflow_dispatch') ||
+      #     contains( github.event.pull_request.labels.*.name, 'examples')
+      #     }}
+      #   name: Run examples
+      #   run: pytest tests/test_examples.py -s -k "cli and cpu and (tgi or tei)"

.github/workflows/test_cli_cpu_pytorch.yaml

@@ -50,5 +50,10 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -s -k "cli and cpu and pytorch"
 
-      # - name: Run examples
+      # - if: ${{
+      #     (github.event_name == 'push') ||
+      #     (github.event_name == 'workflow_dispatch') ||
+      #     contains( github.event.pull_request.labels.*.name, 'examples')
+      #     }}
+      #   name: Run examples
       #   run: pytest tests/test_examples.py -s -k "cli and cpu and pytorch"

.github/workflows/test_cli_cuda_onnxruntime.yaml

@@ -49,5 +49,10 @@ jobs:
         run: |
           pytest tests/test_cli.py -x -s -k "cli and cuda and onnxruntime"
 
-      # - name: Run examples
+      # - if: ${{
+      #     (github.event_name == 'push') ||
+      #     (github.event_name == 'workflow_dispatch') ||
+      #     contains( github.event.pull_request.labels.*.name, 'examples')
+      #     }}
+      #   name: Run examples
       #   run: pytest tests/test_examples.py -x -s -k "cli and cuda and onnxruntime"

.github/workflows/test_cli_cuda_py_txi.yaml

@@ -50,5 +50,10 @@ jobs:
       - name: Run tests
         run: pytest tests/test_cli.py -x -s -k "cli and cuda and py_txi"
 
-      - name: Run examples
-        run: pytest tests/test_examples.py -x -s -k "cli and cuda and tgi"
+      - if: ${{
+          (github.event_name == 'push') ||
+          (github.event_name == 'workflow_dispatch') ||
+          contains( github.event.pull_request.labels.*.name, 'examples')
+          }}
+        name: Run examples
+        run: pytest tests/test_examples.py -x -s -k "cli and cuda and (tgi or tei)"

.github/workflows/test_cli_cuda_pytorch.yaml

@@ -50,7 +50,12 @@ jobs:
         run: |
           pytest tests/test_cli.py -x -s -k "cli and cuda and pytorch and not (dp or ddp or device_map or deepspeed)"
 
-      - name: Run examples
+      - if: ${{
+          (github.event_name == 'push') ||
+          (github.event_name == 'workflow_dispatch') ||
+          contains( github.event.pull_request.labels.*.name, 'examples')
+          }}
+        name: Run examples
         run: pytest tests/test_examples.py -x -s -k "cli and cuda and pytorch"
 
   run_cli_cuda_pytorch_multi_gpu_tests:

.github/workflows/test_cli_cuda_tensorrt_llm.yaml

@@ -46,11 +46,16 @@ jobs:
         run: |
           pip install -e .[testing]
 
-      - name: Run tests
+      - name: Run tests (sequential)
         run: |
-          pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and not (tp or pp)"
+          FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and tensorrt_llm and not (tp or pp)"
 
-      - name: Run examples
+      - if: ${{
+          (github.event_name == 'push') ||
+          (github.event_name == 'workflow_dispatch') ||
+          contains( github.event.pull_request.labels.*.name, 'examples')
+          }}
+        name: Run examples
         run: pytest tests/test_examples.py -x -s -k "cli and cuda and trt"
 
   cli_cuda_tensorrt_llm_multi_gpu_tests:

.github/workflows/test_cli_cuda_vllm.yaml

@@ -50,7 +50,12 @@ jobs:
         run: |
           FORCE_SEQUENTIAL=1 pytest tests/test_cli.py -x -s -k "cli and cuda and vllm and not (tp or pp)"
 
-      - name: Run examples
+      - if: ${{
+          (github.event_name == 'push') ||
+          (github.event_name == 'workflow_dispatch') ||
+          contains( github.event.pull_request.labels.*.name, 'examples')
+          }}
+        name: Run examples
         run: |
           pytest tests/test_examples.py -x -s -k "cli and cuda and vllm"
 

setup.py

@@ -76,6 +76,7 @@
     "py-txi": ["py-txi"],
     "vllm": ["vllm"],
     # optional dependencies
+    "torchao": ["torchao"],
     "autoawq": ["autoawq"],
     "auto-gptq": ["optimum", "auto-gptq"],
     "sentence-transformers": ["sentence-transformers"],