Improve examples (#3091)

### Changes

- Add pattern based on file extension to gitignore for examples
- Use pathlib style instead os.path
- Fix issue with creating ../nncf/~/.cache/datasets directory
- Replace tqdm to nncf.common.logging.track_progress
- Remove unused function arguments and if conditions
- Clean logs, disable progress bar, remove log of validataion
- Pass list of command arguments to subprocess instead of string
- Remove nncf dependencies from tests/cross_fw/examples
- Add `--reuse-venv` to use venv from example directory
- Add progress bar to collect_values_for_each_item_using_prepared_model

### Tests
https://github.com/openvinotoolkit/nncf/actions/runs/11900403097
nightly/job/test_examples/618
nightly/job/windows/job/test-examples/278/

Author: AlexanderDokuchaev

.github/workflows/examples.yml

@@ -44,9 +44,8 @@ jobs:
           cache: pip
       - name: cpuinfo
         run: cat /proc/cpuinfo
-      - name: Install NNCF and test requirements
+      - name: Install test requirements
         run: |
-          pip install -e .
           pip install -r tests/cross_fw/examples/requirements.txt
       - name: Print installed modules
         run: pip list

.gitignore

@@ -119,23 +119,20 @@ nncf_debug/
 
 # NNCF examples
 examples/torch/object_detection/eval/
-examples/post_training_quantization/onnx/mobilenet_v2/mobilenet_v2_*
-examples/post_training_quantization/openvino/mobilenet_v2/mobilenet_v2_*
-examples/post_training_quantization/tensorflow/mobilenet_v2/mobilenet_v2_*
-examples/post_training_quantization/torch/mobilenet_v2/mobilenet_v2_*
-examples/post_training_quantization/torch/ssd300_vgg16/ssd300_vgg16_*
-examples/post_training_quantization/openvino/anomaly_stfpm_quantize_with_accuracy_control/stfpm_*
-examples/post_training_quantization/openvino/yolov8/yolov8n*
-examples/post_training_quantization/openvino/yolov8_quantize_with_accuracy_control/yolov8n*
-examples/**/runs/**
-examples/**/results/**
-examples/llm_compression/openvino/tiny_llama_find_hyperparams/statistics
-compressed_graph.dot
-original_graph.dot
+examples/**/*.xml
+examples/**/*.bin
+examples/**/*.pt
+examples/**/*.onnx
+examples/**/statistics
+examples/**/runs
+examples/**/results
+examples/**/metrics.json
 datasets/**
 
 # Tests
 tests/**/runs/**
 tests/**/tmp*/**
 open_model_zoo/
 nncf-tests.xml
+compressed_graph.dot
+original_graph.dot

examples/post_training_quantization/onnx/mobilenet_v2/main.py

@@ -12,36 +12,37 @@
 import re
 import subprocess
 from pathlib import Path
-from typing import List, Optional
+from typing import List
 
 import numpy as np
 import onnx
 import openvino as ov
 import torch
 from fastdownload import FastDownload
 from fastdownload import download_url
+from rich.progress import track
 from sklearn.metrics import accuracy_score
 from torchvision import datasets
 from torchvision import transforms
-from tqdm import tqdm
 
 import nncf
 
 ROOT = Path(__file__).parent.resolve()
 MODEL_URL = "https://huggingface.co/alexsu52/mobilenet_v2_imagenette/resolve/main/mobilenet_v2_imagenette.onnx"
 DATASET_URL = "https://s3.amazonaws.com/fast-ai-imageclas/imagenette2-320.tgz"
-DATASET_PATH = "~/.cache/nncf/datasets"
-MODEL_PATH = "~/.cache/nncf/models"
+DATASET_PATH = Path().home() / ".cache" / "nncf" / "datasets"
+MODEL_PATH = Path().home() / ".cache" / "nncf" / "models"
 DATASET_CLASSES = 10
 
 
 def download_dataset() -> Path:
-    downloader = FastDownload(base=DATASET_PATH, archive="downloaded", data="extracted")
+    downloader = FastDownload(base=DATASET_PATH.as_posix(), archive="downloaded", data="extracted")
     return downloader.get(DATASET_URL)
 
 
 def download_model() -> Path:
-    return download_url(MODEL_URL, Path(MODEL_PATH).resolve())
+    MODEL_PATH.mkdir(exist_ok=True, parents=True)
+    return download_url(MODEL_URL, MODEL_PATH.resolve())
 
 
 def validate(path_to_model: Path, validation_loader: torch.utils.data.DataLoader) -> float:
@@ -51,7 +52,7 @@ def validate(path_to_model: Path, validation_loader: torch.utils.data.DataLoader
     compiled_model = ov.compile_model(path_to_model, device_name="CPU")
     output = compiled_model.outputs[0]
 
-    for images, target in tqdm(validation_loader):
+    for images, target in track(validation_loader, description="Validating"):
         pred = compiled_model(images)[output]
         predictions.append(np.argmax(pred, axis=1))
         references.append(target)
@@ -61,13 +62,17 @@ def validate(path_to_model: Path, validation_loader: torch.utils.data.DataLoader
     return accuracy_score(predictions, references)
 
 
-def run_benchmark(path_to_model: Path, shape: Optional[List[int]] = None, verbose: bool = True) -> float:
-    command = f"benchmark_app -m {path_to_model} -d CPU -api async -t 15"
-    if shape is not None:
-        command += f' -shape [{",".join(str(x) for x in shape)}]'
-    cmd_output = subprocess.check_output(command, shell=True)  # nosec
-    if verbose:
-        print(*str(cmd_output).split("\\n")[-9:-1], sep="\n")
+def run_benchmark(path_to_model: Path, shape: List[int]) -> float:
+    command = [
+        "benchmark_app",
+        "-m", path_to_model.as_posix(),
+        "-d", "CPU",
+        "-api", "async",
+        "-t", "15",
+        "-shape", str(shape),
+    ]  # fmt: skip
+    cmd_output = subprocess.check_output(command, text=True)
+    print(*cmd_output.splitlines()[-8:], sep="\n")
     match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output))
     return float(match.group(1))
 
@@ -136,9 +141,9 @@ def transform_fn(data_item):
 print(f"[2/7] Save INT8 model: {int8_model_path}")
 
 print("[3/7] Benchmark FP32 model:")
-fp32_fps = run_benchmark(fp32_model_path, shape=[1, 3, 224, 224], verbose=True)
+fp32_fps = run_benchmark(fp32_model_path, shape=[1, 3, 224, 224])
 print("[4/7] Benchmark INT8 model:")
-int8_fps = run_benchmark(int8_model_path, shape=[1, 3, 224, 224], verbose=True)
+int8_fps = run_benchmark(int8_model_path, shape=[1, 3, 224, 224])
 
 print("[5/7] Validate ONNX FP32 model in OpenVINO:")
 fp32_top1 = validate(fp32_model_path, val_loader)

examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/deploy.py

@@ -16,10 +16,10 @@
 
 import openvino as ov
 import torch
-from tqdm import tqdm
+from rich.progress import track
 from ultralytics.cfg import get_cfg
-from ultralytics.engine.validator import BaseValidator as Validator
 from ultralytics.models.yolo import YOLO
+from ultralytics.models.yolo.segment.val import SegmentationValidator
 from ultralytics.utils import DEFAULT_CFG
 from ultralytics.utils.metrics import ConfusionMatrix
 
@@ -37,7 +37,7 @@
 def validate_ov_model(
     ov_model: ov.Model,
     data_loader: torch.utils.data.DataLoader,
-    validator: Validator,
+    validator: SegmentationValidator,
     num_samples: Optional[int] = None,
 ) -> Tuple[Dict, int, int]:
     validator.seen = 0
@@ -47,7 +47,7 @@ def validate_ov_model(
     validator.confusion_matrix = ConfusionMatrix(nc=validator.nc)
     compiled_model = ov.compile_model(ov_model, device_name="CPU")
     num_outputs = len(compiled_model.outputs)
-    for batch_i, batch in enumerate(data_loader):
+    for batch_i, batch in enumerate(track(data_loader, description="Validating")):
         if num_samples is not None and batch_i == num_samples:
             break
         batch = validator.preprocess(batch)
@@ -65,12 +65,17 @@ def validate_ov_model(
     return stats, validator.seen, validator.nt_per_class.sum()
 
 
-def run_benchmark(model_path: str, config) -> float:
-    command = f"benchmark_app -m {model_path} -d CPU -api async -t 30"
-    command += f' -shape "[1,3,{config.imgsz},{config.imgsz}]"'
-    cmd_output = subprocess.check_output(command, shell=True)  # nosec
-
-    match = re.search(r"Throughput\: (.+?) FPS", str(cmd_output))
+def run_benchmark(model_path: Path, config) -> float:
+    command = [
+        "benchmark_app",
+        "-m", model_path.as_posix(),
+        "-d", "CPU",
+        "-api", "async",
+        "-t", "30",
+        "-shape", str([1, 3, config.imgsz, config.imgsz]),
+    ]  # fmt: skip
+    cmd_output = subprocess.check_output(command, text=True)
+    match = re.search(r"Throughput\: (.+?) FPS", cmd_output)
     return float(match.group(1))
 
 
@@ -96,11 +101,11 @@ def run_benchmark(model_path: str, config) -> float:
 validator, data_loader = prepare_validation(YOLO(ROOT / f"{MODEL_NAME}.pt"), args)
 
 print("[5/7] Validate OpenVINO FP32 model:")
-fp32_stats, total_images, total_objects = validate_ov_model(fp32_ov_model, tqdm(data_loader), validator)
+fp32_stats, total_images, total_objects = validate_ov_model(fp32_ov_model, data_loader, validator)
 print_statistics(fp32_stats, total_images, total_objects)
 
 print("[6/7] Validate OpenVINO INT8 model:")
-int8_stats, total_images, total_objects = validate_ov_model(int8_ov_model, tqdm(data_loader), validator)
+int8_stats, total_images, total_objects = validate_ov_model(int8_ov_model, data_loader, validator)
 print_statistics(int8_stats, total_images, total_objects)
 
 print("[7/7] Report:")

examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/main.py

@@ -13,28 +13,32 @@
 from pathlib import Path
 from typing import Any, Dict, Tuple
 
-import numpy as np
 import onnx
 import onnxruntime
 import torch
-from tqdm import tqdm
+from rich.progress import track
 from ultralytics.cfg import get_cfg
 from ultralytics.data.converter import coco80_to_coco91_class
 from ultralytics.data.utils import check_det_dataset
-from ultralytics.engine.validator import BaseValidator as Validator
 from ultralytics.models.yolo import YOLO
+from ultralytics.models.yolo.segment.val import SegmentationValidator
 from ultralytics.utils import DATASETS_DIR
 from ultralytics.utils import DEFAULT_CFG
 from ultralytics.utils import ops
 from ultralytics.utils.metrics import ConfusionMatrix
 
 import nncf
 
+MODEL_NAME = "yolov8n-seg"
+
 ROOT = Path(__file__).parent.resolve()
 
 
 def validate(
-    model: onnx.ModelProto, data_loader: torch.utils.data.DataLoader, validator: Validator, num_samples: int = None
+    model: onnx.ModelProto,
+    data_loader: torch.utils.data.DataLoader,
+    validator: SegmentationValidator,
+    num_samples: int = None,
 ) -> Tuple[Dict, int, int]:
     validator.seen = 0
     validator.jdict = []
@@ -49,7 +53,7 @@ def validate(
     output_names = [output.name for output in session.get_outputs()]
     num_outputs = len(output_names)
 
-    for batch_i, batch in enumerate(data_loader):
+    for batch_i, batch in enumerate(track(data_loader, description="Validating")):
         if num_samples is not None and batch_i == num_samples:
             break
         batch = validator.preprocess(batch)
@@ -71,7 +75,7 @@ def validate(
     return stats, validator.seen, validator.nt_per_class.sum()
 
 
-def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) -> None:
+def print_statistics(stats: Dict[str, float], total_images: int, total_objects: int) -> None:
     print("Metrics(Box):")
     mp, mr, map50, mean_ap = (
         stats["metrics/precision(B)"],
@@ -84,38 +88,35 @@ def print_statistics(stats: np.ndarray, total_images: int, total_objects: int) -
     pf = "%20s" + "%12i" * 2 + "%12.3g" * 4  # print format
     print(pf % ("all", total_images, total_objects, mp, mr, map50, mean_ap))
 
-    # print the mask metrics for segmentation
-    if "metrics/precision(M)" in stats:
-        print("Metrics(Mask):")
-        s_mp, s_mr, s_map50, s_mean_ap = (
-            stats["metrics/precision(M)"],
-            stats["metrics/recall(M)"],
-            stats["metrics/mAP50(M)"],
-            stats["metrics/mAP50-95(M)"],
-        )
-        # Print results
-        s = ("%20s" + "%12s" * 6) % ("Class", "Images", "Labels", "Precision", "Recall", "mAP@.5", "mAP@.5:.95")
-        print(s)
-        pf = "%20s" + "%12i" * 2 + "%12.3g" * 4  # print format
-        print(pf % ("all", total_images, total_objects, s_mp, s_mr, s_map50, s_mean_ap))
-
-
-def prepare_validation(model: YOLO, args: Any) -> Tuple[Validator, torch.utils.data.DataLoader]:
-    validator = model.task_map[model.task]["validator"](args=args)
-    validator.data = check_det_dataset(args.data)
-    validator.stride = 32
+    print("Metrics(Mask):")
+    s_mp, s_mr, s_map50, s_mean_ap = (
+        stats["metrics/precision(M)"],
+        stats["metrics/recall(M)"],
+        stats["metrics/mAP50(M)"],
+        stats["metrics/mAP50-95(M)"],
+    )
+    # Print results
+    s = ("%20s" + "%12s" * 6) % ("Class", "Images", "Labels", "Precision", "Recall", "mAP@.5", "mAP@.5:.95")
+    print(s)
+    pf = "%20s" + "%12i" * 2 + "%12.3g" * 4  # print format
+    print(pf % ("all", total_images, total_objects, s_mp, s_mr, s_map50, s_mean_ap))
 
-    data_loader = validator.get_dataloader(f"{DATASETS_DIR}/coco128-seg", 1)
 
+def prepare_validation(model: YOLO, args: Any) -> Tuple[SegmentationValidator, torch.utils.data.DataLoader]:
+    validator: SegmentationValidator = model.task_map[model.task]["validator"](args=args)
+    validator.data = check_det_dataset(args.data)
+    validator.stride = 32
     validator.is_coco = True
     validator.class_map = coco80_to_coco91_class()
     validator.names = model.model.names
     validator.metrics.names = validator.names
     validator.nc = model.model.model[-1].nc
-    validator.nm = 32
     validator.process = ops.process_mask
     validator.plot_masks = []
 
+    coco_data_path = DATASETS_DIR / "coco128-seg"
+    data_loader = validator.get_dataloader(coco_data_path.as_posix(), 1)
+
     return validator, data_loader
 
 
@@ -129,7 +130,7 @@ def prepare_onnx_model(model: YOLO, model_name: str) -> Tuple[onnx.ModelProto, P
 
 
 def quantize_ac(
-    model: onnx.ModelProto, data_loader: torch.utils.data.DataLoader, validator_ac: Validator
+    model: onnx.ModelProto, data_loader: torch.utils.data.DataLoader, validator_ac: SegmentationValidator
 ) -> onnx.ModelProto:
     input_name = model.graph.input[0].name
 
@@ -140,7 +141,7 @@ def transform_fn(data_item: Dict):
     def validation_ac(
         val_model: onnx.ModelProto,
         validation_loader: torch.utils.data.DataLoader,
-        validator: Validator,
+        validator: SegmentationValidator,
         num_samples: int = None,
     ) -> float:
         validator.seen = 0
@@ -155,7 +156,6 @@ def validation_ac(
         output_names = [output.name for output in session.get_outputs()]
         num_outputs = len(output_names)
 
-        counter = 0
         for batch_i, batch in enumerate(validation_loader):
             if num_samples is not None and batch_i == num_samples:
                 break
@@ -172,13 +172,12 @@ def validation_ac(
                 ]
             preds = validator.postprocess(preds)
             validator.update_metrics(preds, batch)
-            counter += 1
+
         stats = validator.get_stats()
         if num_outputs == 1:
             stats_metrics = stats["metrics/mAP50-95(B)"]
         else:
             stats_metrics = stats["metrics/mAP50-95(M)"]
-        print(f"Validate: dataset length = {counter}, metric value = {stats_metrics:.3f}")
         return stats_metrics, None
 
     quantization_dataset = nncf.Dataset(data_loader, transform_fn)
@@ -213,8 +212,6 @@ def validation_ac(
 
 
 def run_example():
-    MODEL_NAME = "yolov8n-seg"
-
     model = YOLO(ROOT / f"{MODEL_NAME}.pt")
     args = get_cfg(cfg=DEFAULT_CFG)
     args.data = "coco128-seg.yaml"
@@ -231,11 +228,11 @@ def run_example():
     print(f"[2/5] Save INT8 model: {int8_model_path}")
 
     print("[3/5] Validate ONNX FP32 model:")
-    fp_stats, total_images, total_objects = validate(fp32_model, tqdm(data_loader), validator)
+    fp_stats, total_images, total_objects = validate(fp32_model, data_loader, validator)
     print_statistics(fp_stats, total_images, total_objects)
 
     print("[4/5] Validate ONNX INT8 model:")
-    q_stats, total_images, total_objects = validate(int8_model, tqdm(data_loader), validator)
+    q_stats, total_images, total_objects = validate(int8_model, data_loader, validator)
     print_statistics(q_stats, total_images, total_objects)
 
     print("[5/5] Report:")

examples/post_training_quantization/onnx/yolov8_quantize_with_accuracy_control/requirements.txt

@@ -1,3 +1,4 @@
 ultralytics==8.3.22
 onnx==1.17.0
+onnxruntime==1.19.2
 openvino==2024.5