Move speculative decoding from streamer to metrics benchmarking approach (openvinotoolkit#1904)

Co-authored-by: Ilya Lavrenov <ilya.lavrenov@intel.com>
Co-authored-by: Ekaterina Aidova <ekaterina.aidova@intel.com>

Author: 3 people

tools/llm_bench/llm_bench_utils/model_utils.py

@@ -320,3 +320,12 @@ def resolve_media_file_path(file_path, prompt_file_path):
     if not (file_path.startswith("http://") or file_path.startswith("https://")):
         return os.path.join(os.path.dirname(prompt_file_path), file_path.replace("./", ""))
     return file_path
+
+
+def get_version_in_format_to_pars(version):
+    processed_version = version
+    if "-" in processed_version:
+        ov_major_version, dev_info = version.split("-", 1)
+        commit_id = dev_info.split("-")[0]
+        processed_version = f"{ov_major_version}-{commit_id}"
+    return processed_version

tools/llm_bench/llm_bench_utils/ov_utils.py

@@ -11,6 +11,7 @@
 import types
 from llm_bench_utils.hook_common import get_bench_hook
 from llm_bench_utils.hook_forward import MeanStdPair, RawImGenPerfMetrics
+from llm_bench_utils.model_utils import get_version_in_format_to_pars
 from llm_bench_utils.config_class import (
     OV_MODEL_CLASSES_MAPPING,
     TOKENIZE_CLASSES_MAPPING,
@@ -175,6 +176,7 @@ def get_scheduler_config_genai(user_config, config_name="CB config"):
 def create_genai_text_gen_model(model_path, device, ov_config, **kwargs):
     import openvino_genai
     from transformers import AutoTokenizer
+    from packaging.version import parse
 
     if not (model_path / "openvino_tokenizer.xml").exists() or not (model_path / "openvino_detokenizer.xml").exists():
         raise ValueError("OpenVINO Tokenizer model is not found in model directory. Please convert tokenizer using following command:\n"
@@ -190,7 +192,8 @@ def create_genai_text_gen_model(model_path, device, ov_config, **kwargs):
         log.info("Continuous Batching mode activated")
         ov_config["scheduler_config"] = get_scheduler_config_genai(cb_config)
 
-        use_streamer_metrics = not openvino_genai.get_version().startswith("2025.") or draft_model_path
+        version = get_version_in_format_to_pars(openvino_genai.get_version())
+        use_streamer_metrics = parse(version) < parse("2025.0.0") or (draft_model_path and parse(version) < parse("2025.1.0"))
 
     if draft_model_path:
         if not Path(draft_model_path).exists():

tools/llm_bench/task/image_generation.py

@@ -56,12 +56,7 @@ def collects_input_args(image_param, model_name, infer_count=None, height=None,
         from openvino import get_version
         from packaging.version import parse
 
-        version = get_version()
-        # avoid invalid format
-        if "-" in version:
-            ov_major_version, dev_info = version.split("-", 1)
-            commit_id = dev_info.split("-")[0]
-            version = f"{ov_major_version}-{commit_id}"
+        version = model_utils.get_version_in_format_to_pars(get_version())
         is_callback_supported = parse(version) >= parse("2025.0.0")
         if is_callback_supported:
             input_args["callback"] = callback