Add OV Tokenizers Version Check

Author: apaniukov

optimum/exporters/openvino/convert.py

@@ -40,6 +40,7 @@
     _timm_version,
     _torch_version,
     _transformers_version,
+    compare_versions,
 )
 from optimum.utils import DEFAULT_DUMMY_SHAPES, is_diffusers_available
 from optimum.utils.save_utils import maybe_save_preprocessors
@@ -723,8 +724,12 @@ def export_tokenizer(
     if output.exists():
         tokenizer = maybe_convert_tokenizer_to_fast(tokenizer, output)
 
-    if task is not None and task.startswith("text-generation"):
-        logger.info(f"Set padding side to left for `{task}` task.")
+    if (
+        task is not None
+        and task.startswith("text-generation")
+        and compare_versions("openvino-tokenizers", ">=", "2024.3.0.0")
+    ):
+        logger.info(f"Set tokenizer padding side to left for `{task}` task.")
         tokenizer.padding_side = "left"
         tokenizer.truncation_side = "left"
 

tests/openvino/test_exporters_cli.py

@@ -41,7 +41,11 @@
 )
 from optimum.intel.openvino.configuration import _DEFAULT_4BIT_CONFIGS
 from optimum.intel.openvino.utils import _HEAD_TO_AUTOMODELS
-from optimum.intel.utils.import_utils import is_openvino_tokenizers_available, is_transformers_version
+from optimum.intel.utils.import_utils import (
+    compare_versions,
+    is_openvino_tokenizers_available,
+    is_transformers_version,
+)
 
 
 class OVCLIExportTestCase(unittest.TestCase):
@@ -171,8 +175,8 @@ def test_exporters_cli_tokenizers(self, task: str, model_type: str):
             if number_of_tokenizers == 1:
                 self.assertTrue("Detokenizer is not supported, convert tokenizer only." in output, output)
 
-            if task.startswith("text-generation"):
-                self.assertFalse("Set padding side to left" in output, output)
+            if task.startswith("text-generation") and compare_versions("openvino-tokenizers", ">=", "2024.3.0.0"):
+                self.assertTrue("Set tokenizer padding side to left" in output, output)
 
     @parameterized.expand(SUPPORTED_ARCHITECTURES)
     def test_exporters_cli_fp16(self, task: str, model_type: str):