warning

Author: IlyasMoutawwakil

optimum/exporters/openvino/convert.py

@@ -41,6 +41,7 @@
     _torch_version,
     _transformers_version,
     compare_versions,
+    is_tokenizers_version,
     is_transformers_version,
 )
 from optimum.utils import DEFAULT_DUMMY_SHAPES, is_diffusers_available
@@ -730,6 +731,12 @@ def export_tokenizer(
     except ModuleNotFoundError:
         return
 
+    if is_tokenizers_version(">", "0.19"):
+        logger.warning(
+            "Exporting tokenizers to OpenVINO is not supported for tokenizers version > 0.19. "
+            "Please downgrade to tokenizers version <= 0.19 to export tokenizers to OpenVINO."
+        )
+
     if not isinstance(output, Path):
         output = Path(output)