|
22 | 22 |
|
23 | 23 | from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
|
24 | 24 | from requests.exceptions import ConnectionError as RequestsConnectionError
|
25 |
| -from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase |
| 25 | +from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase, ProcessorMixin |
26 | 26 | from transformers.utils import is_torch_available
|
27 | 27 |
|
28 | 28 | from openvino.runtime import Core, Type, save_model
|
@@ -531,10 +531,15 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
|
531 | 531 |
|
532 | 532 | if is_openvino_tokenizers_available():
|
533 | 533 | if library_name != "diffusers" and preprocessors:
|
| 534 | + processor_chat_template = None |
534 | 535 | tokenizer = next(filter(lambda it: isinstance(it, PreTrainedTokenizerBase), preprocessors), None)
|
| 536 | + if len(preprocessors) > 1: |
| 537 | + for processor in preprocessors: |
| 538 | + if isinstance(processor, ProcessorMixin) and hasattr(processor, "chat_template"): |
| 539 | + processor_chat_template = processor.chat_template |
535 | 540 | if tokenizer:
|
536 | 541 | try:
|
537 |
| - export_tokenizer(tokenizer, output, task=task) |
| 542 | + export_tokenizer(tokenizer, output, task=task, processor_chat_template=processor_chat_template) |
538 | 543 | except Exception as exception:
|
539 | 544 | logger.warning(
|
540 | 545 | "Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer "
|
|
0 commit comments