Skip to content

Commit 7cd9492

Browse files
committed
improve template selection logic and add tests
1 parent 6cc2de5 commit 7cd9492

File tree

4 files changed

+138
-36
lines changed

4 files changed

+138
-36
lines changed

optimum/exporters/openvino/__main__.py

+5-5
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222

2323
from huggingface_hub.constants import HUGGINGFACE_HUB_CACHE
2424
from requests.exceptions import ConnectionError as RequestsConnectionError
25-
from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
25+
from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase, ProcessorMixin
2626
from transformers.utils import is_torch_available
2727

2828
from openvino.runtime import Core, Type, save_model
@@ -531,15 +531,15 @@ def maybe_convert_tokenizers(library_name: str, output: Path, model=None, prepro
531531

532532
if is_openvino_tokenizers_available():
533533
if library_name != "diffusers" and preprocessors:
534-
additional_chat_templates = []
534+
processor_chat_template = None
535535
tokenizer = next(filter(lambda it: isinstance(it, PreTrainedTokenizerBase), preprocessors), None)
536536
if len(preprocessors) > 1:
537537
for processor in preprocessors:
538-
if processor != tokenizer and hasattr(processor, "chat_template"):
539-
additional_chat_templates.append(processor.chat_template)
538+
if isinstance(processor, ProcessorMixin) and hasattr(processor, "chat_template"):
539+
processor_chat_template = processor.chat_template
540540
if tokenizer:
541541
try:
542-
export_tokenizer(tokenizer, output, task=task, additional_chat_templates=additional_chat_templates)
542+
export_tokenizer(tokenizer, output, task=task, processor_chat_template=processor_chat_template)
543543
except Exception as exception:
544544
logger.warning(
545545
"Could not load tokenizer using specified model ID or path. OpenVINO tokenizer/detokenizer "

optimum/exporters/openvino/convert.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@
7171
remove_none_from_dummy_inputs,
7272
save_config,
7373
save_preprocessors,
74-
set_simplified_chat_template
74+
set_simplified_chat_template,
7575
)
7676

7777

@@ -826,7 +826,7 @@ def export_tokenizer(
826826
output: Union[str, Path],
827827
suffix: Optional[str] = "",
828828
task: Optional[str] = None,
829-
additional_chat_templates: Optional[List[str]] = None
829+
processor_chat_template: Optional[str] = None,
830830
):
831831
# avoid circular imports
832832
from optimum.intel.openvino import OV_DETOKENIZER_NAME, OV_TOKENIZER_NAME
@@ -860,7 +860,7 @@ def export_tokenizer(
860860

861861
try:
862862
converted = convert_tokenizer(tokenizer, with_detokenizer=True)
863-
set_simplified_chat_template(converted[0], additional_chat_templates)
863+
set_simplified_chat_template(converted[0], processor_chat_template)
864864

865865
except NotImplementedError:
866866
logger.info("Detokenizer is not supported, convert tokenizer only.")

0 commit comments

Comments
 (0)