Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert Tokenizers By Default #580

Merged
merged 26 commits into from
Apr 18, 2024
Merged
Changes from 1 commit
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
6d72fe0
Convert Tokenizers By Default
apaniukov Feb 29, 2024
ffcae81
Add Warning to Deprecated Option
apaniukov Feb 29, 2024
eadb210
Update OV Tokenizers Availability Check
apaniukov Mar 8, 2024
8a22002
Move openvino-tokenizers to openvino dependencies
apaniukov Mar 13, 2024
40f227d
Make Style
apaniukov Mar 13, 2024
d9e3a0f
Merge branch 'main' into del-convert-tokenizer-flag
apaniukov Mar 13, 2024
52e24b5
Change Imports to Absolute
apaniukov Mar 13, 2024
fd1887f
Check openvino-nightly compatibility
apaniukov Mar 14, 2024
92d42f4
Change model skip explanation
apaniukov Mar 14, 2024
feb12dd
Merge branch 'main' into del-convert-tokenizer-flag
apaniukov Mar 14, 2024
86a1eca
Update OV Tokenizers Availability Check
apaniukov Mar 22, 2024
9041fe5
Add Check for OpenVINO Nightly and Archive
apaniukov Mar 25, 2024
277225c
Merge branch 'main' into del-convert-tokenizer-flag
apaniukov Mar 28, 2024
3dcaf9e
Add linux distros compatibility message
apaniukov Mar 28, 2024
241d265
Address Review Comments
apaniukov Mar 28, 2024
4d3df41
Address Review Comments
apaniukov Mar 28, 2024
efd6638
Merge branch 'main' into del-convert-tokenizer-flag
apaniukov Apr 3, 2024
eb05594
Address Review Comments
apaniukov Apr 4, 2024
4ed432d
Fix Style
apaniukov Apr 4, 2024
3710884
Change Warnings to Debug Level
apaniukov Apr 17, 2024
a5ef7b1
Merge branch 'main' into del-convert-tokenizer-flag
apaniukov Apr 17, 2024
cb2b26f
Merge branch 'main' into del-convert-tokenizer-flag
apaniukov Apr 17, 2024
80d4c1d
Fix Tests Debug Message
apaniukov Apr 18, 2024
85c925a
Merge branch 'main' into del-convert-tokenizer-flag
echarlaix Apr 18, 2024
f4b3301
Fix Style
apaniukov Apr 18, 2024
934ea22
Fix Style
apaniukov Apr 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Merge branch 'main' into del-convert-tokenizer-flag
echarlaix authored Apr 18, 2024
commit 85c925a93c73c5564eb57aa7e0ec77c7c1cad52e
76 changes: 58 additions & 18 deletions optimum/commands/export/openvino.py
Original file line number Diff line number Diff line change
@@ -214,21 +214,61 @@ def run(self):
quantization_config["dataset"] = self.args.dataset
ov_config = OVConfig(quantization_config=quantization_config)

if self.args.convert_tokenizer:
logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")

# TODO : add input shapes
main_export(
model_name_or_path=self.args.model,
output=self.args.output,
task=self.args.task,
framework=self.args.framework,
cache_dir=self.args.cache_dir,
trust_remote_code=self.args.trust_remote_code,
pad_token_id=self.args.pad_token_id,
ov_config=ov_config,
stateful=not self.args.disable_stateful,
convert_tokenizer=not self.args.disable_convert_tokenizer,
library_name=self.args.library
# **input_shapes,
)
library_name = TasksManager.infer_library_from_model(self.args.model)

if (
library_name == "diffusers"
and ov_config
and ov_config.quantization_config
and ov_config.quantization_config.dataset is not None
):
if not is_diffusers_available():
raise ValueError(DIFFUSERS_IMPORT_ERROR.format("Export of diffusers models"))

from diffusers import DiffusionPipeline

diffusers_config = DiffusionPipeline.load_config(self.args.model)
class_name = diffusers_config.get("_class_name", None)

if class_name == "LatentConsistencyModelPipeline":
from optimum.intel import OVLatentConsistencyModelPipeline

model_cls = OVLatentConsistencyModelPipeline

elif class_name == "StableDiffusionXLPipeline":
from optimum.intel import OVStableDiffusionXLPipeline

model_cls = OVStableDiffusionXLPipeline
elif class_name == "StableDiffusionPipeline":
from optimum.intel import OVStableDiffusionPipeline

model_cls = OVStableDiffusionPipeline
else:
raise NotImplementedError(f"Quantization in hybrid mode isn't supported for class {class_name}.")

model = model_cls.from_pretrained(
self.args.model, export=True, quantization_config=ov_config.quantization_config
)
model.save_pretrained(self.args.output)

else:
if self.args.convert_tokenizer:
logger.warning("`--convert-tokenizer` option is deprecated. Tokenizer will be converted by default.")

# TODO : add input shapes
main_export(
model_name_or_path=self.args.model,
output=self.args.output,
task=self.args.task,
framework=self.args.framework,
cache_dir=self.args.cache_dir,
trust_remote_code=self.args.trust_remote_code,
pad_token_id=self.args.pad_token_id,
ov_config=ov_config,
stateful=not self.args.disable_stateful,
convert_tokenizer=not self.args.disable_convert_tokenizer,
library_name=library_name,
# **input_shapes,
)


You are viewing a condensed version of this merge commit. You can view the full changes here.