@@ -201,7 +201,7 @@ def from_pretrained(cls, model: PreTrainedModel, **kwargs):
201
201
def quantize (
202
202
self ,
203
203
calibration_dataset : Optional [Union [datasets .Dataset , nncf .Dataset , Iterable ]] = None ,
204
- save_directory : Union [str , Path ] = None ,
204
+ save_directory : Optional [ Union [str , Path ] ] = None ,
205
205
ov_config : OVConfig = None ,
206
206
file_name : Optional [str ] = None ,
207
207
batch_size : int = 1 ,
@@ -217,7 +217,7 @@ def quantize(
217
217
calibration_dataset (`datasets.Dataset` or `nncf.Dataset` or `Iterable`, *optional*):
218
218
A collection of data samples to use for quantization calibration. Is optional for weight-only
219
219
quantization and is required for full quantization.
220
- save_directory (`Union[str, Path]`):
220
+ save_directory (`Union[str, Path]`, *optional* ):
221
221
The directory where the quantized model should be saved.
222
222
ov_config (`OVConfig`, *optional*):
223
223
The configuration containing the parameters related to quantization. If not provided, 8-bit symmetric
@@ -265,10 +265,6 @@ def quantize(
265
265
"as an instance of `OVWeightQuantizationConfig` for weight-only compression or as an instance of `OVQuantizationConfig` for full model quantization."
266
266
)
267
267
268
- if save_directory is None :
269
- # TODO : can be set to self.model.config.name_or_path for OVModels when not provided
270
- raise ValueError ("`save_directory` needs to be specified" )
271
-
272
268
if ov_config is None :
273
269
ov_config = OVConfig ()
274
270
if not isinstance (ov_config , OVConfig ):
@@ -321,21 +317,41 @@ def quantize(
321
317
def _quantize_ovbasemodel (
322
318
self ,
323
319
ov_config : OVConfig ,
324
- save_directory : Union [str , Path ],
320
+ save_directory : Union [str , Path ] = None ,
325
321
calibration_dataset : Optional [Union [datasets .Dataset , nncf .Dataset , Iterable ]] = None ,
326
322
batch_size : int = 1 ,
327
323
data_collator : Optional [DataCollator ] = None ,
328
324
remove_unused_columns : bool = True ,
329
325
** kwargs ,
330
326
):
331
- save_directory = Path (save_directory )
332
- save_directory .mkdir (parents = True , exist_ok = True )
327
+ if save_directory is not None :
328
+ save_directory = Path (save_directory )
329
+ save_directory .mkdir (parents = True , exist_ok = True )
333
330
334
331
quantization_config = ov_config .quantization_config
335
332
if isinstance (quantization_config , OVWeightQuantizationConfig ):
333
+ if calibration_dataset is None and isinstance (quantization_config .dataset , str ):
334
+ from optimum .intel import OVModelForCausalLM
335
+
336
+ if isinstance (self .model , OVModelForCausalLM ):
337
+ from optimum .gptq .data import get_dataset , prepare_dataset
338
+
339
+ tokenizer = AutoTokenizer .from_pretrained (quantization_config .tokenizer )
340
+ nsamples = quantization_config .num_samples if quantization_config .num_samples else 128
341
+ calibration_dataset = get_dataset (
342
+ quantization_config .dataset , tokenizer , seqlen = 32 , nsamples = nsamples
343
+ )
344
+ calibration_dataset = prepare_dataset (calibration_dataset )
345
+ calibration_dataset = nncf .Dataset (calibration_dataset , lambda x : self .model .prepare_inputs (** x ))
346
+ else :
347
+ raise ValueError (
348
+ f"Can't create weight compression calibration dataset from string for { type (self .model )} "
349
+ )
350
+
336
351
_weight_only_quantization (self .model .model , quantization_config , calibration_dataset )
337
- self .model .save_pretrained (save_directory )
338
- ov_config .save_pretrained (save_directory )
352
+ if save_directory is not None :
353
+ self .model .save_pretrained (save_directory )
354
+ ov_config .save_pretrained (save_directory )
339
355
return
340
356
if not isinstance (quantization_config , OVQuantizationConfig ):
341
357
raise ValueError (f"Unsupported type of quantization config: { type (quantization_config )} " )
@@ -387,8 +403,9 @@ def _quantize_ovbasemodel(
387
403
** kwargs ,
388
404
)
389
405
self .model .model = quantized_model
390
- self .model .save_pretrained (save_directory )
391
- ov_config .save_pretrained (save_directory )
406
+ if save_directory is not None :
407
+ self .model .save_pretrained (save_directory )
408
+ ov_config .save_pretrained (save_directory )
392
409
393
410
def _quantize_torchmodel (
394
411
self ,
@@ -401,6 +418,10 @@ def _quantize_torchmodel(
401
418
remove_unused_columns : bool = True ,
402
419
** kwargs ,
403
420
):
421
+ if save_directory is None :
422
+ # TODO : can be set to self.model.config.name_or_path for OVModels when not provided
423
+ raise ValueError ("`save_directory` needs to be specified" )
424
+
404
425
self ._set_task ()
405
426
save_directory = Path (save_directory )
406
427
save_directory .mkdir (parents = True , exist_ok = True )
@@ -660,14 +681,6 @@ def _weight_only_quantization(
660
681
dataset = calibration_dataset
661
682
else :
662
683
dataset = nncf .Dataset (calibration_dataset )
663
- elif config .dataset is not None and isinstance (config .dataset , str ):
664
- tokenizer = AutoTokenizer .from_pretrained (config .tokenizer )
665
-
666
- from optimum .gptq .data import get_dataset , prepare_dataset
667
-
668
- nsamples = config .num_samples if config .num_samples else 128
669
- dataset = get_dataset (config .dataset , tokenizer , seqlen = 32 , nsamples = nsamples )
670
- dataset = prepare_dataset (dataset )
671
684
672
685
sensitivity_metric = None
673
686
if isinstance (config .sensitivity_metric , str ):
0 commit comments