Update CLI argument

Author: echarlaix

optimum/commands/export/openvino.py

@@ -77,7 +77,7 @@ def parse_args_openvino(parser: "ArgumentParser"):
     optional_group.add_argument(
         "--weight-format",
         type=str,
-        choices=["fp32", "fp16", "int8", "int4_sym_g128", "int4_asym_g128", "int4_sym_g64", "int4_asym_g64"],
+        choices=["fp32", "fp16", "int8", "int4", "int4_sym_g128", "int4_asym_g128", "int4_sym_g64", "int4_asym_g64"],
         default=None,
         help=(
             "The weight format of the exporting model, e.g. f32 stands for float32 weights, f16 - for float16 weights, i8 - INT8 weights, int4_* - for INT4 compressed weights."
@@ -86,12 +86,25 @@ def parse_args_openvino(parser: "ArgumentParser"):
     optional_group.add_argument(
         "--ratio",
         type=float,
-        default=0.8,
+        default=None,
         help=(
             "Compression ratio between primary and backup precision. In the case of INT4, NNCF evaluates layer sensitivity and keeps the most impactful layers in INT8"
             "precision (by default 20%% in INT8). This helps to achieve better accuracy after weight compression."
         ),
     )
+    optional_group.add_argument(
+        "--sym",
+        type=bool,
+        default=None,
+        help=("Whether to apply symmetric quantization"),
+    )
+
+    optional_group.add_argument(
+        "--group-size",
+        type=int,
+        default=None,
+        help=("The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization."),
+    )
     optional_group.add_argument(
         "--disable-stateful",
         action="store_true",
@@ -132,6 +145,7 @@ def parse_args(parser: "ArgumentParser"):
 
     def run(self):
         from ...exporters.openvino.__main__ import main_export
+        from ...intel.openvino.configuration import _DEFAULT_4BIT_CONFIGS, OVConfig
 
         if self.args.fp16:
             logger.warning(
@@ -144,6 +158,37 @@ def run(self):
             )
             self.args.weight_format = "int8"
 
+        ov_config = None
+        if self.args.weight_format in {"fp16", "fp32"}:
+            ov_config = OVConfig(dtype=self.args.weight_format)
+        else:
+            is_int8 = self.args.weight_format == "int8"
+
+            # For int4 quantization if not parameter is provided, then use the default config if exist
+            if (
+                not is_int8
+                and self.args.ratio is None
+                and self.args.group_size is None
+                and self.args.sym is None
+                and self.args.model in _DEFAULT_4BIT_CONFIGS
+            ):
+                quantization_config = _DEFAULT_4BIT_CONFIGS[self.args.model]
+            else:
+                quantization_config = {
+                    "bits": 8 if is_int8 else 4,
+                    "ratio": 1 if is_int8 else (self.args.ratio or 0.8),
+                    "sym": self.args.sym or False,
+                    "group_size": -1 if is_int8 else self.args.group_size,
+                }
+
+            if self.args.weight_format in {"int4_sym_g128", "int4_asym_g128", "int4_sym_g64", "int4_asym_g64"}:
+                logger.warning(
+                    f"--weight-format {self.args.weight_format} is deprecated, possible choices are fp32, fp16, int8, int4"
+                )
+                quantization_config["sym"] = "asym" not in self.args.weight_format
+                quantization_config["group_size"] = 128 if "128" in self.args.weight_format else 64
+            ov_config = OVConfig(quantization_config=quantization_config)
+
         # TODO : add input shapes
         main_export(
             model_name_or_path=self.args.model,
@@ -153,8 +198,7 @@ def run(self):
             cache_dir=self.args.cache_dir,
             trust_remote_code=self.args.trust_remote_code,
             pad_token_id=self.args.pad_token_id,
-            compression_option=self.args.weight_format,
-            compression_ratio=self.args.ratio,
+            ov_config=ov_config,
             stateful=not self.args.disable_stateful,
             convert_tokenizer=self.args.convert_tokenizer,
             # **input_shapes,

optimum/exporters/openvino/__main__.py

@@ -14,7 +14,7 @@
 
 import logging
 from pathlib import Path
-from typing import Any, Callable, Dict, Optional, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union
 
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
@@ -41,6 +41,9 @@
     ]
 
 
+if TYPE_CHECKING:
+    from optimum.intel.openvino.configuration import OVConfig
+
 _COMPRESSION_OPTIONS = {
     "int8": {"bits": 8},
     "int4_sym_g128": {"bits": 4, "sym": True, "group_size": 128},

optimum/exporters/openvino/convert.py

@@ -18,7 +18,7 @@
 import logging
 import os
 from pathlib import Path
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, Union
 
 from transformers import T5Tokenizer, T5TokenizerFast
 from transformers.utils import is_tf_available, is_torch_available
@@ -71,6 +71,10 @@
     from transformers.modeling_tf_utils import TFPreTrainedModel
 
 
+if TYPE_CHECKING:
+    from optimum.intel.openvino.configuration import OVConfig
+
+
 def _save_model(model, path: str, ov_config: Optional["OVConfig"] = None):
     compress_to_fp16 = False
     if ov_config is not None:

optimum/intel/openvino/configuration.py

@@ -77,7 +77,7 @@
 }
 
 
-DEFAULT_4BIT_CONFIGS = {
+_DEFAULT_4BIT_CONFIGS = {
     "databricks/dolly-v2-3b": {"bits": 4, "sym": False, "group_size": 32, "ratio": 0.5},
     "EleutherAI/gpt-j-6b": {"bits": 4, "sym": False, "group_size": 64},
     "facebook/opt-6.7b": {"bits": 4, "sym": False, "group_size": 64, "ratio": 0.8},
@@ -241,4 +241,4 @@ def post_init(self):
 
 
 def _check_default_4bit_configs(config: PretrainedConfig):
-    return DEFAULT_4BIT_CONFIGS.get(config.name_or_path, None)
+    return _DEFAULT_4BIT_CONFIGS.get(config.name_or_path, None)

optimum/intel/utils/dummy_openvino_objects.py

@@ -177,7 +177,7 @@ def __init__(self, *args, **kwargs):
 
     @classmethod
     def from_pretrained(cls, *args, **kwargs):
-        requires_backends(self, ["openvino"])
+        requires_backends(cls, ["openvino"])
 
 
 class OVWeightQuantizationConfig(metaclass=DummyObject):
@@ -188,4 +188,4 @@ def __init__(self, *args, **kwargs):
 
     @classmethod
     def from_pretrained(cls, *args, **kwargs):
-        requires_backends(self, ["openvino"])
+        requires_backends(cls, ["openvino"])