Add 'nf4_f8e5m2', 'int4_f8e5m2'; add backup precision

Author: nikita-savelyevv

docs/source/openvino/export.mdx

@@ -32,7 +32,7 @@ Check out the help for more options:
 ```text
 usage: optimum-cli export openvino [-h] -m MODEL [--task TASK] [--framework {pt,tf}] [--trust-remote-code]
                                    [--weight-format {fp32,fp16,int8,int4,mxfp4,nf4}]
-                                   [--quant-mode {int8,f8e4m3,f8e5m2,nf4_f8e4m3,int4_f8e4m3}]
+                                   [--quant-mode {int8,f8e4m3,f8e5m2,nf4_f8e4m3,nf4_f8e5m2,int4_f8e4m3,int4_f8e5m2}]
                                    [--library {transformers,diffusers,timm,sentence_transformers,open_clip}]
                                    [--cache_dir CACHE_DIR] [--pad-token-id PAD_TOKEN_ID] [--ratio RATIO] [--sym]
                                    [--group-size GROUP_SIZE] [--backup-precision {none,int8_sym,int8_asym}]
@@ -68,7 +68,7 @@ Optional arguments:
                         on your local machine arbitrary code present in the model repository.
   --weight-format {fp32,fp16,int8,int4,mxfp4,nf4}
                         The weight format of the exported model.
-  --quant-mode {int8,f8e4m3,f8e5m2,nf4_f8e4m3,int4_f8e4m3}
+  --quant-mode {int8,f8e4m3,f8e5m2,nf4_f8e4m3,nf4_f8e5m2,int4_f8e4m3,int4_f8e5m2}
                         Quantization precision mode. This is used for applying full model quantization including
                         activations.
   --library {transformers,diffusers,timm,sentence_transformers,open_clip}

optimum/commands/export/openvino.py

@@ -78,7 +78,7 @@ def parse_args_openvino(parser: "ArgumentParser"):
     optional_group.add_argument(
         "--quant-mode",
         type=str,
-        choices=["int8", "f8e4m3", "f8e5m2", "nf4_f8e4m3", "int4_f8e4m3"],
+        choices=["int8", "f8e4m3", "f8e5m2", "nf4_f8e4m3", "nf4_f8e5m2", "int4_f8e4m3", "int4_f8e5m2"],
         default=None,
         help=(
             "Quantization precision mode. This is used for applying full model quantization including activations. "
@@ -363,13 +363,13 @@ def run(self):
                         "Dataset is required for full quantization. Please provide it with --dataset argument."
                     )
 
-                if self.args.quant_mode in ["nf4_f8e4m3", "int4_f8e4m3"]:
+                if self.args.quant_mode in ["nf4_f8e4m3", "nf4_f8e5m2", "int4_f8e4m3", "int4_f8e5m2"]:
                     wc_config = prepare_wc_config(self.args, _DEFAULT_4BIT_CONFIG)
-                    weight_dtype_map = {"nf4_f8e4m3": "nf4", "int4_f8e4m3": "int4"}
-                    wc_config["dtype"] = weight_dtype_map[self.args.quant_mode]
+                    wc_dtype, q_dtype = self.args.quant_mode.split("_")
+                    wc_config["dtype"] = wc_dtype
 
                     q_config = prepare_q_config(self.args)
-                    q_config["dtype"] = "f8e4m3"
+                    q_config["dtype"] = q_dtype
 
                     quantization_config = {
                         "weight_quantization_config": wc_config,

optimum/intel/openvino/configuration.py

@@ -483,6 +483,9 @@ def post_init(self):
                 "quantization algorithm is selected and compression ratio is 1.0."
             )
 
+        if self.dtype in ["int4", "int8"]:
+            self.bits = 4 if self.dtype == "int4" else 8
+
         if self.bits not in [4, 8]:
             raise ValueError(f"Only support quantization to [4,8] bits but found {self.bits}")
 
@@ -895,15 +898,24 @@ def __init__(
         """
         if isinstance(weight_quantization_config, dict):
             weight_quantization_config = OVWeightQuantizationConfig.from_dict(weight_quantization_config)
+        else:
+            weight_quantization_config = weight_quantization_config.clone()
         self.weight_quantization_config = weight_quantization_config
+        wqc = self.weight_quantization_config
 
         if isinstance(full_quantization_config, dict):
             full_quantization_config = OVQuantizationConfig.from_dict(full_quantization_config)
+        else:
+            full_quantization_config = full_quantization_config.clone()
         self.full_quantization_config = full_quantization_config
+        fqc = self.full_quantization_config
+
+        if fqc.dtype in ["f8e4m3", "f8e5m2"] and wqc.backup_precision is None:
+            # TODO: remove once there is support for FP8 weight compression in NNCF
+            wqc.backup_precision = "none"
 
         # Pull dataset-related parameters from child configs. This is not the intended use case, but we process it just
         # in case user sets those parameters inside child configs only.
-        wqc, fqc = self.weight_quantization_config, self.full_quantization_config
         num_samples = max((num_samples or 0, wqc.num_samples or 0, fqc.num_samples or 0)) or None
         dataset = dataset or wqc.dataset or fqc.dataset
         tokenizer = tokenizer or wqc.tokenizer or fqc.tokenizer

tests/openvino/test_exporters_cli.py

@@ -149,36 +149,48 @@ class OVCLIExportTestCase(unittest.TestCase):
             "text-generation",
             "llama",
             "nf4_f8e4m3",
-            "--dataset wikitext2 --num-samples 1 --smooth-quant-alpha 0.9 --group-size 16 --trust-remote-code",
+            "--dataset wikitext2 --num-samples 1 --group-size 16 --trust-remote-code --ratio 0.5",
             [
-                13,
+                14,
             ],
             [
-                {"int8": 4, "nf4": 14},
+                {"f8e4m3": 11, "nf4": 5},
             ],
         ),
         (
             "text-generation",
             "llama",
-            "int4_f8e4m3",
-            "--dataset wikitext2 --num-samples 1 --smooth-quant-alpha 0.9 --group-size 16 --trust-remote-code",
+            "nf4_f8e5m2",
+            "--dataset wikitext2 --num-samples 1 --group-size 16 --trust-remote-code --sym --ratio 0.5",
             [
-                13,
+                14,
             ],
             [
-                {"int8": 4, "int4": 28},
+                {"f8e5m2": 11, "nf4": 5},
             ],
         ),
         (
             "text-generation",
             "llama",
             "int4_f8e4m3",
-            "--dataset wikitext2 --num-samples 1 --smooth-quant-alpha 0.9 --group-size 16 --trust-remote-code --sym",
+            "--dataset wikitext2 --num-samples 1 --group-size 16 --trust-remote-code --sym --ratio 0.5",
+            [
+                14,
+            ],
+            [
+                {"f8e4m3": 11, "int4": 5},
+            ],
+        ),
+        (
+            "text-generation",
+            "llama",
+            "int4_f8e5m2",
+            "--dataset wikitext2 --num-samples 1 --group-size 16 --trust-remote-code",
             [
                 13,
             ],
             [
-                {"int8": 4, "int4": 14},
+                {"f8e5m2": 2, "int4": 28},
             ],
         ),
     ]

tests/openvino/test_quantization.py

@@ -138,16 +138,16 @@ class OVQuantizerTest(unittest.TestCase):
             OVModelForCausalLM,
             "llama",
             dict(
-                weight_quantization_config=dict(bits=4, dtype="nf4", group_size=16, weight_only=True),
+                weight_quantization_config=dict(bits=4, dtype="nf4", group_size=16, weight_only=True, ratio=0.5),
                 full_quantization_config=dict(dtype="f8e4m3", weight_only=False),
                 dataset="wikitext2",
                 num_samples=1,
             ),
             [
-                13,
+                14,
             ],
             [
-                {"int8": 4, "nf4": 14},
+                {"f8e4m3": 11, "nf4": 5},
             ],
         ),
         (
@@ -158,6 +158,7 @@ class OVQuantizerTest(unittest.TestCase):
                     bits=4,
                     dtype="nf4",
                     group_size=16,
+                    ratio=0.5,
                     ignored_scope={"patterns": ["^__module.model.layers.0.self_attn"]},
                 ),
                 full_quantization_config=OVQuantizationConfig(
@@ -171,23 +172,64 @@ class OVQuantizerTest(unittest.TestCase):
                 7,
             ],
             [
-                {"int8": 4, "f8e4m3": 4, "nf4": 6},
+                {"f8e4m3": 8, "nf4": 2},
             ],
         ),
         (
             OVModelForCausalLM,
             "llama",
             OVMixedQuantizationConfig(
-                weight_quantization_config=OVWeightQuantizationConfig(bits=4, group_size=16),
+                weight_quantization_config=OVWeightQuantizationConfig(
+                    bits=4,
+                    dtype="nf4",
+                    group_size=16,
+                    ratio=0.5,
+                    ignored_scope={"patterns": ["^__module.model.layers.0.self_attn"]},
+                ),
+                full_quantization_config=OVQuantizationConfig(
+                    dtype="f8e5m2", ignored_scope={"patterns": ["^__module.model.layers.0.mlp"]}
+                ),
+                ignored_scope={"patterns": ["^__module.model.layers.1.self_attn"]},
+                dataset="wikitext2",
+                num_samples=1,
+            ),
+            [
+                7,
+            ],
+            [
+                {"f8e5m2": 8, "nf4": 2},
+            ],
+        ),
+        (
+            OVModelForCausalLM,
+            "llama",
+            OVMixedQuantizationConfig(
+                weight_quantization_config=OVWeightQuantizationConfig(bits=4, group_size=16, ratio=0.5),
                 full_quantization_config=OVQuantizationConfig(dtype="f8e4m3"),
                 dataset="wikitext2",
                 num_samples=1,
             ),
+            [
+                14,
+            ],
+            [
+                {"f8e4m3": 11, "int4": 10},
+            ],
+        ),
+        (
+            OVModelForCausalLM,
+            "llama",
+            OVMixedQuantizationConfig(
+                weight_quantization_config=OVWeightQuantizationConfig(bits=4, group_size=16),
+                full_quantization_config=OVQuantizationConfig(dtype="f8e5m2"),
+                dataset="wikitext2",
+                num_samples=1,
+            ),
             [
                 13,
             ],
             [
-                {"int8": 4, "int4": 28},
+                {"f8e5m2": 2, "int4": 28},
             ],
         ),
     ]