From 8821a044c3679cf057320449b82942e817cd0cd2 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Wed, 5 Feb 2025 19:34:29 +0100 Subject: [PATCH 1/6] add deepcopy_data --- optimum/intel/openvino/quantization.py | 5 +++-- optimum/intel/openvino/utils.py | 24 +++++++++++++++++--- tests/openvino/test_quantization.py | 31 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 5 deletions(-) diff --git a/optimum/intel/openvino/quantization.py b/optimum/intel/openvino/quantization.py index 2ba74244d8..c65726b521 100644 --- a/optimum/intel/openvino/quantization.py +++ b/optimum/intel/openvino/quantization.py @@ -74,6 +74,7 @@ PREDEFINED_SD_DATASETS, PREDEFINED_SPEECH_TO_TEXT_DATASETS, PREDEFINED_VISUAL_LM_DATASETS, + deepcopy_data, ) @@ -131,7 +132,7 @@ def __init__( def collect_inputs(self, inputs): if not self.apply_caching or not isinstance(inputs, dict): - self.collected_inputs.append(copy.deepcopy(inputs)) + self.collected_inputs.append(deepcopy_data(inputs)) return copied_inputs = {} @@ -146,7 +147,7 @@ def collect_inputs(self, inputs): # Avoid data copying if tensor contains data encountered earlier self.tensor_cache.setdefault(k, {}) if data_hash not in self.tensor_cache[k]: - self.tensor_cache[k][data_hash] = copy.deepcopy(v) + self.tensor_cache[k][data_hash] = deepcopy_data(v) copied_inputs[k] = self.tensor_cache[k][data_hash] self.collected_inputs.append(copied_inputs) diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 5e6c4af0ae..f4e7c74d7e 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -19,21 +19,22 @@ import stat import warnings import weakref +from copy import deepcopy from glob import glob from pathlib import Path from tempfile import TemporaryDirectory as OrigTemporaryDirectory from tempfile import mkdtemp -from typing import Tuple, Type, Union +from typing import Tuple, Type, Union, Any import numpy as np import torch from huggingface_hub import model_info -from openvino.runtime import Core, Model, properties +from openvino.runtime import Core, Model, properties, Tensor from openvino.runtime import Type as OVType from packaging.version import Version from transformers import AutoTokenizer, CLIPTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers.onnx.utils import ParameterFormat, compute_serialized_parameters_size - +import openvino from optimum.intel.utils.import_utils import is_torch_version @@ -586,3 +587,20 @@ def check_scale_available(model: Union[Model, str, Path]): if runtime_options is None: return False return runtime_options.find("ACTIVATIONS_SCALE_FACTOR") is not None + + +def deepcopy_data(inputs: Any) -> Any: + if isinstance(inputs, dict): + new_inputs = {} + for k, v in inputs.items(): + new_inputs[deepcopy_data(k)] = deepcopy_data(v) + elif isinstance(inputs, list): + new_inputs = [deepcopy_data(elem) for elem in inputs] + elif isinstance(inputs, tuple): + new_inputs = tuple(deepcopy_data(elem) for elem in inputs) + elif isinstance(inputs, openvino.Tensor): + new_inputs = openvino.Tensor(np.zeros(inputs.shape, dtype=inputs.element_type.to_dtype())) + new_inputs.copy_from(inputs) + else: + new_inputs = deepcopy(inputs) + return new_inputs diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 56b2a77e9a..2b3ef0b45a 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -42,6 +42,9 @@ from transformers.testing_utils import slow from transformers.utils.quantization_config import QuantizationMethod +from optimum.intel.openvino.utils import deepcopy_data + + from optimum.intel import ( OVConfig, OVFluxPipeline, @@ -1354,6 +1357,34 @@ def test_calibration_data_uniqueness(self, model_name, apply_caching): # Without caching, encoder hidden states tensors will be unique for each collected input self.assertGreater(len(data_id_per_key["encoder_hidden_states"]), 2) + def test_deepcopy_data(self): + data = { + "a": torch.tensor([1, 2, 3]), + "b": np.array([1, 2, 3]), + "c": 1, + "d": "string", + "e": {"a": torch.tensor([1, 2, 3]), "b": np.array([1, 2, 3])}, + "f": [ov.Tensor(np.ones((1, 2, 3))), ov.Tensor(np.ones((1, 2, 3)))], + } + copied_data = deepcopy_data(data) + assert copied_data["a"] is not data["a"] + assert copied_data["b"] is not data["b"] + assert copied_data["e"]["a"] is not data["e"]["a"] + assert copied_data["e"]["b"] is not data["e"]["b"] + assert copied_data["f"][0] is not data["f"][0] + assert copied_data["f"][1] is not data["f"][1] + + assert torch.equal(copied_data["a"], data["a"]) + assert np.array_equal(copied_data["b"], data["b"]) + assert copied_data["c"] == data["c"] + assert copied_data["d"] == data["d"] + assert torch.equal(copied_data["e"]["a"], data["e"]["a"]) + assert np.array_equal(copied_data["e"]["b"], data["e"]["b"]) + assert np.array_equal(copied_data["f"][0].data, data["f"][0].data) + assert np.array_equal(copied_data["f"][1].data, data["f"][1].data) + + assert copied_data is not data + def check_optimization_not_applicable_to_optimized_model(model, quantization_config): quantizer = OVQuantizer(model) From 4a4420a96c7166276844e4224a615b307ec339fc Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Thu, 6 Feb 2025 10:13:41 +0100 Subject: [PATCH 2/6] add type param for ov.Tensor --- optimum/intel/openvino/utils.py | 10 +++++----- tests/openvino/test_quantization.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index f4e7c74d7e..54c4183f2f 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -24,17 +24,17 @@ from pathlib import Path from tempfile import TemporaryDirectory as OrigTemporaryDirectory from tempfile import mkdtemp -from typing import Tuple, Type, Union, Any +from typing import Any, Tuple, Type, Union import numpy as np import torch from huggingface_hub import model_info -from openvino.runtime import Core, Model, properties, Tensor +from openvino.runtime import Core, Model, Tensor, properties from openvino.runtime import Type as OVType from packaging.version import Version from transformers import AutoTokenizer, CLIPTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from transformers.onnx.utils import ParameterFormat, compute_serialized_parameters_size -import openvino + from optimum.intel.utils.import_utils import is_torch_version @@ -598,8 +598,8 @@ def deepcopy_data(inputs: Any) -> Any: new_inputs = [deepcopy_data(elem) for elem in inputs] elif isinstance(inputs, tuple): new_inputs = tuple(deepcopy_data(elem) for elem in inputs) - elif isinstance(inputs, openvino.Tensor): - new_inputs = openvino.Tensor(np.zeros(inputs.shape, dtype=inputs.element_type.to_dtype())) + elif isinstance(inputs, Tensor): + new_inputs = Tensor(np.empty(inputs.shape), inputs.shape, inputs.get_element_type()) new_inputs.copy_from(inputs) else: new_inputs = deepcopy(inputs) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 2b3ef0b45a..a59f0bcfcb 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -1364,7 +1364,7 @@ def test_deepcopy_data(self): "c": 1, "d": "string", "e": {"a": torch.tensor([1, 2, 3]), "b": np.array([1, 2, 3])}, - "f": [ov.Tensor(np.ones((1, 2, 3))), ov.Tensor(np.ones((1, 2, 3)))], + "f": [ov.Tensor(np.ones((1, 2, 3)), (1, 2, 3), ov.Type.i4), ov.Tensor(np.ones((1, 2, 3)))], } copied_data = deepcopy_data(data) assert copied_data["a"] is not data["a"] From e4d910b646a756b5fa4cfa2aa58f161171dbe23d Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Thu, 6 Feb 2025 10:15:10 +0100 Subject: [PATCH 3/6] style --- tests/openvino/test_quantization.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index a59f0bcfcb..d8849f929e 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -42,8 +42,6 @@ from transformers.testing_utils import slow from transformers.utils.quantization_config import QuantizationMethod -from optimum.intel.openvino.utils import deepcopy_data - from optimum.intel import ( OVConfig, @@ -77,7 +75,7 @@ _DEFAULT_4BIT_CONFIGS, _DEFAULT_4BIT_CONFIG, ) -from optimum.intel.openvino.utils import TemporaryDirectory +from optimum.intel.openvino.utils import TemporaryDirectory, deepcopy_data from copy import deepcopy from optimum.intel.openvino.quantization import InferRequestWrapper From 5211b77606d05e2769099736cab83f2cba0ab419 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Thu, 6 Feb 2025 10:21:17 +0100 Subject: [PATCH 4/6] assert -> assertTrue --- tests/openvino/test_quantization.py | 40 +++++++++++++++++------------ 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index d8849f929e..bd400c056f 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -1365,23 +1365,29 @@ def test_deepcopy_data(self): "f": [ov.Tensor(np.ones((1, 2, 3)), (1, 2, 3), ov.Type.i4), ov.Tensor(np.ones((1, 2, 3)))], } copied_data = deepcopy_data(data) - assert copied_data["a"] is not data["a"] - assert copied_data["b"] is not data["b"] - assert copied_data["e"]["a"] is not data["e"]["a"] - assert copied_data["e"]["b"] is not data["e"]["b"] - assert copied_data["f"][0] is not data["f"][0] - assert copied_data["f"][1] is not data["f"][1] - - assert torch.equal(copied_data["a"], data["a"]) - assert np.array_equal(copied_data["b"], data["b"]) - assert copied_data["c"] == data["c"] - assert copied_data["d"] == data["d"] - assert torch.equal(copied_data["e"]["a"], data["e"]["a"]) - assert np.array_equal(copied_data["e"]["b"], data["e"]["b"]) - assert np.array_equal(copied_data["f"][0].data, data["f"][0].data) - assert np.array_equal(copied_data["f"][1].data, data["f"][1].data) - - assert copied_data is not data + + # Checks that objects have different IDs + self.assertTrue(copied_data is not data) + self.assertTrue(copied_data["a"] is not data["a"]) + self.assertTrue(copied_data["b"] is not data["b"]) + self.assertTrue(copied_data["e"]["a"] is not data["e"]["a"]) + self.assertTrue(copied_data["e"]["b"] is not data["e"]["b"]) + self.assertTrue(copied_data["f"][0] is not data["f"][0]) + self.assertTrue(copied_data["f"][1] is not data["f"][1]) + + # Checks that constant objects have the same IDs + self.assertTrue(copied_data["c"] is data["c"]) + self.assertTrue(copied_data["d"] is data["d"]) + + # Checks that objects have the same data + self.assertTrue(torch.equal(copied_data["a"], data["a"])) + self.assertTrue(np.array_equal(copied_data["b"], data["b"])) + self.assertTrue(copied_data["c"] == data["c"]) + self.assertTrue(copied_data["d"] == data["d"]) + self.assertTrue(torch.equal(copied_data["e"]["a"], data["e"]["a"])) + self.assertTrue(np.array_equal(copied_data["e"]["b"], data["e"]["b"])) + self.assertTrue(np.array_equal(copied_data["f"][0].data, data["f"][0].data)) + self.assertTrue(np.array_equal(copied_data["f"][1].data, data["f"][1].data)) def check_optimization_not_applicable_to_optimized_model(model, quantization_config): From 46085c2295184bed94205f7954c5d22bb9ee1e35 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Thu, 6 Feb 2025 10:26:17 +0100 Subject: [PATCH 5/6] rm extra line --- tests/openvino/test_quantization.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index bd400c056f..3f47c1fef8 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -42,7 +42,6 @@ from transformers.testing_utils import slow from transformers.utils.quantization_config import QuantizationMethod - from optimum.intel import ( OVConfig, OVFluxPipeline, From af0ab91cf27a1df90f16ba482a411ebaa8443ab4 Mon Sep 17 00:00:00 2001 From: Aleksei Kashapov Date: Thu, 6 Feb 2025 12:12:26 +0100 Subject: [PATCH 6/6] use empty_like --- optimum/intel/openvino/utils.py | 2 +- tests/openvino/test_quantization.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/optimum/intel/openvino/utils.py b/optimum/intel/openvino/utils.py index 54c4183f2f..0a2d8e8440 100644 --- a/optimum/intel/openvino/utils.py +++ b/optimum/intel/openvino/utils.py @@ -599,7 +599,7 @@ def deepcopy_data(inputs: Any) -> Any: elif isinstance(inputs, tuple): new_inputs = tuple(deepcopy_data(elem) for elem in inputs) elif isinstance(inputs, Tensor): - new_inputs = Tensor(np.empty(inputs.shape), inputs.shape, inputs.get_element_type()) + new_inputs = Tensor(np.empty_like(inputs.data), inputs.get_shape(), inputs.get_element_type()) new_inputs.copy_from(inputs) else: new_inputs = deepcopy(inputs) diff --git a/tests/openvino/test_quantization.py b/tests/openvino/test_quantization.py index 3f47c1fef8..b8ce07753f 100644 --- a/tests/openvino/test_quantization.py +++ b/tests/openvino/test_quantization.py @@ -1361,7 +1361,10 @@ def test_deepcopy_data(self): "c": 1, "d": "string", "e": {"a": torch.tensor([1, 2, 3]), "b": np.array([1, 2, 3])}, - "f": [ov.Tensor(np.ones((1, 2, 3)), (1, 2, 3), ov.Type.i4), ov.Tensor(np.ones((1, 2, 3)))], + "f": [ + ov.Tensor(np.ones((1, 2, 3)), (1, 2, 3), ov.Type.i4), + ov.Tensor(np.empty((1, 2, 3)), (2, 3, 1), ov.Type.i4), + ], } copied_data = deepcopy_data(data)