[PTQ][OV] Sequential models support for Engine without TensorStatistics modifications

examples/post_training_quantization/onnx/mobilenet_v2/main.py

@@ -104,6 +104,7 @@ def run_benchmark(path_to_model: str, shape: Optional[List[int]] = None, verbose
 # >> output_names = [output.name for output in sess.get_outputs()]
 # >> for data_item in val_loader:
 # >>    sess.run(output_names, input_feed=transform_fn(data_item))
+
 input_name = model.graph.input[0].name
 
 

examples/post_training_quantization/openvino/mozilla-deepspeech/accuracy_checker.json

@@ -0,0 +1,147 @@
+{
+    "compression": {
+        "algorithms": [
+            {
+                "name": "DefaultQuantization",
+                "params": {
+                    "preset": "performance",
+                    "stat_subset_size": 3
+                }
+            }
+        ],
+        "dump_intermediate_model": true
+    },
+    "engine": {
+        "datasets": [
+            {
+                "metrics": [
+                    {
+                        "type": "wer"
+                    }
+                ],
+                "name": "LibriSpeech_test_clean_wav",
+		"data_source": "/mnt/omz_new/nn_icv_cv_externalN/omz-validation-datasets/librispeech/test/LibriSpeech/test-clean.wav",
+
+		 "annotation_conversion": {
+      		 	"converter": "librispeech",
+      		 	"data_dir": "/mnt/omz_new/nn_icv_cv_externalN/omz-validation-datasets/librispeech/test/LibriSpeech/test-clean.wav"
+		 },
+                "preprocessing": [
+                    {
+                        "int16mode": true,
+                        "type": "audio_normalization"
+                    },
+                    {
+                        "duration": "512 samples",
+                        "overlap": "192 samples",
+                        "type": "clip_audio"
+                    },
+                    {
+                        "base": 512,
+                        "type": "hanning_window"
+                    },
+                    {
+                        "fftbase": 512,
+                        "magnitude_squared": true,
+                        "skip_channels": true,
+                        "type": "audio_spectrogram"
+                    },
+                    {
+                        "base": 257,
+                        "filterbank_channel_count": 40,
+                        "lower_frequency_limit": 20,
+                        "sample_rate": 16000,
+                        "type": "audio_triangle_filtering",
+                        "upper_frequency_limit": 4000
+                    },
+                    {
+                        "filterbank_channel_count": 40,
+                        "numceps": 26,
+                        "type": "audio_dct"
+                    },
+                    {
+                        "context": 9,
+                        "numceps": 26,
+                        "type": "clip_cepstrum"
+                    },
+                    {
+                        "step": 16,
+                        "type": "pack_cepstrum"
+                    }
+                ],
+                "reader": "wav_reader"
+            }
+        ],
+        "launchers": [
+            {
+                "adapter": {
+                    "beam_size": 32,
+                    "lm_alpha": 0.75,
+                    "lm_beta": 1.05,
+                    "lm_file": "/mnt/omz_new/nn_icv_cv_externalN/omz-validation-datasets/model_attributes/mozilla-deepspeech-0.6.1/lm.binary",
+                    "lm_oov_score": -1000,
+                    "lm_vocabulary_length": 4463723,
+                    "lm_vocabulary_offset": 941235601,
+                    "logarithmic_prob": false,
+                    "probability_out": "logits",
+                    "type": "ctc_beam_search_decoder_with_lm"
+                },
+                "framework": "dlsdk",
+                "inputs": [
+                    {
+                        "layout": "NHWC",
+                        "name": "input_node",
+                        "type": "INPUT"
+                    },
+                    {
+                        "name": "previous_state_c",
+                        "type": "LSTM_INPUT",
+                        "value": "cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/BlockLSTM/TensorIterator.2"
+                    },
+                    {
+                        "name": "previous_state_h",
+                        "type": "LSTM_INPUT",
+                        "value": "cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/BlockLSTM/TensorIterator.1"
+                    }
+                ]
+            },
+            {
+                "adapter": {
+                    "beam_size": 32,
+                    "lm_alpha": 0.75,
+                    "lm_beta": 1.05,
+                    "lm_file": "/mnt/omz_new/nn_icv_cv_externalN/omz-validation-datasets/model_attributes/mozilla-deepspeech-0.6.1/lm.binary",
+                    "lm_oov_score": -1000,
+                    "lm_vocabulary_length": 4463723,
+                    "lm_vocabulary_offset": 941235601,
+                    "logarithmic_prob": false,
+                    "probability_out": "logits",
+                    "type": "ctc_beam_search_decoder_with_lm"
+                },
+                "framework": "openvino",
+                "inputs": [
+                    {
+                        "layout": "NHWC",
+                        "name": "input_node",
+                        "type": "INPUT"
+                    },
+                    {
+                        "name": "previous_state_c",
+                        "type": "LSTM_INPUT",
+                        "value": "cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/GatherNd:0"
+                    },
+                    {
+                        "name": "previous_state_h",
+                        "type": "LSTM_INPUT",
+                        "value": "cudnn_lstm/rnn/multi_rnn_cell/cell_0/cudnn_compatible_lstm_cell/GatherNd_1:0"
+                    }
+                ]
+            }
+        ]
+    },
+    "model": {
+        "model": "/mnt/omz/cv_bench_cache/ww18_weekly_23.0.0-10862-40bf400b189-API2.0/mozilla-deepspeech-0.6.1/tf/tf_frozen/FP16/1/dldt/mozilla-deepspeech-0.6.1.xml",
+        "model_name": "mozilla-deepspeech-0.6.1",
+        "weights": "/mnt/omz/cv_bench_cache/ww18_weekly_23.0.0-10862-40bf400b189-API2.0/mozilla-deepspeech-0.6.1/tf/tf_frozen/FP16/1/dldt/mozilla-deepspeech-0.6.1.bin"
+    }
+}

examples/post_training_quantization/openvino/mozilla-deepspeech/main.py

@@ -0,0 +1,55 @@
+import json
+import os
+import subprocess
+
+import numpy as np
+import openvino.runtime as ov
+from openvino.tools.accuracy_checker.evaluators.quantization_model_evaluator import create_model_evaluator
+from openvino.tools.pot.configs.config import Config
+
+import nncf
+
+model_name = "mozilla-deepspeech-0.6.1"
+cache_dir = os.path.dirname(__file__)
+dataset_config = os.path.join(cache_dir, "accuracy_checker.json")
+
+command = f"omz_downloader --name {model_name} --cache_dir {cache_dir}"
+cmd_output = subprocess.call(command, shell=True)  # nosec
+
+model_dir = os.path.join(cache_dir, model_name)
+if not os.path.exists(model_dir):
+    command = f"omz_converter --name {model_name} -o {os.path.join(cache_dir, model_name)}"
+    cmd_output = subprocess.call(command, shell=True)  # nosec
+
+xml_path = os.path.join(model_dir, f"public/{model_name}/FP16/{model_name}.xml")
+ov_model = ov.Core().read_model(xml_path)
+
+config = Config.read_config(dataset_config)
+config.configure_params()
+accuracy_checker_config = config.engine
+
+model_evaluator = create_model_evaluator(accuracy_checker_config)
+model_evaluator.load_network([{"model": ov_model}])
+model_evaluator.select_dataset("")
+
+
+def get_tokens_from_sequence_func(data_item):
+    _, batch_annotation, batch_input, _ = data_item
+    filled_inputs, _, _ = model_evaluator._get_batch_input(batch_input, batch_annotation)
+    for filled_input in filled_inputs:
+        input_data = {}
+        for name, value in filled_input.items():
+            input_data[model_evaluator.launcher.input_to_tensor_name[name]] = value
+        yield input_data
+
+
+def fill_sequential_inputs_fn(model_inputs, model_outputs):
+    # Combine model inputs with state model outputs
+    # or fill state model outputs if model_outputs is None
+    state_inputs = model_evaluator.launcher._fill_lstm_inputs(model_outputs)
+    model_inputs.update(state_inputs)
+    return model_inputs
+
+
+dataset = nncf.RecurentDataset(model_evaluator.dataset, get_tokens_from_sequence_func, fill_sequential_inputs_fn)
+quantized_model = nncf.quantize(ov_model, dataset, subset_size=3)

nncf/__init__.py

@@ -15,6 +15,7 @@
 from nncf.common.logging.logger import set_log_level
 from nncf.config import NNCFConfig
 from nncf.data import Dataset
+from nncf.data import RecurentDataset
 from nncf.parameters import DropType
 from nncf.parameters import ModelType
 from nncf.parameters import TargetDevice

nncf/data/__init__.py

@@ -10,3 +10,5 @@
 # limitations under the License.
 
 from nncf.data.dataset import Dataset
+from nncf.data.dataset import RecurentDataset
+from nncf.data.dataset import Sequence

nncf/data/dataset.py

@@ -9,7 +9,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Callable, Generic, Iterable, List, Optional, TypeVar
+from typing import Any, Callable, Generic, Iterable, List, Optional, TypeVar
 
 from nncf.common.utils.api_marker import api
 
@@ -115,3 +115,30 @@ def _get_iterator_for_iter(
             if idx == indices[pos]:
                 pos = pos + 1
                 yield transform_func(data_item)
+
+
+@api(canonical_alias="nncf.RecurentDataset")
+class RecurentDataset(Dataset):
+    def __init__(self, data_source: Iterable, get_token_from_sequence_func, fill_sequential_inputs_fn):
+        def transform_fn_wrapper(data_item):
+            return Sequence(data_item, get_token_from_sequence_func, fill_sequential_inputs_fn)
+
+        super().__init__(data_source, transform_fn_wrapper)
+
+
+class Sequence:
+    def __init__(
+        self,
+        raw_sequence,
+        get_tokens_from_sequence_func: Callable[[DataItem], ModelInput],
+        fill_sequential_inputs_fn: Callable[[DataItem], ModelInput],
+    ):
+        self._raw_sequence = raw_sequence
+        self._get_tokens_from_sequence_func = get_tokens_from_sequence_func
+        self._fill_sequential_inputs_fn = fill_sequential_inputs_fn
+
+    def get_tokens_iter(self):
+        return self._get_tokens_from_sequence_func(self._raw_sequence)
+
+    def fill_inputs(self, token, model_outputs):
+        return self._fill_sequential_inputs_fn(token, model_outputs)