AC: add support weld porosity detection model as part of common pipeline (openvinotoolkit#1217)

* AC:add support weld porosity detection model as part of common pipeline

* fix spaces

* prevent problems with async execution

* extend tests

Author: eaidova

tools/accuracy_checker/accuracy_checker/data_readers/data_reader.py

@@ -99,7 +99,7 @@ class BaseReader(ClassProvider):
     __provider_type__ = 'reader'
 
     def __init__(self, data_source, config=None, **kwargs):
-        self.config = config
+        self.config = config or {}
         self.data_source = data_source
         self.read_dispatcher = singledispatch(self.read)
         self.read_dispatcher.register(list, self._read_list)
@@ -127,6 +127,7 @@ def __call__(self, context=None, identifier=None, **kwargs):
 
     def configure(self):
         self.data_source = get_path(self.data_source, is_directory=True)
+        self.multi_infer = self.config.get('multi_infer', False)
 
     def validate_config(self):
         pass
@@ -146,7 +147,10 @@ def _read_frames_multi_input(self, data_id):
         return self.read_dispatcher(data_id.frames)
 
     def read_item(self, data_id):
-        return DataRepresentation(self.read_dispatcher(data_id), identifier=data_id)
+        data_rep = DataRepresentation(self.read_dispatcher(data_id), identifier=data_id)
+        if self.multi_infer:
+            data_rep.metadata['multi_infer'] = True
+        return data_rep
 
     @property
     def name(self):
@@ -181,6 +185,7 @@ def configure(self):
             reading_scheme[pattern] = reader
 
         self.reading_scheme = reading_scheme
+        self.multi_infer = self.config.get('multi_infer', False)
 
     def read(self, data_id):
         for pattern, reader in self.reading_scheme.items():
@@ -207,14 +212,15 @@ class OpenCVImageReader(BaseReader):
 
     def validate_config(self):
         if self.config:
-            config_validator = OpenCVImageReaderConfig('opencv_imread_config')
+            config_validator = OpenCVImageReaderConfig(
+                'opencv_imread_config', on_extra_argument=ConfigValidator.IGNORE_ON_EXTRA_ARGUMENT
+            )
             config_validator.validate(self.config)
 
     def configure(self):
         super().configure()
         self.flag = OPENCV_IMREAD_FLAGS[self.config.get('reading_flag', 'color') if self.config else 'color']
 
-
     def read(self, data_id):
         return cv2.imread(str(get_path(self.data_source / data_id)), self.flag)
 
@@ -281,6 +287,7 @@ def _read_sequence(self, data_id):
     def configure(self):
         self.data_source = get_path(self.data_source)
         self.videocap = cv2.VideoCapture(str(self.data_source))
+        self.multi_infer = self.config.get('multi_infer', False)
 
     def reset(self):
         self.current = -1
@@ -301,6 +308,7 @@ def validate_config(self):
 
     def configure(self):
         self.key = self.config.get('key')
+        self.multi_infer = self.config.get('multi_infer', False)
 
     def read(self, data_id):
         data = read_json(str(self.data_source / data_id))
@@ -343,6 +351,7 @@ def configure(self):
         if nib is None:
             raise ImportError('nifty backend for image reading requires nibabel. Please install it before usage.')
         self.channels_first = self.config.get('channels_first', False) if self.config else False
+        self.multi_infer = self.config.get('multi_infer', False)
 
     def read(self, data_id):
         nib_image = nib.load(str(get_path(self.data_source / data_id)))
@@ -353,10 +362,12 @@ def read(self, data_id):
 
         return image
 
+
 class NumpyReaderConfig(ConfigValidator):
     type = StringField(optional=True)
     keys = StringField(optional=True, default="")
 
+
 class NumPyReader(BaseReader):
     __provider__ = 'numpy_reader'
 
@@ -366,8 +377,10 @@ def validate_config(self):
             config_validator.validate(self.config)
 
     def configure(self):
+        self.multi_infer = self.config.get('multi_infer', False)
         self.keys = self.config.get('keys', "") if self.config else ""
         self.keys = [t.strip() for t in self.keys.split(',')] if len(self.keys) > 0 else []
+        self.multi_infer = self.config.get('multi_infer', False)
 
     def read(self, data_id):
         data = np.load(str(self.data_source / data_id))
@@ -384,6 +397,7 @@ def read(self, data_id):
         key = next(iter(data.keys()))
         return data[key]
 
+
 class TensorflowImageReader(BaseReader):
     __provider__ = 'tf_imread'
 
@@ -422,6 +436,7 @@ def configure(self):
         self.single = len(self.feature_list) == 1
         self.counter = 0
         self.subset = range(len(self.data_source))
+        self.multi_infer = self.config.get('multi_infer', False)
 
     def read(self, data_id):
         relevant_annotation = self.data_source[self.subset[self.counter]]

tools/accuracy_checker/accuracy_checker/dataset.py

@@ -273,6 +273,7 @@ def create_subset(annotation, subsample_size, subsample_seed, shuffle=True):
         raise ConfigError('subsample_size should be > 0')
     return make_subset(annotation, subsample_size, subsample_seed, shuffle)
 
+
 class DatasetWrapper:
     def __init__(self, data_reader, annotation_reader=None, tag='', dataset_config=None):
         self.tag = tag
@@ -353,3 +354,7 @@ def full_size(self):
     @property
     def size(self):
         return self.__len__()
+
+    @property
+    def multi_infer(self):
+        return getattr(self.data_reader, 'multi_infer', False)

tools/accuracy_checker/accuracy_checker/evaluators/model_evaluator.py

@@ -129,7 +129,11 @@ def prepare_dataset():
 
         prepare_dataset()
 
-        if self.launcher.allow_reshape_input or self.preprocessor.has_multi_infer_transformations:
+        if (
+                self.launcher.allow_reshape_input or
+                self.preprocessor.has_multi_infer_transformations or
+                getattr(self.reader, 'multi_infer', False)
+        ):
             warning('Model can not to be processed in async mode. Switched to sync.')
             return self.process_dataset_sync(stored_predictions, progress_reporter, *args, **kwargs)
 

tools/accuracy_checker/accuracy_checker/evaluators/quantization_model_evaluator.py

@@ -107,7 +107,11 @@ def completion_callback(status_code, request_id):
 
         self._prepare_to_evaluation(dataset_tag, dump_prediction_to_annotation)
 
-        if self.launcher.allow_reshape_input or self.preprocessor.has_multi_infer_transformations:
+        if (
+                self.launcher.allow_reshape_input or
+                self.preprocessor.has_multi_infer_transformations or
+                self.dataset.multi_infer
+        ):
             warning('Model can not to be processed in async mode. Switched to sync.')
             return self.process_dataset(
                 subset,

tools/accuracy_checker/configs/weld-porosity-detection-0001.yml

@@ -0,0 +1,34 @@
+models:
+  - name: weld-porosity-detection-0001
+    launchers:
+      - framework: dlsdk
+        tags:
+          - FP32
+        model:   intel/weld-porosity-detection-0001/FP32/weld-porosity-detection-0001.xml
+        weights: intel/weld-porosity-detection-0001/FP32/weld-porosity-detection-0001.bin
+        adapter: classification
+
+      - framework: dlsdk
+        tags:
+          - FP16
+        model:   intel/weld-porosity-detection-0001/FP16/weld-porosity-detection-0001.xml
+        weights: intel/weld-porosity-detection-0001/FP16/weld-porosity-detection-0001.bin
+        adapter: classification
+
+    datasets:
+      - name: weld-porosity
+        reader:
+          type: opencv_imread
+          multi_infer: True
+
+        preprocessing:
+          - type: resize
+            size: 224
+            aspect_ratio_scale: greater
+          - type: crop
+            size: 224
+
+        metrics:
+          - type: accuracy
+
+global_definitions: ../dataset_definitions.yml

tools/accuracy_checker/tests/test_model_evaluator.py

@@ -157,6 +157,7 @@ def setup_method(self):
 
         self.dataset = MagicMock()
         self.dataset.__iter__.return_value = [(range(1), self.annotations[0]), (range(1), self.annotations[1])]
+        self.data_reader.multi_infer = False
 
         self.postprocessor.process_batch = Mock(side_effect=[
             ([annotation_container_0], [annotation_container_0]), ([annotation_container_1], [annotation_container_1])
@@ -238,7 +239,7 @@ def test_switch_to_sync_predict_if_need_reshaping(self):
         assert not self.launcher.predict_async.called
         assert self.metric.update_metrics_on_batch.call_count == len(self.annotations)
 
-    def test_switch_to_sync_predict_if_need_multi_infer(self):
+    def test_switch_to_sync_predict_if_need_multi_infer_after_preprocessing(self):
         self.postprocessor.has_dataset_processors = False
         self.launcher.allow_reshape_input = False
         self.preprocessor.has_multi_infer_transformations = True
@@ -250,3 +251,17 @@ def test_switch_to_sync_predict_if_need_multi_infer(self):
         assert self.launcher.predict.called
         assert not self.launcher.predict_async.called
         assert self.metric.update_metrics_on_batch.call_count == len(self.annotations)
+
+    def test_switch_to_sync_predict_if_need_multi_infer(self):
+        self.postprocessor.has_dataset_processors = False
+        self.launcher.allow_reshape_input = False
+        self.preprocessor.has_multi_infer_transformations = False
+        self.data_reader.multi_infer = True
+
+        self.evaluator.process_dataset(None, None)
+
+        assert not self.evaluator.store_predictions.called
+        assert not self.evaluator.load.called
+        assert self.launcher.predict.called
+        assert not self.launcher.predict_async.called
+        assert self.metric.update_metrics_on_batch.call_count == len(self.annotations)