AC: Added processing of device configs for composite devices (openvinotoolkit#1434)

Author: Anna Mironova

tools/accuracy_checker/accuracy_checker/launcher/dlsdk_launcher.py

@@ -20,6 +20,7 @@
 import os
 import platform
 import re
+import warnings
 from collections import OrderedDict
 import numpy as np
 import openvino.inference_engine as ie
@@ -699,10 +700,22 @@ def _prepare_multi_device(self, log=True):
                 print_info('    {} - {}'.format(device, nreq))
 
     def _set_device_config(self, device_config):
-        device_specific_configuration = read_yaml(device_config)
-        if not isinstance(device_specific_configuration, dict):
+        device_configuration = read_yaml(device_config)
+        if not isinstance(device_configuration, dict):
             raise ConfigError('device configuration should be a dict-like')
-        self.ie_core.set_config(device_specific_configuration, self.device)
+        if all(not isinstance(value, dict) for value in device_configuration.values()):
+            self.ie_core.set_config(device_configuration, self.device)
+        else:
+            for key, value in device_configuration.items():
+                if isinstance(value, dict):
+                    if key in ie.known_plugins:
+                        self.ie_core.set_config(value, key)
+                    else:
+                        warnings.warn('Option {key}: {value} will be skipped because device is '
+                                      'unknown'.format(key=key, value=value))
+                else:
+                    warnings.warn('Option {key}: {value} will be skipped because device to which it should be '
+                                  'applied is not specified or option is not a dict-like'.format(key=key, value=value))
 
     def _log_versions(self):
         versions = self.ie_core.get_versions(self._device)

tools/accuracy_checker/accuracy_checker/launcher/dlsdk_launcher_readme.md

@@ -9,7 +9,7 @@ It is possible to specify one or more devices via `-td, --target devices` comman
 * `model` - path to xml file with model for your topology or compiled executable network.
 * `weights` - path to bin file with weights for your topology (Optional, the argument can be omitted if bin file stored in the same directory with model xml or if you use compiled blob).
 
-**Note:** 
+**Note:**
    You can generate executable blob using [compile_tool](https://docs.openvinotoolkit.org/latest/_inference_engine_tools_compile_tool_README.html).
    Before evaluation executable blob, please make sure that selected device support it.
 
@@ -28,7 +28,7 @@ You can provide:
 In case when you want to determine additional parameters for model conversion (data_type, input_shape and so on), you can use `mo_params` for arguments with values and `mo_flags` for positional arguments like `legacy_mxnet_model` .
 Full list of supported parameters you can find in Model Optimizer Developer Guide.
 
-Model will be converted before every evaluation. 
+Model will be converted before every evaluation.
 You can provide `converted_model_dir` for saving converted model in specific folder, otherwise, converted models will be saved in path provided via `-C` command line argument or source model directory.
 
 * `adapter` - approach how raw output will be converted to representation of dataset problem, some adapters can be specific to framework. You can find detailed instruction how to use adapters [here](../adapters/README.md).
@@ -43,14 +43,18 @@ Additionally you can provide device specific parameters:
 * `gpu_extensions` (path to extension *.xml file with OpenCL kernel description for gpu).
 * `bitstream` for running on FPGA.
 
-For setting device specific flags, you are able to use `-dc` or `--device_config` command line option. Device config should be represented as YML file with dictionary, where keys are plugin configuration keys and values are their values respectively.
+Device config contains device specific options which should be set to Inference Engine. For setting device specific flags, you are able to use `-dc` or `--device_config` command line option. Device config should be represented as YML file with dictionary of one of two types:
+1. keys are plugin configuration keys and values are their values respectively. In this way configuration will be applied to current running device.
+2. keys are supported devices and values are plugin configuration for each device. Plugin configuration represented as dictionary where keys are plugin specific configuration keys and values are their values respectively.
+
 Each supported device has own set of supported configuration parameters which can be found on device page in [Inference Engine development guide](https://docs.openvinotoolkit.org/latest/_docs_IE_DG_supported_plugins_Supported_Devices.html)
 
 **Note:** Since OpenVINO 2020.4 on platforms with native bfloat16 support models will be executed on this precision by default. For disabling this behaviour, you need to use device_config with following configuration:
 ```yml
-ENFORCE_BF16: "NO"
+CPU:
+   ENFORCE_BF16: "NO"
 ```
-Device config example can be found <a href="https://github.com/opencv/open_model_zoo/blob/develop/tools/accuracy_checker/sample/disable_bfloat16_device_config.yml">here</a>()
+Device config example can be found <a href="https://github.com/opencv/open_model_zoo/blob/develop/tools/accuracy_checker/sample/disable_bfloat16_device_config.yml">here</a>
 
 Beside that, you can launch model in `async_mode`, enable this option and optionally provide the number of infer requests (`num_requests`), which will be used in evaluation process. By default, if `num_requests` not provided or used value `AUTO`, automatic number request assignment for specific device will be performed
 For multi device configuration async mode used always. You can provide number requests for each device as part device specification: `MULTI:device_1(num_req_1),device_2(num_req_2)` or in `num_requests` config section (for this case comma-separated list of integer numbers or one value if number requests for all devices equal can be used).

tools/accuracy_checker/sample/disable_bfloat16_device_config.yml

@@ -1,10 +1,18 @@
 # Device config example
 
 # Contains device specific options which should be set to Inference Engine.
-# Set of supported keys can be found on device related page in Inference Engine Development guide.
-# Device config should be provided via -dc or --device_config commandline option.
+# For setting device specific flags, you are able to use `-dc` or `--device_config` command line option.
+# Device config should be represented as YML file with dictionary of one of two types:
+#     1. keys are plugin configuration keys and values are their values respectively.
+#        In this way configuration will be applied to current running device.
+#     2. keys are supported devices and values are plugin configuration for each device.
+#        Plugin configuration represented as dictionary where keys are plugin specific configuration keys and values
+#        are their values respectively.
+# Each supported device has own set of supported configuration parameters which can be found on
+# device page in Inference Engine development guide.
 
 # This config example shows how to disable bfloat16 optimizations for CPU.
 # Note: on platforms with native bfloat16 support, ENFORCE_BF16 parameter by default set as "YES",
-# for disabling this behavior you need to specify following configuration.
-ENFORCE_BF16: "NO"
+# for disabling this behavior you need to use device_config with following configuration.
+CPU:
+   ENFORCE_BF16: "NO"

tools/accuracy_checker/tests/test_dlsdk_launcher.py

@@ -1107,6 +1107,50 @@ def test_raises_with_tf_model_and_tf_meta_both_provided(self):
         with pytest.raises(ConfigError):
             DLSDKLauncher(config)
 
+    @pytest.mark.usefixtures('mock_file_exists')
+    def test_dlsdk_launcher_device_config_config_not_dict_like(self, mocker, models_dir):
+        device_config = 'ENFORCE_BF16'
+
+        mocker.patch(
+            'accuracy_checker.launcher.dlsdk_launcher.read_yaml', return_value=device_config
+        )
+
+        with pytest.raises(ConfigError):
+            get_dlsdk_test_model(models_dir, {'_device_config': './device_config.yml'})
+
+    @pytest.mark.usefixtures('mock_file_exists')
+    def test_dlsdk_launcher_device_config_device_unknown(self, mocker, models_dir):
+        device_config = {'device': {'ENFORCE_BF16': 'NO'}}
+
+        mocker.patch(
+            'accuracy_checker.launcher.dlsdk_launcher.read_yaml', return_value=device_config
+        )
+
+        with pytest.warns(Warning):
+            get_dlsdk_test_model(models_dir, {'_device_config': './device_config.yml'})
+
+    @pytest.mark.usefixtures('mock_file_exists')
+    def test_dlsdk_launcher_device_config_one_option_for_device_is_not_dict(self, mocker, models_dir):
+        device_config = {'CPU': {'ENFORCE_BF16': 'NO'}, 'GPU': 'ENFORCE_BF16'}
+
+        mocker.patch(
+            'accuracy_checker.launcher.dlsdk_launcher.read_yaml', return_value=device_config
+        )
+
+        with pytest.warns(Warning):
+            get_dlsdk_test_model(models_dir, {'_device_config': './device_config.yml'})
+
+    @pytest.mark.usefixtures('mock_file_exists')
+    def test_dlsdk_launcher_device_config_one_option_is_not_binding_to_device(self, mocker, models_dir):
+        device_config = {'CPU': {'ENFORCE_BF16': 'NO'}, 'ENFORCE_BF16': 'NO'}
+
+        mocker.patch(
+            'accuracy_checker.launcher.dlsdk_launcher.read_yaml', return_value=device_config
+        )
+
+        with pytest.warns(Warning):
+            get_dlsdk_test_model(models_dir, {'_device_config': './device_config.yml'})
+
 
 @pytest.mark.usefixtures('mock_path_exists', 'mock_inputs', 'mock_inference_engine')
 class TestDLSDKLauncherConfig: