AC: update test model (openvinotoolkit#873)

Author: eaidova

tools/accuracy_checker/accuracy_checker/launcher/dlsdk_launcher_readme.md

@@ -6,8 +6,13 @@ For enabling OpenVINO™ launcher you need to add `framework: dlsdk` in launcher
 Heterogeneous plugin as `HETERO:target_device,fallback_device` and Multi device plugin as `MULTI:target_device1,target_device2`.
 If you have several MYRIAD devices in your machine, you are able to provide specific device id in such way: `MYRIAD.<DEVICE_ID>` (e.g. `MYRIAD.1.2-ma2480`)
 It is possible to specify one or more devices via `-td, --target devices` command line argument. Target device will be selected from command line (in case when several devices provided, evaluations will be run one by one with all specified devices).
-* `model` - path to xml file with Caffe model for your topology.
-* `weights` - path to bin file with weights for your topology.
+* `model` - path to xml file with model for your topology or compiled executable network.
+* `weights` - path to bin file with weights for your topology (Optional, the argument can be omitted if bin file stored in the same directory with model xml or if you use compiled blob).
+
+**Note:** 
+   You can generate executable blob using [compile_tool](http://docs.openvinotoolkit.org/latest/_inference_engine_tools_compile_tool_README.html).
+   Before evaluation executable blob, please make sure that selected device support it.
+
 
 launcher may optionally provide model parameters in source framework format which will be converted to Inference Engine IR using Model Optimizer.
 If you want to use Model Optimizer for model conversion, please view [Model Optimizer Developer Guide](https://software.intel.com/en-us/articles/OpenVINO-ModelOptimizer).

tools/accuracy_checker/data/test_models/SampLeNet.blob

@@ -38,3 +38,10 @@ Used options: `-c` path to evaluation config, `-m` directory where models are st
 If everything worked correctly, you should be able to get `75.02%` accuracy.
 
 Now try edit config, to run SampLeNet on other device or framework (e.g. Caffe, MXNet or OpenCV), or go directly to your topology!
+
+
+###  Additional useful resources
+
+* [config](opencv_sample_config.yml) for running SampleNet via [OpenCV launcher](../accuracy_checker/launcher/opencv_launcher_readme.md)
+* [config](sample_blob_config.yml) for running SampleNet using compiled executable network blob.
+**Note: Not all OpenVINO plugins support compiled network blob execution**

tools/accuracy_checker/sample/README.md

@@ -0,0 +1,71 @@
+models:
+  - name: SampLeNet_example
+
+    # list of launchers for your topology.
+    launchers:
+        # launcher framework (e.g. caffe, dlsdk)
+      - framework: dlsdk
+        # device for infer (e.g. for dlsdk cpu, gpu, hetero:cpu,gpu ...)
+        # Note: not all devices support blob execution, it is the reason why for sample we will use myriad.
+        device: MYRIAD
+        # exported executable network blob
+        # you can export executable network blob from OpenVINO IR using compile_tool, see http://docs.openvinotoolkit.org/latest/_inference_engine_tools_compile_tool_README.html
+        # path to model is prefixed with directory, specified in "-m/--models" option
+        model: SampleNet.blob
+        # launcher returns raw result, so it should be converted
+        # to an appropriate representation with adapter
+        adapter: classification
+
+    # metrics, preprocessing and postprocessing are typically dataset specific, so dataset field
+    # specifies data and all other steps required to validate topology
+    # there is typically definitions file, which contains options for common datasets and which is merged
+    # during evaluation, but since "sample_dataset" is not used anywhere else, this config contains full definition
+    datasets:
+        # uniquely distinguishable name for dataset
+        # note that all other steps are specific for this dataset only
+        # if you need to test topology on multiple datasets, you need to specify
+        # every step explicitly for each dataset
+      - name: sample_dataset
+        # directory where input images are searched.
+        # prefixed with directory specified in "-s/--source" option
+        data_source: sample_dataset/test
+        # parameters for annotation conversion to a common annotation representation format.
+        annotation_conversion:
+          # specified which annotation converter will be used
+          #  In order to do this you need to provide your own annotation converter,
+          # i.e. implement BaseFormatConverter interface.
+          # All annotation converters are stored in accuracy_checker/annotation_converters directory.
+          converter: cifar
+          # converter specific parameters.
+          # Full range available options you can find in accuracy_checker/annotation_converters/README.md
+          # relative paths will be merged with "-s/--source" option
+          data_batch_file: cifar-10-batches-py/test_batch
+          # cifar stores images as binary file, we should convert them to png in first evaluation.
+          # Yo do not need to use these options if you have already converted dataset images.
+          convert_images: True
+          # path to save converted images.
+          converted_images_dir: sample_dataset/test
+          # number of classes in the dataset, used for label_map generation
+          num_classes: 10
+
+        # list of preprocessing, applied to each image during validation
+        # order of entries matters
+        preprocessing:
+            # resize input image to topology input size
+            # you may specify size to which image should be resized
+            # via dst_width, dst_height fields
+          - type: resize
+            size: 32
+            # topology is trained on RGB images, but OpenCV reads in BGR
+            # thence it must be converted to RGB
+          - type: bgr_to_rgb
+            # dataset mean and standard deviation
+          - type: normalization
+            # you may specify precomputed statistics manually or use precomputed values, such as ImageNet as well
+            mean: (125.307, 122.961, 113.8575)
+            std: (51.5865, 50.847, 51.255)
+
+        # list of metrics, calculated on dataset
+        metrics:
+          - type: accuracy
+            top_k: 1

tools/accuracy_checker/sample/sample_blob_config.yml

@@ -34,6 +34,13 @@
 from accuracy_checker.data_readers import DataRepresentation
 from accuracy_checker.utils import contains_all
 
+def no_available_myriad():
+    try:
+        from openvino.inference_engine import IECore
+        return 'MYRIAD' not in IECore().availabe_devices
+    except:
+        return True
+
 
 @pytest.fixture()
 def mock_inference_engine(mocker):
@@ -63,6 +70,20 @@ def get_dlsdk_test_model(models_dir, config_update=None):
     return create_launcher(config)
 
 
+def get_dlsdk_test_blob(models_dir, config_update=None):
+    config = {
+        'framework': 'dlsdk',
+        'model': str(models_dir / 'SampLeNet.blob'),
+        'device': 'MYRIAD',
+        'adapter': 'classification',
+        '_models_prefix': str(models_dir)
+    }
+    if config_update:
+        config.update(config_update)
+
+    return create_launcher(config)
+
+
 def get_image(image_path, input_shape):
     _, _, h, w = input_shape
     img_raw = cv2.imread(str(image_path))
@@ -91,10 +112,21 @@ def test_infer_with_additional_outputs(self, models_dir):
         assert contains_all(outputs, ['fc1', 'fc2', 'fc3'])
         assert dlsdk_test_model.output_blob == 'fc3'
 
-    def test_dlsd_launcher_set_batch_size(self, models_dir):
+    def test_dlsdk_launcher_set_batch_size(self, models_dir):
         dlsdk_test_model = get_dlsdk_test_model(models_dir, {'batch': 2})
         assert dlsdk_test_model.batch == 2
 
+    @pytest.mark.skipif(no_available_myriad(), reason='no myriad device in the system')
+    def test_dlsdk_launcher_import_network(self, data_dir, models_dir):
+        dlsdk_test_model = get_dlsdk_test_blob(models_dir)
+        image = get_image(data_dir / '1.jpg', dlsdk_test_model.inputs['data'].shape)
+        input_blob = np.transpose([image.data], (0, 3, 1, 2))
+        result = dlsdk_test_model.predict([{'data': input_blob.astype(np.float32)}], [image.metadata])
+        assert dlsdk_test_model.output_blob == 'fc3'
+
+        assert np.argmax(result[0][dlsdk_test_model.output_blob]) == 7
+        assert image.metadata['input_shape'] == {'data': [1, 3, 32, 32]}
+
 
 @pytest.mark.usefixtures('mock_path_exists')
 class TestDLSDKLauncherAffinity: