Merge pull request openvinotoolkit#1295 from opencv/develop

Update the release branch with 2020.4 content

Author: Roman Donchenko

CONTRIBUTING.md

@@ -72,7 +72,7 @@ The detailed descriptions of file entries provided below.
 
 **`description`**
 
-Description of the model. Must match with the description from the model [documentation](#documentation).
+Description of the model. Must match with the description from the model [documentation](#documentation). Use [this](./ci/description_updater/documentation_updater.py) script for easy update.
 
 **`task_type`**
 

README.md

@@ -1,5 +1,5 @@
 # [OpenVINO™ Toolkit](https://01.org/openvinotoolkit) - Open Model Zoo repository
-[![Stable release](https://img.shields.io/badge/version-2020.3-green.svg)](https://github.com/opencv/open_model_zoo/releases/tag/2020.3)
+[![Stable release](https://img.shields.io/badge/version-2020.4-green.svg)](https://github.com/opencv/open_model_zoo/releases/tag/2020.4)
 [![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/open_model_zoo/community)
 [![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE)
 

ci/documentation_updater/README.md

@@ -0,0 +1,22 @@
+# Documentation updater
+
+This script updates description in `model.yml` file according to description in markdown documentation from section *Use Case and High-Level Description*.
+
+## Prerequisites
+
+Install `ruamel.yaml` package:
+```
+pip install ruamel.yaml
+```
+
+## Usage
+
+To update description of single model:
+```
+python documentation_updater.py -d <OMZ dir>/models/public/<model dir> --mode update
+```
+
+To check descriptions of all public models:
+```
+python documentation_updater.py -d <OMZ dir>/models/public
+```

ci/documentation_updater/documentation_updater.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+
+"""
+ Copyright (c) 2020 Intel Corporation
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+"""
+
+import argparse
+import logging
+import re
+import ruamel.yaml
+import shlex
+
+from pathlib import Path
+from ruamel.yaml.scalarstring import FoldedScalarString
+from sys import exit
+
+MODES = [
+    'check',
+    'update'
+    ]
+
+LOG_LEVELS = [
+    'CRITICAL',
+    'ERROR',
+    'WARNING',
+    'INFO',
+    'DEBUG',
+]
+
+
+def parse():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-d', '--model-dir', type=Path, default=Path(__file__).resolve().parents[2] / 'models',
+                        help='Path to root directory with models documentation and configuration files')
+    parser.add_argument('--mode', type=str, choices=MODES, default='check',
+                        help='Script work mode: "check" only finds diffs, "update" - updates values')
+    parser.add_argument('--log-level', choices=LOG_LEVELS, default='WARNING',
+                        help='Level of logging')
+    args = parser.parse_args()
+
+    if not args.model_dir.is_dir():
+        logging.critical("Directory {} does not exist. Please check '--model-dir' option."
+                         .format(args.model_dir))
+        exit(1)
+    return args
+
+
+def collect_readme(directory, ignored_files):
+    files = {file.stem: file for file in directory.glob('**/*.md') if file.name not in ignored_files}
+    logging.info('Collected {} description files'.format(len(files)))
+    if not files:
+        logging.error("No markdown file found in {}. Exceptions - {}. Ensure, that you set right directory."
+                      .format(directory, ignored_files))
+        exit(1)
+    return files
+
+
+def convert(lines):
+    result = ''
+    list_signs = ['-', '*']
+    for line in lines:
+        if len(line.strip()) == 0:
+            result += '\n'
+        elif line.lstrip()[0] in list_signs:
+            result += '\n'
+        if len(result) > 0 and not result.endswith('\n'):
+            result += ' '
+        result += line.rstrip('\n')
+    result = re.sub(r"\[(.*?)\]\((.*?)\)", r"\1 <\2>", result) # Links transformation
+    result = result.replace("`", "\"").replace("\*", "*")
+    return result.strip()
+
+
+def collect_descriptions(files):
+    descriptions = {}
+    for name, file in files.items():
+        started = False
+        desc = []
+        with file.open("r", encoding="utf-8") as readme:
+            for line in readme:
+                if line.startswith('##'):
+                    if not started:
+                        started = True
+                        continue
+                    else:
+                        break
+                if started:
+                    desc.append(line)
+        desc = convert(desc)
+        if desc != '':
+            descriptions[name] = desc
+        else:
+            logging.warning('No description found in {} file. '
+                            'Check  compliance with the OMZ Contribution Guide.'.format(file))
+    return descriptions
+
+
+def get_models_from_configs(directory):
+    yaml = ruamel.yaml.YAML()
+    yaml.preserve_quotes = True
+    models = {}
+    model_configs = directory.glob('**/model.yml')
+    for model in model_configs:
+        with model.open("r", encoding="utf-8") as file:
+            models[model.parent.name] = (model, yaml.load(file))
+        if not models[model.parent.name][1]:
+            logging.error("File {} is empty. It will be ignored.".format(model))
+            del models[model.parent.name]
+
+    return models
+
+
+def update_model_descriptions(models, descriptions, mode):
+    update_models = []
+    missed_models = []
+    for name, desc in descriptions.items():
+        model = models.get(name, None)
+        if model is None:
+            logging.error('For description file {}.md no model found'.format(name))
+            missed_models.append(name)
+            continue
+        if not model[1].get('description', None):
+            logging.error('No description found in {} for {} model'.format(model[0], name))
+            missed_models.append(name)
+            continue
+
+        model = model[1]
+        if model.get('description', '') != desc:
+            if mode == 'update':
+                model['description'] = FoldedScalarString(desc)
+            else:
+                logging.debug('Found diff for {} model'.format(name))
+                logging.debug('\n{:12s}{}\n\tvs\n{:12s}{}'
+                              .format('In config:', model['description'], 'In readme:', desc))
+            update_models.append(name)
+    if mode == 'update':
+        msg = 'Description updated for {} models, missed for {} models.'
+        msg_model_list = 'UPDATED:\n\t{}'
+    else:
+        msg = 'Description differs for {} models, missed for {} models.'
+        msg_model_list = 'DIFFERENCE:\n\t{}'
+    logging.info(msg.format(len(update_models), len(missed_models)))
+    if len(update_models) > 0:
+        logging.info(msg_model_list.format("\n\t".join(update_models)))
+    if len(missed_models) > 0:
+        logging.info('FAILED:\n\t{}'.format("\n\t".join(missed_models)))
+    return update_models
+
+
+def update_model_configs(models, descriptions, mode):
+    diffs = update_model_descriptions(models, descriptions, mode)
+    if mode == 'update':
+        for name in diffs:
+            model = models[name]
+            yaml = ruamel.yaml.YAML()
+            yaml.indent(mapping=2, sequence=4, offset=2)
+            yaml.width = 80
+            with model[0].open("w", encoding="utf-8") as file:
+                yaml.dump(model[1], file)
+
+
+def main():
+    args = parse()
+    logging.basicConfig(level=getattr(logging, args.log_level.upper()), format='%(levelname)s: %(message)s')
+
+    ignored_files = ('index.md',)
+    descriptions = collect_descriptions(collect_readme(args.model_dir, ignored_files))
+    models = get_models_from_configs(args.model_dir)
+    update_model_configs(models, descriptions, args.mode)
+
+
+if __name__ == '__main__':
+    main()

ci/get-jobs-for-changes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 # Copyright (c) 2019 Intel Corporation
 #

ci/requirements-ac-test.txt

@@ -1,27 +1,43 @@
-atomicwrites==1.3.0
+# use update-requirements.py to update this file
+
+atomicwrites==1.3.0       # via -r tools/accuracy_checker/requirements-test.in
 attrs==19.3.0             # via pytest
-editdistance==0.5.3
-importlib-metadata==1.5.0  # via pluggy, pytest
+cycler==0.10.0            # via matplotlib
+decorator==4.4.2          # via networkx
+editdistance==0.5.3       # via -r tools/accuracy_checker/requirements.in
+imageio==2.8.0            # via scikit-image
+importlib-metadata==1.6.0  # via pluggy, pytest
 joblib==0.14.1            # via scikit-learn
+kiwisolver==1.1.0         # via matplotlib
+matplotlib==3.0.3         # via scikit-image
 more-itertools==8.2.0     # via pytest
-nibabel==3.0.1
-numpy==1.17.5             # via nibabel, scikit-learn, scipy
-packaging==20.1           # via pytest
+networkx==2.4             # via scikit-image
+nibabel==3.0.2            # via -r tools/accuracy_checker/requirements.in
+numpy==1.17.5             # via -r tools/accuracy_checker/requirements-core.in, imageio, matplotlib, nibabel, pywavelets, scikit-learn, scipy
+packaging==20.3           # via pytest
 pathlib2==2.3.5           # via pytest
-pillow==7.0.0
+pillow==7.1.1             # via -r tools/accuracy_checker/requirements.in, imageio, scikit-image
 pluggy==0.13.1            # via pytest
-py-cpuinfo==4.0.0
+py-cpuinfo==4.0.0         # via -r tools/accuracy_checker/requirements.in
 py==1.8.1                 # via pytest
-pyparsing==2.4.6          # via packaging
-pytest-mock==2.0.0
-pytest==5.3.5
-pyyaml==5.3               # via yamlloader
-scikit-learn==0.22.1
-scipy==1.4.1
-sentencepiece==0.1.85
-shapely==1.7.0
-six==1.14.0               # via packaging, pathlib2
-tqdm==4.42.1
-wcwidth==0.1.8            # via pytest
-yamlloader==0.5.5
-zipp==1.1.0               # via importlib-metadata
+pydicom==1.4.2            # via -r tools/accuracy_checker/requirements.in
+pyparsing==2.4.7          # via matplotlib, packaging
+pytest-mock==2.0.0        # via -r tools/accuracy_checker/requirements-test.in
+pytest==5.4.1             # via -r tools/accuracy_checker/requirements-test.in, pytest-mock
+python-dateutil==2.8.1    # via matplotlib
+pywavelets==1.1.1         # via scikit-image
+pyyaml==5.3.1             # via -r tools/accuracy_checker/requirements-core.in, yamlloader
+scikit-image==0.15.0      # via -r tools/accuracy_checker/requirements.in
+scikit-learn==0.22.2.post1  # via -r tools/accuracy_checker/requirements.in
+scipy==1.4.1              # via -r tools/accuracy_checker/requirements.in, scikit-image, scikit-learn
+sentencepiece==0.1.85     # via -r tools/accuracy_checker/requirements.in
+shapely==1.7.0            # via -r tools/accuracy_checker/requirements.in
+six==1.14.0               # via packaging, pathlib2, python-dateutil
+tokenizers==0.7           # via -r tools/accuracy_checker/requirements.in
+tqdm==4.45.0              # via -r tools/accuracy_checker/requirements.in
+wcwidth==0.1.9            # via pytest
+yamlloader==0.5.5         # via -r tools/accuracy_checker/requirements.in
+zipp==1.2.0               # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

ci/requirements-ac.txt

@@ -1,13 +1,31 @@
-editdistance==0.5.3
+# use update-requirements.py to update this file
+
+cycler==0.10.0            # via matplotlib
+decorator==4.4.2          # via networkx
+editdistance==0.5.3       # via -r tools/accuracy_checker/requirements.in
+imageio==2.8.0            # via scikit-image
 joblib==0.14.1            # via scikit-learn
-nibabel==3.0.1
-numpy==1.17.5             # via nibabel, scikit-learn, scipy
-pillow==7.0.0
-py-cpuinfo==4.0.0
-pyyaml==5.3               # via yamlloader
-scikit-learn==0.22.1
-scipy==1.4.1
-sentencepiece==0.1.85
-shapely==1.7.0
-tqdm==4.42.1
-yamlloader==0.5.5
+kiwisolver==1.1.0         # via matplotlib
+matplotlib==3.0.3         # via scikit-image
+networkx==2.4             # via scikit-image
+nibabel==3.0.2            # via -r tools/accuracy_checker/requirements.in
+numpy==1.17.5             # via -r tools/accuracy_checker/requirements-core.in, imageio, matplotlib, nibabel, pywavelets, scikit-learn, scipy
+pillow==7.1.1             # via -r tools/accuracy_checker/requirements.in, imageio, scikit-image
+py-cpuinfo==4.0.0         # via -r tools/accuracy_checker/requirements.in
+pydicom==1.4.2            # via -r tools/accuracy_checker/requirements.in
+pyparsing==2.4.7          # via matplotlib
+python-dateutil==2.8.1    # via matplotlib
+pywavelets==1.1.1         # via scikit-image
+pyyaml==5.3.1             # via -r tools/accuracy_checker/requirements-core.in, yamlloader
+scikit-image==0.15.0      # via -r tools/accuracy_checker/requirements.in
+scikit-learn==0.22.2.post1  # via -r tools/accuracy_checker/requirements.in
+scipy==1.4.1              # via -r tools/accuracy_checker/requirements.in, scikit-image, scikit-learn
+sentencepiece==0.1.85     # via -r tools/accuracy_checker/requirements.in
+shapely==1.7.0            # via -r tools/accuracy_checker/requirements.in
+six==1.15.0               # via python-dateutil
+tokenizers==0.7           # via -r tools/accuracy_checker/requirements.in
+tqdm==4.45.0              # via -r tools/accuracy_checker/requirements.in
+yamlloader==0.5.5         # via -r tools/accuracy_checker/requirements.in
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools