Image processing C++ demo (openvinotoolkit#2123)

Author: Aleksei Korobeinikov

demos/README.md

@@ -27,6 +27,7 @@ The Open Model Zoo includes the following demos:
 - [Human Pose Estimation C++ Demo](./human_pose_estimation_demo/cpp/README.md) - Human pose estimation demo.
 - [Human Pose Estimation Python\* Demo](./human_pose_estimation_demo/python/README.md) - Human pose estimation demo.
 - [Image Inpainting Python\* Demo](./image_inpainting_demo/python/README.md) - Demo application for GMCNN inpainting network.
+- [Image Processing C++ Demo](./image_processing_demo/cpp/README.md) - Demo application for deblurring and enhancing the resolution of the input image.
 - [Image Retrieval Python\* Demo](./image_retrieval_demo/python/README.md) - The demo demonstrates how to run Image Retrieval models using OpenVINO™.
 - [Image Segmentation C++ Demo](./segmentation_demo/cpp/README.md) - Inference of semantic segmentation networks (supports video and camera inputs).
 - [Image Segmentation Python\* Demo](./segmentation_demo/python/README.md) - Inference of semantic segmentation networks (supports video and camera inputs).
@@ -52,7 +53,6 @@ The Open Model Zoo includes the following demos:
 - [Single Human Pose Estimation Python\* Demo](./single_human_pose_estimation_demo/python/README.md) - 2D human pose estimation demo.
 - [Smart Classroom C++ Demo](./smart_classroom_demo/cpp/README.md) - Face recognition and action detection demo for classroom environment.
 - [Sound Classification Python\* Demo](./sound_classification_demo/python/README.md) - Demo application for sound classification algorithm.
-- [Super Resolution C++ Demo](./super_resolution_demo/cpp/README.md) - Super Resolution demo (the demo supports only images as inputs). It enhances the resolution of the input image.
 - [Text Detection C++ Demo](./text_detection_demo/cpp/README.md) - Text Detection demo. It detects and recognizes multi-oriented scene text on an input image and puts a bounding box around detected area.
 - [Text Spotting Python\* Demo](./text_spotting_demo/python/README.md) - The demo demonstrates how to run Text Spotting models.
 - [Text-to-speech Python\* Demo](./text_to_speech_demo/python/README.md) - Shows an example of using Forward Tacotron and WaveRNN neural networks for text to speech task.

demos/common/cpp/models/include/models/deblurring_model.h

@@ -0,0 +1,37 @@
+/*
+// Copyright (C) 2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writingb  software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#pragma once
+
+#include "image_model.h"
+
+class DeblurringModel : public ImageModel {
+public:
+    /// Constructor
+    /// @param modelFileName name of model to load
+    /// @param inputImgSize size of image to set model input shape
+    DeblurringModel(const std::string& modelFileName, const cv::Size& inputImgSize);
+
+    std::shared_ptr<InternalModelData> preprocess(
+        const InputData& inputData, InferenceEngine::InferRequest::Ptr& request) override;
+    std::unique_ptr<ResultBase> postprocess(InferenceResult& infResult) override;
+
+protected:
+    void prepareInputsOutputs(InferenceEngine::CNNNetwork & cnnNetwork) override;
+    void changeInputSize(InferenceEngine::CNNNetwork& cnnNetwork);
+
+    static const size_t stride = 32;
+};

demos/common/cpp/models/include/models/results.h

@@ -84,10 +84,10 @@ struct RetinaFaceDetectionResult : public DetectionResult {
     std::vector<cv::Point2f> landmarks;
 };
 
-struct SegmentationResult : public ResultBase {
-    SegmentationResult(int64_t frameId = -1, const std::shared_ptr<MetaData>& metaData = nullptr) :
+struct ImageResult : public ResultBase {
+    ImageResult(int64_t frameId = -1, const std::shared_ptr<MetaData>& metaData = nullptr) :
         ResultBase(frameId, metaData) {}
-    cv::Mat mask;
+    cv::Mat resultImage;
 };
 
 struct HumanPose {

demos/common/cpp/models/include/models/super_resolution_model.h

@@ -0,0 +1,34 @@
+/*
+// Copyright (C) 2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writingb  software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#pragma once
+
+#include "image_model.h"
+
+class SuperResolutionModel : public ImageModel {
+public:
+    /// Constructor
+    /// @param modelFileName name of model to load
+    SuperResolutionModel(const std::string& modelFileName, const cv::Size& inputImgSize);
+
+    std::shared_ptr<InternalModelData> preprocess(
+        const InputData& inputData, InferenceEngine::InferRequest::Ptr& request) override;
+    std::unique_ptr<ResultBase> postprocess(InferenceResult& infResult) override;
+
+protected:
+    void changeInputSize(InferenceEngine::CNNNetwork& cnnNetwork, int coeff);
+    void prepareInputsOutputs(InferenceEngine::CNNNetwork & cnnNetwork) override;
+};

demos/common/cpp/models/src/deblurring_model.cpp

@@ -0,0 +1,127 @@
+/*
+// Copyright (C) 2021 Intel Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+*/
+
+#include "models/deblurring_model.h"
+#include "utils/ocv_common.hpp"
+#include <utils/slog.hpp>
+
+using namespace InferenceEngine;
+
+DeblurringModel::DeblurringModel(const std::string& modelFileName, const cv::Size& inputImgSize) :
+    ImageModel(modelFileName, false) {
+        netInputHeight = inputImgSize.height;
+        netInputWidth = inputImgSize.width;
+}
+
+void DeblurringModel::prepareInputsOutputs(CNNNetwork& cnnNetwork) {
+    // --------------------------- Configure input & output -------------------------------------------------
+    // --------------------------- Prepare input blobs ------------------------------------------------------
+    changeInputSize(cnnNetwork);
+
+    ICNNNetwork::InputShapes inputShapes = cnnNetwork.getInputShapes();
+    if (inputShapes.size() != 1)
+        throw std::runtime_error("Demo supports topologies only with 1 input");
+    inputsNames.push_back(inputShapes.begin()->first);
+    SizeVector& inSizeVector = inputShapes.begin()->second;
+    if (inSizeVector.size() != 4 || inSizeVector[0] != 1 || inSizeVector[1] != 3)
+        throw std::runtime_error("3-channel 4-dimensional model's input is expected");
+    InputInfo& inputInfo = *cnnNetwork.getInputsInfo().begin()->second;
+    inputInfo.setPrecision(Precision::U8);
+
+    // --------------------------- Prepare output blobs -----------------------------------------------------
+    const OutputsDataMap& outputInfo = cnnNetwork.getOutputsInfo();
+    if (outputInfo.size() != 1)
+        throw std::runtime_error("Demo supports topologies only with 1 output");
+
+    outputsNames.push_back(outputInfo.begin()->first);
+    Data& data = *outputInfo.begin()->second;
+    data.setPrecision(Precision::FP32);
+    const SizeVector& outSizeVector = data.getTensorDesc().getDims();
+    if (outSizeVector.size() != 4 || outSizeVector[0] != 1 || outSizeVector[1] != 3)
+        throw std::runtime_error("3-channel 4-dimensional model's output is expected");
+}
+
+void DeblurringModel::changeInputSize(CNNNetwork& cnnNetwork) {
+    ICNNNetwork::InputShapes inputShapes = cnnNetwork.getInputShapes();
+    SizeVector& inputDims = inputShapes.begin()->second;
+
+    if (inputDims[2] % stride || inputDims[3] % stride)
+        throw std::runtime_error("The shape of the model input must be divisible by stride");
+
+    netInputHeight = static_cast<int>((netInputHeight + stride - 1) / stride) * stride;
+    netInputWidth = static_cast<int>((netInputWidth + stride - 1) / stride) * stride;
+
+    inputDims[0] = 1;
+    inputDims[2] = netInputHeight;
+    inputDims[3] = netInputWidth;
+
+    cnnNetwork.reshape(inputShapes);
+}
+
+std::shared_ptr<InternalModelData> DeblurringModel::preprocess(const InputData& inputData, InferRequest::Ptr& request) {
+    auto& image = inputData.asRef<ImageInputData>().inputImage;
+    size_t h = image.rows;
+    size_t w = image.cols;
+    cv::Mat resizedImage;
+
+    if (netInputHeight - stride < h && h <= netInputHeight
+        && netInputWidth - stride < w && w <= netInputWidth) {
+        int bottom = netInputHeight - h;
+        int right = netInputWidth - w;
+        cv::copyMakeBorder(image, resizedImage, 0, bottom, 0, right,
+                           cv::BORDER_CONSTANT, 0);
+    } else {
+        slog::warn << "Chosen model aspect ratio doesn't match image aspect ratio\n";
+        cv::resize(image, resizedImage, cv::Size(netInputWidth, netInputHeight));
+    }
+    Blob::Ptr frameBlob = request->GetBlob(inputsNames[0]);
+    matU8ToBlob<uint8_t>(resizedImage, frameBlob);
+
+    return std::make_shared<InternalImageModelData>(image.cols, image.rows);
+}
+
+std::unique_ptr<ResultBase> DeblurringModel::postprocess(InferenceResult& infResult) {
+    ImageResult* result = new ImageResult;
+    *static_cast<ResultBase*>(result) = static_cast<ResultBase&>(infResult);
+
+    const auto& inputImgSize = infResult.internalModelData->asRef<InternalImageModelData>();
+
+    LockedMemory<const void> outMapped = infResult.getFirstOutputBlob()->rmap();
+    const auto outputData = outMapped.as<float*>();
+
+    std::vector<cv::Mat> imgPlanes;
+    const SizeVector& outSizeVector = infResult.getFirstOutputBlob()->getTensorDesc().getDims();
+    size_t outHeight = (int)(outSizeVector[2]);
+    size_t outWidth = (int)(outSizeVector[3]);
+    size_t numOfPixels = outWidth * outHeight;
+    imgPlanes = std::vector<cv::Mat>{
+          cv::Mat(outHeight, outWidth, CV_32FC1, &(outputData[0])),
+          cv::Mat(outHeight, outWidth, CV_32FC1, &(outputData[numOfPixels])),
+          cv::Mat(outHeight, outWidth, CV_32FC1, &(outputData[numOfPixels * 2]))};
+    cv::Mat resultImg;
+    cv::merge(imgPlanes, resultImg);
+
+    if (netInputHeight - stride < static_cast<size_t>(inputImgSize.inputImgHeight) && static_cast<size_t>(inputImgSize.inputImgHeight) <= netInputHeight
+        && netInputWidth - stride < static_cast<size_t>(inputImgSize.inputImgWidth) && static_cast<size_t>(inputImgSize.inputImgWidth) <= netInputWidth) {
+        result->resultImage = resultImg(cv::Rect(0, 0, inputImgSize.inputImgWidth, inputImgSize.inputImgHeight));
+    } else {
+        cv::resize(resultImg, result->resultImage, cv::Size(inputImgSize.inputImgWidth, inputImgSize.inputImgHeight));
+    }
+
+    result->resultImage.convertTo(result->resultImage, CV_8UC3, 255);
+
+    return std::unique_ptr<ResultBase>(result);
+}

demos/common/cpp/models/src/segmentation_model.cpp

@@ -81,7 +81,7 @@ std::shared_ptr<InternalModelData> SegmentationModel::preprocess(const InputData
         /* Just set input blob containing read image. Resize and layout conversionx will be done automatically */
         request->SetBlob(inputsNames[0], wrapMat2Blob(img));
         /* IE::Blob::Ptr from wrapMat2Blob() doesn't own data. Save the image to avoid deallocation before inference */
-         resPtr = std::make_shared<InternalImageMatModelData>(img);
+        resPtr = std::make_shared<InternalImageMatModelData>(img);
     }
     else
     {
@@ -95,20 +95,20 @@ std::shared_ptr<InternalModelData> SegmentationModel::preprocess(const InputData
 }
 
 std::unique_ptr<ResultBase> SegmentationModel::postprocess(InferenceResult& infResult) {
-    SegmentationResult* result = new SegmentationResult(infResult.frameId, infResult.metaData);
+    ImageResult* result = new ImageResult(infResult.frameId, infResult.metaData);
 
     const auto& inputImgSize = infResult.internalModelData->asRef<InternalImageModelData>();
 
     MemoryBlob::Ptr blobPtr = infResult.getFirstOutputBlob();
 
     void* pData = blobPtr->rmap().as<void*>();
 
-    result->mask = cv::Mat(outHeight, outWidth, CV_8UC1);
+    result->resultImage = cv::Mat(outHeight, outWidth, CV_8UC1);
 
     if (outChannels == 1 && blobPtr->getTensorDesc().getPrecision() == Precision::I32)
     {
         cv::Mat predictions(outHeight, outWidth, CV_32SC1, pData);
-        predictions.convertTo(result->mask, CV_8UC1);
+        predictions.convertTo(result->resultImage, CV_8UC1);
     }
     else if (blobPtr->getTensorDesc().getPrecision() == Precision::FP32)
     {
@@ -129,12 +129,12 @@ std::unique_ptr<ResultBase> SegmentationModel::postprocess(InferenceResult& infR
                     }
                 } // nChannels
 
-                result->mask.at<uint8_t>(rowId, colId) = classId;
+                result->resultImage.at<uint8_t>(rowId, colId) = classId;
             } // width
         } // height
     }
 
-    cv::resize(result->mask, result->mask,
+    cv::resize(result->resultImage, result->resultImage,
         cv::Size(inputImgSize.inputImgWidth, inputImgSize.inputImgHeight),
         0, 0, cv::INTER_NEAREST);