From 7c89b2a60f6b9b471dde773ba15062862eb6cf0f Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Thu, 30 Jan 2025 02:29:13 +0900 Subject: [PATCH 01/11] Add tests for py bindings --- .github/workflows/test_accuracy.yml | 4 +++ tests/cpp/accuracy/conftest.py | 8 +++++ tests/cpp/accuracy/test_bindings.py | 56 +++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+) create mode 100644 tests/cpp/accuracy/conftest.py create mode 100644 tests/cpp/accuracy/test_bindings.py diff --git a/.github/workflows/test_accuracy.yml b/.github/workflows/test_accuracy.yml index 6cd93deb..da504c62 100644 --- a/.github/workflows/test_accuracy.yml +++ b/.github/workflows/test_accuracy.yml @@ -49,3 +49,7 @@ jobs: run: | build/test_accuracy -d data -p tests/python/accuracy/public_scope.json DATA=data build/test_YOLOv8 + - name: Run CPP-PY Bindings Test + run: | + source venv/bin/activate + PYTHONPATH="$PYTHONPATH:build/py_bindings" pytest --data=./data --config=./tests/python/accuracy/public_scope.json tests/cpp/accuracy/test_bindings.py diff --git a/tests/cpp/accuracy/conftest.py b/tests/cpp/accuracy/conftest.py new file mode 100644 index 00000000..2b3ab029 --- /dev/null +++ b/tests/cpp/accuracy/conftest.py @@ -0,0 +1,8 @@ +# +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +def pytest_addoption(parser): + parser.addoption("--data", action="store", help="data folder with dataset") + parser.addoption("--config", action="store", help="path to models config") diff --git a/tests/cpp/accuracy/test_bindings.py b/tests/cpp/accuracy/test_bindings.py new file mode 100644 index 00000000..314f2ac6 --- /dev/null +++ b/tests/cpp/accuracy/test_bindings.py @@ -0,0 +1,56 @@ +# +# Copyright (C) 2025 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# + +import pytest +import json +from pathlib import Path + +import cv2 + +from model_api.models import Model +from py_model_api import ClassificationModel + + +def read_config(models_config: str, model_type: str): + with open(models_config, "r") as f: + data = json.load(f) + for item in data: + if item["type"] == model_type: + yield item + + +@pytest.fixture(scope="session") +def data(pytestconfig) -> str: + return pytestconfig.getoption("data") + + +@pytest.fixture(scope="session") +def models_config(pytestconfig) -> str: + return pytestconfig.getoption("config") + + +@pytest.fixture() +def classification_configs(models_config: str): + return read_config(models_config, "ClassificationModel") + + +def test_classification_models(data: str, classification_configs): + for model_data in classification_configs: + name = model_data["name"] + if ".xml" not in name: + continue + if name.endswith(".xml") or name.endswith(".onnx"): + name = f"{data}/{name}" + + model = Model.create_model(name, preload=True) + cpp_model = ClassificationModel.create_model(name, preload=True) + + image_path = Path(data) / next(iter(model_data["test_data"]))["image"] + image = cv2.imread(str(image_path)) + + py_result = model(image) + cpp_result = cpp_model(image) + + assert str(py_result) == str(cpp_result) From 12c7e0db19ad03813b10a10ebba13dc94555526b Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Thu, 30 Jan 2025 02:46:51 +0900 Subject: [PATCH 02/11] Fix linter --- tests/cpp/accuracy/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/cpp/accuracy/conftest.py b/tests/cpp/accuracy/conftest.py index 2b3ab029..c05e5922 100644 --- a/tests/cpp/accuracy/conftest.py +++ b/tests/cpp/accuracy/conftest.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 # + def pytest_addoption(parser): parser.addoption("--data", action="store", help="data folder with dataset") parser.addoption("--config", action="store", help="path to models config") From 18d1cb9ee8b1419b908ed36b87d599f990469dd9 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Thu, 30 Jan 2025 03:10:55 +0900 Subject: [PATCH 03/11] Del outdated OMZ model --- tests/python/accuracy/public_scope.json | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/python/accuracy/public_scope.json b/tests/python/accuracy/public_scope.json index 47f0ea10..a71d2518 100644 --- a/tests/python/accuracy/public_scope.json +++ b/tests/python/accuracy/public_scope.json @@ -145,16 +145,6 @@ } ] }, - { - "name": "resnet-18-pytorch", - "type": "ClassificationModel", - "test_data": [ - { - "image": "coco128/images/train2017/000000000074.jpg", - "reference": ["254 (pug): 0.153, [0], [0], [0]"] - } - ] - }, { "name": "efficientnet-b0-pytorch", "type": "ClassificationModel", From 75ffbf853a7fca3fa958d61ed3b72a8aa80364ca Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 31 Jan 2025 00:00:21 +0900 Subject: [PATCH 04/11] Disable yolo python tests --- .github/workflows/test_accuracy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_accuracy.yml b/.github/workflows/test_accuracy.yml index da504c62..83dfa609 100644 --- a/.github/workflows/test_accuracy.yml +++ b/.github/workflows/test_accuracy.yml @@ -34,7 +34,7 @@ jobs: run: | source venv/bin/activate pytest --data=./data tests/python/accuracy/test_accuracy.py - DATA=data pytest --data=./data tests/python/accuracy/test_YOLOv8.py + # DATA=data pytest --data=./data tests/python/accuracy/test_YOLOv8.py - name: Install CPP dependencies run: | sudo bash src/cpp/install_dependencies.sh From 782974641ceb2f60ae6fc96c260063eed5340845 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 31 Jan 2025 00:18:26 +0900 Subject: [PATCH 05/11] Disable cpp yolo tests as well --- .github/workflows/test_accuracy.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_accuracy.yml b/.github/workflows/test_accuracy.yml index 83dfa609..cb688ccd 100644 --- a/.github/workflows/test_accuracy.yml +++ b/.github/workflows/test_accuracy.yml @@ -34,7 +34,7 @@ jobs: run: | source venv/bin/activate pytest --data=./data tests/python/accuracy/test_accuracy.py - # DATA=data pytest --data=./data tests/python/accuracy/test_YOLOv8.py + # DATA=data pytest --data=./data tests/python/accuracy/test_YOLOv8.py # checkpoints are not available sometimes - name: Install CPP dependencies run: | sudo bash src/cpp/install_dependencies.sh @@ -48,7 +48,7 @@ jobs: - name: Run CPP Test run: | build/test_accuracy -d data -p tests/python/accuracy/public_scope.json - DATA=data build/test_YOLOv8 + # DATA=data build/test_YOLOv8 # checkpoints are not available sometimes - name: Run CPP-PY Bindings Test run: | source venv/bin/activate From df10b258336a5a02417ef2a3aac7630e6731aabb Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 31 Jan 2025 01:04:11 +0900 Subject: [PATCH 06/11] Minor fixes in cpp samples --- examples/cpp/CMakeLists.txt | 2 +- examples/cpp/asynchronous_api/main.cpp | 2 +- examples/cpp/synchronous_api/main.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/cpp/CMakeLists.txt b/examples/cpp/CMakeLists.txt index 2fe5a7fd..b1ebab76 100644 --- a/examples/cpp/CMakeLists.txt +++ b/examples/cpp/CMakeLists.txt @@ -92,7 +92,7 @@ endmacro() find_package(OpenCV REQUIRED COMPONENTS imgcodecs) -set (ENABLE_PY_BINDINGS OFF) +set(ENABLE_PY_BINDINGS OFF) add_subdirectory(../../src/cpp ${Samples_BINARY_DIR}/src/cpp) add_example(NAME asynchronous_api SOURCES ./asynchronous_api/main.cpp DEPENDENCIES model_api) diff --git a/examples/cpp/asynchronous_api/main.cpp b/examples/cpp/asynchronous_api/main.cpp index acd361a2..105a5150 100644 --- a/examples/cpp/asynchronous_api/main.cpp +++ b/examples/cpp/asynchronous_api/main.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/examples/cpp/synchronous_api/main.cpp b/examples/cpp/synchronous_api/main.cpp index ec4ef869..fe3f4908 100644 --- a/examples/cpp/synchronous_api/main.cpp +++ b/examples/cpp/synchronous_api/main.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include From 9dd9fcbdea4723cfbe9c255460a39efeb9584e33 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 31 Jan 2025 01:15:04 +0900 Subject: [PATCH 07/11] Update cmake in tests --- .github/workflows/test_accuracy.yml | 2 +- tests/cpp/accuracy/CMakeLists.txt | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_accuracy.yml b/.github/workflows/test_accuracy.yml index cb688ccd..1ca375bb 100644 --- a/.github/workflows/test_accuracy.yml +++ b/.github/workflows/test_accuracy.yml @@ -52,4 +52,4 @@ jobs: - name: Run CPP-PY Bindings Test run: | source venv/bin/activate - PYTHONPATH="$PYTHONPATH:build/py_bindings" pytest --data=./data --config=./tests/python/accuracy/public_scope.json tests/cpp/accuracy/test_bindings.py + PYTHONPATH="$PYTHONPATH:build/model_api/cpp/py_bindings/" pytest --data=./data --config=./tests/python/accuracy/public_scope.json tests/cpp/accuracy/test_bindings.py diff --git a/tests/cpp/accuracy/CMakeLists.txt b/tests/cpp/accuracy/CMakeLists.txt index 57aa5705..8b4bb42a 100644 --- a/tests/cpp/accuracy/CMakeLists.txt +++ b/tests/cpp/accuracy/CMakeLists.txt @@ -63,8 +63,15 @@ FetchContent_MakeAvailable(json googletest) include(../cmake/common.cmake) +find_package(Python3 REQUIRED) +execute_process( + COMMAND ${Python3_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')" + OUTPUT_VARIABLE OpenVINO_DIR_PY + ERROR_QUIET +) + find_package(OpenCV REQUIRED COMPONENTS core highgui videoio imgproc imgcodecs) -find_package(OpenVINO REQUIRED COMPONENTS Runtime) +find_package(OpenVINO REQUIRED COMPONENTS Runtime HINTS "${OpenVINO_DIR_PY}") add_subdirectory(../../../src/cpp ${tests_BINARY_DIR}/model_api/cpp) From ae532aa271aa8412e6b457efbeb4840a3be0b84e Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 31 Jan 2025 01:38:50 +0900 Subject: [PATCH 08/11] Revert py OV in tests cmake --- tests/cpp/accuracy/CMakeLists.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/tests/cpp/accuracy/CMakeLists.txt b/tests/cpp/accuracy/CMakeLists.txt index 8b4bb42a..57aa5705 100644 --- a/tests/cpp/accuracy/CMakeLists.txt +++ b/tests/cpp/accuracy/CMakeLists.txt @@ -63,15 +63,8 @@ FetchContent_MakeAvailable(json googletest) include(../cmake/common.cmake) -find_package(Python3 REQUIRED) -execute_process( - COMMAND ${Python3_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')" - OUTPUT_VARIABLE OpenVINO_DIR_PY - ERROR_QUIET -) - find_package(OpenCV REQUIRED COMPONENTS core highgui videoio imgproc imgcodecs) -find_package(OpenVINO REQUIRED COMPONENTS Runtime HINTS "${OpenVINO_DIR_PY}") +find_package(OpenVINO REQUIRED COMPONENTS Runtime) add_subdirectory(../../../src/cpp ${tests_BINARY_DIR}/model_api/cpp) From ad192abfd433ba2b6b0b7c3d7e14a8e3374710bc Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 31 Jan 2025 02:43:05 +0900 Subject: [PATCH 09/11] Use python from venv only --- .github/workflows/test_accuracy.yml | 1 + src/cpp/CMakeLists.txt | 5 +++-- src/cpp/py_bindings/CMakeLists.txt | 5 ++--- tests/cpp/accuracy/CMakeLists.txt | 12 +++++++++++- tests/cpp/cmake/common.cmake | 5 ++--- 5 files changed, 19 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_accuracy.yml b/.github/workflows/test_accuracy.yml index 1ca375bb..dab4414d 100644 --- a/.github/workflows/test_accuracy.yml +++ b/.github/workflows/test_accuracy.yml @@ -40,6 +40,7 @@ jobs: sudo bash src/cpp/install_dependencies.sh - name: Build CPP Test run: | + source venv/bin/activate pip install nanobind==2.4.0 pip install typing_extensions==4.12.2 mkdir build && cd build diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 44e9b057..6d864d2c 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -35,9 +35,10 @@ find_package(OpenCV REQUIRED COMPONENTS core imgproc) # Looking for OpenVINO in the python distribution. It doesn't work for cross-compiling build if(NOT CMAKE_CROSSCOMPILING) - find_package(Python3 REQUIRED) + set(Python_FIND_VIRTUALENV ONLY) + find_package(Python COMPONENTS Interpreter Development REQUIRED) execute_process( - COMMAND ${Python3_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')" + COMMAND ${Python_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')" OUTPUT_VARIABLE OpenVINO_DIR_PY ERROR_QUIET ) diff --git a/src/cpp/py_bindings/CMakeLists.txt b/src/cpp/py_bindings/CMakeLists.txt index 91eb70ce..a7c40650 100644 --- a/src/cpp/py_bindings/CMakeLists.txt +++ b/src/cpp/py_bindings/CMakeLists.txt @@ -2,9 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 # -set(DEV_MODULE Development.Module) - -find_package(Python COMPONENTS Interpreter ${DEV_MODULE} REQUIRED) +set(Python_FIND_VIRTUALENV ONLY) +find_package(Python COMPONENTS Interpreter Development REQUIRED) execute_process( COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir diff --git a/tests/cpp/accuracy/CMakeLists.txt b/tests/cpp/accuracy/CMakeLists.txt index 57aa5705..69ef04fa 100644 --- a/tests/cpp/accuracy/CMakeLists.txt +++ b/tests/cpp/accuracy/CMakeLists.txt @@ -64,7 +64,17 @@ FetchContent_MakeAvailable(json googletest) include(../cmake/common.cmake) find_package(OpenCV REQUIRED COMPONENTS core highgui videoio imgproc imgcodecs) -find_package(OpenVINO REQUIRED COMPONENTS Runtime) + +set(Python_FIND_VIRTUALENV ONLY) +find_package(Python REQUIRED) +execute_process( + COMMAND ${Python_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')" + OUTPUT_VARIABLE OpenVINO_DIR_PY + ERROR_QUIET +) +find_package(OpenVINO REQUIRED + COMPONENTS Runtime Threading + HINTS "${OpenVINO_DIR_PY}") add_subdirectory(../../../src/cpp ${tests_BINARY_DIR}/model_api/cpp) diff --git a/tests/cpp/cmake/common.cmake b/tests/cpp/cmake/common.cmake index 8a931e14..dff0a621 100644 --- a/tests/cpp/cmake/common.cmake +++ b/tests/cpp/cmake/common.cmake @@ -35,13 +35,12 @@ macro(add_test) target_include_directories(${TEST_NAME} PRIVATE ${TEST_INCLUDE_DIRECTORIES}) endif() - target_link_libraries(${TEST_NAME} PRIVATE ${OpenCV_LIBRARIES} openvino::runtime ${TEST_DEPENDENCIES}) + target_link_libraries(${TEST_NAME} PRIVATE ${OpenCV_LIBRARIES} ${TEST_DEPENDENCIES}) if(UNIX) target_link_libraries(${TEST_NAME} PRIVATE pthread) endif() - target_link_libraries(${TEST_NAME} PRIVATE gtest gtest_main) - target_link_libraries(${TEST_NAME} PRIVATE nlohmann_json::nlohmann_json) + target_link_libraries(${TEST_NAME} PRIVATE gtest_main gmock_main nlohmann_json::nlohmann_json) endmacro() From 6d2de0c1bd66028f375b2219ef660a98d9fd9c27 Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Fri, 31 Jan 2025 02:55:25 +0900 Subject: [PATCH 10/11] Specify python dir manually --- .github/workflows/test_accuracy.yml | 2 +- src/cpp/CMakeLists.txt | 2 +- src/cpp/py_bindings/CMakeLists.txt | 2 +- tests/cpp/accuracy/CMakeLists.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_accuracy.yml b/.github/workflows/test_accuracy.yml index dab4414d..4c05f79f 100644 --- a/.github/workflows/test_accuracy.yml +++ b/.github/workflows/test_accuracy.yml @@ -44,7 +44,7 @@ jobs: pip install nanobind==2.4.0 pip install typing_extensions==4.12.2 mkdir build && cd build - cmake ../tests/cpp/accuracy/ + cmake ../tests/cpp/accuracy/ -DPython_ROOT_DIR=$(python -c "import sys; print(sys.prefix)") make -j - name: Run CPP Test run: | diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 6d864d2c..971e650c 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -35,7 +35,7 @@ find_package(OpenCV REQUIRED COMPONENTS core imgproc) # Looking for OpenVINO in the python distribution. It doesn't work for cross-compiling build if(NOT CMAKE_CROSSCOMPILING) - set(Python_FIND_VIRTUALENV ONLY) + set(Python_FIND_VIRTUALENV FIRST) find_package(Python COMPONENTS Interpreter Development REQUIRED) execute_process( COMMAND ${Python_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')" diff --git a/src/cpp/py_bindings/CMakeLists.txt b/src/cpp/py_bindings/CMakeLists.txt index a7c40650..b955b8c4 100644 --- a/src/cpp/py_bindings/CMakeLists.txt +++ b/src/cpp/py_bindings/CMakeLists.txt @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # -set(Python_FIND_VIRTUALENV ONLY) +set(Python_FIND_VIRTUALENV FIRST) find_package(Python COMPONENTS Interpreter Development REQUIRED) execute_process( diff --git a/tests/cpp/accuracy/CMakeLists.txt b/tests/cpp/accuracy/CMakeLists.txt index 69ef04fa..fb79fe79 100644 --- a/tests/cpp/accuracy/CMakeLists.txt +++ b/tests/cpp/accuracy/CMakeLists.txt @@ -65,7 +65,7 @@ include(../cmake/common.cmake) find_package(OpenCV REQUIRED COMPONENTS core highgui videoio imgproc imgcodecs) -set(Python_FIND_VIRTUALENV ONLY) +set(Python_FIND_VIRTUALENV FIRST) find_package(Python REQUIRED) execute_process( COMMAND ${Python_EXECUTABLE} -c "from openvino.utils import get_cmake_path; print(get_cmake_path(), end='')" From 16c7bbd1fce7f025ec4f694dc18d1b6f2c46f5db Mon Sep 17 00:00:00 2001 From: Vladisalv Sovrasov Date: Sat, 8 Feb 2025 00:11:24 +0900 Subject: [PATCH 11/11] Enable bindings in cpp accuracy tests --- tests/cpp/accuracy/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cpp/accuracy/CMakeLists.txt b/tests/cpp/accuracy/CMakeLists.txt index c06077ba..2ccbac9a 100644 --- a/tests/cpp/accuracy/CMakeLists.txt +++ b/tests/cpp/accuracy/CMakeLists.txt @@ -65,7 +65,7 @@ include(../cmake/common.cmake) find_package(OpenCV REQUIRED COMPONENTS core highgui videoio imgproc imgcodecs) -set(ENABLE_PY_BINDINGS OFF) +set(ENABLE_PY_BINDINGS ON) add_subdirectory(../../../src/cpp ${tests_BINARY_DIR}/model_api/cpp) add_test(NAME test_accuracy SOURCES test_accuracy.cpp DEPENDENCIES model_api)