Exclude Tokenizers extension (openvinotoolkit#887)

* remove tokenizers code

* ci cleanup

* 3rd-parties cleanup

* Revert "ci cleanup"

This reverts commit f9038b2.

* Revert README

This reverts commit 425bb0e.

Author: mryzhov

.ci/azure/linux.yml

@@ -180,46 +180,6 @@ jobs:
       # need to enable sparse_conv tests with new Open3D release
       python -m pytest -k "not sparse_conv" tests/run_tests.py
     env:
-      CUSTOM_OP_LIB: $(BIN_DIR)/libopenvino_tokenizers.so
+      CUSTOM_OP_LIB: $(BIN_DIR)/libuser_ov_extensions.so
     workingDirectory: $(REPO_DIR)/modules/custom_operations
     displayName: 'Custom user operation tests'
-
-  - script: |
-      set -e
-      source $(WORK_DIR)/.env3/bin/activate && source $(SETUPVARS)
-      python -m pip install build
-      python -m build --wheel --outdir $(BUILD_WHEEL_DIR) $(REPO_DIR)/modules/custom_operations
-    env:
-      CMAKE_ARGS: '-DBUILD_FAST_TOKENIZERS=ON'
-      CMAKE_GENERATOR: 'Unix Makefiles'
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Build tokenizers wheel'
-
-  - publish: $(BUILD_WHEEL_DIR)
-    artifact: Wheels
-
-  - script: |
-      set -e
-      source $(WORK_DIR)/.env3/bin/activate
-      # Find and install wheel
-      pushd $(BUILD_WHEEL_DIR)
-        wheel_name=$(find . -name 'openvino_tokenizers*.whl')
-        python3 -m pip install $wheel_name[all]
-      popd
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Install tokenizers wheel'
-
-  - script: |
-      set -e
-      source $(WORK_DIR)/.env3/bin/activate
-      bandit -c pyproject.toml -r user_ie_extensions/tokenizer/python/
-    workingDirectory: $(REPO_DIR)/modules/custom_operations
-    displayName: 'Tokenizers Bandit Check'
-
-  - script: |
-      set -e
-      source $(WORK_DIR)/.env3/bin/activate
-      python -m pytest tokenizers_test.py
-    continueOnError: true
-    workingDirectory: $(REPO_DIR)/modules/custom_operations/user_ie_extensions/tokenizer/python/tests/
-    displayName: 'Tokenizers extension regression test'

.ci/azure/mac.yml

@@ -179,32 +179,5 @@ jobs:
     workingDirectory: $(WORK_DIR)
     displayName: 'Create virtual env'
 
-  - script: |
-      set -e
-      source $(WORK_DIR)/venv/bin/activate && source $(SETUPVARS)
-      python -m pip install build
-      python -m build --wheel --outdir $(BUILD_WHEEL_DIR) $(REPO_DIR)/modules/custom_operations
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Build tokenizers wheel'
-
   - publish: $(BUILD_WHEEL_DIR)
     artifact: Wheels
-
-  - script: |
-      set -e
-      source $(WORK_DIR)/venv/bin/activate
-      # Find and install wheel
-      pushd $(BUILD_WHEEL_DIR)
-        wheel_name=$(find . -name 'openvino_tokenizers*.whl')
-        python3 -m pip install $wheel_name[all]
-      popd
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Install tokenizers wheel'
-
-  - script: |
-      set -e
-      source $(WORK_DIR)/venv/bin/activate
-      python -m pytest tokenizers_test.py
-    continueOnError: true
-    workingDirectory: $(REPO_DIR)/modules/custom_operations/user_ie_extensions/tokenizer/python/tests/
-    displayName: 'Tokenizers extension regression test'

.ci/azure/windows.yml

@@ -178,37 +178,6 @@ jobs:
       $(PYTHON_EXE) -m pip install -r $(REPO_DIR)\modules\custom_operations\tests\requirements.txt
       $(PYTHON_EXE) -m pytest -k "not sparse_conv" tests\run_tests.py
     env:
-        CUSTOM_OP_LIB: $(BUILD_DIR_CONTRIB)\user_ie_extensions\openvino_tokenizers.dll
+        CUSTOM_OP_LIB: $(BUILD_DIR_CONTRIB)\user_ie_extensions\user_ov_extensions.dll
     workingDirectory: $(REPO_DIR)\modules\custom_operations
     displayName: 'Custom user operation tests'
-
-  - script: |
-      call $(SETUPVARS)
-      $(PYTHON_EXE) -m pip install build
-      $(PYTHON_EXE) -m build --wheel --outdir $(BUILD_WHEEL_DIR) $(REPO_DIR)\modules\custom_operations
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Build tokenizers wheel'
-
-  - publish: $(BUILD_WHEEL_DIR)
-    artifact: Wheels
-
-  - powershell: |
-      # Find and install the core OV wheel
-      $ovCoreWheelPath=Get-ChildItem -Path "$(INSTALL_DIR)\tools" -Filter openvino-*.whl | % { $_.FullName }
-      $(PYTHON_EXE) -m pip install "$ovCoreWheelPath"
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Install OpenVINO Python wheels'
-
-  - powershell: |
-      # Find and install the wheel
-      $wheelPath=Get-ChildItem -Path "$(BUILD_WHEEL_DIR)" -Filter openvino_tokenizers*.whl | % { $_.FullName }
-      $(PYTHON_EXE) -m pip install "$wheelPath[all]"
-    workingDirectory: $(WORK_DIR)
-    displayName: 'Install tokenizers wheel'
-
-  - script: |
-      call $(SETUPVARS) -pyver 3.8 && ^
-      $(PYTHON_EXE) -m pytest tokenizers_test.py
-    continueOnError: true
-    workingDirectory: $(REPO_DIR)\modules\custom_operations\user_ie_extensions\tokenizer\python\tests
-    displayName: 'Tokenizers extension regression test'

modules/custom_operations/CMakeLists.txt

@@ -8,14 +8,4 @@ project(openvino_extensions)
 
 include(cmake/platforms.cmake)
 
-# Find OpenVINODeveloperPackage first to compile with SDL flags
-find_package(OpenVINODeveloperPackage QUIET)
-if(NOT OpenVINODeveloperPackage_FOUND)
-    find_package(OpenVINO REQUIRED COMPONENTS Runtime)
-endif()
-
-# set the project version using OpenVINO major, minor and patch versions
-set(CMAKE_PROJECT_VERSION_TWEAK 0)
-set(CMAKE_PROJECT_VERSION "${OpenVINO_VERSION}.${CMAKE_PROJECT_VERSION_TWEAK}")
-
 add_subdirectory(user_ie_extensions)