Replace openvino_contrib with openvino_tokenizers (openvinotoolkit#231)

Close openvinotoolkit#228

Author: Wovchena

.github/workflows/causal_lm_cpp.yml

@@ -5,7 +5,7 @@ on:
       - .github/workflows/causal_lm_cpp.yml
       - llm_bench/python/**
       - text_generation/causal_lm/cpp/*
-      - thirdparty/openvino_contrib
+      - thirdparty/openvino_tokenizers
       - '!**.md'
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -28,7 +28,7 @@ jobs:
       - name: Download, convert and build
         run: |
           source ./ov/setupvars.sh
-          python -m pip install --upgrade-strategy eager "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_contrib/modules/custom_operations/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu && python ./llm_bench/python/convert.py --model_id openlm-research/open_llama_3b_v2 --output_dir ./open_llama_3b_v2/ --precision FP16 --stateful &
+          python -m pip install --upgrade-strategy eager "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu && python ./llm_bench/python/convert.py --model_id openlm-research/open_llama_3b_v2 --output_dir ./open_llama_3b_v2/ --precision FP16 --stateful &
           cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
           cmake --build ./build/ --config Release -j
           wait
@@ -54,7 +54,7 @@ jobs:
       - name: Download, convert and build
         run: |
           source ./ov/setupvars.sh
-          python -m pip install --upgrade-strategy eager transformers==4.35.2 "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_contrib/modules/custom_operations/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu && python ./llm_bench/python/convert.py --model_id TinyLlama/TinyLlama-1.1B-Chat-v1.0 --output_dir ./TinyLlama-1.1B-Chat-v1.0/ --precision FP16 --stateful &
+          python -m pip install --upgrade-strategy eager "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu && python ./llm_bench/python/convert.py --model_id TinyLlama/TinyLlama-1.1B-Chat-v1.0 --output_dir ./TinyLlama-1.1B-Chat-v1.0/ --precision FP16 --stateful &
           cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
           cmake --build ./build/ --config Release -j
           wait
@@ -94,38 +94,6 @@ jobs:
               predictions = predictions[:idx] + predictions[idx + len(ref):]
           "
           echo Hi passed
-
-          timeout 25s ./build/beam_search_causal_lm ./TinyLlama-1.1B-Chat-v1.0/pytorch/dldt/FP16/ "return 0" > ./pred.txt
-          python -c "
-          import transformers
-          with open('pred.txt', 'r') as file:
-              predictions = file.read()
-          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
-          tokenized = tokenizer('return 0', return_tensors='pt')
-          for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
-              ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) + '\n'
-              idx = predictions.find(ref)
-              if -1 == idx:
-                  raise RuntimeError(f'Missing "{ref=}" from predictions')
-              predictions = predictions[:idx] + predictions[idx + len(ref):]
-          "
-          echo return 0 passed
-
-          ./build/beam_search_causal_lm ./TinyLlama-1.1B-Chat-v1.0/pytorch/dldt/FP16/ "你好！ 你好嗎？" > ./pred.txt
-          python -c "
-          import transformers
-          with open('pred.txt', 'r') as file:
-              predictions = file.read()
-          tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0')
-          tokenized = tokenizer('你好！ 你好嗎？', return_tensors='pt')
-          for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False):
-              ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) + '\n'
-              idx = predictions.find(ref)
-              if -1 == idx:
-                  raise RuntimeError(f'Missing "{ref=}" from predictions')
-              predictions = predictions[:idx] + predictions[idx + len(ref):]
-          "
-          echo 你好！ 你好嗎？ passed
   cpp-beam_search_causal_lm-windows:
     runs-on: windows-latest
     steps:
@@ -144,7 +112,7 @@ jobs:
         shell: cmd
         run: |
           call w_openvino_toolkit_windows_2023.3.0.13775.ceeafaf64f3_x86_64\setupvars.bat
-          python -m pip install --upgrade-strategy eager transformers==4.35.2 "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_contrib/modules/custom_operations/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu
+          python -m pip install --upgrade-strategy eager "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu
           python ./llm_bench/python/convert.py --model_id TinyLlama/TinyLlama-1.1B-Chat-v1.0 --output_dir ./TinyLlama-1.1B-Chat-v1.0/ --precision FP16 --stateful
           cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
           cmake --build ./build/ --config Release -j
@@ -183,7 +151,7 @@ jobs:
       - name: Download, convert and build
         run: |
           source ./ov/setupvars.sh
-          python -m pip install --upgrade-strategy eager transformers==4.35.2 "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_contrib/modules/custom_operations/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu && python ./llm_bench/python/convert.py --model_id Qwen/Qwen-7B-Chat --output_dir ./Qwen-7B-Chat/ --precision FP16 --stateful &
+          python -m pip install --upgrade-strategy eager "optimum>=1.14" -r ./llm_bench/python/requirements.txt ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://download.pytorch.org/whl/cpu && python ./llm_bench/python/convert.py --model_id Qwen/Qwen-7B-Chat --output_dir ./Qwen-7B-Chat/ --precision FP16 --stateful &
           cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
           cmake --build ./build/ --config Release -j
           wait

.github/workflows/lcm_dreamshaper_cpp.yml

@@ -5,7 +5,7 @@ on:
       - image_generation/lcm_dreamshaper_v7/cpp/**
       - image_generation/common/**
       - .github/workflows/lcm_dreamshaper_cpp.yml
-      - thirdparty/openvino_contrib
+      - thirdparty/openvino_tokenizers
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -30,7 +30,7 @@ jobs:
           cd ./image_generation/lcm_dreamshaper_v7/cpp/scripts/
           python -m pip install -U pip
           python -m pip install -r ./requirements.txt
-          python -m pip install ../../../../thirdparty/openvino_contrib/modules/custom_operations/
+          python -m pip install ../../../../thirdparty/openvino_tokenizers/
           python convert_model.py -lcm "SimianLuo/LCM_Dreamshaper_v7" -t "FP16"
       - name: Build app
         run: |
@@ -63,7 +63,7 @@ jobs:
             call w_openvino_toolkit_windows_2023.3.0.13775.ceeafaf64f3_x86_64/setupvars.bat
             cd ./image_generation/lcm_dreamshaper_v7/cpp/scripts/
             python -m pip install -r ./requirements.txt
-            python -m pip install ../../../../thirdparty/openvino_contrib/modules/custom_operations/
+            python -m pip install ../../../../thirdparty/openvino_tokenizers/
             python convert_model.py -lcm "SimianLuo/LCM_Dreamshaper_v7" -t "FP16"
         - name: Build app
           shell: cmd

.github/workflows/stable_diffusion_1_5_cpp.yml

@@ -5,7 +5,7 @@ on:
       - image_generation/stable_diffusion_1_5/cpp/**
       - image_generation/common/**
       - .github/workflows/stable_diffusion_1_5_cpp.yml
-      - thirdparty/openvino_contrib
+      - thirdparty/openvino_tokenizers
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -32,7 +32,7 @@ jobs:
           cd ./image_generation/stable_diffusion_1_5/cpp/scripts/
           python -m pip install -U pip
           python -m pip install -r ./requirements.txt
-          python -m pip install ../../../../thirdparty/openvino_contrib/modules/custom_operations/
+          python -m pip install ../../../../thirdparty/openvino_tokenizers/
           python convert_model.py -sd runwayml/stable-diffusion-v1-5 -b 1 -t FP16 -dyn True
       - name: Build app
         run: |
@@ -67,7 +67,7 @@ jobs:
             call w_openvino_toolkit_windows_2023.3.0.13775.ceeafaf64f3_x86_64/setupvars.bat
             cd ./image_generation/stable_diffusion_1_5/cpp/scripts/
             python -m pip install -r ./requirements.txt
-            python -m pip install ../../../../thirdparty/openvino_contrib/modules/custom_operations/
+            python -m pip install ../../../../thirdparty/openvino_tokenizers/
             python convert_model.py -sd runwayml/stable-diffusion-v1-5 -b 1 -t FP16 -dyn True
         - name: Build app
           shell: cmd

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "thirdparty/openvino_contrib"]
-    path = thirdparty/openvino_contrib
-    url = https://github.com/openvinotoolkit/openvino_contrib.git
+[submodule "thirdparty/openvino_tokenizers"]
+    path = thirdparty/openvino_tokenizers
+    url = https://github.com/openvinotoolkit/openvino_tokenizers.git

image_generation/lcm_dreamshaper_v7/cpp/CMakeLists.txt

@@ -27,9 +27,7 @@ FetchContent_MakeAvailable(cxxopts)
 
 add_subdirectory(../../common/imwrite _deps/imwrite)
 add_subdirectory(../../common/diffusers _deps/diffusers)
-
-set(CUSTOM_OPERATIONS tokenizer)
-add_subdirectory(../../../thirdparty/openvino_contrib/modules/custom_operations/ _deps/tokenizers)
+add_subdirectory(../../../thirdparty/openvino_tokenizers/ _deps/tokenizers)
 
 # create executable
 
@@ -41,6 +39,6 @@ target_link_libraries(${PROJECT_NAME} PRIVATE
     diffusers::diffusers
     imwrite::imwrite)
 
-add_dependencies(${PROJECT_NAME} user_ov_extensions)
+add_dependencies(${PROJECT_NAME} openvino_tokenizers)
 
-target_compile_definitions(${PROJECT_NAME} PRIVATE TOKENIZERS_LIBRARY_PATH=\"$<TARGET_FILE:user_ov_extensions>\")
+target_compile_definitions(${PROJECT_NAME} PRIVATE TOKENIZERS_LIBRARY_PATH=\"$<TARGET_FILE:openvino_tokenizers>\")

image_generation/lcm_dreamshaper_v7/cpp/README.md

@@ -1,5 +1,5 @@
 # OpenVINO Latent Consistency Model C++ image generation  pipeline
-The pure C++ text-to-image pipeline, driven by the OpenVINO native API for SD v1.5 Latent Consistency Model with LCM Scheduler. It includes advanced features like LoRA integration with safetensors and [OpenVINO extension for tokenizers](https://github.com/openvinotoolkit/openvino_contrib/blob/master/modules/custom_operations/user_ie_extensions/tokenizer/python/README.md). Loading `user_ov_extensions` provided by `openvino-tokenizers` to `ov::Core` enables tokenization. [The common folder](../../common/) contains schedulers for image generation and `imwrite()` for saving `bmp` images. This demo has been tested for Linux platform only. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/blob/main/notebooks/263-latent-consistency-models-image-generation/263-lcm-lora-controlnet.ipynb) which provides an example of image generaztion in Python.
+The pure C++ text-to-image pipeline, driven by the OpenVINO native API for SD v1.5 Latent Consistency Model with LCM Scheduler. It includes advanced features like LoRA integration with safetensors and [OpenVINO Tokenizers](https://github.com/openvinotoolkit/openvino_tokenizers). Loading `openvino_tokenizers` to `ov::Core` enables tokenization. [The common folder](../../common/) contains schedulers for image generation and `imwrite()` for saving `bmp` images. This demo has been tested for Linux platform only. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/blob/main/notebooks/263-latent-consistency-models-image-generation/263-lcm-lora-controlnet.ipynb) which provides an example of image generaztion in Python.
 
 > [!NOTE]
 >This tutorial assumes that the current working directory is `<openvino.genai repo>/image_generation/lcm_dreamshaper_v7/cpp/` and all paths are relative to this folder.
@@ -26,7 +26,7 @@ conda install -c conda-forge openvino c-compiler cxx-compiler make
     ```shell
     conda activate openvino_lcm_cpp
     python -m pip install -r scripts/requirements.txt
-    python -m pip install ../../../thirdparty/openvino_contrib/modules/custom_operations/[transformers]
+    python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
     ```
 
 2. Run model conversion script to download and convert PyTorch model to OpenVINO IR via [optimum-intel](https://github.com/huggingface/optimum-intel). Please, use the script `scripts/convert_model.py` to convert the model:
@@ -80,7 +80,7 @@ Usage:
 * `-h, --help`          Print usage
 
 > [!NOTE]
-> The tokenizer model will always be loaded to CPU: [OpenVINO tokenizers](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/custom_operations/user_ie_extensions/tokenizer/python#readme) can be inferred on a CPU device only.
+> The tokenizer model will always be loaded to CPU: [OpenVINO Tokenizers](https://github.com/openvinotoolkit/openvino_tokenizers) can be inferred on a CPU device only.
 
 Example:
 

image_generation/stable_diffusion_1_5/cpp/CMakeLists.txt

@@ -27,9 +27,7 @@ FetchContent_MakeAvailable(cxxopts)
 
 add_subdirectory(../../common/imwrite _deps/imwrite)
 add_subdirectory(../../common/diffusers _deps/diffusers)
-
-set(CUSTOM_OPERATIONS tokenizer)
-add_subdirectory(../../../thirdparty/openvino_contrib/modules/custom_operations/ _deps/tokenizers)
+add_subdirectory(../../../thirdparty/openvino_tokenizers/ _deps/tokenizers)
 
 # create executable
 
@@ -45,6 +43,6 @@ target_link_libraries(${PROJECT_NAME} PRIVATE
     diffusers::diffusers
     imwrite::imwrite)
 
-add_dependencies(${PROJECT_NAME} user_ov_extensions)
+add_dependencies(${PROJECT_NAME} openvino_tokenizers)
 
-target_compile_definitions(${PROJECT_NAME} PRIVATE TOKENIZERS_LIBRARY_PATH=\"$<TARGET_FILE:user_ov_extensions>\")
+target_compile_definitions(${PROJECT_NAME} PRIVATE TOKENIZERS_LIBRARY_PATH=\"$<TARGET_FILE:openvino_tokenizers>\")

image_generation/stable_diffusion_1_5/cpp/README.md

@@ -1,5 +1,5 @@
 # OpenVINO Stable Diffusion (with LoRA) C++ image generation pipeline
-The pure C++ text-to-image pipeline, driven by the OpenVINO native C++ API for Stable Diffusion v1.5 with LMS Discrete Scheduler, supports both static and dynamic model inference. It includes advanced features like [LoRA](https://huggingface.co/docs/peft/conceptual_guides/lora) integration with [safetensors](https://huggingface.co/docs/safetensors/index#format) and [OpenVINO extension for tokenizers](https://github.com/openvinotoolkit/openvino_contrib/blob/master/modules/custom_operations/user_ie_extensions/tokenizer/python/README.md). Loading `user_ov_extensions` provided by `openvino-tokenizers` to `ov::Core` enables tokenization. The sample uses [diffusers](../../common/diffusers) for image generation and [imwrite](../../common/imwrite) for saving `.bmp` images. This demo has been tested on Windows and Unix platforms. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/blob/main/notebooks/225-stable-diffusion-text-to-image/225-stable-diffusion-text-to-image.ipynb) which provides an example of image generation in Python.
+The pure C++ text-to-image pipeline, driven by the OpenVINO native C++ API for Stable Diffusion v1.5 with LMS Discrete Scheduler, supports both static and dynamic model inference. It includes advanced features like [LoRA](https://huggingface.co/docs/peft/conceptual_guides/lora) integration with [safetensors](https://huggingface.co/docs/safetensors/index#format) and [OpenVINO Tokenizers](https://github.com/openvinotoolkit/openvino_tokenizers). Loading `openvino_tokenizers` to `ov::Core` enables tokenization. The sample uses [diffusers](../../common/diffusers) for image generation and [imwrite](../../common/imwrite) for saving `.bmp` images. This demo has been tested on Windows and Unix platforms. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/blob/main/notebooks/225-stable-diffusion-text-to-image/225-stable-diffusion-text-to-image.ipynb) which provides an example of image generation in Python.
 
 > [!NOTE]
 >This tutorial assumes that the current working directory is `<openvino.genai repo>/image_generation/stable_diffusion_1_5/cpp/` and all paths are relative to this folder.
@@ -25,7 +25,7 @@ conda install openvino c-compiler cxx-compiler make
 ```shell
 conda activate openvino_sd_cpp
 python -m pip install -r scripts/requirements.txt
-python -m pip install ../../../thirdparty/openvino_contrib/modules/custom_operations/[transformers]
+python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
 ```
 2. Download a huggingface SD v1.5 model like:
 - [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5)
@@ -92,7 +92,7 @@ Usage:
 * `-h, --help`          Print usage
 
 > [!NOTE]
-> The tokenizer model will always be loaded to CPU: [OpenVINO tokenizers](https://github.com/openvinotoolkit/openvino_contrib/tree/master/modules/custom_operations/user_ie_extensions/tokenizer/python#readme) can be inferred on a CPU device only.
+> The tokenizer model will always be loaded to CPU: [OpenVINO Tokenizers](https://github.com/openvinotoolkit/openvino_tokenizers) can be inferred on a CPU device only.
 
 #### Examples
 

text_generation/causal_lm/cpp/CMakeLists.txt

@@ -4,19 +4,17 @@
 cmake_minimum_required(VERSION 3.15)
 project(causal_lm)
 
-# Build user_ov_extensions
-list(APPEND CUSTOM_OPERATIONS tokenizer)
-add_subdirectory(../../../thirdparty/openvino_contrib/modules/custom_operations/ "${CMAKE_CURRENT_BINARY_DIR}/custom_operations/")
+add_subdirectory(../../../thirdparty/openvino_tokenizers/ "${CMAKE_CURRENT_BINARY_DIR}/openvino_tokenizers/")
 
 add_executable(greedy_causal_lm greedy_causal_lm.cpp)
-target_compile_definitions(greedy_causal_lm PRIVATE USER_OV_EXTENSIONS_PATH=\"$<TARGET_FILE:user_ov_extensions>\")
+target_compile_definitions(greedy_causal_lm PRIVATE OPENVINO_TOKENIZERS_PATH=\"$<TARGET_FILE:openvino_tokenizers>\")
 find_package(OpenVINO REQUIRED COMPONENTS Runtime)
 target_link_libraries(greedy_causal_lm PRIVATE openvino::runtime)
 set_target_properties(greedy_causal_lm PROPERTIES CXX_STANDARD 17)
 set_target_properties(greedy_causal_lm PROPERTIES CXX_STANDARD_REQUIRED ON)
 
 add_executable(beam_search_causal_lm beam_search_causal_lm.cpp)
-target_compile_definitions(beam_search_causal_lm PRIVATE USER_OV_EXTENSIONS_PATH=\"$<TARGET_FILE:user_ov_extensions>\")
+target_compile_definitions(beam_search_causal_lm PRIVATE OPENVINO_TOKENIZERS_PATH=\"$<TARGET_FILE:openvino_tokenizers>\")
 target_include_directories(beam_search_causal_lm PRIVATE ./)
 find_package(OpenVINO REQUIRED COMPONENTS Runtime)
 target_link_libraries(beam_search_causal_lm PRIVATE openvino::runtime)