Skip to content

Commit 5b9195c

Browse files
WovchenaWovchena
authored
Remove convert_tokenizer (#425)
1 parent 387c728 commit 5b9195c

File tree

12 files changed

+27
-60
lines changed

12 files changed

+27
-60
lines changed

‎.github/workflows/causal_lm_cpp.yml

+1-11
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,9 @@ jobs:
3333
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model openlm-research/open_llama_3b_v2 open_llama_3b_v2
3434
cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
3535
cmake --build ./build/ --config Release -j
36-
- name: convert_tokenizer and run
36+
- name: greedy_causal_lm
3737
run: |
3838
source ./ov/setupvars.sh
39-
convert_tokenizer ./open_llama_3b_v2/ --output ./open_llama_3b_v2/ --with-detokenizer
4039
./build/greedy_causal_lm ./open_llama_3b_v2/ "return 0"
4140
4241
cpp-beam_search_causal_lm-ubuntu:
@@ -64,7 +63,6 @@ jobs:
6463
- name: Compare
6564
run: |
6665
source ./ov/setupvars.sh
67-
convert_tokenizer ./TinyLlama-1.1B-Chat-v1.0/ --output ./TinyLlama-1.1B-Chat-v1.0/ --with-detokenizer
6866
6967
timeout 25s ./build/beam_search_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ 69 > ./pred.txt
7068
python -c "
@@ -157,7 +155,6 @@ jobs:
157155
shell: cmd
158156
run: |
159157
call w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64\setupvars.bat
160-
convert_tokenizer .\TinyLlama-1.1B-Chat-v1.0\ --output .\TinyLlama-1.1B-Chat-v1.0\ --with-detokenizer
161158
162159
.\build\Release\beam_search_causal_lm.exe .\TinyLlama-1.1B-Chat-v1.0\ "69" > .\pred.txt
163160
echo import transformers > ref.py
@@ -197,7 +194,6 @@ jobs:
197194
- name: Compare
198195
run: |
199196
source ./ov/setupvars.sh
200-
convert_tokenizer Qwen/Qwen-7B-Chat --output ./Qwen-7B-Chat/ --with-detokenizer --trust-remote-code
201197
timeout 50s ./build/beam_search_causal_lm ./Qwen-7B-Chat/ 69 > ./pred.txt
202198
203199
cpp-beam_search_causal_lm-Qwen1_5-7B-Chat:
@@ -225,7 +221,6 @@ jobs:
225221
- name: Run
226222
run: |
227223
source ./ov/setupvars.sh
228-
convert_tokenizer ./Qwen1.5-7B-Chat/ --output ./Qwen1.5-7B-Chat/ --with-detokenizer --trust-remote-code
229224
timeout 50s ./build/beam_search_causal_lm ./Qwen1.5-7B-Chat/ "你好！" > ./pred_qwen15.txt
230225
231226
cpp-beam_search_causal_lm-Phi-2:
@@ -253,7 +248,6 @@ jobs:
253248
- name: Compare
254249
run: |
255250
source ./ov/setupvars.sh
256-
convert_tokenizer ./phi-2/ --output ./phi-2/ --with-detokenizer --trust-remote-code
257251
timeout 50s ./build/beam_search_causal_lm ./phi-2/ 69 > ./pred.txt
258252
259253
cpp-beam_search_causal_lm-notus-7b-v1:
@@ -281,7 +275,6 @@ jobs:
281275
- name: Compare
282276
run: |
283277
source ./ov/setupvars.sh
284-
convert_tokenizer ./notus-7b-v1/ --output ./notus-7b-v1/ --with-detokenizer --trust-remote-code
285278
timeout 50s ./build/beam_search_causal_lm ./notus-7b-v1/ 69 > ./pred.txt
286279
287280
cpp-speculative_decoding_lm-ubuntu:
@@ -305,8 +298,6 @@ jobs:
305298
python -m pip install ./thirdparty/openvino_tokenizers/[transformers]
306299
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-3b dolly-v2-3b
307300
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-7b dolly-v2-7b
308-
convert_tokenizer ./dolly-v2-3b/ --output ./dolly-v2-3b/ --with-detokenizer
309-
convert_tokenizer ./dolly-v2-7b/ --output ./dolly-v2-7b/ --with-detokenizer
310301
cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
311302
cmake --build ./build/ --config Release -j
312303
- name: run and compare
@@ -348,7 +339,6 @@ jobs:
348339
- name: Run Generation
349340
run: |
350341
source ./ov/setupvars.sh
351-
convert_tokenizer ./phi-1_5/ --output ./phi-1_5/ --with-detokenizer --trust-remote-code
352342
timeout 50s ./build/greedy_causal_lm ./phi-1_5/ "Alan Turing was a" > ./pred_greedy.txt
353343
timeout 50s ./build/beam_search_causal_lm ./phi-1_5/ "Alan Turing was a" > ./pred_beam.txt
354344
- name: Compare

‎.github/workflows/lcm_dreamshaper_cpp.yml

+6-12
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ jobs:
3939
run: |
4040
conda activate openvino_lcm_cpp
4141
conda update -c conda-forge --all
42-
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
42+
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
4343
conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
4444
4545
- name: Install python dependencies
@@ -53,9 +53,7 @@ jobs:
5353
working-directory: ${{ env.working_directory }}
5454
run: |
5555
conda activate openvino_lcm_cpp
56-
export MODEL_PATH="models/lcm_dreamshaper_v7/FP16"
57-
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $MODEL_PATH
58-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
56+
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 models/lcm_dreamshaper_v7/FP16
5957
6058
- name: Build app
6159
working-directory: ${{ env.working_directory }}
@@ -66,8 +64,7 @@ jobs:
6664
6765
- name: Run app
6866
working-directory: ${{ env.working_directory }}
69-
run: |
70-
./build/lcm_dreamshaper
67+
run: ./build/lcm_dreamshaper
7168

7269
lcm_dreamshaper_v7_cpp-windows:
7370
runs-on: windows-latest
@@ -87,7 +84,7 @@ jobs:
8784
run: |
8885
conda activate openvino_lcm_cpp
8986
conda update -c conda-forge --all
90-
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
87+
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
9188
conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
9289
9390
- name: Install python dependencies
@@ -101,9 +98,7 @@ jobs:
10198
working-directory: ${{ env.working_directory }}
10299
run: |
103100
conda activate openvino_lcm_cpp
104-
$env:MODEL_PATH='models/lcm_dreamshaper_v7/FP16'
105-
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $env:MODEL_PATH
106-
convert_tokenizer $env:MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $env:MODEL_PATH/tokenizer/
101+
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 models/lcm_dreamshaper_v7/FP16
107102
108103
- name: Build app
109104
working-directory: ${{ env.working_directory }}
@@ -114,5 +109,4 @@ jobs:
114109
115110
- name: Run app
116111
working-directory: ${{ env.working_directory }}
117-
run: |
118-
& "./build/Release/lcm_dreamshaper.exe" -r --dynamic
112+
run: '& "./build/Release/lcm_dreamshaper.exe" -r --dynamic'

‎.github/workflows/stable_diffusion_1_5_cpp.yml

+6-12
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ jobs:
3838
- name: Install OpenVINO and other conda dependencies
3939
run: |
4040
conda activate openvino_sd_cpp
41-
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
41+
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
4242
conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
4343
4444
- name: Install python dependencies
@@ -52,9 +52,7 @@ jobs:
5252
working-directory: ${{ env.working_directory }}
5353
run: |
5454
conda activate openvino_sd_cpp
55-
export MODEL_PATH="models/stable_diffusion_v1_5_ov/FP16"
56-
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --convert-tokenizer --weight-format fp16 $MODEL_PATH
57-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
55+
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --weight-format fp16 models/stable_diffusion_v1_5_ov/FP16
5856
5957
- name: Build app
6058
working-directory: ${{ env.working_directory }}
@@ -65,8 +63,7 @@ jobs:
6563
6664
- name: Run app
6765
working-directory: ${{ env.working_directory }}
68-
run: |
69-
./build/stable_diffusion -m ./models/stable_diffusion_v1_5_ov -t FP16
66+
run: ./build/stable_diffusion -m ./models/stable_diffusion_v1_5_ov -t FP16
7067

7168
stable_diffusion_1_5_cpp-windows:
7269
runs-on: windows-latest
@@ -85,7 +82,7 @@ jobs:
8582
- name: Install OpenVINO and other conda dependencies
8683
run: |
8784
conda activate openvino_sd_cpp
88-
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
85+
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
8986
9087
- name: Install python dependencies
9188
working-directory: ${{ env.working_directory }}
@@ -98,9 +95,7 @@ jobs:
9895
working-directory: ${{ env.working_directory }}
9996
run: |
10097
conda activate openvino_sd_cpp
101-
$env:MODEL_PATH='models/stable_diffusion_v1_5_ov/FP16'
102-
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --convert-tokenizer --weight-format fp16 $env:MODEL_PATH
103-
convert_tokenizer $env:MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $env:MODEL_PATH/tokenizer/
98+
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --weight-format fp16 models/stable_diffusion_v1_5_ov/FP16
10499
105100
- name: Build app
106101
working-directory: ${{ env.working_directory }}
@@ -111,5 +106,4 @@ jobs:
111106
112107
- name: Run app
113108
working-directory: ${{ env.working_directory }}
114-
run: |
115-
& "./build/Release/stable_diffusion.exe" -m ./models/stable_diffusion_v1_5_ov -t FP16 --dynamic
109+
run: '& "./build/Release/stable_diffusion.exe" -m ./models/stable_diffusion_v1_5_ov -t FP16 --dynamic'

‎image_generation/lcm_dreamshaper_v7/cpp/README.md

+3-8
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Prepare a python environment and install dependencies:
1818
conda create -n openvino_lcm_cpp python==3.10
1919
conda activate openvino_lcm_cpp
2020
conda update -c conda-forge --all
21-
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
21+
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
2222
# Ensure that Conda standard libraries are used
2323
conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
2424
```
@@ -37,13 +37,8 @@ conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
3737
```
3838

3939
2. Download the model from Huggingface and convert it to OpenVINO IR via [optimum-intel CLI](https://github.com/huggingface/optimum-intel). Example command for downloading and exporting FP16 model:
40-
```shell
41-
export MODEL_PATH="models/lcm_dreamshaper_v7/FP16"
42-
# Using optimum-cli for exporting model to OpenVINO format
43-
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $MODEL_PATH
44-
# Converting tokenizer
45-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
46-
```
40+
41+
`optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 models/lcm_dreamshaper_v7/FP16`
4742

4843
### LoRA enabling with safetensors
4944

‎image_generation/lcm_dreamshaper_v7/cpp/requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
--extra-index-url https://download.pytorch.org/whl/cpu
22
torch==2.2.2+cpu
33
diffusers==0.27.2
4-
optimum-intel[nncf,openvino]==1.16.0
4+
optimum-intel[nncf,openvino] @ git+https://github.com/apaniukov/optimum-intel.git@0029e9165a2dad4cfcf787aa63181d9dc0cd49d5

‎image_generation/lcm_dreamshaper_v7/cpp/src/main.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -192,7 +192,7 @@ ov::Tensor text_encoder(StableDiffusionModels models, std::string& pos_prompt) {
192192
tokenizer_req.set_input_tensor(ov::Tensor{ov::element::string, {1}, &pos_prompt});
193193
tokenizer_req.infer();
194194
ov::Tensor input_ids_token = tokenizer_req.get_tensor("input_ids");
195-
std::copy_n(input_ids_token.data<std::int32_t>(), input_ids_token.get_size(), input_ids.data<int32_t>());
195+
std::copy_n(input_ids_token.data<std::int64_t>(), input_ids_token.get_size(), input_ids.data<int32_t>());
196196

197197
// text embeddings
198198
text_encoder_req.set_tensor("input_ids", input_ids);

‎image_generation/stable_diffusion_1_5/cpp/README.md

+3-8
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ Prepare a python environment and install dependencies:
1818
```shell
1919
conda create -n openvino_sd_cpp python==3.10
2020
conda activate openvino_sd_cpp
21-
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler make cmake
21+
conda install -c conda-forge openvino=2024.1.0 c-compiler cxx-compiler git make cmake
2222
# Ensure that Conda standard libraries are used
2323
conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
2424
```
@@ -40,13 +40,8 @@ python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
4040
- [dreamlike-anime-1.0](https://huggingface.co/dreamlike-art/dreamlike-anime-1.0) to run Stable Diffusion with LoRA adapters.
4141

4242
Example command for downloading and exporting FP16 model:
43-
```shell
44-
export MODEL_PATH="models/dreamlike_anime_1_0_ov/FP16"
45-
# Using optimum-cli for exporting model to OpenVINO format
46-
optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --convert-tokenizer --weight-format fp16 $MODEL_PATH
47-
# Converting tokenizer manually (`--convert-tokenizer` flag of `optimum-cli` results in "OpenVINO Tokenizer export for CLIPTokenizer is not supported.")
48-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
49-
```
43+
44+
`optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --weight-format fp16 models/dreamlike_anime_1_0_ov/FP16`
5045

5146
You can also choose other precision and export FP32 or INT8 model.
5247

‎image_generation/stable_diffusion_1_5/cpp/requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
torch==2.2.2+cpu
33
diffusers==0.27.2
44
transformers==4.39.3
5-
optimum-intel[nncf,openvino]==1.16.0
5+
optimum-intel[nncf,openvino] @ git+https://github.com/apaniukov/optimum-intel.git@0029e9165a2dad4cfcf787aa63181d9dc0cd49d5
66
huggingface_hub[cli]==0.22.2

‎image_generation/stable_diffusion_1_5/cpp/src/main.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ ov::Tensor text_encoder(StableDiffusionModels models, std::string& pos_prompt, s
216216
tokenizer_req.set_input_tensor(ov::Tensor{ov::element::string, {1}, &prompt});
217217
tokenizer_req.infer();
218218
ov::Tensor input_ids_token = tokenizer_req.get_tensor("input_ids");
219-
std::copy_n(input_ids_token.data<std::int32_t>(), input_ids_token.get_size(), input_ids.data<int32_t>());
219+
std::copy_n(input_ids_token.data<std::int64_t>(), input_ids_token.get_size(), input_ids.data<std::int32_t>());
220220

221221
// text embeddings
222222
text_encoder_req.set_tensor("input_ids", input_ids);

‎text_generation/causal_lm/cpp/README.md

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Text generation C++ samples that support most popular models like LLaMA 2
22

3-
These examples showcase inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The applications don't have many configuration options to encourage the reader to explore and modify the source code. Loading `openvino_tokenizers` to `ov::Core` enables tokenization. Run `convert_tokenizer` to generate IRs for the samples. [group_beam_searcher.hpp](group_beam_searcher.hpp) implements the algorithm of the same name, which is used by `beam_search_causal_lm`. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/254-llm-chatbot) which provides an example of LLM-powered Chatbot in Python.
3+
These examples showcase inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The applications don't have many configuration options to encourage the reader to explore and modify the source code. Loading `openvino_tokenizers` to `ov::Core` enables tokenization. Run `optimum-cli` to generate IRs for the samples. [group_beam_searcher.hpp](group_beam_searcher.hpp) implements the algorithm of the same name, which is used by `beam_search_causal_lm`. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/254-llm-chatbot) which provides an example of LLM-powered Chatbot in Python.
44

55
## How it works
66

@@ -49,7 +49,7 @@ This approach reduces the need for multiple infer requests to the main model, en
4949
5050
## Install OpenVINO
5151

52-
Install [OpenVINO Archives >= 2024.0](docs.openvino.ai/install). `master` and possibly the latest `releases/*` branch correspond to not yet released OpenVINO versions. https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/ can be used for these branches early testing. `<INSTALL_DIR>` below refers to the extraction location.
52+
Install [OpenVINO Archives >= 2024.1](docs.openvino.ai/install). `master` and possibly the latest `releases/*` branch correspond to not yet released OpenVINO versions. https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/ can be used for these branches early testing. `<INSTALL_DIR>` below refers to the extraction location.
5353

5454
## Build `greedy_causal_lm`, `beam_search_causal_lm` and `openvino_tokenizers`
5555

@@ -81,7 +81,6 @@ python3 -m pip install --upgrade-strategy eager -r requirements.txt
8181
# Update openvino_tokenizers from the submodule
8282
python3 -m pip install ./../../../thirdparty/openvino_tokenizers/[transformers]
8383
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
84-
convert_tokenizer ./TinyLlama-1.1B-Chat-v1.0/ --output ./TinyLlama-1.1B-Chat-v1.0/ --with-detokenizer --trust-remote-code
8584
```
8685

8786
#### Windows
@@ -92,7 +91,6 @@ python -m pip install --upgrade-strategy eager -r requirements.txt
9291
REM Update openvino_tokenizers from the submodule
9392
python -m pip install .\..\..\..\thirdparty\openvino_tokenizers\[transformers]
9493
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
95-
convert_tokenizer .\TinyLlama-1.1B-Chat-v1.0\ --output .\TinyLlama-1.1B-Chat-v1.0\ --with-detokenizer --trust-remote-code
9694
```
9795

9896
## Run

‎text_generation/causal_lm/cpp/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
--extra-index-url https://download.pytorch.org/whl/cpu
22
optimum[openvino]==1.19.1
3+
optimum-intel[openvino] @ git+https://github.com/apaniukov/optimum-intel.git@0029e9165a2dad4cfcf787aa63181d9dc0cd49d5
34
einops==0.7.0 # For Qwen
45
transformers_stream_generator==0.0.4 # For Qwen

‎thirdparty/openvino_tokenizers

0 commit comments

Comments
 (0)