Skip to content

Commit 3fb27c6

Browse files
WovchenaWovchena
committed
Remove convert_tokenizer
1 parent 387c728 commit 3fb27c6

File tree

6 files changed

+15
-49
lines changed

6 files changed

+15
-49
lines changed

‎.github/workflows/causal_lm_cpp.yml

+1-11
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,9 @@ jobs:
3333
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model openlm-research/open_llama_3b_v2 open_llama_3b_v2
3434
cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
3535
cmake --build ./build/ --config Release -j
36-
- name: convert_tokenizer and run
36+
- name: greedy_causal_lm
3737
run: |
3838
source ./ov/setupvars.sh
39-
convert_tokenizer ./open_llama_3b_v2/ --output ./open_llama_3b_v2/ --with-detokenizer
4039
./build/greedy_causal_lm ./open_llama_3b_v2/ "return 0"
4140
4241
cpp-beam_search_causal_lm-ubuntu:
@@ -64,7 +63,6 @@ jobs:
6463
- name: Compare
6564
run: |
6665
source ./ov/setupvars.sh
67-
convert_tokenizer ./TinyLlama-1.1B-Chat-v1.0/ --output ./TinyLlama-1.1B-Chat-v1.0/ --with-detokenizer
6866
6967
timeout 25s ./build/beam_search_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ 69 > ./pred.txt
7068
python -c "
@@ -157,7 +155,6 @@ jobs:
157155
shell: cmd
158156
run: |
159157
call w_openvino_toolkit_windows_2024.1.0.15008.f4afc983258_x86_64\setupvars.bat
160-
convert_tokenizer .\TinyLlama-1.1B-Chat-v1.0\ --output .\TinyLlama-1.1B-Chat-v1.0\ --with-detokenizer
161158
162159
.\build\Release\beam_search_causal_lm.exe .\TinyLlama-1.1B-Chat-v1.0\ "69" > .\pred.txt
163160
echo import transformers > ref.py
@@ -197,7 +194,6 @@ jobs:
197194
- name: Compare
198195
run: |
199196
source ./ov/setupvars.sh
200-
convert_tokenizer Qwen/Qwen-7B-Chat --output ./Qwen-7B-Chat/ --with-detokenizer --trust-remote-code
201197
timeout 50s ./build/beam_search_causal_lm ./Qwen-7B-Chat/ 69 > ./pred.txt
202198
203199
cpp-beam_search_causal_lm-Qwen1_5-7B-Chat:
@@ -225,7 +221,6 @@ jobs:
225221
- name: Run
226222
run: |
227223
source ./ov/setupvars.sh
228-
convert_tokenizer ./Qwen1.5-7B-Chat/ --output ./Qwen1.5-7B-Chat/ --with-detokenizer --trust-remote-code
229224
timeout 50s ./build/beam_search_causal_lm ./Qwen1.5-7B-Chat/ "你好！" > ./pred_qwen15.txt
230225
231226
cpp-beam_search_causal_lm-Phi-2:
@@ -253,7 +248,6 @@ jobs:
253248
- name: Compare
254249
run: |
255250
source ./ov/setupvars.sh
256-
convert_tokenizer ./phi-2/ --output ./phi-2/ --with-detokenizer --trust-remote-code
257251
timeout 50s ./build/beam_search_causal_lm ./phi-2/ 69 > ./pred.txt
258252
259253
cpp-beam_search_causal_lm-notus-7b-v1:
@@ -281,7 +275,6 @@ jobs:
281275
- name: Compare
282276
run: |
283277
source ./ov/setupvars.sh
284-
convert_tokenizer ./notus-7b-v1/ --output ./notus-7b-v1/ --with-detokenizer --trust-remote-code
285278
timeout 50s ./build/beam_search_causal_lm ./notus-7b-v1/ 69 > ./pred.txt
286279
287280
cpp-speculative_decoding_lm-ubuntu:
@@ -305,8 +298,6 @@ jobs:
305298
python -m pip install ./thirdparty/openvino_tokenizers/[transformers]
306299
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-3b dolly-v2-3b
307300
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-7b dolly-v2-7b
308-
convert_tokenizer ./dolly-v2-3b/ --output ./dolly-v2-3b/ --with-detokenizer
309-
convert_tokenizer ./dolly-v2-7b/ --output ./dolly-v2-7b/ --with-detokenizer
310301
cmake -DCMAKE_BUILD_TYPE=Release -S ./text_generation/causal_lm/cpp/ -B ./build/
311302
cmake --build ./build/ --config Release -j
312303
- name: run and compare
@@ -348,7 +339,6 @@ jobs:
348339
- name: Run Generation
349340
run: |
350341
source ./ov/setupvars.sh
351-
convert_tokenizer ./phi-1_5/ --output ./phi-1_5/ --with-detokenizer --trust-remote-code
352342
timeout 50s ./build/greedy_causal_lm ./phi-1_5/ "Alan Turing was a" > ./pred_greedy.txt
353343
timeout 50s ./build/beam_search_causal_lm ./phi-1_5/ "Alan Turing was a" > ./pred_beam.txt
354344
- name: Compare

‎.github/workflows/lcm_dreamshaper_cpp.yml

+4-10
Original file line numberDiff line numberDiff line change
@@ -53,9 +53,7 @@ jobs:
5353
working-directory: ${{ env.working_directory }}
5454
run: |
5555
conda activate openvino_lcm_cpp
56-
export MODEL_PATH="models/lcm_dreamshaper_v7/FP16"
57-
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $MODEL_PATH
58-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
56+
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --convert-tokenizer --weight-format fp16 models/lcm_dreamshaper_v7/FP16
5957
6058
- name: Build app
6159
working-directory: ${{ env.working_directory }}
@@ -66,8 +64,7 @@ jobs:
6664
6765
- name: Run app
6866
working-directory: ${{ env.working_directory }}
69-
run: |
70-
./build/lcm_dreamshaper
67+
run: ./build/lcm_dreamshaper
7168

7269
lcm_dreamshaper_v7_cpp-windows:
7370
runs-on: windows-latest
@@ -101,9 +98,7 @@ jobs:
10198
working-directory: ${{ env.working_directory }}
10299
run: |
103100
conda activate openvino_lcm_cpp
104-
$env:MODEL_PATH='models/lcm_dreamshaper_v7/FP16'
105-
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $env:MODEL_PATH
106-
convert_tokenizer $env:MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $env:MODEL_PATH/tokenizer/
101+
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --convert-tokenizer --weight-format fp16 models/lcm_dreamshaper_v7/FP16
107102
108103
- name: Build app
109104
working-directory: ${{ env.working_directory }}
@@ -114,5 +109,4 @@ jobs:
114109
115110
- name: Run app
116111
working-directory: ${{ env.working_directory }}
117-
run: |
118-
& "./build/Release/lcm_dreamshaper.exe" -r --dynamic
112+
run: '& "./build/Release/lcm_dreamshaper.exe" -r --dynamic'

‎.github/workflows/stable_diffusion_1_5_cpp.yml

+4-10
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,7 @@ jobs:
5252
working-directory: ${{ env.working_directory }}
5353
run: |
5454
conda activate openvino_sd_cpp
55-
export MODEL_PATH="models/stable_diffusion_v1_5_ov/FP16"
56-
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --convert-tokenizer --weight-format fp16 $MODEL_PATH
57-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
55+
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --convert-tokenizer --weight-format fp16 models/stable_diffusion_v1_5_ov/FP16
5856
5957
- name: Build app
6058
working-directory: ${{ env.working_directory }}
@@ -65,8 +63,7 @@ jobs:
6563
6664
- name: Run app
6765
working-directory: ${{ env.working_directory }}
68-
run: |
69-
./build/stable_diffusion -m ./models/stable_diffusion_v1_5_ov -t FP16
66+
run: ./build/stable_diffusion -m ./models/stable_diffusion_v1_5_ov -t FP16
7067

7168
stable_diffusion_1_5_cpp-windows:
7269
runs-on: windows-latest
@@ -98,9 +95,7 @@ jobs:
9895
working-directory: ${{ env.working_directory }}
9996
run: |
10097
conda activate openvino_sd_cpp
101-
$env:MODEL_PATH='models/stable_diffusion_v1_5_ov/FP16'
102-
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --convert-tokenizer --weight-format fp16 $env:MODEL_PATH
103-
convert_tokenizer $env:MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $env:MODEL_PATH/tokenizer/
98+
optimum-cli export openvino --model runwayml/stable-diffusion-v1-5 --task stable-diffusion --convert-tokenizer --weight-format fp16 models/stable_diffusion_v1_5_ov/FP16
10499
105100
- name: Build app
106101
working-directory: ${{ env.working_directory }}
@@ -111,5 +106,4 @@ jobs:
111106
112107
- name: Run app
113108
working-directory: ${{ env.working_directory }}
114-
run: |
115-
& "./build/Release/stable_diffusion.exe" -m ./models/stable_diffusion_v1_5_ov -t FP16 --dynamic
109+
run: '& "./build/Release/stable_diffusion.exe" -m ./models/stable_diffusion_v1_5_ov -t FP16 --dynamic'

‎image_generation/lcm_dreamshaper_v7/cpp/README.md

+2-7
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,8 @@ conda env config vars set LD_LIBRARY_PATH=$CONDA_PREFIX/lib:$LD_LIBRARY_PATH
3737
```
3838

3939
2. Download the model from Huggingface and convert it to OpenVINO IR via [optimum-intel CLI](https://github.com/huggingface/optimum-intel). Example command for downloading and exporting FP16 model:
40-
```shell
41-
export MODEL_PATH="models/lcm_dreamshaper_v7/FP16"
42-
# Using optimum-cli for exporting model to OpenVINO format
43-
optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --weight-format fp16 $MODEL_PATH
44-
# Converting tokenizer
45-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
46-
```
40+
41+
`optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --convert-tokenizer --weight-format fp16 models/lcm_dreamshaper_v7/FP16`
4742

4843
### LoRA enabling with safetensors
4944

‎image_generation/stable_diffusion_1_5/cpp/README.md

+2-7
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,8 @@ python -m pip install ../../../thirdparty/openvino_tokenizers/[transformers]
4040
- [dreamlike-anime-1.0](https://huggingface.co/dreamlike-art/dreamlike-anime-1.0) to run Stable Diffusion with LoRA adapters.
4141

4242
Example command for downloading and exporting FP16 model:
43-
```shell
44-
export MODEL_PATH="models/dreamlike_anime_1_0_ov/FP16"
45-
# Using optimum-cli for exporting model to OpenVINO format
46-
optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --convert-tokenizer --weight-format fp16 $MODEL_PATH
47-
# Converting tokenizer manually (`--convert-tokenizer` flag of `optimum-cli` results in "OpenVINO Tokenizer export for CLIPTokenizer is not supported.")
48-
convert_tokenizer $MODEL_PATH/tokenizer/ --tokenizer-output-type i32 -o $MODEL_PATH/tokenizer/
49-
```
43+
44+
`optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --convert-tokenizer --weight-format fp16 models/dreamlike_anime_1_0_ov/FP16`
5045

5146
You can also choose other precision and export FP32 or INT8 model.
5247

‎text_generation/causal_lm/cpp/README.md

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Text generation C++ samples that support most popular models like LLaMA 2
22

3-
These examples showcase inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The applications don't have many configuration options to encourage the reader to explore and modify the source code. Loading `openvino_tokenizers` to `ov::Core` enables tokenization. Run `convert_tokenizer` to generate IRs for the samples. [group_beam_searcher.hpp](group_beam_searcher.hpp) implements the algorithm of the same name, which is used by `beam_search_causal_lm`. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/254-llm-chatbot) which provides an example of LLM-powered Chatbot in Python.
3+
These examples showcase inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The applications don't have many configuration options to encourage the reader to explore and modify the source code. Loading `openvino_tokenizers` to `ov::Core` enables tokenization. Run `optimum-cli` to generate IRs for the samples. [group_beam_searcher.hpp](group_beam_searcher.hpp) implements the algorithm of the same name, which is used by `beam_search_causal_lm`. There is also a Jupyter [notebook](https://github.com/openvinotoolkit/openvino_notebooks/tree/main/notebooks/254-llm-chatbot) which provides an example of LLM-powered Chatbot in Python.
44

55
## How it works
66

@@ -49,7 +49,7 @@ This approach reduces the need for multiple infer requests to the main model, en
4949
5050
## Install OpenVINO
5151

52-
Install [OpenVINO Archives >= 2024.0](docs.openvino.ai/install). `master` and possibly the latest `releases/*` branch correspond to not yet released OpenVINO versions. https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/ can be used for these branches early testing. `<INSTALL_DIR>` below refers to the extraction location.
52+
Install [OpenVINO Archives >= 2024.1](docs.openvino.ai/install). `master` and possibly the latest `releases/*` branch correspond to not yet released OpenVINO versions. https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/ can be used for these branches early testing. `<INSTALL_DIR>` below refers to the extraction location.
5353

5454
## Build `greedy_causal_lm`, `beam_search_causal_lm` and `openvino_tokenizers`
5555

@@ -81,7 +81,6 @@ python3 -m pip install --upgrade-strategy eager -r requirements.txt
8181
# Update openvino_tokenizers from the submodule
8282
python3 -m pip install ./../../../thirdparty/openvino_tokenizers/[transformers]
8383
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
84-
convert_tokenizer ./TinyLlama-1.1B-Chat-v1.0/ --output ./TinyLlama-1.1B-Chat-v1.0/ --with-detokenizer --trust-remote-code
8584
```
8685

8786
#### Windows
@@ -92,7 +91,6 @@ python -m pip install --upgrade-strategy eager -r requirements.txt
9291
REM Update openvino_tokenizers from the submodule
9392
python -m pip install .\..\..\..\thirdparty\openvino_tokenizers\[transformers]
9493
optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0
95-
convert_tokenizer .\TinyLlama-1.1B-Chat-v1.0\ --output .\TinyLlama-1.1B-Chat-v1.0\ --with-detokenizer --trust-remote-code
9694
```
9795

9896
## Run

0 commit comments

Comments
 (0)