1 file changed
+1
-1
lines changedSubmodule openvino_tokenizers updated 12 files
- README.md+9-22
- python/openvino_tokenizers/hf_parser.py+22-72
- python/openvino_tokenizers/str_pack.py+7-4
- python/openvino_tokenizers/tiktoken_parser.py+4-5
- python/openvino_tokenizers/tokenizer_pipeline.py+15-115
- src/CMakeLists.txt+5-9
- src/bpe_tokenizer.cpp+46-55
- src/bpe_tokenizer.hpp+3-25
- src/regex_split.cpp+14-70
- src/regex_split.hpp+1-10
- tests/pass_rates.json+1-1
- tests/tokenizers_test.py-3
0 commit comments