1 file changed
+1
-1
lines changedSubmodule openvino_tokenizers updated 35 files
- .github/workflows/linux.yml+2-2
- .github/workflows/mac.yml+2-2
- .github/workflows/windows.yml+2-2
- CMakeLists.txt+1-1
- README.md+22-9
- benchmark/README.md+40
- benchmark/benchmark.py+282
- pyproject.toml+9-3
- python/openvino_tokenizers/cli.py+1-1
- python/openvino_tokenizers/hf_parser.py+75-23
- python/openvino_tokenizers/str_pack.py+4-7
- python/openvino_tokenizers/tiktoken_parser.py+5-4
- python/openvino_tokenizers/tokenizer_pipeline.py+115-15
- src/CMakeLists.txt+59-36
- src/bpe_tokenizer.cpp+55-46
- src/bpe_tokenizer.hpp+25-3
- src/case_fold.cpp+1-1
- src/equal_str.cpp+7-12
- src/normalize_unicode.cpp+1-1
- src/ov_extension.cpp+3-1
- src/regex_normalization.hpp+1-1
- src/regex_split.cpp+71-15
- src/regex_split.hpp+10-1
- src/sentence_piece.cpp+39-5
- src/string_tensor_pack.cpp+1-1
- src/string_tensor_unpack.cpp+5-5
- src/string_to_hash_bucket.cpp+218
- src/string_to_hash_bucket.hpp+41
- src/tensorflow_translators.cpp+34-7
- src/tensorflow_translators.hpp+1
- src/tokenizer.hpp+1
- src/utils.cpp+1-1
- tests/pass_rates.json+1-1
- tests/tokenizer_differential_fuzzing.py+18-10
- tests/tokenizers_test.py+3
0 commit comments