1 file changed
+1
-1
lines changedSubmodule openvino_tokenizers updated 35 files
- .github/workflows/linux.yml+2-2
- .github/workflows/mac.yml+2-2
- .github/workflows/windows.yml+2-2
- CMakeLists.txt+1-1
- README.md+9-22
- benchmark/README.md-40
- benchmark/benchmark.py-282
- pyproject.toml+3-9
- python/openvino_tokenizers/cli.py+1-1
- python/openvino_tokenizers/hf_parser.py+23-75
- python/openvino_tokenizers/str_pack.py+7-4
- python/openvino_tokenizers/tiktoken_parser.py+4-5
- python/openvino_tokenizers/tokenizer_pipeline.py+15-115
- src/CMakeLists.txt+38-65
- src/bpe_tokenizer.cpp+46-55
- src/bpe_tokenizer.hpp+3-25
- src/case_fold.cpp+1-1
- src/equal_str.cpp+12-7
- src/normalize_unicode.cpp+1-1
- src/ov_extension.cpp+1-3
- src/regex_normalization.hpp+1-1
- src/regex_split.cpp+15-71
- src/regex_split.hpp+1-10
- src/sentence_piece.cpp+5-39
- src/string_tensor_pack.cpp+1-1
- src/string_tensor_unpack.cpp+5-5
- src/string_to_hash_bucket.cpp-218
- src/string_to_hash_bucket.hpp-41
- src/tensorflow_translators.cpp+7-34
- src/tensorflow_translators.hpp-1
- src/tokenizer.hpp-1
- src/utils.cpp+1-1
- tests/pass_rates.json+1-1
- tests/tokenizer_differential_fuzzing.py+10-18
- tests/tokenizers_test.py-3
0 commit comments