Гse PCRE2 fallback for RegexNormalization'

* add pcre2

* use string with length instead of ZERO_TERMINATION

* add Licence to thirt party

* set m_search_pattern_re to nullptr if PCRE2 is used

* remove unnecessary const char*

* fix typo in m_replace_pattern

* Update Del Control Chars Regex

* Update Pass Rate

* fix win build

* revert added_tests in conftest.py

* revert Readme & pass rates

* update README & pass_rates

* do not log errors

* add comment on malloc

---------

Co-authored-by: Artur Paniukov <chgk1101@gmail.com>

Author: pavel-esir

.gitignore

@@ -0,0 +1 @@
+build-*/

README.md

@@ -433,7 +433,7 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     </tr>
     <tr>
       <td >SentencePiece</td>
-      <td >80.53</td>
+      <td >80.49</td>
       <td >4762</td>
     </tr>
     <tr>
@@ -443,7 +443,7 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     </tr>
     <tr>
       <td >WordPiece</td>
-      <td >91.48</td>
+      <td >99.10</td>
       <td >1327</td>
     </tr>
   </tbody>
@@ -728,13 +728,13 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     <tr>
       <td >SentencePiece</td>
       <td >xlnet-base-cased</td>
-      <td >65.69</td>
+      <td >65.27</td>
       <td >239</td>
     </tr>
     <tr>
       <td >SentencePiece</td>
       <td >xlnet-base-cased_slow</td>
-      <td >61.43</td>
+      <td >60.99</td>
       <td >223</td>
     </tr>
     <tr>
@@ -758,79 +758,79 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     <tr>
       <td >WordPiece</td>
       <td >ProsusAI/finbert</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >bert-base-multilingual-cased</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >bert-base-uncased</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >cointegrated/rubert-tiny2</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >distilbert-base-uncased-finetuned-sst-2-english</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >google/electra-base-discriminator</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >google/mobilebert-uncased</td>
-      <td >94.51</td>
+      <td >100.00</td>
       <td >91</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >jhgan/ko-sbert-sts</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >prajjwal1/bert-mini</td>
-      <td >94.51</td>
+      <td >100.00</td>
       <td >91</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >rajiv003/ernie-finetuned-qqp</td>
-      <td >94.51</td>
+      <td >100.00</td>
       <td >91</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >rasa/LaBSE</td>
-      <td >80.37</td>
+      <td >88.79</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >sentence-transformers/all-MiniLM-L6-v2</td>
-      <td >91.59</td>
+      <td >100.00</td>
       <td >107</td>
     </tr>
     <tr>
       <td >WordPiece</td>
       <td >squeezebert/squeezebert-uncased</td>
-      <td >94.51</td>
+      <td >100.00</td>
       <td >91</td>
     </tr>
   </tbody>

python/openvino_tokenizers/hf_parser.py

@@ -77,9 +77,7 @@ def parse_bert_normalizer(normalizer_dict: Dict[str, Any]) -> List[Normalization
     steps: List[NormalizationStep] = []
 
     if normalizer_dict["clean_text"] is True:
-        pass
-        # TODO: this regex is not supported by re2, skip it until broader syntax support
-        # steps.append(RegexNormalizationStep.del_control_chars_regex())
+        steps.append(RegexNormalizationStep.del_control_chars_regex())
 
     # https://github.com/huggingface/tokenizers/blob/8c9cfb0b689bce00b615b9557a9a767f286d7a33/tokenizers/src/normalizers/bert.rs#L127
     if normalizer_dict.get("strip_accents") or normalizer_dict["lowercase"]:

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -146,10 +146,9 @@ def prepend_regex(cls, string: str) -> "RegexNormalizationStep":
 
     @classmethod
     def del_control_chars_regex(cls) -> "RegexNormalizationStep":
-        # https://github.com/huggingface/tokenizers/blob/8c9cfb0b689bce00b615b9557a9a767f286d7a33/tokenizers/src/normalizers/bert.rs#L17
         return cls(
-            regex_search_pattern=r"((?=[^\n\t\r])\p{Cc})|((?=[^\n\t\r])\p{Cf})",
-            replace_term=" ",
+            regex_search_pattern=r"([\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F])",  # exclude \n\t\r
+            replace_term="",
         )
 
     @classmethod

src/CMakeLists.txt

@@ -278,6 +278,19 @@ set_target_properties(${TARGET_NAME} PROPERTIES COMPILE_OPTIONS "${extra_flags}"
 target_compile_definitions(${TARGET_NAME} PRIVATE IMPLEMENT_OPENVINO_EXTENSION_API)
 target_link_libraries(${TARGET_NAME} PRIVATE openvino::runtime)
 
+# Fetch PCRE2
+include(FetchContent)
+FetchContent_Declare(
+    PCRE2
+    URL https://github.com/PCRE2Project/pcre2/archive/refs/tags/pcre2-10.44.zip
+    URL_HASH SHA256=2d87bd1700bd1993ddea7c56aad2b0373ac2b3d52d9cc78842a6d061ffaf0925
+)
+FetchContent_MakeAvailable(PCRE2)
+
+target_include_directories(${TARGET_NAME} PRIVATE ${PCRE2_BINARY_DIR})
+target_link_libraries(${TARGET_NAME} PRIVATE pcre2-8)
+target_compile_definitions(${TARGET_NAME} PRIVATE PCRE2_CODE_UNIT_WIDTH=8)
+
 if(OpenVINO_Frontend_TensorFlow_FOUND)
   target_link_libraries(${TARGET_NAME} PRIVATE openvino::frontend::tensorflow)
   target_compile_definitions(${TARGET_NAME} PRIVATE OpenVINO_Frontend_TensorFlow_FOUND)

src/regex_normalization.cpp

@@ -2,11 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-
-
 #include "regex_normalization.hpp"
 #include "utils.hpp"
 
+
 using namespace ov;
 
 
@@ -19,32 +18,61 @@ m_global_replace(global_replace) {
     auto replace_pattern_const = as_type_ptr<Constant>(arguments[4].get_node_shared_ptr());
     auto search_pattern_buf = static_cast<const char*>(search_pattern_const->get_data_ptr());
     auto replace_pattern_buf = static_cast<const char*>(replace_pattern_const->get_data_ptr());
-    auto search_pattern = absl::string_view((const char*)search_pattern_buf, search_pattern_const->get_byte_size());
-    m_replace_pattern = absl::string_view((const char*)replace_pattern_buf, replace_pattern_const->get_byte_size());
-    m_search_pattern_re = std::make_shared<re2::RE2>(search_pattern);
+    auto search_pattern = absl::string_view(search_pattern_buf, search_pattern_const->get_byte_size());
+    m_replace_pattern = absl::string_view(replace_pattern_buf, replace_pattern_const->get_byte_size());
+    
+    auto options = re2::RE2::Options();
+    options.set_log_errors(false);    
+    m_search_pattern_re = std::make_shared<re2::RE2>(search_pattern, options);
+    
+    if (m_search_pattern_re->NumberOfCapturingGroups() == -1) {
+        // If RE2 was unable to process pattern.
+        m_search_pattern_pcre2 = std::make_shared<PCRE2Wrapper>(search_pattern);
+        m_search_pattern_re = nullptr;
+    }
+    
     constructor_validate_and_infer_types();
 }
 
 
 RegexNormalization::RegexNormalization(
         const ov::OutputVector& arguments,
         const std::shared_ptr<re2::RE2>& search_pattern_re,
+        const std::shared_ptr<PCRE2Wrapper>& search_pattern_pcre2,
         const absl::string_view replace_pattern,
         bool global_replace
     ) : ov::op::Op(arguments),
         m_search_pattern_re(search_pattern_re),
+        m_search_pattern_pcre2(search_pattern_pcre2),
         m_replace_pattern(replace_pattern),
         m_global_replace(global_replace) {
 
-        if (m_search_pattern_re == nullptr) {
-            auto search_pattern_const = as_type_ptr<Constant>(arguments[3].get_node_shared_ptr());
-            auto replace_pattern_const = as_type_ptr<Constant>(arguments[4].get_node_shared_ptr());
-            auto search_pattern_buf = static_cast<const char*>(search_pattern_const->get_data_ptr());
-            auto replace_pattern_buf = static_cast<const char*>(replace_pattern_const->get_data_ptr());
-            auto search_pattern = absl::string_view((const char*)search_pattern_buf, search_pattern_const->get_byte_size());
-            m_replace_pattern = absl::string_view((const char*)replace_pattern_buf, replace_pattern_const->get_byte_size());
-            m_search_pattern_re = std::make_shared<re2::RE2>(search_pattern);
+        auto search_pattern_const = as_type_ptr<Constant>(arguments[3].get_node_shared_ptr());
+        auto replace_pattern_const = as_type_ptr<Constant>(arguments[4].get_node_shared_ptr());
+        const char* search_pattern_buf;
+        const char* replace_pattern_buf;
+        absl::string_view search_pattern;
+
+        if (m_search_pattern_re == nullptr || m_search_pattern_pcre2 == nullptr) {
+            search_pattern_buf = static_cast<const char*>(search_pattern_const->get_data_ptr());
+            replace_pattern_buf = static_cast<const char*>(replace_pattern_const->get_data_ptr());
+            search_pattern = absl::string_view(search_pattern_buf, search_pattern_const->get_byte_size());
+            m_replace_pattern = absl::string_view(replace_pattern_buf, replace_pattern_const->get_byte_size());
         };
+        
+        auto options = re2::RE2::Options();
+        options.set_log_errors(false);
+        if (m_search_pattern_re == nullptr) {
+            auto options = re2::RE2::Options();
+            options.set_log_errors(false);
+            m_search_pattern_re = std::make_shared<re2::RE2>(search_pattern, options);
+        }
+        
+        if (m_search_pattern_re->NumberOfCapturingGroups() == -1 && m_search_pattern_pcre2 == nullptr) {
+            m_search_pattern_pcre2 = std::make_shared<PCRE2Wrapper>(search_pattern);
+            m_search_pattern_re = nullptr;
+        }
+
         constructor_validate_and_infer_types();
     }
 
@@ -58,24 +86,40 @@ void RegexNormalization::validate_and_infer_types() {
 
 
 bool RegexNormalization::evaluate(ov::TensorVector& outputs, const ov::TensorVector& inputs) const {
-    if (m_search_pattern_re == nullptr) {
-        auto search_pattern = absl::string_view(inputs[3].data<const char>(), inputs[3].get_size());
+    absl::string_view search_pattern;
+    if (m_search_pattern_re == nullptr || m_search_pattern_pcre2 == nullptr) {
+        search_pattern = absl::string_view(inputs[3].data<const char>(), inputs[3].get_size());
         m_replace_pattern = absl::string_view(inputs[4].data<const char>(), inputs[4].get_size());
-        m_search_pattern_re = std::make_shared<re2::RE2>(search_pattern);
-    };
+    }
+
+    if (m_search_pattern_re == nullptr && m_search_pattern_pcre2 == nullptr) {
+        auto options = re2::RE2::Options();
+        options.set_log_errors(false);
+        m_search_pattern_re = std::make_shared<re2::RE2>(search_pattern, options);
+    }
+
+    if ((m_search_pattern_re == nullptr) || (m_search_pattern_re->NumberOfCapturingGroups() == -1 && m_search_pattern_pcre2 == nullptr)) {
+        m_search_pattern_pcre2 = std::make_shared<PCRE2Wrapper>(search_pattern);
+        m_search_pattern_re = nullptr;
+    }
+
     return evaluate_normalization_helper(
         outputs, inputs,
-        [this](const std::string& str) {
-            // FIXME: if regex is not valid re2, return string without changing (use another regex engine)
-            if (m_search_pattern_re->NumberOfCapturingGroups() == -1)
-                return str;
-
+        [this](const std::string& str) -> std::string {
             std::string result = str;
-            if (m_global_replace) {
-                re2::RE2::GlobalReplace(&result, *m_search_pattern_re, m_replace_pattern);
+
+            // Use RE2 where possible, and fallback to PCRE2 if RE2 was not able to process.
+            if (m_search_pattern_re) {
+                if (m_global_replace) {
+                    re2::RE2::GlobalReplace(&result, *m_search_pattern_re, m_replace_pattern);
+                } else {
+                    re2::RE2::Replace(&result, *m_search_pattern_re, m_replace_pattern);
+                };
+                return result;
+            } else if (m_search_pattern_pcre2) {
+                return m_search_pattern_pcre2->substitute(result, m_replace_pattern, m_global_replace);
             } else {
-                re2::RE2::Replace(&result, *m_search_pattern_re, m_replace_pattern);
-            };
-            return result;
+                return result;
+            }
     });
 }

src/regex_normalization.hpp

@@ -5,10 +5,12 @@
 #pragma once
 
 #include "absl/strings/string_view.h"
+#include "utils.hpp"
 
 #include <openvino/op/op.hpp>
 #include "openvino/opsets/opset13.hpp"
 #include "fast_tokenizer/normalizers/normalizers.h"
+#include <pcre2.h>
 
 using namespace ov;
 using namespace ov::opset13;
@@ -25,14 +27,15 @@ class RegexNormalization : public ov::op::Op {
     RegexNormalization(
         const ov::OutputVector& arguments,
         const std::shared_ptr<re2::RE2>& search_pattern_re,
+        const std::shared_ptr<PCRE2Wrapper>& search_pattern_rcre2,
         const absl::string_view replace_pattern,
         bool global_replace = true
     );
 
     void validate_and_infer_types() override;
 
     std::shared_ptr<ov::Node> clone_with_new_inputs(const ov::OutputVector& inputs) const override {
-        return std::make_shared<RegexNormalization>(inputs, m_search_pattern_re, m_replace_pattern, m_global_replace);
+        return std::make_shared<RegexNormalization>(inputs, m_search_pattern_re, m_search_pattern_pcre2, m_replace_pattern, m_global_replace);
     }
 
     bool visit_attributes(ov::AttributeVisitor& visitor) override {
@@ -47,6 +50,7 @@ class RegexNormalization : public ov::op::Op {
     }
 private:
     mutable std::shared_ptr<re2::RE2> m_search_pattern_re;
+    mutable std::shared_ptr<PCRE2Wrapper> m_search_pattern_pcre2;
     mutable absl::string_view m_replace_pattern;
     bool m_global_replace = true;
 };