Add re2 vendoring

Signed-off-by: Petr Shumilov <p.shumilov@vkteam.ru>

.github/workflows/Build.yml

@@ -146,9 +146,9 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install python@3.13 re2c cmake coreutils libiconv re2 pcre yaml-cpp googletest shivammathur/php/php@7.4
+        brew install python@3.13 cmake coreutils libiconv pcre yaml-cpp googletest shivammathur/php/php@7.4
         brew link --overwrite --force shivammathur/php/php@7.4
-        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema
+        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema six
 
     - name: Run cmake
       run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build

.github/workflows/Dockerfile.buster

@@ -22,7 +22,7 @@ RUN apt-get update && \
       git cmake-data=3.18* cmake=3.18* make g++ gperf netcat patch \
       python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \
     pip3 install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libpcre3-dev \
       libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer && \
     rm -rf /var/lib/apt/lists/* && \

.github/workflows/Dockerfile.focal

@@ -15,7 +15,7 @@ RUN apt-get update && \
       git cmake make clang g++ g++-10 gperf netcat patch \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libpcre3-dev \
       libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer unzip && \
     rm -rf /var/lib/apt/lists/*

.github/workflows/Dockerfile.jammy

@@ -18,7 +18,7 @@ RUN apt update && \
     apt install -y --no-install-recommends \
       build-essential devscripts fakeroot git cmake make g++ lld gperf netcat patch \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools && \
-    apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libpcre3-dev \
       libyaml-cpp-dev zlib1g-dev php7.4-dev libnuma-dev unzip  && \
     # Install MySQL-related and PostgreSQL-related libraries from the Ubuntu 20.04 (Focal Fossa) repository.
     # Ubuntu 22.04 has moved to OpenSSL 3.0; however, KPHP relies on OpenSSL 1.1.1.

.github/workflows/macos.yml

@@ -33,9 +33,9 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install python@3.13 re2c cmake coreutils libiconv re2 pcre yaml-cpp googletest shivammathur/php/php@7.4
+        brew install python@3.13 cmake coreutils libiconv pcre yaml-cpp googletest shivammathur/php/php@7.4
         brew link --overwrite --force shivammathur/php/php@7.4
-        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema
+        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema six
       
     - name: Run cmake
       run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build

.gitmodules

@@ -23,3 +23,6 @@
 	path = third-party/zstd
 	url = https://github.com/VKCOM/libzstd.git
 	branch = master
+[submodule "third-party/re2"]
+	path = third-party/re2
+	url = https://github.com/VKCOM/re2.git

CMakeLists.txt

@@ -34,6 +34,9 @@ include(cmake/popular-common.cmake)
 include(${THIRD_PARTY_DIR}/zlib-cmake/zlib.cmake)
 include(${THIRD_PARTY_DIR}/openssl-cmake/openssl.cmake)
 include(${THIRD_PARTY_DIR}/zstd-cmake/zstd.cmake)
+include(${COMMON_DIR}/unicode/unicode.cmake)
+# RE2 depends on unicode
+include(${THIRD_PARTY_DIR}/re2-cmake/re2.cmake)
 
 # TODO: use FetchContent_Declare instead of include_directories
 include_directories(.)
@@ -47,7 +50,6 @@ include(${BASE_DIR}/vkext/vkext.cmake)
 include(${COMMON_DIR}/binlog/binlog.cmake)
 include(${COMMON_DIR}/common.cmake)
 include(${COMMON_DIR}/tl/tl.cmake)
-include(${COMMON_DIR}/unicode/unicode.cmake)
 
 include(${RUNTIME_COMMON_DIR}/runtime-common.cmake)
 

cmake/init-global-vars.cmake

@@ -29,7 +29,8 @@ else()
 endif()
 
 find_package(Git REQUIRED)
-find_package (Python3 COMPONENTS Interpreter REQUIRED)
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
+find_package(Perl REQUIRED)
 
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)

common/common.cmake

@@ -65,8 +65,8 @@ endif()
 
 vk_add_library_no_pic(common-src-no-pic OBJECT ${COMMON_ALL_SOURCES})
 add_dependencies(common-src-no-pic OpenSSL::no-pic::Crypto ZLIB::no-pic::zlib ZSTD::no-pic::zstd)
-target_include_directories(common-src-no-pic PUBLIC ${ZLIB_NO_PIC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIRS})
+target_include_directories(common-src-no-pic PUBLIC ${ZLIB_NO_PIC_INCLUDE_DIRS} ${ZSTD_NO_PIC_INCLUDE_DIRS})
 
 vk_add_library_pic(common-src-pic OBJECT ${COMMON_ALL_SOURCES})
 add_dependencies(common-src-pic OpenSSL::pic::Crypto ZLIB::pic::zlib ZSTD::pic::zstd)
-target_include_directories(common-src-pic PUBLIC ${ZLIB_PIC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIRS})
+target_include_directories(common-src-pic PUBLIC ${ZLIB_PIC_INCLUDE_DIRS} ${ZSTD_PIC_INCLUDE_DIRS})

common/unicode/raw_data/download.sh

@@ -7,6 +7,7 @@ rm -rf *.txt
 wget ${host}/CaseFolding.txt -O CaseFolding.txt
 wget ${host}/extracted/DerivedGeneralCategory.txt -O GeneralCategory.txt
 wget ${host}/UnicodeData.txt -O UnicodeData.txt
+wget ${host}/Scripts.txt -O Scripts.txt # for RE2
 
 touch *.txt
 

common/unicode/unicode.cmake

@@ -24,6 +24,7 @@ add_custom_command(
         COMMAND prepare_unicode_data -c 0,12 ${UNICODE_AUTO_RAW_DATA_DIR}/UnicodeData.txt ${UNICODE_AUTO_DATA_DIR}/UnicodeUppercase.txt
         COMMAND prepare_unicode_data -a -c 0,5 ${UNICODE_AUTO_RAW_DATA_DIR}/UnicodeData.txt ${UNICODE_AUTO_DATA_DIR}/UnicodeNFKD.txt
         COMMENT "unicode raw data generation")
+add_custom_target(unicode-data ALL DEPENDS ${UNICODE_DATA_LIST})
 
 add_custom_command(
         OUTPUT ${AUTO_DIR}/common/unicode-utils-auto.h

compiler/compiler-settings.cpp

@@ -356,7 +356,7 @@ void CompilerSettings::init() {
 
   ld_flags.value_ = extra_ld_flags.get();
   append_apple_options(cxx_default_flags, ld_flags.value_);
-  std::vector<vk::string_view> system_installed_static_libs{"pcre", "re2", "yaml-cpp", "h3", "kphp-timelib"};
+  std::vector<vk::string_view> system_installed_static_libs{"pcre", "yaml-cpp", "h3", "kphp-timelib"};
 
 #ifdef KPHP_TIMELIB_LIB_DIR
   ld_flags.value_ += " -L" KPHP_TIMELIB_LIB_DIR;

compiler/compiler.cmake

@@ -238,6 +238,8 @@ list(APPEND KPHP_COMPILER_SOURCES
      ${AUTO_DIR}/compiler/rewrite-rules/early_opt.cpp)
 
 vk_add_library_no_pic(kphp2cpp_src-no-pic OBJECT ${KPHP_COMPILER_SOURCES})
+add_dependencies(kphp2cpp_src-no-pic OpenSSL::no-pic::Crypto RE2::no-pic::re2)
+target_include_directories(kphp2cpp_src-no-pic PUBLIC ${OPENSSL_NO_PIC_INCLUDE_DIRS} ${RE2_NO_PIC_INCLUDE_DIRS})
 
 file(MAKE_DIRECTORY ${KPHP_COMPILER_AUTO_DIR})
 add_custom_command(OUTPUT ${KEYWORDS_SET}
@@ -274,10 +276,9 @@ set_property(SOURCE ${KPHP_COMPILER_DIR}/kphp2cpp.cpp
 
 add_executable(kphp2cpp ${KPHP_COMPILER_DIR}/kphp2cpp.cpp)
 target_include_directories(kphp2cpp PUBLIC ${KPHP_COMPILER_DIR})
-add_dependencies(kphp2cpp OpenSSL::no-pic::Crypto)
 
-prepare_cross_platform_libs(COMPILER_LIBS yaml-cpp re2)
-set(COMPILER_LIBS vk::no-pic::kphp2cpp_src vk::no-pic::tlo_parsing_src vk::no-pic::popular-common ${COMPILER_LIBS} fmt::fmt OpenSSL::no-pic::Crypto pthread)
+prepare_cross_platform_libs(COMPILER_LIBS yaml-cpp)
+set(COMPILER_LIBS vk::no-pic::kphp2cpp_src vk::no-pic::tlo_parsing_src vk::no-pic::popular-common ${COMPILER_LIBS} fmt::fmt OpenSSL::no-pic::Crypto RE2::no-pic::re2 pthread)
 if(NOT APPLE)
     list(APPEND COMPILER_LIBS stdc++fs)
 endif()

compiler/pipes/register-kphp-configuration.cpp

@@ -4,7 +4,7 @@
 
 #include "compiler/pipes/register-kphp-configuration.h"
 
-#include <re2/re2.h>
+#include "re2/re2.h"
 
 #include "common/algorithms/find.h"
 #include "compiler/compiler-core.h"

docs/kphp-internals/developing-and-extending-kphp/compiling-kphp-from-sources.md

@@ -53,7 +53,7 @@ Install packages
 ```bash
 apt-get update
 apt install git cmake-data=3.16* cmake=3.16* make g++ gperf python3-minimal python3-jsonschema \
-            curl-kphp-vk libuber-h3-dev kphp-timelib libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+            curl-kphp-vk libuber-h3-dev kphp-timelib libfmt-dev libgtest-dev libgmock-dev libpcre3-dev \
             libyaml-cpp-dev php7.4-dev libmysqlclient-dev libnuma-dev \
             postgresql postgresql-server-dev-all libpq-dev libldap-dev libkrb5-dev
 ```

runtime/confdata-keys.h

@@ -4,7 +4,7 @@
 
 #pragma once
 #include <array>
-#include <re2/re2.h>
+#include "re2/re2.h"
 #include <forward_list>
 #include <unordered_map>
 #include <unordered_set>

runtime/regexp.cpp

@@ -5,7 +5,7 @@
 #include "runtime/regexp.h"
 
 #include <cstddef>
-#include <re2/re2.h>
+#include "re2/re2.h"
 #if ASAN_ENABLED
 #include <sanitizer/lsan_interface.h>
 #endif

runtime/runtime.cmake

@@ -1,4 +1,4 @@
-# CURL Build
+# CURL and deps build
 include(${THIRD_PARTY_DIR}/nghttp2-cmake/nghttp2.cmake)
 include(${THIRD_PARTY_DIR}/curl-cmake/curl.cmake)
 
@@ -139,11 +139,11 @@ set(KPHP_RUNTIME_ALL_SOURCES
 allow_deprecated_declarations(${BASE_DIR}/runtime/allocator.cpp ${BASE_DIR}/runtime/openssl.cpp)
 allow_deprecated_declarations_for_apple(${BASE_DIR}/runtime/inter-process-mutex.cpp)
 
-prepare_cross_platform_libs(SYSTEM_INSTALLED_LIBS yaml-cpp re2 h3)
+prepare_cross_platform_libs(SYSTEM_INSTALLED_LIBS yaml-cpp h3)
 
 #### NO PIC
 vk_add_library_no_pic(kphp-runtime-no-pic STATIC ${KPHP_RUNTIME_ALL_SOURCES})
-target_include_directories(kphp-runtime-no-pic PUBLIC ${BASE_DIR} ${OPENSSL_INCLUDE_DIR} ${ZLIB_NO_PIC_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIRS})
+target_include_directories(kphp-runtime-no-pic PUBLIC ${BASE_DIR})
 
 set(RUNTIME_LIBS_NO_PIC
         vk::no-pic::kphp-server
@@ -160,6 +160,7 @@ set(RUNTIME_LIBS_NO_PIC
         ZLIB::no-pic::zlib
         NGHTTP2::no-pic::nghttp2
         ZSTD::no-pic::zstd
+        RE2::no-pic::re2
         m
         pthread
 )
@@ -171,7 +172,7 @@ combine_static_runtime_library(kphp-runtime-no-pic kphp-full-runtime-no-pic)
 
 #### PIC
 vk_add_library_pic(kphp-runtime-pic STATIC ${KPHP_RUNTIME_ALL_SOURCES})
-target_include_directories(kphp-runtime-pic PUBLIC ${BASE_DIR} ${OPENSSL_INCLUDE_DIR} ${ZLIB_PIC_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIRS})
+target_include_directories(kphp-runtime-pic PUBLIC ${BASE_DIR})
 
 set(RUNTIME_LIBS_PIC
         vk::pic::kphp-server
@@ -188,6 +189,7 @@ set(RUNTIME_LIBS_PIC
         ZLIB::pic::zlib
         NGHTTP2::pic::nghttp2
         ZSTD::pic::zstd
+        RE2::pic::re2
         m
         pthread
 )
@@ -213,6 +215,7 @@ set(RUNTIME_LIBS
         ZLIB::${PIC_MODE}::zlib
         NGHTTP2::${PIC_MODE}::nghttp2
         ZSTD::${PIC_MODE}::zstd
+        RE2::${PIC_MODE}::re2
         m
         pthread
 )
@@ -266,7 +269,7 @@ file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/php_lib_version.cpp
 ]])
 
 add_library(php_lib_version_j OBJECT ${CMAKE_CURRENT_BINARY_DIR}/php_lib_version.cpp)
-target_include_directories(php_lib_version_j PUBLIC ${OPENSSL_INCLUDE_DIR} ${ZLIB_NO_PIC_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIRS})
+target_include_directories(php_lib_version_j PUBLIC ${ZLIB_NO_PIC_INCLUDE_DIRS})
 target_compile_options(php_lib_version_j PRIVATE -I. -E)
 add_dependencies(php_lib_version_j kphp-full-runtime-no-pic kphp-full-runtime-pic)
 

server/confdata-binlog-replay.h

@@ -3,7 +3,7 @@
 // Distributed under the GPL v3 License, see LICENSE.notice.txt
 
 #pragma once
-#include <re2/re2.h>
+#include "re2/re2.h"
 
 #include "common/stats/provider.h"
 

server/php-engine.cpp

@@ -14,7 +14,7 @@
 #include <netdb.h>
 #include <netinet/in.h>
 #include <poll.h>
-#include <re2/re2.h>
+#include "re2/re2.h"
 #include <string>
 #include <sys/mman.h>
 #include <sys/socket.h>

server/server.cmake

@@ -73,5 +73,11 @@ set(KPHP_SERVER_ALL_SOURCES
     ${KPHP_DATABASE_DRIVERS_PGSQL_SOURCES})
 
 allow_deprecated_declarations_for_apple(${BASE_DIR}/server/php-runner.cpp)
+
 vk_add_library_no_pic(kphp-server-no-pic OBJECT ${KPHP_SERVER_ALL_SOURCES})
+add_dependencies(kphp-server-no-pic RE2::no-pic::re2)
+target_include_directories(kphp-server-no-pic PUBLIC ${RE2_NO_PIC_INCLUDE_DIRS})
+
 vk_add_library_pic(kphp-server-pic OBJECT ${KPHP_SERVER_ALL_SOURCES})
+add_dependencies(kphp-server-pic RE2::pic::re2)
+target_include_directories(kphp-server-pic PUBLIC ${RE2_NO_PIC_INCLUDE_DIRS})

tests/python/requirements.txt

@@ -11,3 +11,4 @@ pytest-postgresql==4.1.1
 psycopg==3.1.9
 pytest-xdist==3.3.1
 zstandard==0.21.0
+six==1.17.0

third-party/re2-cmake/patches/kphp-unicode-data.patch.template

@@ -0,0 +1,20 @@
+The Unicode data from the KPHP project should be used, rather than
+downloading from unicode.org.
+
+---
+ re2/unicode.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/re2/unicode.py b/re2/unicode.py
+index e0f33ef..651b7cc 100644
+--- a/re2/unicode.py
++++ b/re2/unicode.py
+@@ -13,7 +13,7 @@ import re
+ from six.moves import urllib
+
+ # Directory or URL where Unicode tables reside.
+-_UNICODE_DIR = "/usr/share/unicode"
++_UNICODE_DIR = "${UNICODE_AUTO_RAW_DATA_DIR}"
+
+ # Largest valid Unicode code value.
+ _RUNE_MAX = 0x10FFFF

third-party/re2-cmake/patches/series

@@ -0,0 +1 @@
+kphp-unicode-data.patch