dependencies vendoring, part 2

.github/workflows/Build.yml

@@ -146,9 +146,9 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install python@3.13 re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4
+        brew install python@3.13 re2c cmake coreutils libiconv pcre yaml-cpp googletest shivammathur/php/php@7.4
         brew link --overwrite --force shivammathur/php/php@7.4
-        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema
+        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema six
 
     - name: Run cmake
       run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build

.github/workflows/Dockerfile.buster

@@ -19,11 +19,11 @@ RUN apt-get update && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-      git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \
+      git cmake-data=3.18* cmake=3.18* make g++ gperf netcat patch \
       python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \
     pip3 install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libpcre3-dev \
+      libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer && \
     rm -rf /var/lib/apt/lists/* && \
     update-alternatives --set php /usr/bin/php7.4

.github/workflows/Dockerfile.focal

@@ -12,11 +12,11 @@ RUN apt-get update && \
     add-apt-repository ppa:deadsnakes/ppa && \
     apt-get update && \
     apt-get install -y --no-install-recommends \
-      git cmake make clang g++ g++-10 gperf netcat \
+      git cmake make clang g++ g++-10 gperf netcat patch \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libpcre3-dev \
+      libyaml-cpp-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer unzip && \
     rm -rf /var/lib/apt/lists/*
 

.github/workflows/Dockerfile.jammy

@@ -16,10 +16,10 @@ RUN apt update && \
     add-apt-repository ppa:deadsnakes/ppa && \
     apt update && \
     apt install -y --no-install-recommends \
-      build-essential devscripts fakeroot git cmake make g++ lld gperf netcat \
+      build-essential devscripts fakeroot git cmake make g++ lld gperf netcat patch \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools && \
-    apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip  && \
+    apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libpcre3-dev \
+      libyaml-cpp-dev zlib1g-dev php7.4-dev libnuma-dev unzip  && \
     # Install MySQL-related and PostgreSQL-related libraries from the Ubuntu 20.04 (Focal Fossa) repository.
     # Ubuntu 22.04 has moved to OpenSSL 3.0; however, KPHP relies on OpenSSL 1.1.1.
     # Therefore, we use downgraded libraries that are compatible with OpenSSL 1.1.1.

.github/workflows/macos.yml

@@ -33,9 +33,9 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install python@3.13 re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/php@7.4
+        brew install python@3.13 re2c cmake coreutils libiconv pcre yaml-cpp googletest shivammathur/php/php@7.4
         brew link --overwrite --force shivammathur/php/php@7.4
-        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema
+        /opt/homebrew/opt/python@3.13/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/python@3.13/libexec/bin/pip install --break-system-packages jsonschema six
       
     - name: Run cmake
       run: cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DDOWNLOAD_MISSING_LIBRARIES=On -S $GITHUB_WORKSPACE -B ${{runner.workspace}}/build

.gitmodules

@@ -9,9 +9,20 @@
 	url = https://github.com/PCRE2Project/pcre2.git
 [submodule "third-party/openssl"]
 	path = third-party/openssl
-	url = https://github.com/openssl/openssl.git
-	branch = OpenSSL_1_1_1-stable
+	url = https://github.com/VKCOM/openssl.git
+	branch = debian/buster
 [submodule "third-party/curl"]
 	path = third-party/curl
 	url = https://github.com/VKCOM/curl.git
 	branch = dpkg-build-7.60.0
+[submodule "third-party/nghttp2"]
+	path = third-party/nghttp2
+	url = https://github.com/VKCOM/nghttp2.git
+	branch = buster
+[submodule "third-party/zstd"]
+	path = third-party/zstd
+	url = https://github.com/VKCOM/libzstd.git
+	branch = master
+[submodule "third-party/re2"]
+	path = third-party/re2
+	url = https://github.com/VKCOM/re2.git

CMakeLists.txt

@@ -1,7 +1,7 @@
 cmake_minimum_required(VERSION 3.16)
 project(kphp
         VERSION 1.0.1
-        LANGUAGES CXX ASM
+        LANGUAGES C CXX ASM
         DESCRIPTION "Compiler for PHP (aka KPHP)"
         HOMEPAGE_URL https://github.com/VKCOM/kphp)
 
@@ -30,11 +30,17 @@ include(cmake/external-libraries.cmake)
 include(cmake/init-compilation-flags.cmake)
 include(cmake/popular-common.cmake)
 
-# OpenSSL building
+# Common third-parties build
+include(${THIRD_PARTY_DIR}/zlib-cmake/zlib.cmake)
 include(${THIRD_PARTY_DIR}/openssl-cmake/openssl.cmake)
+include(${THIRD_PARTY_DIR}/zstd-cmake/zstd.cmake)
+include(${COMMON_DIR}/unicode/unicode.cmake)
+# RE2 depends on unicode
+include(${THIRD_PARTY_DIR}/re2-cmake/re2.cmake)
 
 # TODO: use FetchContent_Declare instead of include_directories
 include_directories(.)
+include_directories(${INCLUDE_DIR})
 
 # Custom modules
 include(${BASE_DIR}/flex/flex.cmake)
@@ -44,7 +50,6 @@ include(${BASE_DIR}/vkext/vkext.cmake)
 include(${COMMON_DIR}/binlog/binlog.cmake)
 include(${COMMON_DIR}/common.cmake)
 include(${COMMON_DIR}/tl/tl.cmake)
-include(${COMMON_DIR}/unicode/unicode.cmake)
 
 include(${RUNTIME_COMMON_DIR}/runtime-common.cmake)
 
@@ -68,9 +73,9 @@ endif()
 add_custom_target(kphp ALL DEPENDS ${OBJS_DIR}/php_lib_version.sha256)
 
 if (COMPILE_RUNTIME_LIGHT)
-    add_dependencies(kphp kphp2cpp kphp-light-runtime)
+    add_dependencies(kphp kphp2cpp kphp-light-runtime-pic)
 else ()
-    add_dependencies(kphp kphp2cpp kphp-full-runtime)
+    add_dependencies(kphp kphp2cpp kphp-full-runtime-no-pic kphp-full-runtime-pic)
 endif ()
 
 install(TARGETS kphp2cpp
@@ -79,14 +84,16 @@ install(TARGETS kphp2cpp
 install_symlink(${VK_INSTALL_DIR}/bin/kphp2cpp ${CMAKE_INSTALL_PREFIX}/bin/kphp KPHP)
 
 if (COMPILE_RUNTIME_LIGHT)
-    install(TARGETS kphp-light-runtime
+    install(TARGETS kphp-light-runtime-pic
             COMPONENT KPHP
             LIBRARY DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
 else ()
-    install(TARGETS kphp-full-runtime
+    install(FILES ${OBJS_DIR}/libkphp-full-runtime-no-pic.a
+            COMPONENT KPHP
+            DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
+    install(FILES ${OBJS_DIR}/libkphp-full-runtime-pic.a
             COMPONENT KPHP
-            LIBRARY DESTINATION ${INSTALL_KPHP_SOURCE}/objs
-            ARCHIVE DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
+            DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
 endif ()
 
 install(DIRECTORY ${OBJS_DIR}/include
@@ -121,8 +128,8 @@ install(FILES ${AUTO_DIR}/runtime/runtime-headers.h
         COMPONENT KPHP
         DESTINATION ${INSTALL_KPHP_SOURCE}/objs/generated/auto/runtime/)
 
-set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "vk-flex-data, libuber-h3, libpcre3-dev, libre2-dev, libyaml-cpp-dev, zlib1g-dev, \
-                                       libzstd-dev, g++, libnghttp2-dev, kphp-timelib, libnuma-dev")
+set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "vk-flex-data, libuber-h3, libpcre3-dev, libre2-dev, libyaml-cpp-dev, \
+                                       g++, kphp-timelib, libnuma-dev")
 
 if (PDO_DRIVER_MYSQL)
     set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "${CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS}, libmysqlclient-dev")

cmake/apply_patches.cmake

@@ -0,0 +1,35 @@
+function(apply_patches_from_series build_dir series_file patch_dir)
+    # Check for the presence of the patch utility
+    find_program(PATCH_EXECUTABLE patch)
+    if(NOT PATCH_EXECUTABLE)
+        message(FATAL_ERROR "The 'patch' utility is not found on this system.")
+    endif()
+
+    # Read the series file and apply each patch listed
+    file(READ "${series_file}" series_content)
+    string(REPLACE "\n" ";" series_list "${series_content}")
+
+    foreach(patch IN LISTS series_list)
+        if(NOT patch STREQUAL "")
+            # Construct the full path to the patch file
+            set(patch_file "${patch_dir}${patch}")
+
+            # Apply the patch using GNU patch
+            execute_process(
+                    COMMAND ${PATCH_EXECUTABLE} -p1 -i "${patch_file}"
+                    WORKING_DIRECTORY "${build_dir}"
+                    RESULT_VARIABLE result
+                    OUTPUT_VARIABLE output
+                    ERROR_VARIABLE error
+            )
+
+            if(NOT result EQUAL 0)
+                message(FATAL_ERROR "Failed to apply patch: ${patch}\nOutput: ${output}\nError: ${error}")
+            else()
+                message(STATUS "Applied patch: ${patch}")
+            endif()
+        endif()
+    endforeach()
+endfunction()
+
+apply_patches_from_series(${BUILD_DIR} ${PATCH_SERIES} ${PATCH_DIR})

cmake/external-libraries.cmake

@@ -66,6 +66,7 @@ else()
     )
     message(STATUS "---------------------")
     FetchContent_MakeAvailable(h3)
+    set_target_properties(h3 PROPERTIES DOWNLOADED_LIBRARY 1)
     include_directories(${h3_BINARY_DIR}/src/include)
     add_definitions(-DKPHP_H3_LIB_DIR="${h3_BINARY_DIR}/lib")
     add_link_options(-L${h3_BINARY_DIR}/lib)

cmake/init-global-vars.cmake

@@ -5,20 +5,32 @@ set(COMMON_DIR "${BASE_DIR}/common")
 set(THIRD_PARTY_DIR "${BASE_DIR}/third-party")
 set(OBJS_DIR ${BASE_DIR}/objs)
 set(BIN_DIR ${OBJS_DIR}/bin)
+set(LIB_DIR ${OBJS_DIR}/lib)
+set(INCLUDE_DIR ${OBJS_DIR}/include)
 set(GENERATED_DIR "${OBJS_DIR}/generated")
 set(AUTO_DIR "${GENERATED_DIR}/auto")
 set(RUNTIME_LIGHT_DIR "${BASE_DIR}/runtime-light")
 set(RUNTIME_COMMON_DIR "${BASE_DIR}/runtime-common")
 
+set(PIC_NAMESPACE "pic")
+set(PIC_LIBRARY_SUFFIX "-${PIC_NAMESPACE}")
+set(PIC_LIBRARY_SPECIFIER "PIC")
+
+set(NO_PIC_NAMESPACE "no-pic")
+set(NO_PIC_LIBRARY_SUFFIX "-${NO_PIC_NAMESPACE}")
+set(NO_PIC_LIBRARY_SPECIFIER "NO_PIC")
+
 if(APPLE)
+    detect_xcode_sdk_path(CMAKE_OSX_SYSROOT)
     set(ICONV_LIB iconv)
 else()
     set(RT_LIB rt)
     set(NUMA_LIB numa)
 endif()
 
 find_package(Git REQUIRED)
-find_package (Python3 COMPONENTS Interpreter REQUIRED)
+find_package(Python3 COMPONENTS Interpreter REQUIRED)
+find_package(Perl REQUIRED)
 
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND)
@@ -28,11 +40,15 @@ endif()
 
 find_program(PHP_BIN php REQUIRED)
 
-if(NOT APPLE)
-    check_cxx_compiler_flag(-no-pie NO_PIE_IS_FOUND)
-    if(NO_PIE_IS_FOUND)
-        set(NO_PIE -no-pie)
-    endif()
+check_cxx_compiler_flag(-no-pie NO_PIE_IS_FOUND)
+if(NO_PIE_IS_FOUND)
+    set(PIE_MODE_FLAGS -no-pie)
+    set(PIC_MODE_ENABLED ON)
+    set(PIC_MODE ${NO_PIC_NAMESPACE})
+else()
+    set(PIE_MODE_FLAGS)
+    set(PIC_MODE_ENABLED OFF)
+    set(PIC_MODE ${PIC_NAMESPACE})
 endif()
 
 # add extra build type release without NDEBUG flag
@@ -94,4 +110,8 @@ else()
     set(DEFAULT_KPHP_PATH ${BASE_DIR})
 endif()
 
+# Ensure the dir for dependencies will be created
+file(MAKE_DIRECTORY ${LIB_DIR})
+file(MAKE_DIRECTORY ${INCLUDE_DIR})
+
 cmake_print_variables(CMAKE_INSTALL_PREFIX CPACK_PACKAGING_INSTALL_PREFIX)

cmake/popular-common.cmake

@@ -45,11 +45,10 @@ if(APPLE)
 endif()
 
 if(COMPILE_RUNTIME_LIGHT)
-  vk_add_library(light-common OBJECT ${LIGHT_COMMON_SOURCES})
-  set_property(TARGET light-common PROPERTY POSITION_INDEPENDENT_CODE ON)
-  target_compile_options(light-common PUBLIC -stdlib=libc++ -fPIC)
-  target_link_options(light-common PUBLIC -stdlib=libc++)
+  vk_add_library_pic(light-common-pic OBJECT ${LIGHT_COMMON_SOURCES})
+  target_compile_options(light-common-pic PUBLIC -stdlib=libc++)
+  target_link_options(light-common-pic PUBLIC -stdlib=libc++)
 endif()
 
-vk_add_library(popular_common OBJECT ${POPULAR_COMMON_SOURCES})
-set_property(TARGET popular_common PROPERTY POSITION_INDEPENDENT_CODE ON)
+vk_add_library_no_pic(popular-common-no-pic OBJECT ${POPULAR_COMMON_SOURCES})
+vk_add_library_pic(popular-common-pic OBJECT ${POPULAR_COMMON_SOURCES})