dependencies vendoring, part 1 (#1228)

* Add OpenSSL and CURL vendoring * Fix github CI * Suppress false-positive warining caused by O3 on gcc-8.3 * Disable debug-info for third-party * Use conservative optimization level for third-party Signed-off-by: Petr Shumilov <[email protected]>
VKCOM · Jan 31, 2025 · be4e6c8 · be4e6c8
1 parent 7d80b66
commit be4e6c8
Show file tree

Hide file tree

Showing 30 changed files with 275 additions and 100 deletions.
diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
@@ -42,16 +42,16 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get polyfills repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: 'VKCOM/kphp-polyfills'
         path: 'kphp-polyfills'
 
     - name: Cache docker image
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: docker-image-cache
       with:
         path: kphp-build-env-${{matrix.os}}.tar
@@ -77,11 +77,14 @@ jobs:
 
     - name: Add git safe directory
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
-              "git config --global --add safe.directory ${{env.kphp_root_dir}}"
+              "git config --global --add safe.directory '*'"
+      # This command is used to address potential issues with Git's safe directory feature.
+      # By setting '*' as a safe directory, we allow Git operations to proceed without errors
+      # related to directory safety, ensuring smooth execution of the submodules updating.
 
     - name: Build all
       run: docker exec kphp-build-container-${{matrix.os}} bash -c 
-              "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all"
+              "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=OFF -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all"
 
     - name: Run unit tests
       run: docker exec kphp-build-container-${{matrix.os}} bash -c 
@@ -107,7 +110,7 @@ jobs:
            rm -rf ${{runner.temp}}/_tmp/*/working_dir
 
     - name: Upload python tests artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: steps.python_tests.outcome == 'failure'
       with:
         path: ${{runner.temp}}/_tmp/
@@ -131,7 +134,7 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # because of https://github.com/orgs/Homebrew/discussions/4612
     - name: Check Environment
@@ -143,7 +146,7 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install [email protected] re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
+        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
         brew link --overwrite --force shivammathur/php/[email protected]
         /opt/homebrew/opt/[email protected]/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/[email protected]/libexec/bin/pip install --break-system-packages jsonschema
 

diff --git a/.github/workflows/Dockerfile.buster b/.github/workflows/Dockerfile.buster
@@ -22,7 +22,7 @@ RUN apt-get update && \
       git cmake-data=3.18* cmake=3.18* make g++ gperf netcat \
       python3.7 python3-dev libpython3-dev python3-pip python3-setuptools python3-wheel mysql-server libmysqlclient-dev && \
     pip3 install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer && \
     rm -rf /var/lib/apt/lists/* && \

diff --git a/.github/workflows/Dockerfile.focal b/.github/workflows/Dockerfile.focal
@@ -15,7 +15,7 @@ RUN apt-get update && \
       git cmake make clang g++ g++-10 gperf netcat \
       python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
-    apt-get install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+    apt-get install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
       libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libldap-dev libkrb5-dev \
       libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* libnuma-dev composer unzip && \
     rm -rf /var/lib/apt/lists/*

diff --git a/.github/workflows/Dockerfile.jammy b/.github/workflows/Dockerfile.jammy
@@ -16,12 +16,18 @@ RUN apt update && \
     add-apt-repository ppa:deadsnakes/ppa && \
     apt update && \
     apt install -y --no-install-recommends \
-      git cmake make g++ lld gperf netcat \
-      python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
+      build-essential devscripts fakeroot git cmake make g++ lld gperf netcat \
+      python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools && \
+    apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
+      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip  && \
+    # Install MySQL-related and PostgreSQL-related libraries from the Ubuntu 20.04 (Focal Fossa) repository.
+    # Ubuntu 22.04 has moved to OpenSSL 3.0; however, KPHP relies on OpenSSL 1.1.1.
+    # Therefore, we use downgraded libraries that are compatible with OpenSSL 1.1.1.
+    echo "deb https://archive.ubuntu.com/ubuntu focal main universe" >> /etc/apt/sources.list && \
+    apt update && \
+    apt install -t focal -y --no-install-recommends --allow-downgrades mysql-server libmysqlclient-dev \
+      libldap-dev libtinfo6=6.2* ncurses-bin=6.2* libncurses6=6.2* libncursesw6=6.2* libncurses-dev=6.2* libtinfo-dev=6.2* libcom-err2=1.45* comerr-dev=2.1-1.45* libkrb5support0=1.17* libkrb5-3=1.17* libk5crypto3=1.17* libgssapi-krb5-2=1.17* libkrb5-dev=1.17* libpq5=12.* ssl-cert=1.1.* postgresql-common locales=2.35* postgresql-12 libpq-dev=12.*  && \
     python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
-    apt install -y --no-install-recommends curl-kphp-vk kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip \
-      libldap-dev libkrb5-dev libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* && \
     rm -rf /var/lib/apt/lists/*
 
 # set php7.4 as default

diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml
@@ -25,16 +25,16 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get polyfills repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: 'VKCOM/kphp-polyfills'
         path: 'kphp-polyfills'
 
     - name: Cache docker image
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: docker-image-cache
       with:
         path: kphp-build-env-${{matrix.os}}.tar

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -21,7 +21,7 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
       # because of https://github.com/orgs/Homebrew/discussions/4612
     - name: Check Environment
@@ -33,7 +33,7 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install [email protected] re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
+        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
         brew link --overwrite --force shivammathur/php/[email protected]
         /opt/homebrew/opt/[email protected]/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/[email protected]/libexec/bin/pip install --break-system-packages jsonschema
       

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -35,16 +35,16 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get polyfills repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: 'VKCOM/kphp-polyfills'
         path: 'kphp-polyfills'
 
     - name: Cache docker image
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: docker-image-cache
       with:
         path: kphp-build-env-${{matrix.os}}.tar

diff --git a/.gitmodules b/.gitmodules
@@ -7,3 +7,11 @@
 [submodule "third-party/pcre2"]
 	path = third-party/pcre2
 	url = https://github.com/PCRE2Project/pcre2.git
+[submodule "third-party/openssl"]
+	path = third-party/openssl
+	url = https://github.com/openssl/openssl.git
+	branch = OpenSSL_1_1_1-stable
+[submodule "third-party/curl"]
+	path = third-party/curl
+	url = https://github.com/VKCOM/curl.git
+	branch = dpkg-build-7.60.0
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -20,6 +20,7 @@ include(CheckCXXCompilerFlag)
 include(AddFileDependencies)
 include(FetchContent)
 include(GNUInstallDirs)
+include(ExternalProject)
 
 # Global includes must be before all other includes/add_subdirectories
 include(cmake/utils.cmake)
@@ -29,6 +30,9 @@ include(cmake/external-libraries.cmake)
 include(cmake/init-compilation-flags.cmake)
 include(cmake/popular-common.cmake)
 
+# OpenSSL building
+include(${THIRD_PARTY_DIR}/openssl-cmake/openssl.cmake)
+
 # TODO: use FetchContent_Declare instead of include_directories
 include_directories(.)
 
@@ -85,6 +89,13 @@ else ()
             ARCHIVE DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
 endif ()
 
+install(DIRECTORY ${OBJS_DIR}/include
+        COMPONENT KPHP
+        DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
+install(DIRECTORY ${OBJS_DIR}/lib
+        COMPONENT KPHP
+        DESTINATION ${INSTALL_KPHP_SOURCE}/objs)
+
 install(DIRECTORY ${COMMON_DIR}
                   ${BASE_DIR}/runtime
 		  ${RUNTIME_COMMON_DIR}
@@ -110,7 +121,7 @@ install(FILES ${AUTO_DIR}/runtime/runtime-headers.h
         COMPONENT KPHP
         DESTINATION ${INSTALL_KPHP_SOURCE}/objs/generated/auto/runtime/)
 
-set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "vk-flex-data, curl-kphp-vk, libuber-h3, libpcre3-dev, libre2-dev, libyaml-cpp-dev, libssl-dev, zlib1g-dev, \
+set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "vk-flex-data, libuber-h3, libpcre3-dev, libre2-dev, libyaml-cpp-dev, zlib1g-dev, \
                                        libzstd-dev, g++, libnghttp2-dev, kphp-timelib, libnuma-dev")
 
 if (PDO_DRIVER_MYSQL)

diff --git a/cmake/init-compilation-flags.cmake b/cmake/init-compilation-flags.cmake
@@ -49,9 +49,6 @@ else()
     add_link_options(-fno-lto)
 endif()
 
-set(OPENSSL_USE_STATIC_LIBS TRUE)
-find_package(OpenSSL REQUIRED)
-include_directories(${OPENSSL_INCLUDE_DIR})
 
 option(ADDRESS_SANITIZER "Enable address sanitizer")
 if(ADDRESS_SANITIZER)

diff --git a/cmake/init-global-vars.cmake b/cmake/init-global-vars.cmake
@@ -11,10 +11,8 @@ set(RUNTIME_LIGHT_DIR "${BASE_DIR}/runtime-light")
 set(RUNTIME_COMMON_DIR "${BASE_DIR}/runtime-common")
 
 if(APPLE)
-    set(CURL_LIB curl)
     set(ICONV_LIB iconv)
 else()
-    set(CURL_LIB /opt/curl7600/lib/libcurl.a)
     set(RT_LIB rt)
     set(NUMA_LIB numa)
 endif()

diff --git a/cmake/utils.cmake b/cmake/utils.cmake
@@ -56,19 +56,25 @@ function(check_compiler_version compiler_name compiler_version)
     endif()
 endfunction(check_compiler_version)
 
-# Function to initialize and update Git submodules
-function(update_git_submodules)
-        message(STATUS "Updating Git submodules...")
+# Function to initialize and update specific Git submodule
+function(update_git_submodule submodule_path)
+    message(STATUS "Updating Git submodule ${submodule_path} ...")
 
-        # Update submodules
-        execute_process(
-            COMMAND git submodule update --init --recursive
-            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-            RESULT_VARIABLE update_result
-            ERROR_QUIET
-        )
+    # Update submodules
+    execute_process(
+            COMMAND ${GIT_EXECUTABLE} submodule update --init --remote ${ARGN} ${submodule_path}
+            WORKING_DIRECTORY ${BASE_DIR}
+            RESULT_VARIABLE update_return_code
+            OUTPUT_VARIABLE update_stdout
+            ERROR_VARIABLE update_stderr
+    )
 
-        if(NOT update_result EQUAL 0)
-            message(FATAL_ERROR "Failed to update Git submodules.")
-        endif()
+    if(NOT update_return_code EQUAL 0)
+        message(FATAL_ERROR "Failed to update Git submodule ${submodule_path}: ${update_stdout} ${update_stderr}")
+    endif()
+endfunction()
+
+function(update_git_submodule_recursive submodule_path)
+    set(extra_option --recursive)
+    update_git_submodule(${submodule_path} ${extra_option})
 endfunction()
diff --git a/common/crypto/aes256-generic.cpp b/common/crypto/aes256-generic.cpp
@@ -2,11 +2,11 @@
 // Copyright (c) 2020 LLC «V Kontakte»
 // Distributed under the GPL v3 License, see LICENSE.notice.txt
 
-#include "common/crypto/aes256-generic.h"
-
 #include <string.h>
 
-#include <openssl/aes.h>
+#include "openssl/aes.h"
+
+#include "common/crypto/aes256-generic.h"
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"

diff --git a/common/crypto/aes256.h b/common/crypto/aes256.h
@@ -8,7 +8,7 @@
 #include <sys/cdefs.h>
 #include <stdint.h>
 
-#include <openssl/aes.h>
+#include "openssl/aes.h"
 
 #define AES256_KEY_BITS 256
 

diff --git a/common/kfs/kfs-binlog.cpp b/common/kfs/kfs-binlog.cpp
@@ -8,7 +8,6 @@
 #include <cstring>
 #include <cstdlib>
 #include <fcntl.h>
-#include <openssl/rand.h>
 #include <sys/stat.h>
 #include <unistd.h>
 

diff --git a/common/sha1.h b/common/sha1.h
@@ -5,7 +5,7 @@
 #ifndef __SHA1_H__
 #define __SHA1_H__
 
-#include <openssl/sha.h>
+#include "openssl/sha.h"
 
 void sha1 (unsigned char *input, int ilen, unsigned char output[20]);
 

diff --git a/common/wrappers/openssl.h b/common/wrappers/openssl.h
@@ -5,13 +5,14 @@
 #ifndef KDB_COMMON_OPENSSL_H
 #define KDB_COMMON_OPENSSL_H
 
-#include <openssl/crypto.h>
-#include <openssl/engine.h>
-#include <openssl/evp.h>
-#include <openssl/opensslv.h>
 #include <stddef.h>
 #include <string.h>
 
+#include "openssl/crypto.h"
+#include "openssl/engine.h"
+#include "openssl/evp.h"
+#include "openssl/opensslv.h"
+
 #if OPENSSL_VERSION_NUMBER < 0x10100000L
 
 static inline RSA *EVP_PKEY_get0_RSA(EVP_PKEY *pkey) {