Fix github CI

Signed-off-by: Petr Shumilov <[email protected]>
VKCOM · Jan 30, 2025 · 4ab00e3 · 4ab00e3
1 parent db7276c
commit 4ab00e3
Show file tree

Hide file tree

Showing 14 changed files with 142 additions and 89 deletions.
diff --git a/.github/workflows/Build.yml b/.github/workflows/Build.yml
@@ -42,16 +42,16 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get polyfills repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: 'VKCOM/kphp-polyfills'
         path: 'kphp-polyfills'
 
     - name: Cache docker image
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: docker-image-cache
       with:
         path: kphp-build-env-${{matrix.os}}.tar
@@ -77,11 +77,16 @@ jobs:
 
     - name: Add git safe directory
       run: docker exec kphp-build-container-${{matrix.os}} bash -c
-              "git config --global --add safe.directory ${{env.kphp_root_dir}}"
+              "git config --global --add safe.directory '*'"
+      # This command is used to address potential issues with Git's safe directory feature.
+      # This is particularly relevant in CI environments where the repository
+      # might be checked out by a different user than the one executing the workflow steps.
+      # By setting '*' as a safe directory, we allow Git operations to proceed without errors
+      # related to directory safety, ensuring smooth execution of the workflow.
 
     - name: Build all
       run: docker exec kphp-build-container-${{matrix.os}} bash -c 
-              "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=ON -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all"
+              "cmake -DCMAKE_CXX_COMPILER=${{matrix.compiler}} -DCMAKE_CXX_STANDARD=${{matrix.cpp}} -DADDRESS_SANITIZER=${{matrix.asan}} -DUNDEFINED_SANITIZER=${{matrix.ubsan}} -DPDO_DRIVER_MYSQL=ON -DPDO_DRIVER_PGSQL=ON -DPDO_LIBS_STATIC_LINKING=OFF -S ${{env.kphp_root_dir}} -B ${{env.kphp_build_dir}} && make -C ${{env.kphp_build_dir}} -j$(nproc) all"
 
     - name: Run unit tests
       run: docker exec kphp-build-container-${{matrix.os}} bash -c 
@@ -107,7 +112,7 @@ jobs:
            rm -rf ${{runner.temp}}/_tmp/*/working_dir
 
     - name: Upload python tests artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       if: steps.python_tests.outcome == 'failure'
       with:
         path: ${{runner.temp}}/_tmp/
@@ -131,7 +136,7 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     # because of https://github.com/orgs/Homebrew/discussions/4612
     - name: Check Environment
@@ -143,7 +148,7 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install [email protected] re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
+        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
         brew link --overwrite --force shivammathur/php/[email protected]
         /opt/homebrew/opt/[email protected]/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/[email protected]/libexec/bin/pip install --break-system-packages jsonschema
 

diff --git a/.github/workflows/Dockerfile.jammy b/.github/workflows/Dockerfile.jammy
@@ -16,12 +16,18 @@ RUN apt update && \
     add-apt-repository ppa:deadsnakes/ppa && \
     apt update && \
     apt install -y --no-install-recommends \
-      git cmake make g++ lld gperf netcat \
-      python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools mysql-server libmysqlclient-dev && \
-    python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
+      build-essential devscripts fakeroot git cmake make g++ lld gperf netcat \
+      python3.7 python3-pip python3.7-distutils python3.7-dev libpython3.7-dev python3-jsonschema python3-setuptools && \
     apt install -y --no-install-recommends kphp-timelib libuber-h3-dev libfmt-dev libgtest-dev libgmock-dev libre2-dev libpcre3-dev \
-      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip \
-      libldap-dev libkrb5-dev libpq5=14.* postgresql-14 postgresql-server-dev-14 libpq-dev=14.* && \
+      libzstd-dev libyaml-cpp-dev libnghttp2-dev zlib1g-dev php7.4-dev libnuma-dev unzip  && \
+    # Install MySQL-related and PostgreSQL-related libraries from the Ubuntu 20.04 (Focal Fossa) repository.
+    # Ubuntu 22.04 has moved to OpenSSL 3.0; however, KPHP relies on OpenSSL 1.1.1.
+    # Therefore, we use downgraded libraries that are compatible with OpenSSL 1.1.1.
+    echo "deb https://archive.ubuntu.com/ubuntu focal main universe" >> /etc/apt/sources.list && \
+    apt update && \
+    apt install -t focal -y --no-install-recommends --allow-downgrades mysql-server libmysqlclient-dev \
+      libldap-dev libtinfo6=6.2* ncurses-bin=6.2* libncurses6=6.2* libncursesw6=6.2* libncurses-dev=6.2* libtinfo-dev=6.2* libcom-err2=1.45* comerr-dev=2.1-1.45* libkrb5support0=1.17* libkrb5-3=1.17* libk5crypto3=1.17* libgssapi-krb5-2=1.17* libkrb5-dev=1.17* libpq5=12.* ssl-cert=1.1.* postgresql-common locales=2.35* postgresql-12 libpq-dev=12.*  && \
+    python3.7 -m pip install pip && python3.7 -m pip install -r /tmp/requirements.txt && \
     rm -rf /var/lib/apt/lists/*
 
 # set php7.4 as default

diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml
@@ -25,16 +25,16 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get polyfills repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: 'VKCOM/kphp-polyfills'
         path: 'kphp-polyfills'
 
     - name: Cache docker image
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: docker-image-cache
       with:
         path: kphp-build-env-${{matrix.os}}.tar

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -21,7 +21,7 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
       # because of https://github.com/orgs/Homebrew/discussions/4612
     - name: Check Environment
@@ -33,7 +33,7 @@ jobs:
       run: |
         brew tap shivammathur/php
         brew update
-        brew install [email protected] re2c cmake coreutils openssl libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
+        brew install [email protected] re2c cmake coreutils libiconv re2 pcre yaml-cpp zstd googletest shivammathur/php/[email protected]
         brew link --overwrite --force shivammathur/php/[email protected]
         /opt/homebrew/opt/[email protected]/libexec/bin/python -m pip install --upgrade pip --break-system-packages && /opt/homebrew/opt/[email protected]/libexec/bin/pip install --break-system-packages jsonschema
       

diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -35,16 +35,16 @@ jobs:
     name: "${{matrix.os}}/${{matrix.compiler}}/c++${{matrix.cpp}}/asan=${{matrix.asan}}/ubsan=${{matrix.ubsan}}"
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Get polyfills repo
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
       with:
         repository: 'VKCOM/kphp-polyfills'
         path: 'kphp-polyfills'
 
     - name: Cache docker image
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       id: docker-image-cache
       with:
         path: kphp-build-env-${{matrix.os}}.tar

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -30,9 +30,8 @@ include(cmake/external-libraries.cmake)
 include(cmake/init-compilation-flags.cmake)
 include(cmake/popular-common.cmake)
 
-update_git_submodules()
+# OpenSSL building
 include(${THIRD_PARTY_DIR}/openssl-cmake/openssl.cmake)
-include(${THIRD_PARTY_DIR}/curl-cmake/curl.cmake)
 
 # TODO: use FetchContent_Declare instead of include_directories
 include_directories(.)
@@ -122,7 +121,7 @@ install(FILES ${AUTO_DIR}/runtime/runtime-headers.h
         COMPONENT KPHP
         DESTINATION ${INSTALL_KPHP_SOURCE}/objs/generated/auto/runtime/)
 
-set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "vk-flex-data, curl-kphp-vk, libuber-h3, libpcre3-dev, libre2-dev, libyaml-cpp-dev, libssl-dev, zlib1g-dev, \
+set(CPACK_DEBIAN_KPHP_PACKAGE_DEPENDS "libuber-h3, libpcre3-dev, libre2-dev, libyaml-cpp-dev, zlib1g-dev, \
                                        libzstd-dev, g++, libnghttp2-dev, kphp-timelib, libnuma-dev")
 
 if (PDO_DRIVER_MYSQL)

diff --git a/cmake/init-global-vars.cmake b/cmake/init-global-vars.cmake
@@ -11,10 +11,8 @@ set(RUNTIME_LIGHT_DIR "${BASE_DIR}/runtime-light")
 set(RUNTIME_COMMON_DIR "${BASE_DIR}/runtime-common")
 
 if(APPLE)
-    set(CURL_LIB curl)
     set(ICONV_LIB iconv)
 else()
-    set(CURL_LIB /opt/curl7600/lib/libcurl.a)
     set(RT_LIB rt)
     set(NUMA_LIB numa)
 endif()

diff --git a/cmake/utils.cmake b/cmake/utils.cmake
@@ -56,19 +56,25 @@ function(check_compiler_version compiler_name compiler_version)
     endif()
 endfunction(check_compiler_version)
 
-# Function to initialize and update Git submodules
-function(update_git_submodules)
-        message(STATUS "Updating Git submodules...")
+# Function to initialize and update specific Git submodule
+function(update_git_submodule submodule_path)
+    message(STATUS "Updating Git submodule ${submodule_path} ...")
 
-        # Update submodules
-        execute_process(
-            COMMAND git submodule update --init --recursive
-            WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
-            RESULT_VARIABLE update_result
-            ERROR_QUIET
-        )
+    # Update submodules
+    execute_process(
+            COMMAND ${GIT_EXECUTABLE} submodule update --init --remote ${ARGN} ${submodule_path}
+            WORKING_DIRECTORY ${BASE_DIR}
+            RESULT_VARIABLE update_return_code
+            OUTPUT_VARIABLE update_stdout
+            ERROR_VARIABLE update_stderr
+    )
 
-        if(NOT update_result EQUAL 0)
-            message(FATAL_ERROR "Failed to update Git submodules.")
-        endif()
+    if(NOT update_return_code EQUAL 0)
+        message(FATAL_ERROR "Failed to update Git submodule ${submodule_path}: ${update_stdout} ${update_stderr}")
+    endif()
+endfunction()
+
+function(update_git_submodule_recursive submodule_path)
+    set(extra_option --recursive)
+    update_git_submodule(${submodule_path} ${extra_option})
 endfunction()
diff --git a/compiler/compiler-settings.cpp b/compiler/compiler-settings.cpp
@@ -153,7 +153,7 @@ void append_3dparty_lib(std::string &ld_flags, const std::string &path_to_3dpart
   ld_flags += " " + path_to_3dparty + "lib/lib" + libname + ".a";
 }
 
-void append_curl([[maybe_unused]] std::string &cxx_flags, std::string &ld_flags, const std::string &path_to_3dparty) noexcept {
+void append_curl([[maybe_unused]] std::string &cxx_flags, std::string &ld_flags, [[maybe_unused]] const std::string &path_to_3dparty) noexcept {
   if (!contains_lib(ld_flags, "curl")) {
 #if defined(__APPLE__)
     ld_flags += " -lcurl";
@@ -367,7 +367,7 @@ void CompilerSettings::init() {
   ld_flags.value_ = extra_ld_flags.get();
   append_curl(cxx_default_flags, ld_flags.value_, third_party_path);
   append_apple_options(cxx_default_flags, ld_flags.value_);
-  std::vector<vk::string_view> os_installed_libs{"pcre", "re2", "yaml-cpp", "h3", "z", "zstd", "nghttp2", "kphp-timelib"};
+  std::vector<vk::string_view> system_installed_static_libs{"pcre", "re2", "yaml-cpp", "h3", "z", "zstd", "nghttp2", "kphp-timelib"};
 
 #ifdef KPHP_TIMELIB_LIB_DIR
   ld_flags.value_ += " -L" KPHP_TIMELIB_LIB_DIR;
@@ -387,46 +387,47 @@ void CompilerSettings::init() {
   ld_flags.value_ += " -L /usr/local/lib";
 #endif
 
-  std::vector<vk::string_view> external_libs{"pthread", "m", "dl"};
+  std::vector<vk::string_view> system_installed_dynamic_libs{"pthread", "m", "dl"};
 
 #ifdef PDO_DRIVER_MYSQL
 #ifdef PDO_LIBS_STATIC_LINKING
-  os_installed_libs.emplace_back("mysqlclient");
+  system_installed_static_libs.emplace_back("mysqlclient");
 #else
-  external_libs.emplace_back("mysqlclient");
+  system_installed_dynamic_libs.emplace_back("mysqlclient");
 #endif
 #endif
 
 #ifdef PDO_DRIVER_PGSQL
 #ifdef PDO_LIBS_STATIC_LINKING
   ld_flags.value_ += fmt_format(" -L /usr/lib/postgresql/{}/lib/ ", PDO_DRIVER_PGSQL_VERSION);
-  os_installed_libs.emplace_back("pq");
-  os_installed_libs.emplace_back("pgcommon");
-  os_installed_libs.emplace_back("pgport");
+  system_installed_static_libs.emplace_back("pq");
+  system_installed_static_libs.emplace_back("pgcommon");
+  system_installed_static_libs.emplace_back("pgport");
   // following common libraries are required for libpq.a
-  external_libs.emplace_back("ldap");
-  external_libs.emplace_back("gssapi_krb5");
+  system_installed_dynamic_libs.emplace_back("ldap");
+  system_installed_dynamic_libs.emplace_back("gssapi_krb5");
 #else
-  external_libs.emplace_back("pq");
+  system_installed_dynamic_libs.emplace_back("pq");
 #endif
 #endif
 
-  append_3dparty_headers(cxx_default_flags, third_party_path, "openssl");
-  append_3dparty_lib(ld_flags.value_, third_party_path, "ssl");
-  append_3dparty_lib(ld_flags.value_, third_party_path, "crypto");
-
 #if defined(__APPLE__)
-  append_if_doesnt_contain(ld_flags.value_, os_installed_libs, "-l");
+  append_if_doesnt_contain(ld_flags.value_, system_installed_static_libs, "-l");
   auto flex_prefix = kphp_src_path.value_ + "objs/flex/lib";
   append_if_doesnt_contain(ld_flags.value_, vk::to_array({"vk-flex-data"}), flex_prefix, ".a");
-  external_libs.emplace_back("iconv");
+  system_installed_dynamic_libs.emplace_back("iconv");
 #else
   append_3dparty_lib(ld_flags.value_, third_party_path, "vk-flex-data");
-  os_installed_libs.emplace_back("numa");
-  append_if_doesnt_contain(ld_flags.value_, os_installed_libs, "-l:lib", ".a");
-  external_libs.emplace_back("rt");
+  system_installed_static_libs.emplace_back("numa");
+  append_if_doesnt_contain(ld_flags.value_, system_installed_static_libs, "-l:lib", ".a");
+  system_installed_dynamic_libs.emplace_back("rt");
 #endif
-  append_if_doesnt_contain(ld_flags.value_, external_libs, "-l");
+
+  append_3dparty_headers(cxx_default_flags, third_party_path, "openssl");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "ssl");
+  append_3dparty_lib(ld_flags.value_, third_party_path, "crypto");
+
+  append_if_doesnt_contain(ld_flags.value_, system_installed_dynamic_libs, "-l");
   ld_flags.value_ += " -rdynamic";
 
   runtime_headers.value_ = "runtime-headers.h";

diff --git a/compiler/compiler.cmake b/compiler/compiler.cmake
@@ -274,6 +274,7 @@ set_property(SOURCE ${KPHP_COMPILER_DIR}/kphp2cpp.cpp
 
 add_executable(kphp2cpp ${KPHP_COMPILER_DIR}/kphp2cpp.cpp)
 target_include_directories(kphp2cpp PUBLIC ${KPHP_COMPILER_DIR})
+add_dependencies(kphp2cpp openssl)
 
 prepare_cross_platform_libs(COMPILER_LIBS yaml-cpp re2)
 set(COMPILER_LIBS vk::kphp2cpp_src vk::tlo_parsing_src vk::popular_common ${COMPILER_LIBS} fmt::fmt OpenSSL::Crypto pthread)

diff --git a/runtime-light/runtime-light.cmake b/runtime-light/runtime-light.cmake
@@ -1,5 +1,7 @@
 # prepare third-parties
-update_git_submodules()
+update_git_submodule_recursive(${THIRD_PARTY_DIR}/abseil-cpp)
+update_git_submodule_recursive(${THIRD_PARTY_DIR}/pcre2)
+update_git_submodule_recursive(${THIRD_PARTY_DIR}/zlib)
 include(${THIRD_PARTY_DIR}/abseil-cpp-cmake/abseil-cpp.cmake)
 include(${THIRD_PARTY_DIR}/pcre2-cmake/pcre2.cmake)
 include(${THIRD_PARTY_DIR}/zlib-cmake/zlib.cmake)

diff --git a/runtime/runtime.cmake b/runtime/runtime.cmake
@@ -1,3 +1,6 @@
+# CURL Building
+include(${THIRD_PARTY_DIR}/curl-cmake/curl.cmake)
+
 prepend(KPHP_RUNTIME_DATETIME_SOURCES datetime/
         date_interval.cpp
         datetime.cpp
@@ -148,7 +151,7 @@ target_link_libraries(kphp-full-runtime PUBLIC ${RUNTIME_LIBS})
 set_target_properties(kphp-full-runtime PROPERTIES ARCHIVE_OUTPUT_DIRECTORY ${OBJS_DIR})
 
 prepare_cross_platform_libs(RUNTIME_LINK_TEST_LIBS pcre nghttp2 kphp-timelib)
-set(RUNTIME_LINK_TEST_LIBS vk::flex_data_static CURL::curl OpenSSL::SSL ${NUMA_LIB} ${RUNTIME_LINK_TEST_LIBS} ${EPOLL_SHIM_LIB} ${ICONV_LIB} ${RT_LIB})
+set(RUNTIME_LINK_TEST_LIBS vk::flex_data_static CURL::curl OpenSSL::SSL ${NUMA_LIB} ${RUNTIME_LINK_TEST_LIBS} ${EPOLL_SHIM_LIB} ${ICONV_LIB} ${RT_LIB} dl)
 
 if (PDO_DRIVER_MYSQL)
     list(APPEND RUNTIME_LINK_TEST_LIBS mysqlclient)