From 231e698ab65e990ba2016ec632efb098af9fd9d8 Mon Sep 17 00:00:00 2001 From: David Ostrovsky Date: Tue, 28 May 2024 19:49:03 +0200 Subject: [PATCH] Bazel: Don't build protobuf from the source Gerrit is pure java project. Given that it depends on Google protobuf, and given that Bazel was using @com_google_protobuf toolchain that was built from source, this project ended up bulding the Google protobuf from source. Compiling protoc from source requires a functional C++ toolchain, which is a burden for projects that have no C++ code. Also, Bazel does not ship with a hermetic toolchain, so that it is possible that for many Gerrit developers and contributors the Bazel build is inherently broken. In addition, building protobuf from source made OS upgrades difficult because of the incompatibilities between the source code and the latest XCode versions. That changed in Bazel 7.x release, with the new and shiny option: --incompatible_enable_proto_toolchain_resolution, that allow to register prebuilt protoc toolchains. In addition rules_proto have added support for prebuilt toolchains in context of this tracking issue: [1]. In this change we use toolchains_protoc project to consume predefined protobuf toolchains: [2]. As the side effect of this change we have to consume protobuf-java ourself and not transitively through standard @com_google_protobuf toolchain. Given that Google protobuf is internal Google project and all released versions already available within Google, we add the new dependency to tools/nongoogle.bzl to exempt the updates for it from the Library-Compliance label. Now, that we stop building protobuf from source, we can remove C++ options in .bazelrc as well. [1] https://github.com/bazelbuild/rules_proto/issues/179 [2] https://github.com/aspect-build/toolchains_protoc Release-Notes: Use prebuilt protobuf toolchain to avoid building protoc from source Change-Id: I27975879819c4b632682990474ce88737f722d9a --- .bazelrc | 5 +---- lib/BUILD | 5 +---- lib/nongoogle_test.sh | 1 + tools/BUILD | 9 +++++++++ tools/defs.bzl | 18 ++++++++++++++---- tools/nongoogle.bzl | 42 ++++++++++++++++++++++++++++++++---------- 6 files changed, 58 insertions(+), 22 deletions(-) diff --git a/.bazelrc b/.bazelrc index 480bea79c6c1..74427f34571a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,6 +1,7 @@ # TODO(davido): Migrate all dependencies from WORKSPACE to MODULE.bazel # https://issues.gerritcodereview.com/issues/303819949 common --noenable_bzlmod +common --incompatible_enable_proto_toolchain_resolution build --workspace_status_command="python3 ./tools/workspace_status.py" build --repository_cache=~/.gerritcodereview/bazel-cache/repository @@ -51,10 +52,6 @@ build:remote21_gcp --config=remote21 build:remote21_bb --config=config_bb build:remote21_bb --config=build_java21_shared -# Enable modern C++ features -build --cxxopt=-std=c++17 -build --host_cxxopt=-std=c++17 - # Enable strict_action_env flag to. For more information on this feature see # https://groups.google.com/forum/#!topic/bazel-discuss/_VmRfMyyHBk. # This will be the new default behavior at some point (and the flag was flipped diff --git a/lib/BUILD b/lib/BUILD index f9ece524a8a5..a0debc1d41c7 100644 --- a/lib/BUILD +++ b/lib/BUILD @@ -92,10 +92,7 @@ java_library( name = "protobuf", data = ["//lib:LICENSE-protobuf"], visibility = ["//visibility:public"], - exports = [ - "@com_google_protobuf//:protobuf_java", - "@com_google_protobuf//:protobuf_javalite", - ], + exports = ["@protobuf-java//jar"], ) java_library( diff --git a/lib/nongoogle_test.sh b/lib/nongoogle_test.sh index 37cee0432817..11be9297da7d 100755 --- a/lib/nongoogle_test.sh +++ b/lib/nongoogle_test.sh @@ -49,6 +49,7 @@ mina-core nekohtml objenesis openid-consumer +protobuf-java soy sshd-mina sshd-osgi diff --git a/tools/BUILD b/tools/BUILD index ed55a4f8e40d..dc404274d2e5 100644 --- a/tools/BUILD +++ b/tools/BUILD @@ -4,9 +4,18 @@ load( "default_java_toolchain", ) load("@rules_java//java:defs.bzl", "java_package_configuration") +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") exports_files(["nongoogle.bzl"]) +proto_lang_toolchain( + name = "protoc_java_toolchain", + command_line = "--java_out=%s", + progress_message = "Generating Java proto_library %{label}", + runtime = "@protobuf-java//jar", + toolchain_type = "@rules_java//java/proto:toolchain_type", +) + [default_java_toolchain( name = "error_prone_warnings_toolchain_java" + VERSION, configuration = dict(), diff --git a/tools/defs.bzl b/tools/defs.bzl index 672c6f943117..7149891d6539 100644 --- a/tools/defs.bzl +++ b/tools/defs.bzl @@ -1,11 +1,21 @@ -load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") +load("@bazel_features//:deps.bzl", "bazel_features_deps") +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies") +load("@toolchains_protoc//protoc:repositories.bzl", "rules_protoc_dependencies") +load("@toolchains_protoc//protoc:toolchain.bzl", "protoc_toolchains") def gerrit_init(): """ Initialize the WORKSPACE for gerrit targets """ - protobuf_deps() + rules_protoc_dependencies() - native.register_toolchains("//tools:error_prone_warnings_toolchain_java17_definition") + rules_proto_dependencies() - native.register_toolchains("//tools:error_prone_warnings_toolchain_java21_definition") + bazel_features_deps() + + protoc_toolchains( + name = "toolchains_protoc_hub", + version = "v25.3", + ) + + native.register_toolchains("//tools:all") diff --git a/tools/nongoogle.bzl b/tools/nongoogle.bzl index e185141fc8bd..f298b9bd4ff1 100644 --- a/tools/nongoogle.bzl +++ b/tools/nongoogle.bzl @@ -24,27 +24,43 @@ GUAVA_DOC_URL = "https://google.github.io/guava/releases/" + GUAVA_VERSION + "/a def archive_dependencies(): return [ { - "name": "com_google_protobuf", - "sha256": "9bd87b8280ef720d3240514f884e56a712f2218f0d693b48050c836028940a42", - "strip_prefix": "protobuf-25.1", + "name": "platforms", "urls": [ - "https://github.com/protocolbuffers/protobuf/archive/v25.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.10/platforms-0.0.10.tar.gz", ], + "sha256": "218efe8ee736d26a3572663b374a253c012b716d8af0c07e842e82f238a0a7ee", }, { - "name": "platforms", + "name": "bazel_features", + "strip_prefix": "bazel_features-1.11.0", "urls": [ - "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", - "https://github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", + "https://github.com/bazel-contrib/bazel_features/releases/download/v1.11.0/bazel_features-v1.11.0.tar.gz", ], - "sha256": "3a561c99e7bdbe9173aa653fd579fe849f1d8d67395780ab4770b1f381431d51", + "sha256": "2cd9e57d4c38675d321731d65c15258f3a66438ad531ae09cb8bb14217dc8572", }, { "name": "rules_java", "urls": [ - "https://github.com/bazelbuild/rules_java/releases/download/7.3.1/rules_java-7.3.1.tar.gz", + "https://github.com/bazelbuild/rules_java/releases/download/7.6.1/rules_java-7.6.1.tar.gz", ], - "sha256": "4018e97c93f97680f1650ffd2a7530245b864ac543fd24fae8c02ba447cb2864", + "sha256": "f8ae9ed3887df02f40de9f4f7ac3873e6dd7a471f9cddf63952538b94b59aeb3", + }, + { + "name": "rules_proto", + "strip_prefix": "rules_proto-6.0.0", + "urls": [ + "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0/rules_proto-6.0.0.tar.gz", + ], + "sha256": "303e86e722a520f6f326a50b41cfc16b98fe6d1955ce46642a5b7a67c11c0f5d", + }, + { + "name": "toolchains_protoc", + "strip_prefix": "toolchains_protoc-0.3.0", + "urls": [ + "https://github.com/aspect-build/toolchains_protoc/releases/download/v0.3.0/toolchains_protoc-v0.3.0.tar.gz", + ], + "sha256": "117af61ee2f1b9b014dcac7c9146f374875551abb8a30e51d1b3c5946d25b142", }, { "name": "ubuntu2204_jdk17", @@ -294,6 +310,12 @@ def declare_nongoogle_deps(): sha1 = "b3add478d4382b78ea20b1671390a858002feb6c", ) + maven_jar( + name = "protobuf-java", + artifact = "com.google.protobuf:protobuf-java:3.25.3", + sha1 = "d3200261955f3298e0d85c9892201e70492ce8eb", + ) + # Test-only dependencies below. maven_jar( name = "cglib-3_2",