feat: upgrade to protobuf 27.0 and remove py_proto_library (#1933)

Reintroduce the changes in #1933, which got reverted in commit 8b0eaed (#1948).
bazelbuild · Jun 12, 2024 · dd08165 · dd08165
1 parent 2a19374
commit dd08165
Show file tree

Hide file tree

Showing 18 changed files with 77 additions and 333 deletions.
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
@@ -147,6 +147,14 @@ tasks:
     <<: *reusable_config
     name: "Default: Debian"
     platform: debian11
+    build_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
+    test_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
   macos:
     <<: *reusable_config
     name: "Default: MacOS"
@@ -169,19 +177,39 @@ tasks:
       # build kite cc toolchain.
       - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
       - "--build_tag_filters=-docs"
+    build_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
     test_flags:
       - "--test_tag_filters=-integration-test,-acceptance-test,-docs"
       # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1,
       # which prevents cc toolchain autodetection from working correctly
       # on Bazel 5.4 and earlier. To workaround this, manually specify the
       # build kite cc toolchain.
       - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
+    test_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
   rbe:
     <<: *reusable_config
     name: "RBE: Ubuntu"
     platform: rbe_ubuntu1604
+    build_flags:
+      - "--build_tag_filters=-docs"
+    build_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
+      - '-//docs/...'
     test_flags:
-      - "--test_tag_filters=-integration-test,-acceptance-test"
+      - "--test_tag_filters=-integration-test,-acceptance-test,-docs"
+    test_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
+      - '-//docs/...'
 
   integration_test_build_file_generation_ubuntu_minimum_supported_workspace:
     <<: *minimum_supported_version
@@ -234,6 +262,21 @@ tasks:
     name: "examples/bzlmod: Debian"
     working_directory: examples/bzlmod
     platform: debian11
+    build_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
+    test_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
+    coverage_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
   integration_test_bzlmod_macos:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod
@@ -395,6 +438,14 @@ tasks:
     name: "examples/py_proto_library: Debian, workspace"
     working_directory: examples/py_proto_library
     platform: debian11
+    build_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
+    test_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
   integration_test_py_proto_library_macos_workspace:
     <<: *reusable_build_test_all
     <<: *common_workspace_flags

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,7 +25,9 @@ A brief description of the categories of changes:
 [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x
 
 ### Changed
-* Nothing yet
+* (rules) `py_proto_library` is deprecated in favour of the
+  implementation in https://github.com/protocolbuffers/protobuf. It will be
+  removed in the future release.
 
 ### Fixed
 * Nothing yet

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -9,9 +9,9 @@ bazel_dep(name = "bazel_skylib", version = "1.6.1")
 bazel_dep(name = "rules_cc", version = "0.0.9")
 bazel_dep(name = "platforms", version = "0.0.4")
 
-# Those are loaded only when using py_proto_library
-bazel_dep(name = "rules_proto", version = "6.0.0-rc1")
-bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
+# For backwards compatibility, those are loaded only when using py_proto_library
+# Use py_proto_library directly from protobuf repository
+bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf")
 
 internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps")
 use_repo(

diff --git a/WORKSPACE b/WORKSPACE
@@ -148,9 +148,3 @@ http_file(
         "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
     ],
 )
-
-# rules_proto expects //external:python_headers to point at the python headers.
-bind(
-    name = "python_headers",
-    actual = "//python/cc:current_py_cc_headers",
-)
diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -55,8 +55,3 @@ http_file(
     ],
 )
 
-# rules_proto expects //external:python_headers to point at the python headers.
-bind(
-    name = "python_headers",
-    actual = "//python/cc:current_py_cc_headers",
-)
diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
@@ -11,11 +11,8 @@ local_path_override(
     path = "../..",
 )
 
-# (py_proto_library specific) We are using rules_proto to define rules_proto targets to be consumed by py_proto_library.
-bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
-
 # (py_proto_library specific) Add the protobuf library for well-known types (e.g. `Any`, `Timestamp`, etc)
-bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
+bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf")
 
 # We next initialize the python toolchain using the extension.
 # You can set different Python versions in this block.

diff --git a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "message_proto_py_pb2",

diff --git a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "pricetag_proto_py_pb2",

diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE
@@ -24,24 +24,11 @@ python_register_toolchains(
 # Then we need to setup dependencies in order to use py_proto_library
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-http_archive(
-    name = "rules_proto",
-    sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c",
-    strip_prefix = "rules_proto-6.0.0-rc1",
-    url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz",
-)
-
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
-
-rules_proto_dependencies()
-
-rules_proto_toolchains()
-
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "4fc5ff1b2c339fb86cd3a25f0b5311478ab081e65ad258c6789359cd84d421f8",
-    strip_prefix = "protobuf-26.1",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v26.1.tar.gz"],
+    sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa",
+    strip_prefix = "protobuf-27.0",
+    urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz"],
 )
 
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")

diff --git a/examples/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/py_proto_library/example.com/another_proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "message_proto_py_pb2",

diff --git a/examples/py_proto_library/example.com/proto/BUILD.bazel b/examples/py_proto_library/example.com/proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "pricetag_proto_py_pb2",

diff --git a/internal_deps.bzl b/internal_deps.bzl
@@ -164,20 +164,12 @@ def rules_python_internal_deps():
         ],
     )
 
-    http_archive(
-        name = "rules_proto",
-        sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c",
-        strip_prefix = "rules_proto-6.0.0-rc1",
-        url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz",
-    )
-
     http_archive(
         name = "com_google_protobuf",
-        sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae",
-        strip_prefix = "protobuf-21.7",
+        sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa",
+        strip_prefix = "protobuf-27.0",
         urls = [
-            "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
-            "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
+            "https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz",
         ],
     )
 

diff --git a/internal_setup.bzl b/internal_setup.bzl
@@ -20,7 +20,6 @@ load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies")
 load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries")
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
 load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
 load("//python/private:internal_config_repo.bzl", "internal_config_repo")  # buildifier: disable=bzl-visibility
@@ -35,9 +34,6 @@ def rules_python_internal_setup():
 
     bazel_skylib_workspace()
 
-    rules_proto_dependencies()
-    rules_proto_toolchains()
-
     protobuf_deps()
 
     bazel_integration_test_rules_dependencies()

diff --git a/python/BUILD.bazel b/python/BUILD.bazel
@@ -113,7 +113,7 @@ bzl_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        "//python/private/proto:py_proto_library_bzl",
+        "@com_google_protobuf//bazel:py_proto_library_bzl",
     ],
 )
 

diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
@@ -29,7 +29,6 @@ filegroup(
     srcs = glob(["**"]) + [
         "//python/private/bzlmod:distribution",
         "//python/private/common:distribution",
-        "//python/private/proto:distribution",
         "//python/private/whl_filegroup:distribution",
         "//tools/build_defs/python/private:distribution",
     ],

diff --git a/python/private/proto/BUILD.bazel b/python/private/proto/BUILD.bazel