feat: Upgrade to protobuf 27.0 and remove py_proto_library (#1933)

Protobuf team is taking ownership of `py_proto_library` and the implementation was moved to protobuf repository. Remove py_proto_library from rules_python, to prevent divergent implementations. Make a redirect with a deprecation warning, so that this doesn't break any users. Expected side effect of this change is also that the protobuf version is sufficiently updated that there is no more use of legacy struct providers. Closes #1935 Closes #1924 Closes #1925 Closes #1703 Closes #1707 Closes #1597 Closes #1293 Closes #1080 Fixes #1438
bazelbuild · Jun 5, 2024 · d0e25cf · d0e25cf
1 parent ae1e1a0
commit d0e25cf
Show file tree

Hide file tree

Showing 18 changed files with 77 additions and 332 deletions.
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
@@ -147,6 +147,14 @@ tasks:
     <<: *reusable_config
     name: "Default: Debian"
     platform: debian11
+    build_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
+    test_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
   macos:
     <<: *reusable_config
     name: "Default: MacOS"
@@ -169,19 +177,39 @@ tasks:
       # build kite cc toolchain.
       - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
       - "--build_tag_filters=-docs"
+    build_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
     test_flags:
       - "--test_tag_filters=-integration-test,-acceptance-test,-docs"
       # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1,
       # which prevents cc toolchain autodetection from working correctly
       # on Bazel 5.4 and earlier. To workaround this, manually specify the
       # build kite cc toolchain.
       - "--extra_toolchains=@buildkite_config//config:cc-toolchain"
+    test_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
   rbe:
     <<: *reusable_config
     name: "RBE: Ubuntu"
     platform: rbe_ubuntu1604
+    build_flags:
+      - "--build_tag_filters=-docs"
+    build_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
+      - '-//docs/...'
     test_flags:
-      - "--test_tag_filters=-integration-test,-acceptance-test"
+      - "--test_tag_filters=-integration-test,-acceptance-test,-docs"
+    test_targets:
+      - "--"
+      - "..."
+      - '-//sphinxdocs/...' # protobuf compilation fails
+      - '-//docs/...'
 
   integration_test_build_file_generation_ubuntu_minimum_supported_workspace:
     <<: *minimum_supported_version
@@ -234,6 +262,21 @@ tasks:
     name: "examples/bzlmod: Debian"
     working_directory: examples/bzlmod
     platform: debian11
+    build_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
+    test_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
+    coverage_targets:
+      # For protobuf compilation
+      - "--"
+      - "..."
+      - "-//py_proto_library/..."
   integration_test_bzlmod_macos:
     <<: *reusable_build_test_all
     <<: *coverage_targets_example_bzlmod
@@ -395,6 +438,14 @@ tasks:
     name: "examples/py_proto_library: Debian, workspace"
     working_directory: examples/py_proto_library
     platform: debian11
+    build_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
+    test_flags:
+      # For protobuf compilation
+      - '--host_copt=-Wno-deprecated-declarations'
+      - '--copt=-Wno-deprecated-declarations'
   integration_test_py_proto_library_macos_workspace:
     <<: *reusable_build_test_all
     <<: *common_workspace_flags

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,6 +25,9 @@ A brief description of the categories of changes:
 [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x
 
 ### Changed
+* (rules) `py_proto_library` is deprecated in favour of the
+  implementation in https://github.com/protocolbuffers/protobuf. It will be
+  removed in the future release.
 * (deps) Upgrade the `pip_install` dependencies to pick up a new version of pip.
 * (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and
   `py_library` now depend on the `//python:exec_tools_toolchain_type` for build

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -9,9 +9,9 @@ bazel_dep(name = "bazel_skylib", version = "1.6.1")
 bazel_dep(name = "rules_cc", version = "0.0.9")
 bazel_dep(name = "platforms", version = "0.0.4")
 
-# Those are loaded only when using py_proto_library
-bazel_dep(name = "rules_proto", version = "6.0.0-rc1")
-bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
+# For backwards compatibility, those are loaded only when using py_proto_library
+# Use py_proto_library directly from protobuf repository
+bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf")
 
 internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps")
 use_repo(

diff --git a/WORKSPACE b/WORKSPACE
@@ -148,9 +148,3 @@ http_file(
         "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
     ],
 )
-
-# rules_proto expects //external:python_headers to point at the python headers.
-bind(
-    name = "python_headers",
-    actual = "//python/cc:current_py_cc_headers",
-)
diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -55,8 +55,3 @@ http_file(
     ],
 )
 
-# rules_proto expects //external:python_headers to point at the python headers.
-bind(
-    name = "python_headers",
-    actual = "//python/cc:current_py_cc_headers",
-)
diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel
@@ -11,11 +11,8 @@ local_path_override(
     path = "../..",
 )
 
-# (py_proto_library specific) We are using rules_proto to define rules_proto targets to be consumed by py_proto_library.
-bazel_dep(name = "rules_proto", version = "5.3.0-21.7")
-
 # (py_proto_library specific) Add the protobuf library for well-known types (e.g. `Any`, `Timestamp`, etc)
-bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf")
+bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf")
 
 # We next initialize the python toolchain using the extension.
 # You can set different Python versions in this block.

diff --git a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "message_proto_py_pb2",

diff --git a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "pricetag_proto_py_pb2",

diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE
@@ -24,24 +24,11 @@ python_register_toolchains(
 # Then we need to setup dependencies in order to use py_proto_library
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
-http_archive(
-    name = "rules_proto",
-    sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c",
-    strip_prefix = "rules_proto-6.0.0-rc1",
-    url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz",
-)
-
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
-
-rules_proto_dependencies()
-
-rules_proto_toolchains()
-
 http_archive(
     name = "com_google_protobuf",
-    sha256 = "4fc5ff1b2c339fb86cd3a25f0b5311478ab081e65ad258c6789359cd84d421f8",
-    strip_prefix = "protobuf-26.1",
-    urls = ["https://github.com/protocolbuffers/protobuf/archive/v26.1.tar.gz"],
+    sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa",
+    strip_prefix = "protobuf-27.0",
+    urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz"],
 )
 
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")

diff --git a/examples/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/py_proto_library/example.com/another_proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "message_proto_py_pb2",

diff --git a/examples/py_proto_library/example.com/proto/BUILD.bazel b/examples/py_proto_library/example.com/proto/BUILD.bazel
@@ -1,5 +1,5 @@
-load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:proto.bzl", "py_proto_library")
+load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library")
+load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library")
 
 py_proto_library(
     name = "pricetag_proto_py_pb2",

diff --git a/internal_deps.bzl b/internal_deps.bzl
@@ -164,20 +164,12 @@ def rules_python_internal_deps():
         ],
     )
 
-    http_archive(
-        name = "rules_proto",
-        sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c",
-        strip_prefix = "rules_proto-6.0.0-rc1",
-        url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz",
-    )
-
     http_archive(
         name = "com_google_protobuf",
-        sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae",
-        strip_prefix = "protobuf-21.7",
+        sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa",
+        strip_prefix = "protobuf-27.0",
         urls = [
-            "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
-            "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz",
+            "https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz",
         ],
     )
 

diff --git a/internal_setup.bzl b/internal_setup.bzl
@@ -20,7 +20,6 @@ load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies")
 load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries")
-load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
 load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
 load("//python/private:internal_config_repo.bzl", "internal_config_repo")  # buildifier: disable=bzl-visibility
@@ -35,9 +34,6 @@ def rules_python_internal_setup():
 
     bazel_skylib_workspace()
 
-    rules_proto_dependencies()
-    rules_proto_toolchains()
-
     protobuf_deps()
 
     bazel_integration_test_rules_dependencies()

diff --git a/python/BUILD.bazel b/python/BUILD.bazel
@@ -113,7 +113,7 @@ bzl_library(
     ],
     visibility = ["//visibility:public"],
     deps = [
-        "//python/private/proto:py_proto_library_bzl",
+        "@com_google_protobuf//bazel:py_proto_library_bzl",
     ],
 )
 

diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel
@@ -29,7 +29,6 @@ filegroup(
     srcs = glob(["**"]) + [
         "//python/private/bzlmod:distribution",
         "//python/private/common:distribution",
-        "//python/private/proto:distribution",
         "//python/private/whl_filegroup:distribution",
         "//tools/build_defs/python/private:distribution",
     ],

diff --git a/python/private/proto/BUILD.bazel b/python/private/proto/BUILD.bazel