diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index 30138d3be6..c2d172d7e1 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -147,6 +147,14 @@ tasks: <<: *reusable_config name: "Default: Debian" platform: debian11 + build_flags: + # For protobuf compilation + - '--host_copt=-Wno-deprecated-declarations' + - '--copt=-Wno-deprecated-declarations' + test_flags: + # For protobuf compilation + - '--host_copt=-Wno-deprecated-declarations' + - '--copt=-Wno-deprecated-declarations' macos: <<: *reusable_config name: "Default: MacOS" @@ -169,6 +177,10 @@ tasks: # build kite cc toolchain. - "--extra_toolchains=@buildkite_config//config:cc-toolchain" - "--build_tag_filters=-docs" + build_targets: + - "--" + - "..." + - '-//sphinxdocs/...' # protobuf compilation fails test_flags: - "--test_tag_filters=-integration-test,-acceptance-test,-docs" # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, @@ -176,12 +188,28 @@ tasks: # on Bazel 5.4 and earlier. To workaround this, manually specify the # build kite cc toolchain. - "--extra_toolchains=@buildkite_config//config:cc-toolchain" + test_targets: + - "--" + - "..." + - '-//sphinxdocs/...' # protobuf compilation fails rbe: <<: *reusable_config name: "RBE: Ubuntu" platform: rbe_ubuntu1604 + build_flags: + - "--build_tag_filters=-docs" + build_targets: + - "--" + - "..." + - '-//sphinxdocs/...' # protobuf compilation fails + - '-//docs/...' test_flags: - - "--test_tag_filters=-integration-test,-acceptance-test" + - "--test_tag_filters=-integration-test,-acceptance-test,-docs" + test_targets: + - "--" + - "..." + - '-//sphinxdocs/...' # protobuf compilation fails + - '-//docs/...' integration_test_build_file_generation_ubuntu_minimum_supported_workspace: <<: *minimum_supported_version @@ -234,6 +262,21 @@ tasks: name: "examples/bzlmod: Debian" working_directory: examples/bzlmod platform: debian11 + build_targets: + # For protobuf compilation + - "--" + - "..." + - "-//py_proto_library/..." + test_targets: + # For protobuf compilation + - "--" + - "..." + - "-//py_proto_library/..." + coverage_targets: + # For protobuf compilation + - "--" + - "..." + - "-//py_proto_library/..." integration_test_bzlmod_macos: <<: *reusable_build_test_all <<: *coverage_targets_example_bzlmod @@ -395,6 +438,14 @@ tasks: name: "examples/py_proto_library: Debian, workspace" working_directory: examples/py_proto_library platform: debian11 + build_flags: + # For protobuf compilation + - '--host_copt=-Wno-deprecated-declarations' + - '--copt=-Wno-deprecated-declarations' + test_flags: + # For protobuf compilation + - '--host_copt=-Wno-deprecated-declarations' + - '--copt=-Wno-deprecated-declarations' integration_test_py_proto_library_macos_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags diff --git a/CHANGELOG.md b/CHANGELOG.md index e15be3bbcc..8b2f6e6513 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,9 @@ A brief description of the categories of changes: [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x ### Changed +* (rules) `py_proto_library` is deprecated in favour of the + implementation in https://github.com/protocolbuffers/protobuf. It will be + removed in the future release. * (deps) Upgrade the `pip_install` dependencies to pick up a new version of pip. * (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and `py_library` now depend on the `//python:exec_tools_toolchain_type` for build diff --git a/MODULE.bazel b/MODULE.bazel index 7064dfc84f..04d8fb211e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,9 +9,9 @@ bazel_dep(name = "bazel_skylib", version = "1.6.1") bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "platforms", version = "0.0.4") -# Those are loaded only when using py_proto_library -bazel_dep(name = "rules_proto", version = "6.0.0-rc1") -bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") +# For backwards compatibility, those are loaded only when using py_proto_library +# Use py_proto_library directly from protobuf repository +bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf") internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps") use_repo( diff --git a/WORKSPACE b/WORKSPACE index 90e9305684..d30ccb02fd 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -148,9 +148,3 @@ http_file( "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", ], ) - -# rules_proto expects //external:python_headers to point at the python headers. -bind( - name = "python_headers", - actual = "//python/cc:current_py_cc_headers", -) diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index ca89afe8af..7829af6f21 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -55,8 +55,3 @@ http_file( ], ) -# rules_proto expects //external:python_headers to point at the python headers. -bind( - name = "python_headers", - actual = "//python/cc:current_py_cc_headers", -) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 0d30161147..2e28bd6270 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -11,11 +11,8 @@ local_path_override( path = "../..", ) -# (py_proto_library specific) We are using rules_proto to define rules_proto targets to be consumed by py_proto_library. -bazel_dep(name = "rules_proto", version = "5.3.0-21.7") - # (py_proto_library specific) Add the protobuf library for well-known types (e.g. `Any`, `Timestamp`, etc) -bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") +bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf") # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. diff --git a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel index 806fcb9dcc..80f0470741 100644 --- a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") -load("@rules_python//python:proto.bzl", "py_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") py_proto_library( name = "message_proto_py_pb2", diff --git a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel index fa20f2ce94..3bc9d1b739 100644 --- a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") -load("@rules_python//python:proto.bzl", "py_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") py_proto_library( name = "pricetag_proto_py_pb2", diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index 81f189dbbf..7892c69c4d 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -24,24 +24,11 @@ python_register_toolchains( # Then we need to setup dependencies in order to use py_proto_library load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -http_archive( - name = "rules_proto", - sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", - strip_prefix = "rules_proto-6.0.0-rc1", - url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz", -) - -load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") - -rules_proto_dependencies() - -rules_proto_toolchains() - http_archive( name = "com_google_protobuf", - sha256 = "4fc5ff1b2c339fb86cd3a25f0b5311478ab081e65ad258c6789359cd84d421f8", - strip_prefix = "protobuf-26.1", - urls = ["https://github.com/protocolbuffers/protobuf/archive/v26.1.tar.gz"], + sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa", + strip_prefix = "protobuf-27.0", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz"], ) load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") diff --git a/examples/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/py_proto_library/example.com/another_proto/BUILD.bazel index dd58265bc9..126dd9b46d 100644 --- a/examples/py_proto_library/example.com/another_proto/BUILD.bazel +++ b/examples/py_proto_library/example.com/another_proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") -load("@rules_python//python:proto.bzl", "py_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") py_proto_library( name = "message_proto_py_pb2", diff --git a/examples/py_proto_library/example.com/proto/BUILD.bazel b/examples/py_proto_library/example.com/proto/BUILD.bazel index dc91162aa6..0084a61794 100644 --- a/examples/py_proto_library/example.com/proto/BUILD.bazel +++ b/examples/py_proto_library/example.com/proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@rules_proto//proto:defs.bzl", "proto_library") -load("@rules_python//python:proto.bzl", "py_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") +load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") py_proto_library( name = "pricetag_proto_py_pb2", diff --git a/internal_deps.bzl b/internal_deps.bzl index 8818751644..8c604c4032 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -164,20 +164,12 @@ def rules_python_internal_deps(): ], ) - http_archive( - name = "rules_proto", - sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", - strip_prefix = "rules_proto-6.0.0-rc1", - url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz", - ) - http_archive( name = "com_google_protobuf", - sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", - strip_prefix = "protobuf-21.7", + sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa", + strip_prefix = "protobuf-27.0", urls = [ - "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", - "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + "https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz", ], ) diff --git a/internal_setup.bzl b/internal_setup.bzl index bb62611213..6614ad355e 100644 --- a/internal_setup.bzl +++ b/internal_setup.bzl @@ -20,7 +20,6 @@ load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies") load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies") load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries") -load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS") load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=bzl-visibility @@ -35,9 +34,6 @@ def rules_python_internal_setup(): bazel_skylib_workspace() - rules_proto_dependencies() - rules_proto_toolchains() - protobuf_deps() bazel_integration_test_rules_dependencies() diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 5d31df5e9a..10b785b885 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -112,7 +112,7 @@ bzl_library( ], visibility = ["//visibility:public"], deps = [ - "//python/private/proto:py_proto_library_bzl", + "@com_google_protobuf//bazel:py_proto_library_bzl", ], ) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 3e56208859..35c254eb05 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -29,7 +29,6 @@ filegroup( srcs = glob(["**"]) + [ "//python/private/bzlmod:distribution", "//python/private/common:distribution", - "//python/private/proto:distribution", "//python/private/whl_filegroup:distribution", "//tools/build_defs/python/private:distribution", ], diff --git a/python/private/proto/BUILD.bazel b/python/private/proto/BUILD.bazel deleted file mode 100644 index 65c09444f7..0000000000 --- a/python/private/proto/BUILD.bazel +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright 2022 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -load("@bazel_skylib//:bzl_library.bzl", "bzl_library") -load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") - -package(default_visibility = ["//visibility:private"]) - -licenses(["notice"]) - -filegroup( - name = "distribution", - srcs = glob(["**"]), - visibility = ["//python/private:__pkg__"], -) - -bzl_library( - name = "py_proto_library_bzl", - srcs = ["py_proto_library.bzl"], - visibility = ["//python:__pkg__"], - deps = [ - "//python:defs_bzl", - "@rules_proto//proto:defs", - ], -) - -proto_lang_toolchain( - name = "python_toolchain", - command_line = "--python_out=%s", - progress_message = "Generating Python proto_library %{label}", - runtime = "@com_google_protobuf//:protobuf_python", - # NOTE: This isn't *actually* public. It's an implicit dependency of py_proto_library, - # so must be public so user usages of the rule can reference it. - visibility = ["//visibility:public"], -) diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl deleted file mode 100644 index e123ff8476..0000000000 --- a/python/private/proto/py_proto_library.bzl +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright 2022 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""The implementation of the `py_proto_library` rule and its aspect.""" - -load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") -load("//python:defs.bzl", "PyInfo") - -PY_PROTO_TOOLCHAIN = "@rules_python//python/proto:toolchain_type" - -_PyProtoInfo = provider( - doc = "Encapsulates information needed by the Python proto rules.", - fields = { - "imports": """ - (depset[str]) The field forwarding PyInfo.imports coming from - the proto language runtime dependency.""", - "runfiles_from_proto_deps": """ - (depset[File]) Files from the transitive closure implicit proto - dependencies""", - "transitive_sources": """(depset[File]) The Python sources.""", - }, -) - -def _filter_provider(provider, *attrs): - return [dep[provider] for attr in attrs for dep in attr if provider in dep] - -def _incompatible_toolchains_enabled(): - return getattr(proto_common, "INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION", False) - -def _py_proto_aspect_impl(target, ctx): - """Generates and compiles Python code for a proto_library. - - The function runs protobuf compiler on the `proto_library` target generating - a .py file for each .proto file. - - Args: - target: (Target) A target providing `ProtoInfo`. Usually this means a - `proto_library` target, but not always; you must expect to visit - non-`proto_library` targets, too. - ctx: (RuleContext) The rule context. - - Returns: - ([_PyProtoInfo]) Providers collecting transitive information about - generated files. - """ - _proto_library = ctx.rule.attr - - # Check Proto file names - for proto in target[ProtoInfo].direct_sources: - if proto.is_source and "-" in proto.dirname: - fail("Cannot generate Python code for a .proto whose path contains '-' ({}).".format( - proto.path, - )) - - if _incompatible_toolchains_enabled(): - toolchain = ctx.toolchains[PY_PROTO_TOOLCHAIN] - if not toolchain: - fail("No toolchains registered for '%s'." % PY_PROTO_TOOLCHAIN) - proto_lang_toolchain_info = toolchain.proto - else: - proto_lang_toolchain_info = getattr(ctx.attr, "_aspect_proto_toolchain")[proto_common.ProtoLangToolchainInfo] - - api_deps = [proto_lang_toolchain_info.runtime] - - generated_sources = [] - proto_info = target[ProtoInfo] - proto_root = proto_info.proto_source_root - if proto_info.direct_sources: - # Generate py files - generated_sources = proto_common.declare_generated_files( - actions = ctx.actions, - proto_info = proto_info, - extension = "_pb2.py", - name_mapper = lambda name: name.replace("-", "_").replace(".", "/"), - ) - - # Handles multiple repository and virtual import cases - if proto_root.startswith(ctx.bin_dir.path): - proto_root = proto_root[len(ctx.bin_dir.path) + 1:] - - plugin_output = ctx.bin_dir.path + "/" + proto_root - proto_root = ctx.workspace_name + "/" + proto_root - - proto_common.compile( - actions = ctx.actions, - proto_info = proto_info, - proto_lang_toolchain_info = proto_lang_toolchain_info, - generated_files = generated_sources, - plugin_output = plugin_output, - ) - - # Generated sources == Python sources - python_sources = generated_sources - - deps = _filter_provider(_PyProtoInfo, getattr(_proto_library, "deps", [])) - runfiles_from_proto_deps = depset( - transitive = [dep[DefaultInfo].default_runfiles.files for dep in api_deps] + - [dep.runfiles_from_proto_deps for dep in deps], - ) - transitive_sources = depset( - direct = python_sources, - transitive = [dep.transitive_sources for dep in deps], - ) - - return [ - _PyProtoInfo( - imports = depset( - # Adding to PYTHONPATH so the generated modules can be - # imported. This is necessary when there is - # strip_import_prefix, the Python modules are generated under - # _virtual_imports. But it's undesirable otherwise, because it - # will put the repo root at the top of the PYTHONPATH, ahead of - # directories added through `imports` attributes. - [proto_root] if "_virtual_imports" in proto_root else [], - transitive = [dep[PyInfo].imports for dep in api_deps] + [dep.imports for dep in deps], - ), - runfiles_from_proto_deps = runfiles_from_proto_deps, - transitive_sources = transitive_sources, - ), - ] - -_py_proto_aspect = aspect( - implementation = _py_proto_aspect_impl, - attrs = {} if _incompatible_toolchains_enabled() else { - "_aspect_proto_toolchain": attr.label( - default = ":python_toolchain", - ), - }, - attr_aspects = ["deps"], - required_providers = [ProtoInfo], - provides = [_PyProtoInfo], - toolchains = [PY_PROTO_TOOLCHAIN] if _incompatible_toolchains_enabled() else [], -) - -def _py_proto_library_rule(ctx): - """Merges results of `py_proto_aspect` in `deps`. - - Args: - ctx: (RuleContext) The rule context. - Returns: - ([PyInfo, DefaultInfo, OutputGroupInfo]) - """ - if not ctx.attr.deps: - fail("'deps' attribute mustn't be empty.") - - pyproto_infos = _filter_provider(_PyProtoInfo, ctx.attr.deps) - default_outputs = depset( - transitive = [info.transitive_sources for info in pyproto_infos], - ) - - return [ - DefaultInfo( - files = default_outputs, - default_runfiles = ctx.runfiles(transitive_files = depset( - transitive = - [default_outputs] + - [info.runfiles_from_proto_deps for info in pyproto_infos], - )), - ), - OutputGroupInfo( - default = depset(), - ), - PyInfo( - transitive_sources = default_outputs, - imports = depset(transitive = [info.imports for info in pyproto_infos]), - # Proto always produces 2- and 3- compatible source files - has_py2_only_sources = False, - has_py3_only_sources = False, - ), - ] - -py_proto_library = rule( - implementation = _py_proto_library_rule, - doc = """ - Use `py_proto_library` to generate Python libraries from `.proto` files. - - The convention is to name the `py_proto_library` rule `foo_py_pb2`, - when it is wrapping `proto_library` rule `foo_proto`. - - `deps` must point to a `proto_library` rule. - - Example: - -```starlark -py_library( - name = "lib", - deps = [":foo_py_pb2"], -) - -py_proto_library( - name = "foo_py_pb2", - deps = [":foo_proto"], -) - -proto_library( - name = "foo_proto", - srcs = ["foo.proto"], -) -```""", - attrs = { - "deps": attr.label_list( - doc = """ - The list of `proto_library` rules to generate Python libraries for. - - Usually this is just the one target: the proto library of interest. - It can be any target providing `ProtoInfo`.""", - providers = [ProtoInfo], - aspects = [_py_proto_aspect], - ), - }, - provides = [PyInfo], -) diff --git a/python/proto.bzl b/python/proto.bzl index 3f455aee58..2ea9bdb153 100644 --- a/python/proto.bzl +++ b/python/proto.bzl @@ -11,11 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - """ Python proto library. """ -load("//python/private/proto:py_proto_library.bzl", _py_proto_library = "py_proto_library") +load("@com_google_protobuf//bazel:py_proto_library.bzl", _py_proto_library = "py_proto_library") -py_proto_library = _py_proto_library +def py_proto_library(*, deprecation = "Use py_proto_library from protobuf repository", **kwargs): + _py_proto_library(deprecation = deprecation, **kwargs)