diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index c2d172d7e1..30138d3be6 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -147,14 +147,6 @@ tasks: <<: *reusable_config name: "Default: Debian" platform: debian11 - build_flags: - # For protobuf compilation - - '--host_copt=-Wno-deprecated-declarations' - - '--copt=-Wno-deprecated-declarations' - test_flags: - # For protobuf compilation - - '--host_copt=-Wno-deprecated-declarations' - - '--copt=-Wno-deprecated-declarations' macos: <<: *reusable_config name: "Default: MacOS" @@ -177,10 +169,6 @@ tasks: # build kite cc toolchain. - "--extra_toolchains=@buildkite_config//config:cc-toolchain" - "--build_tag_filters=-docs" - build_targets: - - "--" - - "..." - - '-//sphinxdocs/...' # protobuf compilation fails test_flags: - "--test_tag_filters=-integration-test,-acceptance-test,-docs" # BazelCI sets --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1, @@ -188,28 +176,12 @@ tasks: # on Bazel 5.4 and earlier. To workaround this, manually specify the # build kite cc toolchain. - "--extra_toolchains=@buildkite_config//config:cc-toolchain" - test_targets: - - "--" - - "..." - - '-//sphinxdocs/...' # protobuf compilation fails rbe: <<: *reusable_config name: "RBE: Ubuntu" platform: rbe_ubuntu1604 - build_flags: - - "--build_tag_filters=-docs" - build_targets: - - "--" - - "..." - - '-//sphinxdocs/...' # protobuf compilation fails - - '-//docs/...' test_flags: - - "--test_tag_filters=-integration-test,-acceptance-test,-docs" - test_targets: - - "--" - - "..." - - '-//sphinxdocs/...' # protobuf compilation fails - - '-//docs/...' + - "--test_tag_filters=-integration-test,-acceptance-test" integration_test_build_file_generation_ubuntu_minimum_supported_workspace: <<: *minimum_supported_version @@ -262,21 +234,6 @@ tasks: name: "examples/bzlmod: Debian" working_directory: examples/bzlmod platform: debian11 - build_targets: - # For protobuf compilation - - "--" - - "..." - - "-//py_proto_library/..." - test_targets: - # For protobuf compilation - - "--" - - "..." - - "-//py_proto_library/..." - coverage_targets: - # For protobuf compilation - - "--" - - "..." - - "-//py_proto_library/..." integration_test_bzlmod_macos: <<: *reusable_build_test_all <<: *coverage_targets_example_bzlmod @@ -438,14 +395,6 @@ tasks: name: "examples/py_proto_library: Debian, workspace" working_directory: examples/py_proto_library platform: debian11 - build_flags: - # For protobuf compilation - - '--host_copt=-Wno-deprecated-declarations' - - '--copt=-Wno-deprecated-declarations' - test_flags: - # For protobuf compilation - - '--host_copt=-Wno-deprecated-declarations' - - '--copt=-Wno-deprecated-declarations' integration_test_py_proto_library_macos_workspace: <<: *reusable_build_test_all <<: *common_workspace_flags diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d0e16f020..a2ba32fd76 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,6 @@ A brief description of the categories of changes: [x.x.x]: https://github.com/bazelbuild/rules_python/releases/tag/x.x.x ### Changed -* (rules) `py_proto_library` is deprecated in favour of the - implementation in https://github.com/protocolbuffers/protobuf. It will be - removed in the future release. * (deps) Upgrade the `pip_install` dependencies to pick up a new version of pip. * (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and `py_library` now depend on the `//python:exec_tools_toolchain_type` for build diff --git a/MODULE.bazel b/MODULE.bazel index 7e86bda9b8..53d845ed88 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -9,9 +9,9 @@ bazel_dep(name = "bazel_skylib", version = "1.6.1") bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "platforms", version = "0.0.4") -# For backwards compatibility, those are loaded only when using py_proto_library -# Use py_proto_library directly from protobuf repository -bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf") +# Those are loaded only when using py_proto_library +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") internal_deps = use_extension("//python/private/bzlmod:internal_deps.bzl", "internal_deps") use_repo( diff --git a/WORKSPACE b/WORKSPACE index d30ccb02fd..90e9305684 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -148,3 +148,9 @@ http_file( "https://files.pythonhosted.org/packages/50/67/3e966d99a07d60a21a21d7ec016e9e4c2642a86fea251ec68677daf71d4d/numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", ], ) + +# rules_proto expects //external:python_headers to point at the python headers. +bind( + name = "python_headers", + actual = "//python/cc:current_py_cc_headers", +) diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index 7829af6f21..ca89afe8af 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -55,3 +55,8 @@ http_file( ], ) +# rules_proto expects //external:python_headers to point at the python headers. +bind( + name = "python_headers", + actual = "//python/cc:current_py_cc_headers", +) diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 2e28bd6270..0d30161147 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -11,8 +11,11 @@ local_path_override( path = "../..", ) +# (py_proto_library specific) We are using rules_proto to define rules_proto targets to be consumed by py_proto_library. +bazel_dep(name = "rules_proto", version = "5.3.0-21.7") + # (py_proto_library specific) Add the protobuf library for well-known types (e.g. `Any`, `Timestamp`, etc) -bazel_dep(name = "protobuf", version = "27.0", repo_name = "com_google_protobuf") +bazel_dep(name = "protobuf", version = "21.7", repo_name = "com_google_protobuf") # We next initialize the python toolchain using the extension. # You can set different Python versions in this block. diff --git a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel index 80f0470741..806fcb9dcc 100644 --- a/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/example.com/another_proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") -load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( name = "message_proto_py_pb2", diff --git a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel index 3bc9d1b739..fa20f2ce94 100644 --- a/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel +++ b/examples/bzlmod/py_proto_library/example.com/proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") -load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( name = "pricetag_proto_py_pb2", diff --git a/examples/py_proto_library/WORKSPACE b/examples/py_proto_library/WORKSPACE index 7892c69c4d..81f189dbbf 100644 --- a/examples/py_proto_library/WORKSPACE +++ b/examples/py_proto_library/WORKSPACE @@ -24,11 +24,24 @@ python_register_toolchains( # Then we need to setup dependencies in order to use py_proto_library load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +http_archive( + name = "rules_proto", + sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", + strip_prefix = "rules_proto-6.0.0-rc1", + url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz", +) + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + +rules_proto_dependencies() + +rules_proto_toolchains() + http_archive( name = "com_google_protobuf", - sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa", - strip_prefix = "protobuf-27.0", - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz"], + sha256 = "4fc5ff1b2c339fb86cd3a25f0b5311478ab081e65ad258c6789359cd84d421f8", + strip_prefix = "protobuf-26.1", + urls = ["https://github.com/protocolbuffers/protobuf/archive/v26.1.tar.gz"], ) load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") diff --git a/examples/py_proto_library/example.com/another_proto/BUILD.bazel b/examples/py_proto_library/example.com/another_proto/BUILD.bazel index 126dd9b46d..dd58265bc9 100644 --- a/examples/py_proto_library/example.com/another_proto/BUILD.bazel +++ b/examples/py_proto_library/example.com/another_proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") -load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( name = "message_proto_py_pb2", diff --git a/examples/py_proto_library/example.com/proto/BUILD.bazel b/examples/py_proto_library/example.com/proto/BUILD.bazel index 0084a61794..dc91162aa6 100644 --- a/examples/py_proto_library/example.com/proto/BUILD.bazel +++ b/examples/py_proto_library/example.com/proto/BUILD.bazel @@ -1,5 +1,5 @@ -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") -load("@com_google_protobuf//bazel:py_proto_library.bzl", "py_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_python//python:proto.bzl", "py_proto_library") py_proto_library( name = "pricetag_proto_py_pb2", diff --git a/internal_deps.bzl b/internal_deps.bzl index 8c604c4032..8818751644 100644 --- a/internal_deps.bzl +++ b/internal_deps.bzl @@ -164,12 +164,20 @@ def rules_python_internal_deps(): ], ) + http_archive( + name = "rules_proto", + sha256 = "904a8097fae42a690c8e08d805210e40cccb069f5f9a0f6727cf4faa7bed2c9c", + strip_prefix = "rules_proto-6.0.0-rc1", + url = "https://github.com/bazelbuild/rules_proto/releases/download/6.0.0-rc1/rules_proto-6.0.0-rc1.tar.gz", + ) + http_archive( name = "com_google_protobuf", - sha256 = "da288bf1daa6c04d03a9051781caa52aceb9163586bff9aa6cfb12f69b9395aa", - strip_prefix = "protobuf-27.0", + sha256 = "75be42bd736f4df6d702a0e4e4d30de9ee40eac024c4b845d17ae4cc831fe4ae", + strip_prefix = "protobuf-21.7", urls = [ - "https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protobuf-27.0.tar.gz", + "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", + "https://github.com/protocolbuffers/protobuf/archive/v21.7.tar.gz", ], ) diff --git a/internal_setup.bzl b/internal_setup.bzl index 6614ad355e..bb62611213 100644 --- a/internal_setup.bzl +++ b/internal_setup.bzl @@ -20,6 +20,7 @@ load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies") load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") load("@rules_bazel_integration_test//bazel_integration_test:deps.bzl", "bazel_integration_test_rules_dependencies") load("@rules_bazel_integration_test//bazel_integration_test:repo_defs.bzl", "bazel_binaries") +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS") load("//python/pip_install:repositories.bzl", "pip_install_dependencies") load("//python/private:internal_config_repo.bzl", "internal_config_repo") # buildifier: disable=bzl-visibility @@ -34,6 +35,9 @@ def rules_python_internal_setup(): bazel_skylib_workspace() + rules_proto_dependencies() + rules_proto_toolchains() + protobuf_deps() bazel_integration_test_rules_dependencies() diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 100a8c04a5..cbf29964fb 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -113,7 +113,7 @@ bzl_library( ], visibility = ["//visibility:public"], deps = [ - "@com_google_protobuf//bazel:py_proto_library_bzl", + "//python/private/proto:py_proto_library_bzl", ], ) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index d73cee8ae4..422ed9c7c2 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -29,6 +29,7 @@ filegroup( srcs = glob(["**"]) + [ "//python/private/bzlmod:distribution", "//python/private/common:distribution", + "//python/private/proto:distribution", "//python/private/whl_filegroup:distribution", "//tools/build_defs/python/private:distribution", ], diff --git a/python/private/proto/BUILD.bazel b/python/private/proto/BUILD.bazel new file mode 100644 index 0000000000..65c09444f7 --- /dev/null +++ b/python/private/proto/BUILD.bazel @@ -0,0 +1,46 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") + +package(default_visibility = ["//visibility:private"]) + +licenses(["notice"]) + +filegroup( + name = "distribution", + srcs = glob(["**"]), + visibility = ["//python/private:__pkg__"], +) + +bzl_library( + name = "py_proto_library_bzl", + srcs = ["py_proto_library.bzl"], + visibility = ["//python:__pkg__"], + deps = [ + "//python:defs_bzl", + "@rules_proto//proto:defs", + ], +) + +proto_lang_toolchain( + name = "python_toolchain", + command_line = "--python_out=%s", + progress_message = "Generating Python proto_library %{label}", + runtime = "@com_google_protobuf//:protobuf_python", + # NOTE: This isn't *actually* public. It's an implicit dependency of py_proto_library, + # so must be public so user usages of the rule can reference it. + visibility = ["//visibility:public"], +) diff --git a/python/private/proto/py_proto_library.bzl b/python/private/proto/py_proto_library.bzl new file mode 100644 index 0000000000..e123ff8476 --- /dev/null +++ b/python/private/proto/py_proto_library.bzl @@ -0,0 +1,223 @@ +# Copyright 2022 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The implementation of the `py_proto_library` rule and its aspect.""" + +load("@rules_proto//proto:defs.bzl", "ProtoInfo", "proto_common") +load("//python:defs.bzl", "PyInfo") + +PY_PROTO_TOOLCHAIN = "@rules_python//python/proto:toolchain_type" + +_PyProtoInfo = provider( + doc = "Encapsulates information needed by the Python proto rules.", + fields = { + "imports": """ + (depset[str]) The field forwarding PyInfo.imports coming from + the proto language runtime dependency.""", + "runfiles_from_proto_deps": """ + (depset[File]) Files from the transitive closure implicit proto + dependencies""", + "transitive_sources": """(depset[File]) The Python sources.""", + }, +) + +def _filter_provider(provider, *attrs): + return [dep[provider] for attr in attrs for dep in attr if provider in dep] + +def _incompatible_toolchains_enabled(): + return getattr(proto_common, "INCOMPATIBLE_ENABLE_PROTO_TOOLCHAIN_RESOLUTION", False) + +def _py_proto_aspect_impl(target, ctx): + """Generates and compiles Python code for a proto_library. + + The function runs protobuf compiler on the `proto_library` target generating + a .py file for each .proto file. + + Args: + target: (Target) A target providing `ProtoInfo`. Usually this means a + `proto_library` target, but not always; you must expect to visit + non-`proto_library` targets, too. + ctx: (RuleContext) The rule context. + + Returns: + ([_PyProtoInfo]) Providers collecting transitive information about + generated files. + """ + _proto_library = ctx.rule.attr + + # Check Proto file names + for proto in target[ProtoInfo].direct_sources: + if proto.is_source and "-" in proto.dirname: + fail("Cannot generate Python code for a .proto whose path contains '-' ({}).".format( + proto.path, + )) + + if _incompatible_toolchains_enabled(): + toolchain = ctx.toolchains[PY_PROTO_TOOLCHAIN] + if not toolchain: + fail("No toolchains registered for '%s'." % PY_PROTO_TOOLCHAIN) + proto_lang_toolchain_info = toolchain.proto + else: + proto_lang_toolchain_info = getattr(ctx.attr, "_aspect_proto_toolchain")[proto_common.ProtoLangToolchainInfo] + + api_deps = [proto_lang_toolchain_info.runtime] + + generated_sources = [] + proto_info = target[ProtoInfo] + proto_root = proto_info.proto_source_root + if proto_info.direct_sources: + # Generate py files + generated_sources = proto_common.declare_generated_files( + actions = ctx.actions, + proto_info = proto_info, + extension = "_pb2.py", + name_mapper = lambda name: name.replace("-", "_").replace(".", "/"), + ) + + # Handles multiple repository and virtual import cases + if proto_root.startswith(ctx.bin_dir.path): + proto_root = proto_root[len(ctx.bin_dir.path) + 1:] + + plugin_output = ctx.bin_dir.path + "/" + proto_root + proto_root = ctx.workspace_name + "/" + proto_root + + proto_common.compile( + actions = ctx.actions, + proto_info = proto_info, + proto_lang_toolchain_info = proto_lang_toolchain_info, + generated_files = generated_sources, + plugin_output = plugin_output, + ) + + # Generated sources == Python sources + python_sources = generated_sources + + deps = _filter_provider(_PyProtoInfo, getattr(_proto_library, "deps", [])) + runfiles_from_proto_deps = depset( + transitive = [dep[DefaultInfo].default_runfiles.files for dep in api_deps] + + [dep.runfiles_from_proto_deps for dep in deps], + ) + transitive_sources = depset( + direct = python_sources, + transitive = [dep.transitive_sources for dep in deps], + ) + + return [ + _PyProtoInfo( + imports = depset( + # Adding to PYTHONPATH so the generated modules can be + # imported. This is necessary when there is + # strip_import_prefix, the Python modules are generated under + # _virtual_imports. But it's undesirable otherwise, because it + # will put the repo root at the top of the PYTHONPATH, ahead of + # directories added through `imports` attributes. + [proto_root] if "_virtual_imports" in proto_root else [], + transitive = [dep[PyInfo].imports for dep in api_deps] + [dep.imports for dep in deps], + ), + runfiles_from_proto_deps = runfiles_from_proto_deps, + transitive_sources = transitive_sources, + ), + ] + +_py_proto_aspect = aspect( + implementation = _py_proto_aspect_impl, + attrs = {} if _incompatible_toolchains_enabled() else { + "_aspect_proto_toolchain": attr.label( + default = ":python_toolchain", + ), + }, + attr_aspects = ["deps"], + required_providers = [ProtoInfo], + provides = [_PyProtoInfo], + toolchains = [PY_PROTO_TOOLCHAIN] if _incompatible_toolchains_enabled() else [], +) + +def _py_proto_library_rule(ctx): + """Merges results of `py_proto_aspect` in `deps`. + + Args: + ctx: (RuleContext) The rule context. + Returns: + ([PyInfo, DefaultInfo, OutputGroupInfo]) + """ + if not ctx.attr.deps: + fail("'deps' attribute mustn't be empty.") + + pyproto_infos = _filter_provider(_PyProtoInfo, ctx.attr.deps) + default_outputs = depset( + transitive = [info.transitive_sources for info in pyproto_infos], + ) + + return [ + DefaultInfo( + files = default_outputs, + default_runfiles = ctx.runfiles(transitive_files = depset( + transitive = + [default_outputs] + + [info.runfiles_from_proto_deps for info in pyproto_infos], + )), + ), + OutputGroupInfo( + default = depset(), + ), + PyInfo( + transitive_sources = default_outputs, + imports = depset(transitive = [info.imports for info in pyproto_infos]), + # Proto always produces 2- and 3- compatible source files + has_py2_only_sources = False, + has_py3_only_sources = False, + ), + ] + +py_proto_library = rule( + implementation = _py_proto_library_rule, + doc = """ + Use `py_proto_library` to generate Python libraries from `.proto` files. + + The convention is to name the `py_proto_library` rule `foo_py_pb2`, + when it is wrapping `proto_library` rule `foo_proto`. + + `deps` must point to a `proto_library` rule. + + Example: + +```starlark +py_library( + name = "lib", + deps = [":foo_py_pb2"], +) + +py_proto_library( + name = "foo_py_pb2", + deps = [":foo_proto"], +) + +proto_library( + name = "foo_proto", + srcs = ["foo.proto"], +) +```""", + attrs = { + "deps": attr.label_list( + doc = """ + The list of `proto_library` rules to generate Python libraries for. + + Usually this is just the one target: the proto library of interest. + It can be any target providing `ProtoInfo`.""", + providers = [ProtoInfo], + aspects = [_py_proto_aspect], + ), + }, + provides = [PyInfo], +) diff --git a/python/proto.bzl b/python/proto.bzl index 2ea9bdb153..3f455aee58 100644 --- a/python/proto.bzl +++ b/python/proto.bzl @@ -11,11 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + """ Python proto library. """ -load("@com_google_protobuf//bazel:py_proto_library.bzl", _py_proto_library = "py_proto_library") +load("//python/private/proto:py_proto_library.bzl", _py_proto_library = "py_proto_library") -def py_proto_library(*, deprecation = "Use py_proto_library from protobuf repository", **kwargs): - _py_proto_library(deprecation = deprecation, **kwargs) +py_proto_library = _py_proto_library