From b01f92d71718c7f9950a5457eb2a77d386070ad6 Mon Sep 17 00:00:00 2001 From: njlr Date: Sat, 21 Dec 2024 16:54:34 +0000 Subject: [PATCH 1/7] Add support for Poetry dependency groups --- examples/poetry/example_lock.bzl | 21 +++++++++++++++++++++ examples/poetry/poetry.lock | 10 ++++++++++ examples/poetry/pyproject.toml | 3 +++ examples/poetry/tools/BUILD.bazel | 7 +++++++ examples/poetry/tools/app.py | 3 +++ pycross/private/tools/poetry_translator.py | 12 +++++++++++- 6 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 examples/poetry/tools/app.py diff --git a/examples/poetry/example_lock.bzl b/examples/poetry/example_lock.bzl index f8a4cc31..aec7e575 100755 --- a/examples/poetry/example_lock.bzl +++ b/examples/poetry/example_lock.bzl @@ -22,6 +22,7 @@ PINS = { "charset-normalizer": "charset-normalizer@3.3.2", "click": "click@8.1.7", "cognitojwt": "cognitojwt@1.4.1", + "cowsay": "cowsay@6.1", "cryptography": "cryptography@41.0.5", "cython": "cython@0.29.36", "decorator": "decorator@5.1.1", @@ -367,6 +368,16 @@ def targets(): wheel = ":_wheel_cognitojwt@1.4.1", ) + native.alias( + name = "_wheel_cowsay@6.1", + actual = "@example_lock_wheel_cowsay_6.1_py3_none_any//file", + ) + + pycross_wheel_library( + name = "cowsay@6.1", + wheel = ":_wheel_cowsay@6.1", + ) + _cryptography_41_0_5_deps = [ ":cffi@1.16.0", ] @@ -1874,6 +1885,16 @@ def repositories(): index = "https://pypi.org", ) + maybe( + pypi_file, + name = "example_lock_wheel_cowsay_6.1_py3_none_any", + package_name = "cowsay", + package_version = "6.1", + filename = "cowsay-6.1-py3-none-any.whl", + sha256 = "274b1e6fc1b966d53976333eb90ac94cb07a450a700b455af9fbdf882244b30a", + index = "https://pypi.org", + ) + maybe( pypi_file, name = "example_lock_wheel_cryptography_41.0.5_cp37_abi3_macosx_10_12_universal2", diff --git a/examples/poetry/poetry.lock b/examples/poetry/poetry.lock index ac2774da..df1faade 100644 --- a/examples/poetry/poetry.lock +++ b/examples/poetry/poetry.lock @@ -2632,3 +2632,13 @@ files = [ lock-version = "2.0" python-versions = ">=3.11,<3.13" content-hash = "058183c94e35a7f1f3105fac99063ccd83b2c240b6e0552711b551b34bd7c72d" + +[[package]] +name = "cowsay" +version = "6.1" +description = "The famous cowsay for GNU/Linux is now available for python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "cowsay-6.1-py3-none-any.whl", hash = "sha256:274b1e6fc1b966d53976333eb90ac94cb07a450a700b455af9fbdf882244b30a"}, +] diff --git a/examples/poetry/pyproject.toml b/examples/poetry/pyproject.toml index d93a2afb..5d7a2cc7 100644 --- a/examples/poetry/pyproject.toml +++ b/examples/poetry/pyproject.toml @@ -21,3 +21,6 @@ tree-sitter = "=0.20.2" future = "=0.18.2" opencv-python = "=4.6.0.66" keyring = "=23.9.1" + +[tool.poetry.group.dev.dependencies] +cowsay = "^6.1" diff --git a/examples/poetry/tools/BUILD.bazel b/examples/poetry/tools/BUILD.bazel index 1ccd6559..07789c3f 100644 --- a/examples/poetry/tools/BUILD.bazel +++ b/examples/poetry/tools/BUILD.bazel @@ -18,3 +18,10 @@ py_binary( main = "ipython.py", deps = [requirement("ipython")], ) + +py_binary( + name = "app", + srcs = ["app.py"], + main = "app.py", + deps = [requirement("cowsay")], +) diff --git a/examples/poetry/tools/app.py b/examples/poetry/tools/app.py new file mode 100644 index 00000000..1cb1ab92 --- /dev/null +++ b/examples/poetry/tools/app.py @@ -0,0 +1,3 @@ +import cowsay + +cowsay.cow("Hello rules_pycross") diff --git a/pycross/private/tools/poetry_translator.py b/pycross/private/tools/poetry_translator.py index c80b6edc..9ef179a4 100644 --- a/pycross/private/tools/poetry_translator.py +++ b/pycross/private/tools/poetry_translator.py @@ -113,7 +113,17 @@ def translate(project_file: Path, lock_file: Path) -> RawLockSet: raise Exception(f"Could not load lock file: {lock_file}: {e}") pinned_package_specs = {} - for pin, pin_info in (project_dict.get("tool", {}).get("poetry", {}).get("dependencies", {})).items(): + + dependency_items = [] + + dependency_items.extend((project_dict.get("tool", {}).get("poetry", {}).get("dependencies", {})).items()) + + groups = project_dict.get("tool", {}).get("poetry", {}).get("group", {}) + + for _, group in groups.items(): + dependency_items.extend(group.get("dependencies", {}).items()) + + for pin, pin_info in dependency_items: pin = package_canonical_name(pin) if pin == "python": # Skip the special line indicating python version. From 5f749a93d8f5a47131df0ede2a52327eb4123354 Mon Sep 17 00:00:00 2001 From: njlr Date: Mon, 6 Jan 2025 12:40:31 +0000 Subject: [PATCH 2/7] Implement dep group attrs --- examples/poetry/BUILD.bazel | 1 + pycross/private/lock_attrs.bzl | 10 +++++ pycross/private/poetry_lock_model.bzl | 9 +++++ pycross/private/tools/poetry_translator.py | 43 +++++++++++++++++++--- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/examples/poetry/BUILD.bazel b/examples/poetry/BUILD.bazel index f54608e8..27f5c006 100644 --- a/examples/poetry/BUILD.bazel +++ b/examples/poetry/BUILD.bazel @@ -74,6 +74,7 @@ pycross_poetry_lock_model( name = "example_lock_model", lock_file = "poetry.lock", project_file = "pyproject.toml", + optional_groups = ["dev"], ) pycross_lock_file( diff --git a/pycross/private/lock_attrs.bzl b/pycross/private/lock_attrs.bzl index 89f85b55..8eb09a35 100644 --- a/pycross/private/lock_attrs.bzl +++ b/pycross/private/lock_attrs.bzl @@ -139,6 +139,16 @@ POETRY_IMPORT_ATTRS = dict( allow_single_file = True, mandatory = True, ), + default = attr.bool( + doc = "Whether to install dependencies from the default group.", + default = True, + ), + optional_groups = attr.string_list( + doc = "List of optional dependency groups to install.", + ), + all_optional_groups = attr.bool( + doc = "Install all optional dependencies.", + ), ) def handle_resolve_attrs(attrs, environment_files_and_labels, local_wheel_names_and_labels): diff --git a/pycross/private/poetry_lock_model.bzl b/pycross/private/poetry_lock_model.bzl index ffda441e..59e2b086 100644 --- a/pycross/private/poetry_lock_model.bzl +++ b/pycross/private/poetry_lock_model.bzl @@ -13,6 +13,15 @@ def _pycross_poetry_lock_model_impl(ctx): args.add("--lock-file", ctx.file.lock_file) args.add("--output", out) + if ctx.attr.default: + args.add("--default") + + for group in ctx.attr.optional_groups: + args.add_all(["--optional-group", group]) + + if ctx.attr.all_optional_groups: + args.add("--all-optional-groups") + ctx.actions.run( inputs = ( ctx.files.project_file + diff --git a/pycross/private/tools/poetry_translator.py b/pycross/private/tools/poetry_translator.py index 9ef179a4..a0a82483 100644 --- a/pycross/private/tools/poetry_translator.py +++ b/pycross/private/tools/poetry_translator.py @@ -99,7 +99,13 @@ def get_files_for_package( return result -def translate(project_file: Path, lock_file: Path) -> RawLockSet: +def translate( + project_file: Path, + lock_file: Path, + default_group: bool, + optional_groups: List[str], + all_optional_groups: bool, +) -> RawLockSet: try: with open(project_file, "rb") as f: project_dict = tomli.load(f) @@ -116,12 +122,14 @@ def translate(project_file: Path, lock_file: Path) -> RawLockSet: dependency_items = [] - dependency_items.extend((project_dict.get("tool", {}).get("poetry", {}).get("dependencies", {})).items()) + if default_group: + dependency_items.extend((project_dict.get("tool", {}).get("poetry", {}).get("dependencies", {})).items()) groups = project_dict.get("tool", {}).get("poetry", {}).get("group", {}) - for _, group in groups.items(): - dependency_items.extend(group.get("dependencies", {}).items()) + for group_name, group in groups.items(): + if all_optional_groups or group_name in optional_groups: + dependency_items.extend(group.get("dependencies", {}).items()) for pin, pin_info in dependency_items: pin = package_canonical_name(pin) @@ -249,7 +257,13 @@ def parse_file_info(file_info) -> PackageFile: def main(args: Any) -> None: output = args.output - lock_set = translate(args.project_file, args.lock_file) + lock_set = translate( + project_file=args.project_file, + lock_file=args.lock_file, + default_group=args.default_group, + optional_groups=args.optional_group, + all_optional_groups=args.all_optional_groups, + ) with open(output, "w") as f: f.write(lock_set.to_json(indent=2)) @@ -272,6 +286,25 @@ def parse_flags() -> Any: help="The path to pdm.lock.", ) + parser.add_argument( + "--default-group", + action="store_true", + help="Whether to install dependencies from the default group.", + ) + + parser.add_argument( + "--optional-group", + action="append", + default=[], + help="Optional dependency groups to install.", + ) + + parser.add_argument( + "--all-optional-groups", + action="store_true", + help="Install all optional dependency groups.", + ) + parser.add_argument( "--output", type=Path, From d2a896190840ab042c4bc10f0d2b28dc33eb1b1e Mon Sep 17 00:00:00 2001 From: njlr Date: Mon, 6 Jan 2025 12:47:16 +0000 Subject: [PATCH 3/7] Fix arg string --- pycross/private/tools/poetry_translator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pycross/private/tools/poetry_translator.py b/pycross/private/tools/poetry_translator.py index a0a82483..e767adc1 100644 --- a/pycross/private/tools/poetry_translator.py +++ b/pycross/private/tools/poetry_translator.py @@ -260,7 +260,7 @@ def main(args: Any) -> None: lock_set = translate( project_file=args.project_file, lock_file=args.lock_file, - default_group=args.default_group, + default_group=args.default, optional_groups=args.optional_group, all_optional_groups=args.all_optional_groups, ) @@ -287,7 +287,7 @@ def parse_flags() -> Any: ) parser.add_argument( - "--default-group", + "--default", action="store_true", help="Whether to install dependencies from the default group.", ) From 47ca5c83e07f18104c2a80ca049f7fe68b26f113 Mon Sep 17 00:00:00 2001 From: njlr Date: Mon, 6 Jan 2025 17:11:12 +0000 Subject: [PATCH 4/7] Update docs --- docs/ext_lock_import.md | 8 ++++++-- docs/rules.md | 6 +++++- examples/poetry/BUILD.bazel | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/ext_lock_import.md b/docs/ext_lock_import.md index c3cf4f50..0264558c 100644 --- a/docs/ext_lock_import.md +++ b/docs/ext_lock_import.md @@ -12,8 +12,9 @@ lock_import.import_pdm( default_alias_single_version, default_build_dependencies, development_groups, disallow_builds, local_wheels, lock_file, optional_groups, project_file, repo, require_static_urls, target_environments) -lock_import.import_poetry(default_alias_single_version, default_build_dependencies, disallow_builds, - local_wheels, lock_file, project_file, repo, target_environments) +lock_import.import_poetry(all_optional_groups, default, default_alias_single_version, + default_build_dependencies, disallow_builds, local_wheels, lock_file, + optional_groups, project_file, repo, target_environments) lock_import.import_uv(all_development_groups, all_optional_groups, default, default_alias_single_version, default_build_dependencies, development_groups, disallow_builds, local_wheels, lock_file, optional_groups, project_file, repo, @@ -60,11 +61,14 @@ Import a Poetry lock file. | Name | Description | Type | Mandatory | Default | | :------------- | :------------- | :------------- | :------------- | :------------- | +| all_optional_groups | Install all optional dependencies. | Boolean | optional | `False` | +| default | Whether to install dependencies from the default group. | Boolean | optional | `True` | | default_alias_single_version | Generate aliases for all packages that have a single version in the lock file. | Boolean | optional | `False` | | default_build_dependencies | A list of package keys (name or name@version) that will be used as default build dependencies. | List of strings | optional | `[]` | | disallow_builds | If True, only pre-built wheels are allowed. | Boolean | optional | `False` | | local_wheels | A list of local .whl files to consider when processing lock files. | List of labels | optional | `[]` | | lock_file | The poetry.lock file. | Label | required | | +| optional_groups | List of optional dependency groups to install. | List of strings | optional | `[]` | | project_file | The pyproject.toml file. | Label | required | | | repo | The repository name | String | required | | | target_environments | A list of target environment descriptors. | List of labels | optional | `["@pycross_environments//:environments"]` | diff --git a/docs/rules.md b/docs/rules.md index 5092c834..b4057c50 100644 --- a/docs/rules.md +++ b/docs/rules.md @@ -70,7 +70,8 @@ pycross_pdm_lock_model(name, -pycross_poetry_lock_model(name, lock_file, project_file) +pycross_poetry_lock_model(name, all_optional_groups, default, lock_file, optional_groups, + project_file) @@ -81,7 +82,10 @@ pycross_poetry_lock_model(name, name | A unique name for this target. | Name | required | | +| all_optional_groups | Install all optional dependencies. | Boolean | optional | `False` | +| default | Whether to install dependencies from the default group. | Boolean | optional | `True` | | lock_file | The poetry.lock file. | Label | required | | +| optional_groups | List of optional dependency groups to install. | List of strings | optional | `[]` | | project_file | The pyproject.toml file. | Label | required | | diff --git a/examples/poetry/BUILD.bazel b/examples/poetry/BUILD.bazel index 27f5c006..91e24cb2 100644 --- a/examples/poetry/BUILD.bazel +++ b/examples/poetry/BUILD.bazel @@ -73,8 +73,8 @@ pycross_target_environment( pycross_poetry_lock_model( name = "example_lock_model", lock_file = "poetry.lock", - project_file = "pyproject.toml", optional_groups = ["dev"], + project_file = "pyproject.toml", ) pycross_lock_file( From 07d313ff45b4a6b3e8e2a1d8f2e3e06247884da0 Mon Sep 17 00:00:00 2001 From: njlr Date: Mon, 6 Jan 2025 17:38:49 +0000 Subject: [PATCH 5/7] Fix repo Poetry model --- pycross/private/poetry_lock_model.bzl | 48 +++++++++++++++++---------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/pycross/private/poetry_lock_model.bzl b/pycross/private/poetry_lock_model.bzl index 59e2b086..53dff596 100644 --- a/pycross/private/poetry_lock_model.bzl +++ b/pycross/private/poetry_lock_model.bzl @@ -5,22 +5,33 @@ load(":lock_attrs.bzl", "POETRY_IMPORT_ATTRS") TRANSLATOR_TOOL = Label("//pycross/private/tools:poetry_translator.py") -def _pycross_poetry_lock_model_impl(ctx): - out = ctx.actions.declare_file(ctx.attr.name + ".json") +def _handle_args(attrs, project_file, lock_file, output): + args = [] + args.extend(["--project-file", project_file]) + args.extend(["--lock-file", lock_file]) + args.extend(["--output", output]) - args = ctx.actions.args().use_param_file("--flagfile=%s") - args.add("--project-file", ctx.file.project_file) - args.add("--lock-file", ctx.file.lock_file) - args.add("--output", out) + if attrs.default: + args.append("--default") - if ctx.attr.default: - args.add("--default") + for group in attrs.optional_groups: + args.extend(["--optional-group", group]) - for group in ctx.attr.optional_groups: - args.add_all(["--optional-group", group]) + return args + + +def _pycross_poetry_lock_model_impl(ctx): + out = ctx.actions.declare_file(ctx.attr.name + ".json") - if ctx.attr.all_optional_groups: - args.add("--all-optional-groups") + args = ctx.actions.args().use_param_file("--flagfile=%s") + args.add_all( + _handle_args( + ctx.attr, + ctx.file.project_file.path, + ctx.file.lock_file.path, + out.path, + ), + ) ctx.actions.run( inputs = ( @@ -49,11 +60,14 @@ pycross_poetry_lock_model = rule( } | POETRY_IMPORT_ATTRS, ) -def lock_repo_model_poetry(*, project_file, lock_file): +def lock_repo_model_poetry(*, project_file, lock_file, default = True, optional_groups = [], all_optional_groups = False): return json.encode(dict( model_type = "poetry", project_file = str(project_file), lock_file = str(lock_file), + default = default, + optional_groups = optional_groups, + all_optional_groups = all_optional_groups, )) def repo_create_poetry_model(rctx, params, output): @@ -68,14 +82,12 @@ def repo_create_poetry_model(rctx, params, output): attrs = struct(**params) else: attrs = params - args = [ - "--project-file", + args = _handle_args( + attrs, str(rctx.path(Label(attrs.project_file))), - "--lock-file", str(rctx.path(Label(attrs.lock_file))), - "--output", output, - ] + ) exec_internal_tool( rctx, From f158e89ed224dc77489ca82213362dd9999b248e Mon Sep 17 00:00:00 2001 From: njlr Date: Mon, 6 Jan 2025 17:40:50 +0000 Subject: [PATCH 6/7] Update docs --- docs/workspace_rules.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/workspace_rules.md b/docs/workspace_rules.md index 393f57c5..c06991a0 100644 --- a/docs/workspace_rules.md +++ b/docs/workspace_rules.md @@ -33,7 +33,7 @@ lock_repo_model_pdm(project_file ## lock_repo_model_poetry 
-lock_repo_model_poetry(project_file, lock_file)
+lock_repo_model_poetry(project_file, lock_file, default, optional_groups, all_optional_groups)
@@ -45,6 +45,9 @@ lock_repo_model_poetry(project_fi | :------------- | :------------- | :------------- | | project_file |

-

| none | | lock_file |

-

| none | +| default |

-

| `True` | +| optional_groups |

-

| `[]` | +| all_optional_groups |

-

| `False` | From 997bf9e184c819db9079477addcd48baa0531e33 Mon Sep 17 00:00:00 2001 From: njlr Date: Mon, 6 Jan 2025 17:51:25 +0000 Subject: [PATCH 7/7] Buildifier --- pycross/private/poetry_lock_model.bzl | 1 - 1 file changed, 1 deletion(-) diff --git a/pycross/private/poetry_lock_model.bzl b/pycross/private/poetry_lock_model.bzl index 53dff596..5ba89115 100644 --- a/pycross/private/poetry_lock_model.bzl +++ b/pycross/private/poetry_lock_model.bzl @@ -19,7 +19,6 @@ def _handle_args(attrs, project_file, lock_file, output): return args - def _pycross_poetry_lock_model_impl(ctx): out = ctx.actions.declare_file(ctx.attr.name + ".json")