diff --git a/doc/source/ray-core/handling-dependencies.rst b/doc/source/ray-core/handling-dependencies.rst index 209bb7d728c2..2b5e1ee931d1 100644 --- a/doc/source/ray-core/handling-dependencies.rst +++ b/doc/source/ray-core/handling-dependencies.rst @@ -408,6 +408,9 @@ The ``runtime_env`` is a Python dictionary or a Python class :class:`ray.runtime - ``uv`` (dict | List[str] | str): Alpha version feature. Either (1) a list of uv `requirements specifiers `_, (2) a string containing the path to a local uv `“requirements.txt” `_ file, or (3) a python dictionary that has three fields: (a) ``packages`` (required, List[str]): a list of uv packages, (b) ``uv_version`` (optional, str): the version of uv; Ray will spell the package name "uv" in front of the ``uv_version`` to form the final requirement string. + (c) ``uv_check`` (optional, bool): whether to enable pip check at the end of uv install, default to False. + (d) ``uv_pip_install_options`` (optional, List[str]): user-provided options for ``uv pip install`` command, default to ``["--no-cache"]``. + To override the default options and install without any options, use an empty list ``[]`` as install option value. The syntax of a requirement specifier is the same as ``pip`` requirements. This will be installed in the Ray workers at runtime. Packages in the preinstalled cluster environment will still be available. To use a library like Ray Serve or Ray Tune, you will need to include ``"ray[serve]"`` or ``"ray[tune]"`` here. diff --git a/python/ray/_private/runtime_env/uv.py b/python/ray/_private/runtime_env/uv.py index 78d4ad2c55f4..c182723aa0d3 100644 --- a/python/ray/_private/runtime_env/uv.py +++ b/python/ray/_private/runtime_env/uv.py @@ -166,19 +166,23 @@ async def _install_uv_packages( # # Difference with pip: # 1. `--disable-pip-version-check` has no effect for uv. - # 2. `--no-cache-dir` for `pip` maps to `--no-cache` for uv. - pip_install_cmd = [ + # 2. Allow user to specify their own options to install packages via `uv`. + uv_install_cmd = [ python, "-m", "uv", "pip", "install", - "--no-cache", "-r", requirements_file, ] + + uv_opt_list = self._uv_config.get("uv_pip_install_options", ["--no-cache"]) + if uv_opt_list: + uv_install_cmd += uv_opt_list + logger.info("Installing python requirements to %s", virtualenv_path) - await check_output_cmd(pip_install_cmd, logger=logger, cwd=cwd, env=pip_env) + await check_output_cmd(uv_install_cmd, logger=logger, cwd=cwd, env=pip_env) # Check python environment for conflicts. if self._uv_config.get("uv_check", False): diff --git a/python/ray/_private/runtime_env/validation.py b/python/ray/_private/runtime_env/validation.py index a2f37aa0816e..fa6d9bb16355 100644 --- a/python/ray/_private/runtime_env/validation.py +++ b/python/ray/_private/runtime_env/validation.py @@ -126,6 +126,10 @@ def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]: a) packages (required, List[str]): a list of uv packages, it same as 1). b) uv_check (optional, bool): whether to enable pip check at the end of uv install, default to False. + c) uv_version (optional, str): user provides a specific uv to use; if + unspecified, default version of uv will be used. + d) uv_pip_install_options (optional, List[str]): user-provided options for + `uv pip install` command, default to ["--no-cache"]. The returned parsed value will be a list of packages. If a Ray library (e.g. "ray[serve]") is specified, it will be deleted and replaced by its @@ -146,10 +150,15 @@ def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]: elif isinstance(uv, list) and all(isinstance(dep, str) for dep in uv): result = dict(packages=uv, uv_check=False) elif isinstance(uv, dict): - if set(uv.keys()) - {"packages", "uv_check", "uv_version"}: + if set(uv.keys()) - { + "packages", + "uv_check", + "uv_version", + "uv_pip_install_options", + }: raise ValueError( "runtime_env['uv'] can only have these fields: " - "packages, uv_check and uv_version, but got: " + "packages, uv_check, uv_version and uv_pip_install_options, but got: " f"{list(uv.keys())}" ) if "packages" not in uv: @@ -166,9 +175,25 @@ def parse_and_validate_uv(uv: Union[str, List[str], Dict]) -> Optional[Dict]: "runtime_env['uv']['uv_version'] must be of type str, " f"got {type(uv['uv_version'])}" ) + if "uv_pip_install_options" in uv: + if not isinstance(uv["uv_pip_install_options"], list): + raise TypeError( + "runtime_env['uv']['uv_pip_install_options'] must be of type " + f"list[str] got {type(uv['uv_pip_install_options'])}" + ) + # Check each item in installation option. + for idx, cur_opt in enumerate(uv["uv_pip_install_options"]): + if not isinstance(cur_opt, str): + raise TypeError( + "runtime_env['uv']['uv_pip_install_options'] must be of type " + f"list[str] got {type(cur_opt)} for {idx}-th item." + ) result = uv.copy() result["uv_check"] = uv.get("uv_check", False) + result["uv_pip_install_options"] = uv.get( + "uv_pip_install_options", ["--no-cache"] + ) if not isinstance(uv["packages"], list): raise ValueError( "runtime_env['uv']['packages'] must be of type list, " diff --git a/python/ray/tests/test_runtime_env_uv.py b/python/ray/tests/test_runtime_env_uv.py index b698cd41ba3a..416d067140f6 100644 --- a/python/ray/tests/test_runtime_env_uv.py +++ b/python/ray/tests/test_runtime_env_uv.py @@ -108,6 +108,83 @@ def f(): assert ray.get(f.remote()) == "2.3.0" +# Install different versions of the same package across different tasks, used to check +# uv cache doesn't break runtime env requirement. +def test_package_install_with_different_versions(shutdown_only): + @ray.remote(runtime_env={"uv": {"packages": ["requests==2.3.0"]}}) + def f(): + import requests + + assert requests.__version__ == "2.3.0" + + @ray.remote(runtime_env={"uv": {"packages": ["requests==2.2.0"]}}) + def g(): + import requests + + assert requests.__version__ == "2.2.0" + + ray.get(f.remote()) + ray.get(g.remote()) + + +# Install packages with cache enabled. +def test_package_install_with_cache_enabled(shutdown_only): + @ray.remote( + runtime_env={ + "uv": {"packages": ["requests==2.3.0"], "uv_pip_install_options": []} + } + ) + def f(): + import requests + + assert requests.__version__ == "2.3.0" + + @ray.remote( + runtime_env={ + "uv": {"packages": ["requests==2.2.0"], "uv_pip_install_options": []} + } + ) + def g(): + import requests + + assert requests.__version__ == "2.2.0" + + ray.get(f.remote()) + ray.get(g.remote()) + + +# Testing senario: install packages with `uv` with multiple options. +def test_package_install_with_multiple_options(shutdown_only): + @ray.remote( + runtime_env={ + "uv": { + "packages": ["requests==2.3.0"], + "uv_pip_install_options": ["--no-cache", "--color=auto"], + } + } + ) + def f(): + import requests + + assert requests.__version__ == "2.3.0" + + @ray.remote( + runtime_env={ + "uv": { + "packages": ["requests==2.2.0"], + "uv_pip_install_options": ["--no-cache", "--color=auto"], + } + } + ) + def g(): + import requests + + assert requests.__version__ == "2.2.0" + + ray.get(f.remote()) + ray.get(g.remote()) + + if __name__ == "__main__": if os.environ.get("PARALLEL_CI"): sys.exit(pytest.main(["-n", "auto", "--boxed", "-vs", __file__])) diff --git a/python/ray/tests/unit/test_runtime_env_validation.py b/python/ray/tests/unit/test_runtime_env_validation.py index 341e9fff7995..487569a68a5b 100644 --- a/python/ray/tests/unit/test_runtime_env_validation.py +++ b/python/ray/tests/unit/test_runtime_env_validation.py @@ -32,13 +32,21 @@ class TestVaidationUv: def test_parse_and_validate_uv(self, test_directory): # Valid case w/o duplication. result = validation.parse_and_validate_uv({"packages": ["tensorflow"]}) - assert result == {"packages": ["tensorflow"], "uv_check": False} + assert result == { + "packages": ["tensorflow"], + "uv_check": False, + "uv_pip_install_options": ["--no-cache"], + } # Valid case w/ duplication. result = validation.parse_and_validate_uv( {"packages": ["tensorflow", "tensorflow"]} ) - assert result == {"packages": ["tensorflow"], "uv_check": False} + assert result == { + "packages": ["tensorflow"], + "uv_check": False, + "uv_pip_install_options": ["--no-cache"], + } # Valid case, use `list` to represent necessary packages. result = validation.parse_and_validate_uv( @@ -61,6 +69,7 @@ def test_parse_and_validate_uv(self, test_directory): "packages": ["tensorflow"], "uv_version": "==0.4.30", "uv_check": False, + "uv_pip_install_options": ["--no-cache"], } # Valid requirement files. @@ -76,6 +85,31 @@ def test_parse_and_validate_uv(self, test_directory): with pytest.raises(ValueError): result = validation.parse_and_validate_uv("some random non-existent file") + # Invalid uv install options. + with pytest.raises(TypeError): + result = validation.parse_and_validate_uv( + { + "packages": ["tensorflow"], + "uv_version": "==0.4.30", + "uv_pip_install_options": [1], + } + ) + + # Valid uv install options. + result = validation.parse_and_validate_uv( + { + "packages": ["tensorflow"], + "uv_version": "==0.4.30", + "uv_pip_install_options": ["--no-cache"], + } + ) + assert result == { + "packages": ["tensorflow"], + "uv_check": False, + "uv_pip_install_options": ["--no-cache"], + "uv_version": "==0.4.30", + } + class TestValidatePip: def test_validate_pip_invalid_types(self):