diff --git a/docs/pypi/download.md b/docs/pypi/download.md index e4ab38714d..b9904d4071 100644 --- a/docs/pypi/download.md +++ b/docs/pypi/download.md @@ -121,6 +121,17 @@ Shared library targets can simply depend on the unified hub (e.g., `@pypi//numpy`), and the dependency will automatically resolve to the correct wheel version from the active hub during the build. +To migrate a hub that used to be named `pypi`, the root module can opt into a +`requirements.bzl` in the unified hub with +`pip.default(unified_hub_requirements_bzl = True)`. It has the per-package +macros of a concrete hub's (`requirement`, `whl_requirement`, `data_requirement` +and `dist_info_requirement`), so existing +`load("@pypi//:requirements.bzl", "requirement")` calls keep working after the +rename, and `requirement("numpy")` routes through the active hub like +`@pypi//numpy`. It has no `all_requirements` or other `all_*` lists: those are +fixed at loading time, before the active hub is known, so load them from a +concrete hub instead. Prefer `@pypi//` labels in new code. + ### Declaring Abstract Dependencies (pip.dep) :::{versionadded} 2.2.0 diff --git a/news/4223.added.md b/news/4223.added.md new file mode 100644 index 0000000000..de37e0e0e3 --- /dev/null +++ b/news/4223.added.md @@ -0,0 +1,9 @@ +(pypi) Added `pip.default(unified_hub_requirements_bzl = True)`, which makes the +unified `@pypi` hub generate a `requirements.bzl` with the per-package macros of +a concrete hub's (`requirement`, `whl_requirement`, `data_requirement` and +`dist_info_requirement`). A repo that renames its hub away from the reserved +`pypi` name keeps its `load("@pypi//:requirements.bzl", "requirement")` users +working, and the returned labels route through +{obj}`--@rules_python//python/config_settings:venv`. The `all_*` lists are not +provided, since they could only list one hub's packages; load them from a +concrete hub. diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index cafe245eb2..1b827806f2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -207,6 +207,7 @@ def build_config( A struct with the configuration. """ default_hub = None + unified_hub_requirements_bzl = False defaults = { "platforms": default_platforms(), "python_version": None, @@ -221,6 +222,8 @@ def build_config( if default_hub: fail("Duplicate pip.default tag: only one explicit default PyPI hub is allowed.") default_hub = tag.default_hub + if tag.unified_hub_requirements_bzl and mod.is_root: + unified_hub_requirements_bzl = True pyproject_toml = tag.pyproject_toml if pyproject_toml: pyproject = read_pyproject(module_ctx, pyproject_toml) @@ -270,6 +273,7 @@ def build_config( }, enable_pipstar_extract = enable_pipstar_extract, toml_decode = toml.decode, + unified_hub_requirements_bzl = unified_hub_requirements_bzl, ) def parse_modules( @@ -482,6 +486,7 @@ You cannot use both the additive_build_content and additive_build_content_file a facts = simpleapi_cache.get_facts(), hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, + unified_hub_requirements_bzl = config.unified_hub_requirements_bzl, whl_libraries = whl_libraries, whl_mods = whl_mods, platform_config_settings = { @@ -535,6 +540,7 @@ def _create_unified_hub_repo(mods): extra_aliases = extra_aliases, hubs = hubs, packages = packages, + requirements_bzl = mods.unified_hub_requirements_bzl, ) def _pip_impl(module_ctx): @@ -753,6 +759,26 @@ This is designed to work with dependency management tools like Renovate. :::{versionadded} 2.3.0 ::: +""", + ), + "unified_hub_requirements_bzl": attr.bool( + default = False, + doc = """\ +Generate a `requirements.bzl` in the [Unified @pypi Hub](unified-pypi-hub) with +the per-package macros of a concrete hub's: `requirement`, `whl_requirement`, +`data_requirement` and `dist_info_requirement`. The labels they return are in +the unified hub, so they follow {flag}`--venv` like `@pypi//`. + +This is a migration aid for a repository whose hub used to be named `pypi`: +after renaming it, its existing `load("@pypi//:requirements.bzl", "requirement")` +calls keep working and can move to `@pypi//` labels one at a time. The +`all_*` lists are not generated, since they are fixed at loading time and could +only list one hub's packages; load those from a concrete hub. + +Only the root module's value is used. + +:::{versionadded} VERSION_NEXT_FEATURE +::: """, ), "whl_abi_tags": attr.string_list( diff --git a/python/private/pypi/unified_hub_repo.bzl b/python/private/pypi/unified_hub_repo.bzl index f88db75f19..ddd15c0d86 100644 --- a/python/private/pypi/unified_hub_repo.bzl +++ b/python/private/pypi/unified_hub_repo.bzl @@ -7,7 +7,7 @@ _ROOT_BUILD_TMPL = """\ load("@rules_python//python/private/pypi:unified_hub_setup.bzl", "define_venv_flag_config_settings") package(default_visibility = ["//visibility:public"]) - +{exports_requirements_bzl} define_venv_flag_config_settings( name = "venv_config_settings", hubs = {hubs}, @@ -28,6 +28,35 @@ define_pypi_package_targets( ) """ +_EXPORTS_REQUIREMENTS_BZL = """ +# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it +exports_files(["requirements.bzl"]) +""" + +# The per-package macros of a concrete hub's requirements.bzl. The `all_*` +# lists are left out on purpose: they are fixed at loading time, before the +# venv flag is known, so they could only ever list one hub's packages. +_REQUIREMENTS_BZL_TMPL = '''\ +"""Starlark representation of the unified PyPI hub's packages. + +@generated by rules_python pip.parse bzlmod extension. +""" + +load("@rules_python//python:pip.bzl", "pip_utils") + +def requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg") + +def whl_requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl") + +def data_requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data") + +def dist_info_requirement(name): + return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info") +''' + def _unified_hub_repo_impl(rctx): hubs = rctx.attr.hubs default_hub = rctx.attr.default_hub or None @@ -35,7 +64,10 @@ def _unified_hub_repo_impl(rctx): # 1. Generate Root BUILD.bazel with shared config settings rctx.file( "BUILD.bazel", - _ROOT_BUILD_TMPL.format(hubs = hubs), + _ROOT_BUILD_TMPL.format( + exports_requirements_bzl = _EXPORTS_REQUIREMENTS_BZL if rctx.attr.requirements_bzl else "", + hubs = hubs, + ), ) # 2. Organize extra aliases by package @@ -60,6 +92,23 @@ def _unified_hub_repo_impl(rctx): ), ) + # 4. Optionally generate requirements.bzl with a concrete hub's per-package + # macros, so a hub renamed away from the reserved "pypi" name keeps its + # `load("@pypi//:requirements.bzl", "requirement")` users working. The + # macros return labels in this repo, so they route through the venv flag + # like any other `@pypi//` label. + # + # NOTE: we are using the canonical name with the double '@' in order to + # always uniquely identify a repository, as the labels are being passed as + # a string and the resolution of the label happens at the call-site of the + # `requirement`, et al. macros. + if rctx.attr.requirements_bzl: + macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name) + rctx.file( + "requirements.bzl", + _REQUIREMENTS_BZL_TMPL.replace("%%MACRO_TMPL%%", macro_tmpl), + ) + unified_hub_repo = repository_rule( implementation = _unified_hub_repo_impl, attrs = { @@ -77,11 +126,15 @@ unified_hub_repo = repository_rule( mandatory = True, doc = "Dictionary mapping package names to a list of hubs that contain them.", ), + "requirements_bzl": attr.bool( + default = False, + doc = "Whether to generate a requirements.bzl with the per-package `requirement` macros.", + ), }, doc = "Private repository rule creating the automatic Unified PyPI Hub.", ) -def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {}): +def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {}, requirements_bzl = False): """Creates a Unified PyPI Hub repository for WORKSPACE mode by loading requirements from hubs. Args: @@ -90,6 +143,8 @@ def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = { e.g. {"dev_pip": dev_pip_requirements, "pypi_alpha": pypi_alpha_requirements} default_hub: Optional default hub name. extra_aliases: Dictionary mapping 'package:alias' to a list of hubs that support it. + requirements_bzl: Whether to generate a requirements.bzl with the + per-package `requirement` macros. """ packages = {} for hub_name, req_map in hubs.items(): @@ -108,4 +163,5 @@ def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = { extra_aliases = extra_aliases, hubs = sorted(hubs.keys()), packages = packages, + requirements_bzl = requirements_bzl, ) diff --git a/tests/integration/unified_pypi/BUILD.bazel b/tests/integration/unified_pypi/BUILD.bazel index c0b9905fa6..c0aee2de5d 100644 --- a/tests/integration/unified_pypi/BUILD.bazel +++ b/tests/integration/unified_pypi/BUILD.bazel @@ -1,8 +1,28 @@ +load("@pypi//:requirements.bzl", "requirement") load("@rules_python//python:py_binary.bzl", "py_binary") load("@rules_python//python:py_test.bzl", "py_test") package(default_visibility = ["//visibility:public"]) +# `requirement()` from the unified hub normalizes the name and resolves +# through the venv flag like a plain `@pypi//` label. +py_test( + name = "test_requirement_default", + srcs = ["test_default.py"], + main = "test_default.py", + deps = [requirement("Colorama")], +) + +py_test( + name = "test_requirement_a", + srcs = ["test_a.py"], + config_settings = { + "@rules_python//python/config_settings:venv": "pypi_a", + }, + main = "test_a.py", + deps = [requirement("colorama")], +) + py_test( name = "test_default", srcs = ["test_default.py"], diff --git a/tests/integration/unified_pypi/MODULE.bazel b/tests/integration/unified_pypi/MODULE.bazel index 6a4b87e015..6dd2e25dd5 100644 --- a/tests/integration/unified_pypi/MODULE.bazel +++ b/tests/integration/unified_pypi/MODULE.bazel @@ -45,6 +45,7 @@ pip.parse( use_repo(pip, "pypi_b") pip.default(default_hub = "pypi_b") +pip.default(unified_hub_requirements_bzl = True) pip.dep( name = "declared-only-pkg", extra_targets = ["declared-only-alias"], diff --git a/tests/integration/unified_pypi_test.py b/tests/integration/unified_pypi_test.py index a19a9fcfbe..08f7b029c1 100644 --- a/tests/integration/unified_pypi_test.py +++ b/tests/integration/unified_pypi_test.py @@ -13,6 +13,31 @@ def test_default_fallback_hub(self): def test_transitioned_hub(self): self.run_bazel("test", "//:test_a") + def test_requirement_macro_default_hub(self): + self.run_bazel("test", "//:test_requirement_default") + + def test_requirement_macro_transitioned_hub(self): + self.run_bazel("test", "//:test_requirement_a") + + def test_requirements_bzl_requires_opt_in(self): + module_bazel = self.repo_root / "MODULE.bazel" + without_opt_in = module_bazel.read_text().replace( + "pip.default(unified_hub_requirements_bzl = True)\n", + "", + ) + with self._temp_modify_file(module_bazel, without_opt_in): + result = self.run_bazel( + "cquery", "//:test_requirement_default", check=False + ) + self.assertNotEqual( + result.exit_code, + 0, + "Expected loading @pypi//:requirements.bzl to fail without the opt-in", + ) + self.assert_result_matches( + result, r"cannot load '[^']*//:requirements\.bzl': no such file" + ) + def test_cli_override(self): self.run_bazel( "run", diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl index 97849e882d..6d7e971132 100644 --- a/tests/pypi/extension/extension_tests.bzl +++ b/tests/pypi/extension/extension_tests.bzl @@ -52,6 +52,7 @@ def _default( os_name = None, platform = None, pyproject_toml = None, + unified_hub_requirements_bzl = False, whl_platform_tags = None, whl_abi_tags = None): return struct( @@ -66,6 +67,7 @@ def _default( os_name = os_name, platform = platform, pyproject_toml = pyproject_toml, + unified_hub_requirements_bzl = unified_hub_requirements_bzl, whl_abi_tags = whl_abi_tags or [], whl_platform_tags = whl_platform_tags or [], ) @@ -120,6 +122,7 @@ def _parse_modules(env, **kwargs): exposed_packages = subjects.dict, hub_group_map = subjects.dict, hub_whl_map = subjects.dict, + unified_hub_requirements_bzl = subjects.bool, whl_libraries = subjects.dict, whl_mods = subjects.dict, ), @@ -497,6 +500,66 @@ def _test_default_hub_precedence(env): _tests.append(_test_default_hub_precedence) +def _parse_unified_hub_requirements_bzl(env, *modules): + return _parse_modules( + env, + module_ctx = _pypi_mock_mctx( + os_name = "linux", + arch_name = "x86_64", + *modules + ), + available_interpreters = { + "python_3_15_host": "unit_test_interpreter_target", + }, + minor_mapping = {"3.15": "3.15.19"}, + ) + +def _unified_hub_requirements_bzl_mod(name, is_root, unified_hub_requirements_bzl): + return _mod( + name = name, + is_root = is_root, + default = _default_tags_default + [ + _default(unified_hub_requirements_bzl = unified_hub_requirements_bzl), + ], + parse = [ + _parse( + hub_name = name + "_pypi", + python_version = "3.15", + simpleapi_skip = ["simple"], + requirements_lock = "requirements.txt", + ), + ], + ) + +def _test_unified_hub_requirements_bzl_off_by_default(env): + pypi = _parse_unified_hub_requirements_bzl( + env, + _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = False), + ) + pypi.unified_hub_requirements_bzl().equals(False) + +_tests.append(_test_unified_hub_requirements_bzl_off_by_default) + +def _test_unified_hub_requirements_bzl_root_opt_in(env): + pypi = _parse_unified_hub_requirements_bzl( + env, + _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = True), + ) + pypi.unified_hub_requirements_bzl().equals(True) + +_tests.append(_test_unified_hub_requirements_bzl_root_opt_in) + +def _test_unified_hub_requirements_bzl_ignored_outside_root(env): + """Only the root module decides whether the unified hub gets a requirements.bzl.""" + pypi = _parse_unified_hub_requirements_bzl( + env, + _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = False), + _unified_hub_requirements_bzl_mod("rules_python", is_root = False, unified_hub_requirements_bzl = True), + ) + pypi.unified_hub_requirements_bzl().equals(False) + +_tests.append(_test_unified_hub_requirements_bzl_ignored_outside_root) + def _test_extension_dep(env): pypi = _parse_modules( env,