feat(pypi): generate requirements.bzl in the unified @pypi hub

docs/pypi/download.md

@@ -121,6 +121,17 @@ Shared library targets can simply depend on the unified hub (e.g.,
 `@pypi//numpy`), and the dependency will automatically resolve to the correct
 wheel version from the active hub during the build.
 
+To migrate a hub that used to be named `pypi`, the root module can opt into a
+`requirements.bzl` in the unified hub with
+`pip.default(unified_hub_requirements_bzl = True)`. It has the per-package
+macros of a concrete hub's (`requirement`, `whl_requirement`, `data_requirement`
+and `dist_info_requirement`), so existing
+`load("@pypi//:requirements.bzl", "requirement")` calls keep working after the
+rename, and `requirement("numpy")` routes through the active hub like
+`@pypi//numpy`. It has no `all_requirements` or other `all_*` lists: those are
+fixed at loading time, before the active hub is known, so load them from a
+concrete hub instead. Prefer `@pypi//<pkg>` labels in new code.
+
 ### Declaring Abstract Dependencies (pip.dep)
 
 :::{versionadded} 2.2.0

news/4223.added.md

@@ -0,0 +1,9 @@
+(pypi) Added `pip.default(unified_hub_requirements_bzl = True)`, which makes the
+unified `@pypi` hub generate a `requirements.bzl` with the per-package macros of
+a concrete hub's (`requirement`, `whl_requirement`, `data_requirement` and
+`dist_info_requirement`). A repo that renames its hub away from the reserved
+`pypi` name keeps its `load("@pypi//:requirements.bzl", "requirement")` users
+working, and the returned labels route through
+{obj}`--@rules_python//python/config_settings:venv`. The `all_*` lists are not
+provided, since they could only list one hub's packages; load them from a
+concrete hub.

python/private/pypi/extension.bzl

@@ -207,6 +207,7 @@ def build_config(
         A struct with the configuration.
     """
     default_hub = None
+    unified_hub_requirements_bzl = False
     defaults = {
         "platforms": default_platforms(),
         "python_version": None,
@@ -221,6 +222,8 @@ def build_config(
                     if default_hub:
                         fail("Duplicate pip.default tag: only one explicit default PyPI hub is allowed.")
                     default_hub = tag.default_hub
+            if tag.unified_hub_requirements_bzl and mod.is_root:
+                unified_hub_requirements_bzl = True
             pyproject_toml = tag.pyproject_toml
             if pyproject_toml:
                 pyproject = read_pyproject(module_ctx, pyproject_toml)
@@ -270,6 +273,7 @@ def build_config(
         },
         enable_pipstar_extract = enable_pipstar_extract,
         toml_decode = toml.decode,
+        unified_hub_requirements_bzl = unified_hub_requirements_bzl,
     )
 
 def parse_modules(
@@ -482,6 +486,7 @@ You cannot use both the additive_build_content and additive_build_content_file a
         facts = simpleapi_cache.get_facts(),
         hub_group_map = hub_group_map,
         hub_whl_map = hub_whl_map,
+        unified_hub_requirements_bzl = config.unified_hub_requirements_bzl,
         whl_libraries = whl_libraries,
         whl_mods = whl_mods,
         platform_config_settings = {
@@ -535,6 +540,7 @@ def _create_unified_hub_repo(mods):
         extra_aliases = extra_aliases,
         hubs = hubs,
         packages = packages,
+        requirements_bzl = mods.unified_hub_requirements_bzl,
     )
 
 def _pip_impl(module_ctx):
@@ -753,6 +759,26 @@ This is designed to work with dependency management tools like Renovate.
 
 :::{versionadded} 2.3.0
 :::
+""",
+    ),
+    "unified_hub_requirements_bzl": attr.bool(
+        default = False,
+        doc = """\
+Generate a `requirements.bzl` in the [Unified @pypi Hub](unified-pypi-hub) with
+the per-package macros of a concrete hub's: `requirement`, `whl_requirement`,
+`data_requirement` and `dist_info_requirement`. The labels they return are in
+the unified hub, so they follow {flag}`--venv` like `@pypi//<pkg>`.
+
+This is a migration aid for a repository whose hub used to be named `pypi`:
+after renaming it, its existing `load("@pypi//:requirements.bzl", "requirement")`
+calls keep working and can move to `@pypi//<pkg>` labels one at a time. The
+`all_*` lists are not generated, since they are fixed at loading time and could
+only list one hub's packages; load those from a concrete hub.
+
+Only the root module's value is used.
+
+:::{versionadded} VERSION_NEXT_FEATURE
+:::
 """,
     ),
     "whl_abi_tags": attr.string_list(

python/private/pypi/unified_hub_repo.bzl

@@ -7,7 +7,7 @@ _ROOT_BUILD_TMPL = """\
 load("@rules_python//python/private/pypi:unified_hub_setup.bzl", "define_venv_flag_config_settings")
 
 package(default_visibility = ["//visibility:public"])
-
+{exports_requirements_bzl}
 define_venv_flag_config_settings(
     name = "venv_config_settings",
     hubs = {hubs},
@@ -28,14 +28,46 @@ define_pypi_package_targets(
 )
 """
 
+_EXPORTS_REQUIREMENTS_BZL = """
+# Ensure the `requirements.bzl` source can be accessed by stardoc, since users load() from it
+exports_files(["requirements.bzl"])
+"""
+
+# The per-package macros of a concrete hub's requirements.bzl. The `all_*`
+# lists are left out on purpose: they are fixed at loading time, before the
+# venv flag is known, so they could only ever list one hub's packages.
+_REQUIREMENTS_BZL_TMPL = '''\
+"""Starlark representation of the unified PyPI hub's packages.
+
+@generated by rules_python pip.parse bzlmod extension.
+"""
+
+load("@rules_python//python:pip.bzl", "pip_utils")
+
+def requirement(name):
+    return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "pkg")
+
+def whl_requirement(name):
+    return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "whl")
+
+def data_requirement(name):
+    return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "data")
+
+def dist_info_requirement(name):
+    return "%%MACRO_TMPL%%".format(pip_utils.normalize_name(name), "dist_info")
+'''
+
 def _unified_hub_repo_impl(rctx):
     hubs = rctx.attr.hubs
     default_hub = rctx.attr.default_hub or None
 
     # 1. Generate Root BUILD.bazel with shared config settings
     rctx.file(
         "BUILD.bazel",
-        _ROOT_BUILD_TMPL.format(hubs = hubs),
+        _ROOT_BUILD_TMPL.format(
+            exports_requirements_bzl = _EXPORTS_REQUIREMENTS_BZL if rctx.attr.requirements_bzl else "",
+            hubs = hubs,
+        ),
     )
 
     # 2. Organize extra aliases by package
@@ -60,6 +92,23 @@ def _unified_hub_repo_impl(rctx):
             ),
         )
 
+    # 4. Optionally generate requirements.bzl with a concrete hub's per-package
+    # macros, so a hub renamed away from the reserved "pypi" name keeps its
+    # `load("@pypi//:requirements.bzl", "requirement")` users working. The
+    # macros return labels in this repo, so they route through the venv flag
+    # like any other `@pypi//<pkg>` label.
+    #
+    # NOTE: we are using the canonical name with the double '@' in order to
+    # always uniquely identify a repository, as the labels are being passed as
+    # a string and the resolution of the label happens at the call-site of the
+    # `requirement`, et al. macros.
+    if rctx.attr.requirements_bzl:
+        macro_tmpl = "@@{name}//{{}}:{{}}".format(name = rctx.attr.name)
+        rctx.file(
+            "requirements.bzl",
+            _REQUIREMENTS_BZL_TMPL.replace("%%MACRO_TMPL%%", macro_tmpl),
+        )
+
 unified_hub_repo = repository_rule(
     implementation = _unified_hub_repo_impl,
     attrs = {
@@ -77,11 +126,15 @@ unified_hub_repo = repository_rule(
             mandatory = True,
             doc = "Dictionary mapping package names to a list of hubs that contain them.",
         ),
+        "requirements_bzl": attr.bool(
+            default = False,
+            doc = "Whether to generate a requirements.bzl with the per-package `requirement` macros.",
+        ),
     },
     doc = "Private repository rule creating the automatic Unified PyPI Hub.",
 )
 
-def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {}):
+def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {}, requirements_bzl = False):
     """Creates a Unified PyPI Hub repository for WORKSPACE mode by loading requirements from hubs.
 
     Args:
@@ -90,6 +143,8 @@ def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {
               e.g. {"dev_pip": dev_pip_requirements, "pypi_alpha": pypi_alpha_requirements}
         default_hub: Optional default hub name.
         extra_aliases: Dictionary mapping 'package:alias' to a list of hubs that support it.
+        requirements_bzl: Whether to generate a requirements.bzl with the
+            per-package `requirement` macros.
     """
     packages = {}
     for hub_name, req_map in hubs.items():
@@ -108,4 +163,5 @@ def unified_workspace_hub_repo(name, hubs, default_hub = None, extra_aliases = {
         extra_aliases = extra_aliases,
         hubs = sorted(hubs.keys()),
         packages = packages,
+        requirements_bzl = requirements_bzl,
     )

tests/integration/unified_pypi/BUILD.bazel

@@ -1,8 +1,28 @@
+load("@pypi//:requirements.bzl", "requirement")
 load("@rules_python//python:py_binary.bzl", "py_binary")
 load("@rules_python//python:py_test.bzl", "py_test")
 
 package(default_visibility = ["//visibility:public"])
 
+# `requirement()` from the unified hub normalizes the name and resolves
+# through the venv flag like a plain `@pypi//<pkg>` label.
+py_test(
+    name = "test_requirement_default",
+    srcs = ["test_default.py"],
+    main = "test_default.py",
+    deps = [requirement("Colorama")],
+)
+
+py_test(
+    name = "test_requirement_a",
+    srcs = ["test_a.py"],
+    config_settings = {
+        "@rules_python//python/config_settings:venv": "pypi_a",
+    },
+    main = "test_a.py",
+    deps = [requirement("colorama")],
+)
+
 py_test(
     name = "test_default",
     srcs = ["test_default.py"],

tests/integration/unified_pypi/MODULE.bazel

@@ -45,6 +45,7 @@ pip.parse(
 use_repo(pip, "pypi_b")
 
 pip.default(default_hub = "pypi_b")
+pip.default(unified_hub_requirements_bzl = True)
 pip.dep(
     name = "declared-only-pkg",
     extra_targets = ["declared-only-alias"],

tests/integration/unified_pypi_test.py

@@ -13,6 +13,31 @@ def test_default_fallback_hub(self):
     def test_transitioned_hub(self):
         self.run_bazel("test", "//:test_a")
 
+    def test_requirement_macro_default_hub(self):
+        self.run_bazel("test", "//:test_requirement_default")
+
+    def test_requirement_macro_transitioned_hub(self):
+        self.run_bazel("test", "//:test_requirement_a")
+
+    def test_requirements_bzl_requires_opt_in(self):
+        module_bazel = self.repo_root / "MODULE.bazel"
+        without_opt_in = module_bazel.read_text().replace(
+            "pip.default(unified_hub_requirements_bzl = True)\n",
+            "",
+        )
+        with self._temp_modify_file(module_bazel, without_opt_in):
+            result = self.run_bazel(
+                "cquery", "//:test_requirement_default", check=False
+            )
+            self.assertNotEqual(
+                result.exit_code,
+                0,
+                "Expected loading @pypi//:requirements.bzl to fail without the opt-in",
+            )
+            self.assert_result_matches(
+                result, r"cannot load '[^']*//:requirements\.bzl': no such file"
+            )
+
     def test_cli_override(self):
         self.run_bazel(
             "run",

tests/pypi/extension/extension_tests.bzl

@@ -52,6 +52,7 @@ def _default(
         os_name = None,
         platform = None,
         pyproject_toml = None,
+        unified_hub_requirements_bzl = False,
         whl_platform_tags = None,
         whl_abi_tags = None):
     return struct(
@@ -66,6 +67,7 @@ def _default(
         os_name = os_name,
         platform = platform,
         pyproject_toml = pyproject_toml,
+        unified_hub_requirements_bzl = unified_hub_requirements_bzl,
         whl_abi_tags = whl_abi_tags or [],
         whl_platform_tags = whl_platform_tags or [],
     )
@@ -120,6 +122,7 @@ def _parse_modules(env, **kwargs):
             exposed_packages = subjects.dict,
             hub_group_map = subjects.dict,
             hub_whl_map = subjects.dict,
+            unified_hub_requirements_bzl = subjects.bool,
             whl_libraries = subjects.dict,
             whl_mods = subjects.dict,
         ),
@@ -497,6 +500,66 @@ def _test_default_hub_precedence(env):
 
 _tests.append(_test_default_hub_precedence)
 
+def _parse_unified_hub_requirements_bzl(env, *modules):
+    return _parse_modules(
+        env,
+        module_ctx = _pypi_mock_mctx(
+            os_name = "linux",
+            arch_name = "x86_64",
+            *modules
+        ),
+        available_interpreters = {
+            "python_3_15_host": "unit_test_interpreter_target",
+        },
+        minor_mapping = {"3.15": "3.15.19"},
+    )
+
+def _unified_hub_requirements_bzl_mod(name, is_root, unified_hub_requirements_bzl):
+    return _mod(
+        name = name,
+        is_root = is_root,
+        default = _default_tags_default + [
+            _default(unified_hub_requirements_bzl = unified_hub_requirements_bzl),
+        ],
+        parse = [
+            _parse(
+                hub_name = name + "_pypi",
+                python_version = "3.15",
+                simpleapi_skip = ["simple"],
+                requirements_lock = "requirements.txt",
+            ),
+        ],
+    )
+
+def _test_unified_hub_requirements_bzl_off_by_default(env):
+    pypi = _parse_unified_hub_requirements_bzl(
+        env,
+        _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = False),
+    )
+    pypi.unified_hub_requirements_bzl().equals(False)
+
+_tests.append(_test_unified_hub_requirements_bzl_off_by_default)
+
+def _test_unified_hub_requirements_bzl_root_opt_in(env):
+    pypi = _parse_unified_hub_requirements_bzl(
+        env,
+        _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = True),
+    )
+    pypi.unified_hub_requirements_bzl().equals(True)
+
+_tests.append(_test_unified_hub_requirements_bzl_root_opt_in)
+
+def _test_unified_hub_requirements_bzl_ignored_outside_root(env):
+    """Only the root module decides whether the unified hub gets a requirements.bzl."""
+    pypi = _parse_unified_hub_requirements_bzl(
+        env,
+        _unified_hub_requirements_bzl_mod("root", is_root = True, unified_hub_requirements_bzl = False),
+        _unified_hub_requirements_bzl_mod("rules_python", is_root = False, unified_hub_requirements_bzl = True),
+    )
+    pypi.unified_hub_requirements_bzl().equals(False)
+
+_tests.append(_test_unified_hub_requirements_bzl_ignored_outside_root)
+
 def _test_extension_dep(env):
     pypi = _parse_modules(
         env,