From 69a356c22ccecff1f949bd9ff2177d04c78370bd Mon Sep 17 00:00:00 2001 From: Ruben Arts Date: Mon, 26 Oct 2026 15:01:27 +0100 Subject: [PATCH] Skip already built recipes by exact build string Skipping by package name hides missing variants: once a package is published for python 3.14, its 3.12 build was never generated, and the builds of its dependents failed to solve. With -m, vinca now renders the written recipes with rattler-build and removes only those whose every output (name, version, build string) is published; the build string carries the variant hash and the build number. rattler-build --skip-existing=all then skips the published variants of the rest. --- pixi.lock | 99 ++++++++++++++++++++--- pixi.toml | 1 + vinca/main.py | 20 +++++ vinca/skip_existing.py | 94 +++++++++++++++++++++ vinca/test_skip_existing_builds.py | 126 +++++++++++++++++++++++++++++ 5 files changed, 327 insertions(+), 13 deletions(-) create mode 100644 vinca/skip_existing.py create mode 100644 vinca/test_skip_existing_builds.py diff --git a/pixi.lock b/pixi.lock index 0d6ab27..7a51044 100644 --- a/pixi.lock +++ b/pixi.lock @@ -44,10 +44,12 @@ environments: - conda: https://prefix.dev/conda-forge/linux-64/libzlib-1.3.1-hb9d3cd8_2.conda - conda: https://prefix.dev/conda-forge/linux-64/markupsafe-3.0.3-py314h67df5f8_1.conda - conda: https://prefix.dev/conda-forge/linux-64/ncurses-6.5-h2d0b736_3.conda - - conda: https://prefix.dev/conda-forge/linux-64/openssl-3.5.4-h26f9b46_0.conda + - conda: https://prefix.dev/conda-forge/linux-64/openssl-3.6.5-h781a0a9_0.conda + - conda: https://prefix.dev/conda-forge/linux-64/patchelf-0.19.2-hee9eb32_0.conda - conda: https://prefix.dev/conda-forge/linux-64/pyrefly-1.2.0-hb17b654_0.conda - conda: https://prefix.dev/conda-forge/linux-64/python-3.14.0-h32b2ec7_102_cp314.conda - conda: https://prefix.dev/conda-forge/linux-64/pyyaml-6.0.3-py314h67df5f8_1.conda + - conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.77.0-hf01adef_1.conda - conda: https://prefix.dev/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda - conda: https://prefix.dev/conda-forge/linux-64/ruamel.yaml-0.17.17-py314h5bd0f2a_4.conda - conda: https://prefix.dev/conda-forge/linux-64/ruamel.yaml.clib-0.2.15-py314hfe1a184_3.conda @@ -154,6 +156,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-64/pyrefly-1.2.0-h19f9e61_0.conda - conda: https://prefix.dev/conda-forge/osx-64/python-3.14.0-hf88997e_102_cp314.conda - conda: https://prefix.dev/conda-forge/osx-64/pyyaml-6.0.3-py314h10d0514_1.conda + - conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.77.0-h9a0f642_1.conda - conda: https://prefix.dev/conda-forge/osx-64/readline-8.2-h7cca4af_2.conda - conda: https://prefix.dev/conda-forge/osx-64/ruamel.yaml-0.17.17-py314h6482030_4.conda - conda: https://prefix.dev/conda-forge/osx-64/ruamel.yaml.clib-0.2.15-py314h17af80e_3.conda @@ -221,6 +224,7 @@ environments: - conda: https://prefix.dev/conda-forge/osx-arm64/pyrefly-1.2.0-h6fdd925_0.conda - conda: https://prefix.dev/conda-forge/osx-arm64/python-3.14.0-h40d2674_102_cp314.conda - conda: https://prefix.dev/conda-forge/osx-arm64/pyyaml-6.0.3-py314h6e9b3f0_1.conda + - conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.77.0-h2ace698_1.conda - conda: https://prefix.dev/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda - conda: https://prefix.dev/conda-forge/osx-arm64/ruamel.yaml-0.17.17-py314h0612a62_4.conda - conda: https://prefix.dev/conda-forge/osx-arm64/ruamel.yaml.clib-0.2.15-py314hd98292b_3.conda @@ -286,6 +290,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/pyrefly-1.2.0-h18a1a76_0.conda - conda: https://prefix.dev/conda-forge/win-64/python-3.14.0-h4b44e0e_102_cp314.conda - conda: https://prefix.dev/conda-forge/win-64/pyyaml-6.0.3-py314h2359020_1.conda + - conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.77.0-he94b42d_1.conda - conda: https://prefix.dev/conda-forge/win-64/ruamel.yaml-0.17.17-py314h5a2d7ad_4.conda - conda: https://prefix.dev/conda-forge/win-64/ruamel.yaml.clib-0.2.15-py314hc5dbbe4_3.conda - conda: https://prefix.dev/conda-forge/win-64/ruff-0.14.2-h3e3edff_0.conda @@ -730,18 +735,6 @@ packages: - ncurses >=6.6,<7.0a0 size: 911196 timestamp: 1786355078102 -- conda: https://prefix.dev/conda-forge/linux-64/openssl-3.5.4-h26f9b46_0.conda - sha256: e807f3bad09bdf4075dbb4168619e14b0c0360bacb2e12ef18641a834c8c5549 - md5: 14edad12b59ccbfa3910d42c72adc2a0 - depends: - - __glibc >=2.17,<3.0.a0 - - ca-certificates - - libgcc >=14 - license: Apache-2.0 - license_family: Apache - purls: [] - size: 3119624 - timestamp: 1759324353651 - conda: https://prefix.dev/conda-forge/linux-64/openssl-3.6.4-h781a0a9_0.conda sha256: 4747b2d6a8336f52343bceb8a1ebf41a9e6e665b9d2e3f989972de53a310599e md5: 16e3034a330cc625f2c685edec60cf4e @@ -756,6 +749,32 @@ packages: - openssl >=3.6.4,<4.0a0 size: 3202955 timestamp: 1787698780103 +- conda: https://prefix.dev/conda-forge/linux-64/openssl-3.6.5-h781a0a9_0.conda + sha256: 9c1533f1cf784f0e250919c0c9762d0b0e6bc727a79290ffb7cea1ceca1d038c + md5: 5a51a2ea653f120240b1a9bf76e0aae9 + depends: + - __glibc >=2.17,<3.0.a0 + - ca-certificates + - libgcc >=15 + license: Apache-2.0 + license_family: Apache + run_exports: + weak: + - openssl >=3.6.5,<4.0a0 + size: 3195299 + timestamp: 1791098052301 +- conda: https://prefix.dev/conda-forge/linux-64/patchelf-0.19.2-hee9eb32_0.conda + sha256: 74ba13a7cd793cae441317d1963f824b1c45f49adde34f9801b3c766986592c5 + md5: 0fd1c32796736ad1b43e1445cad9a2fb + depends: + - libstdcxx >=15 + - libgcc >=15 + - __glibc >=2.17,<3.0.a0 + license: GPL-3.0-or-later + license_family: GPL + run_exports: {} + size: 152126 + timestamp: 1790866799280 - conda: https://prefix.dev/conda-forge/linux-64/pyrefly-1.2.0-hb17b654_0.conda sha256: 937a57a855f4a7f0994a561fbee5a089d509faf9b0668ae2fd02a71dc606dc4f md5: 2c8a7d2685e9b1aff86433c47a883431 @@ -844,6 +863,22 @@ packages: run_exports: {} size: 202391 timestamp: 1770223462836 +- conda: https://prefix.dev/conda-forge/linux-64/rattler-build-0.77.0-hf01adef_1.conda + sha256: 7bcadeeda659a37783003d43a157252c23826d6e81e1df9c0777cb61f9996f39 + md5: 1b82a8513a170c17d1eaea225ea79420 + depends: + - patchelf + - libstdcxx >=15 + - libgcc >=15 + - __glibc >=2.17,<3.0.a0 + - openssl >=3.5.9,<4.0a0 + constrains: + - __glibc >=2.17 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 21828116 + timestamp: 1792604777652 - conda: https://prefix.dev/conda-forge/linux-64/readline-8.2-h8c095d6_2.conda sha256: 2d6d0c026902561ed77cd646b5021aef2d4db22e57a5b0178dfc669231e06d2c md5: 283b96675859b20a825f8fa30f311446 @@ -1969,6 +2004,19 @@ packages: run_exports: {} size: 191947 timestamp: 1770226344240 +- conda: https://prefix.dev/conda-forge/osx-64/rattler-build-0.77.0-h9a0f642_1.conda + sha256: 73e2612701c4d2ef4d77cfde2c322e45511a0b6039d68bcb256ccd1f21891c34 + md5: 754bcf356018f9f1b16a96376cfc5336 + depends: + - __osx >=11.0 + - libcxx >=21 + constrains: + - __osx >=11.0 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 21461377 + timestamp: 1792604950742 - conda: https://prefix.dev/conda-forge/osx-64/readline-8.2-h7cca4af_2.conda sha256: 53017e80453c4c1d97aaf78369040418dea14cf8f46a2fa999f31bd70b36c877 md5: 342570f8e02f2f022147a7f841475784 @@ -2492,6 +2540,19 @@ packages: run_exports: {} size: 189475 timestamp: 1770223788648 +- conda: https://prefix.dev/conda-forge/osx-arm64/rattler-build-0.77.0-h2ace698_1.conda + sha256: 1905e563b8ea0ed5d750a1aa879c42def85b7a4ce4bfa0d131e9dbb1688b8029 + md5: f83ca3594af92932486aa219ff2d1635 + depends: + - libcxx >=21 + - __osx >=11.0 + constrains: + - __osx >=11.0 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 19719448 + timestamp: 1792604807546 - conda: https://prefix.dev/conda-forge/osx-arm64/readline-8.2-h1d1bf99_2.conda sha256: 7db04684d3904f6151eff8673270922d31da1eea7fa73254d01c437f49702e34 md5: 63ef3f6e6d6d5c589e64f11263dc5676 @@ -2996,6 +3057,18 @@ packages: run_exports: {} size: 181257 timestamp: 1770223460931 +- conda: https://prefix.dev/conda-forge/win-64/rattler-build-0.77.0-he94b42d_1.conda + sha256: 5e62743cf0648a451a604b8097862e41791fa10da90ef4ed46198ef2559ff4b3 + md5: a33b0a9d59da14acd79d2264b9f67f49 + depends: + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + license: BSD-3-Clause + license_family: BSD + run_exports: {} + size: 22565961 + timestamp: 1792604776870 - conda: https://prefix.dev/conda-forge/win-64/ruamel.yaml-0.17.17-py314h5a2d7ad_4.conda sha256: f313c788da41d77b15acd3c096a7c8763de45a93d5e6f47dcc844ac456639063 md5: 9edc657959c912dd5f74d2dd24d801ad diff --git a/pixi.toml b/pixi.toml index 786ce49..67527e9 100644 --- a/pixi.toml +++ b/pixi.toml @@ -17,6 +17,7 @@ pyrefly = "*" pytest = "*" ruff = "*" vinca.path = "." +rattler-build = ">=0.77.0,<0.78" [package.build] backend.name = "pixi-build-python" diff --git a/vinca/main.py b/vinca/main.py index 308113c..a871b0b 100644 --- a/vinca/main.py +++ b/vinca/main.py @@ -6,6 +6,7 @@ import glob import os import sys +from pathlib import Path from typing import Any, Optional import catkin_pkg @@ -38,6 +39,7 @@ from .recipes import generate_output as build_output from .recipes import get_depmods as get_depmods from .resolve import resolve_pkgname +from .skip_existing import published_builds, remove_built_recipes from .sources import ( generate_fat_source as build_fat_source, ) @@ -709,8 +711,12 @@ def main(): write_recipe(sources, outputs, vinca_conf, distro) else: + # (name, version, build string) of the published builds; None: skip nothing + published = None if arguments.skip_already_built_repodata or vinca_conf.get("skip_existing"): + published = set() skip_built_packages = set() + vinca_conf["skip_built_packages"] = skip_built_packages fn = arguments.skip_already_built_repodata if not fn: fn = vinca_conf.get("skip_existing") @@ -745,6 +751,10 @@ def main(): print(f"Fetching repodata: {fn}") repodata = get_repodata(fn, get_conda_subdir()) + if arguments.multiple_file: + # compared by exact build string once the recipes are written + published |= published_builds([repodata], vinca_conf["ros_distro"]) + continue # currently we don't check the build numbers of local repodatas, # only URLs if "://" in fn: @@ -801,6 +811,16 @@ def main(): if arguments.multiple_file: write_recipe(source, outputs, vinca_conf, distro, False) + if published is not None: + variant_config = Path(base_dir) / "conda_build_config.yaml" + removed = remove_built_recipes( + Path("recipes").absolute(), + get_conda_subdir(), + variant_config if variant_config.is_file() else None, + published, + ) + print(f"Skipping {len(removed)} recipes whose builds are all published") + outputs = [o for o in outputs if o["package"]["name"] not in removed] else: write_recipe(source, outputs, vinca_conf, distro) diff --git a/vinca/skip_existing.py b/vinca/skip_existing.py new file mode 100644 index 0000000..3588d57 --- /dev/null +++ b/vinca/skip_existing.py @@ -0,0 +1,94 @@ +"""Skip recipes whose exact builds are already published. + +A package is only already built when a build with the same name, version and build +string is on a channel: the build string carries the variant hash (e.g. ``py312`` / +``py314``) and the build number, so a new python version or a bumped build number is +built even though the package name already exists. vinca renders the generated +recipes with rattler-build to get those build strings and removes the recipes whose +every output is published; rattler-build (``--skip-existing=all``) then skips the +published variants of the recipes that remain. +""" + +from __future__ import annotations + +import json +import shutil +import subprocess +from collections.abc import Iterable, Mapping +from pathlib import Path +from typing import Any, Optional + +from vinca.utils import built_for_distro + +Build = tuple[str, str, str] + + +def published_builds( + repodatas: Iterable[Mapping[str, Any]], ros_distro: str +) -> set[Build]: + """(name, version, build string) of every record built for *ros_distro*.""" + builds = set() + for repodata in repodatas: + records = {**repodata.get("packages", {}), **repodata.get("packages.conda", {})} + for record in records.values(): + if built_for_distro(record, ros_distro): + builds.add((record["name"], record["version"], record["build"])) + return builds + + +def render_outputs( + recipes_dir: Path, target_platform: str, variant_config: Optional[Path] +) -> dict[str, list[Build]]: + """Render every recipe in *recipes_dir*, grouped by package name (the name of + the recipe's directory, see template.write_recipe).""" + cmd = [ + "rattler-build", + "build", + "--render-only", + "--recipe-dir", + str(recipes_dir), + "--target-platform", + target_platform, + ] + if variant_config is not None: + cmd += ["-m", str(variant_config)] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError( + f"rendering the recipes in {recipes_dir} failed:\n{proc.stderr}" + ) + rendered: dict[str, list[Build]] = {} + for output in json.loads(proc.stdout): + package = output["recipe"]["package"] + build = ( + package["name"], + package["version"], + output["recipe"]["build"]["string"], + ) + rendered.setdefault(package["name"], []).append(build) + return rendered + + +def built_recipes( + rendered: Mapping[str, list[Build]], published: set[Build] +) -> set[str]: + """Recipes whose every rendered output is published.""" + return { + name + for name, builds in rendered.items() + if builds and all(build in published for build in builds) + } + + +def remove_built_recipes( + recipes_dir: Path, + target_platform: str, + variant_config: Optional[Path], + published: set[Build], +) -> set[str]: + """Delete the recipe directories that have nothing left to build; return their names.""" + rendered = render_outputs(recipes_dir, target_platform, variant_config) + removed = built_recipes(rendered, published) + for name in removed: + shutil.rmtree(recipes_dir / name) + return removed diff --git a/vinca/test_skip_existing_builds.py b/vinca/test_skip_existing_builds.py new file mode 100644 index 0000000..f3c9c35 --- /dev/null +++ b/vinca/test_skip_existing_builds.py @@ -0,0 +1,126 @@ +"""Tests for skipping recipes whose exact builds are already published.""" + +import shutil +from pathlib import Path + +import pytest + +from vinca import skip_existing +from vinca.skip_existing import ( + built_recipes, + published_builds, + remove_built_recipes, + render_outputs, +) + +LYRICAL_MUTEX = "ros2-distro-mutex 0.22.* lyrical_*" +ROLLING_MUTEX = "ros2-distro-mutex 0.22.* rolling_*" + + +def _record(name, version, build, mutex=LYRICAL_MUTEX): + return {"name": name, "version": version, "build": build, "depends": [mutex]} + + +def test_published_builds_are_name_version_and_build_string(): + repodata = { + "packages": {"a.tar.bz2": _record("ros2-rcl", "10.2.0", "hb2aaf94_30")}, + "packages.conda": { + "b.conda": _record("ros2-rclpy", "10.0.1", "np2py314h40712d6_30") + }, + } + + assert published_builds([repodata], "lyrical") == { + ("ros2-rcl", "10.2.0", "hb2aaf94_30"), + ("ros2-rclpy", "10.0.1", "np2py314h40712d6_30"), + } + + +def test_published_builds_leave_out_other_distributions(): + repodata = { + "packages.conda": { + "a.conda": _record("ros2-rcl", "10.2.0", "h1_30", ROLLING_MUTEX) + } + } + + assert published_builds([repodata], "lyrical") == set() + + +def test_recipe_with_every_output_published_is_built(): + rendered = {"ros2-rcl": [("ros2-rcl", "10.2.0", "hb2aaf94_30")]} + published = {("ros2-rcl", "10.2.0", "hb2aaf94_30")} + + assert built_recipes(rendered, published) == {"ros2-rcl"} + + +def test_recipe_with_a_missing_python_variant_is_kept(): + rendered = { + "ros2-rclpy": [ + ("ros2-rclpy", "10.0.1", "np2py312hfd3a116_30"), + ("ros2-rclpy", "10.0.1", "np2py314h40712d6_30"), + ] + } + published = {("ros2-rclpy", "10.0.1", "np2py314h40712d6_30")} + + assert built_recipes(rendered, published) == set() + + +def test_recipe_with_a_new_build_number_is_kept(): + rendered = {"ros2-rcl": [("ros2-rcl", "10.2.0", "hb2aaf94_31")]} + published = {("ros2-rcl", "10.2.0", "hb2aaf94_30")} + + assert built_recipes(rendered, published) == set() + + +def test_recipe_without_rendered_outputs_is_kept(): + assert built_recipes({"ros2-rcl": []}, set()) == set() + + +def test_remove_built_recipes_deletes_only_complete_ones(tmp_path, monkeypatch): + for name in ("ros2-rcl", "ros2-rclpy"): + (tmp_path / name).mkdir() + (tmp_path / name / "recipe.yaml").write_text("") + rendered = { + "ros2-rcl": [("ros2-rcl", "10.2.0", "h1_30")], + "ros2-rclpy": [ + ("ros2-rclpy", "10.0.1", "np2py312h2_30"), + ("ros2-rclpy", "10.0.1", "np2py314h3_30"), + ], + } + monkeypatch.setattr(skip_existing, "render_outputs", lambda *_args: rendered) + published = { + ("ros2-rcl", "10.2.0", "h1_30"), + ("ros2-rclpy", "10.0.1", "np2py314h3_30"), + } + + removed = remove_built_recipes(tmp_path, "linux-64", None, published) + + assert removed == {"ros2-rcl"} + assert sorted(p.name for p in tmp_path.iterdir()) == ["ros2-rclpy"] + + +RECIPE = """\ +package: + name: {name} + version: "1.0.0" +build: + number: 30 +requirements: + host: +{host} +""" + + +@pytest.mark.skipif(shutil.which("rattler-build") is None, reason="needs rattler-build") +def test_render_outputs_gives_one_build_string_per_variant(tmp_path: Path): + recipes = tmp_path / "recipes" + for name, host in (("ros2-py", " - python"), ("ros2-cpp", " - zlib")): + (recipes / name).mkdir(parents=True) + (recipes / name / "recipe.yaml").write_text(RECIPE.format(name=name, host=host)) + variants = tmp_path / "conda_build_config.yaml" + variants.write_text("python:\n - 3.12.* *_cpython\n - 3.14.* *_cp314\n") + + rendered = render_outputs(recipes, "linux-64", variants) + + assert sorted(b[2][:5] for b in rendered["ros2-py"]) == ["py312", "py314"] + assert len(rendered["ros2-cpp"]) == 1 + assert "py" not in rendered["ros2-cpp"][0][2]