diff --git a/.ci/scripts/test_wheel_package_qnn.sh b/.ci/scripts/test_wheel_package_qnn.sh index 568c90ed14c..646d7c1f42c 100644 --- a/.ci/scripts/test_wheel_package_qnn.sh +++ b/.ci/scripts/test_wheel_package_qnn.sh @@ -98,8 +98,7 @@ install_qnn export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" install_executorch -EXECUTORCH_BUILDING_WHEEL=1 python setup.py bdist_wheel -unset EXECUTORCH_BUILDING_WHEEL +EXECUTORCH_BUILDING_WHEEL=1 EXECUTORCH_RELEASE_WHEEL_METADATA=1 EXECUTORCH_WHEEL_VARIANT=cpu python setup.py bdist_wheel WHEEL_FILE=$(ls dist/*.whl | head -n 1) echo "Found wheel: $WHEEL_FILE" diff --git a/.ci/scripts/tests/test_cu134_dependencies.py b/.ci/scripts/tests/test_cu134_dependencies.py index ab424f41212..014b85f869a 100644 --- a/.ci/scripts/tests/test_cu134_dependencies.py +++ b/.ci/scripts/tests/test_cu134_dependencies.py @@ -5,12 +5,14 @@ # LICENSE file in the root directory of this source tree. import ast +import functools import importlib.util import os import subprocess import sys import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch from packaging.requirements import Requirement @@ -28,6 +30,7 @@ def load_module(name): class TestCu134Dependencies(unittest.TestCase): def setUp(self): self.utils = load_module("install_utils") + self.release_versions = load_module("scripts/release/release_versions") self.modules = patch.dict(sys.modules, {"install_utils": self.utils}) self.modules.start() self.addCleanup(self.modules.stop) @@ -57,12 +60,10 @@ def test_all_install_steps_preserve_exact_cu134_selection(self): with self.subTest(machine=machine): commands = self.install_commands((13, 4), machine) self.assertEqual(len(commands), 4) - expected = { - "torch==2.14.0.dev20260810+cu134", - "torchvision==0.29.0.dev20260811+cu134", - "torchaudio==2.11.0.dev20260811+cu134", - f"torchao=={self.installer.CU134_TORCHAO_NIGHTLY_VERSION}+{ao_variant}", - } + expected = set(self.installer.CU134_TORCH_PACKAGES) + expected.add( + f"torchao=={self.installer.CU134_TORCHAO_NIGHTLY_VERSION}+{ao_variant}" + ) for index, command in enumerate(commands): required = ( expected @@ -81,17 +82,29 @@ def test_all_install_steps_preserve_exact_cu134_selection(self): for arg in command ) ) - self.assertIn( - "https://download.pytorch.org/whl/nightly/cu134", command - ) - self.assertNotIn( - "https://download.pytorch.org/whl/test/cu134", command + expected_base = ( + self.installer.TORCH_URL_BASE + if self.installer.RELEASE_WHEEL + else self.installer.TORCHAO_URL_BASE ) + self.assertIn(f"{expected_base}/cu134", command) self.assertNotIn("--no-deps", command) if machine == "aarch64": - self.assertIn( - "https://download.pytorch.org/whl/nightly/cpu", command - ) + self.assertIn(f"{self.installer.TORCHAO_URL_BASE}/cpu", command) + + release_packages = [ + "torch==2.15.0+cu134", + "torchvision==0.30.0+cu134", + "torchaudio==2.12.0+cu134", + ] + with ( + patch.object(self.installer, "RELEASE_WHEEL", True), + patch.object(self.installer, "CU134_TORCH_PACKAGES", release_packages), + ): + commands = self.install_commands((13, 4)) + for command in commands: + self.assertIn("https://download.pytorch.org/whl/test/cu134", command) + self.assertTrue(set(release_packages).issubset(commands[2])) def test_other_cuda_trains_keep_existing_pins(self): for cuda in ((12, 6), (13, 0), (13, 2)): @@ -100,12 +113,16 @@ def test_other_cuda_trains_keep_existing_pins(self): core, local, domains, examples = self.install_commands( cuda, machine ) - self.assertIn("torch==2.14.0", core) + self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) self.assertIn( f"torchao=={self.installer.TORCHAO_NIGHTLY_VERSION}", core ) - self.assertIn("torchvision==0.29.0", domains) - self.assertIn("torchaudio==2.11.0", domains) + self.assertIn( + f"torchvision=={self.installer.TORCHVISION_VERSION}", domains + ) + self.assertIn( + f"torchaudio=={self.installer.TORCHAUDIO_VERSION}", domains + ) self.assertFalse(any("==" in arg for arg in local)) self.assertFalse(any("==" in arg for arg in examples)) @@ -114,21 +131,21 @@ def test_source_pinned_torch_is_not_replaced(self): with self.subTest(cuda=cuda): core, _, domains, _ = self.install_commands(cuda, nightly=False) self.assertIn("torch", core) - self.assertNotIn("torch==2.14.0.dev20260810+cu134", core) + self.assertNotIn(self.installer.CU134_TORCH_PACKAGES[0], core) self.assertIn("torchvision", domains) self.assertIn("torchaudio", domains) def test_no_cuda_keeps_default_pins(self): core, _, domains, _ = self.install_commands(None) - self.assertIn("torch==2.14.0", core) + self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) self.assertIn(f"torchao=={self.installer.TORCHAO_NIGHTLY_VERSION}", core) - self.assertIn("torchvision==0.29.0", domains) + self.assertIn(f"torchvision=={self.installer.TORCHVISION_VERSION}", domains) self.assertIn("https://download.pytorch.org/whl/test/cpu", core) def test_windows_does_not_select_cu134(self): core, _, domains, _ = self.install_commands((13, 4), system="Windows") - self.assertIn("torch==2.14.0", core) - self.assertIn("torchvision==0.29.0", domains) + self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) + self.assertIn(f"torchvision=={self.installer.TORCHVISION_VERSION}", domains) self.assertIn("https://download.pytorch.org/whl/test/cpu", core) def test_failure_is_not_retried_with_another_cuda_train(self): @@ -145,26 +162,63 @@ def test_failure_is_not_retried_with_another_cuda_train(self): self.installer.install_requirements(True) self.assertEqual(run.call_count, 1) - def torchao_requirement(self): + def setup_requirement( + self, + function_name, + *, + installed_torch="2.15.0", + building_wheel=True, + wheel_variant="cpu", + ): path = ROOT / "setup.py" tree = ast.parse(path.read_text()) - function = next( + functions = [ node for node in tree.body - if isinstance(node, ast.FunctionDef) and node.name == "_torchao_requirement" - ) + if isinstance(node, ast.FunctionDef) + and node.name + in { + "_load_install_requirements", + "_torchao_requirement", + "_release_torch_requirement", + } + ] namespace = { "__file__": str(path), "Path": Path, + "List": list, + "functools": functools, "importlib": importlib, + "os": os, "sys": sys, "install_utils": self.utils, + "release_versions": self.release_versions, + "torch_pin": SimpleNamespace(RELEASE_WHEEL=True, TORCH_VERSION="2.15.0"), } exec( - compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), + compile(ast.Module(body=functions, type_ignores=[]), str(path), "exec"), namespace, ) - return namespace["_torchao_requirement"]() + environment = ( + { + "EXECUTORCH_RELEASE_WHEEL_METADATA": "1", + "EXECUTORCH_WHEEL_VARIANT": wheel_variant, + } + if building_wheel + else {} + ) + with ( + patch.dict(os.environ, environment, clear=True), + patch("importlib.metadata.version", return_value=installed_torch), + ): + return namespace[function_name]() + + def torchao_requirement(self): + return self.setup_requirement("_torchao_requirement") + + def release_torch_requirement(self, **kwargs): + requirements = self.setup_requirement("_release_torch_requirement", **kwargs) + return requirements[0] if requirements else None def test_package_install_preserves_source_pinned_torchao(self): with patch.dict(sys.modules, {"install_requirements": self.installer}): @@ -255,6 +309,27 @@ def test_cu134_keeps_explicit_torchao_source_build(self): f"{source_version}+git{source_commit[:7]}", metadata.specifier ) + def test_release_wheel_torch_bound_matches_selected_train(self): + for wheel_variant, installed_torch, expected_torch in ( + ("cu132", "2.15.0+cu132", "torch==2.15.0+cu132"), + ("cpu", "2.15.0", "torch>=2.15.0,<2.16"), + ): + with self.subTest(wheel_variant=wheel_variant): + self.assertEqual( + self.release_torch_requirement( + installed_torch=installed_torch, + wheel_variant=wheel_variant, + ), + expected_torch, + ) + + with self.assertRaisesRegex(RuntimeError, "for Torch 2.15.0"): + self.release_torch_requirement( + installed_torch="2.14.0.dev20260810+cu134", + wheel_variant="cu134", + ) + self.assertIsNone(self.release_torch_requirement(building_wheel=False)) + def test_wheel_torchao_bound_matches_selected_train(self): for cuda in ((13, 4), (13, 2), None): version = ( diff --git a/.ci/scripts/tests/test_filter_cuda_matrix.py b/.ci/scripts/tests/test_filter_cuda_matrix.py index 35a36cfe9df..6d40889ebe6 100644 --- a/.ci/scripts/tests/test_filter_cuda_matrix.py +++ b/.ci/scripts/tests/test_filter_cuda_matrix.py @@ -16,6 +16,7 @@ import json import os import re +import runpy import subprocess import unittest from pathlib import Path @@ -37,6 +38,7 @@ def _load_module(name, path): "filter_cuda_matrix", ROOT / ".github" / "scripts" / "filter_cuda_matrix.py" ) INSTALL_UTILS = _load_module("install_utils", ROOT / "install_utils.py") +RELEASE_WHEEL = runpy.run_path(str(ROOT / "torch_pin.py"))["RELEASE_WHEEL"] def _full_matrix(): @@ -284,7 +286,17 @@ class TestPublishedSets(unittest.TestCase): """ def test_published_cuda_versions(self): - self.assertEqual(FILTER.SUPPORTED_CUDA_VERSIONS, ["cu130", "cu132", "cu134"]) + expected = {"cu130", "cu132", "cu134"} + self.assertEqual(set(FILTER.RELEASE_CUDA_CANDIDATES), expected) + self.assertLessEqual( + set(FILTER.SUPPORTED_CUDA_VERSIONS), + set(FILTER.RELEASE_CUDA_CANDIDATES), + ) + if RELEASE_WHEEL: + self.assertTrue(FILTER.SUPPORTED_CUDA_VERSIONS) + self.assertLessEqual(set(FILTER.SUPPORTED_CUDA_VERSIONS), expected) + else: + self.assertEqual(set(FILTER.SUPPORTED_CUDA_VERSIONS), expected) def test_published_cuda_versions_are_documented(self): # The install table on the getting started page is the only place a user is told @@ -297,11 +309,15 @@ def test_published_cuda_versions_are_documented(self): self.assertIn(heading, text, f"no Installation section in {page.name}") section = text.split(heading, 1)[1].split("\n## ", 1)[0] documented = set(re.findall(r"cu\d+", section)) - self.assertEqual( - documented, - set(FILTER.SUPPORTED_CUDA_VERSIONS), - f"the install table in {page.name} lists {sorted(documented)}", - ) + published = set(FILTER.SUPPORTED_CUDA_VERSIONS) + if RELEASE_WHEEL: + self.assertLessEqual(published, documented) + else: + self.assertEqual( + documented, + published, + f"the install table in {page.name} lists {sorted(documented)}", + ) def test_published_cuda_versions_are_supported_by_the_installer(self): supported = { diff --git a/.ci/scripts/wheel/cuda_arch_list.sh b/.ci/scripts/wheel/cuda_arch_list.sh index 9e74af6c644..af83d299c98 100644 --- a/.ci/scripts/wheel/cuda_arch_list.sh +++ b/.ci/scripts/wheel/cuda_arch_list.sh @@ -73,6 +73,17 @@ _executorch_unknown_train() { return 64 } +# Normalize and validate the CUDA wheel variant once for every wheel script. +executorch_cuda_variant() { + local raw="${CU_VERSION:-${DESIRED_CUDA:-}}" + local digits="${raw//[!0-9]/}" + if [[ ! "${digits}" =~ ^[0-9]{3}$ ]]; then + echo "CU_VERSION or DESIRED_CUDA must identify a CUDA wheel variant, got '${raw}'" >&2 + return 64 + fi + printf 'cu%s' "${digits}" +} + # The architectures for the current row, space separated in the dotted form PyTorch expects. executorch_cuda_arch_list() { local machine @@ -97,8 +108,10 @@ executorch_cuda_arch_list() { ;; esac # The value arrives as cu130, while some callers pass 13.0 instead. + if ! train="$(executorch_cuda_variant)"; then + return 64 + fi train="${train#cu}" - train="${train//./}" case "${machine}" in aarch64 | arm64) diff --git a/.ci/scripts/wheel/envvar_base.sh b/.ci/scripts/wheel/envvar_base.sh index f57f6131f46..ab3dad76ca9 100755 --- a/.ci/scripts/wheel/envvar_base.sh +++ b/.ci/scripts/wheel/envvar_base.sh @@ -11,3 +11,9 @@ # Ensure that CMAKE_ARGS is defined before referencing it. Defaults to empty # if not defined. export CMAKE_ARGS="${CMAKE_ARGS:-}" + +# setup.py only adds release dependency metadata for artifacts built by the +# binary wheel jobs. A source install on a release branch must preserve the +# PyTorch build already installed by that checkout's CI job. +export EXECUTORCH_RELEASE_WHEEL_METADATA=1 +export EXECUTORCH_WHEEL_VARIANT=cpu diff --git a/.ci/scripts/wheel/envvar_cuda_linux.sh b/.ci/scripts/wheel/envvar_cuda_linux.sh index fda1a6b88a3..65563d4a0db 100644 --- a/.ci/scripts/wheel/envvar_cuda_linux.sh +++ b/.ci/scripts/wheel/envvar_cuda_linux.sh @@ -9,6 +9,12 @@ # any variables so that subprocesses will see them. source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh" +source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/cuda_arch_list.sh" + +if ! _executorch_cuda_variant="$(executorch_cuda_variant)"; then + exit 1 +fi +export EXECUTORCH_WHEEL_VARIANT="${_executorch_cuda_variant}" # Ask for the CUDA delegate explicitly rather than letting the build detect a toolkit. A detected # build is fine locally, but a release row states what it is producing, and a row that silently @@ -31,7 +37,6 @@ fi # Compile device code for the GPUs this release row claims, rather than for whichever GPU the # builder happens to have. A wheel built by detection alone installs on every machine the row covers # and then fails when a model runs on a different generation. -source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/cuda_arch_list.sh" # The status is checked rather than only the output, so an unrecognised row reports why it stopped. # A bare assignment would end the build on the lookup's own exit status with no message, since this # file is sourced into a shell that exits on a failing command. diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index 65943277324..b9c721926fa 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -21,6 +21,8 @@ import json import os +import re +import runpy import subprocess import sys from pathlib import Path @@ -45,12 +47,94 @@ # Distributions that are legitimately present without being declared. # -# torch, because the wheel deliberately does not declare it: a consumer brings the build -# matching their platform and accelerator. The rest are what torch itself requires, so they are -# guaranteed alongside it. +# Development and minimal wheels deliberately do not declare torch: a consumer brings the build +# matching their platform and accelerator. Release wheels also reach it through their declared +# requirements. The rest are what torch itself requires, so they are guaranteed alongside it. ASSUMED_PRESENT: Set[str] = {"torch", "executorch"} +def test_release_pytorch_requirement() -> None: + """Release wheels must declare the PyTorch release in torch_pin.py.""" + import importlib.metadata as metadata + + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name + from packaging.version import Version + + repo_root = Path(__file__).resolve().parents[3] + config = runpy.run_path(str(repo_root / "torch_pin.py")) + release_versions = runpy.run_path( + str(repo_root / "scripts/release/release_versions.py") + ) + ci_refs = ( + os.environ.get("GITHUB_REF_NAME", ""), + os.environ.get("GITHUB_BASE_REF", ""), + os.environ.get("GITHUB_REF", ""), + ) + release_ci = any( + ref.startswith("release/") + or re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+(?:-rc\d+)?", ref) + for ref in ci_refs + ) + assert not release_ci or config["RELEASE_WHEEL"], ( + "this wheel is being built from a release ref, but torch_pin.py does not enable " + "release metadata; run scripts/release/apply-release-changes.sh" + ) + torch_requirements = [ + Requirement(raw) + for raw in metadata.requires("executorch") or [] + if canonicalize_name(Requirement(raw).name) == "torch" + ] + + if config["RELEASE_WHEEL"]: + assert len(torch_requirements) == 1, ( + "a release wheel must declare exactly one PyTorch dependency, but found " + f"{[str(requirement) for requirement in torch_requirements]}" + ) + requirement = torch_requirements[0] + pins = [ + specifier.version + for specifier in requirement.specifier + if specifier.operator == "==" + ] + configured_variant = os.environ.get("EXECUTORCH_WHEEL_VARIANT", "") + if len(pins) == 1 and "+" in pins[0]: + local_version = Version(pins[0]).local or "" + variants = re.findall(r"(?:^|\.)(cu\d+)(?:\.|$)", local_version) + assert ( + len(variants) == 1 + ), f"release wheel has an invalid CUDA Torch pin: {requirement}" + assert configured_variant in ( + "", + variants[0], + ), f"{configured_variant} wheel declares the {variants[0]} Torch build" + expected_raw = release_versions["torch_requirement"]( + config["TORCH_VERSION"], variants[0], pins[0] + ) + else: + assert not configured_variant.startswith( + "cu" + ), f"{configured_variant} wheel declares a generic CPU Torch requirement" + expected_raw = release_versions["torch_requirement"]( + config["TORCH_VERSION"], "cpu" + ) + expected = str(Requirement(expected_raw).specifier) + assert str(requirement.specifier) == expected, ( + f"release wheel declares {requirement}, expected torch{expected} " + "from torch_pin.py" + ) + installed_torch = Version(metadata.version("torch")) + assert requirement.specifier.contains( + installed_torch, prereleases=True + ), f"installed torch {installed_torch} does not satisfy {requirement}" + print(f"āœ“ release wheel declares torch{expected}") + else: + assert not torch_requirements, ( + "a development wheel unexpectedly declares PyTorch; only release wheels should " + f"declare it, found {[str(requirement) for requirement in torch_requirements]}" + ) + + def _normalise(name: str) -> str: return name.lower().replace("-", "_").replace(".", "_") @@ -169,6 +253,7 @@ def _blocked_modules(allowed: Set[str]) -> Set[str]: def run_tests(work_dir: Path) -> None: + test_release_pytorch_requirement() allowed = _allowed_distributions() blocked = _blocked_modules(allowed) print( diff --git a/.ci/scripts/wheel/test_cuda_linux.py b/.ci/scripts/wheel/test_cuda_linux.py index 914fafaa3f3..3d0a060fb8b 100644 --- a/.ci/scripts/wheel/test_cuda_linux.py +++ b/.ci/scripts/wheel/test_cuda_linux.py @@ -34,6 +34,7 @@ from typing import Optional, Set import test_base +import test_clean_install import test_cpp_sdk import test_shared_libraries from examples.models import Backend, Model @@ -583,6 +584,7 @@ def test_a_model_runs_through_the_delegate() -> None: if __name__ == "__main__": assert platform.system() == "Linux", "the CUDA rows are Linux only" + test_clean_install.test_release_pytorch_requirement() test_cuda_libraries_are_shipped() test_cuda_runtime_is_declared() test_cuda_libraries_resolve_relatively() diff --git a/.ci/scripts/wheel/vc_env_helper.bat b/.ci/scripts/wheel/vc_env_helper.bat index d7fa2c1a596..5d0965df7f0 100644 --- a/.ci/scripts/wheel/vc_env_helper.bat +++ b/.ci/scripts/wheel/vc_env_helper.bat @@ -26,6 +26,8 @@ if "%VSDEVCMD_ARGS%" == "" ( if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" set DISTUTILS_USE_SDK=1 +set EXECUTORCH_RELEASE_WHEEL_METADATA=1 +set EXECUTORCH_WHEEL_VARIANT=cpu set args=%1 shift diff --git a/.githooks/README.md b/.githooks/README.md index 796274c2e48..59b34a9c80c 100644 --- a/.githooks/README.md +++ b/.githooks/README.md @@ -9,8 +9,9 @@ This directory contains Git hooks for the ExecuTorch repository. It is used as Runs on every commit: -1. **torch_pin sync** ā€” when `torch_pin.py` is staged, updates the PyTorch commit - pin in `.ci/docker/ci_commit_pins/pytorch.txt` and syncs grafted c10 files. +1. **torch_pin sync** ā€” when a development `torch_pin.py` is staged, updates the + PyTorch commit pin and grafted c10 files. Prepared release branches retain + the release commit selected by the branch-cut or finalization script. 2. **lintrunner** ā€” runs `lintrunner -a --revision HEAD^ --skip MYPY` on changed files. Auto-fixes formatting and blocks on lint errors. Soft-fails if lintrunner is not installed. Runs `lintrunner init` automatically when `.lintrunner.toml` diff --git a/.githooks/pre-commit b/.githooks/pre-commit index da2aaf27ee2..2ccbb3062f7 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -5,25 +5,29 @@ # Check if torch_pin.py is being committed if git diff --cached --name-only | grep -q "^torch_pin.py$"; then echo "šŸ” Detected changes to torch_pin.py" - echo "šŸ“ Updating PyTorch commit pin and syncing c10 directories..." - - # Run the update script (which now also syncs c10 directories) - if python .github/scripts/update_pytorch_pin.py; then - # Stage any modified files (pytorch.txt and grafted c10 files) - if ! git diff --quiet .ci/docker/ci_commit_pins/pytorch.txt; then - git add .ci/docker/ci_commit_pins/pytorch.txt - echo "šŸ“Œ Staged .ci/docker/ci_commit_pins/pytorch.txt" - fi - - # Stage any grafted c10 files - if ! git diff --quiet runtime/core/portable_type/c10/; then - git add runtime/core/portable_type/c10/ - echo "šŸ“Œ Staged grafted c10 files" - fi + if git show :torch_pin.py | grep -q '^RELEASE_WHEEL = True$'; then + echo "šŸ“Œ Keeping the PyTorch release commit selected by release preparation" else - echo "āŒ Failed to update PyTorch commit pin" - echo "Please run: python .github/scripts/update_pytorch_pin.py" - exit 1 + echo "šŸ“ Updating PyTorch commit pin and syncing c10 directories..." + + # Run the update script (which now also syncs c10 directories) + if python .github/scripts/update_pytorch_pin.py; then + # Stage any modified files (pytorch.txt and grafted c10 files) + if ! git diff --quiet .ci/docker/ci_commit_pins/pytorch.txt; then + git add .ci/docker/ci_commit_pins/pytorch.txt + echo "šŸ“Œ Staged .ci/docker/ci_commit_pins/pytorch.txt" + fi + + # Stage any grafted c10 files + if ! git diff --quiet runtime/core/portable_type/c10/; then + git add runtime/core/portable_type/c10/ + echo "šŸ“Œ Staged grafted c10 files" + fi + else + echo "āŒ Failed to update PyTorch commit pin" + echo "Please run: python .github/scripts/update_pytorch_pin.py" + exit 1 + fi fi fi diff --git a/.github/scripts/filter_cuda_matrix.py b/.github/scripts/filter_cuda_matrix.py index bdce1c3feab..eb6bf2045eb 100644 --- a/.github/scripts/filter_cuda_matrix.py +++ b/.github/scripts/filter_cuda_matrix.py @@ -39,6 +39,11 @@ # not on that list is rejected whether or not it appears here. DISABLED_PYTHON_VERSIONS: List[str] = ["3.13t", "3.14t", "3.15", "3.15t"] +# CUDA versions release preparation may publish, when the selected upstream +# release provides every required package. This list stays intact on release +# branches so a rerun can restore a train whose artifacts arrived late. +RELEASE_CUDA_CANDIDATES: List[str] = ["cu130", "cu132", "cu134"] + # CUDA versions to publish, when the generator offers them. # # Chosen so that every consumer row can find a matching wheel rather than by what is diff --git a/.github/scripts/update_pytorch_pin.py b/.github/scripts/update_pytorch_pin.py index c6a6318e965..c535e98045d 100644 --- a/.github/scripts/update_pytorch_pin.py +++ b/.github/scripts/update_pytorch_pin.py @@ -1,14 +1,15 @@ #!/usr/bin/env python3 -import base64 +import argparse import json +import os import re import runpy import subprocess import sys import urllib.request from pathlib import Path -from urllib.parse import unquote +from urllib.parse import quote, unquote TORCHAO_INDEX_URL = "https://download.pytorch.org/whl/nightly" @@ -53,6 +54,8 @@ def get_json(url): req = urllib.request.Request(url) req.add_header("Accept", "application/vnd.github.v3+json") req.add_header("User-Agent", "ExecuTorch-Bot") + if os.environ.get("GITHUB_TOKEN"): + req.add_header("Authorization", f"Bearer {os.environ['GITHUB_TOKEN']}") with urllib.request.urlopen(req) as response: return json.loads(response.read().decode()) @@ -68,7 +71,7 @@ def get_commit_hash_for_nightly(date_str): Commit hash string """ api_url = "https://api.github.com/repos/pytorch/pytorch/commits" - params = f"?sha=nightly&per_page=50" + params = "?sha=nightly&per_page=50" url = api_url + params try: @@ -227,7 +230,7 @@ def should_skip_file(filename): def fetch_file_content(commit_hash, file_path): """ - Fetch file content from GitHub API. + Fetch file content from GitHub's raw-content service. Args: commit_hash: Commit hash to fetch from @@ -236,26 +239,27 @@ def fetch_file_content(commit_hash, file_path): Returns: File content as bytes """ - api_url = f"https://api.github.com/repos/pytorch/pytorch/contents/{file_path}?ref={commit_hash}" + raw_url = ( + "https://raw.githubusercontent.com/pytorch/pytorch/" # @lint-ignore + f"{quote(commit_hash, safe='')}/{quote(file_path)}" + ) - req = urllib.request.Request(api_url) - req.add_header("Accept", "application/vnd.github.v3+json") + req = urllib.request.Request(raw_url) req.add_header("User-Agent", "ExecuTorch-Bot") + if os.environ.get("GITHUB_TOKEN"): + req.add_header("Authorization", f"Bearer {os.environ['GITHUB_TOKEN']}") try: with urllib.request.urlopen(req) as response: - data = json.loads(response.read().decode()) - # Content is base64 encoded - content = base64.b64decode(data["content"]) - return content + return response.read() except urllib.request.HTTPError as e: print(f"Error fetching file {file_path}: {e}", file=sys.stderr) raise -def sync_directory(et_dir, pt_path, commit_hash): +def directory_updates(et_dir, pt_path, commit_hash): """ - Sync files from PyTorch to ExecuTorch using GitHub API. + Sync files from PyTorch to ExecuTorch using GitHub raw content. Only syncs files that already exist in ExecuTorch - does not add new files. Args: @@ -263,15 +267,15 @@ def sync_directory(et_dir, pt_path, commit_hash): pt_path: PyTorch directory path in the repository (e.g., "c10") commit_hash: Commit hash to fetch from - Returns: - Number of files grafted + Returns a list of (path, content) updates without mutating the checkout. """ - files_grafted = 0 print(f"Checking {et_dir} vs pytorch/{pt_path}...") if not et_dir.exists(): print(f"Warning: ExecuTorch directory {et_dir} does not exist, skipping") - return 0 + return [] + + updates = [] # Loop through files in ExecuTorch directory for et_file in et_dir.rglob("*"): @@ -293,19 +297,12 @@ def sync_directory(et_dir, pt_path, commit_hash): if pt_content != et_content: print(f"āš ļø Difference detected in {rel_path}") - print(f"šŸ“‹ Grafting from PyTorch commit {commit_hash}...") - - et_file.write_bytes(pt_content) - print(f"āœ… Grafted {et_file}") - files_grafted += 1 + updates.append((et_file, pt_content)) except urllib.request.HTTPError as e: - if e.code != 404: # It's ok to have more files in ET than pytorch/pytorch. - print(f"Error fetching {rel_path} from PyTorch: {e}") - except Exception as e: - print(f"Error syncing {rel_path}: {e}") - continue + if e.code != 404: + raise - return files_grafted + return updates def sync_c10_directories(commit_hash): @@ -336,51 +333,62 @@ def sync_c10_directories(commit_hash): ), ] - total_grafted = 0 + updates = [] for et_dir, pt_path in dir_pairs: - files_grafted = sync_directory(et_dir, pt_path, commit_hash) - total_grafted += files_grafted + updates.extend(directory_updates(et_dir, pt_path, commit_hash)) - if total_grafted > 0: - print(f"\nāœ… Successfully grafted {total_grafted} file(s) from PyTorch") + for et_file, pt_content in updates: + print(f"šŸ“‹ Grafting from PyTorch commit {commit_hash}...") + et_file.write_bytes(pt_content) + print(f"āœ… Grafted {et_file}") + + if updates: + print(f"\nāœ… Successfully grafted {len(updates)} file(s) from PyTorch") else: print("\nāœ… No differences found - c10 is in sync") - return total_grafted + return len(updates) def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--commit", + help="PyTorch release commit to use instead of resolving NIGHTLY_VERSION", + ) + args = parser.parse_args() try: - # Read NIGHTLY_VERSION from torch_pin.py - nightly_version = get_torch_nightly_version() - print(f"Found NIGHTLY_VERSION: {nightly_version}") - - # Parse to date string - date_str = parse_nightly_version(nightly_version) - print(f"Parsed date: {date_str}") - - # Fetch commit hash from PyTorch nightly branch - commit_hash = get_commit_hash_for_nightly(date_str) - print(f"Found commit hash: {commit_hash}") - - # Update the pin file - update_pytorch_pin(commit_hash) - - # Sync c10 directories from PyTorch + if args.commit: + commit_hash = args.commit + print(f"Using requested PyTorch commit: {commit_hash}") + else: + nightly_version = get_torch_nightly_version() + print(f"Found NIGHTLY_VERSION: {nightly_version}") + date_str = parse_nightly_version(nightly_version) + print(f"Parsed date: {date_str}") + commit_hash = get_commit_hash_for_nightly(date_str) + print(f"Found commit hash: {commit_hash}") + + # Fetch every header before changing the checkout. If the network or + # rate limit fails, the existing source pin and headers stay together. sync_c10_directories(commit_hash) + update_pytorch_pin(commit_hash) - # Select the newest TorchAO nightly available for every supported CUDA - # channel and align the source submodule with the commit that built it. - max_torchao_date = date_str.replace("-", "") - torchao_version = get_latest_torchao_nightly(max_torchao_date) - print(f"Found TorchAO nightly version: {torchao_version}") - torchao_commit_hash = get_torchao_commit_hash(torchao_version) - print(f"Found TorchAO commit hash: {torchao_commit_hash}") - update_torchao_pins(torchao_version, torchao_commit_hash) - - print( - "\nāœ… Successfully updated PyTorch and TorchAO pins and synced c10 directories!" - ) + if args.commit: + print("\nāœ… Successfully updated PyTorch pin and synced c10 directories!") + else: + # Select the newest TorchAO nightly available for every supported CUDA + # channel and align the source submodule with the commit that built it. + max_torchao_date = date_str.replace("-", "") + torchao_version = get_latest_torchao_nightly(max_torchao_date) + print(f"Found TorchAO nightly version: {torchao_version}") + torchao_commit_hash = get_torchao_commit_hash(torchao_version) + print(f"Found TorchAO commit hash: {torchao_commit_hash}") + update_torchao_pins(torchao_version, torchao_commit_hash) + print( + "\nāœ… Successfully updated PyTorch and TorchAO pins and synced " + "c10 directories!" + ) except Exception as e: print(f"Error: {e}", file=sys.stderr) diff --git a/.github/workflows/build-wheels-aarch64-linux.yml b/.github/workflows/build-wheels-aarch64-linux.yml index eb79a9786c0..3db59284048 100644 --- a/.github/workflows/build-wheels-aarch64-linux.yml +++ b/.github/workflows/build-wheels-aarch64-linux.yml @@ -11,7 +11,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/build-wheels-cuda-aarch64-linux.yml b/.github/workflows/build-wheels-cuda-aarch64-linux.yml index 882184d4dcc..134a1d4e9ab 100644 --- a/.github/workflows/build-wheels-cuda-aarch64-linux.yml +++ b/.github/workflows/build-wheels-cuda-aarch64-linux.yml @@ -15,6 +15,7 @@ on: - install_utils.py - pyproject.toml - setup.py + - torch_pin.py - docs/source/using-executorch-cpp.md - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories diff --git a/.github/workflows/build-wheels-cuda-linux.yml b/.github/workflows/build-wheels-cuda-linux.yml index 8de4ae4036c..3d324d75132 100644 --- a/.github/workflows/build-wheels-cuda-linux.yml +++ b/.github/workflows/build-wheels-cuda-linux.yml @@ -15,6 +15,7 @@ on: - install_utils.py - pyproject.toml - setup.py + - torch_pin.py - docs/source/using-executorch-cpp.md - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml index 15e6af41fb4..cf3332c456e 100644 --- a/.github/workflows/build-wheels-linux.yml +++ b/.github/workflows/build-wheels-linux.yml @@ -11,7 +11,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml index feb8b8f7b69..af33d67043e 100644 --- a/.github/workflows/build-wheels-macos.yml +++ b/.github/workflows/build-wheels-macos.yml @@ -11,7 +11,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml index 8d6cac2c9c6..31907d6fe95 100644 --- a/.github/workflows/build-wheels-windows.yml +++ b/.github/workflows/build-wheels-windows.yml @@ -10,7 +10,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 66c1e7d0e92..7b6781faa6b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -49,11 +49,13 @@ jobs: - name: Install dependencies run: | TORCH_VERSION=$(python -c "from torch_pin import TORCH_VERSION; print(TORCH_VERSION)") + TORCHVISION_VERSION=$(python -c "from torch_pin import TORCHVISION_VERSION; print(TORCHVISION_VERSION)") + TORCHAUDIO_VERSION=$(python -c "from torch_pin import TORCHAUDIO_VERSION; print(TORCHAUDIO_VERSION)") pip install \ "torch==${TORCH_VERSION}" \ - torchvision \ - torchaudio \ - --index-url https://download.pytorch.org/whl/cpu + "torchvision==${TORCHVISION_VERSION}" \ + "torchaudio==${TORCHAUDIO_VERSION}" \ + --index-url "https://download.pytorch.org/whl/test/cpu" pip install lintrunner==0.12.7 lintrunner-adapters==0.14.1 pip install -r requirements-lintrunner.txt USE_CPP=0 pip install --no-build-isolation third-party/ao @@ -105,6 +107,13 @@ jobs: pip install -r requirements-lintrunner.txt lintrunner init + - name: Test release preparation + run: python scripts/release/test_prepare_release.py + + - name: Validate release configuration + if: startsWith(github.ref_name, 'release/') || startsWith(github.base_ref, 'release/') + run: python scripts/release/prepare_release.py --check + - name: Run linters env: CHANGED_FILES: ${{ needs.get-changed-files.outputs.changed-files }} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 5c4391ee69e..bc8092ee935 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -91,6 +91,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, 'extension/flat_tensor') || contains(needs.changed-files.outputs.changed-files, 'extension/pytree') || contains(needs.changed-files.outputs.changed-files, 'pyproject.toml') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || contains(needs.changed-files.outputs.changed-files, 'schema/') || contains(needs.changed-files.outputs.changed-files, 'setup.py') || contains(needs.changed-files.outputs.changed-files, 'tools/cmake/') diff --git a/.github/workflows/weekly-pytorch-pin-bump.yml b/.github/workflows/weekly-pytorch-pin-bump.yml index 0aecdaa7594..422cef20761 100644 --- a/.github/workflows/weekly-pytorch-pin-bump.yml +++ b/.github/workflows/weekly-pytorch-pin-bump.yml @@ -29,17 +29,27 @@ jobs: NIGHTLY_VERSION="dev${NIGHTLY_DATE}" echo "version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT" - - name: Read current TORCH_VERSION - id: torch - run: | - TORCH_VERSION=$(python -c "exec(open('torch_pin.py').read()); print(TORCH_VERSION)") - echo "version=${TORCH_VERSION}" >> "$GITHUB_OUTPUT" - - name: Update torch_pin.py with new NIGHTLY_VERSION + env: + NIGHTLY_VERSION: ${{ steps.nightly.outputs.version }} run: | - printf 'TORCH_VERSION = "%s"\nNIGHTLY_VERSION = "%s"\n' \ - "${{ steps.torch.outputs.version }}" \ - "${{ steps.nightly.outputs.version }}" > torch_pin.py + python - <<'PY' + import os + import pathlib + import re + + path = pathlib.Path("torch_pin.py") + updated, count = re.subn( + r'^NIGHTLY_VERSION\s*=\s*"[^"]+"$', + f'NIGHTLY_VERSION = "{os.environ["NIGHTLY_VERSION"]}"', + path.read_text(), + count=1, + flags=re.MULTILINE, + ) + if count != 1: + raise RuntimeError("expected exactly one NIGHTLY_VERSION assignment") + path.write_text(updated) + PY - name: Run pin bump script run: python .github/scripts/update_pytorch_pin.py diff --git a/.github/workflows/windows-msvc.yml b/.github/workflows/windows-msvc.yml index 269f2cf2381..3a1fa15fb71 100644 --- a/.github/workflows/windows-msvc.yml +++ b/.github/workflows/windows-msvc.yml @@ -12,6 +12,7 @@ on: - .ci/docker/ci_commit_pins/pytorch.txt - .ci/scripts/** - .github/workflows/windows-msvc.yml + - torch_pin.py workflow_dispatch: concurrency: @@ -43,6 +44,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/docker/ci_commit_pins/pytorch.txt') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/windows-msvc.yml') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || needs.run-decision.outputs.is-full-run == 'true' uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: @@ -66,6 +68,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/docker/ci_commit_pins/pytorch.txt') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/windows-msvc.yml') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || needs.run-decision.outputs.is-full-run == 'true' # test-infra's Windows setup assumes the AWS runner image; use the GitHub # hosted Windows ARM64 image directly. @@ -91,7 +94,9 @@ jobs: - name: Install build dependencies shell: pwsh - run: python -m pip install pyyaml torch==2.14.0 --extra-index-url https://download.pytorch.org/whl/test/cpu + run: | + $torchVersion = python -c "from torch_pin import TORCH_VERSION; print(TORCH_VERSION)" + python -m pip install pyyaml "torch==$torchVersion" --extra-index-url https://download.pytorch.org/whl/test/cpu - name: Build ExecuTorch shell: pwsh diff --git a/install_requirements.py b/install_requirements.py index fccd65eaead..6f594c0aea1 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -8,11 +8,18 @@ import argparse import os import platform +import runpy import subprocess import sys from install_utils import determine_torch_url, is_intel_mac_os, python_is_compatible +_TORCH_PIN = runpy.run_path(os.path.join(os.path.dirname(__file__), "torch_pin.py")) +TORCH_VERSION = _TORCH_PIN["TORCH_VERSION"] +TORCHVISION_VERSION = _TORCH_PIN["TORCHVISION_VERSION"] +TORCHAUDIO_VERSION = _TORCH_PIN["TORCHAUDIO_VERSION"] +RELEASE_WHEEL = _TORCH_PIN["RELEASE_WHEEL"] + # The pip repository that hosts nightly torch packages. # This will be dynamically set based on CUDA availability and CUDA backend enabled/disabled. TORCH_URL_BASE = "https://download.pytorch.org/whl/test" @@ -22,7 +29,8 @@ # Newest rocm7.2 TorchAO nightly with compiled kernels (cp310-abi3); later ROCm nightlies # are py3-none-any only, and none are published after 0.19.0.dev20260831. ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260805" -# These wheels' metadata pairs August 11 domain libraries with August 10 torch. +# cu134 needs explicit local-version pins. Release preparation replaces this +# development-nightly set with compatible wheels from the test index. CU134_TORCH_PACKAGES = [ "torch==2.14.0.dev20260810+cu134", "torchvision==0.29.0.dev20260811+cu134", @@ -80,9 +88,10 @@ def install_requirements(use_pytorch_nightly): torch_url = determine_torch_url(TORCH_URL_BASE) cu134_packages = cu134_requirements(torch_url) if cu134_packages: - torch_url = determine_torch_url(TORCHAO_URL_BASE) if not use_pytorch_nightly: cu134_packages[0] = "torch" + if not RELEASE_WHEEL: + torch_url = determine_torch_url(TORCHAO_URL_BASE) # torchao's CUDA channel publishes x86_64 only, so asking for a CUDA build makes the pin # unsatisfiable on aarch64. Only that case is special-cased: falling back everywhere would # change which torchao a CPU x86_64 install resolves, and the CUDA build is genuinely wanted @@ -103,7 +112,7 @@ def install_requirements(use_pytorch_nightly): # Setting use_pytorch_nightly to false to test the pinned PyTorch commit. Note # that we don't need to set any version number there because they have already # been installed on CI before this step, so pip won't reinstall them - ("torch==2.14.0" if use_pytorch_nightly else "torch"), + (f"torch=={TORCH_VERSION}" if use_pytorch_nightly else "torch"), f"torchao=={TORCHAO_NIGHTLY_VERSION}", ] @@ -176,18 +185,30 @@ def install_optional_example_requirements(use_pytorch_nightly): if use_pytorch_nightly else [] ) - if cu134_packages: + if cu134_packages and not RELEASE_WHEEL: torch_url = determine_torch_url(TORCHAO_URL_BASE) torchao_index = ( - ["--extra-index-url", f"{TORCHAO_URL_BASE}/cpu"] - if cu134_packages and platform.machine().lower() in ("aarch64", "arm64") + [ + "--extra-index-url", + ( + f"{TORCHAO_URL_BASE}/cpu" + if platform.machine().lower() in ("aarch64", "arm64") + else determine_torch_url(TORCHAO_URL_BASE) + ), + ] + if cu134_packages + and (RELEASE_WHEEL or platform.machine().lower() in ("aarch64", "arm64")) else [] ) print("Installing torch domain libraries") DOMAIN_LIBRARIES = cu134_packages or [ - ("torchvision==0.29.0" if use_pytorch_nightly else "torchvision"), - ("torchaudio==2.11.0" if use_pytorch_nightly else "torchaudio"), + ( + f"torchvision=={TORCHVISION_VERSION}" + if use_pytorch_nightly + else "torchvision" + ), + (f"torchaudio=={TORCHAUDIO_VERSION}" if use_pytorch_nightly else "torchaudio"), ] # Then install domain libraries subprocess.run( diff --git a/scripts/release/apply-release-changes.sh b/scripts/release/apply-release-changes.sh index a880687ddaf..991798b352e 100755 --- a/scripts/release/apply-release-changes.sh +++ b/scripts/release/apply-release-changes.sh @@ -6,14 +6,17 @@ # LICENSE file in the root directory of this source tree. : ' -# Step 2 after branch cut is complete. -# -# Creates PR with release only changes. +# Prepare an existing release branch. cut-release-branch.sh invokes this +# automatically; it may also be rerun safely by itself. # # Usage (run from root of project): # TEST_INFRA_BRANCH=release/2.3 ./scripts/release/apply-release-changes.sh # -# TEST_INFRA_BRANCH: The release branch of test-infra that houses all reusable +# TEST_INFRA_BRANCH: Optional override for the test-infra release branch. It +# normally defaults to the release/X.Y branch matching the selected PyTorch RC. +# The script also enables the PyTorch dependency for full release wheels. +# TORCH_VERSION: Optional PyTorch RC version override. By default, the newest +# release-form version available on the PyTorch test index is selected. ' set -eou pipefail @@ -21,43 +24,30 @@ set -eou pipefail GIT_TOP_DIR=$(git rev-parse --show-toplevel) RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} RELEASE_BRANCH="release/${RELEASE_VERSION}" +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} # Check out to Release Branch -if git ls-remote --exit-code origin ${RELEASE_BRANCH} >/dev/null 2>&1; then +if git show-ref --verify --quiet "refs/heads/${RELEASE_BRANCH}"; then + echo "Check out local Release Branch '${RELEASE_BRANCH}'" + git checkout "${RELEASE_BRANCH}" +elif git ls-remote --exit-code origin "${RELEASE_BRANCH}" >/dev/null 2>&1; then echo "Check out to Release Branch '${RELEASE_BRANCH}'" - git checkout ${RELEASE_BRANCH} + git checkout "${RELEASE_BRANCH}" else echo "Error: Remote branch '${RELEASE_BRANCH}' not found. Please run 'cut-release-branch.sh' first." exit 1 fi -# Change all GitHub Actions to reference the test-infra release branch -# as opposed to main. -echo "Applying release-only changes to workflows" -for i in .github/workflows/*.yml; do - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' -e s#@main#@"${TEST_INFRA_BRANCH}"# $i; - sed -i '' -e s#test-infra-ref:[[:space:]]main#"test-infra-ref: ${TEST_INFRA_BRANCH}"# $i; - else - sed -i -e s#@main#@"${TEST_INFRA_BRANCH}"# $i; - sed -i -e s#test-infra-ref:[[:space:]]main#"test-infra-ref: ${TEST_INFRA_BRANCH}"# $i; - fi -done - -# Update documentation to use release branch instead of viable/strict -echo "Updating documentation branch references" -for doc in $(grep -rl "viable/strict" docs/); do - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' -e "s#-b viable/strict#-b ${RELEASE_BRANCH}#g" "$doc" - else - sed -i -e "s#-b viable/strict#-b ${RELEASE_BRANCH}#g" "$doc" - fi -done +PREPARE_ARGS=( + --release-version "${RELEASE_VERSION}" +) +if [[ -n "${TEST_INFRA_BRANCH:-}" ]]; then + PREPARE_ARGS+=(--test-infra-branch "${TEST_INFRA_BRANCH}") +fi +if [[ -n "${TORCH_VERSION:-}" ]]; then + PREPARE_ARGS+=(--torch-version "${TORCH_VERSION}") +fi +"${PYTHON_EXECUTABLE}" "${GIT_TOP_DIR}/scripts/release/prepare_release.py" "${PREPARE_ARGS[@]}" -echo "You'll need to manually commit the changes and create a PR. Here are the steps:" -echo "1. Stage the changes:" -echo " git add .github/workflows/*.yml docs/" -echo "2. Commit the changes:" -echo " git commit -m \"[RELEASE-ONLY CHANGES] Branch Cut for Release ${RELEASE_VERSION}\"" -echo "3. After committing, create a pull request to merge the changes." +echo "Release changes are ready for review. This script does not commit or push them." diff --git a/scripts/release/cut-release-branch.sh b/scripts/release/cut-release-branch.sh index 60e9237ff0b..44ef3805ef1 100755 --- a/scripts/release/cut-release-branch.sh +++ b/scripts/release/cut-release-branch.sh @@ -9,46 +9,111 @@ So you are looking to cut a release branch? Well you came to the right script. -This script can be used to cut any branch on any repository - -For `pytorch/executorch` usage would be like: +For `pytorch/executorch`, run: > DRY_RUN=disabled ./scripts/release/cut-release-branch.sh -or to cut from main branch: -> DRY_RUN=disabled GIT_BRANCH_TO_CUT_FROM=main ./scripts/release/cut-release-branch.sh +The script always cuts from origin/viable/strict. RELEASE_VERSION defaults to +the MAJOR.MINOR value in its version.txt (for example, 1.10.0a0 +becomes release/1.10). It preserves the unmodified cut as orig/release/X.Y, +then prepares, commits, and pushes release/X.Y. TORCH_VERSION may override the +newest PyTorch release candidate selected from the test wheel index. ' set -eou pipefail GIT_TOP_DIR=$(git rev-parse --show-toplevel) GIT_REMOTE=${GIT_REMOTE:-origin} -GIT_BRANCH_TO_CUT_FROM=${GIT_BRANCH_TO_CUT_FROM:-viable/strict} - -# should output something like 1.11 -RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} -DRY_RUN_FLAG="--dry-run" -if [[ ${DRY_RUN:-enabled} == "disabled" ]]; then - DRY_RUN_FLAG="" +if [[ -n "$(git status --porcelain)" ]]; then + echo "Error: release cuts require a clean checkout." + exit 1 fi - ( set -x git fetch --all - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" ) -for branch in "release/${RELEASE_VERSION}" "orig/release/${RELEASE_VERSION}"; do - if git rev-parse --verify "${branch}" >/dev/null 2>/dev/null; then - echo "+ Branch ${branch} already exists, skipping..." - continue +VIABLE_BRANCH="${GIT_REMOTE}/viable/strict" +if [[ -z "${RELEASE_VERSION:-}" ]]; then + # A failed first run may already have preserved the cut point. Resume that + # release even if viable/strict has since moved to the next line. + PENDING_RELEASES=() + while IFS= read -r ORIGINAL_REF; do + CANDIDATE=${ORIGINAL_REF#refs/remotes/${GIT_REMOTE}/orig/release/} + if ! git show-ref --verify --quiet "refs/remotes/${GIT_REMOTE}/release/${CANDIDATE}"; then + PENDING_RELEASES+=("${CANDIDATE}") + fi + done < <(git for-each-ref --format='%(refname)' "refs/remotes/${GIT_REMOTE}/orig/release/") + if [[ ${#PENDING_RELEASES[@]} -gt 1 ]]; then + echo "Error: multiple unfinished release cuts: ${PENDING_RELEASES[*]}" + exit 1 + elif [[ ${#PENDING_RELEASES[@]} -eq 1 ]]; then + RELEASE_VERSION="${PENDING_RELEASES[0]}" else - ( - set -x - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" - git checkout -b "${branch}" - git push -q ${DRY_RUN_FLAG} "${GIT_REMOTE}" "${branch}" - ) + # Read version.txt directly from viable/strict so the caller's starting + # branch cannot influence a new release cut. + SOURCE_VERSION=$(git show "${VIABLE_BRANCH}:version.txt") + RELEASE_VERSION=$(printf '%s\n' "${SOURCE_VERSION}" | cut -d'.' -f1-2) + fi +fi +RELEASE_BRANCH="release/${RELEASE_VERSION}" +ORIGINAL_BRANCH="orig/${RELEASE_BRANCH}" +ARM_MANIFEST="backends/arm/public_api_manifests/api_manifest_${RELEASE_VERSION//./_}.toml" + +# The API snapshot protects main against backwards-incompatible changes, so it +# must be reviewed and merged there before the branch is cut. Creating it only +# on the release branch leaves main unprotected and guarantees later conflicts. +if git cat-file -e "${VIABLE_BRANCH}:${ARM_MANIFEST}" 2>/dev/null; then + : +elif git cat-file -e "${VIABLE_BRANCH}:backends/arm/public_api_manifests/api_manifest_running.toml" 2>/dev/null; then + echo "Error: ${ARM_MANIFEST} must be merged into ${VIABLE_BRANCH} before the release cut." + exit 1 +fi + +if [[ ${DRY_RUN:-enabled} != "disabled" ]]; then + echo "Dry run: would preserve ${VIABLE_BRANCH} as ${ORIGINAL_BRANCH}" + echo "Dry run: would prepare, commit, and push ${RELEASE_BRANCH}" + exit 0 +fi + +if git ls-remote --exit-code "${GIT_REMOTE}" "refs/heads/${ORIGINAL_BRANCH}" >/dev/null 2>&1; then + # A previous attempt already fixed the cut point. Always resume from it, + # even if viable/strict advanced after that attempt failed. + CUT_SOURCE="${GIT_REMOTE}/${ORIGINAL_BRANCH}" +else + git push "${GIT_REMOTE}" "${VIABLE_BRANCH}:refs/heads/${ORIGINAL_BRANCH}" + git fetch "${GIT_REMOTE}" "${ORIGINAL_BRANCH}:refs/remotes/${GIT_REMOTE}/${ORIGINAL_BRANCH}" + CUT_SOURCE="${GIT_REMOTE}/${ORIGINAL_BRANCH}" +fi + +if git show-ref --verify --quiet "refs/heads/${RELEASE_BRANCH}"; then + if ! git merge-base --is-ancestor "${CUT_SOURCE}" "refs/heads/${RELEASE_BRANCH}"; then + echo "Error: local ${RELEASE_BRANCH} does not descend from preserved cut ${CUT_SOURCE}." + exit 1 + fi + git checkout "${RELEASE_BRANCH}" +elif git ls-remote --exit-code "${GIT_REMOTE}" "refs/heads/${RELEASE_BRANCH}" >/dev/null 2>&1; then + if ! git merge-base --is-ancestor "${CUT_SOURCE}" "${GIT_REMOTE}/${RELEASE_BRANCH}"; then + echo "Error: remote ${RELEASE_BRANCH} does not descend from preserved cut ${CUT_SOURCE}." + exit 1 fi -done + git checkout -b "${RELEASE_BRANCH}" "${GIT_REMOTE}/${RELEASE_BRANCH}" +else + git checkout -b "${RELEASE_BRANCH}" "${CUT_SOURCE}" +fi + +( + set -x + RELEASE_VERSION="${RELEASE_VERSION}" \ + "${GIT_TOP_DIR}/scripts/release/apply-release-changes.sh" +) + +git add -A +if git diff --cached --quiet; then + echo "Release branch is already prepared; nothing to commit." +else + git commit -m "[RELEASE-ONLY CHANGES] Branch Cut for Release ${RELEASE_VERSION}" +fi +git push --set-upstream "${GIT_REMOTE}" "${RELEASE_BRANCH}" diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py new file mode 100644 index 00000000000..59a84009e7e --- /dev/null +++ b/scripts/release/prepare_release.py @@ -0,0 +1,980 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Apply and validate the deterministic changes required on release branches.""" + +import argparse +import html +import io +import re +import runpy +import subprocess +import sys +import urllib.request +import zipfile +from dataclasses import dataclass +from email.parser import Parser +from functools import lru_cache +from html.parser import HTMLParser +from pathlib import Path +from typing import Iterable, List +from urllib.error import HTTPError +from urllib.parse import unquote, urljoin + +from release_versions import ( # type: ignore[import-not-found] + is_release_version, + release_key, + torch_requirement, +) + +_RELEASE_VERSION_PATTERN = re.compile(r"\d+\.\d+") +_RELEASE_WHEEL_PATTERN = re.compile( + r"^RELEASE_WHEEL\s*=\s*(?:True|False)$", re.MULTILINE +) +_TORCH_VERSION_PATTERN = re.compile(r'^TORCH_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE) +_TORCHVISION_VERSION_PATTERN = re.compile( + r'^TORCHVISION_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE +) +_TORCHAUDIO_VERSION_PATTERN = re.compile( + r'^TORCHAUDIO_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE +) +_CU134_TORCH_PACKAGES_PATTERN = re.compile( + r"^CU134_TORCH_PACKAGES = \[\n.*?^\]\n", re.MULTILINE | re.DOTALL +) +_TORCHAO_URL_PATTERN = re.compile(r'^TORCHAO_URL_BASE\s*=\s*"[^"]+"$', re.MULTILINE) +_TORCHAO_VERSION_PATTERN = re.compile( + r'^TORCHAO_NIGHTLY_VERSION\s*=\s*"[^"]+"$', re.MULTILINE +) +_CU134_TORCHAO_VERSION_PATTERN = re.compile( + r'^CU134_TORCHAO_NIGHTLY_VERSION\s*=\s*"[^"]+"$', re.MULTILINE +) +_SUPPORTED_CUDA_PATTERN = re.compile( + r"^SUPPORTED_CUDA_VERSIONS: List\[str\] = \[[^\n]*\]$", re.MULTILINE +) +_RELEASE_CUDA_CANDIDATES_PATTERN = re.compile( + r"^RELEASE_CUDA_CANDIDATES: List\[str\] = \[[^\n]*\]$", re.MULTILINE +) +_TEST_INFRA_MAIN_PATTERN = re.compile(r"(pytorch/test-infra/[^\s\"']+)@main\b") +_TEST_INFRA_REF_MAIN_PATTERN = re.compile(r"(test-infra-ref:\s*)main\b") +_TEST_INFRA_BRANCH_PATTERN = re.compile(r"pytorch/test-infra/[^\s\"'@]+@([^\s\"']+)") +_TEST_INFRA_INPUT_PATTERN = re.compile(r"test-infra-ref:\s*([^\s#]+)") +_CLONE_BRANCH_PATTERN = re.compile(r"(?P-b\s+)viable/strict\b") +_STABLE_SWIFTPM_PATTERN = re.compile(r"swiftpm-\d+\.\d+\.\d+(?![.\d-])") +_DOCUMENTATION_PATHS = ( + "CONTRIBUTING.md", + "docs", + "extension/benchmark/apple/Benchmark", +) + + +@dataclass(frozen=True) +class WheelLink: + """One package-index wheel and its optional PEP 658 metadata.""" + + version: str + installed_version: str + url: str + metadata_url: str | None + + +class _WheelIndexParser(HTMLParser): + def __init__(self, index_url: str, package: str) -> None: + super().__init__() + self.index_url = index_url + self.package = package + self.wheels: dict[str, list[WheelLink]] = {} + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag != "a": + return + attributes = dict(attrs) + href = attributes.get("href") + if not href: + return + wheel_url = urljoin(self.index_url, html.unescape(href).split("#", 1)[0]) + filename = unquote(wheel_url).rsplit("/", 1)[-1] + match = re.match(rf"{re.escape(self.package)}-([^-]+)-.*\.whl$", filename) + if match is None: + return + installed_version = match.group(1) + version = installed_version.partition("+")[0] + if not is_release_version(version): + return + has_metadata = any( + name in attributes + for name in ("data-core-metadata", "data-dist-info-metadata") + ) + self.wheels.setdefault(version, []).append( + WheelLink( + version=version, + installed_version=installed_version, + url=wheel_url, + metadata_url=f"{wheel_url}.metadata" if has_metadata else None, + ) + ) + + +class _RemoteWheel(io.RawIOBase): + """Seekable range reader used to inspect a wheel without downloading it.""" + + def __init__(self, url: str) -> None: + self.url = url + self.position = 0 + request = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) + with urllib.request.urlopen(request, timeout=30) as response: + content_range = response.headers.get("Content-Range", "") + if "/" not in content_range: + raise RuntimeError(f"wheel server does not support byte ranges for {url}") + self.length = int(content_range.rsplit("/", 1)[1]) + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self.position + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + if whence == io.SEEK_SET: + self.position = offset + elif whence == io.SEEK_CUR: + self.position += offset + elif whence == io.SEEK_END: + self.position = self.length + offset + else: + raise ValueError(f"invalid seek mode {whence}") + return self.position + + def read(self, size: int = -1) -> bytes: + if size == 0 or self.position >= self.length: + return b"" + end = ( + self.length - 1 + if size < 0 + else min(self.length - 1, self.position + size - 1) + ) + request = urllib.request.Request( + self.url, headers={"Range": f"bytes={self.position}-{end}"} + ) + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read() + self.position += len(data) + return data + + +def _validate_release_version(release_version: str) -> None: + if _RELEASE_VERSION_PATTERN.fullmatch(release_version) is None: + raise RuntimeError( + f"release version must be MAJOR.MINOR, received {release_version!r}" + ) + + +def _write_if_changed(path: Path, updated: str) -> bool: + original = path.read_text() + if updated == original: + return False + path.write_text(updated) + return True + + +def enable_release_wheel(torch_pin_path: Path) -> bool: + """Enable release-only wheel metadata in the PyTorch pin file.""" + text = torch_pin_path.read_text() + updated, count = _RELEASE_WHEEL_PATTERN.subn("RELEASE_WHEEL = True", text) + if count != 1: + raise RuntimeError( + f"expected exactly one RELEASE_WHEEL assignment in {torch_pin_path}" + ) + return _write_if_changed(torch_pin_path, updated) + + +def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: + """Set the Torch release pin used by installs and release wheel metadata.""" + if not is_release_version(torch_version, allow_prerelease=False): + raise RuntimeError(f"invalid Torch release version {torch_version!r}") + text = torch_pin_path.read_text() + updated, count = _TORCH_VERSION_PATTERN.subn( + f'TORCH_VERSION = "{torch_version}"', text + ) + if count != 1: + raise RuntimeError( + f"expected exactly one TORCH_VERSION assignment in {torch_pin_path}" + ) + return _write_if_changed(torch_pin_path, updated) + + +def set_companion_versions( + torch_pin_path: Path, torchvision: str, torchaudio: str +) -> bool: + """Set domain-library releases that declare the selected Torch version.""" + text = torch_pin_path.read_text() + updated, vision_count = _TORCHVISION_VERSION_PATTERN.subn( + f'TORCHVISION_VERSION = "{torchvision}"', text + ) + updated, audio_count = _TORCHAUDIO_VERSION_PATTERN.subn( + f'TORCHAUDIO_VERSION = "{torchaudio}"', updated + ) + if vision_count != 1 or audio_count != 1: + raise RuntimeError( + f"expected exactly one torchvision and torchaudio assignment in {torch_pin_path}" + ) + return _write_if_changed(torch_pin_path, updated) + + +def set_cu134_versions( + install_requirements_path: Path, + torch_version: str, + torchvision: str, + torchaudio: str, +) -> bool: + """Move the exceptional cu134 package list to the selected release train.""" + text = install_requirements_path.read_text() + packages = ( + "CU134_TORCH_PACKAGES = [\n" + f' "torch=={torch_version}+cu134",\n' + f' "torchvision=={torchvision}+cu134",\n' + f' "torchaudio=={torchaudio}+cu134",\n' + "]\n" + ) + updated, count = _CU134_TORCH_PACKAGES_PATTERN.subn(packages, text) + if count != 1: + raise RuntimeError( + f"expected one CU134_TORCH_PACKAGES list in {install_requirements_path}" + ) + return _write_if_changed(install_requirements_path, updated) + + +def set_release_torchao(install_requirements_path: Path, version: str) -> bool: + """Pin TorchAO to a retained test-index release instead of a nightly.""" + if not is_release_version(version, allow_prerelease=False): + raise RuntimeError(f"invalid TorchAO release version {version!r}") + text = install_requirements_path.read_text() + updated, url_count = _TORCHAO_URL_PATTERN.subn( + 'TORCHAO_URL_BASE = "https://download.pytorch.org/whl/test"', text + ) + updated, version_count = _TORCHAO_VERSION_PATTERN.subn( + f'TORCHAO_NIGHTLY_VERSION = "{version}"', updated + ) + updated, cu_count = _CU134_TORCHAO_VERSION_PATTERN.subn( + f'CU134_TORCHAO_NIGHTLY_VERSION = "{version}"', updated + ) + if (url_count, version_count, cu_count) != (1, 1, 1): + raise RuntimeError( + f"expected one TorchAO URL and two version pins in {install_requirements_path}" + ) + return _write_if_changed(install_requirements_path, updated) + + +def configured_cuda_variants(filter_path: Path) -> list[str]: + """Read the CUDA wheel trains selected by the release matrix filter.""" + match = _SUPPORTED_CUDA_PATTERN.search(filter_path.read_text()) + if match is None: + raise RuntimeError(f"could not read supported CUDA versions from {filter_path}") + return re.findall(r'"(cu\d+)"', match.group(0)) + + +def release_cuda_candidates(filter_path: Path) -> list[str]: + """Read every CUDA train that release preparation may select.""" + match = _RELEASE_CUDA_CANDIDATES_PATTERN.search(filter_path.read_text()) + if match is None: + raise RuntimeError(f"could not read release CUDA candidates from {filter_path}") + return re.findall(r'"(cu\d+)"', match.group(0)) + + +def set_cuda_variants(filter_path: Path, variants: Iterable[str]) -> bool: + """Drop CUDA trains for which the selected upstream release has no wheels.""" + selected = list(variants) + if not selected: + raise RuntimeError("the selected PyTorch release has no supported CUDA trains") + if any(re.fullmatch(r"cu\d+", variant) is None for variant in selected): + raise RuntimeError(f"invalid CUDA release variants {selected!r}") + replacement = "SUPPORTED_CUDA_VERSIONS: List[str] = " + repr(selected).replace( + "'", '"' + ) + text = filter_path.read_text() + updated, count = _SUPPORTED_CUDA_PATTERN.subn(replacement, text) + if count != 1: + raise RuntimeError(f"expected one supported CUDA list in {filter_path}") + return _write_if_changed(filter_path, updated) + + +def newest_torch_test_release( + available_versions: Iterable[str], newer_than: str = "" +) -> str: + """Return the newest final-form wheel on a newer PyTorch release train.""" + candidates = [] + for version in available_versions: + if not is_release_version(version, allow_prerelease=False): + continue + key = release_key(version) + if newer_than and key[:2] <= release_key(newer_than)[:2]: + continue + candidates.append((key, version)) + if not candidates: + qualifier = f" newer than {newer_than}" if newer_than else "" + raise RuntimeError(f"PyTorch test index has no release wheel{qualifier}") + return max(candidates)[1] + + +@lru_cache(maxsize=None) +def _test_index_wheels( + package: str, variant: str = "cpu" +) -> dict[str, list[WheelLink]]: + index_url = ( + f"https://download.pytorch.org/whl/test/{variant}/{package}/" # @lint-ignore + ) + try: + with urllib.request.urlopen(index_url, timeout=30) as response: + index = response.read().decode() + except HTTPError as error: + if error.code in (403, 404) and variant != "cpu": + return {} + raise + parser = _WheelIndexParser(index_url, package) + parser.feed(index) + return parser.wheels + + +def _test_index_versions(package: str, variant: str = "cpu") -> set[str]: + return set(_test_index_wheels(package, variant)) + + +@lru_cache(maxsize=None) +def _wheel_member(wheel_url: str, suffix: str) -> str: + with zipfile.ZipFile(_RemoteWheel(wheel_url)) as wheel: + matches = [name for name in wheel.namelist() if name.endswith(suffix)] + if len(matches) != 1: + raise RuntimeError( + f"expected one {suffix} in {wheel_url}, found {len(matches)}" + ) + return wheel.read(matches[0]).decode() + + +@lru_cache(maxsize=None) +def _wheel_metadata(wheel: WheelLink) -> str: + if wheel.metadata_url: + with urllib.request.urlopen(wheel.metadata_url, timeout=30) as response: + return response.read().decode() + return _wheel_member(wheel.url, ".dist-info/METADATA") + + +def _preferred_wheel(wheels: list[WheelLink]) -> WheelLink: + """Choose a common Linux wheel, with a deterministic fallback.""" + return min( + wheels, + key=lambda wheel: ( + "manylinux" not in wheel.url or "x86_64" not in wheel.url, + "cp310" not in wheel.url, + wheel.url, + ), + ) + + +def companion_release_for_torch( + package: str, torch_version: str, torch_installed_version: str = "" +) -> str: + """Find the newest domain-library wheel compatible with this Torch build.""" + wheels = _test_index_wheels(package) + installed = torch_installed_version or torch_version + for version in sorted(wheels, key=release_key, reverse=True): + if companion_release_supports_torch(package, version, installed): + return version + raise RuntimeError( + f"PyTorch test index has no {package} release compatible with torch " + f"{installed}" + ) + + +def companion_release_supports_torch( + package: str, package_version: str, torch_installed_version: str +) -> bool: + """Whether any wheel for a companion release accepts the selected Torch build.""" + for wheel in _test_index_wheels(package).get(package_version, []): + metadata = Parser().parsestr(_wheel_metadata(wheel)) + for raw_requirement in metadata.get_all("Requires-Dist", []): + match = re.match( + r"\s*torch\s*(?:\(\s*([^)]*)\s*\)|([^;]*))?", + raw_requirement, + re.IGNORECASE, + ) + if match and _specifier_allows_version( + (match.group(1) or match.group(2) or "").strip(), + torch_installed_version, + ): + return True + return False + + +def _specifier_allows_version(specifier: str, installed_version: str) -> bool: + """Evaluate the release specifiers used by PyTorch companion wheels.""" + installed_public = installed_version.partition("+")[0] + installed_key = release_key(installed_public) + for clause in filter(None, (part.strip() for part in specifier.split(","))): + match = re.fullmatch(r"(===|==|!=|<=|>=|<|>|~=)\s*([^\s]+)", clause) + if match is None: + raise RuntimeError(f"unsupported Torch requirement {specifier!r}") + operator, wanted = match.groups() + wanted_public = wanted.partition("+")[0] + if re.fullmatch(r"\d+\.\d+", wanted_public): + wanted_public += ".0" + wanted_key = release_key(wanted_public) + equal = ( + installed_version == wanted + if "+" in wanted + else installed_public == wanted_public + ) + accepted = { + "==": equal, + "===": installed_version == wanted, + "!=": not equal, + "<=": installed_key <= wanted_key, + ">=": installed_key >= wanted_key, + "<": installed_key < wanted_key, + ">": installed_key > wanted_key, + }.get(operator) + if operator == "~=": + release = wanted_public.split(".") + upper = ( + (int(release[0]) + 1, 0) + if len(release) == 2 + else ( + int(release[0]), + int(release[1]) + 1, + ) + ) + accepted = installed_key >= wanted_key and installed_key[:2] < upper + if not accepted: + return False + return True + + +def companion_releases_for_torch(torch_version: str) -> dict[str, str]: + """Resolve compatible CPU companions for the selected Torch wheel.""" + torch_wheels = _test_index_wheels("torch").get(torch_version, []) + if not torch_wheels: + raise RuntimeError(f"PyTorch test index does not contain torch {torch_version}") + torch_installed_version = _preferred_wheel(torch_wheels).installed_version + expected = { + package: companion_release_for_torch( + package, torch_version, torch_installed_version + ) + for package in ("torchvision", "torchaudio") + } + expected["torch"] = torch_version + return expected + + +def available_cuda_variants( + candidates: Iterable[str], releases: dict[str, str] +) -> list[str]: + """Keep CUDA trains for which every selected PyTorch package exists.""" + return [ + variant + for variant in candidates + if all( + version in _test_index_versions(package, variant) + for package, version in releases.items() + ) + ] + + +def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: + """Choose a pin once, then preserve it on subsequent preparation runs.""" + config = runpy.run_path(str(torch_pin_path)) + current = config["TORCH_VERSION"] + if override: + if not is_release_version(override, allow_prerelease=False): + raise RuntimeError( + "Torch wheel versions on the test index must use final X.Y.Z form, " + f"received {override!r}" + ) + current_key = release_key(current) + override_key = release_key(override) + already_prepared = config.get("RELEASE_WHEEL") is True + invalid = ( + override_key < current_key + if already_prepared + else override_key[:2] <= current_key[:2] + ) + if invalid: + raise RuntimeError( + f"Torch release {override} cannot replace current version {current}" + ) + return override + if config.get("RELEASE_WHEEL") is True: + return current + return newest_torch_test_release(_test_index_versions("torch"), newer_than=current) + + +def test_infra_branch_for_torch(torch_version: str) -> str: + """Return the test-infra release branch matching a PyTorch release.""" + major, minor, _patch, _stage, _stage_number = release_key(torch_version) + return f"release/{major}.{minor}" + + +def require_test_infra_branch(test_infra_branch: str) -> None: + """Fail before editing if the selected test-infra branch does not exist.""" + result = subprocess.run( + [ + "git", + "ls-remote", + "--exit-code", + "--heads", + "https://github.com/pytorch/test-infra.git", + test_infra_branch, + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + raise RuntimeError( + f"pytorch/test-infra does not have branch {test_infra_branch!r}" + ) + + +def pytorch_commit_for_wheel(torch_version: str) -> str: + """Read the exact source commit embedded in the selected binary wheel.""" + wheels = _test_index_wheels("torch").get(torch_version, []) + if not wheels: + raise RuntimeError(f"PyTorch test index has no torch {torch_version} wheel") + version_module = _wheel_member(_preferred_wheel(wheels).url, "torch/version.py") + version_match = re.search( + r"^__version__\s*=\s*['\"]([^'\"]+)", version_module, re.MULTILINE + ) + commit_match = re.search( + r"^git_version\s*=\s*['\"]([0-9a-f]{40})", version_module, re.MULTILINE + ) + if version_match is None or commit_match is None: + raise RuntimeError("selected PyTorch wheel does not record its build commit") + if version_match.group(1).partition("+")[0] != torch_version: + raise RuntimeError( + f"selected wheel reports {version_match.group(1)}, expected {torch_version}" + ) + return commit_match.group(1) + + +def sync_pytorch_source(repo_root: Path, commit: str) -> None: + """Pin PyTorch source and synchronize the vendored c10 headers.""" + subprocess.run( + [ + sys.executable, + str(repo_root / ".github/scripts/update_pytorch_pin.py"), + "--commit", + commit, + ], + cwd=repo_root, + check=True, + ) + + +def _release_torchao_version(install_requirements_path: Path) -> tuple[str, str]: + requirements_text = install_requirements_path.read_text() + url_match = _TORCHAO_URL_PATTERN.search(requirements_text) + version_match = _TORCHAO_VERSION_PATTERN.search(requirements_text) + if url_match is None or "/whl/test" not in url_match.group(0): + raise RuntimeError("release TorchAO must come from the retained test index") + if version_match is None: + raise RuntimeError("release TorchAO pin is missing") + torchao_version = version_match.group(0).split('"')[1] + if not is_release_version(torchao_version, allow_prerelease=False): + raise RuntimeError("release TorchAO must use a non-nightly version") + return torchao_version, requirements_text + + +def _validate_cuda_releases( + releases: dict[str, str], + requirements_text: str, + cuda_filter_path: Path, + verify_index: bool, +) -> None: + variants = configured_cuda_variants(cuda_filter_path) + if verify_index: + missing = [ + f"{package}=={package_version}+{variant}" + for variant in variants + for package, package_version in releases.items() + if package_version not in _test_index_versions(package, variant) + ] + if missing: + raise RuntimeError( + "configured CUDA trains are missing release packages: " + + ", ".join(missing) + ) + if "cu134" not in variants: + return + expected = { + f"{package}=={package_version}+cu134" + for package, package_version in releases.items() + if package != "torchao" + } + actual = set( + re.findall( + r'"(torch(?:vision|audio)?==[^"]+\+cu134)"', + requirements_text, + ) + ) + if actual != expected: + raise RuntimeError( + f"cu134 pins {sorted(actual)!r} do not match {sorted(expected)!r}" + ) + + +def validate_release_build( + torch_pin_path: Path, + install_requirements_path: Path | None = None, + cuda_filter_path: Path | None = None, + verify_index: bool = True, +) -> str: + """Return the release PyTorch requirement or raise for invalid config.""" + config = runpy.run_path(str(torch_pin_path)) + if config.get("RELEASE_WHEEL") is not True: + raise RuntimeError(f"{torch_pin_path} does not enable release wheel metadata") + + version = config.get("TORCH_VERSION") + if not isinstance(version, str) or not is_release_version( + version, allow_prerelease=False + ): + raise RuntimeError(f"{torch_pin_path} has invalid TORCH_VERSION {version!r}") + for name in ("TORCHVISION_VERSION", "TORCHAUDIO_VERSION"): + if not is_release_version(config.get(name), allow_prerelease=False): + raise RuntimeError( + f"{torch_pin_path} has invalid {name} {config.get(name)!r}" + ) + if install_requirements_path is None or cuda_filter_path is None: + return torch_requirement(version, "cpu") + + releases = {"torch": version} + for package, name in ( + ("torchvision", "TORCHVISION_VERSION"), + ("torchaudio", "TORCHAUDIO_VERSION"), + ): + package_version = config[name] + releases[package] = package_version + + if verify_index: + torch_wheels = _test_index_wheels("torch").get(version, []) + if not torch_wheels: + raise RuntimeError(f"test index no longer contains torch {version}") + installed = _preferred_wheel(torch_wheels).installed_version + for package, name in ( + ("torchvision", "TORCHVISION_VERSION"), + ("torchaudio", "TORCHAUDIO_VERSION"), + ): + if not companion_release_supports_torch(package, config[name], installed): + raise RuntimeError( + f"{package} {config[name]} is not compatible with torch {installed}" + ) + + torchao_version, requirements_text = _release_torchao_version( + install_requirements_path + ) + releases["torchao"] = torchao_version + _validate_cuda_releases(releases, requirements_text, cuda_filter_path, verify_index) + return torch_requirement(version, "cpu") + + +def configure_release_version(version_path: Path, release_version: str) -> str: + """Finalize a development version without resetting an existing patch release.""" + current = version_path.read_text().strip() + if re.fullmatch(rf"{re.escape(release_version)}\.\d+", current): + return current + release_full_version = f"{release_version}.0" + _write_if_changed(version_path, f"{release_full_version}\n") + return release_full_version + + +def configure_workflows(workflow_paths: Iterable[Path], test_infra_branch: str) -> int: + """Pin test-infra actions and reusable workflows to its release branch.""" + changed = 0 + for path in workflow_paths: + text = path.read_text() + updated = _TEST_INFRA_MAIN_PATTERN.sub(rf"\1@{test_infra_branch}", text) + updated = _TEST_INFRA_REF_MAIN_PATTERN.sub(rf"\1{test_infra_branch}", updated) + changed += _write_if_changed(path, updated) + return changed + + +def documentation_paths(repo_root: Path) -> List[Path]: + """Return the maintained Markdown files that contain branch instructions.""" + paths: List[Path] = [] + for relative in _DOCUMENTATION_PATHS: + path = repo_root / relative + if path.is_file(): + paths.append(path) + elif path.is_dir(): + paths.extend(path.rglob("*.md")) + return sorted(paths) + + +def configure_documentation( + paths: Iterable[Path], release_branch: str, release_full_version: str +) -> int: + """Point checkout and stable SwiftPM examples at the new release.""" + changed = 0 + for path in paths: + text = path.read_text() + updated = _CLONE_BRANCH_PATTERN.sub(rf"\g{release_branch}", text) + updated = _STABLE_SWIFTPM_PATTERN.sub( + f"swiftpm-{release_full_version}", updated + ) + changed += _write_if_changed(path, updated) + return changed + + +def validate_release_references( + workflow_paths: Iterable[Path], + documentation: Iterable[Path], + test_infra_branch: str, + release_version: str, + release_full_version: str, +) -> None: + """Reject release branches that retain moving branch references.""" + errors = [] + found_test_infra_reference = False + for path in workflow_paths: + text = path.read_text() + references = _TEST_INFRA_BRANCH_PATTERN.findall(text) + references.extend(_TEST_INFRA_INPUT_PATTERN.findall(text)) + found_test_infra_reference |= bool(references) + for reference in references: + if reference != test_infra_branch: + errors.append( + f"{path}: test-infra references {reference}, expected " + f"{test_infra_branch}" + ) + if not found_test_infra_reference: + errors.append("no test-infra workflow references were found") + expected_release_branch = f"release/{release_version}" + expected_swiftpm_branch = f"swiftpm-{release_full_version}" + for path in documentation: + text = path.read_text() + if _CLONE_BRANCH_PATTERN.search(text): + errors.append( + f"{path}: clone command still references viable/strict, expected " + f"{expected_release_branch}" + ) + for swiftpm_branch in _STABLE_SWIFTPM_PATTERN.findall(text): + if swiftpm_branch != expected_swiftpm_branch: + errors.append( + f"{path}: stable SwiftPM example references {swiftpm_branch}, " + f"expected {expected_swiftpm_branch}" + ) + if errors: + raise RuntimeError("release preparation is incomplete:\n" + "\n".join(errors)) + + +def validate_release_files(repo_root: Path, release_version: str) -> str: + """Validate the release version.""" + actual_version = (repo_root / "version.txt").read_text().strip() + if re.fullmatch(rf"{re.escape(release_version)}\.\d+", actual_version) is None: + raise RuntimeError( + f"version.txt contains {actual_version!r}, expected a final " + f"{release_version}.x version" + ) + + manifest_dir = repo_root / "backends/arm/public_api_manifests" + if (manifest_dir / "api_manifest_running.toml").exists(): + expected_manifest = ( + manifest_dir / f"api_manifest_{release_version.replace('.', '_')}.toml" + ) + if not expected_manifest.exists(): + raise RuntimeError( + f"missing {expected_manifest}; merge the Arm API snapshot into main " + "before cutting the release branch" + ) + + return actual_version + + +def configured_test_infra_branch(workflow_paths: Iterable[Path]) -> str: + """Read the one test-infra branch already recorded in release workflows.""" + references = set() + for path in workflow_paths: + text = path.read_text() + references.update(_TEST_INFRA_BRANCH_PATTERN.findall(text)) + references.update(_TEST_INFRA_INPUT_PATTERN.findall(text)) + if len(references) != 1: + raise RuntimeError( + "expected exactly one configured test-infra branch, found " + + repr(sorted(references)) + ) + return references.pop() + + +def prepare_release( + repo_root: Path, + release_version: str, + test_infra_branch: str, + torch_version: str, + companions: dict[str, str], + torchao_version: str, + cuda_variants: list[str], + verify_index: bool = True, +) -> str: + """Apply every deterministic branch-cut edit and return the Torch requirement.""" + _validate_release_version(release_version) + if not test_infra_branch: + raise RuntimeError("test-infra branch must not be empty") + + workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + documentation = documentation_paths(repo_root) + set_torch_version(repo_root / "torch_pin.py", torch_version) + set_companion_versions( + repo_root / "torch_pin.py", + companions["torchvision"], + companions["torchaudio"], + ) + install_requirements_path = repo_root / "install_requirements.py" + if "cu134" in cuda_variants: + set_cu134_versions( + install_requirements_path, + torch_version, + companions["torchvision"], + companions["torchaudio"], + ) + set_release_torchao(install_requirements_path, torchao_version) + cuda_filter_path = repo_root / ".github/scripts/filter_cuda_matrix.py" + set_cuda_variants(cuda_filter_path, cuda_variants) + enable_release_wheel(repo_root / "torch_pin.py") + release_full_version = configure_release_version( + repo_root / "version.txt", release_version + ) + workflow_count = configure_workflows(workflow_paths, test_infra_branch) + documentation_count = configure_documentation( + documentation, f"release/{release_version}", release_full_version + ) + validate_release_references( + workflow_paths, + documentation, + test_infra_branch=test_infra_branch, + release_version=release_version, + release_full_version=release_full_version, + ) + validate_release_files(repo_root, release_version) + requirement = validate_release_build( + repo_root / "torch_pin.py", + install_requirements_path, + cuda_filter_path, + verify_index=verify_index, + ) + print( + f"Prepared release/{release_version}: {requirement}; changed " + f"{workflow_count} workflow files and {documentation_count} documentation files" + ) + return requirement + + +def _release_version_from_file(repo_root: Path) -> str: + version = (repo_root / "version.txt").read_text().strip() + match = re.match(r"(\d+\.\d+)", version) + if match is None: + raise RuntimeError(f"could not derive release version from {version!r}") + return match.group(1) + + +def main() -> None: + """Update or validate the repository's release configuration.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + action="store_true", + help="validate the release configuration without changing it", + ) + parser.add_argument("--release-version", help="release version in MAJOR.MINOR form") + parser.add_argument( + "--test-infra-branch", + help="test-infra branch (defaults to the selected PyTorch release line)", + ) + parser.add_argument( + "--torch-version", + help="Torch wheel version (defaults to the newest release on the test index)", + ) + parser.add_argument( + "--repo-root", type=Path, default=Path(__file__).resolve().parents[2] + ) + args = parser.parse_args() + + repo_root = args.repo_root.resolve() + release_version = args.release_version or _release_version_from_file(repo_root) + workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + documentation = documentation_paths(repo_root) + + if args.check: + _validate_release_version(release_version) + requirement = validate_release_build( + repo_root / "torch_pin.py", + repo_root / "install_requirements.py", + repo_root / ".github/scripts/filter_cuda_matrix.py", + ) + test_infra_branch = args.test_infra_branch or configured_test_infra_branch( + workflow_paths + ) + release_full_version = validate_release_files(repo_root, release_version) + validate_release_references( + workflow_paths, + documentation, + test_infra_branch=test_infra_branch, + release_version=release_version, + release_full_version=release_full_version, + ) + print(f"Release configuration is valid: {requirement}") + else: + current_config = runpy.run_path(str(repo_root / "torch_pin.py")) + reuse_selected_dependencies = ( + current_config.get("RELEASE_WHEEL") is True and not args.torch_version + ) + torch_version = torch_version_for_release( + repo_root / "torch_pin.py", args.torch_version or "" + ) + cuda_filter_path = repo_root / ".github/scripts/filter_cuda_matrix.py" + if reuse_selected_dependencies: + companions = { + "torch": torch_version, + "torchvision": current_config["TORCHVISION_VERSION"], + "torchaudio": current_config["TORCHAUDIO_VERSION"], + } + requirements_text = (repo_root / "install_requirements.py").read_text() + torchao_match = _TORCHAO_VERSION_PATTERN.search(requirements_text) + if torchao_match is None: + raise RuntimeError("release TorchAO pin is missing") + torchao_version = torchao_match.group(0).split('"')[1] + releases = dict(companions) + releases["torchao"] = torchao_version + cuda_variants = available_cuda_variants( + release_cuda_candidates(cuda_filter_path), releases + ) + else: + companions = companion_releases_for_torch(torch_version) + torchao_version = newest_torch_test_release(_test_index_versions("torchao")) + releases = dict(companions) + releases["torchao"] = torchao_version + cuda_variants = available_cuda_variants( + release_cuda_candidates(cuda_filter_path), releases + ) + pytorch_commit = pytorch_commit_for_wheel(torch_version) + test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( + torch_version + ) + require_test_infra_branch(test_infra_branch) + sync_pytorch_source(repo_root, pytorch_commit) + prepare_release( + repo_root, + release_version, + test_infra_branch, + torch_version, + companions, + torchao_version, + cuda_variants, + verify_index=True, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/release_versions.py b/scripts/release/release_versions.py new file mode 100644 index 00000000000..4d3c0e31071 --- /dev/null +++ b/scripts/release/release_versions.py @@ -0,0 +1,67 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Shared version rules for release preparation and wheel metadata.""" + +import re + +_RELEASE_PATTERN = re.compile(r"(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?") + + +def release_parts(version: str) -> tuple[int, int, int, str | None, int]: + match = _RELEASE_PATTERN.fullmatch(version) + if match is None: + raise RuntimeError(f"invalid release version {version!r}") + major, minor, patch = (int(part) for part in match.groups()[:3]) + return major, minor, patch, match.group(4), int(match.group(5) or 0) + + +def is_release_version(version: object, allow_prerelease: bool = True) -> bool: + if not isinstance(version, str): + return False + try: + _major, _minor, _patch, stage, _stage_number = release_parts(version) + except RuntimeError: + return False + return allow_prerelease or stage is None + + +def release_key(version: str) -> tuple[int, int, int, int, int]: + major, minor, patch, stage, stage_number = release_parts(version) + stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} + return major, minor, patch, stage_rank[stage], stage_number + + +def torch_release_tag(torch_version: str) -> str: + major, minor, patch, stage, stage_number = release_parts(torch_version) + suffix = f"-{stage}{stage_number}" if stage else "" + return f"v{major}.{minor}.{patch}{suffix}" + + +def torch_requirement( + torch_version: str, wheel_variant: str, installed_version: str = "" +) -> str: + major, minor, _patch, _stage, _stage_number = release_key(torch_version) + if wheel_variant == "cpu": + return f"torch>={torch_version},<{major}.{minor + 1}" + if re.fullmatch(r"cu\d+", wheel_variant) is None: + raise RuntimeError(f"invalid release wheel variant {wheel_variant!r}") + if not installed_version: + raise RuntimeError( + f"the {wheel_variant} wheel build did not report its Torch version" + ) + public_version, separator, local_version = installed_version.partition("+") + if public_version != torch_version: + raise RuntimeError( + f"building a {wheel_variant} wheel for Torch {torch_version} with " + f"torch {installed_version}" + ) + if not separator or wheel_variant not in local_version.split("."): + raise RuntimeError( + f"building a {wheel_variant} wheel with torch {installed_version}; " + "the installed torch build must use the same CUDA variant" + ) + return f"torch=={installed_version}" diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py new file mode 100644 index 00000000000..8eb6d613f9d --- /dev/null +++ b/scripts/release/test_prepare_release.py @@ -0,0 +1,343 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from prepare_release import ( # type: ignore[import-not-found] + _WheelIndexParser, + companion_release_for_torch, + configured_cuda_variants, + configured_test_infra_branch, + main as prepare_release_main, + newest_torch_test_release, + prepare_release, + pytorch_commit_for_wheel, + release_cuda_candidates, + set_cuda_variants, + test_infra_branch_for_torch as _test_infra_branch_for_torch, + torch_version_for_release, + WheelLink, +) + + +class PrepareReleaseTest(unittest.TestCase): + """Tests for release-only branch preparation.""" + + @staticmethod + def _git(directory: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=directory, + check=True, + capture_output=True, + text=True, + ) + + def test_selects_release_candidate_once(self) -> None: + """A branch cut selects the final-form RC wheel and preserves it.""" + releases = ["2.14.1", "2.15.0a1", "2.15.0rc1", "2.15.0"] + self.assertEqual( + newest_torch_test_release(releases, newer_than="2.14.0"), + "2.15.0", + ) + with self.assertRaises(RuntimeError): + newest_torch_test_release(["2.14.1"], newer_than="2.14.0") + self.assertEqual(_test_infra_branch_for_torch("2.15.0"), "release/2.15") + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "torch_pin.py" + path.write_text('TORCH_VERSION = "2.15.0"\nRELEASE_WHEEL = True\n') + + self.assertEqual(torch_version_for_release(path), "2.15.0") + + wheel = WheelLink( + "2.12.0", + "2.12.0", + "https://example/torchaudio.whl", # @lint-ignore + None, + ) + with patch( + "prepare_release._test_index_wheels", + return_value={"2.12.0": [wheel]}, + ), patch( + "prepare_release._wheel_metadata", + return_value="Requires-Dist: torch (>=2.15.0,<2.16)\r\n", + ): + self.assertEqual( + companion_release_for_torch("torchaudio", "2.15.0", "2.15.0+cpu"), + "2.12.0", + ) + + torch_wheel = WheelLink( + "2.15.0", + "2.15.0+cpu", + "https://example/torch.whl", # @lint-ignore + None, + ) + with patch( + "prepare_release._test_index_wheels", + return_value={"2.15.0": [torch_wheel]}, + ), patch( + "prepare_release._wheel_member", + return_value=( + "__version__ = '2.15.0+cpu'\n" f"git_version = '{'2' * 40}'\n" + ), + ): + self.assertEqual( + pytorch_commit_for_wheel("2.15.0"), + "2" * 40, + ) + + parser = _WheelIndexParser("https://example/simple/", "torch") # @lint-ignore + parser.feed( + 'a' + 'b' + ) + self.assertEqual(len(parser.wheels["2.15.0"]), 2) + self.assertIsNone(parser.wheels["2.15.0"][0].metadata_url) + self.assertIsNotNone(parser.wheels["2.15.0"][1].metadata_url) + + def test_prepares_complete_repository(self) -> None: + """The top-level operation applies dependency, workflow, and docs edits.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / ".github/workflows").mkdir(parents=True) + (root / "docs").mkdir() + (root / "version.txt").write_text("1.6.0a0\n") + (root / "torch_pin.py").write_text( + 'TORCH_VERSION = "2.14.0"\n' + 'TORCHVISION_VERSION = "0.29.0"\n' + 'TORCHAUDIO_VERSION = "2.11.0"\n' + "RELEASE_WHEEL = False\n" + ) + (root / "install_requirements.py").write_text( + 'TORCHAO_URL_BASE = "https://download.pytorch.org/whl/nightly"\n' + 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' + 'CU134_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' + "CU134_TORCH_PACKAGES = [\n" + ' "torch==2.14.0.dev20260810+cu134",\n' + ' "torchvision==0.29.0.dev20260811+cu134",\n' + ' "torchaudio==2.11.0.dev20260811+cu134",\n' + "]\n" + ) + (root / ".github/scripts").mkdir(parents=True) + cuda_filter = root / ".github/scripts/filter_cuda_matrix.py" + cuda_filter.write_text( + 'RELEASE_CUDA_CANDIDATES: List[str] = ["cu130", "cu134"]\n' + 'SUPPORTED_CUDA_VERSIONS: List[str] = ["cu130", "cu134"]\n' + ) + workflow = root / ".github/workflows/test.yml" + workflow.write_text( + "uses: pytorch/test-infra/.github/workflows/test.yml@main\n" + "test-infra-ref: main\n" + "uses: example/action@main\n" + ) + doc = root / "docs/README.md" + doc.write_text( + "git clone -b viable/strict repo\n" + "tutorial: git clone -b release/1.0 legacy-example\n" + "stable: swiftpm-1.5.1\n" + "nightly: swiftpm-1.6.0.20260929\n" + ) + companions = {"torchvision": "0.30.0", "torchaudio": "2.12.0"} + requirement = prepare_release( + root, + "1.6", + "release/2.15", + "2.15.0", + companions, + "0.20.0", + ["cu130", "cu134"], + verify_index=False, + ) + (root / "version.txt").write_text("1.6.1\n") + second_requirement = prepare_release( + root, + "1.6", + "release/2.15", + "2.15.0", + companions, + "0.20.0", + ["cu130", "cu134"], + verify_index=False, + ) + + self.assertEqual(requirement, "torch>=2.15.0,<2.16") + self.assertEqual(second_requirement, requirement) + self.assertIn("RELEASE_WHEEL = True", (root / "torch_pin.py").read_text()) + self.assertIn( + 'TORCHVISION_VERSION = "0.30.0"', + (root / "torch_pin.py").read_text(), + ) + self.assertIn( + 'TORCHAUDIO_VERSION = "2.12.0"', + (root / "torch_pin.py").read_text(), + ) + requirements = (root / "install_requirements.py").read_text() + self.assertIn('"torch==2.15.0+cu134"', requirements) + self.assertIn('"torchvision==0.30.0+cu134"', requirements) + self.assertIn('"torchaudio==2.12.0+cu134"', requirements) + self.assertIn('TORCHAO_NIGHTLY_VERSION = "0.20.0"', requirements) + self.assertIn("/whl/test", requirements) + self.assertIn("@release/2.15", workflow.read_text()) + self.assertIn("test-infra-ref: release/2.15", workflow.read_text()) + self.assertEqual(configured_test_infra_branch([workflow]), "release/2.15") + self.assertIn("example/action@main", workflow.read_text()) + self.assertIn("-b release/1.6", doc.read_text()) + self.assertIn("-b release/1.0", doc.read_text()) + self.assertIn("stable: swiftpm-1.6.1", doc.read_text()) + self.assertIn("nightly: swiftpm-1.6.0.20260929", doc.read_text()) + self.assertEqual((root / "version.txt").read_text(), "1.6.1\n") + + def test_release_cuda_candidates_survive_matrix_pruning(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "filter_cuda_matrix.py" + path.write_text( + 'RELEASE_CUDA_CANDIDATES: List[str] = ["cu130", "cu132", "cu134"]\n' + 'SUPPORTED_CUDA_VERSIONS: List[str] = ["cu130", "cu132", "cu134"]\n' + ) + + set_cuda_variants(path, ["cu130"]) + + self.assertEqual(configured_cuda_variants(path), ["cu130"]) + self.assertEqual(release_cuda_candidates(path), ["cu130", "cu132", "cu134"]) + + def test_partial_preparation_revalidates_and_resynchronizes_source(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "torch_pin.py").write_text( + 'TORCH_VERSION = "2.15.0"\n' + 'TORCHVISION_VERSION = "0.30.0"\n' + 'TORCHAUDIO_VERSION = "2.12.0"\n' + "RELEASE_WHEEL = True\n" + ) + (root / "install_requirements.py").write_text( + 'TORCHAO_NIGHTLY_VERSION = "0.20.0"\n' + ) + (root / ".github/scripts").mkdir(parents=True) + (root / ".github/scripts/filter_cuda_matrix.py").write_text( + 'RELEASE_CUDA_CANDIDATES: List[str] = ["cu130", "cu132", "cu134"]\n' + 'SUPPORTED_CUDA_VERSIONS: List[str] = ["cu130"]\n' + ) + calls: list[str] = [] + + with ( + patch.object( + sys, + "argv", + [ + "prepare_release.py", + "--repo-root", + str(root), + "--release-version", + "1.6", + ], + ), + patch("prepare_release.require_test_infra_branch"), + patch( + "prepare_release.pytorch_commit_for_wheel", + return_value="2" * 40, + ), + patch( + "prepare_release.available_cuda_variants", + return_value=["cu130", "cu132"], + ) as available_variants, + patch( + "prepare_release.sync_pytorch_source", + side_effect=lambda *_args: calls.append("sync"), + ) as sync_source, + patch( + "prepare_release.prepare_release", + side_effect=lambda *_args, **_kwargs: calls.append("prepare"), + ) as apply_changes, + ): + prepare_release_main() + + available_variants.assert_called_once() + self.assertEqual( + available_variants.call_args.args[0], ["cu130", "cu132", "cu134"] + ) + sync_source.assert_called_once_with(root.resolve(), "2" * 40) + self.assertTrue(apply_changes.call_args.kwargs["verify_index"]) + self.assertEqual(apply_changes.call_args.args[6], ["cu130", "cu132"]) + self.assertEqual(calls, ["sync", "prepare"]) + + def test_unrelated_local_release_branch_is_rejected(self) -> None: + script = Path(__file__).with_name("cut-release-branch.sh") + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + remote = root / "remote.git" + seed = root / "seed" + checkout = root / "checkout" + subprocess.run( + ["git", "init", "--bare", str(remote)], + check=True, + capture_output=True, + text=True, + ) + seed.mkdir() + self._git(seed, "init") + self._git(seed, "config", "user.email", "release-test@example.com") + self._git(seed, "config", "user.name", "Release Test") + (seed / "version.txt").write_text("1.6.0a0\n") + self._git(seed, "add", "version.txt") + self._git(seed, "commit", "-m", "cut source") + self._git(seed, "branch", "-M", "main") + self._git(seed, "remote", "add", "origin", str(remote)) + self._git(seed, "push", "origin", "main") + self._git(seed, "push", "origin", "HEAD:refs/heads/viable/strict") + self._git(seed, "push", "origin", "HEAD:refs/heads/orig/release/1.6") + subprocess.run( + ["git", "clone", "--branch", "main", str(remote), str(checkout)], + check=True, + capture_output=True, + text=True, + ) + self._git(checkout, "config", "user.email", "release-test@example.com") + self._git(checkout, "config", "user.name", "Release Test") + tree = self._git(checkout, "rev-parse", "HEAD^{tree}").stdout.strip() + unrelated = self._git( + checkout, "commit-tree", tree, "-m", "unrelated history" + ).stdout.strip() + self._git(checkout, "branch", "release/1.6", unrelated) + + result = subprocess.run( + ["bash", str(script)], + cwd=checkout, + env={ + **os.environ, + "DRY_RUN": "disabled", + "RELEASE_VERSION": "1.6", + }, + capture_output=True, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("does not descend from preserved cut", result.stdout) + remote_release = subprocess.run( + [ + "git", + "ls-remote", + "--exit-code", + str(remote), + "refs/heads/release/1.6", + ], + capture_output=True, + text=True, + ) + self.assertNotEqual(remote_release.returncode, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/setup.py b/setup.py index c0a23cb0195..8b604657289 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,26 @@ raise ImportError(f"Module spec has no loader for {_install_utils_path}") _spec.loader.exec_module(install_utils) +_torch_pin_path = Path(__file__).parent / "torch_pin.py" +_spec = importlib.util.spec_from_file_location("torch_pin", _torch_pin_path) +if _spec is None: + raise ImportError(f"Could not create module spec for {_torch_pin_path}") +torch_pin = importlib.util.module_from_spec(_spec) +if _spec.loader is None: + raise ImportError(f"Could not load {_torch_pin_path}") +_spec.loader.exec_module(torch_pin) + +_release_versions_path = Path(__file__).parent / "scripts/release/release_versions.py" +_spec = importlib.util.spec_from_file_location( + "release_versions", _release_versions_path +) +if _spec is None: + raise ImportError(f"Could not create module spec for {_release_versions_path}") +release_versions = importlib.util.module_from_spec(_spec) +if _spec.loader is None: + raise ImportError(f"Could not load {_release_versions_path}") +_spec.loader.exec_module(release_versions) + from setuptools import Distribution, Extension, find_namespace_packages, setup from setuptools.command.build import build from setuptools.command.build_ext import build_ext @@ -1085,17 +1105,9 @@ def _package_relative_depth(library: Path) -> int: return max(len(parts) - index - 2, 0) -def _torchao_requirement() -> str: - """The torchao dependency, pinned to the series install_requirements.py installs. - - Derived from that module rather than written out, so a nightly bump cannot move the - pin without moving this bound with it. A bump into the next series would otherwise - silently stop satisfying the lower bound, and installing this package over a - development checkout would replace the torchao that was just installed. - - Loaded by path, the way install_utils is above, because setuptools executes this - file without the project directory on sys.path, so a plain import does not resolve. - """ +@functools.lru_cache(maxsize=1) +def _load_install_requirements(): + """Load release dependency pins once without relying on the checkout's sys.path.""" path = Path(__file__).parent / "install_requirements.py" spec = importlib.util.spec_from_file_location("install_requirements", path) if spec is None or spec.loader is None: @@ -1105,6 +1117,12 @@ def _torchao_requirement() -> str: # because that name is registered below. sys.modules.setdefault("install_utils", install_utils) spec.loader.exec_module(module) + return module + + +def _torchao_requirement() -> str: + """The torchao dependency, pinned to the series development installs use.""" + module = _load_install_requirements() version = module.TORCHAO_NIGHTLY_VERSION if ( @@ -1116,6 +1134,24 @@ def _torchao_requirement() -> str: return f"torchao>={version},<{major}.{minor + 1}" +def _release_torch_requirement() -> List[str]: + if ( + not torch_pin.RELEASE_WHEEL + or os.environ.get("EXECUTORCH_RELEASE_WHEEL_METADATA") != "1" + ): + return [] + + from importlib import metadata + + variant = os.environ.get("EXECUTORCH_WHEEL_VARIANT", "") + installed_version = metadata.version("torch") if variant.startswith("cu") else "" + return [ + release_versions.torch_requirement( + torch_pin.TORCH_VERSION, variant, installed_version + ) + ] + + def _base_dependencies() -> List[str]: """Runtime dependencies for the full wheel. @@ -1197,7 +1233,8 @@ def _minimal_dependencies() -> List[str]: Derived as the subset of _base_dependencies() that executorch.exir needs to lower and serialize a .pte, so version pins and markers stay in sync with the - full set. torch is intentionally absent from both (consumers bring their own). + full set. torch is intentionally absent from the minimal wheel, including on + release branches (consumers bring their own). mpmath is intentionally dropped too: it is pulled transitively by sympy, whose "mpmath<1.4" cap resolves to the same 1.3.0 the full wheel pins. Keep the name set below in sync with the `expected` set in .ci/scripts/test_minimal_wheel.sh. @@ -2852,7 +2889,9 @@ def iter_distribution_names(self): setup_kwargs["packages"] = _full_packages() # A CUDA wheel links the CUDA runtime but does not bundle it, so the wheels that # carry it are declared here. A CPU wheel adds nothing. - setup_kwargs["install_requires"] = _base_dependencies() + _cuda_dependencies() + setup_kwargs["install_requires"] = ( + _base_dependencies() + _release_torch_requirement() + _cuda_dependencies() + ) setup( diff --git a/torch_pin.py b/torch_pin.py index 3010f4a22ed..58660177313 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -1,2 +1,11 @@ +# PyTorch release used by development installs and release-wheel metadata. TORCH_VERSION = "2.14.0" +TORCHVISION_VERSION = "0.29.0" +TORCHAUDIO_VERSION = "2.11.0" +# Date used by the main-branch weekly pin bot. Release preparation resolves the +# selected TORCH_VERSION tag and updates the source commit separately. NIGHTLY_VERSION = "dev20260913" + +# Changed to True by scripts/release/apply-release-changes.sh. Release wheels +# declare the PyTorch release above; development and minimal wheels do not. +RELEASE_WHEEL = False