From 1e327d7c96d4b73d35cacafe049f872232f08742 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Wed, 30 Sep 2026 15:17:49 -0700 Subject: [PATCH 1/8] Revert "Split release finalization into follow-up" This reverts commit 97fe0c16305a886bb5c63f043fa9526b4ac7acb4. --- .ci/scripts/wheel/test_clean_install.py | 18 +- .github/workflows/lint.yml | 4 +- scripts/release/apply-release-changes.sh | 2 + scripts/release/finalize_release.py | 432 +++++++++++++++++++++++ scripts/release/test_finalize_release.py | 106 ++++++ torch_pin.py | 3 + 6 files changed, 562 insertions(+), 3 deletions(-) create mode 100644 scripts/release/finalize_release.py create mode 100644 scripts/release/test_finalize_release.py diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index b9c721926fa..b9af511e900 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -71,15 +71,29 @@ def test_release_pytorch_requirement() -> None: os.environ.get("GITHUB_BASE_REF", ""), os.environ.get("GITHUB_REF", ""), ) - release_ci = any( + release_tag_ci = any( + re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+", ref) for ref in ci_refs + ) + release_ci = release_tag_ci or any( ref.startswith("release/") - or re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+(?:-rc\d+)?", ref) + or re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+-rc\d+", ref) for ref in ci_refs ) assert not release_ci or config["RELEASE_WHEEL"], ( "this wheel is being built from a release ref, but torch_pin.py does not enable " "release metadata; run scripts/release/apply-release-changes.sh" ) + dependencies_finalized = config.get("RELEASE_DEPENDENCIES_FINALIZED") is True + assert not release_tag_ci or dependencies_finalized, ( + "this wheel is being built from a release tag, but stable dependencies were not " + "finalized; run scripts/release/finalize_release.py before tagging" + ) + assert not dependencies_finalized or re.fullmatch( + r"\d+\.\d+\.\d+", config["TORCH_VERSION"] + ), ( + "release dependencies are marked finalized, but TORCH_VERSION is not a final " + f"PyTorch release: {config['TORCH_VERSION']}" + ) torch_requirements = [ Requirement(raw) for raw in metadata.requires("executorch") or [] diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7b6781faa6b..cdcb9e4ca55 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -108,7 +108,9 @@ jobs: lintrunner init - name: Test release preparation - run: python scripts/release/test_prepare_release.py + run: | + python scripts/release/test_prepare_release.py + python scripts/release/test_finalize_release.py - name: Validate release configuration if: startsWith(github.ref_name, 'release/') || startsWith(github.base_ref, 'release/') diff --git a/scripts/release/apply-release-changes.sh b/scripts/release/apply-release-changes.sh index 991798b352e..8c5cc7851d7 100755 --- a/scripts/release/apply-release-changes.sh +++ b/scripts/release/apply-release-changes.sh @@ -51,3 +51,5 @@ fi "${PYTHON_EXECUTABLE}" "${GIT_TOP_DIR}/scripts/release/prepare_release.py" "${PREPARE_ARGS[@]}" echo "Release changes are ready for review. This script does not commit or push them." +echo "When stable dependency artifacts are published, run:" +echo " python scripts/release/finalize_release.py" diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py new file mode 100644 index 00000000000..fba2fadbb94 --- /dev/null +++ b/scripts/release/finalize_release.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Finalize release dependencies after their stable artifacts are published.""" + +import argparse +import json +import re +import runpy +import subprocess +import urllib.error +import urllib.request +from pathlib import Path +from typing import Iterable + +from prepare_release import ( + _release_version_from_file, + _TORCH_VERSION_PATTERN, + _write_if_changed, +) + +_VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") +_TORCHCODEC_PATTERN = re.compile(r"torchcodec==\d+\.\d+\.\d+") +_TOKENIZERS_REQUIREMENT_PATTERN = re.compile( + r'(?m)^(?P\s*)"pytorch-tokenizers(?:>=\d+\.\d+\.\d+)?",$' +) +_TORCHAO_ASSIGNMENT_PATTERN = re.compile( + r'^TORCHAO_NIGHTLY_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE +) +_ROCM_TORCHAO_ASSIGNMENT_PATTERN = re.compile( + r'^ROCM_TORCHAO_NIGHTLY_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE +) +_ROCM_VERSION_PATTERN = re.compile( + r'^ROCM_VERSION="\$\{ROCM_VERSION:-(?P\d+\.\d+)\}"$', re.MULTILINE +) +_FINALIZED_PATTERN = re.compile( + r"^RELEASE_DEPENDENCIES_FINALIZED\s*=\s*(?:True|False)$", re.MULTILINE +) +_TORCHCODEC_PATHS = ( + ".ci/scripts/test-rocm-voxtral.sh", + ".ci/scripts/test_model_e2e.sh", + "examples/models/moshi/mimi/install_requirements.sh", +) +_ROCM_PATHS = ( + ".ci/scripts/test-rocm-aoti.sh", + ".ci/scripts/test-rocm-voxtral.sh", +) +_QNN_TEST_INDEX = ( + '"$PIPBIN" install torch=="${TORCH_VERSION}" ' + '--extra-index-url "https://download.pytorch.org/whl/test"' +) +_QNN_STABLE_INDEX = ( + '"$PIPBIN" install --no-cache-dir torch=="${TORCH_VERSION}" ' + '--index-url "https://download.pytorch.org/whl/cpu"' +) +_SUBMODULE_RELEASES = ( + ("extension/llm/tokenizers", "https://github.com/meta-pytorch/tokenizers.git"), + ("third-party/ao", "https://github.com/pytorch/ao.git"), +) + + +def stable_base_version(version: str) -> str: + """Convert a development or prerelease version to its final base version.""" + match = re.match(r"(\d+\.\d+\.\d+)", version) + if match is None: + raise RuntimeError(f"could not derive a stable version from {version!r}") + return match.group(1) + + +def current_torchao_version(install_requirements_path: Path) -> str: + """Read the generic TorchAO pin without importing the installer module.""" + match = _TORCHAO_ASSIGNMENT_PATTERN.search(install_requirements_path.read_text()) + if match is None: + raise RuntimeError( + f"could not find TORCHAO_NIGHTLY_VERSION in {install_requirements_path}" + ) + return match.group("version") + + +def latest_pypi_version(package: str) -> str: + """Return the latest stable version published for a package.""" + with urllib.request.urlopen( + f"https://pypi.org/pypi/{package}/json", timeout=30 + ) as response: + version = json.load(response)["info"]["version"] + if _VERSION_PATTERN.fullmatch(version) is None: + raise RuntimeError(f"latest {package} version is not stable: {version!r}") + return version + + +def require_pypi_release(package: str, version: str) -> None: + """Fail before mutation if an expected stable package is unavailable.""" + if _VERSION_PATTERN.fullmatch(version) is None: + raise RuntimeError(f"invalid {package} version {version!r}") + try: + with urllib.request.urlopen( + f"https://pypi.org/pypi/{package}/{version}/json", timeout=30 + ): + pass + except urllib.error.HTTPError as error: + if error.code == 404: + raise RuntimeError( + f"{package}=={version} is not published on PyPI" + ) from error + raise + + +def require_remote_tag(repository: str, version: str) -> None: + """Fail before mutation if a matching submodule release tag is unavailable.""" + result = subprocess.run( + [ + "git", + "ls-remote", + "--exit-code", + "--tags", + repository, + f"refs/tags/v{version}", + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + raise RuntimeError(f"{repository} does not have tag v{version}") + + +def require_rocm_torchao_release(repo_root: Path, version: str) -> None: + """Verify the stable ROCm TorchAO artifact used by release CI exists.""" + rocm_script = repo_root / _ROCM_PATHS[0] + match = _ROCM_VERSION_PATTERN.search(rocm_script.read_text()) + if match is None: + raise RuntimeError(f"could not determine ROCm version from {rocm_script}") + rocm_version = match.group("version") + url = ( + f"https://download.pytorch.org/whl/rocm{rocm_version}/" + f"torchao-{version}%2Brocm{rocm_version}-" + "cp310-abi3-manylinux_2_28_x86_64.whl" + ) + try: + with urllib.request.urlopen(url, timeout=30): + pass + except urllib.error.HTTPError as error: + if error.code == 404: + raise RuntimeError( + f"TorchAO {version} is not published for ROCm {rocm_version}" + ) from error + raise + + +def planned_torch_release(torch_pin_path: Path, torch_version: str) -> str: + """Plan the RC-to-final pin update without changing the checkout.""" + if _VERSION_PATTERN.fullmatch(torch_version) is None: + raise RuntimeError( + f"final PyTorch version must be X.Y.Z, got {torch_version!r}" + ) + text = torch_pin_path.read_text() + updated, version_count = _TORCH_VERSION_PATTERN.subn( + f'TORCH_VERSION = "{torch_version}"', text + ) + if version_count != 1: + raise RuntimeError(f"expected one TORCH_VERSION assignment in {torch_pin_path}") + updated, count = _FINALIZED_PATTERN.subn( + "RELEASE_DEPENDENCIES_FINALIZED = True", updated + ) + if count != 1: + raise RuntimeError( + f"expected one RELEASE_DEPENDENCIES_FINALIZED assignment in {torch_pin_path}" + ) + return updated + + +def finalize_torch_release(torch_pin_path: Path, torch_version: str) -> int: + """Replace the RC pin with the final release and mark finalization complete.""" + updated = planned_torch_release(torch_pin_path, torch_version) + return int(_write_if_changed(torch_pin_path, updated)) + + +def plan_dependency_text( + repo_root: Path, + torchao_version: str, + tokenizers_version: str, + torchcodec_version: str, +) -> dict[Path, str]: + """Validate and calculate every text edit without changing the checkout.""" + updates: dict[Path, str] = {} + + def read(path: Path) -> str: + if not path.exists(): + raise RuntimeError(f"expected release dependency file {path}") + return updates.get(path, path.read_text()) + + path = repo_root / "install_requirements.py" + updated, count = _TORCHAO_ASSIGNMENT_PATTERN.subn( + f'TORCHAO_NIGHTLY_VERSION = "{torchao_version}"', read(path) + ) + if count != 1: + raise RuntimeError(f"expected one TORCHAO_NIGHTLY_VERSION in {path}") + updated, count = _ROCM_TORCHAO_ASSIGNMENT_PATTERN.subn( + f'ROCM_TORCHAO_NIGHTLY_VERSION = "{torchao_version}"', updated + ) + if count != 1: + raise RuntimeError(f"expected one ROCM_TORCHAO_NIGHTLY_VERSION in {path}") + updates[path] = updated + + path = repo_root / "setup.py" + updated, count = _TOKENIZERS_REQUIREMENT_PATTERN.subn( + rf'\g"pytorch-tokenizers>={tokenizers_version}",', read(path) + ) + if count != 1: + raise RuntimeError(f"expected one pytorch-tokenizers requirement in {path}") + updates[path] = updated + + for relative in _TORCHCODEC_PATHS: + path = repo_root / relative + updated, count = _TORCHCODEC_PATTERN.subn( + f"torchcodec=={torchcodec_version}", read(path) + ) + if count == 0: + raise RuntimeError(f"expected a torchcodec pin in {path}") + updates[path] = updated.replace( + "--extra-index-url https://download.pytorch.org/whl/test/cpu", + "--index-url https://download.pytorch.org/whl/cpu", + ) + + for relative in _ROCM_PATHS: + path = repo_root / relative + updates[path] = ( + read(path) + .replace( + "https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}", + "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", + ) + .replace( + "https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}", + "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", + ) + ) + + qnn_path = repo_root / ".ci/scripts/test_wheel_package_qnn.sh" + updated = read(qnn_path).replace(_QNN_TEST_INDEX, _QNN_STABLE_INDEX) + if _QNN_STABLE_INDEX not in updated: + raise RuntimeError( + f"could not find the QNN Torch install command in {qnn_path}" + ) + updates[qnn_path] = updated + return {path: text for path, text in updates.items() if text != path.read_text()} + + +def finalize_dependency_text( + repo_root: Path, + torchao_version: str, + tokenizers_version: str, + torchcodec_version: str, +) -> int: + """Update all recurring stable dependency declarations and index URLs.""" + updates = plan_dependency_text( + repo_root, torchao_version, tokenizers_version, torchcodec_version + ) + for path, updated in updates.items(): + _write_if_changed(path, updated) + return len(updates) + + +def prepare_submodule_tags( + repo_root: Path, releases: Iterable[tuple[str, str, str]] +) -> list[tuple[Path, str, str]]: + """Fetch and resolve every submodule tag before changing any gitlink.""" + prepared = [] + for relative, _repository, version in releases: + path = repo_root / relative + if (path / ".git").exists() and subprocess.run( + ["git", "status", "--porcelain"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout: + raise RuntimeError(f"submodule {path} has local changes") + subprocess.run( + ["git", "submodule", "update", "--init", "--", relative], + cwd=repo_root, + check=True, + ) + if subprocess.run( + ["git", "status", "--porcelain"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout: + raise RuntimeError(f"submodule {path} has local changes") + subprocess.run( + ["git", "fetch", "--depth=1", "origin", "tag", f"v{version}"], + cwd=path, + check=True, + ) + current = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + target = subprocess.run( + ["git", "rev-parse", f"v{version}^{{commit}}"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + prepared.append((path, current, target)) + return prepared + + +def update_submodule_tags(prepared: Iterable[tuple[Path, str, str]]) -> None: + """Apply prepared submodule updates, restoring prior heads on failure.""" + changed = [] + try: + for path, current, target in prepared: + subprocess.run( + ["git", "checkout", "--detach", target], cwd=path, check=True + ) + changed.append((path, current)) + except Exception: + for path, current in reversed(changed): + subprocess.run( + ["git", "checkout", "--detach", current], cwd=path, check=True + ) + raise + + +def main() -> None: + """Preflight and apply stable dependency finalization.""" + parser = argparse.ArgumentParser() + parser.add_argument("--release-version", help="release version in MAJOR.MINOR form") + parser.add_argument("--torch-version", help="final stable PyTorch version") + parser.add_argument("--torchao-version", help="stable TorchAO version") + parser.add_argument( + "--tokenizers-version", help="stable pytorch-tokenizers version" + ) + parser.add_argument("--torchcodec-version", help="stable TorchCodec version") + parser.add_argument( + "--preflight-only", + action="store_true", + help="verify packages and tags without changing the checkout", + ) + parser.add_argument( + "--repo-root", type=Path, default=Path(__file__).resolve().parents[2] + ) + args = parser.parse_args() + + repo_root = args.repo_root.resolve() + release_version = args.release_version or _release_version_from_file(repo_root) + release_config = runpy.run_path(str(repo_root / "torch_pin.py")) + if release_config.get("RELEASE_WHEEL") is not True: + raise RuntimeError( + "release preparation has not run; use cut-release-branch.sh first" + ) + torch_version = args.torch_version or stable_base_version( + release_config["TORCH_VERSION"] + ) + torchao_version = args.torchao_version or stable_base_version( + current_torchao_version(repo_root / "install_requirements.py") + ) + tokenizers_version = args.tokenizers_version or f"{release_version}.0" + torchcodec_version = args.torchcodec_version or latest_pypi_version("torchcodec") + + releases = ( + ("extension/llm/tokenizers", _SUBMODULE_RELEASES[0][1], tokenizers_version), + ("third-party/ao", _SUBMODULE_RELEASES[1][1], torchao_version), + ) + for package, version in ( + ("torch", torch_version), + ("torchao", torchao_version), + ("pytorch-tokenizers", tokenizers_version), + ("torchcodec", torchcodec_version), + ): + require_pypi_release(package, version) + for _path, repository, version in releases: + require_remote_tag(repository, version) + require_rocm_torchao_release(repo_root, torchao_version) + + print( + "Release dependency artifacts are available: " + f"torch=={torch_version}, torchao=={torchao_version}, " + f"pytorch-tokenizers=={tokenizers_version}, torchcodec=={torchcodec_version}" + ) + if args.preflight_only: + return + + dependency_updates = plan_dependency_text( + repo_root, torchao_version, tokenizers_version, torchcodec_version + ) + torch_pin_path = repo_root / "torch_pin.py" + torch_pin_update = planned_torch_release(torch_pin_path, torch_version) + prepared_submodules = prepare_submodule_tags(repo_root, releases) + text_updates = {**dependency_updates, torch_pin_path: torch_pin_update} + original_text = {path: path.read_text() for path in text_updates} + + try: + update_submodule_tags(prepared_submodules) + for path, updated in text_updates.items(): + _write_if_changed(path, updated) + except Exception: + for path, original in original_text.items(): + path.write_text(original) + for path, original, _target in reversed(prepared_submodules): + subprocess.run( + ["git", "checkout", "--detach", original], cwd=path, check=False + ) + raise + + changed = sum( + updated != original_text[path] for path, updated in text_updates.items() + ) + print( + f"Finalized release dependencies; changed {changed} text files and 2 submodules" + ) + print( + "Review and stage the resulting release-only changes with git add " + "install_requirements.py setup.py torch_pin.py .ci/scripts/ " + "examples/models/moshi/mimi/install_requirements.sh " + "extension/llm/tokenizers third-party/ao" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py new file mode 100644 index 00000000000..7d8ba9791a2 --- /dev/null +++ b/scripts/release/test_finalize_release.py @@ -0,0 +1,106 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import tempfile +import unittest +from pathlib import Path + +from finalize_release import ( + finalize_dependency_text, + finalize_torch_release, + plan_dependency_text, + stable_base_version, +) + + +class FinalizeReleaseTest(unittest.TestCase): + """Tests for delayed release dependency finalization.""" + + def test_promotes_release_candidate_to_final(self) -> None: + self.assertEqual(stable_base_version("0.19.0.dev20260907"), "0.19.0") + self.assertEqual(stable_base_version("0.19.0rc2"), "0.19.0") + self.assertEqual(stable_base_version("0.19.0"), "0.19.0") + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "torch_pin.py" + path.write_text( + 'TORCH_VERSION = "2.15.0rc2"\n' + "RELEASE_WHEEL = True\n" + "RELEASE_DEPENDENCIES_FINALIZED = False\n" + ) + + self.assertEqual(finalize_torch_release(path, "2.15.0"), 1) + self.assertIn('TORCH_VERSION = "2.15.0"', path.read_text()) + self.assertIn("RELEASE_DEPENDENCIES_FINALIZED = True", path.read_text()) + + def test_finalizes_dependency_text(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "install_requirements.py").write_text( + 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' + 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260805"\n' + ) + (root / "setup.py").write_text(' "pytorch-tokenizers",\n') + qnn_path = root / ".ci/scripts/test_wheel_package_qnn.sh" + qnn_path.parent.mkdir(parents=True) + qnn_path.write_text( + '"$PIPBIN" install torch=="${TORCH_VERSION}" ' + '--extra-index-url "https://download.pytorch.org/whl/test"\n' + ) + for relative in ( + ".ci/scripts/test-rocm-aoti.sh", + ".ci/scripts/test-rocm-voxtral.sh", + ".ci/scripts/test_model_e2e.sh", + "examples/models/moshi/mimi/install_requirements.sh", + ): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "pip install torchcodec==0.11.0 " + "--extra-index-url https://download.pytorch.org/whl/test/cpu\n" + 'PYTORCH_ROCM_INDEX="https://download.pytorch.org/whl/test/' + 'rocm${ROCM_VERSION}"\n' + 'TORCHAO_ROCM_WHEEL_BASE="https://download.pytorch.org/whl/' + 'nightly/rocm${ROCM_VERSION}"\n' + ) + + planned = plan_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") + self.assertIn( + 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"', + (root / "install_requirements.py").read_text(), + ) + self.assertEqual(len(planned), 7) + + changed = finalize_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") + self.assertEqual(changed, 7) + self.assertIn( + 'TORCHAO_NIGHTLY_VERSION = "0.19.0"', + (root / "install_requirements.py").read_text(), + ) + self.assertIn( + 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0"', + (root / "install_requirements.py").read_text(), + ) + self.assertIn( + '"pytorch-tokenizers>=1.6.0"', (root / "setup.py").read_text() + ) + for relative in ( + ".ci/scripts/test-rocm-voxtral.sh", + ".ci/scripts/test_model_e2e.sh", + "examples/models/moshi/mimi/install_requirements.sh", + ): + text = (root / relative).read_text() + self.assertIn("torchcodec==0.17.0", text) + self.assertNotIn("whl/test/cpu", text) + rocm = (root / ".ci/scripts/test-rocm-voxtral.sh").read_text() + self.assertNotIn("whl/test/rocm", rocm) + self.assertNotIn("whl/nightly/rocm", rocm) + self.assertIn("--no-cache-dir", qnn_path.read_text()) + self.assertIn("--index-url", qnn_path.read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/torch_pin.py b/torch_pin.py index 58660177313..1d55e68e1f5 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -9,3 +9,6 @@ # Changed to True by scripts/release/apply-release-changes.sh. Release wheels # declare the PyTorch release above; development and minimal wheels do not. RELEASE_WHEEL = False + +# Changed to True after stable third-party artifacts and submodule tags exist. +RELEASE_DEPENDENCIES_FINALIZED = False From 6959b1496637f3af351cf2c659a261fb1831c46f Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 09:22:47 -0700 Subject: [PATCH 2/8] Fix release finalizer mypy imports --- scripts/release/finalize_release.py | 2 +- scripts/release/test_finalize_release.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index fba2fadbb94..9fa6be89afe 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -17,7 +17,7 @@ from pathlib import Path from typing import Iterable -from prepare_release import ( +from prepare_release import ( # type: ignore[import-not-found] _release_version_from_file, _TORCH_VERSION_PATTERN, _write_if_changed, diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py index 7d8ba9791a2..5f70fcf92d3 100644 --- a/scripts/release/test_finalize_release.py +++ b/scripts/release/test_finalize_release.py @@ -8,7 +8,7 @@ import unittest from pathlib import Path -from finalize_release import ( +from finalize_release import ( # type: ignore[import-not-found] finalize_dependency_text, finalize_torch_release, plan_dependency_text, From 885cd3061b66380dec7cd603da152e7ec8c6529b Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 09:37:34 -0700 Subject: [PATCH 3/8] Finalize PyTorch companion releases --- .github/workflows/lint.yml | 3 +- scripts/release/finalize_release.py | 36 +++++++++++++++++++++++- scripts/release/test_finalize_release.py | 4 +++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index cdcb9e4ca55..9b88482fa04 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -51,11 +51,12 @@ jobs: TORCH_VERSION=$(python -c "from torch_pin import TORCH_VERSION; print(TORCH_VERSION)") TORCHVISION_VERSION=$(python -c "from torch_pin import TORCHVISION_VERSION; print(TORCHVISION_VERSION)") TORCHAUDIO_VERSION=$(python -c "from torch_pin import TORCHAUDIO_VERSION; print(TORCHAUDIO_VERSION)") + TORCH_INDEX=$(python -c "from torch_pin import RELEASE_WHEEL, RELEASE_DEPENDENCIES_FINALIZED; print('https://download.pytorch.org/whl/test/cpu' if RELEASE_WHEEL and not RELEASE_DEPENDENCIES_FINALIZED else 'https://download.pytorch.org/whl/cpu')") pip install \ "torch==${TORCH_VERSION}" \ "torchvision==${TORCHVISION_VERSION}" \ "torchaudio==${TORCHAUDIO_VERSION}" \ - --index-url "https://download.pytorch.org/whl/test/cpu" + --index-url "${TORCH_INDEX}" pip install lintrunner==0.12.7 lintrunner-adapters==0.14.1 pip install -r requirements-lintrunner.txt USE_CPP=0 pip install --no-build-isolation third-party/ao diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index 9fa6be89afe..886a7312835 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -24,6 +24,12 @@ ) _VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") +_TORCHVISION_VERSION_PATTERN = re.compile( + r'^TORCHVISION_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE +) +_TORCHAUDIO_VERSION_PATTERN = re.compile( + r'^TORCHAUDIO_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE +) _TORCHCODEC_PATTERN = re.compile(r"torchcodec==\d+\.\d+\.\d+") _TOKENIZERS_REQUIREMENT_PATTERN = re.compile( r'(?m)^(?P\s*)"pytorch-tokenizers(?:>=\d+\.\d+\.\d+)?",$' @@ -163,6 +169,29 @@ def planned_torch_release(torch_pin_path: Path, torch_version: str) -> str: ) if version_count != 1: raise RuntimeError(f"expected one TORCH_VERSION assignment in {torch_pin_path}") + match = re.fullmatch(r"\d+\.(\d+)\.(\d+)", torch_version) + assert match is not None + expected_companions = { + _TORCHVISION_VERSION_PATTERN: f"0.{int(match.group(1)) + 15}.{match.group(2)}", + _TORCHAUDIO_VERSION_PATTERN: torch_version, + } + for pattern, expected in expected_companions.items(): + current = pattern.search(updated) + if current is None: + raise RuntimeError( + f"expected companion version assignment in {torch_pin_path}" + ) + if stable_base_version(current.group("version")) != expected: + raise RuntimeError( + f"companion release {current.group('version')!r} does not match " + f"Torch {torch_version}" + ) + updated = pattern.sub( + lambda version_match: version_match.group(0).replace( + version_match.group("version"), expected + ), + updated, + ) updated, count = _FINALIZED_PATTERN.subn( "RELEASE_DEPENDENCIES_FINALIZED = True", updated ) @@ -363,6 +392,8 @@ def main() -> None: torch_version = args.torch_version or stable_base_version( release_config["TORCH_VERSION"] ) + torchvision_version = stable_base_version(release_config["TORCHVISION_VERSION"]) + torchaudio_version = stable_base_version(release_config["TORCHAUDIO_VERSION"]) torchao_version = args.torchao_version or stable_base_version( current_torchao_version(repo_root / "install_requirements.py") ) @@ -375,6 +406,8 @@ def main() -> None: ) for package, version in ( ("torch", torch_version), + ("torchvision", torchvision_version), + ("torchaudio", torchaudio_version), ("torchao", torchao_version), ("pytorch-tokenizers", tokenizers_version), ("torchcodec", torchcodec_version), @@ -386,7 +419,8 @@ def main() -> None: print( "Release dependency artifacts are available: " - f"torch=={torch_version}, torchao=={torchao_version}, " + f"torch=={torch_version}, torchvision=={torchvision_version}, " + f"torchaudio=={torchaudio_version}, torchao=={torchao_version}, " f"pytorch-tokenizers=={tokenizers_version}, torchcodec=={torchcodec_version}" ) if args.preflight_only: diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py index 5f70fcf92d3..210eb3e8da7 100644 --- a/scripts/release/test_finalize_release.py +++ b/scripts/release/test_finalize_release.py @@ -28,12 +28,16 @@ def test_promotes_release_candidate_to_final(self) -> None: path = Path(directory) / "torch_pin.py" path.write_text( 'TORCH_VERSION = "2.15.0rc2"\n' + 'TORCHVISION_VERSION = "0.30.0rc2"\n' + 'TORCHAUDIO_VERSION = "2.15.0rc2"\n' "RELEASE_WHEEL = True\n" "RELEASE_DEPENDENCIES_FINALIZED = False\n" ) self.assertEqual(finalize_torch_release(path, "2.15.0"), 1) self.assertIn('TORCH_VERSION = "2.15.0"', path.read_text()) + self.assertIn('TORCHVISION_VERSION = "0.30.0"', path.read_text()) + self.assertIn('TORCHAUDIO_VERSION = "2.15.0"', path.read_text()) self.assertIn("RELEASE_DEPENDENCIES_FINALIZED = True", path.read_text()) def test_finalizes_dependency_text(self) -> None: From 768337d3a0b732a94c25b66d5aa042bba5be2fde Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 13:14:35 -0700 Subject: [PATCH 4/8] Keep final release pins on matching trains --- scripts/release/finalize_release.py | 65 +++++++++++++++--------- scripts/release/test_finalize_release.py | 4 +- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index 886a7312835..3e08df1e344 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -21,7 +21,10 @@ _release_version_from_file, _TORCH_VERSION_PATTERN, _write_if_changed, + pytorch_commit_for_release, + sync_pytorch_source, ) +from release_versions import torchvision_version # type: ignore[import-not-found] _VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") _TORCHVISION_VERSION_PATTERN = re.compile( @@ -169,29 +172,28 @@ def planned_torch_release(torch_pin_path: Path, torch_version: str) -> str: ) if version_count != 1: raise RuntimeError(f"expected one TORCH_VERSION assignment in {torch_pin_path}") - match = re.fullmatch(r"\d+\.(\d+)\.(\d+)", torch_version) - assert match is not None - expected_companions = { - _TORCHVISION_VERSION_PATTERN: f"0.{int(match.group(1)) + 15}.{match.group(2)}", - _TORCHAUDIO_VERSION_PATTERN: torch_version, - } - for pattern, expected in expected_companions.items(): - current = pattern.search(updated) - if current is None: - raise RuntimeError( - f"expected companion version assignment in {torch_pin_path}" - ) - if stable_base_version(current.group("version")) != expected: - raise RuntimeError( - f"companion release {current.group('version')!r} does not match " - f"Torch {torch_version}" - ) - updated = pattern.sub( - lambda version_match: version_match.group(0).replace( - version_match.group("version"), expected - ), - updated, + vision = _TORCHVISION_VERSION_PATTERN.search(updated) + if vision is None: + raise RuntimeError(f"expected torchvision assignment in {torch_pin_path}") + expected_vision = torchvision_version(torch_version) + if stable_base_version(vision.group("version")) != expected_vision: + raise RuntimeError( + f"torchvision release {vision.group('version')!r} does not match " + f"Torch {torch_version}" ) + updated = _TORCHVISION_VERSION_PATTERN.sub( + lambda match: match.group(0).replace(match.group("version"), expected_vision), + updated, + ) + + audio = _TORCHAUDIO_VERSION_PATTERN.search(updated) + if audio is None: + raise RuntimeError(f"expected torchaudio assignment in {torch_pin_path}") + expected_audio = stable_base_version(audio.group("version")) + updated = _TORCHAUDIO_VERSION_PATTERN.sub( + lambda match: match.group(0).replace(match.group("version"), expected_audio), + updated, + ) updated, count = _FINALIZED_PATTERN.subn( "RELEASE_DEPENDENCIES_FINALIZED = True", updated ) @@ -399,6 +401,7 @@ def main() -> None: ) tokenizers_version = args.tokenizers_version or f"{release_version}.0" torchcodec_version = args.torchcodec_version or latest_pypi_version("torchcodec") + pytorch_commit = pytorch_commit_for_release(torch_version) releases = ( ("extension/llm/tokenizers", _SUBMODULE_RELEASES[0][1], tokenizers_version), @@ -434,14 +437,26 @@ def main() -> None: prepared_submodules = prepare_submodule_tags(repo_root, releases) text_updates = {**dependency_updates, torch_pin_path: torch_pin_update} original_text = {path: path.read_text() for path in text_updates} + source_paths = [repo_root / ".ci/docker/ci_commit_pins/pytorch.txt"] + for relative in ( + "runtime/core/portable_type/c10/c10", + "runtime/core/portable_type/c10/torch/headeronly", + ): + source_paths.extend( + path for path in (repo_root / relative).rglob("*") if path.is_file() + ) + original_source = {path: path.read_bytes() for path in source_paths} try: + sync_pytorch_source(repo_root, pytorch_commit) update_submodule_tags(prepared_submodules) for path, updated in text_updates.items(): _write_if_changed(path, updated) except Exception: for path, original in original_text.items(): path.write_text(original) + for path, original in original_source.items(): + path.write_bytes(original) for path, original, _target in reversed(prepared_submodules): subprocess.run( ["git", "checkout", "--detach", original], cwd=path, check=False @@ -451,8 +466,12 @@ def main() -> None: changed = sum( updated != original_text[path] for path, updated in text_updates.items() ) + changed_source = sum( + path.read_bytes() != original for path, original in original_source.items() + ) print( - f"Finalized release dependencies; changed {changed} text files and 2 submodules" + f"Finalized release dependencies; changed {changed} text files, " + f"{changed_source} PyTorch source files, and 2 submodules" ) print( "Review and stage the resulting release-only changes with git add " diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py index 210eb3e8da7..4b9cbfa4915 100644 --- a/scripts/release/test_finalize_release.py +++ b/scripts/release/test_finalize_release.py @@ -29,7 +29,7 @@ def test_promotes_release_candidate_to_final(self) -> None: path.write_text( 'TORCH_VERSION = "2.15.0rc2"\n' 'TORCHVISION_VERSION = "0.30.0rc2"\n' - 'TORCHAUDIO_VERSION = "2.15.0rc2"\n' + 'TORCHAUDIO_VERSION = "2.11.0rc2"\n' "RELEASE_WHEEL = True\n" "RELEASE_DEPENDENCIES_FINALIZED = False\n" ) @@ -37,7 +37,7 @@ def test_promotes_release_candidate_to_final(self) -> None: self.assertEqual(finalize_torch_release(path, "2.15.0"), 1) self.assertIn('TORCH_VERSION = "2.15.0"', path.read_text()) self.assertIn('TORCHVISION_VERSION = "0.30.0"', path.read_text()) - self.assertIn('TORCHAUDIO_VERSION = "2.15.0"', path.read_text()) + self.assertIn('TORCHAUDIO_VERSION = "2.11.0"', path.read_text()) self.assertIn("RELEASE_DEPENDENCIES_FINALIZED = True", path.read_text()) def test_finalizes_dependency_text(self) -> None: From 24549c7ef3d4c153ea2292aab7707719925c8a14 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 13:16:09 -0700 Subject: [PATCH 5/8] List finalized PyTorch source files --- scripts/release/finalize_release.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index 3e08df1e344..a94ff2cf329 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -476,6 +476,7 @@ def main() -> None: print( "Review and stage the resulting release-only changes with git add " "install_requirements.py setup.py torch_pin.py .ci/scripts/ " + ".ci/docker/ci_commit_pins/pytorch.txt runtime/core/portable_type/c10/ " "examples/models/moshi/mimi/install_requirements.sh " "extension/llm/tokenizers third-party/ao" ) From dc846a3acf77d2b88a492e1dd1347a417dd82301 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 14:50:45 -0700 Subject: [PATCH 6/8] Finalize release indexes and source tag --- scripts/release/finalize_release.py | 82 +++++++++++++++--------- scripts/release/prepare_release.py | 13 ++-- scripts/release/release_versions.py | 9 +++ scripts/release/test_finalize_release.py | 5 ++ 4 files changed, 72 insertions(+), 37 deletions(-) diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index a94ff2cf329..df7c5addd3c 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -15,18 +15,21 @@ import urllib.error import urllib.request from pathlib import Path -from typing import Iterable +from typing import Iterable, Match from prepare_release import ( # type: ignore[import-not-found] _release_version_from_file, + _test_index_versions, _TORCH_VERSION_PATTERN, _write_if_changed, pytorch_commit_for_release, sync_pytorch_source, ) -from release_versions import torchvision_version # type: ignore[import-not-found] +from release_versions import ( # type: ignore[import-not-found] + is_release_version, + release_base_version, +) -_VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") _TORCHVISION_VERSION_PATTERN = re.compile( r'^TORCHVISION_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE ) @@ -46,6 +49,10 @@ _ROCM_VERSION_PATTERN = re.compile( r'^ROCM_VERSION="\$\{ROCM_VERSION:-(?P\d+\.\d+)\}"$', re.MULTILINE ) +_TORCH_URL_BASE_PATTERN = re.compile( + r'^TORCH_URL_BASE\s*=\s*"https://download\.pytorch\.org/whl(?:/test)?"$', + re.MULTILINE, +) _FINALIZED_PATTERN = re.compile( r"^RELEASE_DEPENDENCIES_FINALIZED\s*=\s*(?:True|False)$", re.MULTILINE ) @@ -74,10 +81,7 @@ def stable_base_version(version: str) -> str: """Convert a development or prerelease version to its final base version.""" - match = re.match(r"(\d+\.\d+\.\d+)", version) - if match is None: - raise RuntimeError(f"could not derive a stable version from {version!r}") - return match.group(1) + return release_base_version(version) def current_torchao_version(install_requirements_path: Path) -> str: @@ -96,14 +100,14 @@ def latest_pypi_version(package: str) -> str: f"https://pypi.org/pypi/{package}/json", timeout=30 ) as response: version = json.load(response)["info"]["version"] - if _VERSION_PATTERN.fullmatch(version) is None: + if not is_release_version(version, allow_prerelease=False): raise RuntimeError(f"latest {package} version is not stable: {version!r}") return version def require_pypi_release(package: str, version: str) -> None: """Fail before mutation if an expected stable package is unavailable.""" - if _VERSION_PATTERN.fullmatch(version) is None: + if not is_release_version(version, allow_prerelease=False): raise RuntimeError(f"invalid {package} version {version!r}") try: with urllib.request.urlopen( @@ -118,6 +122,19 @@ def require_pypi_release(package: str, version: str) -> None: raise +def require_stable_cuda_releases(versions: Iterable[tuple[str, str]]) -> None: + """Fail unless every final cu134 package is on the stable index.""" + missing = [ + f"{package}=={version}+cu134" + for package, version in versions + if version not in _test_index_versions(package, "cu134", channel="") + ] + if missing: + raise RuntimeError( + "stable PyTorch cu134 releases are unavailable: " + ", ".join(missing) + ) + + def require_remote_tag(repository: str, version: str) -> None: """Fail before mutation if a matching submodule release tag is unavailable.""" result = subprocess.run( @@ -162,7 +179,7 @@ def require_rocm_torchao_release(repo_root: Path, version: str) -> None: def planned_torch_release(torch_pin_path: Path, torch_version: str) -> str: """Plan the RC-to-final pin update without changing the checkout.""" - if _VERSION_PATTERN.fullmatch(torch_version) is None: + if not is_release_version(torch_version, allow_prerelease=False): raise RuntimeError( f"final PyTorch version must be X.Y.Z, got {torch_version!r}" ) @@ -172,28 +189,19 @@ def planned_torch_release(torch_pin_path: Path, torch_version: str) -> str: ) if version_count != 1: raise RuntimeError(f"expected one TORCH_VERSION assignment in {torch_pin_path}") - vision = _TORCHVISION_VERSION_PATTERN.search(updated) - if vision is None: - raise RuntimeError(f"expected torchvision assignment in {torch_pin_path}") - expected_vision = torchvision_version(torch_version) - if stable_base_version(vision.group("version")) != expected_vision: - raise RuntimeError( - f"torchvision release {vision.group('version')!r} does not match " - f"Torch {torch_version}" - ) - updated = _TORCHVISION_VERSION_PATTERN.sub( - lambda match: match.group(0).replace(match.group("version"), expected_vision), - updated, - ) + for pattern, package in ( + (_TORCHVISION_VERSION_PATTERN, "torchvision"), + (_TORCHAUDIO_VERSION_PATTERN, "torchaudio"), + ): + current = pattern.search(updated) + if current is None: + raise RuntimeError(f"expected {package} assignment in {torch_pin_path}") + expected = stable_base_version(current.group("version")) - audio = _TORCHAUDIO_VERSION_PATTERN.search(updated) - if audio is None: - raise RuntimeError(f"expected torchaudio assignment in {torch_pin_path}") - expected_audio = stable_base_version(audio.group("version")) - updated = _TORCHAUDIO_VERSION_PATTERN.sub( - lambda match: match.group(0).replace(match.group("version"), expected_audio), - updated, - ) + def replace_version(match: Match[str], replacement: str = expected) -> str: + return match.group(0).replace(match.group("version"), replacement) + + updated = pattern.sub(replace_version, updated) updated, count = _FINALIZED_PATTERN.subn( "RELEASE_DEPENDENCIES_FINALIZED = True", updated ) @@ -235,6 +243,11 @@ def read(path: Path) -> str: ) if count != 1: raise RuntimeError(f"expected one ROCM_TORCHAO_NIGHTLY_VERSION in {path}") + updated, count = _TORCH_URL_BASE_PATTERN.subn( + 'TORCH_URL_BASE = "https://download.pytorch.org/whl"', updated + ) + if count != 1: + raise RuntimeError(f"expected one TORCH_URL_BASE in {path}") updates[path] = updated path = repo_root / "setup.py" @@ -416,6 +429,13 @@ def main() -> None: ("torchcodec", torchcodec_version), ): require_pypi_release(package, version) + require_stable_cuda_releases( + ( + ("torch", torch_version), + ("torchvision", torchvision_version), + ("torchaudio", torchaudio_version), + ) + ) for _path, repository, version in releases: require_remote_tag(repository, version) require_rocm_torchao_release(repo_root, torchao_version) diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index 0567e4c4482..73da01c5f4a 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -313,11 +313,10 @@ def newest_torch_test_release( @lru_cache(maxsize=None) def _test_index_wheels( - package: str, variant: str = "cpu" + package: str, variant: str = "cpu", channel: str = "test" ) -> dict[str, list[WheelLink]]: - index_url = ( - f"https://download.pytorch.org/whl/test/{variant}/{package}/" # @lint-ignore - ) + channel_path = f"{channel}/" if channel else "" + index_url = f"https://download.pytorch.org/whl/{channel_path}{variant}/{package}/" # @lint-ignore try: with urllib.request.urlopen(index_url, timeout=30) as response: index = response.read().decode() @@ -330,8 +329,10 @@ def _test_index_wheels( return parser.wheels -def _test_index_versions(package: str, variant: str = "cpu") -> set[str]: - return set(_test_index_wheels(package, variant)) +def _test_index_versions( + package: str, variant: str = "cpu", channel: str = "test" +) -> set[str]: + return set(_test_index_wheels(package, variant, channel)) @lru_cache(maxsize=None) diff --git a/scripts/release/release_versions.py b/scripts/release/release_versions.py index 4d3c0e31071..6a9656c6d53 100644 --- a/scripts/release/release_versions.py +++ b/scripts/release/release_versions.py @@ -29,6 +29,15 @@ def is_release_version(version: object, allow_prerelease: bool = True) -> bool: return allow_prerelease or stage is None +def release_base_version(version: str) -> str: + if ".dev" in version: + version, development = version.rsplit(".dev", 1) + if not development.isdigit(): + raise RuntimeError(f"invalid release version {version!r}") + major, minor, patch, _stage, _stage_number = release_parts(version) + return f"{major}.{minor}.{patch}" + + def release_key(version: str) -> tuple[int, int, int, int, int]: major, minor, patch, stage, stage_number = release_parts(version) stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py index 4b9cbfa4915..933d30fb081 100644 --- a/scripts/release/test_finalize_release.py +++ b/scripts/release/test_finalize_release.py @@ -44,6 +44,7 @@ def test_finalizes_dependency_text(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) (root / "install_requirements.py").write_text( + 'TORCH_URL_BASE = "https://download.pytorch.org/whl/test"\n' 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260805"\n' ) @@ -88,6 +89,10 @@ def test_finalizes_dependency_text(self) -> None: 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0"', (root / "install_requirements.py").read_text(), ) + self.assertIn( + 'TORCH_URL_BASE = "https://download.pytorch.org/whl"', + (root / "install_requirements.py").read_text(), + ) self.assertIn( '"pytorch-tokenizers>=1.6.0"', (root / "setup.py").read_text() ) From 5aa762e83e573849ac95e6b3728469afb1ffa47c Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 14:53:01 -0700 Subject: [PATCH 7/8] Skip generated index pattern in URL lint --- scripts/release/finalize_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index df7c5addd3c..b9e940a5d62 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -50,7 +50,7 @@ r'^ROCM_VERSION="\$\{ROCM_VERSION:-(?P\d+\.\d+)\}"$', re.MULTILINE ) _TORCH_URL_BASE_PATTERN = re.compile( - r'^TORCH_URL_BASE\s*=\s*"https://download\.pytorch\.org/whl(?:/test)?"$', + r'^TORCH_URL_BASE\s*=\s*"https://download\.pytorch\.org/whl(?:/test)?"$', # @lint-ignore re.MULTILINE, ) _FINALIZED_PATTERN = re.compile( From 3eb605eb66f1e370ca8c241593044d79497f82b9 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 15:17:20 -0700 Subject: [PATCH 8/8] Finalize from published stable wheels --- scripts/release/finalize_release.py | 43 ++++++++++++++++++------ scripts/release/prepare_release.py | 38 ++++++++++++++------- scripts/release/release_versions.py | 15 --------- scripts/release/test_finalize_release.py | 11 ++++-- 4 files changed, 66 insertions(+), 41 deletions(-) diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index b9e940a5d62..76e802edd97 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -22,12 +22,13 @@ _test_index_versions, _TORCH_VERSION_PATTERN, _write_if_changed, - pytorch_commit_for_release, + configured_cuda_variants, + pytorch_commit_for_wheel, sync_pytorch_source, ) from release_versions import ( # type: ignore[import-not-found] is_release_version, - release_base_version, + release_parts, ) _TORCHVISION_VERSION_PATTERN = re.compile( @@ -53,6 +54,10 @@ r'^TORCH_URL_BASE\s*=\s*"https://download\.pytorch\.org/whl(?:/test)?"$', # @lint-ignore re.MULTILINE, ) +_TORCHAO_URL_BASE_PATTERN = re.compile( + r'^TORCHAO_URL_BASE\s*=\s*"https://download\.pytorch\.org/whl(?:/test)?"$', # @lint-ignore + re.MULTILINE, +) _FINALIZED_PATTERN = re.compile( r"^RELEASE_DEPENDENCIES_FINALIZED\s*=\s*(?:True|False)$", re.MULTILINE ) @@ -81,7 +86,12 @@ def stable_base_version(version: str) -> str: """Convert a development or prerelease version to its final base version.""" - return release_base_version(version) + if ".dev" in version: + version, development = version.rsplit(".dev", 1) + if not development.isdigit(): + raise RuntimeError(f"invalid release version {version!r}") + major, minor, patch, _stage, _stage_number = release_parts(version) + return f"{major}.{minor}.{patch}" def current_torchao_version(install_requirements_path: Path) -> str: @@ -122,16 +132,20 @@ def require_pypi_release(package: str, version: str) -> None: raise -def require_stable_cuda_releases(versions: Iterable[tuple[str, str]]) -> None: - """Fail unless every final cu134 package is on the stable index.""" +def require_stable_cuda_releases( + versions: Iterable[tuple[str, str]], variants: Iterable[str] +) -> None: + """Fail unless every retained CUDA train is on the stable index.""" + releases = tuple(versions) missing = [ - f"{package}=={version}+cu134" - for package, version in versions - if version not in _test_index_versions(package, "cu134", channel="") + f"{package}=={version}+{variant}" + for variant in variants + for package, version in releases + if version not in _test_index_versions(package, variant, channel="") ] if missing: raise RuntimeError( - "stable PyTorch cu134 releases are unavailable: " + ", ".join(missing) + "stable CUDA releases are unavailable: " + ", ".join(missing) ) @@ -248,6 +262,11 @@ def read(path: Path) -> str: ) if count != 1: raise RuntimeError(f"expected one TORCH_URL_BASE in {path}") + updated, count = _TORCHAO_URL_BASE_PATTERN.subn( + 'TORCHAO_URL_BASE = "https://download.pytorch.org/whl"', updated + ) + if count != 1: + raise RuntimeError(f"expected one TORCHAO_URL_BASE in {path}") updates[path] = updated path = repo_root / "setup.py" @@ -414,7 +433,7 @@ def main() -> None: ) tokenizers_version = args.tokenizers_version or f"{release_version}.0" torchcodec_version = args.torchcodec_version or latest_pypi_version("torchcodec") - pytorch_commit = pytorch_commit_for_release(torch_version) + pytorch_commit = pytorch_commit_for_wheel(torch_version, channel="") releases = ( ("extension/llm/tokenizers", _SUBMODULE_RELEASES[0][1], tokenizers_version), @@ -434,7 +453,9 @@ def main() -> None: ("torch", torch_version), ("torchvision", torchvision_version), ("torchaudio", torchaudio_version), - ) + ("torchao", torchao_version), + ), + configured_cuda_variants(repo_root / ".github/scripts/filter_cuda_matrix.py"), ) for _path, repository, version in releases: require_remote_tag(repository, version) diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index 73da01c5f4a..d581402c26e 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -382,10 +382,13 @@ def companion_release_for_torch( def companion_release_supports_torch( - package: str, package_version: str, torch_installed_version: str + package: str, + package_version: str, + torch_installed_version: str, + channel: str = "test", ) -> bool: """Whether any wheel for a companion release accepts the selected Torch build.""" - for wheel in _test_index_wheels(package).get(package_version, []): + for wheel in _test_index_wheels(package, channel=channel).get(package_version, []): metadata = Parser().parsestr(_wheel_metadata(wheel)) for raw_requirement in metadata.get_all("Requires-Dist", []): match = re.match( @@ -529,9 +532,9 @@ def require_test_infra_branch(test_infra_branch: str) -> None: ) -def pytorch_commit_for_wheel(torch_version: str) -> str: +def pytorch_commit_for_wheel(torch_version: str, channel: str = "test") -> str: """Read the exact source commit embedded in the selected binary wheel.""" - wheels = _test_index_wheels("torch").get(torch_version, []) + wheels = _test_index_wheels("torch", channel=channel).get(torch_version, []) if not wheels: raise RuntimeError(f"PyTorch test index has no torch {torch_version} wheel") version_module = _wheel_member(_preferred_wheel(wheels).url, "torch/version.py") @@ -564,12 +567,15 @@ def sync_pytorch_source(repo_root: Path, commit: str) -> None: ) -def _release_torchao_version(install_requirements_path: Path) -> tuple[str, str]: +def _release_torchao_version( + install_requirements_path: Path, finalized: bool +) -> tuple[str, str]: requirements_text = install_requirements_path.read_text() url_match = _TORCHAO_URL_PATTERN.search(requirements_text) version_match = _TORCHAO_VERSION_PATTERN.search(requirements_text) - if url_match is None or "/whl/test" not in url_match.group(0): - raise RuntimeError("release TorchAO must come from the retained test index") + expected_index = "/whl" if finalized else "/whl/test" + if url_match is None or not url_match.group(0).endswith(f'{expected_index}"'): + raise RuntimeError(f"release TorchAO must come from the {expected_index} index") if version_match is None: raise RuntimeError("release TorchAO pin is missing") torchao_version = version_match.group(0).split('"')[1] @@ -583,6 +589,7 @@ def _validate_cuda_releases( requirements_text: str, cuda_filter_path: Path, verify_index: bool, + channel: str, ) -> None: variants = configured_cuda_variants(cuda_filter_path) if verify_index: @@ -590,7 +597,8 @@ def _validate_cuda_releases( f"{package}=={package_version}+{variant}" for variant in variants for package, package_version in releases.items() - if package_version not in _test_index_versions(package, variant) + if package_version + not in _test_index_versions(package, variant, channel=channel) ] if missing: raise RuntimeError( @@ -648,8 +656,10 @@ def validate_release_build( package_version = config[name] releases[package] = package_version + finalized = config.get("RELEASE_DEPENDENCIES_FINALIZED") is True + channel = "" if finalized else "test" if verify_index: - torch_wheels = _test_index_wheels("torch").get(version, []) + torch_wheels = _test_index_wheels("torch", channel=channel).get(version, []) if not torch_wheels: raise RuntimeError(f"test index no longer contains torch {version}") installed = _preferred_wheel(torch_wheels).installed_version @@ -657,16 +667,20 @@ def validate_release_build( ("torchvision", "TORCHVISION_VERSION"), ("torchaudio", "TORCHAUDIO_VERSION"), ): - if not companion_release_supports_torch(package, config[name], installed): + if not companion_release_supports_torch( + package, config[name], installed, channel=channel + ): raise RuntimeError( f"{package} {config[name]} is not compatible with torch {installed}" ) torchao_version, requirements_text = _release_torchao_version( - install_requirements_path + install_requirements_path, finalized ) releases["torchao"] = torchao_version - _validate_cuda_releases(releases, requirements_text, cuda_filter_path, verify_index) + _validate_cuda_releases( + releases, requirements_text, cuda_filter_path, verify_index, channel + ) return torch_requirement(version, "cpu") diff --git a/scripts/release/release_versions.py b/scripts/release/release_versions.py index 6a9656c6d53..b7dd948958f 100644 --- a/scripts/release/release_versions.py +++ b/scripts/release/release_versions.py @@ -29,27 +29,12 @@ def is_release_version(version: object, allow_prerelease: bool = True) -> bool: return allow_prerelease or stage is None -def release_base_version(version: str) -> str: - if ".dev" in version: - version, development = version.rsplit(".dev", 1) - if not development.isdigit(): - raise RuntimeError(f"invalid release version {version!r}") - major, minor, patch, _stage, _stage_number = release_parts(version) - return f"{major}.{minor}.{patch}" - - def release_key(version: str) -> tuple[int, int, int, int, int]: major, minor, patch, stage, stage_number = release_parts(version) stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} return major, minor, patch, stage_rank[stage], stage_number -def torch_release_tag(torch_version: str) -> str: - major, minor, patch, stage, stage_number = release_parts(torch_version) - suffix = f"-{stage}{stage_number}" if stage else "" - return f"v{major}.{minor}.{patch}{suffix}" - - def torch_requirement( torch_version: str, wheel_variant: str, installed_version: str = "" ) -> str: diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py index 933d30fb081..6d40bc52fbf 100644 --- a/scripts/release/test_finalize_release.py +++ b/scripts/release/test_finalize_release.py @@ -27,9 +27,9 @@ def test_promotes_release_candidate_to_final(self) -> None: with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "torch_pin.py" path.write_text( - 'TORCH_VERSION = "2.15.0rc2"\n' - 'TORCHVISION_VERSION = "0.30.0rc2"\n' - 'TORCHAUDIO_VERSION = "2.11.0rc2"\n' + 'TORCH_VERSION = "2.15.0"\n' + 'TORCHVISION_VERSION = "0.30.0"\n' + 'TORCHAUDIO_VERSION = "2.11.0"\n' "RELEASE_WHEEL = True\n" "RELEASE_DEPENDENCIES_FINALIZED = False\n" ) @@ -45,6 +45,7 @@ def test_finalizes_dependency_text(self) -> None: root = Path(directory) (root / "install_requirements.py").write_text( 'TORCH_URL_BASE = "https://download.pytorch.org/whl/test"\n' + 'TORCHAO_URL_BASE = "https://download.pytorch.org/whl/test"\n' 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260805"\n' ) @@ -93,6 +94,10 @@ def test_finalizes_dependency_text(self) -> None: 'TORCH_URL_BASE = "https://download.pytorch.org/whl"', (root / "install_requirements.py").read_text(), ) + self.assertIn( + 'TORCHAO_URL_BASE = "https://download.pytorch.org/whl"', + (root / "install_requirements.py").read_text(), + ) self.assertIn( '"pytorch-tokenizers>=1.6.0"', (root / "setup.py").read_text() )