From ef79b86b53188c20316eb191f7847714a4fc2b1e Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Wed, 30 Sep 2026 13:52:02 -0700 Subject: [PATCH 01/12] Automate release branch preparation and finalization --- .ci/scripts/tests/test_cu134_dependencies.py | 6 +- .ci/scripts/wheel/test_clean_install.py | 68 +++- .ci/scripts/wheel/test_cuda_linux.py | 2 + .../workflows/build-wheels-aarch64-linux.yml | 2 + .../build-wheels-cuda-aarch64-linux.yml | 1 + .github/workflows/build-wheels-cuda-linux.yml | 1 + .github/workflows/build-wheels-linux.yml | 2 + .github/workflows/build-wheels-macos.yml | 2 + .github/workflows/build-wheels-windows.yml | 2 + .github/workflows/lint.yml | 9 + .github/workflows/pull.yml | 1 + .github/workflows/weekly-pytorch-pin-bump.yml | 28 +- .github/workflows/windows-msvc.yml | 7 +- install_requirements.py | 7 +- scripts/release/apply-release-changes.sh | 52 ++- scripts/release/cut-release-branch.sh | 13 + scripts/release/finalize_release.py | 347 +++++++++++++++++ scripts/release/prepare_release.py | 362 ++++++++++++++++++ scripts/release/test_finalize_release.py | 94 +++++ scripts/release/test_prepare_release.py | 82 ++++ setup.py | 19 +- torch_pin.py | 8 + 22 files changed, 1069 insertions(+), 46 deletions(-) create mode 100644 scripts/release/finalize_release.py create mode 100644 scripts/release/prepare_release.py create mode 100644 scripts/release/test_finalize_release.py create mode 100644 scripts/release/test_prepare_release.py diff --git a/.ci/scripts/tests/test_cu134_dependencies.py b/.ci/scripts/tests/test_cu134_dependencies.py index 689b6be3e05..d087cea17dc 100644 --- a/.ci/scripts/tests/test_cu134_dependencies.py +++ b/.ci/scripts/tests/test_cu134_dependencies.py @@ -100,7 +100,7 @@ def test_other_cuda_trains_keep_existing_pins(self): core, local, domains, examples = self.install_commands( cuda, machine ) - self.assertIn("torch==2.14.0", core) + self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) self.assertIn("torchao==0.19.0.dev20260907", core) self.assertIn("torchvision==0.29.0", domains) self.assertIn("torchaudio==2.11.0", domains) @@ -118,14 +118,14 @@ def test_source_pinned_torch_is_not_replaced(self): def test_no_cuda_keeps_default_pins(self): core, _, domains, _ = self.install_commands(None) - self.assertIn("torch==2.14.0", core) + self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) self.assertIn("torchao==0.19.0.dev20260907", core) self.assertIn("torchvision==0.29.0", domains) self.assertIn("https://download.pytorch.org/whl/test/cpu", core) def test_windows_does_not_select_cu134(self): core, _, domains, _ = self.install_commands((13, 4), system="Windows") - self.assertIn("torch==2.14.0", core) + self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) self.assertIn("torchvision==0.29.0", domains) self.assertIn("https://download.pytorch.org/whl/test/cpu", core) diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index 65943277324..82e26de6f8b 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -21,6 +21,8 @@ import json import os +import re +import runpy import subprocess import sys from pathlib import Path @@ -45,12 +47,71 @@ # Distributions that are legitimately present without being declared. # -# torch, because the wheel deliberately does not declare it: a consumer brings the build -# matching their platform and accelerator. The rest are what torch itself requires, so they are -# guaranteed alongside it. +# Development and minimal wheels deliberately do not declare torch: a consumer brings the build +# matching their platform and accelerator. Release wheels also reach it through their declared +# requirements. The rest are what torch itself requires, so they are guaranteed alongside it. ASSUMED_PRESENT: Set[str] = {"torch", "executorch"} +def test_release_pytorch_requirement() -> None: + """Release wheels must declare the PyTorch release in torch_pin.py.""" + import importlib.metadata as metadata + + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name + + repo_root = Path(__file__).resolve().parents[3] + config = runpy.run_path(str(repo_root / "torch_pin.py")) + ci_refs = ( + os.environ.get("GITHUB_REF_NAME", ""), + os.environ.get("GITHUB_BASE_REF", ""), + os.environ.get("GITHUB_REF", "").removeprefix("refs/heads/"), + ) + release_tag_ci = any( + re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+(?:-rc\d+)?", ref) for ref in ci_refs + ) + release_ci = any(ref.startswith("release/") for ref in ci_refs) or any( + re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+(?:-rc\d+)?", ref) for ref in ci_refs + ) + assert not release_ci or config["RELEASE_WHEEL"], ( + "this wheel is being built from a release ref, but torch_pin.py does not enable " + "release metadata; run scripts/release/apply-release-changes.sh" + ) + dependencies_finalized = config.get("RELEASE_DEPENDENCIES_FINALIZED") is True + assert not release_tag_ci or dependencies_finalized, ( + "this wheel is being built from a release tag, but stable dependencies were not " + "finalized; run scripts/release/finalize_release.py before tagging" + ) + assert not dependencies_finalized or re.fullmatch( + r"\d+\.\d+\.\d+", config["TORCH_VERSION"] + ), ( + "release dependencies are marked finalized, but TORCH_VERSION is not a final " + f"PyTorch release: {config['TORCH_VERSION']}" + ) + torch_requirements = [ + Requirement(raw) + for raw in metadata.requires("executorch") or [] + if canonicalize_name(Requirement(raw).name) == "torch" + ] + + if config["RELEASE_WHEEL"]: + expected = f">={config['TORCH_VERSION']}" + assert len(torch_requirements) == 1, ( + "a release wheel must declare exactly one PyTorch dependency, but found " + f"{[str(requirement) for requirement in torch_requirements]}" + ) + assert str(torch_requirements[0].specifier) == expected, ( + f"release wheel declares {torch_requirements[0]}, expected torch{expected} " + "from torch_pin.py" + ) + print(f"✓ release wheel declares torch{expected}") + else: + assert not torch_requirements, ( + "a development wheel unexpectedly declares PyTorch; only release wheels should " + f"declare it, found {[str(requirement) for requirement in torch_requirements]}" + ) + + def _normalise(name: str) -> str: return name.lower().replace("-", "_").replace(".", "_") @@ -169,6 +230,7 @@ def _blocked_modules(allowed: Set[str]) -> Set[str]: def run_tests(work_dir: Path) -> None: + test_release_pytorch_requirement() allowed = _allowed_distributions() blocked = _blocked_modules(allowed) print( diff --git a/.ci/scripts/wheel/test_cuda_linux.py b/.ci/scripts/wheel/test_cuda_linux.py index 914fafaa3f3..3d0a060fb8b 100644 --- a/.ci/scripts/wheel/test_cuda_linux.py +++ b/.ci/scripts/wheel/test_cuda_linux.py @@ -34,6 +34,7 @@ from typing import Optional, Set import test_base +import test_clean_install import test_cpp_sdk import test_shared_libraries from examples.models import Backend, Model @@ -583,6 +584,7 @@ def test_a_model_runs_through_the_delegate() -> None: if __name__ == "__main__": assert platform.system() == "Linux", "the CUDA rows are Linux only" + test_clean_install.test_release_pytorch_requirement() test_cuda_libraries_are_shipped() test_cuda_runtime_is_declared() test_cuda_libraries_resolve_relatively() diff --git a/.github/workflows/build-wheels-aarch64-linux.yml b/.github/workflows/build-wheels-aarch64-linux.yml index eb79a9786c0..3db59284048 100644 --- a/.github/workflows/build-wheels-aarch64-linux.yml +++ b/.github/workflows/build-wheels-aarch64-linux.yml @@ -11,7 +11,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/build-wheels-cuda-aarch64-linux.yml b/.github/workflows/build-wheels-cuda-aarch64-linux.yml index 882184d4dcc..134a1d4e9ab 100644 --- a/.github/workflows/build-wheels-cuda-aarch64-linux.yml +++ b/.github/workflows/build-wheels-cuda-aarch64-linux.yml @@ -15,6 +15,7 @@ on: - install_utils.py - pyproject.toml - setup.py + - torch_pin.py - docs/source/using-executorch-cpp.md - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories diff --git a/.github/workflows/build-wheels-cuda-linux.yml b/.github/workflows/build-wheels-cuda-linux.yml index 8de4ae4036c..3d324d75132 100644 --- a/.github/workflows/build-wheels-cuda-linux.yml +++ b/.github/workflows/build-wheels-cuda-linux.yml @@ -15,6 +15,7 @@ on: - install_utils.py - pyproject.toml - setup.py + - torch_pin.py - docs/source/using-executorch-cpp.md - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories diff --git a/.github/workflows/build-wheels-linux.yml b/.github/workflows/build-wheels-linux.yml index 15e6af41fb4..cf3332c456e 100644 --- a/.github/workflows/build-wheels-linux.yml +++ b/.github/workflows/build-wheels-linux.yml @@ -11,7 +11,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/build-wheels-macos.yml b/.github/workflows/build-wheels-macos.yml index feb8b8f7b69..af33d67043e 100644 --- a/.github/workflows/build-wheels-macos.yml +++ b/.github/workflows/build-wheels-macos.yml @@ -11,7 +11,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/build-wheels-windows.yml b/.github/workflows/build-wheels-windows.yml index 8d6cac2c9c6..31907d6fe95 100644 --- a/.github/workflows/build-wheels-windows.yml +++ b/.github/workflows/build-wheels-windows.yml @@ -10,7 +10,9 @@ on: - pyproject.toml - setup.py - docs/source/using-executorch-cpp.md + - install_requirements.py - install_utils.py + - torch_pin.py - tools/cmake/**/* # The wheel ships these as its C++ SDK. Whole trees rather than the exact directories # setup.py copies from, so adding one there cannot silently drop it from this list. diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 66c1e7d0e92..a71a879cb5a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -105,6 +105,15 @@ jobs: pip install -r requirements-lintrunner.txt lintrunner init + - name: Test release preparation + run: | + python scripts/release/test_prepare_release.py + python scripts/release/test_finalize_release.py + + - name: Validate release configuration + if: startsWith(github.ref_name, 'release/') || startsWith(github.base_ref, 'release/') + run: python scripts/release/prepare_release.py --check + - name: Run linters env: CHANGED_FILES: ${{ needs.get-changed-files.outputs.changed-files }} diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index b672d691446..9caf1311e7e 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -91,6 +91,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, 'extension/flat_tensor') || contains(needs.changed-files.outputs.changed-files, 'extension/pytree') || contains(needs.changed-files.outputs.changed-files, 'pyproject.toml') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || contains(needs.changed-files.outputs.changed-files, 'schema/') || contains(needs.changed-files.outputs.changed-files, 'setup.py') || contains(needs.changed-files.outputs.changed-files, 'tools/cmake/') diff --git a/.github/workflows/weekly-pytorch-pin-bump.yml b/.github/workflows/weekly-pytorch-pin-bump.yml index 5441ad8b836..0ac6d738e0d 100644 --- a/.github/workflows/weekly-pytorch-pin-bump.yml +++ b/.github/workflows/weekly-pytorch-pin-bump.yml @@ -29,17 +29,27 @@ jobs: NIGHTLY_VERSION="dev${NIGHTLY_DATE}" echo "version=${NIGHTLY_VERSION}" >> "$GITHUB_OUTPUT" - - name: Read current TORCH_VERSION - id: torch - run: | - TORCH_VERSION=$(python -c "exec(open('torch_pin.py').read()); print(TORCH_VERSION)") - echo "version=${TORCH_VERSION}" >> "$GITHUB_OUTPUT" - - name: Update torch_pin.py with new NIGHTLY_VERSION + env: + NIGHTLY_VERSION: ${{ steps.nightly.outputs.version }} run: | - printf 'TORCH_VERSION = "%s"\nNIGHTLY_VERSION = "%s"\n' \ - "${{ steps.torch.outputs.version }}" \ - "${{ steps.nightly.outputs.version }}" > torch_pin.py + python - <<'PY' + import os + import pathlib + import re + + path = pathlib.Path("torch_pin.py") + updated, count = re.subn( + r'^NIGHTLY_VERSION\s*=\s*"[^"]+"$', + f'NIGHTLY_VERSION = "{os.environ["NIGHTLY_VERSION"]}"', + path.read_text(), + count=1, + flags=re.MULTILINE, + ) + if count != 1: + raise RuntimeError("expected exactly one NIGHTLY_VERSION assignment") + path.write_text(updated) + PY - name: Run pin bump script run: python .github/scripts/update_pytorch_pin.py diff --git a/.github/workflows/windows-msvc.yml b/.github/workflows/windows-msvc.yml index 269f2cf2381..3a1fa15fb71 100644 --- a/.github/workflows/windows-msvc.yml +++ b/.github/workflows/windows-msvc.yml @@ -12,6 +12,7 @@ on: - .ci/docker/ci_commit_pins/pytorch.txt - .ci/scripts/** - .github/workflows/windows-msvc.yml + - torch_pin.py workflow_dispatch: concurrency: @@ -43,6 +44,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/docker/ci_commit_pins/pytorch.txt') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/windows-msvc.yml') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || needs.run-decision.outputs.is-full-run == 'true' uses: pytorch/test-infra/.github/workflows/windows_job.yml@main with: @@ -66,6 +68,7 @@ jobs: contains(needs.changed-files.outputs.changed-files, '.ci/docker/ci_commit_pins/pytorch.txt') || contains(needs.changed-files.outputs.changed-files, '.ci/scripts/') || contains(needs.changed-files.outputs.changed-files, '.github/workflows/windows-msvc.yml') || + contains(needs.changed-files.outputs.changed-files, 'torch_pin.py') || needs.run-decision.outputs.is-full-run == 'true' # test-infra's Windows setup assumes the AWS runner image; use the GitHub # hosted Windows ARM64 image directly. @@ -91,7 +94,9 @@ jobs: - name: Install build dependencies shell: pwsh - run: python -m pip install pyyaml torch==2.14.0 --extra-index-url https://download.pytorch.org/whl/test/cpu + run: | + $torchVersion = python -c "from torch_pin import TORCH_VERSION; print(TORCH_VERSION)" + python -m pip install pyyaml "torch==$torchVersion" --extra-index-url https://download.pytorch.org/whl/test/cpu - name: Build ExecuTorch shell: pwsh diff --git a/install_requirements.py b/install_requirements.py index fccd65eaead..f7ac388ec45 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -8,11 +8,16 @@ import argparse import os import platform +import runpy import subprocess import sys from install_utils import determine_torch_url, is_intel_mac_os, python_is_compatible +TORCH_VERSION = runpy.run_path(os.path.join(os.path.dirname(__file__), "torch_pin.py"))[ + "TORCH_VERSION" +] + # The pip repository that hosts nightly torch packages. # This will be dynamically set based on CUDA availability and CUDA backend enabled/disabled. TORCH_URL_BASE = "https://download.pytorch.org/whl/test" @@ -103,7 +108,7 @@ def install_requirements(use_pytorch_nightly): # Setting use_pytorch_nightly to false to test the pinned PyTorch commit. Note # that we don't need to set any version number there because they have already # been installed on CI before this step, so pip won't reinstall them - ("torch==2.14.0" if use_pytorch_nightly else "torch"), + (f"torch=={TORCH_VERSION}" if use_pytorch_nightly else "torch"), f"torchao=={TORCHAO_NIGHTLY_VERSION}", ] diff --git a/scripts/release/apply-release-changes.sh b/scripts/release/apply-release-changes.sh index a880687ddaf..f7462b959f2 100755 --- a/scripts/release/apply-release-changes.sh +++ b/scripts/release/apply-release-changes.sh @@ -6,14 +6,17 @@ # LICENSE file in the root directory of this source tree. : ' -# Step 2 after branch cut is complete. -# -# Creates PR with release only changes. +# Prepare an existing release branch. cut-release-branch.sh invokes this +# automatically; it may also be rerun safely by itself. # # Usage (run from root of project): # TEST_INFRA_BRANCH=release/2.3 ./scripts/release/apply-release-changes.sh # # TEST_INFRA_BRANCH: The release branch of test-infra that houses all reusable +# workflows. Defaults to the matching release/X.Y branch. The script also +# enables the PyTorch dependency for full release wheels using torch_pin.py. +# TORCH_VERSION: Optional PyTorch RC version override. By default, the newest +# release-form version available on the PyTorch test index is selected. ' set -eou pipefail @@ -21,43 +24,36 @@ set -eou pipefail GIT_TOP_DIR=$(git rev-parse --show-toplevel) RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} RELEASE_BRANCH="release/${RELEASE_VERSION}" +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} +TEST_INFRA_BRANCH=${TEST_INFRA_BRANCH:-${RELEASE_BRANCH}} # Check out to Release Branch -if git ls-remote --exit-code origin ${RELEASE_BRANCH} >/dev/null 2>&1; then +if git show-ref --verify --quiet "refs/heads/${RELEASE_BRANCH}"; then + echo "Check out local Release Branch '${RELEASE_BRANCH}'" + git checkout "${RELEASE_BRANCH}" +elif git ls-remote --exit-code origin "${RELEASE_BRANCH}" >/dev/null 2>&1; then echo "Check out to Release Branch '${RELEASE_BRANCH}'" - git checkout ${RELEASE_BRANCH} + git checkout "${RELEASE_BRANCH}" else echo "Error: Remote branch '${RELEASE_BRANCH}' not found. Please run 'cut-release-branch.sh' first." exit 1 fi -# Change all GitHub Actions to reference the test-infra release branch -# as opposed to main. -echo "Applying release-only changes to workflows" -for i in .github/workflows/*.yml; do - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' -e s#@main#@"${TEST_INFRA_BRANCH}"# $i; - sed -i '' -e s#test-infra-ref:[[:space:]]main#"test-infra-ref: ${TEST_INFRA_BRANCH}"# $i; - else - sed -i -e s#@main#@"${TEST_INFRA_BRANCH}"# $i; - sed -i -e s#test-infra-ref:[[:space:]]main#"test-infra-ref: ${TEST_INFRA_BRANCH}"# $i; - fi -done - -# Update documentation to use release branch instead of viable/strict -echo "Updating documentation branch references" -for doc in $(grep -rl "viable/strict" docs/); do - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' -e "s#-b viable/strict#-b ${RELEASE_BRANCH}#g" "$doc" - else - sed -i -e "s#-b viable/strict#-b ${RELEASE_BRANCH}#g" "$doc" - fi -done +PREPARE_ARGS=( + --release-version "${RELEASE_VERSION}" + --test-infra-branch "${TEST_INFRA_BRANCH}" +) +if [[ -n "${TORCH_VERSION:-}" ]]; then + PREPARE_ARGS+=(--torch-version "${TORCH_VERSION}") +fi +"${PYTHON_EXECUTABLE}" "${GIT_TOP_DIR}/scripts/release/prepare_release.py" "${PREPARE_ARGS[@]}" echo "You'll need to manually commit the changes and create a PR. Here are the steps:" echo "1. Stage the changes:" -echo " git add .github/workflows/*.yml docs/" +echo " git add .github/workflows/*.yml CONTRIBUTING.md docs/ extension/benchmark/apple/Benchmark/ backends/arm/public_api_manifests/ version.txt torch_pin.py" echo "2. Commit the changes:" echo " git commit -m \"[RELEASE-ONLY CHANGES] Branch Cut for Release ${RELEASE_VERSION}\"" echo "3. After committing, create a pull request to merge the changes." +echo "4. When stable dependency artifacts are published, run:" +echo " python scripts/release/finalize_release.py" diff --git a/scripts/release/cut-release-branch.sh b/scripts/release/cut-release-branch.sh index 60e9237ff0b..36b09b5b89e 100755 --- a/scripts/release/cut-release-branch.sh +++ b/scripts/release/cut-release-branch.sh @@ -16,6 +16,11 @@ For `pytorch/executorch` usage would be like: or to cut from main branch: > DRY_RUN=disabled GIT_BRANCH_TO_CUT_FROM=main ./scripts/release/cut-release-branch.sh + +After creating release/X.Y and orig/release/X.Y at the selected commit, this +also applies the deterministic release-only changes to the local release/X.Y +branch. TEST_INFRA_BRANCH defaults to release/X.Y. TORCH_VERSION may override +the newest PyTorch release candidate selected from the test wheel index. ' set -eou pipefail @@ -26,6 +31,7 @@ GIT_BRANCH_TO_CUT_FROM=${GIT_BRANCH_TO_CUT_FROM:-viable/strict} # should output something like 1.11 RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} +TEST_INFRA_BRANCH=${TEST_INFRA_BRANCH:-release/${RELEASE_VERSION}} DRY_RUN_FLAG="--dry-run" if [[ ${DRY_RUN:-enabled} == "disabled" ]]; then @@ -52,3 +58,10 @@ for branch in "release/${RELEASE_VERSION}" "orig/release/${RELEASE_VERSION}"; do ) fi done + +( + set -x + RELEASE_VERSION="${RELEASE_VERSION}" \ + TEST_INFRA_BRANCH="${TEST_INFRA_BRANCH}" \ + "${GIT_TOP_DIR}/scripts/release/apply-release-changes.sh" +) diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py new file mode 100644 index 00000000000..626a62fc1f0 --- /dev/null +++ b/scripts/release/finalize_release.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Finalize release dependencies after their stable artifacts are published.""" + +import argparse +import json +import re +import runpy +import subprocess +import urllib.error +import urllib.request +from pathlib import Path +from typing import Iterable + +from prepare_release import _release_version_from_file, set_torch_version + +_VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") +_TORCHCODEC_PATTERN = re.compile(r"torchcodec==\d+\.\d+\.\d+") +_TOKENIZERS_REQUIREMENT_PATTERN = re.compile( + r'(?m)^(?P\s*)"pytorch-tokenizers(?:>=\d+\.\d+\.\d+)?",$' +) +_TORCHAO_ASSIGNMENT_PATTERN = re.compile( + r'^TORCHAO_NIGHTLY_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE +) +_FINALIZED_PATTERN = re.compile( + r"^RELEASE_DEPENDENCIES_FINALIZED\s*=\s*(?:True|False)$", re.MULTILINE +) +_TORCHCODEC_PATHS = ( + ".ci/scripts/test-rocm-voxtral.sh", + ".ci/scripts/test_model_e2e.sh", + "examples/models/moshi/mimi/install_requirements.sh", +) +_ROCM_PATHS = ( + ".ci/scripts/test-rocm-aoti.sh", + ".ci/scripts/test-rocm-voxtral.sh", +) +_QNN_TEST_INDEX = ( + '"$PIPBIN" install torch=="${TORCH_VERSION}" ' + '--extra-index-url "https://download.pytorch.org/whl/test"' +) +_QNN_STABLE_INDEX = ( + '"$PIPBIN" install --no-cache-dir torch=="${TORCH_VERSION}" ' + '--index-url "https://download.pytorch.org/whl/cpu"' +) +_SUBMODULE_RELEASES = ( + ("extension/llm/tokenizers", "https://github.com/meta-pytorch/tokenizers.git"), + ("third-party/ao", "https://github.com/pytorch/ao.git"), +) + + +def stable_base_version(version: str) -> str: + """Convert a development or prerelease version to its final base version.""" + match = re.match(r"(\d+\.\d+\.\d+)", version) + if match is None: + raise RuntimeError(f"could not derive a stable version from {version!r}") + return match.group(1) + + +def current_torchao_version(install_requirements_path: Path) -> str: + """Read the generic TorchAO pin without importing the installer module.""" + match = _TORCHAO_ASSIGNMENT_PATTERN.search(install_requirements_path.read_text()) + if match is None: + raise RuntimeError( + f"could not find TORCHAO_NIGHTLY_VERSION in {install_requirements_path}" + ) + return match.group("version") + + +def latest_pypi_version(package: str) -> str: + """Return the latest stable version published for a package.""" + with urllib.request.urlopen( + f"https://pypi.org/pypi/{package}/json", timeout=30 + ) as response: + version = json.load(response)["info"]["version"] + if _VERSION_PATTERN.fullmatch(version) is None: + raise RuntimeError(f"latest {package} version is not stable: {version!r}") + return version + + +def require_pypi_release(package: str, version: str) -> None: + """Fail before mutation if an expected stable package is unavailable.""" + if _VERSION_PATTERN.fullmatch(version) is None: + raise RuntimeError(f"invalid {package} version {version!r}") + try: + with urllib.request.urlopen( + f"https://pypi.org/pypi/{package}/{version}/json", timeout=30 + ): + pass + except urllib.error.HTTPError as error: + if error.code == 404: + raise RuntimeError( + f"{package}=={version} is not published on PyPI" + ) from error + raise + + +def require_remote_tag(repository: str, version: str) -> None: + """Fail before mutation if a matching submodule release tag is unavailable.""" + result = subprocess.run( + [ + "git", + "ls-remote", + "--exit-code", + "--tags", + repository, + f"refs/tags/v{version}", + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + raise RuntimeError(f"{repository} does not have tag v{version}") + + +def _write_if_changed(path: Path, updated: str) -> bool: + original = path.read_text() + if updated == original: + return False + path.write_text(updated) + return True + + +def mark_release_dependencies_finalized(torch_pin_path: Path) -> bool: + """Record that delayed dependency finalization completed successfully.""" + text = torch_pin_path.read_text() + updated, count = _FINALIZED_PATTERN.subn( + "RELEASE_DEPENDENCIES_FINALIZED = True", text + ) + if count != 1: + raise RuntimeError( + f"expected one RELEASE_DEPENDENCIES_FINALIZED assignment in {torch_pin_path}" + ) + return _write_if_changed(torch_pin_path, updated) + + +def finalize_torch_release(torch_pin_path: Path, torch_version: str) -> int: + """Replace the RC pin with the final release and mark finalization complete.""" + changed = int(set_torch_version(torch_pin_path, torch_version)) + changed += int(mark_release_dependencies_finalized(torch_pin_path)) + return changed + + +def _finalize_torchao(repo_root: Path, version: str) -> set[Path]: + path = repo_root / "install_requirements.py" + text = path.read_text() + updated, count = _TORCHAO_ASSIGNMENT_PATTERN.subn( + f'TORCHAO_NIGHTLY_VERSION = "{version}"', text + ) + if count != 1: + raise RuntimeError(f"expected one TORCHAO_NIGHTLY_VERSION in {path}") + return {path} if _write_if_changed(path, updated) else set() + + +def _finalize_tokenizers(repo_root: Path, version: str) -> set[Path]: + path = repo_root / "setup.py" + text = path.read_text() + updated, count = _TOKENIZERS_REQUIREMENT_PATTERN.subn( + rf'\g"pytorch-tokenizers>={version}",', text + ) + if count != 1: + raise RuntimeError(f"expected one pytorch-tokenizers requirement in {path}") + return {path} if _write_if_changed(path, updated) else set() + + +def _finalize_torchcodec(repo_root: Path, version: str) -> set[Path]: + changed_paths = set() + for relative in _TORCHCODEC_PATHS: + path = repo_root / relative + if not path.exists(): + raise RuntimeError(f"expected release dependency file {path}") + text = path.read_text() + updated, count = _TORCHCODEC_PATTERN.subn(f"torchcodec=={version}", text) + if count == 0: + raise RuntimeError(f"expected a torchcodec pin in {path}") + updated = updated.replace( + "--extra-index-url https://download.pytorch.org/whl/test/cpu", + "--index-url https://download.pytorch.org/whl/cpu", + ) + if _write_if_changed(path, updated): + changed_paths.add(path) + return changed_paths + + +def _finalize_stable_indexes(repo_root: Path) -> set[Path]: + changed_paths = set() + for relative in _ROCM_PATHS: + path = repo_root / relative + if not path.exists(): + raise RuntimeError(f"expected release dependency file {path}") + text = path.read_text() + updated = text.replace( + "https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}", + "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", + ).replace( + "https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}", + "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", + ) + if _write_if_changed(path, updated): + changed_paths.add(path) + + qnn_path = repo_root / ".ci/scripts/test_wheel_package_qnn.sh" + if not qnn_path.exists(): + raise RuntimeError(f"expected release dependency file {qnn_path}") + text = qnn_path.read_text() + updated = text.replace(_QNN_TEST_INDEX, _QNN_STABLE_INDEX) + if _QNN_STABLE_INDEX not in updated: + raise RuntimeError( + f"could not find the QNN Torch install command in {qnn_path}" + ) + if _write_if_changed(qnn_path, updated): + changed_paths.add(qnn_path) + return changed_paths + + +def finalize_dependency_text( + repo_root: Path, + torchao_version: str, + tokenizers_version: str, + torchcodec_version: str, +) -> int: + """Update all recurring stable dependency declarations and index URLs.""" + changed_paths = _finalize_torchao(repo_root, torchao_version) + changed_paths |= _finalize_tokenizers(repo_root, tokenizers_version) + changed_paths |= _finalize_torchcodec(repo_root, torchcodec_version) + changed_paths |= _finalize_stable_indexes(repo_root) + return len(changed_paths) + + +def update_submodule_tags( + repo_root: Path, releases: Iterable[tuple[str, str, str]] +) -> None: + """Check out verified release tags in the dependency submodules.""" + for relative, _repository, version in releases: + path = repo_root / relative + if (path / ".git").exists() and subprocess.run( + ["git", "status", "--porcelain"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout: + raise RuntimeError(f"submodule {path} has local changes") + subprocess.run( + ["git", "submodule", "update", "--init", "--", relative], + cwd=repo_root, + check=True, + ) + if subprocess.run( + ["git", "status", "--porcelain"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout: + raise RuntimeError(f"submodule {path} has local changes") + subprocess.run( + ["git", "fetch", "--depth=1", "origin", "tag", f"v{version}"], + cwd=path, + check=True, + ) + subprocess.run( + ["git", "checkout", "--detach", f"v{version}"], cwd=path, check=True + ) + + +def main() -> None: + """Preflight and apply stable dependency finalization.""" + parser = argparse.ArgumentParser() + parser.add_argument("--release-version", help="release version in MAJOR.MINOR form") + parser.add_argument("--torch-version", help="final stable PyTorch version") + parser.add_argument("--torchao-version", help="stable TorchAO version") + parser.add_argument( + "--tokenizers-version", help="stable pytorch-tokenizers version" + ) + parser.add_argument("--torchcodec-version", help="stable TorchCodec version") + parser.add_argument( + "--preflight-only", + action="store_true", + help="verify packages and tags without changing the checkout", + ) + parser.add_argument( + "--repo-root", type=Path, default=Path(__file__).resolve().parents[2] + ) + args = parser.parse_args() + + repo_root = args.repo_root.resolve() + release_version = args.release_version or _release_version_from_file(repo_root) + release_config = runpy.run_path(str(repo_root / "torch_pin.py")) + if release_config.get("RELEASE_WHEEL") is not True: + raise RuntimeError( + "release preparation has not run; use cut-release-branch.sh first" + ) + torch_version = args.torch_version or stable_base_version( + release_config["TORCH_VERSION"] + ) + torchao_version = args.torchao_version or stable_base_version( + current_torchao_version(repo_root / "install_requirements.py") + ) + tokenizers_version = args.tokenizers_version or f"{release_version}.0" + torchcodec_version = args.torchcodec_version or latest_pypi_version("torchcodec") + + releases = ( + ("extension/llm/tokenizers", _SUBMODULE_RELEASES[0][1], tokenizers_version), + ("third-party/ao", _SUBMODULE_RELEASES[1][1], torchao_version), + ) + for package, version in ( + ("torch", torch_version), + ("torchao", torchao_version), + ("pytorch-tokenizers", tokenizers_version), + ("torchcodec", torchcodec_version), + ): + require_pypi_release(package, version) + for _path, repository, version in releases: + require_remote_tag(repository, version) + + print( + "Release dependency artifacts are available: " + f"torch=={torch_version}, torchao=={torchao_version}, " + f"pytorch-tokenizers=={tokenizers_version}, torchcodec=={torchcodec_version}" + ) + if args.preflight_only: + return + + update_submodule_tags(repo_root, releases) + changed = finalize_dependency_text( + repo_root, torchao_version, tokenizers_version, torchcodec_version + ) + changed += finalize_torch_release(repo_root / "torch_pin.py", torch_version) + print( + f"Finalized release dependencies; changed {changed} text files and 2 submodules" + ) + print( + "Review and stage the resulting release-only changes with git add " + "install_requirements.py setup.py torch_pin.py .ci/scripts/ " + "examples/models/moshi/mimi/install_requirements.sh " + "extension/llm/tokenizers third-party/ao" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py new file mode 100644 index 00000000000..7c297cf3a26 --- /dev/null +++ b/scripts/release/prepare_release.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Apply and validate the deterministic changes required on release branches.""" + +import argparse +import re +import runpy +import shutil +import urllib.request +from pathlib import Path +from typing import Iterable, List +from urllib.parse import unquote + +_RELEASE_VERSION_PATTERN = re.compile(r"\d+\.\d+") +_RELEASE_WHEEL_PATTERN = re.compile( + r"^RELEASE_WHEEL\s*=\s*(?:True|False)$", re.MULTILINE +) +_TORCH_VERSION_PATTERN = re.compile(r'^TORCH_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE) +_TEST_INFRA_MAIN_PATTERN = re.compile(r"(pytorch/test-infra/[^\s\"']+)@main\b") +_TEST_INFRA_REF_MAIN_PATTERN = re.compile(r"(test-infra-ref:\s*)main\b") +_TEST_INFRA_BRANCH_PATTERN = re.compile(r"pytorch/test-infra/[^\s\"'@]+@([^\s\"']+)") +_TEST_INFRA_INPUT_PATTERN = re.compile(r"test-infra-ref:\s*([^\s#]+)") +_CLONE_BRANCH_PATTERN = re.compile( + r"(?P-b\s+)(?Pviable/strict|release/\d+\.\d+)\b" +) +_STABLE_SWIFTPM_PATTERN = re.compile(r"swiftpm-\d+\.\d+\.\d+(?![.\d-])") +_DOCUMENTATION_PATHS = ( + "CONTRIBUTING.md", + "docs", + "extension/benchmark/apple/Benchmark", +) + + +def _validate_release_version(release_version: str) -> None: + if _RELEASE_VERSION_PATTERN.fullmatch(release_version) is None: + raise RuntimeError( + f"release version must be MAJOR.MINOR, received {release_version!r}" + ) + + +def _write_if_changed(path: Path, original: str, updated: str) -> bool: + if updated == original: + return False + path.write_text(updated) + return True + + +def enable_release_wheel(torch_pin_path: Path) -> bool: + """Enable release-only wheel metadata in the PyTorch pin file.""" + text = torch_pin_path.read_text() + updated, count = _RELEASE_WHEEL_PATTERN.subn("RELEASE_WHEEL = True", text) + if count != 1: + raise RuntimeError( + f"expected exactly one RELEASE_WHEEL assignment in {torch_pin_path}" + ) + return _write_if_changed(torch_pin_path, text, updated) + + +def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: + """Set the Torch release pin used by installs and release wheel metadata.""" + if re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", torch_version) is None: + raise RuntimeError(f"invalid Torch release version {torch_version!r}") + text = torch_pin_path.read_text() + updated, count = _TORCH_VERSION_PATTERN.subn( + f'TORCH_VERSION = "{torch_version}"', text + ) + if count != 1: + raise RuntimeError( + f"expected exactly one TORCH_VERSION assignment in {torch_pin_path}" + ) + return _write_if_changed(torch_pin_path, text, updated) + + +def newest_torch_test_release(available_versions: Iterable[str]) -> str: + """Return the newest RC or final-form version on the PyTorch test index.""" + candidates = [] + stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} + for version in available_versions: + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?", version) + if match is None: + continue + major, minor, patch = (int(part) for part in match.groups()[:3]) + stage = match.group(4) + stage_number = int(match.group(5) or 0) + candidates.append( + ((major, minor, patch, stage_rank[stage], stage_number), version) + ) + if not candidates: + raise RuntimeError("PyTorch test index has no release candidate wheels") + return max(candidates)[1] + + +def latest_torch_test_release() -> str: + """Look up the newest PyTorch release candidate on its CPU test index.""" + with urllib.request.urlopen( + "https://download.pytorch.org/whl/test/cpu/torch/", timeout=30 + ) as response: + index = unquote(response.read().decode()) + versions = re.findall( + r"torch-(\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?)(?:\+[^-]+)?-", index + ) + return newest_torch_test_release(versions) + + +def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: + """Choose a pin once, then preserve it on subsequent preparation runs.""" + if override: + return override + config = runpy.run_path(str(torch_pin_path)) + if config.get("RELEASE_WHEEL") is True: + return config["TORCH_VERSION"] + return latest_torch_test_release() + + +def validate_release_build(torch_pin_path: Path) -> str: + """Return the release PyTorch requirement or raise for invalid config.""" + config = runpy.run_path(str(torch_pin_path)) + if config.get("RELEASE_WHEEL") is not True: + raise RuntimeError(f"{torch_pin_path} does not enable release wheel metadata") + + version = config.get("TORCH_VERSION") + if ( + not isinstance(version, str) + or re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", version) is None + ): + raise RuntimeError(f"{torch_pin_path} has invalid TORCH_VERSION {version!r}") + return f"torch>={version}" + + +def configure_release_version(version_path: Path, release_version: str) -> bool: + """Set the package version to the first final release in this series.""" + return _write_if_changed( + version_path, version_path.read_text(), f"{release_version}.0\n" + ) + + +def configure_workflows(workflow_paths: Iterable[Path], test_infra_branch: str) -> int: + """Pin test-infra actions and reusable workflows to its release branch.""" + changed = 0 + for path in workflow_paths: + text = path.read_text() + updated = _TEST_INFRA_MAIN_PATTERN.sub(rf"\1@{test_infra_branch}", text) + updated = _TEST_INFRA_REF_MAIN_PATTERN.sub(rf"\1{test_infra_branch}", updated) + changed += _write_if_changed(path, text, updated) + return changed + + +def documentation_paths(repo_root: Path) -> List[Path]: + """Return the maintained Markdown files that contain branch instructions.""" + paths: List[Path] = [] + for relative in _DOCUMENTATION_PATHS: + path = repo_root / relative + if path.is_file(): + paths.append(path) + elif path.is_dir(): + paths.extend(path.rglob("*.md")) + return sorted(paths) + + +def configure_documentation( + paths: Iterable[Path], release_branch: str, release_version: str +) -> int: + """Point checkout and stable SwiftPM examples at the new release.""" + changed = 0 + for path in paths: + text = path.read_text() + updated = _CLONE_BRANCH_PATTERN.sub(rf"\g{release_branch}", text) + updated = _STABLE_SWIFTPM_PATTERN.sub(f"swiftpm-{release_version}.0", updated) + changed += _write_if_changed(path, text, updated) + return changed + + +def freeze_arm_public_api(repo_root: Path, release_version: str) -> bool: + """Snapshot the Arm API and retain the two newest release manifests.""" + manifest_dir = repo_root / "backends/arm/public_api_manifests" + running = manifest_dir / "api_manifest_running.toml" + if not running.exists(): + return False + + target = manifest_dir / f"api_manifest_{release_version.replace('.', '_')}.toml" + created = False + if not target.exists(): + shutil.copyfile(running, target) + created = True + + static_manifests = sorted( + manifest_dir.glob("api_manifest_[0-9]*_[0-9]*.toml"), + key=lambda path: tuple( + int(part) for part in path.stem.removeprefix("api_manifest_").split("_") + ), + ) + for obsolete in static_manifests[:-2]: + obsolete.unlink() + return created + + +def validate_release_references( + workflow_paths: Iterable[Path], + documentation: Iterable[Path], + test_infra_branch: str, + release_version: str, + release_full_version: str, +) -> None: + """Reject release branches that retain moving branch references.""" + errors = [] + found_test_infra_reference = False + for path in workflow_paths: + text = path.read_text() + references = _TEST_INFRA_BRANCH_PATTERN.findall(text) + references.extend(_TEST_INFRA_INPUT_PATTERN.findall(text)) + found_test_infra_reference |= bool(references) + for reference in references: + if reference != test_infra_branch: + errors.append( + f"{path}: test-infra references {reference}, expected " + f"{test_infra_branch}" + ) + if not found_test_infra_reference: + errors.append("no test-infra workflow references were found") + expected_release_branch = f"release/{release_version}" + expected_swiftpm_branch = f"swiftpm-{release_full_version}" + for path in documentation: + text = path.read_text() + for match in _CLONE_BRANCH_PATTERN.finditer(text): + if match.group("branch") != expected_release_branch: + errors.append( + f"{path}: clone command references {match.group('branch')}, " + f"expected {expected_release_branch}" + ) + for swiftpm_branch in _STABLE_SWIFTPM_PATTERN.findall(text): + if swiftpm_branch != expected_swiftpm_branch: + errors.append( + f"{path}: stable SwiftPM example references {swiftpm_branch}, " + f"expected {expected_swiftpm_branch}" + ) + if errors: + raise RuntimeError("release preparation is incomplete:\n" + "\n".join(errors)) + + +def validate_release_files(repo_root: Path, release_version: str) -> str: + """Validate release version, API snapshot, and stable QNN installation.""" + actual_version = (repo_root / "version.txt").read_text().strip() + if re.fullmatch(rf"{re.escape(release_version)}\.\d+", actual_version) is None: + raise RuntimeError( + f"version.txt contains {actual_version!r}, expected a final " + f"{release_version}.x version" + ) + + manifest_dir = repo_root / "backends/arm/public_api_manifests" + if manifest_dir.exists(): + expected_manifest = ( + manifest_dir / f"api_manifest_{release_version.replace('.', '_')}.toml" + ) + if not expected_manifest.exists(): + raise RuntimeError(f"missing release API snapshot {expected_manifest}") + static_manifests = list(manifest_dir.glob("api_manifest_[0-9]*_[0-9]*.toml")) + if len(static_manifests) > 2: + raise RuntimeError("more than two static Arm API manifests remain") + + return actual_version + + +def prepare_release( + repo_root: Path, + release_version: str, + test_infra_branch: str, + torch_version: str, +) -> str: + """Apply every deterministic branch-cut edit and return the Torch requirement.""" + _validate_release_version(release_version) + if not test_infra_branch: + raise RuntimeError("test-infra branch must not be empty") + + workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + documentation = documentation_paths(repo_root) + set_torch_version(repo_root / "torch_pin.py", torch_version) + enable_release_wheel(repo_root / "torch_pin.py") + configure_release_version(repo_root / "version.txt", release_version) + workflow_count = configure_workflows(workflow_paths, test_infra_branch) + documentation_count = configure_documentation( + documentation, f"release/{release_version}", release_version + ) + manifest_created = freeze_arm_public_api(repo_root, release_version) + validate_release_references( + workflow_paths, + documentation, + test_infra_branch=test_infra_branch, + release_version=release_version, + release_full_version=f"{release_version}.0", + ) + validate_release_files(repo_root, release_version) + requirement = validate_release_build(repo_root / "torch_pin.py") + print( + f"Prepared release/{release_version}: {requirement}; changed " + f"{workflow_count} workflow files and {documentation_count} documentation files; " + f"Arm API snapshot {'created' if manifest_created else 'already present'}" + ) + return requirement + + +def _release_version_from_file(repo_root: Path) -> str: + version = (repo_root / "version.txt").read_text().strip() + match = re.match(r"(\d+\.\d+)", version) + if match is None: + raise RuntimeError(f"could not derive release version from {version!r}") + return match.group(1) + + +def main() -> None: + """Update or validate the repository's release configuration.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--check", + action="store_true", + help="validate the release configuration without changing it", + ) + parser.add_argument("--release-version", help="release version in MAJOR.MINOR form") + parser.add_argument( + "--test-infra-branch", + help="test-infra release branch (defaults to release/MAJOR.MINOR)", + ) + parser.add_argument( + "--torch-version", + help="Torch RC version (defaults to the newest release on the test index)", + ) + parser.add_argument( + "--repo-root", type=Path, default=Path(__file__).resolve().parents[2] + ) + args = parser.parse_args() + + repo_root = args.repo_root.resolve() + release_version = args.release_version or _release_version_from_file(repo_root) + test_infra_branch = args.test_infra_branch or f"release/{release_version}" + workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) + documentation = documentation_paths(repo_root) + + if args.check: + _validate_release_version(release_version) + requirement = validate_release_build(repo_root / "torch_pin.py") + release_full_version = validate_release_files(repo_root, release_version) + validate_release_references( + workflow_paths, + documentation, + test_infra_branch=test_infra_branch, + release_version=release_version, + release_full_version=release_full_version, + ) + print(f"Release configuration is valid: {requirement}") + else: + torch_version = torch_version_for_release( + repo_root / "torch_pin.py", args.torch_version or "" + ) + prepare_release(repo_root, release_version, test_infra_branch, torch_version) + + +if __name__ == "__main__": + main() diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py new file mode 100644 index 00000000000..63fd933475b --- /dev/null +++ b/scripts/release/test_finalize_release.py @@ -0,0 +1,94 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import tempfile +import unittest +from pathlib import Path + +from finalize_release import ( + finalize_dependency_text, + finalize_torch_release, + stable_base_version, +) + + +class FinalizeReleaseTest(unittest.TestCase): + """Tests for delayed release dependency finalization.""" + + def test_promotes_release_candidate_to_final(self) -> None: + self.assertEqual(stable_base_version("0.19.0.dev20260907"), "0.19.0") + self.assertEqual(stable_base_version("0.19.0rc2"), "0.19.0") + self.assertEqual(stable_base_version("0.19.0"), "0.19.0") + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "torch_pin.py" + path.write_text( + 'TORCH_VERSION = "2.15.0rc2"\n' + "RELEASE_WHEEL = True\n" + "RELEASE_DEPENDENCIES_FINALIZED = False\n" + ) + + self.assertEqual(finalize_torch_release(path, "2.15.0"), 2) + self.assertIn('TORCH_VERSION = "2.15.0"', path.read_text()) + self.assertIn("RELEASE_DEPENDENCIES_FINALIZED = True", path.read_text()) + + def test_finalizes_dependency_text(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "install_requirements.py").write_text( + 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' + ) + (root / "setup.py").write_text(' "pytorch-tokenizers",\n') + qnn_path = root / ".ci/scripts/test_wheel_package_qnn.sh" + qnn_path.parent.mkdir(parents=True) + qnn_path.write_text( + '"$PIPBIN" install torch=="${TORCH_VERSION}" ' + '--extra-index-url "https://download.pytorch.org/whl/test"\n' + ) + for relative in ( + ".ci/scripts/test-rocm-aoti.sh", + ".ci/scripts/test-rocm-voxtral.sh", + ".ci/scripts/test_model_e2e.sh", + "examples/models/moshi/mimi/install_requirements.sh", + ): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + "pip install torchcodec==0.11.0 " + "--extra-index-url https://download.pytorch.org/whl/test/cpu\n" + 'PYTORCH_ROCM_INDEX="https://download.pytorch.org/whl/test/' + 'rocm${ROCM_VERSION}"\n' + 'TORCHAO_ROCM_WHEEL_BASE="https://download.pytorch.org/whl/' + 'nightly/rocm${ROCM_VERSION}"\n' + ) + + changed = finalize_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") + + self.assertEqual(changed, 7) + self.assertIn( + 'TORCHAO_NIGHTLY_VERSION = "0.19.0"', + (root / "install_requirements.py").read_text(), + ) + self.assertIn( + '"pytorch-tokenizers>=1.6.0"', (root / "setup.py").read_text() + ) + for relative in ( + ".ci/scripts/test-rocm-voxtral.sh", + ".ci/scripts/test_model_e2e.sh", + "examples/models/moshi/mimi/install_requirements.sh", + ): + text = (root / relative).read_text() + self.assertIn("torchcodec==0.17.0", text) + self.assertNotIn("whl/test/cpu", text) + rocm = (root / ".ci/scripts/test-rocm-voxtral.sh").read_text() + self.assertNotIn("whl/test/rocm", rocm) + self.assertNotIn("whl/nightly/rocm", rocm) + self.assertIn("--no-cache-dir", qnn_path.read_text()) + self.assertIn("--index-url", qnn_path.read_text()) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py new file mode 100644 index 00000000000..b3bab16d6f1 --- /dev/null +++ b/scripts/release/test_prepare_release.py @@ -0,0 +1,82 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +import tempfile +import unittest +from pathlib import Path + +from prepare_release import ( + newest_torch_test_release, + prepare_release, + torch_version_for_release, +) + + +class PrepareReleaseTest(unittest.TestCase): + """Tests for release-only branch preparation.""" + + def test_selects_release_candidate_once(self) -> None: + """A branch cut selects the newest RC and preserves it on reruns.""" + releases = ["2.14.1", "2.15.0a1", "2.15.0b2", "2.15.0rc1"] + self.assertEqual(newest_torch_test_release(releases), "2.15.0rc1") + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "torch_pin.py" + path.write_text('TORCH_VERSION = "2.15.0rc1"\nRELEASE_WHEEL = True\n') + + self.assertEqual(torch_version_for_release(path), "2.15.0rc1") + + def test_prepares_complete_repository(self) -> None: + """The top-level operation applies dependency, workflow, and docs edits.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / ".github/workflows").mkdir(parents=True) + (root / "docs").mkdir() + (root / "version.txt").write_text("1.6.0a0\n") + (root / "torch_pin.py").write_text( + 'TORCH_VERSION = "2.15.0"\nRELEASE_WHEEL = False\n' + ) + workflow = root / ".github/workflows/test.yml" + workflow.write_text( + "uses: pytorch/test-infra/.github/workflows/test.yml@main\n" + "test-infra-ref: main\n" + "uses: example/action@main\n" + ) + doc = root / "docs/README.md" + doc.write_text( + "git clone -b viable/strict repo\n" + "stable: swiftpm-1.5.1\n" + "nightly: swiftpm-1.6.0.20260929\n" + ) + manifests = root / "backends/arm/public_api_manifests" + manifests.mkdir(parents=True) + (manifests / "api_manifest_1_4.toml").write_text("old\n") + (manifests / "api_manifest_1_5.toml").write_text("previous\n") + (manifests / "api_manifest_running.toml").write_text("current\n") + + requirement = prepare_release(root, "1.6", "release/1.6", "2.15.0rc1") + second_requirement = prepare_release( + root, "1.6", "release/1.6", "2.15.0rc1" + ) + + self.assertEqual(requirement, "torch>=2.15.0rc1") + self.assertEqual(second_requirement, requirement) + self.assertIn("RELEASE_WHEEL = True", (root / "torch_pin.py").read_text()) + self.assertIn("@release/1.6", workflow.read_text()) + self.assertIn("test-infra-ref: release/1.6", workflow.read_text()) + self.assertIn("example/action@main", workflow.read_text()) + self.assertIn("-b release/1.6", doc.read_text()) + self.assertIn("stable: swiftpm-1.6.0", doc.read_text()) + self.assertIn("nightly: swiftpm-1.6.0.20260929", doc.read_text()) + self.assertEqual((root / "version.txt").read_text(), "1.6.0\n") + self.assertFalse((manifests / "api_manifest_1_4.toml").exists()) + self.assertEqual( + (manifests / "api_manifest_1_6.toml").read_text(), "current\n" + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/setup.py b/setup.py index c0a23cb0195..cb33757a75c 100644 --- a/setup.py +++ b/setup.py @@ -78,6 +78,15 @@ raise ImportError(f"Module spec has no loader for {_install_utils_path}") _spec.loader.exec_module(install_utils) +_torch_pin_path = Path(__file__).parent / "torch_pin.py" +_spec = importlib.util.spec_from_file_location("torch_pin", _torch_pin_path) +if _spec is None: + raise ImportError(f"Could not create module spec for {_torch_pin_path}") +torch_pin = importlib.util.module_from_spec(_spec) +if _spec.loader is None: + raise ImportError(f"Could not load {_torch_pin_path}") +_spec.loader.exec_module(torch_pin) + from setuptools import Distribution, Extension, find_namespace_packages, setup from setuptools.command.build import build from setuptools.command.build_ext import build_ext @@ -1116,6 +1125,12 @@ def _torchao_requirement() -> str: return f"torchao>={version},<{major}.{minor + 1}" +def _release_torch_requirement() -> List[str]: + if not torch_pin.RELEASE_WHEEL: + return [] + return [f"torch>={torch_pin.TORCH_VERSION}"] + + def _base_dependencies() -> List[str]: """Runtime dependencies for the full wheel. @@ -1140,6 +1155,7 @@ def _base_dependencies() -> List[str]: "py-cpuinfo", "requests", "pytorch-tokenizers", + *_release_torch_requirement(), # Shipped code imports torchao at module scope in many places, so a plain install cannot # lower a model without it. Among others: the XNNPACK utilities the partitioner uses # (backends/xnnpack/utils/utils.py), the Core ML quantizer, and executorch.export itself. @@ -1197,7 +1213,8 @@ def _minimal_dependencies() -> List[str]: Derived as the subset of _base_dependencies() that executorch.exir needs to lower and serialize a .pte, so version pins and markers stay in sync with the - full set. torch is intentionally absent from both (consumers bring their own). + full set. torch is intentionally absent from the minimal wheel, including on + release branches (consumers bring their own). mpmath is intentionally dropped too: it is pulled transitively by sympy, whose "mpmath<1.4" cap resolves to the same 1.3.0 the full wheel pins. Keep the name set below in sync with the `expected` set in .ci/scripts/test_minimal_wheel.sh. diff --git a/torch_pin.py b/torch_pin.py index 3010f4a22ed..b1ee639bb66 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -1,2 +1,10 @@ +# PyTorch release used by development installs and release-wheel metadata. TORCH_VERSION = "2.14.0" NIGHTLY_VERSION = "dev20260913" + +# Changed to True by scripts/release/apply-release-changes.sh. Release wheels +# declare the PyTorch release above; development and minimal wheels do not. +RELEASE_WHEEL = False + +# Changed to True after stable third-party artifacts and submodule tags exist. +RELEASE_DEPENDENCIES_FINALIZED = False From 41a9655850ab9cd240f9dff2feb2fd6abed1470b Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Wed, 30 Sep 2026 14:00:52 -0700 Subject: [PATCH 02/12] Default release cuts from viable strict --- scripts/release/cut-release-branch.sh | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/scripts/release/cut-release-branch.sh b/scripts/release/cut-release-branch.sh index 36b09b5b89e..e8be3cec616 100755 --- a/scripts/release/cut-release-branch.sh +++ b/scripts/release/cut-release-branch.sh @@ -9,15 +9,12 @@ So you are looking to cut a release branch? Well you came to the right script. -This script can be used to cut any branch on any repository - -For `pytorch/executorch` usage would be like: +For `pytorch/executorch`, run: > DRY_RUN=disabled ./scripts/release/cut-release-branch.sh -or to cut from main branch: -> DRY_RUN=disabled GIT_BRANCH_TO_CUT_FROM=main ./scripts/release/cut-release-branch.sh - -After creating release/X.Y and orig/release/X.Y at the selected commit, this +The script always cuts from origin/viable/strict. RELEASE_VERSION defaults to +the MAJOR.MINOR value in its version.txt (for example, 1.10.0a0 +becomes release/1.10). After creating release/X.Y and orig/release/X.Y, this also applies the deterministic release-only changes to the local release/X.Y branch. TEST_INFRA_BRANCH defaults to release/X.Y. TORCH_VERSION may override the newest PyTorch release candidate selected from the test wheel index. @@ -27,11 +24,6 @@ set -eou pipefail GIT_TOP_DIR=$(git rev-parse --show-toplevel) GIT_REMOTE=${GIT_REMOTE:-origin} -GIT_BRANCH_TO_CUT_FROM=${GIT_BRANCH_TO_CUT_FROM:-viable/strict} - -# should output something like 1.11 -RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} -TEST_INFRA_BRANCH=${TEST_INFRA_BRANCH:-release/${RELEASE_VERSION}} DRY_RUN_FLAG="--dry-run" if [[ ${DRY_RUN:-enabled} == "disabled" ]]; then @@ -42,9 +34,15 @@ fi ( set -x git fetch --all - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" + git checkout "${GIT_REMOTE}/viable/strict" ) +# Read the version only after checking out viable/strict, so the caller's +# starting branch cannot influence which release is cut. This preserves all +# numeric components (for example, 1.10 does not become 1.1). +RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} +TEST_INFRA_BRANCH=${TEST_INFRA_BRANCH:-release/${RELEASE_VERSION}} + for branch in "release/${RELEASE_VERSION}" "orig/release/${RELEASE_VERSION}"; do if git rev-parse --verify "${branch}" >/dev/null 2>/dev/null; then echo "+ Branch ${branch} already exists, skipping..." @@ -52,7 +50,7 @@ for branch in "release/${RELEASE_VERSION}" "orig/release/${RELEASE_VERSION}"; do else ( set -x - git checkout "${GIT_REMOTE}/${GIT_BRANCH_TO_CUT_FROM}" + git checkout "${GIT_REMOTE}/viable/strict" git checkout -b "${branch}" git push -q ${DRY_RUN_FLAG} "${GIT_REMOTE}" "${branch}" ) From e672963ad14509a830e26a9017c51417faf75b90 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Wed, 30 Sep 2026 14:19:19 -0700 Subject: [PATCH 03/12] Fix release automation preflight and RC handling --- .ci/scripts/tests/test_cu134_dependencies.py | 41 +++- .ci/scripts/wheel/test_clean_install.py | 15 +- scripts/release/apply-release-changes.sh | 20 +- scripts/release/cut-release-branch.sh | 68 +++--- scripts/release/finalize_release.py | 223 +++++++++++++------ scripts/release/prepare_release.py | 96 +++++--- scripts/release/test_finalize_release.py | 16 +- scripts/release/test_prepare_release.py | 17 +- setup.py | 15 ++ 9 files changed, 358 insertions(+), 153 deletions(-) diff --git a/.ci/scripts/tests/test_cu134_dependencies.py b/.ci/scripts/tests/test_cu134_dependencies.py index d087cea17dc..46402dcb907 100644 --- a/.ci/scripts/tests/test_cu134_dependencies.py +++ b/.ci/scripts/tests/test_cu134_dependencies.py @@ -11,6 +11,7 @@ import sys import unittest from pathlib import Path +from types import SimpleNamespace from unittest.mock import patch from packaging.requirements import Requirement @@ -164,6 +165,30 @@ def torchao_requirement(self): ) return namespace["_torchao_requirement"]() + def release_torch_requirement(self): + path = ROOT / "setup.py" + tree = ast.parse(path.read_text()) + function = next( + node + for node in tree.body + if isinstance(node, ast.FunctionDef) + and node.name == "_release_torch_requirement" + ) + namespace = { + "__file__": str(path), + "Path": Path, + "List": list, + "importlib": importlib, + "sys": sys, + "install_utils": self.utils, + "torch_pin": SimpleNamespace(RELEASE_WHEEL=True, TORCH_VERSION="2.15.0rc1"), + } + exec( + compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), + namespace, + ) + return namespace["_release_torch_requirement"]()[0] + def test_package_install_preserves_source_pinned_torchao(self): with patch.dict(sys.modules, {"install_requirements": self.installer}): package_installer = load_module("install_executorch") @@ -241,11 +266,11 @@ def test_cu134_keeps_explicit_torchao_source_build(self): self.assertIn("torch==2.14.0.dev20260810+cu134", commands[-1]) self.assertIn("0.19.0+gitb7ac3aa", metadata.specifier) - def test_wheel_torchao_bound_matches_selected_train(self): - for cuda, expected in ( - ((13, 4), "torchao>=0.19.0.dev20260907,<0.20"), - ((13, 2), "torchao>=0.19.0.dev20260907,<0.20"), - (None, "torchao>=0.19.0.dev20260907,<0.20"), + def test_wheel_bounds_match_selected_train(self): + for cuda, expected_torch in ( + ((13, 4), "torch==2.14.0.dev20260810+cu134"), + ((13, 2), "torch>=2.15.0rc1"), + (None, "torch>=2.15.0rc1"), ): self.utils.determine_torch_url.cache_clear() with ( @@ -257,7 +282,11 @@ def test_wheel_torchao_bound_matches_selected_train(self): ), patch.object(self.installer.platform, "system", return_value="Linux"), ): - self.assertEqual(self.torchao_requirement(), expected) + self.assertEqual( + self.torchao_requirement(), + "torchao>=0.19.0.dev20260907,<0.20", + ) + self.assertEqual(self.release_torch_requirement(), expected_torch) if __name__ == "__main__": diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index 82e26de6f8b..36e6bd556f7 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -59,6 +59,7 @@ def test_release_pytorch_requirement() -> None: from packaging.requirements import Requirement from packaging.utils import canonicalize_name + from packaging.version import Version repo_root = Path(__file__).resolve().parents[3] config = runpy.run_path(str(repo_root / "torch_pin.py")) @@ -68,10 +69,12 @@ def test_release_pytorch_requirement() -> None: os.environ.get("GITHUB_REF", "").removeprefix("refs/heads/"), ) release_tag_ci = any( - re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+(?:-rc\d+)?", ref) for ref in ci_refs + re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+", ref) for ref in ci_refs ) - release_ci = any(ref.startswith("release/") for ref in ci_refs) or any( - re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+(?:-rc\d+)?", ref) for ref in ci_refs + release_ci = release_tag_ci or any( + ref.startswith("release/") + or re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+-rc\d+", ref) + for ref in ci_refs ) assert not release_ci or config["RELEASE_WHEEL"], ( "this wheel is being built from a release ref, but torch_pin.py does not enable " @@ -95,7 +98,11 @@ def test_release_pytorch_requirement() -> None: ] if config["RELEASE_WHEEL"]: - expected = f">={config['TORCH_VERSION']}" + installed_torch = Version(metadata.version("torch")) + if installed_torch.local and "cu134" in installed_torch.local: + expected = f"=={installed_torch}" + else: + expected = f">={config['TORCH_VERSION']}" assert len(torch_requirements) == 1, ( "a release wheel must declare exactly one PyTorch dependency, but found " f"{[str(requirement) for requirement in torch_requirements]}" diff --git a/scripts/release/apply-release-changes.sh b/scripts/release/apply-release-changes.sh index f7462b959f2..8c5cc7851d7 100755 --- a/scripts/release/apply-release-changes.sh +++ b/scripts/release/apply-release-changes.sh @@ -12,9 +12,9 @@ # Usage (run from root of project): # TEST_INFRA_BRANCH=release/2.3 ./scripts/release/apply-release-changes.sh # -# TEST_INFRA_BRANCH: The release branch of test-infra that houses all reusable -# workflows. Defaults to the matching release/X.Y branch. The script also -# enables the PyTorch dependency for full release wheels using torch_pin.py. +# TEST_INFRA_BRANCH: Optional override for the test-infra release branch. It +# normally defaults to the release/X.Y branch matching the selected PyTorch RC. +# The script also enables the PyTorch dependency for full release wheels. # TORCH_VERSION: Optional PyTorch RC version override. By default, the newest # release-form version available on the PyTorch test index is selected. ' @@ -25,7 +25,6 @@ GIT_TOP_DIR=$(git rev-parse --show-toplevel) RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} RELEASE_BRANCH="release/${RELEASE_VERSION}" PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} -TEST_INFRA_BRANCH=${TEST_INFRA_BRANCH:-${RELEASE_BRANCH}} # Check out to Release Branch @@ -42,18 +41,15 @@ fi PREPARE_ARGS=( --release-version "${RELEASE_VERSION}" - --test-infra-branch "${TEST_INFRA_BRANCH}" ) +if [[ -n "${TEST_INFRA_BRANCH:-}" ]]; then + PREPARE_ARGS+=(--test-infra-branch "${TEST_INFRA_BRANCH}") +fi if [[ -n "${TORCH_VERSION:-}" ]]; then PREPARE_ARGS+=(--torch-version "${TORCH_VERSION}") fi "${PYTHON_EXECUTABLE}" "${GIT_TOP_DIR}/scripts/release/prepare_release.py" "${PREPARE_ARGS[@]}" -echo "You'll need to manually commit the changes and create a PR. Here are the steps:" -echo "1. Stage the changes:" -echo " git add .github/workflows/*.yml CONTRIBUTING.md docs/ extension/benchmark/apple/Benchmark/ backends/arm/public_api_manifests/ version.txt torch_pin.py" -echo "2. Commit the changes:" -echo " git commit -m \"[RELEASE-ONLY CHANGES] Branch Cut for Release ${RELEASE_VERSION}\"" -echo "3. After committing, create a pull request to merge the changes." -echo "4. When stable dependency artifacts are published, run:" +echo "Release changes are ready for review. This script does not commit or push them." +echo "When stable dependency artifacts are published, run:" echo " python scripts/release/finalize_release.py" diff --git a/scripts/release/cut-release-branch.sh b/scripts/release/cut-release-branch.sh index e8be3cec616..4ccc78875ef 100755 --- a/scripts/release/cut-release-branch.sh +++ b/scripts/release/cut-release-branch.sh @@ -14,52 +14,64 @@ For `pytorch/executorch`, run: The script always cuts from origin/viable/strict. RELEASE_VERSION defaults to the MAJOR.MINOR value in its version.txt (for example, 1.10.0a0 -becomes release/1.10). After creating release/X.Y and orig/release/X.Y, this -also applies the deterministic release-only changes to the local release/X.Y -branch. TEST_INFRA_BRANCH defaults to release/X.Y. TORCH_VERSION may override -the newest PyTorch release candidate selected from the test wheel index. +becomes release/1.10). It preserves the unmodified cut as orig/release/X.Y, +then prepares, commits, and pushes release/X.Y. TORCH_VERSION may override the +newest PyTorch release candidate selected from the test wheel index. ' set -eou pipefail GIT_TOP_DIR=$(git rev-parse --show-toplevel) GIT_REMOTE=${GIT_REMOTE:-origin} +PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python3} -DRY_RUN_FLAG="--dry-run" -if [[ ${DRY_RUN:-enabled} == "disabled" ]]; then - DRY_RUN_FLAG="" +if [[ -n "$(git status --porcelain)" ]]; then + echo "Error: release cuts require a clean checkout." + exit 1 fi - ( set -x git fetch --all - git checkout "${GIT_REMOTE}/viable/strict" ) -# Read the version only after checking out viable/strict, so the caller's -# starting branch cannot influence which release is cut. This preserves all -# numeric components (for example, 1.10 does not become 1.1). -RELEASE_VERSION=${RELEASE_VERSION:-$(cut -d'.' -f1-2 "${GIT_TOP_DIR}/version.txt")} -TEST_INFRA_BRANCH=${TEST_INFRA_BRANCH:-release/${RELEASE_VERSION}} +# Read version.txt directly from viable/strict so the caller's starting branch +# cannot influence which release is cut. This preserves all numeric components +# (for example, 1.10 does not become 1.1). +SOURCE_BRANCH="${GIT_REMOTE}/viable/strict" +SOURCE_VERSION=$(git show "${SOURCE_BRANCH}:version.txt") +RELEASE_VERSION=${RELEASE_VERSION:-$(printf '%s\n' "${SOURCE_VERSION}" | cut -d'.' -f1-2)} +RELEASE_BRANCH="release/${RELEASE_VERSION}" +ORIGINAL_BRANCH="orig/${RELEASE_BRANCH}" + +if [[ ${DRY_RUN:-enabled} != "disabled" ]]; then + echo "Dry run: would preserve ${SOURCE_BRANCH} as ${ORIGINAL_BRANCH}" + echo "Dry run: would prepare, commit, and push ${RELEASE_BRANCH}" + exit 0 +fi + +if ! git ls-remote --exit-code "${GIT_REMOTE}" "refs/heads/${ORIGINAL_BRANCH}" >/dev/null 2>&1; then + git push "${GIT_REMOTE}" "${SOURCE_BRANCH}:refs/heads/${ORIGINAL_BRANCH}" +fi -for branch in "release/${RELEASE_VERSION}" "orig/release/${RELEASE_VERSION}"; do - if git rev-parse --verify "${branch}" >/dev/null 2>/dev/null; then - echo "+ Branch ${branch} already exists, skipping..." - continue - else - ( - set -x - git checkout "${GIT_REMOTE}/viable/strict" - git checkout -b "${branch}" - git push -q ${DRY_RUN_FLAG} "${GIT_REMOTE}" "${branch}" - ) - fi -done +if git show-ref --verify --quiet "refs/heads/${RELEASE_BRANCH}"; then + git checkout "${RELEASE_BRANCH}" +elif git ls-remote --exit-code "${GIT_REMOTE}" "refs/heads/${RELEASE_BRANCH}" >/dev/null 2>&1; then + git checkout -b "${RELEASE_BRANCH}" "${GIT_REMOTE}/${RELEASE_BRANCH}" +else + git checkout -b "${RELEASE_BRANCH}" "${SOURCE_BRANCH}" +fi ( set -x RELEASE_VERSION="${RELEASE_VERSION}" \ - TEST_INFRA_BRANCH="${TEST_INFRA_BRANCH}" \ "${GIT_TOP_DIR}/scripts/release/apply-release-changes.sh" ) + +git add -A +if git diff --cached --quiet; then + echo "Release branch is already prepared; nothing to commit." +else + git commit -m "[RELEASE-ONLY CHANGES] Branch Cut for Release ${RELEASE_VERSION}" +fi +git push --set-upstream "${GIT_REMOTE}" "${RELEASE_BRANCH}" diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py index 626a62fc1f0..fba2fadbb94 100644 --- a/scripts/release/finalize_release.py +++ b/scripts/release/finalize_release.py @@ -17,7 +17,11 @@ from pathlib import Path from typing import Iterable -from prepare_release import _release_version_from_file, set_torch_version +from prepare_release import ( + _release_version_from_file, + _TORCH_VERSION_PATTERN, + _write_if_changed, +) _VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") _TORCHCODEC_PATTERN = re.compile(r"torchcodec==\d+\.\d+\.\d+") @@ -27,6 +31,12 @@ _TORCHAO_ASSIGNMENT_PATTERN = re.compile( r'^TORCHAO_NIGHTLY_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE ) +_ROCM_TORCHAO_ASSIGNMENT_PATTERN = re.compile( + r'^ROCM_TORCHAO_NIGHTLY_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE +) +_ROCM_VERSION_PATTERN = re.compile( + r'^ROCM_VERSION="\$\{ROCM_VERSION:-(?P\d+\.\d+)\}"$', re.MULTILINE +) _FINALIZED_PATTERN = re.compile( r"^RELEASE_DEPENDENCIES_FINALIZED\s*=\s*(?:True|False)$", re.MULTILINE ) @@ -118,104 +128,126 @@ def require_remote_tag(repository: str, version: str) -> None: raise RuntimeError(f"{repository} does not have tag v{version}") -def _write_if_changed(path: Path, updated: str) -> bool: - original = path.read_text() - if updated == original: - return False - path.write_text(updated) - return True +def require_rocm_torchao_release(repo_root: Path, version: str) -> None: + """Verify the stable ROCm TorchAO artifact used by release CI exists.""" + rocm_script = repo_root / _ROCM_PATHS[0] + match = _ROCM_VERSION_PATTERN.search(rocm_script.read_text()) + if match is None: + raise RuntimeError(f"could not determine ROCm version from {rocm_script}") + rocm_version = match.group("version") + url = ( + f"https://download.pytorch.org/whl/rocm{rocm_version}/" + f"torchao-{version}%2Brocm{rocm_version}-" + "cp310-abi3-manylinux_2_28_x86_64.whl" + ) + try: + with urllib.request.urlopen(url, timeout=30): + pass + except urllib.error.HTTPError as error: + if error.code == 404: + raise RuntimeError( + f"TorchAO {version} is not published for ROCm {rocm_version}" + ) from error + raise -def mark_release_dependencies_finalized(torch_pin_path: Path) -> bool: - """Record that delayed dependency finalization completed successfully.""" +def planned_torch_release(torch_pin_path: Path, torch_version: str) -> str: + """Plan the RC-to-final pin update without changing the checkout.""" + if _VERSION_PATTERN.fullmatch(torch_version) is None: + raise RuntimeError( + f"final PyTorch version must be X.Y.Z, got {torch_version!r}" + ) text = torch_pin_path.read_text() + updated, version_count = _TORCH_VERSION_PATTERN.subn( + f'TORCH_VERSION = "{torch_version}"', text + ) + if version_count != 1: + raise RuntimeError(f"expected one TORCH_VERSION assignment in {torch_pin_path}") updated, count = _FINALIZED_PATTERN.subn( - "RELEASE_DEPENDENCIES_FINALIZED = True", text + "RELEASE_DEPENDENCIES_FINALIZED = True", updated ) if count != 1: raise RuntimeError( f"expected one RELEASE_DEPENDENCIES_FINALIZED assignment in {torch_pin_path}" ) - return _write_if_changed(torch_pin_path, updated) + return updated def finalize_torch_release(torch_pin_path: Path, torch_version: str) -> int: """Replace the RC pin with the final release and mark finalization complete.""" - changed = int(set_torch_version(torch_pin_path, torch_version)) - changed += int(mark_release_dependencies_finalized(torch_pin_path)) - return changed + updated = planned_torch_release(torch_pin_path, torch_version) + return int(_write_if_changed(torch_pin_path, updated)) -def _finalize_torchao(repo_root: Path, version: str) -> set[Path]: +def plan_dependency_text( + repo_root: Path, + torchao_version: str, + tokenizers_version: str, + torchcodec_version: str, +) -> dict[Path, str]: + """Validate and calculate every text edit without changing the checkout.""" + updates: dict[Path, str] = {} + + def read(path: Path) -> str: + if not path.exists(): + raise RuntimeError(f"expected release dependency file {path}") + return updates.get(path, path.read_text()) + path = repo_root / "install_requirements.py" - text = path.read_text() updated, count = _TORCHAO_ASSIGNMENT_PATTERN.subn( - f'TORCHAO_NIGHTLY_VERSION = "{version}"', text + f'TORCHAO_NIGHTLY_VERSION = "{torchao_version}"', read(path) ) if count != 1: raise RuntimeError(f"expected one TORCHAO_NIGHTLY_VERSION in {path}") - return {path} if _write_if_changed(path, updated) else set() - + updated, count = _ROCM_TORCHAO_ASSIGNMENT_PATTERN.subn( + f'ROCM_TORCHAO_NIGHTLY_VERSION = "{torchao_version}"', updated + ) + if count != 1: + raise RuntimeError(f"expected one ROCM_TORCHAO_NIGHTLY_VERSION in {path}") + updates[path] = updated -def _finalize_tokenizers(repo_root: Path, version: str) -> set[Path]: path = repo_root / "setup.py" - text = path.read_text() updated, count = _TOKENIZERS_REQUIREMENT_PATTERN.subn( - rf'\g"pytorch-tokenizers>={version}",', text + rf'\g"pytorch-tokenizers>={tokenizers_version}",', read(path) ) if count != 1: raise RuntimeError(f"expected one pytorch-tokenizers requirement in {path}") - return {path} if _write_if_changed(path, updated) else set() - + updates[path] = updated -def _finalize_torchcodec(repo_root: Path, version: str) -> set[Path]: - changed_paths = set() for relative in _TORCHCODEC_PATHS: path = repo_root / relative - if not path.exists(): - raise RuntimeError(f"expected release dependency file {path}") - text = path.read_text() - updated, count = _TORCHCODEC_PATTERN.subn(f"torchcodec=={version}", text) + updated, count = _TORCHCODEC_PATTERN.subn( + f"torchcodec=={torchcodec_version}", read(path) + ) if count == 0: raise RuntimeError(f"expected a torchcodec pin in {path}") - updated = updated.replace( + updates[path] = updated.replace( "--extra-index-url https://download.pytorch.org/whl/test/cpu", "--index-url https://download.pytorch.org/whl/cpu", ) - if _write_if_changed(path, updated): - changed_paths.add(path) - return changed_paths - -def _finalize_stable_indexes(repo_root: Path) -> set[Path]: - changed_paths = set() for relative in _ROCM_PATHS: path = repo_root / relative - if not path.exists(): - raise RuntimeError(f"expected release dependency file {path}") - text = path.read_text() - updated = text.replace( - "https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}", - "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", - ).replace( - "https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}", - "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", + updates[path] = ( + read(path) + .replace( + "https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}", + "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", + ) + .replace( + "https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}", + "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", + ) ) - if _write_if_changed(path, updated): - changed_paths.add(path) qnn_path = repo_root / ".ci/scripts/test_wheel_package_qnn.sh" - if not qnn_path.exists(): - raise RuntimeError(f"expected release dependency file {qnn_path}") - text = qnn_path.read_text() - updated = text.replace(_QNN_TEST_INDEX, _QNN_STABLE_INDEX) + updated = read(qnn_path).replace(_QNN_TEST_INDEX, _QNN_STABLE_INDEX) if _QNN_STABLE_INDEX not in updated: raise RuntimeError( f"could not find the QNN Torch install command in {qnn_path}" ) - if _write_if_changed(qnn_path, updated): - changed_paths.add(qnn_path) - return changed_paths + updates[qnn_path] = updated + return {path: text for path, text in updates.items() if text != path.read_text()} def finalize_dependency_text( @@ -225,17 +257,19 @@ def finalize_dependency_text( torchcodec_version: str, ) -> int: """Update all recurring stable dependency declarations and index URLs.""" - changed_paths = _finalize_torchao(repo_root, torchao_version) - changed_paths |= _finalize_tokenizers(repo_root, tokenizers_version) - changed_paths |= _finalize_torchcodec(repo_root, torchcodec_version) - changed_paths |= _finalize_stable_indexes(repo_root) - return len(changed_paths) + updates = plan_dependency_text( + repo_root, torchao_version, tokenizers_version, torchcodec_version + ) + for path, updated in updates.items(): + _write_if_changed(path, updated) + return len(updates) -def update_submodule_tags( +def prepare_submodule_tags( repo_root: Path, releases: Iterable[tuple[str, str, str]] -) -> None: - """Check out verified release tags in the dependency submodules.""" +) -> list[tuple[Path, str, str]]: + """Fetch and resolve every submodule tag before changing any gitlink.""" + prepared = [] for relative, _repository, version in releases: path = repo_root / relative if (path / ".git").exists() and subprocess.run( @@ -264,9 +298,39 @@ def update_submodule_tags( cwd=path, check=True, ) - subprocess.run( - ["git", "checkout", "--detach", f"v{version}"], cwd=path, check=True - ) + current = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + target = subprocess.run( + ["git", "rev-parse", f"v{version}^{{commit}}"], + cwd=path, + check=True, + capture_output=True, + text=True, + ).stdout.strip() + prepared.append((path, current, target)) + return prepared + + +def update_submodule_tags(prepared: Iterable[tuple[Path, str, str]]) -> None: + """Apply prepared submodule updates, restoring prior heads on failure.""" + changed = [] + try: + for path, current, target in prepared: + subprocess.run( + ["git", "checkout", "--detach", target], cwd=path, check=True + ) + changed.append((path, current)) + except Exception: + for path, current in reversed(changed): + subprocess.run( + ["git", "checkout", "--detach", current], cwd=path, check=True + ) + raise def main() -> None: @@ -318,6 +382,7 @@ def main() -> None: require_pypi_release(package, version) for _path, repository, version in releases: require_remote_tag(repository, version) + require_rocm_torchao_release(repo_root, torchao_version) print( "Release dependency artifacts are available: " @@ -327,11 +392,31 @@ def main() -> None: if args.preflight_only: return - update_submodule_tags(repo_root, releases) - changed = finalize_dependency_text( + dependency_updates = plan_dependency_text( repo_root, torchao_version, tokenizers_version, torchcodec_version ) - changed += finalize_torch_release(repo_root / "torch_pin.py", torch_version) + torch_pin_path = repo_root / "torch_pin.py" + torch_pin_update = planned_torch_release(torch_pin_path, torch_version) + prepared_submodules = prepare_submodule_tags(repo_root, releases) + text_updates = {**dependency_updates, torch_pin_path: torch_pin_update} + original_text = {path: path.read_text() for path in text_updates} + + try: + update_submodule_tags(prepared_submodules) + for path, updated in text_updates.items(): + _write_if_changed(path, updated) + except Exception: + for path, original in original_text.items(): + path.write_text(original) + for path, original, _target in reversed(prepared_submodules): + subprocess.run( + ["git", "checkout", "--detach", original], cwd=path, check=False + ) + raise + + changed = sum( + updated != original_text[path] for path, updated in text_updates.items() + ) print( f"Finalized release dependencies; changed {changed} text files and 2 submodules" ) diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index 7c297cf3a26..d5a8fc8f6d6 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -11,6 +11,7 @@ import re import runpy import shutil +import subprocess import urllib.request from pathlib import Path from typing import Iterable, List @@ -25,9 +26,7 @@ _TEST_INFRA_REF_MAIN_PATTERN = re.compile(r"(test-infra-ref:\s*)main\b") _TEST_INFRA_BRANCH_PATTERN = re.compile(r"pytorch/test-infra/[^\s\"'@]+@([^\s\"']+)") _TEST_INFRA_INPUT_PATTERN = re.compile(r"test-infra-ref:\s*([^\s#]+)") -_CLONE_BRANCH_PATTERN = re.compile( - r"(?P-b\s+)(?Pviable/strict|release/\d+\.\d+)\b" -) +_CLONE_BRANCH_PATTERN = re.compile(r"(?P-b\s+)viable/strict\b") _STABLE_SWIFTPM_PATTERN = re.compile(r"swiftpm-\d+\.\d+\.\d+(?![.\d-])") _DOCUMENTATION_PATHS = ( "CONTRIBUTING.md", @@ -43,7 +42,8 @@ def _validate_release_version(release_version: str) -> None: ) -def _write_if_changed(path: Path, original: str, updated: str) -> bool: +def _write_if_changed(path: Path, updated: str) -> bool: + original = path.read_text() if updated == original: return False path.write_text(updated) @@ -58,7 +58,7 @@ def enable_release_wheel(torch_pin_path: Path) -> bool: raise RuntimeError( f"expected exactly one RELEASE_WHEEL assignment in {torch_pin_path}" ) - return _write_if_changed(torch_pin_path, text, updated) + return _write_if_changed(torch_pin_path, updated) def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: @@ -73,7 +73,7 @@ def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: raise RuntimeError( f"expected exactly one TORCH_VERSION assignment in {torch_pin_path}" ) - return _write_if_changed(torch_pin_path, text, updated) + return _write_if_changed(torch_pin_path, updated) def newest_torch_test_release(available_versions: Iterable[str]) -> str: @@ -117,6 +117,37 @@ def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: return latest_torch_test_release() +def test_infra_branch_for_torch(torch_version: str) -> str: + """Return the test-infra release branch matching a PyTorch release.""" + match = re.match(r"(\d+)\.(\d+)\.", torch_version) + if match is None: + raise RuntimeError( + f"could not derive a test-infra branch from Torch {torch_version!r}" + ) + return f"release/{match.group(1)}.{match.group(2)}" + + +def require_test_infra_branch(test_infra_branch: str) -> None: + """Fail before editing if the selected test-infra branch does not exist.""" + result = subprocess.run( + [ + "git", + "ls-remote", + "--exit-code", + "--heads", + "https://github.com/pytorch/test-infra.git", + test_infra_branch, + ], + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if result.returncode != 0: + raise RuntimeError( + f"pytorch/test-infra does not have branch {test_infra_branch!r}" + ) + + def validate_release_build(torch_pin_path: Path) -> str: """Return the release PyTorch requirement or raise for invalid config.""" config = runpy.run_path(str(torch_pin_path)) @@ -132,11 +163,14 @@ def validate_release_build(torch_pin_path: Path) -> str: return f"torch>={version}" -def configure_release_version(version_path: Path, release_version: str) -> bool: - """Set the package version to the first final release in this series.""" - return _write_if_changed( - version_path, version_path.read_text(), f"{release_version}.0\n" - ) +def configure_release_version(version_path: Path, release_version: str) -> str: + """Finalize a development version without resetting an existing patch release.""" + current = version_path.read_text().strip() + if re.fullmatch(rf"{re.escape(release_version)}\.\d+", current): + return current + release_full_version = f"{release_version}.0" + _write_if_changed(version_path, f"{release_full_version}\n") + return release_full_version def configure_workflows(workflow_paths: Iterable[Path], test_infra_branch: str) -> int: @@ -146,7 +180,7 @@ def configure_workflows(workflow_paths: Iterable[Path], test_infra_branch: str) text = path.read_text() updated = _TEST_INFRA_MAIN_PATTERN.sub(rf"\1@{test_infra_branch}", text) updated = _TEST_INFRA_REF_MAIN_PATTERN.sub(rf"\1{test_infra_branch}", updated) - changed += _write_if_changed(path, text, updated) + changed += _write_if_changed(path, updated) return changed @@ -163,15 +197,17 @@ def documentation_paths(repo_root: Path) -> List[Path]: def configure_documentation( - paths: Iterable[Path], release_branch: str, release_version: str + paths: Iterable[Path], release_branch: str, release_full_version: str ) -> int: """Point checkout and stable SwiftPM examples at the new release.""" changed = 0 for path in paths: text = path.read_text() updated = _CLONE_BRANCH_PATTERN.sub(rf"\g{release_branch}", text) - updated = _STABLE_SWIFTPM_PATTERN.sub(f"swiftpm-{release_version}.0", updated) - changed += _write_if_changed(path, text, updated) + updated = _STABLE_SWIFTPM_PATTERN.sub( + f"swiftpm-{release_full_version}", updated + ) + changed += _write_if_changed(path, updated) return changed @@ -226,12 +262,11 @@ def validate_release_references( expected_swiftpm_branch = f"swiftpm-{release_full_version}" for path in documentation: text = path.read_text() - for match in _CLONE_BRANCH_PATTERN.finditer(text): - if match.group("branch") != expected_release_branch: - errors.append( - f"{path}: clone command references {match.group('branch')}, " - f"expected {expected_release_branch}" - ) + if _CLONE_BRANCH_PATTERN.search(text): + errors.append( + f"{path}: clone command still references viable/strict, expected " + f"{expected_release_branch}" + ) for swiftpm_branch in _STABLE_SWIFTPM_PATTERN.findall(text): if swiftpm_branch != expected_swiftpm_branch: errors.append( @@ -280,10 +315,12 @@ def prepare_release( documentation = documentation_paths(repo_root) set_torch_version(repo_root / "torch_pin.py", torch_version) enable_release_wheel(repo_root / "torch_pin.py") - configure_release_version(repo_root / "version.txt", release_version) + release_full_version = configure_release_version( + repo_root / "version.txt", release_version + ) workflow_count = configure_workflows(workflow_paths, test_infra_branch) documentation_count = configure_documentation( - documentation, f"release/{release_version}", release_version + documentation, f"release/{release_version}", release_full_version ) manifest_created = freeze_arm_public_api(repo_root, release_version) validate_release_references( @@ -291,7 +328,7 @@ def prepare_release( documentation, test_infra_branch=test_infra_branch, release_version=release_version, - release_full_version=f"{release_version}.0", + release_full_version=release_full_version, ) validate_release_files(repo_root, release_version) requirement = validate_release_build(repo_root / "torch_pin.py") @@ -322,7 +359,7 @@ def main() -> None: parser.add_argument("--release-version", help="release version in MAJOR.MINOR form") parser.add_argument( "--test-infra-branch", - help="test-infra release branch (defaults to release/MAJOR.MINOR)", + help="test-infra branch (defaults to the selected PyTorch release line)", ) parser.add_argument( "--torch-version", @@ -335,13 +372,16 @@ def main() -> None: repo_root = args.repo_root.resolve() release_version = args.release_version or _release_version_from_file(repo_root) - test_infra_branch = args.test_infra_branch or f"release/{release_version}" workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) documentation = documentation_paths(repo_root) if args.check: _validate_release_version(release_version) requirement = validate_release_build(repo_root / "torch_pin.py") + torch_version = runpy.run_path(str(repo_root / "torch_pin.py"))["TORCH_VERSION"] + test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( + torch_version + ) release_full_version = validate_release_files(repo_root, release_version) validate_release_references( workflow_paths, @@ -355,6 +395,10 @@ def main() -> None: torch_version = torch_version_for_release( repo_root / "torch_pin.py", args.torch_version or "" ) + test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( + torch_version + ) + require_test_infra_branch(test_infra_branch) prepare_release(repo_root, release_version, test_infra_branch, torch_version) diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py index 63fd933475b..7d8ba9791a2 100644 --- a/scripts/release/test_finalize_release.py +++ b/scripts/release/test_finalize_release.py @@ -11,6 +11,7 @@ from finalize_release import ( finalize_dependency_text, finalize_torch_release, + plan_dependency_text, stable_base_version, ) @@ -31,7 +32,7 @@ def test_promotes_release_candidate_to_final(self) -> None: "RELEASE_DEPENDENCIES_FINALIZED = False\n" ) - self.assertEqual(finalize_torch_release(path, "2.15.0"), 2) + self.assertEqual(finalize_torch_release(path, "2.15.0"), 1) self.assertIn('TORCH_VERSION = "2.15.0"', path.read_text()) self.assertIn("RELEASE_DEPENDENCIES_FINALIZED = True", path.read_text()) @@ -40,6 +41,7 @@ def test_finalizes_dependency_text(self) -> None: root = Path(directory) (root / "install_requirements.py").write_text( 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' + 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260805"\n' ) (root / "setup.py").write_text(' "pytorch-tokenizers",\n') qnn_path = root / ".ci/scripts/test_wheel_package_qnn.sh" @@ -65,13 +67,23 @@ def test_finalizes_dependency_text(self) -> None: 'nightly/rocm${ROCM_VERSION}"\n' ) - changed = finalize_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") + planned = plan_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") + self.assertIn( + 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"', + (root / "install_requirements.py").read_text(), + ) + self.assertEqual(len(planned), 7) + changed = finalize_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") self.assertEqual(changed, 7) self.assertIn( 'TORCHAO_NIGHTLY_VERSION = "0.19.0"', (root / "install_requirements.py").read_text(), ) + self.assertIn( + 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0"', + (root / "install_requirements.py").read_text(), + ) self.assertIn( '"pytorch-tokenizers>=1.6.0"', (root / "setup.py").read_text() ) diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index b3bab16d6f1..05525f66937 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -11,6 +11,7 @@ from prepare_release import ( newest_torch_test_release, prepare_release, + test_infra_branch_for_torch, torch_version_for_release, ) @@ -22,6 +23,7 @@ def test_selects_release_candidate_once(self) -> None: """A branch cut selects the newest RC and preserves it on reruns.""" releases = ["2.14.1", "2.15.0a1", "2.15.0b2", "2.15.0rc1"] self.assertEqual(newest_torch_test_release(releases), "2.15.0rc1") + self.assertEqual(test_infra_branch_for_torch("2.15.0rc1"), "release/2.15") with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "torch_pin.py" @@ -48,6 +50,7 @@ def test_prepares_complete_repository(self) -> None: doc = root / "docs/README.md" doc.write_text( "git clone -b viable/strict repo\n" + "tutorial: git clone -b release/1.0 legacy-example\n" "stable: swiftpm-1.5.1\n" "nightly: swiftpm-1.6.0.20260929\n" ) @@ -57,21 +60,23 @@ def test_prepares_complete_repository(self) -> None: (manifests / "api_manifest_1_5.toml").write_text("previous\n") (manifests / "api_manifest_running.toml").write_text("current\n") - requirement = prepare_release(root, "1.6", "release/1.6", "2.15.0rc1") + requirement = prepare_release(root, "1.6", "release/2.15", "2.15.0rc1") + (root / "version.txt").write_text("1.6.1\n") second_requirement = prepare_release( - root, "1.6", "release/1.6", "2.15.0rc1" + root, "1.6", "release/2.15", "2.15.0rc1" ) self.assertEqual(requirement, "torch>=2.15.0rc1") self.assertEqual(second_requirement, requirement) self.assertIn("RELEASE_WHEEL = True", (root / "torch_pin.py").read_text()) - self.assertIn("@release/1.6", workflow.read_text()) - self.assertIn("test-infra-ref: release/1.6", workflow.read_text()) + self.assertIn("@release/2.15", workflow.read_text()) + self.assertIn("test-infra-ref: release/2.15", workflow.read_text()) self.assertIn("example/action@main", workflow.read_text()) self.assertIn("-b release/1.6", doc.read_text()) - self.assertIn("stable: swiftpm-1.6.0", doc.read_text()) + self.assertIn("-b release/1.0", doc.read_text()) + self.assertIn("stable: swiftpm-1.6.1", doc.read_text()) self.assertIn("nightly: swiftpm-1.6.0.20260929", doc.read_text()) - self.assertEqual((root / "version.txt").read_text(), "1.6.0\n") + self.assertEqual((root / "version.txt").read_text(), "1.6.1\n") self.assertFalse((manifests / "api_manifest_1_4.toml").exists()) self.assertEqual( (manifests / "api_manifest_1_6.toml").read_text(), "current\n" diff --git a/setup.py b/setup.py index cb33757a75c..f874a901703 100644 --- a/setup.py +++ b/setup.py @@ -1128,6 +1128,21 @@ def _torchao_requirement() -> str: def _release_torch_requirement() -> List[str]: if not torch_pin.RELEASE_WHEEL: return [] + + path = Path(__file__).parent / "install_requirements.py" + spec = importlib.util.spec_from_file_location("install_requirements", path) + if spec is None or spec.loader is None: + raise ImportError(f"Could not load {path}") + module = importlib.util.module_from_spec(spec) + sys.modules.setdefault("install_utils", install_utils) + spec.loader.exec_module(module) + torch_url = install_utils.determine_torch_url(module.TORCH_URL_BASE) + cu134_requirements = module.cu134_requirements(torch_url) + if cu134_requirements: + # The CUDA 13.4 build currently uses a PyTorch development wheel. A + # stable lower bound would reject and replace the binary it was built + # against, so this variant must retain its exact build-time dependency. + return [cu134_requirements[0]] return [f"torch>={torch_pin.TORCH_VERSION}"] From d1529f98560cbd1402cf4572c1045910d3c89453 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Wed, 30 Sep 2026 15:17:37 -0700 Subject: [PATCH 04/12] Split release finalization into follow-up --- .ci/scripts/wheel/test_clean_install.py | 18 +- .github/workflows/lint.yml | 4 +- scripts/release/apply-release-changes.sh | 2 - scripts/release/finalize_release.py | 432 ----------------------- scripts/release/test_finalize_release.py | 106 ------ torch_pin.py | 3 - 6 files changed, 3 insertions(+), 562 deletions(-) delete mode 100644 scripts/release/finalize_release.py delete mode 100644 scripts/release/test_finalize_release.py diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index 36e6bd556f7..209de1e03e3 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -68,29 +68,15 @@ def test_release_pytorch_requirement() -> None: os.environ.get("GITHUB_BASE_REF", ""), os.environ.get("GITHUB_REF", "").removeprefix("refs/heads/"), ) - release_tag_ci = any( - re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+", ref) for ref in ci_refs - ) - release_ci = release_tag_ci or any( + release_ci = any( ref.startswith("release/") - or re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+-rc\d+", ref) + or re.fullmatch(r"refs/tags/v\d+\.\d+\.\d+(?:-rc\d+)?", ref) for ref in ci_refs ) assert not release_ci or config["RELEASE_WHEEL"], ( "this wheel is being built from a release ref, but torch_pin.py does not enable " "release metadata; run scripts/release/apply-release-changes.sh" ) - dependencies_finalized = config.get("RELEASE_DEPENDENCIES_FINALIZED") is True - assert not release_tag_ci or dependencies_finalized, ( - "this wheel is being built from a release tag, but stable dependencies were not " - "finalized; run scripts/release/finalize_release.py before tagging" - ) - assert not dependencies_finalized or re.fullmatch( - r"\d+\.\d+\.\d+", config["TORCH_VERSION"] - ), ( - "release dependencies are marked finalized, but TORCH_VERSION is not a final " - f"PyTorch release: {config['TORCH_VERSION']}" - ) torch_requirements = [ Requirement(raw) for raw in metadata.requires("executorch") or [] diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a71a879cb5a..ce9347bf560 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -106,9 +106,7 @@ jobs: lintrunner init - name: Test release preparation - run: | - python scripts/release/test_prepare_release.py - python scripts/release/test_finalize_release.py + run: python scripts/release/test_prepare_release.py - name: Validate release configuration if: startsWith(github.ref_name, 'release/') || startsWith(github.base_ref, 'release/') diff --git a/scripts/release/apply-release-changes.sh b/scripts/release/apply-release-changes.sh index 8c5cc7851d7..991798b352e 100755 --- a/scripts/release/apply-release-changes.sh +++ b/scripts/release/apply-release-changes.sh @@ -51,5 +51,3 @@ fi "${PYTHON_EXECUTABLE}" "${GIT_TOP_DIR}/scripts/release/prepare_release.py" "${PREPARE_ARGS[@]}" echo "Release changes are ready for review. This script does not commit or push them." -echo "When stable dependency artifacts are published, run:" -echo " python scripts/release/finalize_release.py" diff --git a/scripts/release/finalize_release.py b/scripts/release/finalize_release.py deleted file mode 100644 index fba2fadbb94..00000000000 --- a/scripts/release/finalize_release.py +++ /dev/null @@ -1,432 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -"""Finalize release dependencies after their stable artifacts are published.""" - -import argparse -import json -import re -import runpy -import subprocess -import urllib.error -import urllib.request -from pathlib import Path -from typing import Iterable - -from prepare_release import ( - _release_version_from_file, - _TORCH_VERSION_PATTERN, - _write_if_changed, -) - -_VERSION_PATTERN = re.compile(r"\d+\.\d+\.\d+") -_TORCHCODEC_PATTERN = re.compile(r"torchcodec==\d+\.\d+\.\d+") -_TOKENIZERS_REQUIREMENT_PATTERN = re.compile( - r'(?m)^(?P\s*)"pytorch-tokenizers(?:>=\d+\.\d+\.\d+)?",$' -) -_TORCHAO_ASSIGNMENT_PATTERN = re.compile( - r'^TORCHAO_NIGHTLY_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE -) -_ROCM_TORCHAO_ASSIGNMENT_PATTERN = re.compile( - r'^ROCM_TORCHAO_NIGHTLY_VERSION\s*=\s*"(?P[^"]+)"$', re.MULTILINE -) -_ROCM_VERSION_PATTERN = re.compile( - r'^ROCM_VERSION="\$\{ROCM_VERSION:-(?P\d+\.\d+)\}"$', re.MULTILINE -) -_FINALIZED_PATTERN = re.compile( - r"^RELEASE_DEPENDENCIES_FINALIZED\s*=\s*(?:True|False)$", re.MULTILINE -) -_TORCHCODEC_PATHS = ( - ".ci/scripts/test-rocm-voxtral.sh", - ".ci/scripts/test_model_e2e.sh", - "examples/models/moshi/mimi/install_requirements.sh", -) -_ROCM_PATHS = ( - ".ci/scripts/test-rocm-aoti.sh", - ".ci/scripts/test-rocm-voxtral.sh", -) -_QNN_TEST_INDEX = ( - '"$PIPBIN" install torch=="${TORCH_VERSION}" ' - '--extra-index-url "https://download.pytorch.org/whl/test"' -) -_QNN_STABLE_INDEX = ( - '"$PIPBIN" install --no-cache-dir torch=="${TORCH_VERSION}" ' - '--index-url "https://download.pytorch.org/whl/cpu"' -) -_SUBMODULE_RELEASES = ( - ("extension/llm/tokenizers", "https://github.com/meta-pytorch/tokenizers.git"), - ("third-party/ao", "https://github.com/pytorch/ao.git"), -) - - -def stable_base_version(version: str) -> str: - """Convert a development or prerelease version to its final base version.""" - match = re.match(r"(\d+\.\d+\.\d+)", version) - if match is None: - raise RuntimeError(f"could not derive a stable version from {version!r}") - return match.group(1) - - -def current_torchao_version(install_requirements_path: Path) -> str: - """Read the generic TorchAO pin without importing the installer module.""" - match = _TORCHAO_ASSIGNMENT_PATTERN.search(install_requirements_path.read_text()) - if match is None: - raise RuntimeError( - f"could not find TORCHAO_NIGHTLY_VERSION in {install_requirements_path}" - ) - return match.group("version") - - -def latest_pypi_version(package: str) -> str: - """Return the latest stable version published for a package.""" - with urllib.request.urlopen( - f"https://pypi.org/pypi/{package}/json", timeout=30 - ) as response: - version = json.load(response)["info"]["version"] - if _VERSION_PATTERN.fullmatch(version) is None: - raise RuntimeError(f"latest {package} version is not stable: {version!r}") - return version - - -def require_pypi_release(package: str, version: str) -> None: - """Fail before mutation if an expected stable package is unavailable.""" - if _VERSION_PATTERN.fullmatch(version) is None: - raise RuntimeError(f"invalid {package} version {version!r}") - try: - with urllib.request.urlopen( - f"https://pypi.org/pypi/{package}/{version}/json", timeout=30 - ): - pass - except urllib.error.HTTPError as error: - if error.code == 404: - raise RuntimeError( - f"{package}=={version} is not published on PyPI" - ) from error - raise - - -def require_remote_tag(repository: str, version: str) -> None: - """Fail before mutation if a matching submodule release tag is unavailable.""" - result = subprocess.run( - [ - "git", - "ls-remote", - "--exit-code", - "--tags", - repository, - f"refs/tags/v{version}", - ], - check=False, - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - ) - if result.returncode != 0: - raise RuntimeError(f"{repository} does not have tag v{version}") - - -def require_rocm_torchao_release(repo_root: Path, version: str) -> None: - """Verify the stable ROCm TorchAO artifact used by release CI exists.""" - rocm_script = repo_root / _ROCM_PATHS[0] - match = _ROCM_VERSION_PATTERN.search(rocm_script.read_text()) - if match is None: - raise RuntimeError(f"could not determine ROCm version from {rocm_script}") - rocm_version = match.group("version") - url = ( - f"https://download.pytorch.org/whl/rocm{rocm_version}/" - f"torchao-{version}%2Brocm{rocm_version}-" - "cp310-abi3-manylinux_2_28_x86_64.whl" - ) - try: - with urllib.request.urlopen(url, timeout=30): - pass - except urllib.error.HTTPError as error: - if error.code == 404: - raise RuntimeError( - f"TorchAO {version} is not published for ROCm {rocm_version}" - ) from error - raise - - -def planned_torch_release(torch_pin_path: Path, torch_version: str) -> str: - """Plan the RC-to-final pin update without changing the checkout.""" - if _VERSION_PATTERN.fullmatch(torch_version) is None: - raise RuntimeError( - f"final PyTorch version must be X.Y.Z, got {torch_version!r}" - ) - text = torch_pin_path.read_text() - updated, version_count = _TORCH_VERSION_PATTERN.subn( - f'TORCH_VERSION = "{torch_version}"', text - ) - if version_count != 1: - raise RuntimeError(f"expected one TORCH_VERSION assignment in {torch_pin_path}") - updated, count = _FINALIZED_PATTERN.subn( - "RELEASE_DEPENDENCIES_FINALIZED = True", updated - ) - if count != 1: - raise RuntimeError( - f"expected one RELEASE_DEPENDENCIES_FINALIZED assignment in {torch_pin_path}" - ) - return updated - - -def finalize_torch_release(torch_pin_path: Path, torch_version: str) -> int: - """Replace the RC pin with the final release and mark finalization complete.""" - updated = planned_torch_release(torch_pin_path, torch_version) - return int(_write_if_changed(torch_pin_path, updated)) - - -def plan_dependency_text( - repo_root: Path, - torchao_version: str, - tokenizers_version: str, - torchcodec_version: str, -) -> dict[Path, str]: - """Validate and calculate every text edit without changing the checkout.""" - updates: dict[Path, str] = {} - - def read(path: Path) -> str: - if not path.exists(): - raise RuntimeError(f"expected release dependency file {path}") - return updates.get(path, path.read_text()) - - path = repo_root / "install_requirements.py" - updated, count = _TORCHAO_ASSIGNMENT_PATTERN.subn( - f'TORCHAO_NIGHTLY_VERSION = "{torchao_version}"', read(path) - ) - if count != 1: - raise RuntimeError(f"expected one TORCHAO_NIGHTLY_VERSION in {path}") - updated, count = _ROCM_TORCHAO_ASSIGNMENT_PATTERN.subn( - f'ROCM_TORCHAO_NIGHTLY_VERSION = "{torchao_version}"', updated - ) - if count != 1: - raise RuntimeError(f"expected one ROCM_TORCHAO_NIGHTLY_VERSION in {path}") - updates[path] = updated - - path = repo_root / "setup.py" - updated, count = _TOKENIZERS_REQUIREMENT_PATTERN.subn( - rf'\g"pytorch-tokenizers>={tokenizers_version}",', read(path) - ) - if count != 1: - raise RuntimeError(f"expected one pytorch-tokenizers requirement in {path}") - updates[path] = updated - - for relative in _TORCHCODEC_PATHS: - path = repo_root / relative - updated, count = _TORCHCODEC_PATTERN.subn( - f"torchcodec=={torchcodec_version}", read(path) - ) - if count == 0: - raise RuntimeError(f"expected a torchcodec pin in {path}") - updates[path] = updated.replace( - "--extra-index-url https://download.pytorch.org/whl/test/cpu", - "--index-url https://download.pytorch.org/whl/cpu", - ) - - for relative in _ROCM_PATHS: - path = repo_root / relative - updates[path] = ( - read(path) - .replace( - "https://download.pytorch.org/whl/test/rocm${ROCM_VERSION}", - "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", - ) - .replace( - "https://download.pytorch.org/whl/nightly/rocm${ROCM_VERSION}", - "https://download.pytorch.org/whl/rocm${ROCM_VERSION}", - ) - ) - - qnn_path = repo_root / ".ci/scripts/test_wheel_package_qnn.sh" - updated = read(qnn_path).replace(_QNN_TEST_INDEX, _QNN_STABLE_INDEX) - if _QNN_STABLE_INDEX not in updated: - raise RuntimeError( - f"could not find the QNN Torch install command in {qnn_path}" - ) - updates[qnn_path] = updated - return {path: text for path, text in updates.items() if text != path.read_text()} - - -def finalize_dependency_text( - repo_root: Path, - torchao_version: str, - tokenizers_version: str, - torchcodec_version: str, -) -> int: - """Update all recurring stable dependency declarations and index URLs.""" - updates = plan_dependency_text( - repo_root, torchao_version, tokenizers_version, torchcodec_version - ) - for path, updated in updates.items(): - _write_if_changed(path, updated) - return len(updates) - - -def prepare_submodule_tags( - repo_root: Path, releases: Iterable[tuple[str, str, str]] -) -> list[tuple[Path, str, str]]: - """Fetch and resolve every submodule tag before changing any gitlink.""" - prepared = [] - for relative, _repository, version in releases: - path = repo_root / relative - if (path / ".git").exists() and subprocess.run( - ["git", "status", "--porcelain"], - cwd=path, - check=True, - capture_output=True, - text=True, - ).stdout: - raise RuntimeError(f"submodule {path} has local changes") - subprocess.run( - ["git", "submodule", "update", "--init", "--", relative], - cwd=repo_root, - check=True, - ) - if subprocess.run( - ["git", "status", "--porcelain"], - cwd=path, - check=True, - capture_output=True, - text=True, - ).stdout: - raise RuntimeError(f"submodule {path} has local changes") - subprocess.run( - ["git", "fetch", "--depth=1", "origin", "tag", f"v{version}"], - cwd=path, - check=True, - ) - current = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=path, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - target = subprocess.run( - ["git", "rev-parse", f"v{version}^{{commit}}"], - cwd=path, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - prepared.append((path, current, target)) - return prepared - - -def update_submodule_tags(prepared: Iterable[tuple[Path, str, str]]) -> None: - """Apply prepared submodule updates, restoring prior heads on failure.""" - changed = [] - try: - for path, current, target in prepared: - subprocess.run( - ["git", "checkout", "--detach", target], cwd=path, check=True - ) - changed.append((path, current)) - except Exception: - for path, current in reversed(changed): - subprocess.run( - ["git", "checkout", "--detach", current], cwd=path, check=True - ) - raise - - -def main() -> None: - """Preflight and apply stable dependency finalization.""" - parser = argparse.ArgumentParser() - parser.add_argument("--release-version", help="release version in MAJOR.MINOR form") - parser.add_argument("--torch-version", help="final stable PyTorch version") - parser.add_argument("--torchao-version", help="stable TorchAO version") - parser.add_argument( - "--tokenizers-version", help="stable pytorch-tokenizers version" - ) - parser.add_argument("--torchcodec-version", help="stable TorchCodec version") - parser.add_argument( - "--preflight-only", - action="store_true", - help="verify packages and tags without changing the checkout", - ) - parser.add_argument( - "--repo-root", type=Path, default=Path(__file__).resolve().parents[2] - ) - args = parser.parse_args() - - repo_root = args.repo_root.resolve() - release_version = args.release_version or _release_version_from_file(repo_root) - release_config = runpy.run_path(str(repo_root / "torch_pin.py")) - if release_config.get("RELEASE_WHEEL") is not True: - raise RuntimeError( - "release preparation has not run; use cut-release-branch.sh first" - ) - torch_version = args.torch_version or stable_base_version( - release_config["TORCH_VERSION"] - ) - torchao_version = args.torchao_version or stable_base_version( - current_torchao_version(repo_root / "install_requirements.py") - ) - tokenizers_version = args.tokenizers_version or f"{release_version}.0" - torchcodec_version = args.torchcodec_version or latest_pypi_version("torchcodec") - - releases = ( - ("extension/llm/tokenizers", _SUBMODULE_RELEASES[0][1], tokenizers_version), - ("third-party/ao", _SUBMODULE_RELEASES[1][1], torchao_version), - ) - for package, version in ( - ("torch", torch_version), - ("torchao", torchao_version), - ("pytorch-tokenizers", tokenizers_version), - ("torchcodec", torchcodec_version), - ): - require_pypi_release(package, version) - for _path, repository, version in releases: - require_remote_tag(repository, version) - require_rocm_torchao_release(repo_root, torchao_version) - - print( - "Release dependency artifacts are available: " - f"torch=={torch_version}, torchao=={torchao_version}, " - f"pytorch-tokenizers=={tokenizers_version}, torchcodec=={torchcodec_version}" - ) - if args.preflight_only: - return - - dependency_updates = plan_dependency_text( - repo_root, torchao_version, tokenizers_version, torchcodec_version - ) - torch_pin_path = repo_root / "torch_pin.py" - torch_pin_update = planned_torch_release(torch_pin_path, torch_version) - prepared_submodules = prepare_submodule_tags(repo_root, releases) - text_updates = {**dependency_updates, torch_pin_path: torch_pin_update} - original_text = {path: path.read_text() for path in text_updates} - - try: - update_submodule_tags(prepared_submodules) - for path, updated in text_updates.items(): - _write_if_changed(path, updated) - except Exception: - for path, original in original_text.items(): - path.write_text(original) - for path, original, _target in reversed(prepared_submodules): - subprocess.run( - ["git", "checkout", "--detach", original], cwd=path, check=False - ) - raise - - changed = sum( - updated != original_text[path] for path, updated in text_updates.items() - ) - print( - f"Finalized release dependencies; changed {changed} text files and 2 submodules" - ) - print( - "Review and stage the resulting release-only changes with git add " - "install_requirements.py setup.py torch_pin.py .ci/scripts/ " - "examples/models/moshi/mimi/install_requirements.sh " - "extension/llm/tokenizers third-party/ao" - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/release/test_finalize_release.py b/scripts/release/test_finalize_release.py deleted file mode 100644 index 7d8ba9791a2..00000000000 --- a/scripts/release/test_finalize_release.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (c) Meta Platforms, Inc. and affiliates. -# All rights reserved. -# -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -import tempfile -import unittest -from pathlib import Path - -from finalize_release import ( - finalize_dependency_text, - finalize_torch_release, - plan_dependency_text, - stable_base_version, -) - - -class FinalizeReleaseTest(unittest.TestCase): - """Tests for delayed release dependency finalization.""" - - def test_promotes_release_candidate_to_final(self) -> None: - self.assertEqual(stable_base_version("0.19.0.dev20260907"), "0.19.0") - self.assertEqual(stable_base_version("0.19.0rc2"), "0.19.0") - self.assertEqual(stable_base_version("0.19.0"), "0.19.0") - - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "torch_pin.py" - path.write_text( - 'TORCH_VERSION = "2.15.0rc2"\n' - "RELEASE_WHEEL = True\n" - "RELEASE_DEPENDENCIES_FINALIZED = False\n" - ) - - self.assertEqual(finalize_torch_release(path, "2.15.0"), 1) - self.assertIn('TORCH_VERSION = "2.15.0"', path.read_text()) - self.assertIn("RELEASE_DEPENDENCIES_FINALIZED = True", path.read_text()) - - def test_finalizes_dependency_text(self) -> None: - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - (root / "install_requirements.py").write_text( - 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' - 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260805"\n' - ) - (root / "setup.py").write_text(' "pytorch-tokenizers",\n') - qnn_path = root / ".ci/scripts/test_wheel_package_qnn.sh" - qnn_path.parent.mkdir(parents=True) - qnn_path.write_text( - '"$PIPBIN" install torch=="${TORCH_VERSION}" ' - '--extra-index-url "https://download.pytorch.org/whl/test"\n' - ) - for relative in ( - ".ci/scripts/test-rocm-aoti.sh", - ".ci/scripts/test-rocm-voxtral.sh", - ".ci/scripts/test_model_e2e.sh", - "examples/models/moshi/mimi/install_requirements.sh", - ): - path = root / relative - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - "pip install torchcodec==0.11.0 " - "--extra-index-url https://download.pytorch.org/whl/test/cpu\n" - 'PYTORCH_ROCM_INDEX="https://download.pytorch.org/whl/test/' - 'rocm${ROCM_VERSION}"\n' - 'TORCHAO_ROCM_WHEEL_BASE="https://download.pytorch.org/whl/' - 'nightly/rocm${ROCM_VERSION}"\n' - ) - - planned = plan_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") - self.assertIn( - 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"', - (root / "install_requirements.py").read_text(), - ) - self.assertEqual(len(planned), 7) - - changed = finalize_dependency_text(root, "0.19.0", "1.6.0", "0.17.0") - self.assertEqual(changed, 7) - self.assertIn( - 'TORCHAO_NIGHTLY_VERSION = "0.19.0"', - (root / "install_requirements.py").read_text(), - ) - self.assertIn( - 'ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0"', - (root / "install_requirements.py").read_text(), - ) - self.assertIn( - '"pytorch-tokenizers>=1.6.0"', (root / "setup.py").read_text() - ) - for relative in ( - ".ci/scripts/test-rocm-voxtral.sh", - ".ci/scripts/test_model_e2e.sh", - "examples/models/moshi/mimi/install_requirements.sh", - ): - text = (root / relative).read_text() - self.assertIn("torchcodec==0.17.0", text) - self.assertNotIn("whl/test/cpu", text) - rocm = (root / ".ci/scripts/test-rocm-voxtral.sh").read_text() - self.assertNotIn("whl/test/rocm", rocm) - self.assertNotIn("whl/nightly/rocm", rocm) - self.assertIn("--no-cache-dir", qnn_path.read_text()) - self.assertIn("--index-url", qnn_path.read_text()) - - -if __name__ == "__main__": - unittest.main() diff --git a/torch_pin.py b/torch_pin.py index b1ee639bb66..7f9551200bd 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -5,6 +5,3 @@ # Changed to True by scripts/release/apply-release-changes.sh. Release wheels # declare the PyTorch release above; development and minimal wheels do not. RELEASE_WHEEL = False - -# Changed to True after stable third-party artifacts and submodule tags exist. -RELEASE_DEPENDENCIES_FINALIZED = False From 6ca694c12ed538a0a58e5aa193f009abea15a691 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 09:21:32 -0700 Subject: [PATCH 05/12] Fix release prep test mypy import --- scripts/release/test_prepare_release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index 05525f66937..4a9febf8cc6 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -8,7 +8,7 @@ import unittest from pathlib import Path -from prepare_release import ( +from prepare_release import ( # type: ignore[import-not-found] newest_torch_test_release, prepare_release, test_infra_branch_for_torch, From 0d92965ad040aec1684834705ab4e8f41a939dd0 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 09:36:19 -0700 Subject: [PATCH 06/12] Address release preparation review failures --- .ci/scripts/tests/test_cu134_dependencies.py | 72 ++++--- .ci/scripts/wheel/envvar_base.sh | 5 + .ci/scripts/wheel/test_clean_install.py | 14 +- .github/workflows/lint.yml | 9 +- install_requirements.py | 15 +- scripts/release/cut-release-branch.sh | 29 ++- scripts/release/prepare_release.py | 200 +++++++++++++------ scripts/release/test_prepare_release.py | 34 ++-- setup.py | 60 +++--- torch_pin.py | 2 + 10 files changed, 294 insertions(+), 146 deletions(-) diff --git a/.ci/scripts/tests/test_cu134_dependencies.py b/.ci/scripts/tests/test_cu134_dependencies.py index 46402dcb907..234b35baa9c 100644 --- a/.ci/scripts/tests/test_cu134_dependencies.py +++ b/.ci/scripts/tests/test_cu134_dependencies.py @@ -5,8 +5,10 @@ # LICENSE file in the root directory of this source tree. import ast +import functools import importlib.util import os +import re import subprocess import sys import unittest @@ -144,50 +146,51 @@ def test_failure_is_not_retried_with_another_cuda_train(self): self.installer.install_requirements(True) self.assertEqual(run.call_count, 1) - def torchao_requirement(self): - path = ROOT / "setup.py" - tree = ast.parse(path.read_text()) - function = next( - node - for node in tree.body - if isinstance(node, ast.FunctionDef) and node.name == "_torchao_requirement" - ) - namespace = { - "__file__": str(path), - "Path": Path, - "importlib": importlib, - "sys": sys, - "install_utils": self.utils, - } - exec( - compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), - namespace, - ) - return namespace["_torchao_requirement"]() - - def release_torch_requirement(self): + def setup_requirement( + self, function_name, *, installed_torch="2.15.0rc1", building_wheel=True + ): path = ROOT / "setup.py" tree = ast.parse(path.read_text()) - function = next( + functions = [ node for node in tree.body if isinstance(node, ast.FunctionDef) - and node.name == "_release_torch_requirement" - ) + and node.name + in { + "_load_install_requirements", + "_torchao_requirement", + "_release_torch_requirement", + } + ] namespace = { "__file__": str(path), "Path": Path, "List": list, + "functools": functools, "importlib": importlib, + "os": os, + "re": re, "sys": sys, "install_utils": self.utils, "torch_pin": SimpleNamespace(RELEASE_WHEEL=True, TORCH_VERSION="2.15.0rc1"), } exec( - compile(ast.Module(body=[function], type_ignores=[]), str(path), "exec"), + compile(ast.Module(body=functions, type_ignores=[]), str(path), "exec"), namespace, ) - return namespace["_release_torch_requirement"]()[0] + environment = {"EXECUTORCH_BUILDING_WHEEL": "1"} if building_wheel else {} + with ( + patch.dict(os.environ, environment, clear=True), + patch("importlib.metadata.version", return_value=installed_torch), + ): + return namespace[function_name]() + + def torchao_requirement(self): + return self.setup_requirement("_torchao_requirement") + + def release_torch_requirement(self, **kwargs): + requirements = self.setup_requirement("_release_torch_requirement", **kwargs) + return requirements[0] if requirements else None def test_package_install_preserves_source_pinned_torchao(self): with patch.dict(sys.modules, {"install_requirements": self.installer}): @@ -267,10 +270,10 @@ def test_cu134_keeps_explicit_torchao_source_build(self): self.assertIn("0.19.0+gitb7ac3aa", metadata.specifier) def test_wheel_bounds_match_selected_train(self): - for cuda, expected_torch in ( - ((13, 4), "torch==2.14.0.dev20260810+cu134"), - ((13, 2), "torch>=2.15.0rc1"), - (None, "torch>=2.15.0rc1"), + for cuda, installed_torch, expected_torch in ( + ((13, 4), "2.14.0.dev20260810+cu134", "torch==2.14.0.dev20260810+cu134"), + ((13, 2), "2.15.0rc1+cu132", "torch==2.15.0rc1+cu132"), + (None, "2.15.0rc1", "torch>=2.15.0rc1,<2.16"), ): self.utils.determine_torch_url.cache_clear() with ( @@ -286,7 +289,12 @@ def test_wheel_bounds_match_selected_train(self): self.torchao_requirement(), "torchao>=0.19.0.dev20260907,<0.20", ) - self.assertEqual(self.release_torch_requirement(), expected_torch) + self.assertEqual( + self.release_torch_requirement(installed_torch=installed_torch), + expected_torch, + ) + + self.assertIsNone(self.release_torch_requirement(building_wheel=False)) if __name__ == "__main__": diff --git a/.ci/scripts/wheel/envvar_base.sh b/.ci/scripts/wheel/envvar_base.sh index f57f6131f46..ba97136ebda 100755 --- a/.ci/scripts/wheel/envvar_base.sh +++ b/.ci/scripts/wheel/envvar_base.sh @@ -11,3 +11,8 @@ # Ensure that CMAKE_ARGS is defined before referencing it. Defaults to empty # if not defined. export CMAKE_ARGS="${CMAKE_ARGS:-}" + +# setup.py only adds release dependency metadata for artifacts built by the +# binary wheel jobs. A source install on a release branch must preserve the +# PyTorch build already installed by that checkout's CI job. +export EXECUTORCH_BUILDING_WHEEL=1 diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index 209de1e03e3..ffe04f8d1a6 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -66,7 +66,7 @@ def test_release_pytorch_requirement() -> None: ci_refs = ( os.environ.get("GITHUB_REF_NAME", ""), os.environ.get("GITHUB_BASE_REF", ""), - os.environ.get("GITHUB_REF", "").removeprefix("refs/heads/"), + os.environ.get("GITHUB_REF", ""), ) release_ci = any( ref.startswith("release/") @@ -85,10 +85,16 @@ def test_release_pytorch_requirement() -> None: if config["RELEASE_WHEEL"]: installed_torch = Version(metadata.version("torch")) - if installed_torch.local and "cu134" in installed_torch.local: - expected = f"=={installed_torch}" + if installed_torch.local and re.search( + r"(?:^|\.)cu\d+(?:\.|$)", installed_torch.local + ): + expected_requirement = Requirement(f"torch=={installed_torch}") else: - expected = f">={config['TORCH_VERSION']}" + major, minor, *_ = config["TORCH_VERSION"].split(".") + expected_requirement = Requirement( + f"torch>={config['TORCH_VERSION']},<{major}.{int(minor) + 1}" + ) + expected = str(expected_requirement.specifier) assert len(torch_requirements) == 1, ( "a release wheel must declare exactly one PyTorch dependency, but found " f"{[str(requirement) for requirement in torch_requirements]}" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ce9347bf560..6983539bf8b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -49,11 +49,14 @@ jobs: - name: Install dependencies run: | TORCH_VERSION=$(python -c "from torch_pin import TORCH_VERSION; print(TORCH_VERSION)") + TORCHVISION_VERSION=$(python -c "from torch_pin import TORCHVISION_VERSION; print(TORCHVISION_VERSION)") + TORCHAUDIO_VERSION=$(python -c "from torch_pin import TORCHAUDIO_VERSION; print(TORCHAUDIO_VERSION)") + TORCH_INDEX=$(python -c "from torch_pin import RELEASE_WHEEL; print('https://download.pytorch.org/whl/test/cpu' if RELEASE_WHEEL else 'https://download.pytorch.org/whl/cpu')") pip install \ "torch==${TORCH_VERSION}" \ - torchvision \ - torchaudio \ - --index-url https://download.pytorch.org/whl/cpu + "torchvision==${TORCHVISION_VERSION}" \ + "torchaudio==${TORCHAUDIO_VERSION}" \ + --index-url "${TORCH_INDEX}" pip install lintrunner==0.12.7 lintrunner-adapters==0.14.1 pip install -r requirements-lintrunner.txt USE_CPP=0 pip install --no-build-isolation third-party/ao diff --git a/install_requirements.py b/install_requirements.py index f7ac388ec45..009c67c7099 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -14,9 +14,10 @@ from install_utils import determine_torch_url, is_intel_mac_os, python_is_compatible -TORCH_VERSION = runpy.run_path(os.path.join(os.path.dirname(__file__), "torch_pin.py"))[ - "TORCH_VERSION" -] +_TORCH_PIN = runpy.run_path(os.path.join(os.path.dirname(__file__), "torch_pin.py")) +TORCH_VERSION = _TORCH_PIN["TORCH_VERSION"] +TORCHVISION_VERSION = _TORCH_PIN["TORCHVISION_VERSION"] +TORCHAUDIO_VERSION = _TORCH_PIN["TORCHAUDIO_VERSION"] # The pip repository that hosts nightly torch packages. # This will be dynamically set based on CUDA availability and CUDA backend enabled/disabled. @@ -191,8 +192,12 @@ def install_optional_example_requirements(use_pytorch_nightly): print("Installing torch domain libraries") DOMAIN_LIBRARIES = cu134_packages or [ - ("torchvision==0.29.0" if use_pytorch_nightly else "torchvision"), - ("torchaudio==2.11.0" if use_pytorch_nightly else "torchaudio"), + ( + f"torchvision=={TORCHVISION_VERSION}" + if use_pytorch_nightly + else "torchvision" + ), + (f"torchaudio=={TORCHAUDIO_VERSION}" if use_pytorch_nightly else "torchaudio"), ] # Then install domain libraries subprocess.run( diff --git a/scripts/release/cut-release-branch.sh b/scripts/release/cut-release-branch.sh index 4ccc78875ef..0bfc2718fb5 100755 --- a/scripts/release/cut-release-branch.sh +++ b/scripts/release/cut-release-branch.sh @@ -38,20 +38,37 @@ fi # Read version.txt directly from viable/strict so the caller's starting branch # cannot influence which release is cut. This preserves all numeric components # (for example, 1.10 does not become 1.1). -SOURCE_BRANCH="${GIT_REMOTE}/viable/strict" -SOURCE_VERSION=$(git show "${SOURCE_BRANCH}:version.txt") +VIABLE_BRANCH="${GIT_REMOTE}/viable/strict" +SOURCE_VERSION=$(git show "${VIABLE_BRANCH}:version.txt") RELEASE_VERSION=${RELEASE_VERSION:-$(printf '%s\n' "${SOURCE_VERSION}" | cut -d'.' -f1-2)} RELEASE_BRANCH="release/${RELEASE_VERSION}" ORIGINAL_BRANCH="orig/${RELEASE_BRANCH}" +ARM_MANIFEST="backends/arm/public_api_manifests/api_manifest_${RELEASE_VERSION//./_}.toml" + +# The API snapshot protects main against backwards-incompatible changes, so it +# must be reviewed and merged there before the branch is cut. Creating it only +# on the release branch leaves main unprotected and guarantees later conflicts. +if git cat-file -e "${VIABLE_BRANCH}:${ARM_MANIFEST}" 2>/dev/null; then + : +elif git cat-file -e "${VIABLE_BRANCH}:backends/arm/public_api_manifests/api_manifest_running.toml" 2>/dev/null; then + echo "Error: ${ARM_MANIFEST} must be merged into ${VIABLE_BRANCH} before the release cut." + exit 1 +fi if [[ ${DRY_RUN:-enabled} != "disabled" ]]; then - echo "Dry run: would preserve ${SOURCE_BRANCH} as ${ORIGINAL_BRANCH}" + echo "Dry run: would preserve ${VIABLE_BRANCH} as ${ORIGINAL_BRANCH}" echo "Dry run: would prepare, commit, and push ${RELEASE_BRANCH}" exit 0 fi -if ! git ls-remote --exit-code "${GIT_REMOTE}" "refs/heads/${ORIGINAL_BRANCH}" >/dev/null 2>&1; then - git push "${GIT_REMOTE}" "${SOURCE_BRANCH}:refs/heads/${ORIGINAL_BRANCH}" +if git ls-remote --exit-code "${GIT_REMOTE}" "refs/heads/${ORIGINAL_BRANCH}" >/dev/null 2>&1; then + # A previous attempt already fixed the cut point. Always resume from it, + # even if viable/strict advanced after that attempt failed. + CUT_SOURCE="${GIT_REMOTE}/${ORIGINAL_BRANCH}" +else + git push "${GIT_REMOTE}" "${VIABLE_BRANCH}:refs/heads/${ORIGINAL_BRANCH}" + git fetch "${GIT_REMOTE}" "${ORIGINAL_BRANCH}:refs/remotes/${GIT_REMOTE}/${ORIGINAL_BRANCH}" + CUT_SOURCE="${GIT_REMOTE}/${ORIGINAL_BRANCH}" fi if git show-ref --verify --quiet "refs/heads/${RELEASE_BRANCH}"; then @@ -59,7 +76,7 @@ if git show-ref --verify --quiet "refs/heads/${RELEASE_BRANCH}"; then elif git ls-remote --exit-code "${GIT_REMOTE}" "refs/heads/${RELEASE_BRANCH}" >/dev/null 2>&1; then git checkout -b "${RELEASE_BRANCH}" "${GIT_REMOTE}/${RELEASE_BRANCH}" else - git checkout -b "${RELEASE_BRANCH}" "${SOURCE_BRANCH}" + git checkout -b "${RELEASE_BRANCH}" "${CUT_SOURCE}" fi ( diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index d5a8fc8f6d6..592d6cc9d7f 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -10,7 +10,6 @@ import argparse import re import runpy -import shutil import subprocess import urllib.request from pathlib import Path @@ -22,6 +21,12 @@ r"^RELEASE_WHEEL\s*=\s*(?:True|False)$", re.MULTILINE ) _TORCH_VERSION_PATTERN = re.compile(r'^TORCH_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE) +_TORCHVISION_VERSION_PATTERN = re.compile( + r'^TORCHVISION_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE +) +_TORCHAUDIO_VERSION_PATTERN = re.compile( + r'^TORCHAUDIO_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE +) _TEST_INFRA_MAIN_PATTERN = re.compile(r"(pytorch/test-infra/[^\s\"']+)@main\b") _TEST_INFRA_REF_MAIN_PATTERN = re.compile(r"(test-infra-ref:\s*)main\b") _TEST_INFRA_BRANCH_PATTERN = re.compile(r"pytorch/test-infra/[^\s\"'@]+@([^\s\"']+)") @@ -76,45 +81,122 @@ def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: return _write_if_changed(torch_pin_path, updated) -def newest_torch_test_release(available_versions: Iterable[str]) -> str: - """Return the newest RC or final-form version on the PyTorch test index.""" - candidates = [] +def set_companion_versions(torch_pin_path: Path, torch_version: str) -> bool: + """Move torchvision and torchaudio to the selected PyTorch release train.""" + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)((?:a|b|rc)\d+)?", torch_version) + if match is None: + raise RuntimeError(f"invalid Torch release version {torch_version!r}") + _major, minor, patch, suffix = match.groups() + torchvision_version = f"0.{int(minor) + 15}.{patch}{suffix or ''}" + torchaudio_version = torch_version + + text = torch_pin_path.read_text() + updated, vision_count = _TORCHVISION_VERSION_PATTERN.subn( + f'TORCHVISION_VERSION = "{torchvision_version}"', text + ) + updated, audio_count = _TORCHAUDIO_VERSION_PATTERN.subn( + f'TORCHAUDIO_VERSION = "{torchaudio_version}"', updated + ) + if vision_count != 1 or audio_count != 1: + raise RuntimeError( + f"expected exactly one torchvision and torchaudio assignment in {torch_pin_path}" + ) + return _write_if_changed(torch_pin_path, updated) + + +def _release_key(version: str) -> tuple[int, int, int, int, int]: + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?", version) + if match is None: + raise RuntimeError(f"invalid release version {version!r}") stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} + major, minor, patch = (int(part) for part in match.groups()[:3]) + return major, minor, patch, stage_rank[match.group(4)], int(match.group(5) or 0) + + +def newest_torch_test_release( + available_versions: Iterable[str], newer_than: str = "" +) -> str: + """Return the newest RC/final on a newer PyTorch release train.""" + candidates = [] for version in available_versions: - match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?", version) + match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:(rc)(\d+))?", version) if match is None: continue - major, minor, patch = (int(part) for part in match.groups()[:3]) - stage = match.group(4) - stage_number = int(match.group(5) or 0) - candidates.append( - ((major, minor, patch, stage_rank[stage], stage_number), version) - ) + key = _release_key(version) + if newer_than and key[:2] <= _release_key(newer_than)[:2]: + continue + candidates.append((key, version)) if not candidates: - raise RuntimeError("PyTorch test index has no release candidate wheels") + qualifier = f" newer than {newer_than}" if newer_than else "" + raise RuntimeError( + f"PyTorch test index has no release candidate or final release{qualifier}" + ) return max(candidates)[1] -def latest_torch_test_release() -> str: - """Look up the newest PyTorch release candidate on its CPU test index.""" +def _test_index_versions(package: str) -> set[str]: with urllib.request.urlopen( - "https://download.pytorch.org/whl/test/cpu/torch/", timeout=30 + f"https://download.pytorch.org/whl/test/cpu/{package}/", timeout=30 ) as response: index = unquote(response.read().decode()) - versions = re.findall( - r"torch-(\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?)(?:\+[^-]+)?-", index + return set( + re.findall( + rf"{re.escape(package)}-(\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?)(?:\+[^-]+)?-", + index, + ) + ) + + +def latest_torch_test_release(newer_than: str) -> str: + """Look up the newest PyTorch release candidate on its CPU test index.""" + return newest_torch_test_release( + _test_index_versions("torch"), newer_than=newer_than ) - return newest_torch_test_release(versions) + + +def require_companion_releases(torch_version: str) -> None: + """Fail before mutation unless matching domain-library releases exist.""" + match = re.fullmatch(r"\d+\.(\d+)\.(\d+)((?:a|b|rc)\d+)?", torch_version) + if match is None: + raise RuntimeError(f"invalid Torch release version {torch_version!r}") + minor, patch, suffix = match.groups() + expected = { + "torchvision": f"0.{int(minor) + 15}.{patch}{suffix or ''}", + "torchaudio": torch_version, + } + missing = [ + f"{package}=={version}" + for package, version in expected.items() + if version not in _test_index_versions(package) + ] + if missing: + raise RuntimeError( + "PyTorch companion releases are not available on the test index: " + + ", ".join(missing) + ) def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: """Choose a pin once, then preserve it on subsequent preparation runs.""" + config = runpy.run_path(str(torch_pin_path)) + current = config["TORCH_VERSION"] if override: + current_key = _release_key(current) + override_key = _release_key(override) + already_prepared = config.get("RELEASE_WHEEL") is True + invalid = ( + override_key < current_key + if already_prepared + else override_key[:2] <= current_key[:2] + ) + if invalid: + raise RuntimeError( + f"Torch release {override} cannot replace current version {current}" + ) return override - config = runpy.run_path(str(torch_pin_path)) if config.get("RELEASE_WHEEL") is True: - return config["TORCH_VERSION"] - return latest_torch_test_release() + return current + return latest_torch_test_release(current) def test_infra_branch_for_torch(torch_version: str) -> str: @@ -160,7 +242,20 @@ def validate_release_build(torch_pin_path: Path) -> str: or re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", version) is None ): raise RuntimeError(f"{torch_pin_path} has invalid TORCH_VERSION {version!r}") - return f"torch>={version}" + match = re.fullmatch(r"\d+\.(\d+)\.(\d+)((?:a|b|rc)\d+)?", version) + assert match is not None + minor, patch, suffix = match.groups() + expected_versions = { + "TORCHVISION_VERSION": f"0.{int(minor) + 15}.{patch}{suffix or ''}", + "TORCHAUDIO_VERSION": version, + } + for name, expected in expected_versions.items(): + if config.get(name) != expected: + raise RuntimeError( + f"{torch_pin_path} has {name}={config.get(name)!r}, expected {expected!r}" + ) + next_minor = int(minor) + 1 + return f"torch>={version},<{version.split('.')[0]}.{next_minor}" def configure_release_version(version_path: Path, release_version: str) -> str: @@ -211,30 +306,6 @@ def configure_documentation( return changed -def freeze_arm_public_api(repo_root: Path, release_version: str) -> bool: - """Snapshot the Arm API and retain the two newest release manifests.""" - manifest_dir = repo_root / "backends/arm/public_api_manifests" - running = manifest_dir / "api_manifest_running.toml" - if not running.exists(): - return False - - target = manifest_dir / f"api_manifest_{release_version.replace('.', '_')}.toml" - created = False - if not target.exists(): - shutil.copyfile(running, target) - created = True - - static_manifests = sorted( - manifest_dir.glob("api_manifest_[0-9]*_[0-9]*.toml"), - key=lambda path: tuple( - int(part) for part in path.stem.removeprefix("api_manifest_").split("_") - ), - ) - for obsolete in static_manifests[:-2]: - obsolete.unlink() - return created - - def validate_release_references( workflow_paths: Iterable[Path], documentation: Iterable[Path], @@ -278,7 +349,7 @@ def validate_release_references( def validate_release_files(repo_root: Path, release_version: str) -> str: - """Validate release version, API snapshot, and stable QNN installation.""" + """Validate the release version.""" actual_version = (repo_root / "version.txt").read_text().strip() if re.fullmatch(rf"{re.escape(release_version)}\.\d+", actual_version) is None: raise RuntimeError( @@ -287,19 +358,34 @@ def validate_release_files(repo_root: Path, release_version: str) -> str: ) manifest_dir = repo_root / "backends/arm/public_api_manifests" - if manifest_dir.exists(): + if (manifest_dir / "api_manifest_running.toml").exists(): expected_manifest = ( manifest_dir / f"api_manifest_{release_version.replace('.', '_')}.toml" ) if not expected_manifest.exists(): - raise RuntimeError(f"missing release API snapshot {expected_manifest}") - static_manifests = list(manifest_dir.glob("api_manifest_[0-9]*_[0-9]*.toml")) - if len(static_manifests) > 2: - raise RuntimeError("more than two static Arm API manifests remain") + raise RuntimeError( + f"missing {expected_manifest}; merge the Arm API snapshot into main " + "before cutting the release branch" + ) return actual_version +def configured_test_infra_branch(workflow_paths: Iterable[Path]) -> str: + """Read the one test-infra branch already recorded in release workflows.""" + references = set() + for path in workflow_paths: + text = path.read_text() + references.update(_TEST_INFRA_BRANCH_PATTERN.findall(text)) + references.update(_TEST_INFRA_INPUT_PATTERN.findall(text)) + if len(references) != 1: + raise RuntimeError( + "expected exactly one configured test-infra branch, found " + + repr(sorted(references)) + ) + return references.pop() + + def prepare_release( repo_root: Path, release_version: str, @@ -314,6 +400,7 @@ def prepare_release( workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) documentation = documentation_paths(repo_root) set_torch_version(repo_root / "torch_pin.py", torch_version) + set_companion_versions(repo_root / "torch_pin.py", torch_version) enable_release_wheel(repo_root / "torch_pin.py") release_full_version = configure_release_version( repo_root / "version.txt", release_version @@ -322,7 +409,6 @@ def prepare_release( documentation_count = configure_documentation( documentation, f"release/{release_version}", release_full_version ) - manifest_created = freeze_arm_public_api(repo_root, release_version) validate_release_references( workflow_paths, documentation, @@ -334,8 +420,7 @@ def prepare_release( requirement = validate_release_build(repo_root / "torch_pin.py") print( f"Prepared release/{release_version}: {requirement}; changed " - f"{workflow_count} workflow files and {documentation_count} documentation files; " - f"Arm API snapshot {'created' if manifest_created else 'already present'}" + f"{workflow_count} workflow files and {documentation_count} documentation files" ) return requirement @@ -379,8 +464,8 @@ def main() -> None: _validate_release_version(release_version) requirement = validate_release_build(repo_root / "torch_pin.py") torch_version = runpy.run_path(str(repo_root / "torch_pin.py"))["TORCH_VERSION"] - test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( - torch_version + test_infra_branch = args.test_infra_branch or configured_test_infra_branch( + workflow_paths ) release_full_version = validate_release_files(repo_root, release_version) validate_release_references( @@ -395,6 +480,7 @@ def main() -> None: torch_version = torch_version_for_release( repo_root / "torch_pin.py", args.torch_version or "" ) + require_companion_releases(torch_version) test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( torch_version ) diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index 4a9febf8cc6..aa8021dc662 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -9,6 +9,7 @@ from pathlib import Path from prepare_release import ( # type: ignore[import-not-found] + configured_test_infra_branch, newest_torch_test_release, prepare_release, test_infra_branch_for_torch, @@ -22,7 +23,12 @@ class PrepareReleaseTest(unittest.TestCase): def test_selects_release_candidate_once(self) -> None: """A branch cut selects the newest RC and preserves it on reruns.""" releases = ["2.14.1", "2.15.0a1", "2.15.0b2", "2.15.0rc1"] - self.assertEqual(newest_torch_test_release(releases), "2.15.0rc1") + self.assertEqual( + newest_torch_test_release(releases, newer_than="2.14.0"), + "2.15.0rc1", + ) + with self.assertRaises(RuntimeError): + newest_torch_test_release(["2.14.1"], newer_than="2.14.0") self.assertEqual(test_infra_branch_for_torch("2.15.0rc1"), "release/2.15") with tempfile.TemporaryDirectory() as directory: @@ -39,7 +45,10 @@ def test_prepares_complete_repository(self) -> None: (root / "docs").mkdir() (root / "version.txt").write_text("1.6.0a0\n") (root / "torch_pin.py").write_text( - 'TORCH_VERSION = "2.15.0"\nRELEASE_WHEEL = False\n' + 'TORCH_VERSION = "2.14.0"\n' + 'TORCHVISION_VERSION = "0.29.0"\n' + 'TORCHAUDIO_VERSION = "2.11.0"\n' + "RELEASE_WHEEL = False\n" ) workflow = root / ".github/workflows/test.yml" workflow.write_text( @@ -54,33 +63,32 @@ def test_prepares_complete_repository(self) -> None: "stable: swiftpm-1.5.1\n" "nightly: swiftpm-1.6.0.20260929\n" ) - manifests = root / "backends/arm/public_api_manifests" - manifests.mkdir(parents=True) - (manifests / "api_manifest_1_4.toml").write_text("old\n") - (manifests / "api_manifest_1_5.toml").write_text("previous\n") - (manifests / "api_manifest_running.toml").write_text("current\n") - requirement = prepare_release(root, "1.6", "release/2.15", "2.15.0rc1") (root / "version.txt").write_text("1.6.1\n") second_requirement = prepare_release( root, "1.6", "release/2.15", "2.15.0rc1" ) - self.assertEqual(requirement, "torch>=2.15.0rc1") + self.assertEqual(requirement, "torch>=2.15.0rc1,<2.16") self.assertEqual(second_requirement, requirement) self.assertIn("RELEASE_WHEEL = True", (root / "torch_pin.py").read_text()) + self.assertIn( + 'TORCHVISION_VERSION = "0.30.0rc1"', + (root / "torch_pin.py").read_text(), + ) + self.assertIn( + 'TORCHAUDIO_VERSION = "2.15.0rc1"', + (root / "torch_pin.py").read_text(), + ) self.assertIn("@release/2.15", workflow.read_text()) self.assertIn("test-infra-ref: release/2.15", workflow.read_text()) + self.assertEqual(configured_test_infra_branch([workflow]), "release/2.15") self.assertIn("example/action@main", workflow.read_text()) self.assertIn("-b release/1.6", doc.read_text()) self.assertIn("-b release/1.0", doc.read_text()) self.assertIn("stable: swiftpm-1.6.1", doc.read_text()) self.assertIn("nightly: swiftpm-1.6.0.20260929", doc.read_text()) self.assertEqual((root / "version.txt").read_text(), "1.6.1\n") - self.assertFalse((manifests / "api_manifest_1_4.toml").exists()) - self.assertEqual( - (manifests / "api_manifest_1_6.toml").read_text(), "current\n" - ) if __name__ == "__main__": diff --git a/setup.py b/setup.py index f874a901703..ae52ac7e602 100644 --- a/setup.py +++ b/setup.py @@ -1094,17 +1094,9 @@ def _package_relative_depth(library: Path) -> int: return max(len(parts) - index - 2, 0) -def _torchao_requirement() -> str: - """The torchao dependency, pinned to the series install_requirements.py installs. - - Derived from that module rather than written out, so a nightly bump cannot move the - pin without moving this bound with it. A bump into the next series would otherwise - silently stop satisfying the lower bound, and installing this package over a - development checkout would replace the torchao that was just installed. - - Loaded by path, the way install_utils is above, because setuptools executes this - file without the project directory on sys.path, so a plain import does not resolve. - """ +@functools.lru_cache(maxsize=1) +def _load_install_requirements(): + """Load release dependency pins once without relying on the checkout's sys.path.""" path = Path(__file__).parent / "install_requirements.py" spec = importlib.util.spec_from_file_location("install_requirements", path) if spec is None or spec.loader is None: @@ -1114,6 +1106,12 @@ def _torchao_requirement() -> str: # because that name is registered below. sys.modules.setdefault("install_utils", install_utils) spec.loader.exec_module(module) + return module + + +def _torchao_requirement() -> str: + """The torchao dependency, pinned to the series development installs use.""" + module = _load_install_requirements() version = module.TORCHAO_NIGHTLY_VERSION if ( @@ -1126,24 +1124,34 @@ def _torchao_requirement() -> str: def _release_torch_requirement() -> List[str]: - if not torch_pin.RELEASE_WHEEL: + if ( + not torch_pin.RELEASE_WHEEL + or os.environ.get("EXECUTORCH_BUILDING_WHEEL") != "1" + ): return [] - path = Path(__file__).parent / "install_requirements.py" - spec = importlib.util.spec_from_file_location("install_requirements", path) - if spec is None or spec.loader is None: - raise ImportError(f"Could not load {path}") - module = importlib.util.module_from_spec(spec) - sys.modules.setdefault("install_utils", install_utils) - spec.loader.exec_module(module) + from importlib import metadata + + module = _load_install_requirements() torch_url = install_utils.determine_torch_url(module.TORCH_URL_BASE) - cu134_requirements = module.cu134_requirements(torch_url) - if cu134_requirements: - # The CUDA 13.4 build currently uses a PyTorch development wheel. A - # stable lower bound would reject and replace the binary it was built - # against, so this variant must retain its exact build-time dependency. - return [cu134_requirements[0]] - return [f"torch>={torch_pin.TORCH_VERSION}"] + variant = torch_url.rstrip("/").rsplit("/", 1)[-1] + if re.fullmatch(r"cu\d+", variant): + installed_version = metadata.version("torch") + if f"+{variant}" not in installed_version: + raise RuntimeError( + f"building a {variant} wheel with torch {installed_version}; " + "the installed torch build must use the same CUDA variant" + ) + return [f"torch=={installed_version}"] + + match = re.fullmatch( + r"(\d+)\.(\d+)\.\d+(?:(?:a|b|rc)\d+)?", torch_pin.TORCH_VERSION + ) + if match is None: + raise RuntimeError(f"invalid release Torch version {torch_pin.TORCH_VERSION!r}") + return [ + f"torch>={torch_pin.TORCH_VERSION},<{match.group(1)}.{int(match.group(2)) + 1}" + ] def _base_dependencies() -> List[str]: diff --git a/torch_pin.py b/torch_pin.py index 7f9551200bd..8f7a7986220 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -1,5 +1,7 @@ # PyTorch release used by development installs and release-wheel metadata. TORCH_VERSION = "2.14.0" +TORCHVISION_VERSION = "0.29.0" +TORCHAUDIO_VERSION = "2.11.0" NIGHTLY_VERSION = "dev20260913" # Changed to True by scripts/release/apply-release-changes.sh. Release wheels From d508ab3c70ee9514fe827455712cba94eb524478 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 13:13:07 -0700 Subject: [PATCH 07/12] Fix release preparation dependency handling --- .ci/scripts/test_wheel_package_qnn.sh | 3 +- .ci/scripts/tests/test_cu134_dependencies.py | 48 ++++--- .ci/scripts/wheel/envvar_base.sh | 1 + .ci/scripts/wheel/envvar_cuda_linux.sh | 3 + .ci/scripts/wheel/test_clean_install.py | 57 +++++++-- .ci/scripts/wheel/vc_env_helper.bat | 2 + .github/scripts/update_pytorch_pin.py | 105 +++++++-------- scripts/release/cut-release-branch.sh | 27 +++- scripts/release/prepare_release.py | 128 ++++++++++++------- scripts/release/release_versions.py | 64 ++++++++++ scripts/release/test_prepare_release.py | 6 +- setup.py | 39 +++--- torch_pin.py | 2 + 13 files changed, 325 insertions(+), 160 deletions(-) create mode 100644 scripts/release/release_versions.py diff --git a/.ci/scripts/test_wheel_package_qnn.sh b/.ci/scripts/test_wheel_package_qnn.sh index 568c90ed14c..133ec885321 100644 --- a/.ci/scripts/test_wheel_package_qnn.sh +++ b/.ci/scripts/test_wheel_package_qnn.sh @@ -98,8 +98,7 @@ install_qnn export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" install_executorch -EXECUTORCH_BUILDING_WHEEL=1 python setup.py bdist_wheel -unset EXECUTORCH_BUILDING_WHEEL +EXECUTORCH_BUILDING_WHEEL=1 EXECUTORCH_WHEEL_VARIANT=cpu python setup.py bdist_wheel WHEEL_FILE=$(ls dist/*.whl | head -n 1) echo "Found wheel: $WHEEL_FILE" diff --git a/.ci/scripts/tests/test_cu134_dependencies.py b/.ci/scripts/tests/test_cu134_dependencies.py index 234b35baa9c..1b9f18455e4 100644 --- a/.ci/scripts/tests/test_cu134_dependencies.py +++ b/.ci/scripts/tests/test_cu134_dependencies.py @@ -8,7 +8,6 @@ import functools import importlib.util import os -import re import subprocess import sys import unittest @@ -31,6 +30,7 @@ def load_module(name): class TestCu134Dependencies(unittest.TestCase): def setUp(self): self.utils = load_module("install_utils") + self.release_versions = load_module("scripts/release/release_versions") self.modules = patch.dict(sys.modules, {"install_utils": self.utils}) self.modules.start() self.addCleanup(self.modules.stop) @@ -147,7 +147,12 @@ def test_failure_is_not_retried_with_another_cuda_train(self): self.assertEqual(run.call_count, 1) def setup_requirement( - self, function_name, *, installed_torch="2.15.0rc1", building_wheel=True + self, + function_name, + *, + installed_torch="2.15.0rc1", + building_wheel=True, + wheel_variant="cpu", ): path = ROOT / "setup.py" tree = ast.parse(path.read_text()) @@ -169,16 +174,23 @@ def setup_requirement( "functools": functools, "importlib": importlib, "os": os, - "re": re, "sys": sys, "install_utils": self.utils, + "release_versions": self.release_versions, "torch_pin": SimpleNamespace(RELEASE_WHEEL=True, TORCH_VERSION="2.15.0rc1"), } exec( compile(ast.Module(body=functions, type_ignores=[]), str(path), "exec"), namespace, ) - environment = {"EXECUTORCH_BUILDING_WHEEL": "1"} if building_wheel else {} + environment = ( + { + "EXECUTORCH_BUILDING_WHEEL": "1", + "EXECUTORCH_WHEEL_VARIANT": wheel_variant, + } + if building_wheel + else {} + ) with ( patch.dict(os.environ, environment, clear=True), patch("importlib.metadata.version", return_value=installed_torch), @@ -270,30 +282,28 @@ def test_cu134_keeps_explicit_torchao_source_build(self): self.assertIn("0.19.0+gitb7ac3aa", metadata.specifier) def test_wheel_bounds_match_selected_train(self): - for cuda, installed_torch, expected_torch in ( - ((13, 4), "2.14.0.dev20260810+cu134", "torch==2.14.0.dev20260810+cu134"), - ((13, 2), "2.15.0rc1+cu132", "torch==2.15.0rc1+cu132"), - (None, "2.15.0rc1", "torch>=2.15.0rc1,<2.16"), + for wheel_variant, installed_torch, expected_torch in ( + ("cu132", "2.15.0rc1+cu132", "torch==2.15.0rc1+cu132"), + ("cpu", "2.15.0rc1", "torch>=2.15.0rc1,<2.16"), ): - self.utils.determine_torch_url.cache_clear() - with ( - patch.object( - self.utils, - "_get_cuda_version", - return_value=cuda, - side_effect=RuntimeError("no nvcc") if cuda is None else None, - ), - patch.object(self.installer.platform, "system", return_value="Linux"), - ): + with self.subTest(wheel_variant=wheel_variant): self.assertEqual( self.torchao_requirement(), "torchao>=0.19.0.dev20260907,<0.20", ) self.assertEqual( - self.release_torch_requirement(installed_torch=installed_torch), + self.release_torch_requirement( + installed_torch=installed_torch, + wheel_variant=wheel_variant, + ), expected_torch, ) + with self.assertRaisesRegex(RuntimeError, "for Torch 2.15.0rc1"): + self.release_torch_requirement( + installed_torch="2.14.0.dev20260810+cu134", + wheel_variant="cu134", + ) self.assertIsNone(self.release_torch_requirement(building_wheel=False)) diff --git a/.ci/scripts/wheel/envvar_base.sh b/.ci/scripts/wheel/envvar_base.sh index ba97136ebda..dd06905564d 100755 --- a/.ci/scripts/wheel/envvar_base.sh +++ b/.ci/scripts/wheel/envvar_base.sh @@ -16,3 +16,4 @@ export CMAKE_ARGS="${CMAKE_ARGS:-}" # binary wheel jobs. A source install on a release branch must preserve the # PyTorch build already installed by that checkout's CI job. export EXECUTORCH_BUILDING_WHEEL=1 +export EXECUTORCH_WHEEL_VARIANT=cpu diff --git a/.ci/scripts/wheel/envvar_cuda_linux.sh b/.ci/scripts/wheel/envvar_cuda_linux.sh index fda1a6b88a3..282c499226c 100644 --- a/.ci/scripts/wheel/envvar_cuda_linux.sh +++ b/.ci/scripts/wheel/envvar_cuda_linux.sh @@ -10,6 +10,9 @@ source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh" +_executorch_cuda_version="${CU_VERSION:-${DESIRED_CUDA:-}}" +export EXECUTORCH_WHEEL_VARIANT="cu${_executorch_cuda_version//[!0-9]/}" + # Ask for the CUDA delegate explicitly rather than letting the build detect a toolkit. A detected # build is fine locally, but a release row states what it is producing, and a row that silently # produced a CPU wheel because the toolkit was missing would publish under a CUDA name. diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index ffe04f8d1a6..f0107516a59 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -63,6 +63,9 @@ def test_release_pytorch_requirement() -> None: repo_root = Path(__file__).resolve().parents[3] config = runpy.run_path(str(repo_root / "torch_pin.py")) + release_versions = runpy.run_path( + str(repo_root / "scripts/release/release_versions.py") + ) ci_refs = ( os.environ.get("GITHUB_REF_NAME", ""), os.environ.get("GITHUB_BASE_REF", ""), @@ -84,25 +87,53 @@ def test_release_pytorch_requirement() -> None: ] if config["RELEASE_WHEEL"]: - installed_torch = Version(metadata.version("torch")) - if installed_torch.local and re.search( - r"(?:^|\.)cu\d+(?:\.|$)", installed_torch.local - ): - expected_requirement = Requirement(f"torch=={installed_torch}") - else: - major, minor, *_ = config["TORCH_VERSION"].split(".") - expected_requirement = Requirement( - f"torch>={config['TORCH_VERSION']},<{major}.{int(minor) + 1}" - ) - expected = str(expected_requirement.specifier) assert len(torch_requirements) == 1, ( "a release wheel must declare exactly one PyTorch dependency, but found " f"{[str(requirement) for requirement in torch_requirements]}" ) - assert str(torch_requirements[0].specifier) == expected, ( - f"release wheel declares {torch_requirements[0]}, expected torch{expected} " + requirement = torch_requirements[0] + pins = [ + specifier.version + for specifier in requirement.specifier + if specifier.operator == "==" + ] + configured_variant = os.environ.get("EXECUTORCH_WHEEL_VARIANT", "") + cuda_version = os.environ.get("CU_VERSION") or os.environ.get( + "DESIRED_CUDA", "" + ) + if not configured_variant and cuda_version: + configured_variant = "cu" + "".join( + character for character in cuda_version if character.isdigit() + ) + if len(pins) == 1 and "+" in pins[0]: + local_version = Version(pins[0]).local or "" + variants = re.findall(r"(?:^|\.)(cu\d+)(?:\.|$)", local_version) + assert ( + len(variants) == 1 + ), f"release wheel has an invalid CUDA Torch pin: {requirement}" + assert configured_variant in ( + "", + variants[0], + ), f"{configured_variant} wheel declares the {variants[0]} Torch build" + expected_raw = release_versions["torch_requirement"]( + config["TORCH_VERSION"], variants[0], pins[0] + ) + else: + assert not configured_variant.startswith( + "cu" + ), f"{configured_variant} wheel declares a generic CPU Torch requirement" + expected_raw = release_versions["torch_requirement"]( + config["TORCH_VERSION"], "cpu" + ) + expected = str(Requirement(expected_raw).specifier) + assert str(requirement.specifier) == expected, ( + f"release wheel declares {requirement}, expected torch{expected} " "from torch_pin.py" ) + installed_torch = Version(metadata.version("torch")) + assert requirement.specifier.contains( + installed_torch, prereleases=True + ), f"installed torch {installed_torch} does not satisfy {requirement}" print(f"✓ release wheel declares torch{expected}") else: assert not torch_requirements, ( diff --git a/.ci/scripts/wheel/vc_env_helper.bat b/.ci/scripts/wheel/vc_env_helper.bat index d7fa2c1a596..13c0436c2d5 100644 --- a/.ci/scripts/wheel/vc_env_helper.bat +++ b/.ci/scripts/wheel/vc_env_helper.bat @@ -26,6 +26,8 @@ if "%VSDEVCMD_ARGS%" == "" ( if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" set DISTUTILS_USE_SDK=1 +set EXECUTORCH_BUILDING_WHEEL=1 +set EXECUTORCH_WHEEL_VARIANT=cpu set args=%1 shift diff --git a/.github/scripts/update_pytorch_pin.py b/.github/scripts/update_pytorch_pin.py index dbc48552d9b..810d8bf13fd 100644 --- a/.github/scripts/update_pytorch_pin.py +++ b/.github/scripts/update_pytorch_pin.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 -import base64 -import hashlib +import argparse import json +import os import re import sys import urllib.request from pathlib import Path +from urllib.parse import quote def parse_nightly_version(nightly_version): @@ -55,12 +56,14 @@ def get_commit_hash_for_nightly(date_str): Commit hash string """ api_url = "https://api.github.com/repos/pytorch/pytorch/commits" - params = f"?sha=nightly&per_page=50" + params = "?sha=nightly&per_page=50" url = api_url + params req = urllib.request.Request(url) req.add_header("Accept", "application/vnd.github.v3+json") req.add_header("User-Agent", "ExecuTorch-Bot") + if os.environ.get("GITHUB_TOKEN"): + req.add_header("Authorization", f"Bearer {os.environ['GITHUB_TOKEN']}") try: with urllib.request.urlopen(req) as response: @@ -120,7 +123,7 @@ def should_skip_file(filename): def fetch_file_content(commit_hash, file_path): """ - Fetch file content from GitHub API. + Fetch file content from GitHub's raw-content service. Args: commit_hash: Commit hash to fetch from @@ -129,26 +132,27 @@ def fetch_file_content(commit_hash, file_path): Returns: File content as bytes """ - api_url = f"https://api.github.com/repos/pytorch/pytorch/contents/{file_path}?ref={commit_hash}" + raw_url = ( + "https://raw.githubusercontent.com/pytorch/pytorch/" + f"{quote(commit_hash, safe='')}/{quote(file_path)}" + ) - req = urllib.request.Request(api_url) - req.add_header("Accept", "application/vnd.github.v3+json") + req = urllib.request.Request(raw_url) req.add_header("User-Agent", "ExecuTorch-Bot") + if os.environ.get("GITHUB_TOKEN"): + req.add_header("Authorization", f"Bearer {os.environ['GITHUB_TOKEN']}") try: with urllib.request.urlopen(req) as response: - data = json.loads(response.read().decode()) - # Content is base64 encoded - content = base64.b64decode(data["content"]) - return content + return response.read() except urllib.request.HTTPError as e: print(f"Error fetching file {file_path}: {e}", file=sys.stderr) raise -def sync_directory(et_dir, pt_path, commit_hash): +def directory_updates(et_dir, pt_path, commit_hash): """ - Sync files from PyTorch to ExecuTorch using GitHub API. + Sync files from PyTorch to ExecuTorch using GitHub raw content. Only syncs files that already exist in ExecuTorch - does not add new files. Args: @@ -156,15 +160,15 @@ def sync_directory(et_dir, pt_path, commit_hash): pt_path: PyTorch directory path in the repository (e.g., "c10") commit_hash: Commit hash to fetch from - Returns: - Number of files grafted + Returns a list of (path, content) updates without mutating the checkout. """ - files_grafted = 0 print(f"Checking {et_dir} vs pytorch/{pt_path}...") if not et_dir.exists(): print(f"Warning: ExecuTorch directory {et_dir} does not exist, skipping") - return 0 + return [] + + updates = [] # Loop through files in ExecuTorch directory for et_file in et_dir.rglob("*"): @@ -186,19 +190,12 @@ def sync_directory(et_dir, pt_path, commit_hash): if pt_content != et_content: print(f"⚠️ Difference detected in {rel_path}") - print(f"📋 Grafting from PyTorch commit {commit_hash}...") - - et_file.write_bytes(pt_content) - print(f"✅ Grafted {et_file}") - files_grafted += 1 + updates.append((et_file, pt_content)) except urllib.request.HTTPError as e: - if e.code != 404: # It's ok to have more files in ET than pytorch/pytorch. - print(f"Error fetching {rel_path} from PyTorch: {e}") - except Exception as e: - print(f"Error syncing {rel_path}: {e}") - continue + if e.code != 404: + raise - return files_grafted + return updates def sync_c10_directories(commit_hash): @@ -229,38 +226,46 @@ def sync_c10_directories(commit_hash): ), ] - total_grafted = 0 + updates = [] for et_dir, pt_path in dir_pairs: - files_grafted = sync_directory(et_dir, pt_path, commit_hash) - total_grafted += files_grafted + updates.extend(directory_updates(et_dir, pt_path, commit_hash)) + + for et_file, pt_content in updates: + print(f"📋 Grafting from PyTorch commit {commit_hash}...") + et_file.write_bytes(pt_content) + print(f"✅ Grafted {et_file}") - if total_grafted > 0: - print(f"\n✅ Successfully grafted {total_grafted} file(s) from PyTorch") + if updates: + print(f"\n✅ Successfully grafted {len(updates)} file(s) from PyTorch") else: print("\n✅ No differences found - c10 is in sync") - return total_grafted + return len(updates) def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--commit", + help="PyTorch release commit to use instead of resolving NIGHTLY_VERSION", + ) + args = parser.parse_args() try: - # Read NIGHTLY_VERSION from torch_pin.py - nightly_version = get_torch_nightly_version() - print(f"Found NIGHTLY_VERSION: {nightly_version}") - - # Parse to date string - date_str = parse_nightly_version(nightly_version) - print(f"Parsed date: {date_str}") - - # Fetch commit hash from PyTorch nightly branch - commit_hash = get_commit_hash_for_nightly(date_str) - print(f"Found commit hash: {commit_hash}") - - # Update the pin file - update_pytorch_pin(commit_hash) - - # Sync c10 directories from PyTorch + if args.commit: + commit_hash = args.commit + print(f"Using requested PyTorch commit: {commit_hash}") + else: + nightly_version = get_torch_nightly_version() + print(f"Found NIGHTLY_VERSION: {nightly_version}") + date_str = parse_nightly_version(nightly_version) + print(f"Parsed date: {date_str}") + commit_hash = get_commit_hash_for_nightly(date_str) + print(f"Found commit hash: {commit_hash}") + + # Fetch every header before changing the checkout. If the network or + # rate limit fails, the existing source pin and headers stay together. sync_c10_directories(commit_hash) + update_pytorch_pin(commit_hash) print( "\n✅ Successfully updated PyTorch commit pin and synced c10 directories!" diff --git a/scripts/release/cut-release-branch.sh b/scripts/release/cut-release-branch.sh index 0bfc2718fb5..eea98e156da 100755 --- a/scripts/release/cut-release-branch.sh +++ b/scripts/release/cut-release-branch.sh @@ -35,12 +35,29 @@ fi git fetch --all ) -# Read version.txt directly from viable/strict so the caller's starting branch -# cannot influence which release is cut. This preserves all numeric components -# (for example, 1.10 does not become 1.1). VIABLE_BRANCH="${GIT_REMOTE}/viable/strict" -SOURCE_VERSION=$(git show "${VIABLE_BRANCH}:version.txt") -RELEASE_VERSION=${RELEASE_VERSION:-$(printf '%s\n' "${SOURCE_VERSION}" | cut -d'.' -f1-2)} +if [[ -z "${RELEASE_VERSION:-}" ]]; then + # A failed first run may already have preserved the cut point. Resume that + # release even if viable/strict has since moved to the next line. + PENDING_RELEASES=() + while IFS= read -r ORIGINAL_REF; do + CANDIDATE=${ORIGINAL_REF#refs/remotes/${GIT_REMOTE}/orig/release/} + if ! git show-ref --verify --quiet "refs/remotes/${GIT_REMOTE}/release/${CANDIDATE}"; then + PENDING_RELEASES+=("${CANDIDATE}") + fi + done < <(git for-each-ref --format='%(refname)' "refs/remotes/${GIT_REMOTE}/orig/release/") + if [[ ${#PENDING_RELEASES[@]} -gt 1 ]]; then + echo "Error: multiple unfinished release cuts: ${PENDING_RELEASES[*]}" + exit 1 + elif [[ ${#PENDING_RELEASES[@]} -eq 1 ]]; then + RELEASE_VERSION="${PENDING_RELEASES[0]}" + else + # Read version.txt directly from viable/strict so the caller's starting + # branch cannot influence a new release cut. + SOURCE_VERSION=$(git show "${VIABLE_BRANCH}:version.txt") + RELEASE_VERSION=$(printf '%s\n' "${SOURCE_VERSION}" | cut -d'.' -f1-2) + fi +fi RELEASE_BRANCH="release/${RELEASE_VERSION}" ORIGINAL_BRANCH="orig/${RELEASE_BRANCH}" ARM_MANIFEST="backends/arm/public_api_manifests/api_manifest_${RELEASE_VERSION//./_}.toml" diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index 592d6cc9d7f..f902aff930f 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -11,11 +11,19 @@ import re import runpy import subprocess +import sys import urllib.request from pathlib import Path from typing import Iterable, List from urllib.parse import unquote +from release_versions import ( # type: ignore[import-not-found] + release_key, + torch_release_tag, + torch_requirement, + torchvision_version, +) + _RELEASE_VERSION_PATTERN = re.compile(r"\d+\.\d+") _RELEASE_WHEEL_PATTERN = re.compile( r"^RELEASE_WHEEL\s*=\s*(?:True|False)$", re.MULTILINE @@ -82,37 +90,18 @@ def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: def set_companion_versions(torch_pin_path: Path, torch_version: str) -> bool: - """Move torchvision and torchaudio to the selected PyTorch release train.""" - match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)((?:a|b|rc)\d+)?", torch_version) - if match is None: - raise RuntimeError(f"invalid Torch release version {torch_version!r}") - _major, minor, patch, suffix = match.groups() - torchvision_version = f"0.{int(minor) + 15}.{patch}{suffix or ''}" - torchaudio_version = torch_version - + """Move torchvision to the selected PyTorch release train.""" text = torch_pin_path.read_text() updated, vision_count = _TORCHVISION_VERSION_PATTERN.subn( - f'TORCHVISION_VERSION = "{torchvision_version}"', text + f'TORCHVISION_VERSION = "{torchvision_version(torch_version)}"', text ) - updated, audio_count = _TORCHAUDIO_VERSION_PATTERN.subn( - f'TORCHAUDIO_VERSION = "{torchaudio_version}"', updated - ) - if vision_count != 1 or audio_count != 1: + if vision_count != 1 or _TORCHAUDIO_VERSION_PATTERN.search(updated) is None: raise RuntimeError( f"expected exactly one torchvision and torchaudio assignment in {torch_pin_path}" ) return _write_if_changed(torch_pin_path, updated) -def _release_key(version: str) -> tuple[int, int, int, int, int]: - match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?", version) - if match is None: - raise RuntimeError(f"invalid release version {version!r}") - stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} - major, minor, patch = (int(part) for part in match.groups()[:3]) - return major, minor, patch, stage_rank[match.group(4)], int(match.group(5) or 0) - - def newest_torch_test_release( available_versions: Iterable[str], newer_than: str = "" ) -> str: @@ -122,8 +111,8 @@ def newest_torch_test_release( match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:(rc)(\d+))?", version) if match is None: continue - key = _release_key(version) - if newer_than and key[:2] <= _release_key(newer_than)[:2]: + key = release_key(version) + if newer_than and key[:2] <= release_key(newer_than)[:2]: continue candidates.append((key, version)) if not candidates: @@ -154,15 +143,12 @@ def latest_torch_test_release(newer_than: str) -> str: ) -def require_companion_releases(torch_version: str) -> None: +def require_companion_releases(torch_pin_path: Path, torch_version: str) -> None: """Fail before mutation unless matching domain-library releases exist.""" - match = re.fullmatch(r"\d+\.(\d+)\.(\d+)((?:a|b|rc)\d+)?", torch_version) - if match is None: - raise RuntimeError(f"invalid Torch release version {torch_version!r}") - minor, patch, suffix = match.groups() + config = runpy.run_path(str(torch_pin_path)) expected = { - "torchvision": f"0.{int(minor) + 15}.{patch}{suffix or ''}", - "torchaudio": torch_version, + "torchvision": torchvision_version(torch_version), + "torchaudio": config["TORCHAUDIO_VERSION"], } missing = [ f"{package}=={version}" @@ -181,8 +167,8 @@ def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: config = runpy.run_path(str(torch_pin_path)) current = config["TORCH_VERSION"] if override: - current_key = _release_key(current) - override_key = _release_key(override) + current_key = release_key(current) + override_key = release_key(override) already_prepared = config.get("RELEASE_WHEEL") is True invalid = ( override_key < current_key @@ -230,6 +216,49 @@ def require_test_infra_branch(test_infra_branch: str) -> None: ) +def pytorch_commit_for_release(torch_version: str) -> str: + """Resolve the selected PyTorch release tag before changing the checkout.""" + tag = torch_release_tag(torch_version) + result = subprocess.run( + [ + "git", + "ls-remote", + "https://github.com/pytorch/pytorch.git", + f"refs/tags/{tag}", + f"refs/tags/{tag}^{{}}", + ], + check=True, + capture_output=True, + text=True, + ) + references = { + reference: commit + for commit, reference in ( + line.split("\t", 1) for line in result.stdout.splitlines() if "\t" in line + ) + } + commit = references.get(f"refs/tags/{tag}^{{}}") or references.get( + f"refs/tags/{tag}" + ) + if commit is None: + raise RuntimeError(f"PyTorch release tag {tag} does not exist") + return commit + + +def sync_pytorch_source(repo_root: Path, commit: str) -> None: + """Pin PyTorch source and synchronize the vendored c10 headers.""" + subprocess.run( + [ + sys.executable, + str(repo_root / ".github/scripts/update_pytorch_pin.py"), + "--commit", + commit, + ], + cwd=repo_root, + check=True, + ) + + def validate_release_build(torch_pin_path: Path) -> str: """Return the release PyTorch requirement or raise for invalid config.""" config = runpy.run_path(str(torch_pin_path)) @@ -242,20 +271,21 @@ def validate_release_build(torch_pin_path: Path) -> str: or re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", version) is None ): raise RuntimeError(f"{torch_pin_path} has invalid TORCH_VERSION {version!r}") - match = re.fullmatch(r"\d+\.(\d+)\.(\d+)((?:a|b|rc)\d+)?", version) - assert match is not None - minor, patch, suffix = match.groups() - expected_versions = { - "TORCHVISION_VERSION": f"0.{int(minor) + 15}.{patch}{suffix or ''}", - "TORCHAUDIO_VERSION": version, - } - for name, expected in expected_versions.items(): - if config.get(name) != expected: - raise RuntimeError( - f"{torch_pin_path} has {name}={config.get(name)!r}, expected {expected!r}" - ) - next_minor = int(minor) + 1 - return f"torch>={version},<{version.split('.')[0]}.{next_minor}" + expected_vision = torchvision_version(version) + if config.get("TORCHVISION_VERSION") != expected_vision: + raise RuntimeError( + f"{torch_pin_path} has TORCHVISION_VERSION=" + f"{config.get('TORCHVISION_VERSION')!r}, expected {expected_vision!r}" + ) + audio_version = config.get("TORCHAUDIO_VERSION") + if ( + not isinstance(audio_version, str) + or re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", audio_version) is None + ): + raise RuntimeError( + f"{torch_pin_path} has invalid TORCHAUDIO_VERSION {audio_version!r}" + ) + return torch_requirement(version, "cpu") def configure_release_version(version_path: Path, release_version: str) -> str: @@ -480,12 +510,14 @@ def main() -> None: torch_version = torch_version_for_release( repo_root / "torch_pin.py", args.torch_version or "" ) - require_companion_releases(torch_version) + require_companion_releases(repo_root / "torch_pin.py", torch_version) test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( torch_version ) require_test_infra_branch(test_infra_branch) + pytorch_commit = pytorch_commit_for_release(torch_version) prepare_release(repo_root, release_version, test_infra_branch, torch_version) + sync_pytorch_source(repo_root, pytorch_commit) if __name__ == "__main__": diff --git a/scripts/release/release_versions.py b/scripts/release/release_versions.py new file mode 100644 index 00000000000..b4ebdc63cc7 --- /dev/null +++ b/scripts/release/release_versions.py @@ -0,0 +1,64 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +"""Shared version rules for release preparation and wheel metadata.""" + +import re + +_RELEASE_PATTERN = re.compile(r"(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?") + + +def release_key(version: str) -> tuple[int, int, int, int, int]: + match = _RELEASE_PATTERN.fullmatch(version) + if match is None: + raise RuntimeError(f"invalid release version {version!r}") + stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} + major, minor, patch = (int(part) for part in match.groups()[:3]) + return major, minor, patch, stage_rank[match.group(4)], int(match.group(5) or 0) + + +def torchvision_version(torch_version: str) -> str: + match = _RELEASE_PATTERN.fullmatch(torch_version) + if match is None: + raise RuntimeError(f"invalid Torch release version {torch_version!r}") + _major, minor, patch, stage, stage_number = match.groups() + suffix = f"{stage}{stage_number}" if stage else "" + return f"0.{int(minor) + 15}.{patch}{suffix}" + + +def torch_release_tag(torch_version: str) -> str: + match = _RELEASE_PATTERN.fullmatch(torch_version) + if match is None: + raise RuntimeError(f"invalid Torch release version {torch_version!r}") + major, minor, patch, stage, stage_number = match.groups() + suffix = f"-{stage}{stage_number}" if stage else "" + return f"v{major}.{minor}.{patch}{suffix}" + + +def torch_requirement( + torch_version: str, wheel_variant: str, installed_version: str = "" +) -> str: + major, minor, _patch, _stage, _stage_number = release_key(torch_version) + if wheel_variant == "cpu": + return f"torch>={torch_version},<{major}.{minor + 1}" + if re.fullmatch(r"cu\d+", wheel_variant) is None: + raise RuntimeError(f"invalid release wheel variant {wheel_variant!r}") + if not installed_version: + raise RuntimeError( + f"the {wheel_variant} wheel build did not report its Torch version" + ) + public_version, separator, local_version = installed_version.partition("+") + if public_version != torch_version: + raise RuntimeError( + f"building a {wheel_variant} wheel for Torch {torch_version} with " + f"torch {installed_version}" + ) + if not separator or wheel_variant not in local_version.split("."): + raise RuntimeError( + f"building a {wheel_variant} wheel with torch {installed_version}; " + "the installed torch build must use the same CUDA variant" + ) + return f"torch=={installed_version}" diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index aa8021dc662..4042f75d2e2 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -12,7 +12,7 @@ configured_test_infra_branch, newest_torch_test_release, prepare_release, - test_infra_branch_for_torch, + test_infra_branch_for_torch as _test_infra_branch_for_torch, torch_version_for_release, ) @@ -29,7 +29,7 @@ def test_selects_release_candidate_once(self) -> None: ) with self.assertRaises(RuntimeError): newest_torch_test_release(["2.14.1"], newer_than="2.14.0") - self.assertEqual(test_infra_branch_for_torch("2.15.0rc1"), "release/2.15") + self.assertEqual(_test_infra_branch_for_torch("2.15.0rc1"), "release/2.15") with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "torch_pin.py" @@ -77,7 +77,7 @@ def test_prepares_complete_repository(self) -> None: (root / "torch_pin.py").read_text(), ) self.assertIn( - 'TORCHAUDIO_VERSION = "2.15.0rc1"', + 'TORCHAUDIO_VERSION = "2.11.0"', (root / "torch_pin.py").read_text(), ) self.assertIn("@release/2.15", workflow.read_text()) diff --git a/setup.py b/setup.py index ae52ac7e602..c1f4017ea7f 100644 --- a/setup.py +++ b/setup.py @@ -87,6 +87,17 @@ raise ImportError(f"Could not load {_torch_pin_path}") _spec.loader.exec_module(torch_pin) +_release_versions_path = Path(__file__).parent / "scripts/release/release_versions.py" +_spec = importlib.util.spec_from_file_location( + "release_versions", _release_versions_path +) +if _spec is None: + raise ImportError(f"Could not create module spec for {_release_versions_path}") +release_versions = importlib.util.module_from_spec(_spec) +if _spec.loader is None: + raise ImportError(f"Could not load {_release_versions_path}") +_spec.loader.exec_module(release_versions) + from setuptools import Distribution, Extension, find_namespace_packages, setup from setuptools.command.build import build from setuptools.command.build_ext import build_ext @@ -1132,25 +1143,12 @@ def _release_torch_requirement() -> List[str]: from importlib import metadata - module = _load_install_requirements() - torch_url = install_utils.determine_torch_url(module.TORCH_URL_BASE) - variant = torch_url.rstrip("/").rsplit("/", 1)[-1] - if re.fullmatch(r"cu\d+", variant): - installed_version = metadata.version("torch") - if f"+{variant}" not in installed_version: - raise RuntimeError( - f"building a {variant} wheel with torch {installed_version}; " - "the installed torch build must use the same CUDA variant" - ) - return [f"torch=={installed_version}"] - - match = re.fullmatch( - r"(\d+)\.(\d+)\.\d+(?:(?:a|b|rc)\d+)?", torch_pin.TORCH_VERSION - ) - if match is None: - raise RuntimeError(f"invalid release Torch version {torch_pin.TORCH_VERSION!r}") + variant = os.environ.get("EXECUTORCH_WHEEL_VARIANT", "") + installed_version = metadata.version("torch") if variant.startswith("cu") else "" return [ - f"torch>={torch_pin.TORCH_VERSION},<{match.group(1)}.{int(match.group(2)) + 1}" + release_versions.torch_requirement( + torch_pin.TORCH_VERSION, variant, installed_version + ) ] @@ -1178,7 +1176,6 @@ def _base_dependencies() -> List[str]: "py-cpuinfo", "requests", "pytorch-tokenizers", - *_release_torch_requirement(), # Shipped code imports torchao at module scope in many places, so a plain install cannot # lower a model without it. Among others: the XNNPACK utilities the partitioner uses # (backends/xnnpack/utils/utils.py), the Core ML quantizer, and executorch.export itself. @@ -2892,7 +2889,9 @@ def iter_distribution_names(self): setup_kwargs["packages"] = _full_packages() # A CUDA wheel links the CUDA runtime but does not bundle it, so the wheels that # carry it are declared here. A CPU wheel adds nothing. - setup_kwargs["install_requires"] = _base_dependencies() + _cuda_dependencies() + setup_kwargs["install_requires"] = ( + _base_dependencies() + _release_torch_requirement() + _cuda_dependencies() + ) setup( diff --git a/torch_pin.py b/torch_pin.py index 8f7a7986220..58660177313 100644 --- a/torch_pin.py +++ b/torch_pin.py @@ -2,6 +2,8 @@ TORCH_VERSION = "2.14.0" TORCHVISION_VERSION = "0.29.0" TORCHAUDIO_VERSION = "2.11.0" +# Date used by the main-branch weekly pin bot. Release preparation resolves the +# selected TORCH_VERSION tag and updates the source commit separately. NIGHTLY_VERSION = "dev20260913" # Changed to True by scripts/release/apply-release-changes.sh. Release wheels From 5c4b955f4ed3f99f62a27c501470e3aa39d6b047 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 13:15:28 -0700 Subject: [PATCH 08/12] Skip constructed raw URL in link lint --- .github/scripts/update_pytorch_pin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/update_pytorch_pin.py b/.github/scripts/update_pytorch_pin.py index 810d8bf13fd..2886602a9b1 100644 --- a/.github/scripts/update_pytorch_pin.py +++ b/.github/scripts/update_pytorch_pin.py @@ -133,7 +133,7 @@ def fetch_file_content(commit_hash, file_path): File content as bytes """ raw_url = ( - "https://raw.githubusercontent.com/pytorch/pytorch/" + "https://raw.githubusercontent.com/pytorch/pytorch/" # @lint-ignore f"{quote(commit_hash, safe='')}/{quote(file_path)}" ) From 4ee539ef20ad3a0f398d12ff49d34dfa5875e3ee Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 14:48:01 -0700 Subject: [PATCH 09/12] Resolve release candidate wheel compatibility --- .ci/scripts/test_wheel_package_qnn.sh | 2 +- .ci/scripts/tests/test_cu134_dependencies.py | 26 ++- .ci/scripts/wheel/envvar_base.sh | 2 +- .ci/scripts/wheel/envvar_cuda_linux.sh | 7 +- .ci/scripts/wheel/vc_env_helper.bat | 2 +- .githooks/README.md | 5 +- .githooks/pre-commit | 40 ++-- .github/workflows/lint.yml | 3 +- install_requirements.py | 21 +- scripts/release/prepare_release.py | 227 +++++++++++++------ scripts/release/release_versions.py | 36 +-- scripts/release/test_prepare_release.py | 64 +++++- setup.py | 2 +- 13 files changed, 305 insertions(+), 132 deletions(-) diff --git a/.ci/scripts/test_wheel_package_qnn.sh b/.ci/scripts/test_wheel_package_qnn.sh index 133ec885321..646d7c1f42c 100644 --- a/.ci/scripts/test_wheel_package_qnn.sh +++ b/.ci/scripts/test_wheel_package_qnn.sh @@ -98,7 +98,7 @@ install_qnn export LD_LIBRARY_PATH="${QNN_SDK_ROOT}/lib/x86_64-linux-clang/:${LD_LIBRARY_PATH:-}" install_executorch -EXECUTORCH_BUILDING_WHEEL=1 EXECUTORCH_WHEEL_VARIANT=cpu python setup.py bdist_wheel +EXECUTORCH_BUILDING_WHEEL=1 EXECUTORCH_RELEASE_WHEEL_METADATA=1 EXECUTORCH_WHEEL_VARIANT=cpu python setup.py bdist_wheel WHEEL_FILE=$(ls dist/*.whl | head -n 1) echo "Found wheel: $WHEEL_FILE" diff --git a/.ci/scripts/tests/test_cu134_dependencies.py b/.ci/scripts/tests/test_cu134_dependencies.py index 1b9f18455e4..df4fd79d9dc 100644 --- a/.ci/scripts/tests/test_cu134_dependencies.py +++ b/.ci/scripts/tests/test_cu134_dependencies.py @@ -96,6 +96,20 @@ def test_all_install_steps_preserve_exact_cu134_selection(self): "https://download.pytorch.org/whl/nightly/cpu", command ) + release_packages = [ + "torch==2.15.0+cu134", + "torchvision==0.30.0+cu134", + "torchaudio==2.12.0+cu134", + ] + with ( + patch.object(self.installer, "RELEASE_WHEEL", True), + patch.object(self.installer, "CU134_TORCH_PACKAGES", release_packages), + ): + commands = self.install_commands((13, 4)) + for command in commands: + self.assertIn("https://download.pytorch.org/whl/test/cu134", command) + self.assertTrue(set(release_packages).issubset(commands[2])) + def test_other_cuda_trains_keep_existing_pins(self): for cuda in ((12, 6), (13, 0), (13, 2)): for machine in ("x86_64", "aarch64"): @@ -150,7 +164,7 @@ def setup_requirement( self, function_name, *, - installed_torch="2.15.0rc1", + installed_torch="2.15.0", building_wheel=True, wheel_variant="cpu", ): @@ -177,7 +191,7 @@ def setup_requirement( "sys": sys, "install_utils": self.utils, "release_versions": self.release_versions, - "torch_pin": SimpleNamespace(RELEASE_WHEEL=True, TORCH_VERSION="2.15.0rc1"), + "torch_pin": SimpleNamespace(RELEASE_WHEEL=True, TORCH_VERSION="2.15.0"), } exec( compile(ast.Module(body=functions, type_ignores=[]), str(path), "exec"), @@ -185,7 +199,7 @@ def setup_requirement( ) environment = ( { - "EXECUTORCH_BUILDING_WHEEL": "1", + "EXECUTORCH_RELEASE_WHEEL_METADATA": "1", "EXECUTORCH_WHEEL_VARIANT": wheel_variant, } if building_wheel @@ -283,8 +297,8 @@ def test_cu134_keeps_explicit_torchao_source_build(self): def test_wheel_bounds_match_selected_train(self): for wheel_variant, installed_torch, expected_torch in ( - ("cu132", "2.15.0rc1+cu132", "torch==2.15.0rc1+cu132"), - ("cpu", "2.15.0rc1", "torch>=2.15.0rc1,<2.16"), + ("cu132", "2.15.0+cu132", "torch==2.15.0+cu132"), + ("cpu", "2.15.0", "torch>=2.15.0,<2.16"), ): with self.subTest(wheel_variant=wheel_variant): self.assertEqual( @@ -299,7 +313,7 @@ def test_wheel_bounds_match_selected_train(self): expected_torch, ) - with self.assertRaisesRegex(RuntimeError, "for Torch 2.15.0rc1"): + with self.assertRaisesRegex(RuntimeError, "for Torch 2.15.0"): self.release_torch_requirement( installed_torch="2.14.0.dev20260810+cu134", wheel_variant="cu134", diff --git a/.ci/scripts/wheel/envvar_base.sh b/.ci/scripts/wheel/envvar_base.sh index dd06905564d..ab3dad76ca9 100755 --- a/.ci/scripts/wheel/envvar_base.sh +++ b/.ci/scripts/wheel/envvar_base.sh @@ -15,5 +15,5 @@ export CMAKE_ARGS="${CMAKE_ARGS:-}" # setup.py only adds release dependency metadata for artifacts built by the # binary wheel jobs. A source install on a release branch must preserve the # PyTorch build already installed by that checkout's CI job. -export EXECUTORCH_BUILDING_WHEEL=1 +export EXECUTORCH_RELEASE_WHEEL_METADATA=1 export EXECUTORCH_WHEEL_VARIANT=cpu diff --git a/.ci/scripts/wheel/envvar_cuda_linux.sh b/.ci/scripts/wheel/envvar_cuda_linux.sh index 282c499226c..e041d2c5c32 100644 --- a/.ci/scripts/wheel/envvar_cuda_linux.sh +++ b/.ci/scripts/wheel/envvar_cuda_linux.sh @@ -11,7 +11,12 @@ source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh" _executorch_cuda_version="${CU_VERSION:-${DESIRED_CUDA:-}}" -export EXECUTORCH_WHEEL_VARIANT="cu${_executorch_cuda_version//[!0-9]/}" +_executorch_cuda_digits="${_executorch_cuda_version//[!0-9]/}" +if [[ ! "${_executorch_cuda_digits}" =~ ^[0-9]{3}$ ]]; then + echo "CU_VERSION or DESIRED_CUDA must identify a CUDA wheel variant, got '${_executorch_cuda_version}'" >&2 + exit 1 +fi +export EXECUTORCH_WHEEL_VARIANT="cu${_executorch_cuda_digits}" # Ask for the CUDA delegate explicitly rather than letting the build detect a toolkit. A detected # build is fine locally, but a release row states what it is producing, and a row that silently diff --git a/.ci/scripts/wheel/vc_env_helper.bat b/.ci/scripts/wheel/vc_env_helper.bat index 13c0436c2d5..5d0965df7f0 100644 --- a/.ci/scripts/wheel/vc_env_helper.bat +++ b/.ci/scripts/wheel/vc_env_helper.bat @@ -26,7 +26,7 @@ if "%VSDEVCMD_ARGS%" == "" ( if "%CU_VERSION%" == "xpu" call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" set DISTUTILS_USE_SDK=1 -set EXECUTORCH_BUILDING_WHEEL=1 +set EXECUTORCH_RELEASE_WHEEL_METADATA=1 set EXECUTORCH_WHEEL_VARIANT=cpu set args=%1 diff --git a/.githooks/README.md b/.githooks/README.md index 796274c2e48..59b34a9c80c 100644 --- a/.githooks/README.md +++ b/.githooks/README.md @@ -9,8 +9,9 @@ This directory contains Git hooks for the ExecuTorch repository. It is used as Runs on every commit: -1. **torch_pin sync** — when `torch_pin.py` is staged, updates the PyTorch commit - pin in `.ci/docker/ci_commit_pins/pytorch.txt` and syncs grafted c10 files. +1. **torch_pin sync** — when a development `torch_pin.py` is staged, updates the + PyTorch commit pin and grafted c10 files. Prepared release branches retain + the release commit selected by the branch-cut or finalization script. 2. **lintrunner** — runs `lintrunner -a --revision HEAD^ --skip MYPY` on changed files. Auto-fixes formatting and blocks on lint errors. Soft-fails if lintrunner is not installed. Runs `lintrunner init` automatically when `.lintrunner.toml` diff --git a/.githooks/pre-commit b/.githooks/pre-commit index da2aaf27ee2..2ccbb3062f7 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -5,25 +5,29 @@ # Check if torch_pin.py is being committed if git diff --cached --name-only | grep -q "^torch_pin.py$"; then echo "🔍 Detected changes to torch_pin.py" - echo "📝 Updating PyTorch commit pin and syncing c10 directories..." - - # Run the update script (which now also syncs c10 directories) - if python .github/scripts/update_pytorch_pin.py; then - # Stage any modified files (pytorch.txt and grafted c10 files) - if ! git diff --quiet .ci/docker/ci_commit_pins/pytorch.txt; then - git add .ci/docker/ci_commit_pins/pytorch.txt - echo "📌 Staged .ci/docker/ci_commit_pins/pytorch.txt" - fi - - # Stage any grafted c10 files - if ! git diff --quiet runtime/core/portable_type/c10/; then - git add runtime/core/portable_type/c10/ - echo "📌 Staged grafted c10 files" - fi + if git show :torch_pin.py | grep -q '^RELEASE_WHEEL = True$'; then + echo "📌 Keeping the PyTorch release commit selected by release preparation" else - echo "❌ Failed to update PyTorch commit pin" - echo "Please run: python .github/scripts/update_pytorch_pin.py" - exit 1 + echo "📝 Updating PyTorch commit pin and syncing c10 directories..." + + # Run the update script (which now also syncs c10 directories) + if python .github/scripts/update_pytorch_pin.py; then + # Stage any modified files (pytorch.txt and grafted c10 files) + if ! git diff --quiet .ci/docker/ci_commit_pins/pytorch.txt; then + git add .ci/docker/ci_commit_pins/pytorch.txt + echo "📌 Staged .ci/docker/ci_commit_pins/pytorch.txt" + fi + + # Stage any grafted c10 files + if ! git diff --quiet runtime/core/portable_type/c10/; then + git add runtime/core/portable_type/c10/ + echo "📌 Staged grafted c10 files" + fi + else + echo "❌ Failed to update PyTorch commit pin" + echo "Please run: python .github/scripts/update_pytorch_pin.py" + exit 1 + fi fi fi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6983539bf8b..7b6781faa6b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -51,12 +51,11 @@ jobs: TORCH_VERSION=$(python -c "from torch_pin import TORCH_VERSION; print(TORCH_VERSION)") TORCHVISION_VERSION=$(python -c "from torch_pin import TORCHVISION_VERSION; print(TORCHVISION_VERSION)") TORCHAUDIO_VERSION=$(python -c "from torch_pin import TORCHAUDIO_VERSION; print(TORCHAUDIO_VERSION)") - TORCH_INDEX=$(python -c "from torch_pin import RELEASE_WHEEL; print('https://download.pytorch.org/whl/test/cpu' if RELEASE_WHEEL else 'https://download.pytorch.org/whl/cpu')") pip install \ "torch==${TORCH_VERSION}" \ "torchvision==${TORCHVISION_VERSION}" \ "torchaudio==${TORCHAUDIO_VERSION}" \ - --index-url "${TORCH_INDEX}" + --index-url "https://download.pytorch.org/whl/test/cpu" pip install lintrunner==0.12.7 lintrunner-adapters==0.14.1 pip install -r requirements-lintrunner.txt USE_CPP=0 pip install --no-build-isolation third-party/ao diff --git a/install_requirements.py b/install_requirements.py index 009c67c7099..6f594c0aea1 100644 --- a/install_requirements.py +++ b/install_requirements.py @@ -18,6 +18,7 @@ TORCH_VERSION = _TORCH_PIN["TORCH_VERSION"] TORCHVISION_VERSION = _TORCH_PIN["TORCHVISION_VERSION"] TORCHAUDIO_VERSION = _TORCH_PIN["TORCHAUDIO_VERSION"] +RELEASE_WHEEL = _TORCH_PIN["RELEASE_WHEEL"] # The pip repository that hosts nightly torch packages. # This will be dynamically set based on CUDA availability and CUDA backend enabled/disabled. @@ -28,7 +29,8 @@ # Newest rocm7.2 TorchAO nightly with compiled kernels (cp310-abi3); later ROCm nightlies # are py3-none-any only, and none are published after 0.19.0.dev20260831. ROCM_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260805" -# These wheels' metadata pairs August 11 domain libraries with August 10 torch. +# cu134 needs explicit local-version pins. Release preparation replaces this +# development-nightly set with compatible wheels from the test index. CU134_TORCH_PACKAGES = [ "torch==2.14.0.dev20260810+cu134", "torchvision==0.29.0.dev20260811+cu134", @@ -86,9 +88,10 @@ def install_requirements(use_pytorch_nightly): torch_url = determine_torch_url(TORCH_URL_BASE) cu134_packages = cu134_requirements(torch_url) if cu134_packages: - torch_url = determine_torch_url(TORCHAO_URL_BASE) if not use_pytorch_nightly: cu134_packages[0] = "torch" + if not RELEASE_WHEEL: + torch_url = determine_torch_url(TORCHAO_URL_BASE) # torchao's CUDA channel publishes x86_64 only, so asking for a CUDA build makes the pin # unsatisfiable on aarch64. Only that case is special-cased: falling back everywhere would # change which torchao a CPU x86_64 install resolves, and the CUDA build is genuinely wanted @@ -182,11 +185,19 @@ def install_optional_example_requirements(use_pytorch_nightly): if use_pytorch_nightly else [] ) - if cu134_packages: + if cu134_packages and not RELEASE_WHEEL: torch_url = determine_torch_url(TORCHAO_URL_BASE) torchao_index = ( - ["--extra-index-url", f"{TORCHAO_URL_BASE}/cpu"] - if cu134_packages and platform.machine().lower() in ("aarch64", "arm64") + [ + "--extra-index-url", + ( + f"{TORCHAO_URL_BASE}/cpu" + if platform.machine().lower() in ("aarch64", "arm64") + else determine_torch_url(TORCHAO_URL_BASE) + ), + ] + if cu134_packages + and (RELEASE_WHEEL or platform.machine().lower() in ("aarch64", "arm64")) else [] ) diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index f902aff930f..219b3fc7f0e 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -8,6 +8,7 @@ """Apply and validate the deterministic changes required on release branches.""" import argparse +import html import re import runpy import subprocess @@ -15,13 +16,14 @@ import urllib.request from pathlib import Path from typing import Iterable, List -from urllib.parse import unquote +from urllib.parse import unquote, urljoin from release_versions import ( # type: ignore[import-not-found] + is_release_version, + release_base_version, release_key, torch_release_tag, torch_requirement, - torchvision_version, ) _RELEASE_VERSION_PATTERN = re.compile(r"\d+\.\d+") @@ -35,6 +37,9 @@ _TORCHAUDIO_VERSION_PATTERN = re.compile( r'^TORCHAUDIO_VERSION\s*=\s*"([^"]+)"$', re.MULTILINE ) +_CU134_TORCH_PACKAGES_PATTERN = re.compile( + r"^CU134_TORCH_PACKAGES = \[\n.*?^\]\n", re.MULTILINE | re.DOTALL +) _TEST_INFRA_MAIN_PATTERN = re.compile(r"(pytorch/test-infra/[^\s\"']+)@main\b") _TEST_INFRA_REF_MAIN_PATTERN = re.compile(r"(test-infra-ref:\s*)main\b") _TEST_INFRA_BRANCH_PATTERN = re.compile(r"pytorch/test-infra/[^\s\"'@]+@([^\s\"']+)") @@ -76,7 +81,7 @@ def enable_release_wheel(torch_pin_path: Path) -> bool: def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: """Set the Torch release pin used by installs and release wheel metadata.""" - if re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", torch_version) is None: + if not is_release_version(torch_version, allow_prerelease=False): raise RuntimeError(f"invalid Torch release version {torch_version!r}") text = torch_pin_path.read_text() updated, count = _TORCH_VERSION_PATTERN.subn( @@ -89,27 +94,54 @@ def set_torch_version(torch_pin_path: Path, torch_version: str) -> bool: return _write_if_changed(torch_pin_path, updated) -def set_companion_versions(torch_pin_path: Path, torch_version: str) -> bool: - """Move torchvision to the selected PyTorch release train.""" +def set_companion_versions( + torch_pin_path: Path, torchvision: str, torchaudio: str +) -> bool: + """Set domain-library releases that declare the selected Torch version.""" text = torch_pin_path.read_text() updated, vision_count = _TORCHVISION_VERSION_PATTERN.subn( - f'TORCHVISION_VERSION = "{torchvision_version(torch_version)}"', text + f'TORCHVISION_VERSION = "{torchvision}"', text + ) + updated, audio_count = _TORCHAUDIO_VERSION_PATTERN.subn( + f'TORCHAUDIO_VERSION = "{torchaudio}"', updated ) - if vision_count != 1 or _TORCHAUDIO_VERSION_PATTERN.search(updated) is None: + if vision_count != 1 or audio_count != 1: raise RuntimeError( f"expected exactly one torchvision and torchaudio assignment in {torch_pin_path}" ) return _write_if_changed(torch_pin_path, updated) +def set_cu134_versions( + install_requirements_path: Path, + torch_version: str, + torchvision: str, + torchaudio: str, +) -> bool: + """Move the exceptional cu134 package list to the selected release train.""" + text = install_requirements_path.read_text() + packages = ( + "CU134_TORCH_PACKAGES = [\n" + f' "torch=={torch_version}+cu134",\n' + f' "torchvision=={torchvision}+cu134",\n' + f' "torchaudio=={torchaudio}+cu134",\n' + "]\n" + ) + updated, count = _CU134_TORCH_PACKAGES_PATTERN.subn(packages, text) + if count != 1: + raise RuntimeError( + f"expected one CU134_TORCH_PACKAGES list in {install_requirements_path}" + ) + return _write_if_changed(install_requirements_path, updated) + + def newest_torch_test_release( available_versions: Iterable[str], newer_than: str = "" ) -> str: - """Return the newest RC/final on a newer PyTorch release train.""" + """Return the newest final-form wheel on a newer PyTorch release train.""" candidates = [] for version in available_versions: - match = re.fullmatch(r"(\d+)\.(\d+)\.(\d+)(?:(rc)(\d+))?", version) - if match is None: + if not is_release_version(version, allow_prerelease=False): continue key = release_key(version) if newer_than and key[:2] <= release_key(newer_than)[:2]: @@ -117,22 +149,54 @@ def newest_torch_test_release( candidates.append((key, version)) if not candidates: qualifier = f" newer than {newer_than}" if newer_than else "" - raise RuntimeError( - f"PyTorch test index has no release candidate or final release{qualifier}" - ) + raise RuntimeError(f"PyTorch test index has no release wheel{qualifier}") return max(candidates)[1] -def _test_index_versions(package: str) -> set[str]: - with urllib.request.urlopen( - f"https://download.pytorch.org/whl/test/cpu/{package}/", timeout=30 - ) as response: - index = unquote(response.read().decode()) - return set( - re.findall( - rf"{re.escape(package)}-(\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?)(?:\+[^-]+)?-", - index, +def _test_index_wheels(package: str, variant: str = "cpu") -> dict[str, str]: + index_url = ( + f"https://download.pytorch.org/whl/test/{variant}/{package}/" # @lint-ignore + ) + with urllib.request.urlopen(index_url, timeout=30) as response: + index = response.read().decode() + wheels: dict[str, str] = {} + for href in re.findall( + r']+data-(?:core|dist-info)-metadata=', index + ): + filename = unquote(html.unescape(href)).rsplit("/", 1)[-1] + match = re.match( + rf"{re.escape(package)}-([^-+]+)(?:\+[^-]+)?-", + filename, + ) + if match is None or not is_release_version(match.group(1)): + continue + version = match.group(1) + wheels.setdefault( + version, + urljoin(index_url, html.unescape(href).split("#", 1)[0]) + ".metadata", ) + return wheels + + +def _test_index_versions(package: str, variant: str = "cpu") -> set[str]: + return set(_test_index_wheels(package, variant)) + + +def companion_release_for_torch(package: str, torch_version: str) -> str: + """Find the newest domain-library wheel that requires this Torch release.""" + wheels = _test_index_wheels(package) + requirement = re.compile( + rf"^Requires-Dist:\s*torch\s*(?:\(\s*)?==\s*{re.escape(torch_version)}" + r"(?:\s*\))?(?:\s*;.*)?\s*$", + re.MULTILINE, + ) + for version in sorted(wheels, key=release_key, reverse=True): + with urllib.request.urlopen(wheels[version], timeout=30) as response: + metadata = response.read().decode() + if requirement.search(metadata): + return version + raise RuntimeError( + f"PyTorch test index has no {package} release requiring torch=={torch_version}" ) @@ -143,23 +207,24 @@ def latest_torch_test_release(newer_than: str) -> str: ) -def require_companion_releases(torch_pin_path: Path, torch_version: str) -> None: - """Fail before mutation unless matching domain-library releases exist.""" - config = runpy.run_path(str(torch_pin_path)) +def companion_releases_for_torch(torch_version: str) -> dict[str, str]: + """Resolve compatible CPU companions and require the cu134 release train.""" expected = { - "torchvision": torchvision_version(torch_version), - "torchaudio": config["TORCHAUDIO_VERSION"], + package: companion_release_for_torch(package, torch_version) + for package in ("torchvision", "torchaudio") } + expected["torch"] = torch_version missing = [ - f"{package}=={version}" + f"{package}=={version}+cu134" for package, version in expected.items() - if version not in _test_index_versions(package) + if version not in _test_index_versions(package, "cu134") ] if missing: raise RuntimeError( - "PyTorch companion releases are not available on the test index: " + "PyTorch cu134 releases are not available on the test index: " + ", ".join(missing) ) + return expected def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: @@ -167,6 +232,11 @@ def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: config = runpy.run_path(str(torch_pin_path)) current = config["TORCH_VERSION"] if override: + if not is_release_version(override, allow_prerelease=False): + raise RuntimeError( + "Torch wheel versions on the test index must use final X.Y.Z form, " + f"received {override!r}" + ) current_key = release_key(current) override_key = release_key(override) already_prepared = config.get("RELEASE_WHEEL") is True @@ -187,12 +257,8 @@ def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: def test_infra_branch_for_torch(torch_version: str) -> str: """Return the test-infra release branch matching a PyTorch release.""" - match = re.match(r"(\d+)\.(\d+)\.", torch_version) - if match is None: - raise RuntimeError( - f"could not derive a test-infra branch from Torch {torch_version!r}" - ) - return f"release/{match.group(1)}.{match.group(2)}" + major, minor, _patch, _stage, _stage_number = release_key(torch_version) + return f"release/{major}.{minor}" def require_test_infra_branch(test_infra_branch: str) -> None: @@ -216,16 +282,23 @@ def require_test_infra_branch(test_infra_branch: str) -> None: ) -def pytorch_commit_for_release(torch_version: str) -> str: - """Resolve the selected PyTorch release tag before changing the checkout.""" - tag = torch_release_tag(torch_version) +def pytorch_commit_for_release( + torch_version: str, release_candidate: bool = False +) -> str: + """Resolve the final tag or newest RC tag for a wheel release version.""" + base_version = release_base_version(torch_version) + tag = torch_release_tag(base_version) + reference_patterns = ( + [f"refs/tags/{tag}-rc*"] + if release_candidate + else [f"refs/tags/{tag}", f"refs/tags/{tag}^{{}}"] + ) result = subprocess.run( [ "git", "ls-remote", "https://github.com/pytorch/pytorch.git", - f"refs/tags/{tag}", - f"refs/tags/{tag}^{{}}", + *reference_patterns, ], check=True, capture_output=True, @@ -237,11 +310,21 @@ def pytorch_commit_for_release(torch_version: str) -> str: line.split("\t", 1) for line in result.stdout.splitlines() if "\t" in line ) } - commit = references.get(f"refs/tags/{tag}^{{}}") or references.get( - f"refs/tags/{tag}" - ) + if release_candidate: + candidates = [] + pattern = re.compile(rf"refs/tags/{re.escape(tag)}-rc(\d+)" + r"(\^\{\})?$") + for reference, commit in references.items(): + match = pattern.fullmatch(reference) + if match: + candidates.append((int(match.group(1)), bool(match.group(2)), commit)) + commit = max(candidates)[2] if candidates else None + else: + commit = references.get(f"refs/tags/{tag}^{{}}") or references.get( + f"refs/tags/{tag}" + ) if commit is None: - raise RuntimeError(f"PyTorch release tag {tag} does not exist") + qualifier = " release-candidate" if release_candidate else "" + raise RuntimeError(f"PyTorch{qualifier} tag for {tag} does not exist") return commit @@ -266,25 +349,13 @@ def validate_release_build(torch_pin_path: Path) -> str: raise RuntimeError(f"{torch_pin_path} does not enable release wheel metadata") version = config.get("TORCH_VERSION") - if ( - not isinstance(version, str) - or re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", version) is None - ): + if not is_release_version(version, allow_prerelease=False): raise RuntimeError(f"{torch_pin_path} has invalid TORCH_VERSION {version!r}") - expected_vision = torchvision_version(version) - if config.get("TORCHVISION_VERSION") != expected_vision: - raise RuntimeError( - f"{torch_pin_path} has TORCHVISION_VERSION=" - f"{config.get('TORCHVISION_VERSION')!r}, expected {expected_vision!r}" - ) - audio_version = config.get("TORCHAUDIO_VERSION") - if ( - not isinstance(audio_version, str) - or re.fullmatch(r"\d+\.\d+\.\d+(?:(?:a|b|rc)\d+)?", audio_version) is None - ): - raise RuntimeError( - f"{torch_pin_path} has invalid TORCHAUDIO_VERSION {audio_version!r}" - ) + for name in ("TORCHVISION_VERSION", "TORCHAUDIO_VERSION"): + if not is_release_version(config.get(name), allow_prerelease=False): + raise RuntimeError( + f"{torch_pin_path} has invalid {name} {config.get(name)!r}" + ) return torch_requirement(version, "cpu") @@ -421,6 +492,7 @@ def prepare_release( release_version: str, test_infra_branch: str, torch_version: str, + companions: dict[str, str], ) -> str: """Apply every deterministic branch-cut edit and return the Torch requirement.""" _validate_release_version(release_version) @@ -430,7 +502,17 @@ def prepare_release( workflow_paths = sorted((repo_root / ".github/workflows").glob("*.yml")) documentation = documentation_paths(repo_root) set_torch_version(repo_root / "torch_pin.py", torch_version) - set_companion_versions(repo_root / "torch_pin.py", torch_version) + set_companion_versions( + repo_root / "torch_pin.py", + companions["torchvision"], + companions["torchaudio"], + ) + set_cu134_versions( + repo_root / "install_requirements.py", + torch_version, + companions["torchvision"], + companions["torchaudio"], + ) enable_release_wheel(repo_root / "torch_pin.py") release_full_version = configure_release_version( repo_root / "version.txt", release_version @@ -478,7 +560,7 @@ def main() -> None: ) parser.add_argument( "--torch-version", - help="Torch RC version (defaults to the newest release on the test index)", + help="Torch wheel version (defaults to the newest release on the test index)", ) parser.add_argument( "--repo-root", type=Path, default=Path(__file__).resolve().parents[2] @@ -493,7 +575,6 @@ def main() -> None: if args.check: _validate_release_version(release_version) requirement = validate_release_build(repo_root / "torch_pin.py") - torch_version = runpy.run_path(str(repo_root / "torch_pin.py"))["TORCH_VERSION"] test_infra_branch = args.test_infra_branch or configured_test_infra_branch( workflow_paths ) @@ -510,13 +591,21 @@ def main() -> None: torch_version = torch_version_for_release( repo_root / "torch_pin.py", args.torch_version or "" ) - require_companion_releases(repo_root / "torch_pin.py", torch_version) + companions = companion_releases_for_torch(torch_version) test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( torch_version ) require_test_infra_branch(test_infra_branch) - pytorch_commit = pytorch_commit_for_release(torch_version) - prepare_release(repo_root, release_version, test_infra_branch, torch_version) + pytorch_commit = pytorch_commit_for_release( + torch_version, release_candidate=True + ) + prepare_release( + repo_root, + release_version, + test_infra_branch, + torch_version, + companions, + ) sync_pytorch_source(repo_root, pytorch_commit) diff --git a/scripts/release/release_versions.py b/scripts/release/release_versions.py index b4ebdc63cc7..982ea2692e4 100644 --- a/scripts/release/release_versions.py +++ b/scripts/release/release_versions.py @@ -11,29 +11,37 @@ _RELEASE_PATTERN = re.compile(r"(\d+)\.(\d+)\.(\d+)(?:(a|b|rc)(\d+))?") -def release_key(version: str) -> tuple[int, int, int, int, int]: +def release_parts(version: str) -> tuple[int, int, int, str | None, int]: match = _RELEASE_PATTERN.fullmatch(version) if match is None: raise RuntimeError(f"invalid release version {version!r}") - stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} major, minor, patch = (int(part) for part in match.groups()[:3]) - return major, minor, patch, stage_rank[match.group(4)], int(match.group(5) or 0) + return major, minor, patch, match.group(4), int(match.group(5) or 0) -def torchvision_version(torch_version: str) -> str: - match = _RELEASE_PATTERN.fullmatch(torch_version) - if match is None: - raise RuntimeError(f"invalid Torch release version {torch_version!r}") - _major, minor, patch, stage, stage_number = match.groups() - suffix = f"{stage}{stage_number}" if stage else "" - return f"0.{int(minor) + 15}.{patch}{suffix}" +def is_release_version(version: object, allow_prerelease: bool = True) -> bool: + if not isinstance(version, str): + return False + try: + _major, _minor, _patch, stage, _stage_number = release_parts(version) + except RuntimeError: + return False + return allow_prerelease or stage is None + + +def release_base_version(version: str) -> str: + major, minor, patch, _stage, _stage_number = release_parts(version) + return f"{major}.{minor}.{patch}" + + +def release_key(version: str) -> tuple[int, int, int, int, int]: + major, minor, patch, stage, stage_number = release_parts(version) + stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} + return major, minor, patch, stage_rank[stage], stage_number def torch_release_tag(torch_version: str) -> str: - match = _RELEASE_PATTERN.fullmatch(torch_version) - if match is None: - raise RuntimeError(f"invalid Torch release version {torch_version!r}") - major, minor, patch, stage, stage_number = match.groups() + major, minor, patch, stage, stage_number = release_parts(torch_version) suffix = f"-{stage}{stage_number}" if stage else "" return f"v{major}.{minor}.{patch}{suffix}" diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index 4042f75d2e2..6892151a6d1 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -7,11 +7,15 @@ import tempfile import unittest from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch from prepare_release import ( # type: ignore[import-not-found] + companion_release_for_torch, configured_test_infra_branch, newest_torch_test_release, prepare_release, + pytorch_commit_for_release, test_infra_branch_for_torch as _test_infra_branch_for_torch, torch_version_for_release, ) @@ -21,21 +25,45 @@ class PrepareReleaseTest(unittest.TestCase): """Tests for release-only branch preparation.""" def test_selects_release_candidate_once(self) -> None: - """A branch cut selects the newest RC and preserves it on reruns.""" - releases = ["2.14.1", "2.15.0a1", "2.15.0b2", "2.15.0rc1"] + """A branch cut selects the final-form RC wheel and preserves it.""" + releases = ["2.14.1", "2.15.0a1", "2.15.0rc1", "2.15.0"] self.assertEqual( newest_torch_test_release(releases, newer_than="2.14.0"), - "2.15.0rc1", + "2.15.0", ) with self.assertRaises(RuntimeError): newest_torch_test_release(["2.14.1"], newer_than="2.14.0") - self.assertEqual(_test_infra_branch_for_torch("2.15.0rc1"), "release/2.15") + self.assertEqual(_test_infra_branch_for_torch("2.15.0"), "release/2.15") with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "torch_pin.py" - path.write_text('TORCH_VERSION = "2.15.0rc1"\nRELEASE_WHEEL = True\n') + path.write_text('TORCH_VERSION = "2.15.0"\nRELEASE_WHEEL = True\n') - self.assertEqual(torch_version_for_release(path), "2.15.0rc1") + self.assertEqual(torch_version_for_release(path), "2.15.0") + + metadata = Path(directory) / "METADATA" + metadata.write_text("Requires-Dist: torch (==2.15.0)\r\n") + with patch( + "prepare_release._test_index_wheels", + return_value={"2.12.0": metadata.as_uri()}, + ): + self.assertEqual( + companion_release_for_torch("torchaudio", "2.15.0"), + "2.12.0", + ) + + tags = ( + f'{"1" * 40}\trefs/tags/v2.15.0-rc2\n' + f'{"2" * 40}\trefs/tags/v2.15.0-rc10\n' + ) + with patch( + "prepare_release.subprocess.run", + return_value=SimpleNamespace(stdout=tags), + ): + self.assertEqual( + pytorch_commit_for_release("2.15.0", release_candidate=True), + "2" * 40, + ) def test_prepares_complete_repository(self) -> None: """The top-level operation applies dependency, workflow, and docs edits.""" @@ -50,6 +78,13 @@ def test_prepares_complete_repository(self) -> None: 'TORCHAUDIO_VERSION = "2.11.0"\n' "RELEASE_WHEEL = False\n" ) + (root / "install_requirements.py").write_text( + "CU134_TORCH_PACKAGES = [\n" + ' "torch==2.14.0.dev20260810+cu134",\n' + ' "torchvision==0.29.0.dev20260811+cu134",\n' + ' "torchaudio==2.11.0.dev20260811+cu134",\n' + "]\n" + ) workflow = root / ".github/workflows/test.yml" workflow.write_text( "uses: pytorch/test-infra/.github/workflows/test.yml@main\n" @@ -63,23 +98,30 @@ def test_prepares_complete_repository(self) -> None: "stable: swiftpm-1.5.1\n" "nightly: swiftpm-1.6.0.20260929\n" ) - requirement = prepare_release(root, "1.6", "release/2.15", "2.15.0rc1") + companions = {"torchvision": "0.30.0", "torchaudio": "2.12.0"} + requirement = prepare_release( + root, "1.6", "release/2.15", "2.15.0", companions + ) (root / "version.txt").write_text("1.6.1\n") second_requirement = prepare_release( - root, "1.6", "release/2.15", "2.15.0rc1" + root, "1.6", "release/2.15", "2.15.0", companions ) - self.assertEqual(requirement, "torch>=2.15.0rc1,<2.16") + self.assertEqual(requirement, "torch>=2.15.0,<2.16") self.assertEqual(second_requirement, requirement) self.assertIn("RELEASE_WHEEL = True", (root / "torch_pin.py").read_text()) self.assertIn( - 'TORCHVISION_VERSION = "0.30.0rc1"', + 'TORCHVISION_VERSION = "0.30.0"', (root / "torch_pin.py").read_text(), ) self.assertIn( - 'TORCHAUDIO_VERSION = "2.11.0"', + 'TORCHAUDIO_VERSION = "2.12.0"', (root / "torch_pin.py").read_text(), ) + requirements = (root / "install_requirements.py").read_text() + self.assertIn('"torch==2.15.0+cu134"', requirements) + self.assertIn('"torchvision==0.30.0+cu134"', requirements) + self.assertIn('"torchaudio==2.12.0+cu134"', requirements) self.assertIn("@release/2.15", workflow.read_text()) self.assertIn("test-infra-ref: release/2.15", workflow.read_text()) self.assertEqual(configured_test_infra_branch([workflow]), "release/2.15") diff --git a/setup.py b/setup.py index c1f4017ea7f..8b604657289 100644 --- a/setup.py +++ b/setup.py @@ -1137,7 +1137,7 @@ def _torchao_requirement() -> str: def _release_torch_requirement() -> List[str]: if ( not torch_pin.RELEASE_WHEEL - or os.environ.get("EXECUTORCH_BUILDING_WHEEL") != "1" + or os.environ.get("EXECUTORCH_RELEASE_WHEEL_METADATA") != "1" ): return [] From 93bf590c74476914abee46f4a87904f5e21fb7f3 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 15:15:23 -0700 Subject: [PATCH 10/12] Harden release dependency selection --- .ci/scripts/tests/test_cu134_dependencies.py | 44 +- .ci/scripts/tests/test_filter_cuda_matrix.py | 23 +- .ci/scripts/wheel/cuda_arch_list.sh | 15 +- .ci/scripts/wheel/envvar_cuda_linux.sh | 9 +- .ci/scripts/wheel/test_clean_install.py | 7 - scripts/release/prepare_release.py | 554 +++++++++++++++---- scripts/release/release_versions.py | 5 - scripts/release/test_prepare_release.py | 74 ++- 8 files changed, 564 insertions(+), 167 deletions(-) diff --git a/.ci/scripts/tests/test_cu134_dependencies.py b/.ci/scripts/tests/test_cu134_dependencies.py index df4fd79d9dc..ccd1530094d 100644 --- a/.ci/scripts/tests/test_cu134_dependencies.py +++ b/.ci/scripts/tests/test_cu134_dependencies.py @@ -60,12 +60,10 @@ def test_all_install_steps_preserve_exact_cu134_selection(self): with self.subTest(machine=machine): commands = self.install_commands((13, 4), machine) self.assertEqual(len(commands), 4) - expected = { - "torch==2.14.0.dev20260810+cu134", - "torchvision==0.29.0.dev20260811+cu134", - "torchaudio==2.11.0.dev20260811+cu134", - f"torchao==0.19.0.dev20260907+{ao_variant}", - } + expected = set(self.installer.CU134_TORCH_PACKAGES) + expected.add( + f"torchao=={self.installer.CU134_TORCHAO_NIGHTLY_VERSION}+{ao_variant}" + ) for index, command in enumerate(commands): required = ( expected @@ -84,17 +82,15 @@ def test_all_install_steps_preserve_exact_cu134_selection(self): for arg in command ) ) - self.assertIn( - "https://download.pytorch.org/whl/nightly/cu134", command - ) - self.assertNotIn( - "https://download.pytorch.org/whl/test/cu134", command + expected_base = ( + self.installer.TORCH_URL_BASE + if self.installer.RELEASE_WHEEL + else self.installer.TORCHAO_URL_BASE ) + self.assertIn(f"{expected_base}/cu134", command) self.assertNotIn("--no-deps", command) if machine == "aarch64": - self.assertIn( - "https://download.pytorch.org/whl/nightly/cpu", command - ) + self.assertIn(f"{self.installer.TORCHAO_URL_BASE}/cpu", command) release_packages = [ "torch==2.15.0+cu134", @@ -118,9 +114,15 @@ def test_other_cuda_trains_keep_existing_pins(self): cuda, machine ) self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) - self.assertIn("torchao==0.19.0.dev20260907", core) - self.assertIn("torchvision==0.29.0", domains) - self.assertIn("torchaudio==2.11.0", domains) + self.assertIn( + f"torchao=={self.installer.TORCHAO_NIGHTLY_VERSION}", core + ) + self.assertIn( + f"torchvision=={self.installer.TORCHVISION_VERSION}", domains + ) + self.assertIn( + f"torchaudio=={self.installer.TORCHAUDIO_VERSION}", domains + ) self.assertFalse(any("==" in arg for arg in local)) self.assertFalse(any("==" in arg for arg in examples)) @@ -129,21 +131,21 @@ def test_source_pinned_torch_is_not_replaced(self): with self.subTest(cuda=cuda): core, _, domains, _ = self.install_commands(cuda, nightly=False) self.assertIn("torch", core) - self.assertNotIn("torch==2.14.0.dev20260810+cu134", core) + self.assertNotIn(self.installer.CU134_TORCH_PACKAGES[0], core) self.assertIn("torchvision", domains) self.assertIn("torchaudio", domains) def test_no_cuda_keeps_default_pins(self): core, _, domains, _ = self.install_commands(None) self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) - self.assertIn("torchao==0.19.0.dev20260907", core) - self.assertIn("torchvision==0.29.0", domains) + self.assertIn(f"torchao=={self.installer.TORCHAO_NIGHTLY_VERSION}", core) + self.assertIn(f"torchvision=={self.installer.TORCHVISION_VERSION}", domains) self.assertIn("https://download.pytorch.org/whl/test/cpu", core) def test_windows_does_not_select_cu134(self): core, _, domains, _ = self.install_commands((13, 4), system="Windows") self.assertIn(f"torch=={self.installer.TORCH_VERSION}", core) - self.assertIn("torchvision==0.29.0", domains) + self.assertIn(f"torchvision=={self.installer.TORCHVISION_VERSION}", domains) self.assertIn("https://download.pytorch.org/whl/test/cpu", core) def test_failure_is_not_retried_with_another_cuda_train(self): diff --git a/.ci/scripts/tests/test_filter_cuda_matrix.py b/.ci/scripts/tests/test_filter_cuda_matrix.py index 35a36cfe9df..2ab485b821a 100644 --- a/.ci/scripts/tests/test_filter_cuda_matrix.py +++ b/.ci/scripts/tests/test_filter_cuda_matrix.py @@ -16,6 +16,7 @@ import json import os import re +import runpy import subprocess import unittest from pathlib import Path @@ -37,6 +38,7 @@ def _load_module(name, path): "filter_cuda_matrix", ROOT / ".github" / "scripts" / "filter_cuda_matrix.py" ) INSTALL_UTILS = _load_module("install_utils", ROOT / "install_utils.py") +RELEASE_WHEEL = runpy.run_path(str(ROOT / "torch_pin.py"))["RELEASE_WHEEL"] def _full_matrix(): @@ -284,7 +286,12 @@ class TestPublishedSets(unittest.TestCase): """ def test_published_cuda_versions(self): - self.assertEqual(FILTER.SUPPORTED_CUDA_VERSIONS, ["cu130", "cu132", "cu134"]) + expected = {"cu130", "cu132", "cu134"} + if RELEASE_WHEEL: + self.assertTrue(FILTER.SUPPORTED_CUDA_VERSIONS) + self.assertLessEqual(set(FILTER.SUPPORTED_CUDA_VERSIONS), expected) + else: + self.assertEqual(set(FILTER.SUPPORTED_CUDA_VERSIONS), expected) def test_published_cuda_versions_are_documented(self): # The install table on the getting started page is the only place a user is told @@ -297,11 +304,15 @@ def test_published_cuda_versions_are_documented(self): self.assertIn(heading, text, f"no Installation section in {page.name}") section = text.split(heading, 1)[1].split("\n## ", 1)[0] documented = set(re.findall(r"cu\d+", section)) - self.assertEqual( - documented, - set(FILTER.SUPPORTED_CUDA_VERSIONS), - f"the install table in {page.name} lists {sorted(documented)}", - ) + published = set(FILTER.SUPPORTED_CUDA_VERSIONS) + if RELEASE_WHEEL: + self.assertLessEqual(published, documented) + else: + self.assertEqual( + documented, + published, + f"the install table in {page.name} lists {sorted(documented)}", + ) def test_published_cuda_versions_are_supported_by_the_installer(self): supported = { diff --git a/.ci/scripts/wheel/cuda_arch_list.sh b/.ci/scripts/wheel/cuda_arch_list.sh index 9e74af6c644..af83d299c98 100644 --- a/.ci/scripts/wheel/cuda_arch_list.sh +++ b/.ci/scripts/wheel/cuda_arch_list.sh @@ -73,6 +73,17 @@ _executorch_unknown_train() { return 64 } +# Normalize and validate the CUDA wheel variant once for every wheel script. +executorch_cuda_variant() { + local raw="${CU_VERSION:-${DESIRED_CUDA:-}}" + local digits="${raw//[!0-9]/}" + if [[ ! "${digits}" =~ ^[0-9]{3}$ ]]; then + echo "CU_VERSION or DESIRED_CUDA must identify a CUDA wheel variant, got '${raw}'" >&2 + return 64 + fi + printf 'cu%s' "${digits}" +} + # The architectures for the current row, space separated in the dotted form PyTorch expects. executorch_cuda_arch_list() { local machine @@ -97,8 +108,10 @@ executorch_cuda_arch_list() { ;; esac # The value arrives as cu130, while some callers pass 13.0 instead. + if ! train="$(executorch_cuda_variant)"; then + return 64 + fi train="${train#cu}" - train="${train//./}" case "${machine}" in aarch64 | arm64) diff --git a/.ci/scripts/wheel/envvar_cuda_linux.sh b/.ci/scripts/wheel/envvar_cuda_linux.sh index e041d2c5c32..65563d4a0db 100644 --- a/.ci/scripts/wheel/envvar_cuda_linux.sh +++ b/.ci/scripts/wheel/envvar_cuda_linux.sh @@ -9,14 +9,12 @@ # any variables so that subprocesses will see them. source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/envvar_base.sh" +source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/cuda_arch_list.sh" -_executorch_cuda_version="${CU_VERSION:-${DESIRED_CUDA:-}}" -_executorch_cuda_digits="${_executorch_cuda_version//[!0-9]/}" -if [[ ! "${_executorch_cuda_digits}" =~ ^[0-9]{3}$ ]]; then - echo "CU_VERSION or DESIRED_CUDA must identify a CUDA wheel variant, got '${_executorch_cuda_version}'" >&2 +if ! _executorch_cuda_variant="$(executorch_cuda_variant)"; then exit 1 fi -export EXECUTORCH_WHEEL_VARIANT="cu${_executorch_cuda_digits}" +export EXECUTORCH_WHEEL_VARIANT="${_executorch_cuda_variant}" # Ask for the CUDA delegate explicitly rather than letting the build detect a toolkit. A detected # build is fine locally, but a release row states what it is producing, and a row that silently @@ -39,7 +37,6 @@ fi # Compile device code for the GPUs this release row claims, rather than for whichever GPU the # builder happens to have. A wheel built by detection alone installs on every machine the row covers # and then fails when a model runs on a different generation. -source "${GITHUB_WORKSPACE}/${REPOSITORY}/.ci/scripts/wheel/cuda_arch_list.sh" # The status is checked rather than only the output, so an unrecognised row reports why it stopped. # A bare assignment would end the build on the lookup's own exit status with no message, since this # file is sourced into a shell that exits on a failing command. diff --git a/.ci/scripts/wheel/test_clean_install.py b/.ci/scripts/wheel/test_clean_install.py index f0107516a59..b9c721926fa 100644 --- a/.ci/scripts/wheel/test_clean_install.py +++ b/.ci/scripts/wheel/test_clean_install.py @@ -98,13 +98,6 @@ def test_release_pytorch_requirement() -> None: if specifier.operator == "==" ] configured_variant = os.environ.get("EXECUTORCH_WHEEL_VARIANT", "") - cuda_version = os.environ.get("CU_VERSION") or os.environ.get( - "DESIRED_CUDA", "" - ) - if not configured_variant and cuda_version: - configured_variant = "cu" + "".join( - character for character in cuda_version if character.isdigit() - ) if len(pins) == 1 and "+" in pins[0]: local_version = Version(pins[0]).local or "" variants = re.findall(r"(?:^|\.)(cu\d+)(?:\.|$)", local_version) diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index 219b3fc7f0e..6adaf735b15 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -9,20 +9,25 @@ import argparse import html +import io import re import runpy import subprocess import sys import urllib.request +import zipfile +from dataclasses import dataclass +from email.parser import Parser +from functools import lru_cache +from html.parser import HTMLParser from pathlib import Path from typing import Iterable, List +from urllib.error import HTTPError from urllib.parse import unquote, urljoin from release_versions import ( # type: ignore[import-not-found] is_release_version, - release_base_version, release_key, - torch_release_tag, torch_requirement, ) @@ -40,6 +45,16 @@ _CU134_TORCH_PACKAGES_PATTERN = re.compile( r"^CU134_TORCH_PACKAGES = \[\n.*?^\]\n", re.MULTILINE | re.DOTALL ) +_TORCHAO_URL_PATTERN = re.compile(r'^TORCHAO_URL_BASE\s*=\s*"[^"]+"$', re.MULTILINE) +_TORCHAO_VERSION_PATTERN = re.compile( + r'^TORCHAO_NIGHTLY_VERSION\s*=\s*"[^"]+"$', re.MULTILINE +) +_CU134_TORCHAO_VERSION_PATTERN = re.compile( + r'^CU134_TORCHAO_NIGHTLY_VERSION\s*=\s*"[^"]+"$', re.MULTILINE +) +_SUPPORTED_CUDA_PATTERN = re.compile( + r"^SUPPORTED_CUDA_VERSIONS: List\[str\] = \[[^\n]*\]$", re.MULTILINE +) _TEST_INFRA_MAIN_PATTERN = re.compile(r"(pytorch/test-infra/[^\s\"']+)@main\b") _TEST_INFRA_REF_MAIN_PATTERN = re.compile(r"(test-infra-ref:\s*)main\b") _TEST_INFRA_BRANCH_PATTERN = re.compile(r"pytorch/test-infra/[^\s\"'@]+@([^\s\"']+)") @@ -53,6 +68,103 @@ ) +@dataclass(frozen=True) +class WheelLink: + """One package-index wheel and its optional PEP 658 metadata.""" + + version: str + installed_version: str + url: str + metadata_url: str | None + + +class _WheelIndexParser(HTMLParser): + def __init__(self, index_url: str, package: str) -> None: + super().__init__() + self.index_url = index_url + self.package = package + self.wheels: dict[str, list[WheelLink]] = {} + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag != "a": + return + attributes = dict(attrs) + href = attributes.get("href") + if not href: + return + wheel_url = urljoin(self.index_url, html.unescape(href).split("#", 1)[0]) + filename = unquote(wheel_url).rsplit("/", 1)[-1] + match = re.match(rf"{re.escape(self.package)}-([^-]+)-.*\.whl$", filename) + if match is None: + return + installed_version = match.group(1) + version = installed_version.partition("+")[0] + if not is_release_version(version): + return + has_metadata = any( + name in attributes + for name in ("data-core-metadata", "data-dist-info-metadata") + ) + self.wheels.setdefault(version, []).append( + WheelLink( + version=version, + installed_version=installed_version, + url=wheel_url, + metadata_url=f"{wheel_url}.metadata" if has_metadata else None, + ) + ) + + +class _RemoteWheel(io.RawIOBase): + """Seekable range reader used to inspect a wheel without downloading it.""" + + def __init__(self, url: str) -> None: + self.url = url + self.position = 0 + request = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) + with urllib.request.urlopen(request, timeout=30) as response: + content_range = response.headers.get("Content-Range", "") + if "/" not in content_range: + raise RuntimeError(f"wheel server does not support byte ranges for {url}") + self.length = int(content_range.rsplit("/", 1)[1]) + + def readable(self) -> bool: + return True + + def seekable(self) -> bool: + return True + + def tell(self) -> int: + return self.position + + def seek(self, offset: int, whence: int = io.SEEK_SET) -> int: + if whence == io.SEEK_SET: + self.position = offset + elif whence == io.SEEK_CUR: + self.position += offset + elif whence == io.SEEK_END: + self.position = self.length + offset + else: + raise ValueError(f"invalid seek mode {whence}") + return self.position + + def read(self, size: int = -1) -> bytes: + if size == 0 or self.position >= self.length: + return b"" + end = ( + self.length - 1 + if size < 0 + else min(self.length - 1, self.position + size - 1) + ) + request = urllib.request.Request( + self.url, headers={"Range": f"bytes={self.position}-{end}"} + ) + with urllib.request.urlopen(request, timeout=30) as response: + data = response.read() + self.position += len(data) + return data + + def _validate_release_version(release_version: str) -> None: if _RELEASE_VERSION_PATTERN.fullmatch(release_version) is None: raise RuntimeError( @@ -135,6 +247,52 @@ def set_cu134_versions( return _write_if_changed(install_requirements_path, updated) +def set_release_torchao(install_requirements_path: Path, version: str) -> bool: + """Pin TorchAO to a retained test-index release instead of a nightly.""" + if not is_release_version(version, allow_prerelease=False): + raise RuntimeError(f"invalid TorchAO release version {version!r}") + text = install_requirements_path.read_text() + updated, url_count = _TORCHAO_URL_PATTERN.subn( + 'TORCHAO_URL_BASE = "https://download.pytorch.org/whl/test"', text + ) + updated, version_count = _TORCHAO_VERSION_PATTERN.subn( + f'TORCHAO_NIGHTLY_VERSION = "{version}"', updated + ) + updated, cu_count = _CU134_TORCHAO_VERSION_PATTERN.subn( + f'CU134_TORCHAO_NIGHTLY_VERSION = "{version}"', updated + ) + if (url_count, version_count, cu_count) != (1, 1, 1): + raise RuntimeError( + f"expected one TorchAO URL and two version pins in {install_requirements_path}" + ) + return _write_if_changed(install_requirements_path, updated) + + +def configured_cuda_variants(filter_path: Path) -> list[str]: + """Read the CUDA wheel trains selected by the release matrix filter.""" + match = _SUPPORTED_CUDA_PATTERN.search(filter_path.read_text()) + if match is None: + raise RuntimeError(f"could not read supported CUDA versions from {filter_path}") + return re.findall(r'"(cu\d+)"', match.group(0)) + + +def set_cuda_variants(filter_path: Path, variants: Iterable[str]) -> bool: + """Drop CUDA trains for which the selected upstream release has no wheels.""" + selected = list(variants) + if not selected: + raise RuntimeError("the selected PyTorch release has no supported CUDA trains") + if any(re.fullmatch(r"cu\d+", variant) is None for variant in selected): + raise RuntimeError(f"invalid CUDA release variants {selected!r}") + replacement = "SUPPORTED_CUDA_VERSIONS: List[str] = " + repr(selected).replace( + "'", '"' + ) + text = filter_path.read_text() + updated, count = _SUPPORTED_CUDA_PATTERN.subn(replacement, text) + if count != 1: + raise RuntimeError(f"expected one supported CUDA list in {filter_path}") + return _write_if_changed(filter_path, updated) + + def newest_torch_test_release( available_versions: Iterable[str], newer_than: str = "" ) -> str: @@ -153,80 +311,168 @@ def newest_torch_test_release( return max(candidates)[1] -def _test_index_wheels(package: str, variant: str = "cpu") -> dict[str, str]: +@lru_cache(maxsize=None) +def _test_index_wheels( + package: str, variant: str = "cpu" +) -> dict[str, list[WheelLink]]: index_url = ( f"https://download.pytorch.org/whl/test/{variant}/{package}/" # @lint-ignore ) - with urllib.request.urlopen(index_url, timeout=30) as response: - index = response.read().decode() - wheels: dict[str, str] = {} - for href in re.findall( - r']+data-(?:core|dist-info)-metadata=', index - ): - filename = unquote(html.unescape(href)).rsplit("/", 1)[-1] - match = re.match( - rf"{re.escape(package)}-([^-+]+)(?:\+[^-]+)?-", - filename, - ) - if match is None or not is_release_version(match.group(1)): - continue - version = match.group(1) - wheels.setdefault( - version, - urljoin(index_url, html.unescape(href).split("#", 1)[0]) + ".metadata", - ) - return wheels + try: + with urllib.request.urlopen(index_url, timeout=30) as response: + index = response.read().decode() + except HTTPError as error: + if error.code in (403, 404) and variant != "cpu": + return {} + raise + parser = _WheelIndexParser(index_url, package) + parser.feed(index) + return parser.wheels def _test_index_versions(package: str, variant: str = "cpu") -> set[str]: return set(_test_index_wheels(package, variant)) -def companion_release_for_torch(package: str, torch_version: str) -> str: - """Find the newest domain-library wheel that requires this Torch release.""" - wheels = _test_index_wheels(package) - requirement = re.compile( - rf"^Requires-Dist:\s*torch\s*(?:\(\s*)?==\s*{re.escape(torch_version)}" - r"(?:\s*\))?(?:\s*;.*)?\s*$", - re.MULTILINE, +@lru_cache(maxsize=None) +def _wheel_member(wheel_url: str, suffix: str) -> str: + with zipfile.ZipFile(_RemoteWheel(wheel_url)) as wheel: + matches = [name for name in wheel.namelist() if name.endswith(suffix)] + if len(matches) != 1: + raise RuntimeError( + f"expected one {suffix} in {wheel_url}, found {len(matches)}" + ) + return wheel.read(matches[0]).decode() + + +@lru_cache(maxsize=None) +def _wheel_metadata(wheel: WheelLink) -> str: + if wheel.metadata_url: + with urllib.request.urlopen(wheel.metadata_url, timeout=30) as response: + return response.read().decode() + return _wheel_member(wheel.url, ".dist-info/METADATA") + + +def _preferred_wheel(wheels: list[WheelLink]) -> WheelLink: + """Choose a common Linux wheel, with a deterministic fallback.""" + return min( + wheels, + key=lambda wheel: ( + "manylinux" not in wheel.url or "x86_64" not in wheel.url, + "cp310" not in wheel.url, + wheel.url, + ), ) + + +def companion_release_for_torch( + package: str, torch_version: str, torch_installed_version: str = "" +) -> str: + """Find the newest domain-library wheel compatible with this Torch build.""" + wheels = _test_index_wheels(package) + installed = torch_installed_version or torch_version for version in sorted(wheels, key=release_key, reverse=True): - with urllib.request.urlopen(wheels[version], timeout=30) as response: - metadata = response.read().decode() - if requirement.search(metadata): + if companion_release_supports_torch(package, version, installed): return version raise RuntimeError( - f"PyTorch test index has no {package} release requiring torch=={torch_version}" + f"PyTorch test index has no {package} release compatible with torch " + f"{installed}" ) -def latest_torch_test_release(newer_than: str) -> str: - """Look up the newest PyTorch release candidate on its CPU test index.""" - return newest_torch_test_release( - _test_index_versions("torch"), newer_than=newer_than - ) +def companion_release_supports_torch( + package: str, package_version: str, torch_installed_version: str +) -> bool: + """Whether any wheel for a companion release accepts the selected Torch build.""" + for wheel in _test_index_wheels(package).get(package_version, []): + metadata = Parser().parsestr(_wheel_metadata(wheel)) + for raw_requirement in metadata.get_all("Requires-Dist", []): + match = re.match( + r"\s*torch\s*(?:\(\s*([^)]*)\s*\)|([^;]*))?", + raw_requirement, + re.IGNORECASE, + ) + if match and _specifier_allows_version( + (match.group(1) or match.group(2) or "").strip(), + torch_installed_version, + ): + return True + return False + + +def _specifier_allows_version(specifier: str, installed_version: str) -> bool: + """Evaluate the release specifiers used by PyTorch companion wheels.""" + installed_public = installed_version.partition("+")[0] + installed_key = release_key(installed_public) + for clause in filter(None, (part.strip() for part in specifier.split(","))): + match = re.fullmatch(r"(===|==|!=|<=|>=|<|>|~=)\s*([^\s]+)", clause) + if match is None: + raise RuntimeError(f"unsupported Torch requirement {specifier!r}") + operator, wanted = match.groups() + wanted_public = wanted.partition("+")[0] + if re.fullmatch(r"\d+\.\d+", wanted_public): + wanted_public += ".0" + wanted_key = release_key(wanted_public) + equal = ( + installed_version == wanted + if "+" in wanted + else installed_public == wanted_public + ) + accepted = { + "==": equal, + "===": installed_version == wanted, + "!=": not equal, + "<=": installed_key <= wanted_key, + ">=": installed_key >= wanted_key, + "<": installed_key < wanted_key, + ">": installed_key > wanted_key, + }.get(operator) + if operator == "~=": + release = wanted_public.split(".") + upper = ( + (int(release[0]) + 1, 0) + if len(release) == 2 + else ( + int(release[0]), + int(release[1]) + 1, + ) + ) + accepted = installed_key >= wanted_key and installed_key[:2] < upper + if not accepted: + return False + return True def companion_releases_for_torch(torch_version: str) -> dict[str, str]: - """Resolve compatible CPU companions and require the cu134 release train.""" + """Resolve compatible CPU companions for the selected Torch wheel.""" + torch_wheels = _test_index_wheels("torch").get(torch_version, []) + if not torch_wheels: + raise RuntimeError(f"PyTorch test index does not contain torch {torch_version}") + torch_installed_version = _preferred_wheel(torch_wheels).installed_version expected = { - package: companion_release_for_torch(package, torch_version) + package: companion_release_for_torch( + package, torch_version, torch_installed_version + ) for package in ("torchvision", "torchaudio") } expected["torch"] = torch_version - missing = [ - f"{package}=={version}+cu134" - for package, version in expected.items() - if version not in _test_index_versions(package, "cu134") - ] - if missing: - raise RuntimeError( - "PyTorch cu134 releases are not available on the test index: " - + ", ".join(missing) - ) return expected +def available_cuda_variants( + candidates: Iterable[str], releases: dict[str, str] +) -> list[str]: + """Keep CUDA trains for which every selected PyTorch package exists.""" + return [ + variant + for variant in candidates + if all( + version in _test_index_versions(package, variant) + for package, version in releases.items() + ) + ] + + def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: """Choose a pin once, then preserve it on subsequent preparation runs.""" config = runpy.run_path(str(torch_pin_path)) @@ -252,7 +498,7 @@ def torch_version_for_release(torch_pin_path: Path, override: str = "") -> str: return override if config.get("RELEASE_WHEEL") is True: return current - return latest_torch_test_release(current) + return newest_torch_test_release(_test_index_versions("torch"), newer_than=current) def test_infra_branch_for_torch(torch_version: str) -> str: @@ -282,50 +528,25 @@ def require_test_infra_branch(test_infra_branch: str) -> None: ) -def pytorch_commit_for_release( - torch_version: str, release_candidate: bool = False -) -> str: - """Resolve the final tag or newest RC tag for a wheel release version.""" - base_version = release_base_version(torch_version) - tag = torch_release_tag(base_version) - reference_patterns = ( - [f"refs/tags/{tag}-rc*"] - if release_candidate - else [f"refs/tags/{tag}", f"refs/tags/{tag}^{{}}"] +def pytorch_commit_for_wheel(torch_version: str) -> str: + """Read the exact source commit embedded in the selected binary wheel.""" + wheels = _test_index_wheels("torch").get(torch_version, []) + if not wheels: + raise RuntimeError(f"PyTorch test index has no torch {torch_version} wheel") + version_module = _wheel_member(_preferred_wheel(wheels).url, "torch/version.py") + version_match = re.search( + r"^__version__\s*=\s*['\"]([^'\"]+)", version_module, re.MULTILINE ) - result = subprocess.run( - [ - "git", - "ls-remote", - "https://github.com/pytorch/pytorch.git", - *reference_patterns, - ], - check=True, - capture_output=True, - text=True, + commit_match = re.search( + r"^git_version\s*=\s*['\"]([0-9a-f]{40})", version_module, re.MULTILINE ) - references = { - reference: commit - for commit, reference in ( - line.split("\t", 1) for line in result.stdout.splitlines() if "\t" in line - ) - } - if release_candidate: - candidates = [] - pattern = re.compile(rf"refs/tags/{re.escape(tag)}-rc(\d+)" + r"(\^\{\})?$") - for reference, commit in references.items(): - match = pattern.fullmatch(reference) - if match: - candidates.append((int(match.group(1)), bool(match.group(2)), commit)) - commit = max(candidates)[2] if candidates else None - else: - commit = references.get(f"refs/tags/{tag}^{{}}") or references.get( - f"refs/tags/{tag}" + if version_match is None or commit_match is None: + raise RuntimeError("selected PyTorch wheel does not record its build commit") + if version_match.group(1).partition("+")[0] != torch_version: + raise RuntimeError( + f"selected wheel reports {version_match.group(1)}, expected {torch_version}" ) - if commit is None: - qualifier = " release-candidate" if release_candidate else "" - raise RuntimeError(f"PyTorch{qualifier} tag for {tag} does not exist") - return commit + return commit_match.group(1) def sync_pytorch_source(repo_root: Path, commit: str) -> None: @@ -342,20 +563,95 @@ def sync_pytorch_source(repo_root: Path, commit: str) -> None: ) -def validate_release_build(torch_pin_path: Path) -> str: +def validate_release_build( + torch_pin_path: Path, + install_requirements_path: Path | None = None, + cuda_filter_path: Path | None = None, + verify_index: bool = True, +) -> str: """Return the release PyTorch requirement or raise for invalid config.""" config = runpy.run_path(str(torch_pin_path)) if config.get("RELEASE_WHEEL") is not True: raise RuntimeError(f"{torch_pin_path} does not enable release wheel metadata") version = config.get("TORCH_VERSION") - if not is_release_version(version, allow_prerelease=False): + if not isinstance(version, str) or not is_release_version( + version, allow_prerelease=False + ): raise RuntimeError(f"{torch_pin_path} has invalid TORCH_VERSION {version!r}") for name in ("TORCHVISION_VERSION", "TORCHAUDIO_VERSION"): if not is_release_version(config.get(name), allow_prerelease=False): raise RuntimeError( f"{torch_pin_path} has invalid {name} {config.get(name)!r}" ) + if install_requirements_path is None or cuda_filter_path is None: + return torch_requirement(version, "cpu") + + releases = {"torch": version} + for package, name in ( + ("torchvision", "TORCHVISION_VERSION"), + ("torchaudio", "TORCHAUDIO_VERSION"), + ): + package_version = config[name] + releases[package] = package_version + + if verify_index: + torch_wheels = _test_index_wheels("torch").get(version, []) + if not torch_wheels: + raise RuntimeError(f"test index no longer contains torch {version}") + installed = _preferred_wheel(torch_wheels).installed_version + for package, name in ( + ("torchvision", "TORCHVISION_VERSION"), + ("torchaudio", "TORCHAUDIO_VERSION"), + ): + if not companion_release_supports_torch(package, config[name], installed): + raise RuntimeError( + f"{package} {config[name]} is not compatible with torch {installed}" + ) + + requirements_text = install_requirements_path.read_text() + url_match = _TORCHAO_URL_PATTERN.search(requirements_text) + torchao_match = _TORCHAO_VERSION_PATTERN.search(requirements_text) + if url_match is None or "/whl/test" not in url_match.group(0): + raise RuntimeError("release TorchAO must come from the retained test index") + if torchao_match is None: + raise RuntimeError("release TorchAO pin is missing") + torchao_version = re.search(r'"([^"]+)"', torchao_match.group(0)) + if torchao_version is None or not is_release_version( + torchao_version.group(1), allow_prerelease=False + ): + raise RuntimeError("release TorchAO must use a non-nightly version") + releases["torchao"] = torchao_version.group(1) + + variants = configured_cuda_variants(cuda_filter_path) + if verify_index: + missing = [ + f"{package}=={package_version}+{variant}" + for variant in variants + for package, package_version in releases.items() + if package_version not in _test_index_versions(package, variant) + ] + if missing: + raise RuntimeError( + "configured CUDA trains are missing release packages: " + + ", ".join(missing) + ) + if "cu134" in variants: + expected = { + f"{package}=={package_version}+cu134" + for package, package_version in releases.items() + if package != "torchao" + } + actual = set( + re.findall( + r'"(torch(?:vision|audio)?==[^"]+\+cu134)"', + requirements_text, + ) + ) + if actual != expected: + raise RuntimeError( + f"cu134 pins {sorted(actual)!r} do not match {sorted(expected)!r}" + ) return torch_requirement(version, "cpu") @@ -493,6 +789,9 @@ def prepare_release( test_infra_branch: str, torch_version: str, companions: dict[str, str], + torchao_version: str, + cuda_variants: list[str], + verify_index: bool = True, ) -> str: """Apply every deterministic branch-cut edit and return the Torch requirement.""" _validate_release_version(release_version) @@ -507,12 +806,17 @@ def prepare_release( companions["torchvision"], companions["torchaudio"], ) - set_cu134_versions( - repo_root / "install_requirements.py", - torch_version, - companions["torchvision"], - companions["torchaudio"], - ) + install_requirements_path = repo_root / "install_requirements.py" + if "cu134" in cuda_variants: + set_cu134_versions( + install_requirements_path, + torch_version, + companions["torchvision"], + companions["torchaudio"], + ) + set_release_torchao(install_requirements_path, torchao_version) + cuda_filter_path = repo_root / ".github/scripts/filter_cuda_matrix.py" + set_cuda_variants(cuda_filter_path, cuda_variants) enable_release_wheel(repo_root / "torch_pin.py") release_full_version = configure_release_version( repo_root / "version.txt", release_version @@ -529,7 +833,12 @@ def prepare_release( release_full_version=release_full_version, ) validate_release_files(repo_root, release_version) - requirement = validate_release_build(repo_root / "torch_pin.py") + requirement = validate_release_build( + repo_root / "torch_pin.py", + install_requirements_path, + cuda_filter_path, + verify_index=verify_index, + ) print( f"Prepared release/{release_version}: {requirement}; changed " f"{workflow_count} workflow files and {documentation_count} documentation files" @@ -574,7 +883,11 @@ def main() -> None: if args.check: _validate_release_version(release_version) - requirement = validate_release_build(repo_root / "torch_pin.py") + requirement = validate_release_build( + repo_root / "torch_pin.py", + repo_root / "install_requirements.py", + repo_root / ".github/scripts/filter_cuda_matrix.py", + ) test_infra_branch = args.test_infra_branch or configured_test_infra_branch( workflow_paths ) @@ -588,25 +901,52 @@ def main() -> None: ) print(f"Release configuration is valid: {requirement}") else: + current_config = runpy.run_path(str(repo_root / "torch_pin.py")) + already_prepared = ( + current_config.get("RELEASE_WHEEL") is True and not args.torch_version + ) torch_version = torch_version_for_release( repo_root / "torch_pin.py", args.torch_version or "" ) - companions = companion_releases_for_torch(torch_version) + cuda_filter_path = repo_root / ".github/scripts/filter_cuda_matrix.py" + if already_prepared: + companions = { + "torch": torch_version, + "torchvision": current_config["TORCHVISION_VERSION"], + "torchaudio": current_config["TORCHAUDIO_VERSION"], + } + requirements_text = (repo_root / "install_requirements.py").read_text() + torchao_match = _TORCHAO_VERSION_PATTERN.search(requirements_text) + if torchao_match is None: + raise RuntimeError("release TorchAO pin is missing") + torchao_version = torchao_match.group(0).split('"')[1] + cuda_variants = configured_cuda_variants(cuda_filter_path) + pytorch_commit = None + else: + companions = companion_releases_for_torch(torch_version) + torchao_version = newest_torch_test_release(_test_index_versions("torchao")) + releases = dict(companions) + releases["torchao"] = torchao_version + cuda_variants = available_cuda_variants( + configured_cuda_variants(cuda_filter_path), releases + ) + pytorch_commit = pytorch_commit_for_wheel(torch_version) test_infra_branch = args.test_infra_branch or test_infra_branch_for_torch( torch_version ) require_test_infra_branch(test_infra_branch) - pytorch_commit = pytorch_commit_for_release( - torch_version, release_candidate=True - ) prepare_release( repo_root, release_version, test_infra_branch, torch_version, companions, + torchao_version, + cuda_variants, + verify_index=not already_prepared, ) - sync_pytorch_source(repo_root, pytorch_commit) + if pytorch_commit is not None: + sync_pytorch_source(repo_root, pytorch_commit) if __name__ == "__main__": diff --git a/scripts/release/release_versions.py b/scripts/release/release_versions.py index 982ea2692e4..4d3c0e31071 100644 --- a/scripts/release/release_versions.py +++ b/scripts/release/release_versions.py @@ -29,11 +29,6 @@ def is_release_version(version: object, allow_prerelease: bool = True) -> bool: return allow_prerelease or stage is None -def release_base_version(version: str) -> str: - major, minor, patch, _stage, _stage_number = release_parts(version) - return f"{major}.{minor}.{patch}" - - def release_key(version: str) -> tuple[int, int, int, int, int]: major, minor, patch, stage, stage_number = release_parts(version) stage_rank = {"a": 0, "b": 1, "rc": 2, None: 3} diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index 6892151a6d1..e6ca2da4ab0 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -7,15 +7,16 @@ import tempfile import unittest from pathlib import Path -from types import SimpleNamespace from unittest.mock import patch from prepare_release import ( # type: ignore[import-not-found] + WheelLink, + _WheelIndexParser, companion_release_for_torch, configured_test_infra_branch, newest_torch_test_release, prepare_release, - pytorch_commit_for_release, + pytorch_commit_for_wheel, test_infra_branch_for_torch as _test_infra_branch_for_torch, torch_version_for_release, ) @@ -41,30 +42,51 @@ def test_selects_release_candidate_once(self) -> None: self.assertEqual(torch_version_for_release(path), "2.15.0") - metadata = Path(directory) / "METADATA" - metadata.write_text("Requires-Dist: torch (==2.15.0)\r\n") + wheel = WheelLink( + "2.12.0", "2.12.0", "https://example/torchaudio.whl", None + ) with patch( "prepare_release._test_index_wheels", - return_value={"2.12.0": metadata.as_uri()}, + return_value={"2.12.0": [wheel]}, + ), patch( + "prepare_release._wheel_metadata", + return_value="Requires-Dist: torch (>=2.15.0,<2.16)\r\n", ): self.assertEqual( - companion_release_for_torch("torchaudio", "2.15.0"), + companion_release_for_torch("torchaudio", "2.15.0", "2.15.0+cpu"), "2.12.0", ) - tags = ( - f'{"1" * 40}\trefs/tags/v2.15.0-rc2\n' - f'{"2" * 40}\trefs/tags/v2.15.0-rc10\n' + torch_wheel = WheelLink( + "2.15.0", + "2.15.0+cpu", + "https://example/torch.whl", + None, ) with patch( - "prepare_release.subprocess.run", - return_value=SimpleNamespace(stdout=tags), + "prepare_release._test_index_wheels", + return_value={"2.15.0": [torch_wheel]}, + ), patch( + "prepare_release._wheel_member", + return_value=( + "__version__ = '2.15.0+cpu'\n" f"git_version = '{'2' * 40}'\n" + ), ): self.assertEqual( - pytorch_commit_for_release("2.15.0", release_candidate=True), + pytorch_commit_for_wheel("2.15.0"), "2" * 40, ) + parser = _WheelIndexParser("https://example/simple/", "torch") + parser.feed( + 'a' + 'b' + ) + self.assertEqual(len(parser.wheels["2.15.0"]), 2) + self.assertIsNone(parser.wheels["2.15.0"][0].metadata_url) + self.assertIsNotNone(parser.wheels["2.15.0"][1].metadata_url) + def test_prepares_complete_repository(self) -> None: """The top-level operation applies dependency, workflow, and docs edits.""" with tempfile.TemporaryDirectory() as directory: @@ -79,12 +101,20 @@ def test_prepares_complete_repository(self) -> None: "RELEASE_WHEEL = False\n" ) (root / "install_requirements.py").write_text( + 'TORCHAO_URL_BASE = "https://download.pytorch.org/whl/nightly"\n' + 'TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' + 'CU134_TORCHAO_NIGHTLY_VERSION = "0.19.0.dev20260907"\n' "CU134_TORCH_PACKAGES = [\n" ' "torch==2.14.0.dev20260810+cu134",\n' ' "torchvision==0.29.0.dev20260811+cu134",\n' ' "torchaudio==2.11.0.dev20260811+cu134",\n' "]\n" ) + (root / ".github/scripts").mkdir(parents=True) + cuda_filter = root / ".github/scripts/filter_cuda_matrix.py" + cuda_filter.write_text( + 'SUPPORTED_CUDA_VERSIONS: List[str] = ["cu130", "cu134"]\n' + ) workflow = root / ".github/workflows/test.yml" workflow.write_text( "uses: pytorch/test-infra/.github/workflows/test.yml@main\n" @@ -100,11 +130,25 @@ def test_prepares_complete_repository(self) -> None: ) companions = {"torchvision": "0.30.0", "torchaudio": "2.12.0"} requirement = prepare_release( - root, "1.6", "release/2.15", "2.15.0", companions + root, + "1.6", + "release/2.15", + "2.15.0", + companions, + "0.20.0", + ["cu130", "cu134"], + verify_index=False, ) (root / "version.txt").write_text("1.6.1\n") second_requirement = prepare_release( - root, "1.6", "release/2.15", "2.15.0", companions + root, + "1.6", + "release/2.15", + "2.15.0", + companions, + "0.20.0", + ["cu130", "cu134"], + verify_index=False, ) self.assertEqual(requirement, "torch>=2.15.0,<2.16") @@ -122,6 +166,8 @@ def test_prepares_complete_repository(self) -> None: self.assertIn('"torch==2.15.0+cu134"', requirements) self.assertIn('"torchvision==0.30.0+cu134"', requirements) self.assertIn('"torchaudio==2.12.0+cu134"', requirements) + self.assertIn('TORCHAO_NIGHTLY_VERSION = "0.20.0"', requirements) + self.assertIn("/whl/test", requirements) self.assertIn("@release/2.15", workflow.read_text()) self.assertIn("test-infra-ref: release/2.15", workflow.read_text()) self.assertEqual(configured_test_infra_branch([workflow]), "release/2.15") From 23ec183ce4ad0a0e7429a681e7dd7a20722439a4 Mon Sep 17 00:00:00 2001 From: Jacob Szwejbka Date: Thu, 1 Oct 2026 15:18:22 -0700 Subject: [PATCH 11/12] Skip synthetic release test URLs in lint --- scripts/release/test_prepare_release.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index e6ca2da4ab0..b1d3cb6c393 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -43,7 +43,10 @@ def test_selects_release_candidate_once(self) -> None: self.assertEqual(torch_version_for_release(path), "2.15.0") wheel = WheelLink( - "2.12.0", "2.12.0", "https://example/torchaudio.whl", None + "2.12.0", + "2.12.0", + "https://example/torchaudio.whl", # @lint-ignore + None, ) with patch( "prepare_release._test_index_wheels", @@ -60,7 +63,7 @@ def test_selects_release_candidate_once(self) -> None: torch_wheel = WheelLink( "2.15.0", "2.15.0+cpu", - "https://example/torch.whl", + "https://example/torch.whl", # @lint-ignore None, ) with patch( @@ -77,7 +80,7 @@ def test_selects_release_candidate_once(self) -> None: "2" * 40, ) - parser = _WheelIndexParser("https://example/simple/", "torch") + parser = _WheelIndexParser("https://example/simple/", "torch") # @lint-ignore parser.feed( 'a' ' Date: Thu, 1 Oct 2026 15:21:28 -0700 Subject: [PATCH 12/12] Satisfy release preparation lint --- scripts/release/prepare_release.py | 100 ++++++++++++++---------- scripts/release/test_prepare_release.py | 2 +- 2 files changed, 58 insertions(+), 44 deletions(-) diff --git a/scripts/release/prepare_release.py b/scripts/release/prepare_release.py index 6adaf735b15..0567e4c4482 100644 --- a/scripts/release/prepare_release.py +++ b/scripts/release/prepare_release.py @@ -563,6 +563,58 @@ def sync_pytorch_source(repo_root: Path, commit: str) -> None: ) +def _release_torchao_version(install_requirements_path: Path) -> tuple[str, str]: + requirements_text = install_requirements_path.read_text() + url_match = _TORCHAO_URL_PATTERN.search(requirements_text) + version_match = _TORCHAO_VERSION_PATTERN.search(requirements_text) + if url_match is None or "/whl/test" not in url_match.group(0): + raise RuntimeError("release TorchAO must come from the retained test index") + if version_match is None: + raise RuntimeError("release TorchAO pin is missing") + torchao_version = version_match.group(0).split('"')[1] + if not is_release_version(torchao_version, allow_prerelease=False): + raise RuntimeError("release TorchAO must use a non-nightly version") + return torchao_version, requirements_text + + +def _validate_cuda_releases( + releases: dict[str, str], + requirements_text: str, + cuda_filter_path: Path, + verify_index: bool, +) -> None: + variants = configured_cuda_variants(cuda_filter_path) + if verify_index: + missing = [ + f"{package}=={package_version}+{variant}" + for variant in variants + for package, package_version in releases.items() + if package_version not in _test_index_versions(package, variant) + ] + if missing: + raise RuntimeError( + "configured CUDA trains are missing release packages: " + + ", ".join(missing) + ) + if "cu134" not in variants: + return + expected = { + f"{package}=={package_version}+cu134" + for package, package_version in releases.items() + if package != "torchao" + } + actual = set( + re.findall( + r'"(torch(?:vision|audio)?==[^"]+\+cu134)"', + requirements_text, + ) + ) + if actual != expected: + raise RuntimeError( + f"cu134 pins {sorted(actual)!r} do not match {sorted(expected)!r}" + ) + + def validate_release_build( torch_pin_path: Path, install_requirements_path: Path | None = None, @@ -609,49 +661,11 @@ def validate_release_build( f"{package} {config[name]} is not compatible with torch {installed}" ) - requirements_text = install_requirements_path.read_text() - url_match = _TORCHAO_URL_PATTERN.search(requirements_text) - torchao_match = _TORCHAO_VERSION_PATTERN.search(requirements_text) - if url_match is None or "/whl/test" not in url_match.group(0): - raise RuntimeError("release TorchAO must come from the retained test index") - if torchao_match is None: - raise RuntimeError("release TorchAO pin is missing") - torchao_version = re.search(r'"([^"]+)"', torchao_match.group(0)) - if torchao_version is None or not is_release_version( - torchao_version.group(1), allow_prerelease=False - ): - raise RuntimeError("release TorchAO must use a non-nightly version") - releases["torchao"] = torchao_version.group(1) - - variants = configured_cuda_variants(cuda_filter_path) - if verify_index: - missing = [ - f"{package}=={package_version}+{variant}" - for variant in variants - for package, package_version in releases.items() - if package_version not in _test_index_versions(package, variant) - ] - if missing: - raise RuntimeError( - "configured CUDA trains are missing release packages: " - + ", ".join(missing) - ) - if "cu134" in variants: - expected = { - f"{package}=={package_version}+cu134" - for package, package_version in releases.items() - if package != "torchao" - } - actual = set( - re.findall( - r'"(torch(?:vision|audio)?==[^"]+\+cu134)"', - requirements_text, - ) - ) - if actual != expected: - raise RuntimeError( - f"cu134 pins {sorted(actual)!r} do not match {sorted(expected)!r}" - ) + torchao_version, requirements_text = _release_torchao_version( + install_requirements_path + ) + releases["torchao"] = torchao_version + _validate_cuda_releases(releases, requirements_text, cuda_filter_path, verify_index) return torch_requirement(version, "cpu") diff --git a/scripts/release/test_prepare_release.py b/scripts/release/test_prepare_release.py index b1d3cb6c393..e117f0cf68b 100644 --- a/scripts/release/test_prepare_release.py +++ b/scripts/release/test_prepare_release.py @@ -10,7 +10,6 @@ from unittest.mock import patch from prepare_release import ( # type: ignore[import-not-found] - WheelLink, _WheelIndexParser, companion_release_for_torch, configured_test_infra_branch, @@ -19,6 +18,7 @@ pytorch_commit_for_wheel, test_infra_branch_for_torch as _test_infra_branch_for_torch, torch_version_for_release, + WheelLink, )