Run stdlib tests via on-demand action

.github/workflows/stdlib.yml

@@ -0,0 +1,217 @@
+name: stdlib
+
+on:
+  workflow_dispatch:
+    inputs:
+      sha:
+        description: "The full SHA of a commit with completed build workflows."
+        required: true
+        type: string
+      filters:
+        description: "Optional CI filters, restricted to available artifacts (e.g., 'platform:linux,python:3.14,build:pgo')."
+        required: false
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}-${{ inputs.sha }}-${{ inputs.filters }}
+  cancel-in-progress: false
+
+env:
+  FORCE_COLOR: 1
+
+permissions:
+  actions: read
+  contents: read
+
+jobs:
+  generate-matrix:
+    name: Generate test matrix
+    runs-on: ubuntu-latest
+    outputs:
+      linux-matrix-0: ${{ steps.set-matrix.outputs.linux-matrix-0 }}
+      linux-matrix-1: ${{ steps.set-matrix.outputs.linux-matrix-1 }}
+      macos-matrix: ${{ steps.set-matrix.outputs.macos-matrix }}
+      windows-matrix: ${{ steps.set-matrix.outputs.windows-matrix }}
+      any-linux-0: ${{ steps.set-matrix.outputs.any-linux-0 }}
+      any-linux-1: ${{ steps.set-matrix.outputs.any-linux-1 }}
+      any-macos: ${{ steps.set-matrix.outputs.any-macos }}
+      any-windows: ${{ steps.set-matrix.outputs.any-windows }}
+      linux-run-id: ${{ steps.set-matrix.outputs.linux-run-id }}
+      macos-run-id: ${{ steps.set-matrix.outputs.macos-run-id }}
+      windows-run-id: ${{ steps.set-matrix.outputs.windows-run-id }}
+    steps:
+      # The helper may not exist at the commit whose artifacts we are testing.
+      - name: Check out workflow helper
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ github.workflow_sha }}
+          path: workflow-tools
+          sparse-checkout: scripts/stdlib_test_matrix.py
+          sparse-checkout-cone-mode: false
+          persist-credentials: false
+
+      - name: Check out target configuration
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.sha }}
+          path: target
+          persist-credentials: false
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: false
+
+      - name: Generate test matrix
+        id: set-matrix
+        working-directory: target
+        env:
+          GH_TOKEN: ${{ github.token }}
+          INPUT_SHA: ${{ inputs.sha }}
+          INPUT_FILTERS: ${{ inputs.filters }}
+        run: |
+          uv run ci-matrix.py --platform linux --event push --labels "${INPUT_FILTERS}" > linux.json
+          uv run ci-matrix.py --platform darwin --event push --labels "${INPUT_FILTERS}" > macos.json
+          uv run ci-matrix.py --platform windows --event push --labels "${INPUT_FILTERS}" > windows.json
+
+          uv run --script ../workflow-tools/scripts/stdlib_test_matrix.py \
+            --repository "${GITHUB_REPOSITORY}" \
+            --sha "${INPUT_SHA}" \
+            --matrix-dir . \
+            --github-output "${GITHUB_OUTPUT}"
+
+  linux-0:
+    name: linux / ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
+    if: ${{ needs.generate-matrix.outputs.any-linux-0 == 'true' }}
+    needs: generate-matrix
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.linux-matrix-0) }}
+    steps: &linux_steps
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.sha }}
+          persist-credentials: false
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: false
+
+      - name: Download distribution
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ needs.generate-matrix.outputs.linux-run-id }}
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}
+          path: dist
+
+      - name: Run stdlib tests
+        env:
+          JUNIT_FILENAME: junit-cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}.xml
+          MATRIX_LIBC: ${{ matrix.libc }}
+        run: |
+          if [ "${MATRIX_LIBC}" = "musl" ]; then
+            sudo apt install --yes musl-dev
+            unset LD_LIBRARY_PATH
+          fi
+
+          uv run --no-dev test-distribution.py --junit-xml "${JUNIT_FILENAME}" --stdlib dist/*.tar.zst
+
+      - name: Upload JUnit test results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: junit-cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}.xml
+
+  linux-1:
+    name: linux / ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
+    if: ${{ needs.generate-matrix.outputs.any-linux-1 == 'true' }}
+    needs: generate-matrix
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.linux-matrix-1) }}
+    steps: *linux_steps
+
+  macos:
+    name: macos / ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
+    if: ${{ needs.generate-matrix.outputs.any-macos == 'true' }}
+    needs: generate-matrix
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.macos-matrix) }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.sha }}
+          persist-credentials: false
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: false
+
+      - name: Download distribution
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ needs.generate-matrix.outputs.macos-run-id }}
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}
+          path: dist
+
+      - name: Run stdlib tests
+        env:
+          JUNIT_FILENAME: junit-cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}.xml
+        run: uv run --no-dev test-distribution.py --junit-xml "${JUNIT_FILENAME}" --stdlib dist/*.tar.zst
+
+      - name: Upload JUnit test results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: junit-cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}.xml
+
+  windows:
+    name: windows / ${{ matrix.target_triple }} / ${{ matrix.python }} / ${{ matrix.build_options }}
+    if: ${{ needs.generate-matrix.outputs.any-windows == 'true' }}
+    needs: generate-matrix
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.generate-matrix.outputs.windows-matrix) }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.sha }}
+          persist-credentials: false
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
+        with:
+          enable-cache: false
+
+      - name: Download distribution
+        uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+        with:
+          github-token: ${{ github.token }}
+          run-id: ${{ needs.generate-matrix.outputs.windows-run-id }}
+          name: cpython-${{ matrix.python }}-${{ matrix.vcvars }}-${{ matrix.build_options }}
+          path: dist
+
+      - name: Run stdlib tests
+        env:
+          JUNIT_FILENAME: junit-cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}.xml
+        run: |
+          $Dists = Resolve-Path -Path "dist/*.tar.zst" -Relative
+          uv run --no-dev test-distribution.py --junit-xml $env:JUNIT_FILENAME --stdlib $Dists
+
+      - name: Upload JUnit test results
+        if: ${{ always() }}
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}-junit
+          path: junit-cpython-${{ matrix.python }}-${{ matrix.target_triple }}-${{ matrix.build_options }}.xml

cpython-unix/build-cpython.sh

@@ -66,10 +66,14 @@ tar -xf "Python-${PYTHON_VERSION}.tar.xz"
 PIP_WHEEL="${ROOT}/pip-${PIP_VERSION}-py3-none-any.whl"
 SETUPTOOLS_WHEEL="${ROOT}/setuptools-${SETUPTOOLS_VERSION}-py3-none-any.whl"
 
-cat Setup.local
-mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
+# Put critical config files in logs to aid debugging.
+for f in Setup.local Makefile.extra stdlib-test-annotations.json; do
+  echo "BEGIN $f"
+  cat $f
+  echo "END $f"
+done
 
-cat Makefile.extra
+mv Setup.local "Python-${PYTHON_VERSION}/Modules/Setup.local"
 
 pushd "Python-${PYTHON_VERSION}"
 
@@ -1438,14 +1442,18 @@ cp -av Modules/config.c.in "${ROOT}/out/python/build/Modules/"
 cp -av Python/frozen.c "${ROOT}/out/python/build/Python/"
 cp -av Modules/Setup* "${ROOT}/out/python/build/Modules/"
 
-# Copy the test hardness runner for convenience.
+# Copy the test harness runner for convenience.
 # As of Python 3.13, the test harness runner has been removed so we provide a compatibility script
 if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" ]; then
     cp -av "${ROOT}/run_tests-13.py" "${ROOT}/out/python/build/run_tests.py"
 else
     cp -av Tools/scripts/run_tests.py "${ROOT}/out/python/build/"
 fi
 
+# Copy standard library test annotations so it is in the artifact and the
+# distribution self-describes expected test wonkiness.
+cp -av "${ROOT}/stdlib-test-annotations.json" "${ROOT}/out/python/build/stdlib-test-annotations.json"
+
 # Don't hard-code the build-time prefix into the pkg-config files. See
 # the description of `pcfiledir` in `man pkg-config`.
 find "${ROOT}/out/python/install/lib/pkgconfig" -name \*.pc -type f -exec \

cpython-unix/build.py

@@ -25,6 +25,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_setup_line,
+    stdlib_test_annotations,
 )
 from pythonbuild.docker import (
     build_docker_image,
@@ -58,6 +59,7 @@
 SUPPORT = ROOT / "cpython-unix"
 EXTENSION_MODULES = SUPPORT / "extension-modules.yml"
 TARGETS_CONFIG = SUPPORT / "targets.yml"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LINUX_ALLOW_SYSTEM_LIBRARIES = {
     "c",
@@ -771,6 +773,13 @@ def build_cpython(
     setuptools_archive = download_entry("setuptools", DOWNLOADS_PATH)
     pip_archive = download_entry("pip", DOWNLOADS_PATH)
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     ems = extension_modules_config(EXTENSION_MODULES)
 
     setup = derive_setup_local(
@@ -847,6 +856,14 @@ def build_cpython(
 
             build_env.copy_file(fh.name, dest_name="Makefile.extra")
 
+        # Install the derived test annotations.
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8") as fh:
+            os.chmod(fh.name, 0o644)
+            test_annotations.json_dump(fh)
+            fh.flush()
+
+            build_env.copy_file(fh.name, dest_name="stdlib-test-annotations.json")
+
         env = {
             "PIP_VERSION": DOWNLOADS["pip"]["version"],
             "PYTHON_VERSION": python_version,
@@ -1139,7 +1156,7 @@ def main():
             write_dockerfiles(SUPPORT, BUILD)
         elif action == "makefiles":
             targets = get_targets(TARGETS_CONFIG)
-            write_triples_makefiles(targets, BUILD, SUPPORT)
+            write_triples_makefiles(targets, ROOT, BUILD, SUPPORT)
             write_target_settings(targets, BUILD / "targets")
             write_package_versions(BUILD / "versions")
 

cpython-windows/build.py

@@ -22,6 +22,7 @@
     meets_python_maximum_version,
     meets_python_minimum_version,
     parse_config_c,
+    stdlib_test_annotations,
 )
 from pythonbuild.downloads import DOWNLOADS
 from pythonbuild.utils import (
@@ -39,6 +40,7 @@
 BUILD = ROOT / "build"
 DIST = ROOT / "dist"
 SUPPORT = ROOT / "cpython-windows"
+STDLIB_TEST_ANNOTATIONS = ROOT / "stdlib-test-annotations.yml"
 
 LOG_PREFIX = [None]
 LOG_FH = [None]
@@ -128,7 +130,6 @@
     "_zstd": ["zstd"],
 }
 
-
 # Tests to run during PGO profiling.
 #
 # This set was copied from test.libregrtest.pgo in the CPython source
@@ -1457,6 +1458,13 @@ def build_cpython(
     if freethreaded and meets_python_minimum_version(python_version, "3.15"):
         pcbuild_directory = f"{build_directory}t"
 
+    test_annotations = stdlib_test_annotations(
+        STDLIB_TEST_ANNOTATIONS,
+        python_version,
+        target_triple,
+        parsed_build_options,
+    )
+
     tempdir_opts = (
         {"ignore_cleanup_errors": True} if sys.version_info >= (3, 12) else {}
     )
@@ -1816,6 +1824,12 @@ def build_cpython(
                 out_dir / "python" / "build" / "run_tests.py",
             )
 
+        # Install a JSON file annotating tests.
+        with (out_dir / "python" / "build" / "stdlib-test-annotations.json").open(
+            "w", encoding="utf-8"
+        ) as fh:
+            test_annotations.json_dump(fh)
+
         licenses_dir = out_dir / "python" / "licenses"
         licenses_dir.mkdir()
         for f in sorted(os.listdir(ROOT)):

pyproject.toml

@@ -3,13 +3,14 @@ name = "python-build-standalone"
 version = "0.1.0"
 description = "Produces standalone, highly-redistributable builds of Python."
 readme = "README.rst"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dependencies = [
     "boto3>=1.37",
     "boto3-stubs[s3]>=1.42",
     "docker>=7.1.0",
     "jinja2>=3.1.5",
     "jsonschema>=4.23.0",
+    "junitparser>=4.0.2",
     "pyyaml>=6.0.2",
     "six>=1.17.0",
     "tomli>=2.2.1",