From 2bdf22ef77c98df1ba1b82d34644bc00b27570de Mon Sep 17 00:00:00 2001 From: Helge Klein Date: Mon, 26 Oct 2026 22:44:41 +0100 Subject: [PATCH 1/5] Implementation --- .github/dependabot.yml | 14 ++ .github/workflows/docker-image-validate.yml | 23 ++ Dockerfile | 14 +- backend/build-tools/requirements.txt | 2 + .../tests/test_bootstrap_build_pip_script.py | 197 ++++++++++++++++++ scripts/bootstrap-build-pip.py | 121 +++++++++++ .../docs-reports/docs-structure-report.html | 2 +- .../dependency-update-workflow/index.md | 31 +++ 8 files changed, 402 insertions(+), 2 deletions(-) create mode 100644 backend/build-tools/requirements.txt create mode 100644 backend/tests/test_bootstrap_build_pip_script.py create mode 100644 scripts/bootstrap-build-pip.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a988e8516..9bc15488d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -110,6 +110,20 @@ updates: - "ruff" - "smbprotocol" + - package-ecosystem: pip + directory: "/backend/build-tools" + commit-message: + prefix: "deps(build)" + schedule: + interval: weekly + day: monday + time: "05:15" + timezone: UTC + open-pull-requests-limit: 3 + labels: + - dependencies + - python + - package-ecosystem: npm directory: "/frontend" commit-message: diff --git a/.github/workflows/docker-image-validate.yml b/.github/workflows/docker-image-validate.yml index f3b90a719..8afdadfa7 100644 --- a/.github/workflows/docker-image-validate.yml +++ b/.github/workflows/docker-image-validate.yml @@ -37,6 +37,7 @@ on: - 'backend/**' - 'frontend/**' - 'scripts/install-system-deps' + - 'scripts/bootstrap-build-pip.py' pull_request: branches: [ main ] paths: @@ -70,6 +71,7 @@ on: - 'backend/**' - 'frontend/**' - 'scripts/install-system-deps' + - 'scripts/bootstrap-build-pip.py' workflow_dispatch: concurrency: @@ -217,6 +219,27 @@ jobs: test -x /app/scripts/reset-archive-v2-cutover-state ' + - name: Verify production packaging tools are absent + shell: bash + run: | + docker exec sambee-smoke-${{ matrix.platform_id }} python -c ' + import importlib.util + import pathlib + import sys + import sysconfig + + modules = ("pip", "ensurepip") + remaining = [name for name in modules if importlib.util.find_spec(name) is not None] + remaining.extend(str(path) for path in pathlib.Path(sysconfig.get_path("scripts")).glob("pip*")) + remaining.extend(str(path) for path in pathlib.Path(sysconfig.get_path("purelib")).glob("pip-*.dist-info")) + bootstrap = pathlib.Path(sysconfig.get_path("stdlib")) / "ensurepip" + if bootstrap.exists(): + remaining.append(str(bootstrap)) + if remaining: + sys.exit("Production packaging tools remain: " + ", ".join(remaining)) + print("Production pip, launchers, metadata, and ensurepip are absent.") + ' + - name: Report production runtime contract shell: bash run: | diff --git a/Dockerfile b/Dockerfile index 469558921..9c095865e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,10 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ UPGRADE_EXISTING_PACKAGES=1 bash /tmp/install-system-deps && \ rm /tmp/install-system-deps +COPY backend/build-tools/requirements.txt scripts/bootstrap-build-pip.py /tmp/build-pip/ +RUN python /tmp/build-pip/bootstrap-build-pip.py /tmp/build-pip/requirements.txt && \ + rm -rf /tmp/build-pip + # Development target: inherits the runtime contract, then adds editor and # companion tooling without upgrading shared runtime packages again. FROM runtime-base AS devcontainer @@ -108,7 +112,10 @@ FROM --platform=$BUILDPLATFORM ${PYTHON_BASE_IMAGE} AS pyvips-wheel-builder WORKDIR /tmp/pyvips-wheel-builder COPY backend/requirements.lock.txt ./ COPY scripts/extract-pyvips-wheel-requirement.py /usr/local/bin/ +COPY backend/build-tools/requirements.txt scripts/bootstrap-build-pip.py /tmp/build-pip/ RUN --mount=type=cache,target=/root/.cache/pip \ + python /tmp/build-pip/bootstrap-build-pip.py /tmp/build-pip/requirements.txt && \ + rm -rf /tmp/build-pip && \ python /usr/local/bin/extract-pyvips-wheel-requirement.py \ requirements.lock.txt /tmp/pyvips-wheel-requirement.txt && \ pip wheel --wheel-dir /tmp/wheels --require-hashes --no-deps \ @@ -128,13 +135,15 @@ RUN useradd --create-home --shell /bin/bash --uid 1000 vscode && \ chown -R vscode:vscode /home/vscode COPY backend/requirements-dev.lock.txt /tmp/requirements-dev.lock.txt COPY --from=pyvips-wheel-builder /tmp/wheels /tmp/wheels +COPY backend/build-tools/requirements.txt scripts/bootstrap-build-pip.py /tmp/build-pip/ RUN python -m venv /workspace/backend/.venv && \ + /workspace/backend/.venv/bin/python /tmp/build-pip/bootstrap-build-pip.py /tmp/build-pip/requirements.txt && \ /workspace/backend/.venv/bin/python -m pip install \ --require-hashes \ --find-links=/tmp/wheels \ -r /tmp/requirements-dev.lock.txt && \ chown vscode:vscode /workspace/backend && \ - rm -rf /tmp/wheels + rm -rf /tmp/wheels /tmp/build-pip COPY --chown=vscode:vscode archive-contract/ ./archive-contract/ COPY --chown=vscode:vscode companion/ ./companion/ COPY --chown=vscode:vscode backend/ ./backend/ @@ -161,6 +170,9 @@ COPY --from=pyvips-wheel-builder /tmp/wheels /tmp/wheels # Install Python dependencies before copying full backend (changes rarely - better layer caching) RUN --mount=type=cache,target=/root/.cache/pip \ pip install --root-user-action=ignore --disable-pip-version-check --require-hashes --find-links=/tmp/wheels -r requirements.lock.txt && \ + python -m pip check && \ + python -m pip --disable-pip-version-check uninstall --yes --root-user-action=ignore pip && \ + python -c 'import shutil, sysconfig; from pathlib import Path; shutil.rmtree(Path(sysconfig.get_path("stdlib")) / "ensurepip")' && \ rm -rf /tmp/wheels # The production image needs only the application package and startup guards. diff --git a/backend/build-tools/requirements.txt b/backend/build-tools/requirements.txt new file mode 100644 index 000000000..0a4be4b78 --- /dev/null +++ b/backend/build-tools/requirements.txt @@ -0,0 +1,2 @@ +pip==26.2.2 \ + --hash=sha256:fd7f4dfa8190e014013b9f08f3e973f4e4c26ffd179a204895533a7254813a70 diff --git a/backend/tests/test_bootstrap_build_pip_script.py b/backend/tests/test_bootstrap_build_pip_script.py new file mode 100644 index 000000000..c0e610619 --- /dev/null +++ b/backend/tests/test_bootstrap_build_pip_script.py @@ -0,0 +1,197 @@ +import hashlib +import importlib.util +import io +import json +import shlex +import subprocess +import sys +import sysconfig +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from unittest.mock import Mock + +import pytest +import yaml + +SCRIPT = Path(__file__).parents[2] / "scripts/bootstrap-build-pip.py" +SPEC = spec_from_file_location("bootstrap_build_pip", SCRIPT) +assert SPEC and SPEC.loader +MODULE = module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + +VERSION = "26.2.2" +WHEEL_NAME = f"pip-{VERSION}-py3-none-any.whl" +WHEEL_URL = f"https://files.pythonhosted.org/packages/{WHEEL_NAME}" +WHEEL_BYTES = b"verified wheel fixture" +WHEEL_HASH = hashlib.sha256(WHEEL_BYTES).hexdigest() + + +@pytest.fixture +def requirements(tmp_path: Path) -> Path: + path = tmp_path / "requirements.txt" + path.write_text(f"pip=={VERSION} \\\n --hash=sha256:{WHEEL_HASH}\n", encoding="utf-8") + return path + + +def metadata(**overrides: object) -> bytes: + wheel = {"filename": WHEEL_NAME, "packagetype": "bdist_wheel", "url": WHEEL_URL, "yanked": False} + wheel.update(overrides) + return json.dumps({"urls": [wheel]}).encode() + + +def test_reads_hash_locked_requirement(requirements: Path) -> None: + assert MODULE.read_requirement(requirements) == (VERSION, {WHEEL_HASH}) + + +def test_reads_comments_and_multiple_hashes(requirements: Path) -> None: + other_hash = "a" * 64 + requirements.write_text( + f"# Build installer\npip=={VERSION} \\\n --hash=sha256:{WHEEL_HASH} \\\n --hash=sha256:{other_hash}\n", + encoding="utf-8", + ) + assert MODULE.read_requirement(requirements) == (VERSION, {WHEEL_HASH, other_hash}) + + +@pytest.mark.parametrize("content", ["", "pip>=26.2.2", "pip==26.2.2", "pip==26.2.2 --hash=sha256:bad", "--index-url https://example.com"]) +def test_rejects_unsafe_requirements(requirements: Path, content: str) -> None: + requirements.write_text(content, encoding="utf-8") + with pytest.raises(ValueError, match="only pip=="): + MODULE.read_requirement(requirements) + + +def test_rejects_additional_packages(requirements: Path) -> None: + with requirements.open("a") as stream: + stream.write(f"other==1.0 --hash=sha256:{WHEEL_HASH}\n") + with pytest.raises(ValueError, match="only pip=="): + MODULE.read_requirement(requirements) + + +def test_executes_only_verified_wheel_offline(requirements: Path, monkeypatch: pytest.MonkeyPatch) -> None: + downloader = Mock(side_effect=[metadata(), WHEEL_BYTES]) + monkeypatch.setattr(MODULE, "download", downloader) + + def run(command: list[str], *, check: bool) -> None: + assert check + assert command[:4] == [sys.executable, "-I", "-c", MODULE.WHEEL_ENTRYPOINT] + wheel = Path(command[4]) + assert wheel.read_bytes() == WHEEL_BYTES + assert wheel.name == WHEEL_NAME + for flag in ("--no-index", "--no-deps", "--require-hashes", "--upgrade"): + assert flag in command + assert command[-2:] == ["-r", str(requirements.resolve())] + + runner = Mock(side_effect=run) + monkeypatch.setattr(MODULE.subprocess, "run", runner) + MODULE.bootstrap(requirements) + runner.assert_called_once() + assert not Path(runner.call_args.args[0][4]).exists() + + +def test_hash_mismatch_does_not_execute_wheel(requirements: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(MODULE, "download", Mock(side_effect=[metadata(), b"tampered wheel"])) + runner = Mock() + monkeypatch.setattr(MODULE.subprocess, "run", runner) + with pytest.raises(ValueError, match="SHA-256 mismatch"): + MODULE.bootstrap(requirements) + runner.assert_not_called() + + +@pytest.mark.parametrize("changes", [{"yanked": True}, {"filename": "unexpected.whl"}]) +def test_rejects_unusable_release(requirements: Path, monkeypatch: pytest.MonkeyPatch, changes: dict[str, object]) -> None: + monkeypatch.setattr(MODULE, "download", Mock(return_value=metadata(**changes))) + with pytest.raises(ValueError, match="non-yanked universal wheel"): + MODULE.bootstrap(requirements) + + +@pytest.mark.parametrize("url", ["http://files.pythonhosted.org/pip.whl", "https://example.com/pip.whl"]) +def test_rejects_untrusted_download_url(url: str) -> None: + with pytest.raises(ValueError, match="HTTPS download"): + MODULE.download(url, MODULE.WHEEL_HOST) + + +def test_download_uses_system_tls_and_timeout(monkeypatch: pytest.MonkeyPatch) -> None: + response = io.BytesIO(WHEEL_BYTES) + response.url = WHEEL_URL + opener = Mock(return_value=response) + context = object() + monkeypatch.setattr(MODULE, "urlopen", opener) + monkeypatch.setattr(MODULE.ssl, "create_default_context", Mock(return_value=context)) + assert MODULE.download(WHEEL_URL, MODULE.WHEEL_HOST) == WHEEL_BYTES + opener.assert_called_once_with(WHEEL_URL, context=context, timeout=MODULE.DOWNLOAD_TIMEOUT_SECONDS) + + +def test_rejects_redirect_to_untrusted_host(monkeypatch: pytest.MonkeyPatch) -> None: + response = io.BytesIO(WHEEL_BYTES) + response.url = "https://example.com/pip.whl" + monkeypatch.setattr(MODULE, "urlopen", Mock(return_value=response)) + with pytest.raises(ValueError, match="redirected outside"): + MODULE.download(WHEEL_URL, MODULE.WHEEL_HOST) + + +def test_rejects_oversized_download(monkeypatch: pytest.MonkeyPatch) -> None: + response = io.BytesIO(WHEEL_BYTES) + response.url = WHEEL_URL + monkeypatch.setattr(MODULE, "MAX_DOWNLOAD_BYTES", 1) + monkeypatch.setattr(MODULE, "urlopen", Mock(return_value=response)) + with pytest.raises(ValueError, match="exceeds"): + MODULE.download(WHEEL_URL, MODULE.WHEEL_HOST) + + +def test_cli_reports_invalid_requirement(tmp_path: Path) -> None: + requirements = tmp_path / "requirements.txt" + requirements.write_text("pip==26.2.2", encoding="utf-8") + result = subprocess.run([sys.executable, str(SCRIPT), str(requirements)], capture_output=True, text=True, check=False) + assert result.returncode == 1 + assert "Build pip bootstrap failed:" in result.stderr + + +def test_docker_build_upgrades_before_downloads_and_removes_runtime_pip() -> None: + dockerfile = (SCRIPT.parents[1] / "Dockerfile").read_text(encoding="utf-8") + runtime = dockerfile.split("AS runtime-base", 1)[1].split("FROM runtime-base AS devcontainer", 1)[0] + wheel_builder = dockerfile.split("AS pyvips-wheel-builder", 1)[1].split("FROM runtime-base AS backend-test", 1)[0] + backend_test = dockerfile.split("FROM runtime-base AS backend-test", 1)[1].split("FROM runtime-base AS production", 1)[0] + production = dockerfile.split("FROM runtime-base AS production", 1)[1] + assert "bootstrap-build-pip.py" in runtime + assert wheel_builder.index("python /tmp/build-pip/bootstrap-build-pip.py") < wheel_builder.index("pip wheel") + assert backend_test.index("/bin/python /tmp/build-pip/bootstrap-build-pip.py") < backend_test.index("/bin/python -m pip install") + assert production.index("pip install") < production.index("python -m pip check") < production.index("uninstall --yes") + assert 'shutil.rmtree(Path(sysconfig.get_path("stdlib")) / "ensurepip")' in production + assert production.index("uninstall --yes") < production.index("COPY --link backend/app") + + +def test_committed_build_requirement_is_valid() -> None: + version, hashes = MODULE.read_requirement(SCRIPT.parents[1] / "backend/build-tools/requirements.txt") + assert version + assert hashes + + +def test_dependabot_manages_build_requirements() -> None: + configuration = yaml.safe_load((SCRIPT.parents[1] / ".github/dependabot.yml").read_text(encoding="utf-8")) + assert any( + update["package-ecosystem"] == "pip" and update["directory"] == "/backend/build-tools" for update in configuration["updates"] + ) + + +@pytest.mark.parametrize("remaining", [None, "module", "launcher", "metadata", "ensurepip"]) +def test_production_ci_check_rejects_packaging_tools(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, remaining: str | None) -> None: + configuration = yaml.safe_load((SCRIPT.parents[1] / ".github/workflows/docker-image-validate.yml").read_text(encoding="utf-8")) + step = next( + step for step in configuration["jobs"]["validate"]["steps"] if step.get("name") == "Verify production packaging tools are absent" + ) + code = shlex.split(step["run"])[-1] + directories = {name: tmp_path / name for name in ("scripts", "purelib", "stdlib")} + for directory in directories.values(): + directory.mkdir() + monkeypatch.setattr(importlib.util, "find_spec", lambda name: object() if remaining == "module" and name == "pip" else None) + monkeypatch.setattr(sysconfig, "get_path", lambda name: str(directories[name])) + if remaining == "launcher": + (directories["scripts"] / "pip3").touch() + elif remaining == "metadata": + (directories["purelib"] / f"pip-{VERSION}.dist-info").mkdir() + elif remaining == "ensurepip": + (directories["stdlib"] / "ensurepip").mkdir() + if remaining is None: + exec(compile(code, "production-packaging-check", "exec"), {}) + else: + with pytest.raises(SystemExit, match="Production packaging tools remain"): + exec(compile(code, "production-packaging-check", "exec"), {}) diff --git a/scripts/bootstrap-build-pip.py b/scripts/bootstrap-build-pip.py new file mode 100644 index 000000000..43bc2b9e4 --- /dev/null +++ b/scripts/bootstrap-build-pip.py @@ -0,0 +1,121 @@ +"""Install hash-locked build pip without running the interpreter's bundled pip.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import ssl +import subprocess +import sys +from pathlib import Path +from tempfile import TemporaryDirectory +from urllib.error import URLError +from urllib.parse import urlsplit +from urllib.request import urlopen + +PYPI_API = "https://pypi.org/pypi/pip/{version}/json" +WHEEL_HOST = "files.pythonhosted.org" +DOWNLOAD_TIMEOUT_SECONDS = 60 +MAX_DOWNLOAD_BYTES = 8 * 1024 * 1024 +REQUIREMENT_PATTERN = re.compile( + r"pip==(?P[0-9]+(?:\.[0-9]+){1,2})(?P(?:\s+--hash=sha256:[0-9a-f]{64})+)" +) +HASH_PATTERN = re.compile(r"--hash=sha256:([0-9a-f]{64})") +WHEEL_ENTRYPOINT = "import runpy, sys; sys.path.insert(0, sys.argv.pop(1)); runpy.run_module('pip', run_name='__main__')" + + +def read_requirement(requirements: Path) -> tuple[str, set[str]]: + """Accept only one exact pip pin with SHA-256 hashes.""" + logical_lines = " ".join( + line.split("#", 1)[0].strip().removesuffix("\\") + for line in requirements.read_text(encoding="utf-8").splitlines() + ) + match = REQUIREMENT_PATTERN.fullmatch(logical_lines.strip()) + if match is None: + raise ValueError( + "Build requirements must contain only pip== and SHA-256 hashes." + ) + return match.group("version"), set(HASH_PATTERN.findall(match.group("hashes"))) + + +def download(url: str, expected_host: str) -> bytes: + """Use the system CA store and limit network response size.""" + parsed = urlsplit(url) + if parsed.scheme != "https" or parsed.netloc != expected_host: + raise ValueError(f"Expected an HTTPS download from {expected_host}.") + with urlopen( + url, context=ssl.create_default_context(), timeout=DOWNLOAD_TIMEOUT_SECONDS + ) as response: + final_url = urlsplit(response.url) + if final_url.scheme != "https" or final_url.netloc != expected_host: + raise ValueError(f"Download redirected outside {expected_host}.") + data = bytes(response.read(MAX_DOWNLOAD_BYTES + 1)) + if len(data) > MAX_DOWNLOAD_BYTES: + raise ValueError(f"Download exceeds {MAX_DOWNLOAD_BYTES} bytes.") + return data + + +def bootstrap(requirements: Path) -> None: + version, hashes = read_requirement(requirements) + filename = f"pip-{version}-py3-none-any.whl" + metadata = json.loads(download(PYPI_API.format(version=version), "pypi.org")) + wheels = [ + item + for item in metadata["urls"] + if item["filename"] == filename and item["packagetype"] == "bdist_wheel" + ] + if len(wheels) != 1 or wheels[0].get("yanked"): + raise ValueError(f"Expected one non-yanked universal wheel for pip {version}.") + wheel_data = download(wheels[0]["url"], WHEEL_HOST) + if hashlib.sha256(wheel_data).hexdigest() not in hashes: + raise ValueError( + f"SHA-256 mismatch for {filename}; check the build requirements before retrying." + ) + with TemporaryDirectory(prefix="sambee-build-pip-") as directory: + wheel = Path(directory) / filename + wheel.write_bytes(wheel_data) + subprocess.run( + [ + sys.executable, + "-I", + "-c", + WHEEL_ENTRYPOINT, + str(wheel), + "install", + "--upgrade", + "--no-index", + "--no-deps", + "--only-binary=:all:", + "--require-hashes", + "--disable-pip-version-check", + "--root-user-action=ignore", + "--find-links", + directory, + "-r", + str(requirements.resolve()), + ], + check=True, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("requirements", type=Path) + arguments = parser.parse_args() + try: + bootstrap(arguments.requirements) + except ( + OSError, + URLError, + ValueError, + KeyError, + TypeError, + subprocess.CalledProcessError, + ) as error: + parser.exit(1, f"Build pip bootstrap failed: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/website-meta/docs-reports/docs-structure-report.html b/website-meta/docs-reports/docs-structure-report.html index 2e4521085..6a4436391 100644 --- a/website-meta/docs-reports/docs-structure-report.html +++ b/website-meta/docs-reports/docs-structure-report.html @@ -268,7 +268,7 @@

Content diff

- + + +