From a3fa75d7bfa2bc4c4ac1d6bc2770f27732c45f32 Mon Sep 17 00:00:00 2001 From: KuGouGo <62388728+KuGouGo@users.noreply.github.com> Date: Wed, 15 Jul 2026 15:51:29 +0800 Subject: [PATCH] refactor: harden artifact build and publication --- .github/workflows/build.yml | 1 + .github/workflows/pull-request.yml | 11 +- Makefile | 5 +- README.md | 2 +- docs/DEVELOPMENT.md | 21 +- docs/STRUCTURE.md | 12 +- docs/TROUBLESHOOTING.md | 12 +- .../commands/build-artifacts-transaction.sh | 238 ++++++++++++++++- scripts/commands/build-custom.sh | 171 ++++++++++-- scripts/commands/check-runtime.sh | 51 ++++ .../commands/generate-artifact-manifest.sh | 7 + scripts/commands/publish-branches.sh | 139 +++++++++- scripts/commands/restore-artifacts.sh | 45 +++- scripts/commands/select-build-scope.sh | 203 ++++++++++++++- scripts/commands/sync-upstream.sh | 16 +- scripts/commands/verify-artifact-manifest.sh | 7 + scripts/lib/rules.sh | 32 +++ scripts/tests/test-artifact-manifest.sh | 123 ++++++++- .../tests/test-artifact-transaction-atomic.sh | 149 ++++++++++- .../tests/test-binary-artifact-verifier.sh | 30 ++- scripts/tests/test-build-scope.sh | 115 +++++++- scripts/tests/test-custom-build-atomic.sh | 1 + scripts/tests/test-publish-branches.sh | 239 ++++++++++++++++- scripts/tests/test-runtime.sh | 37 ++- scripts/tests/test-shell-utils.sh | 17 ++ scripts/tools/artifact_manifest.py | 118 ++++++++- scripts/tools/artifact_verifier.py | 245 ++++++++++++++---- 27 files changed, 1860 insertions(+), 187 deletions(-) create mode 100755 scripts/commands/check-runtime.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 20eececc9..a3e11fc29 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -85,6 +85,7 @@ jobs: ARTIFACT_GENERATION_ID: ${{ github.run_id }}-${{ github.run_attempt }} ARTIFACT_BUILD_ID: ${{ github.run_id }} ARTIFACT_SOURCE_SHA: ${{ github.sha }} + ARTIFACT_BASELINE_FILE: ${{ steps.scope.outputs.baseline_file }} run: ./scripts/commands/build-artifacts-transaction.sh - name: Show summary diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 796aed622..37ee509ad 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -42,6 +42,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.11' + - name: Select release candidate scope id: scope env: @@ -53,12 +58,6 @@ jobs: - name: Show selected release candidate scope run: echo "${{ steps.scope.outputs.scope }} - ${{ steps.scope.outputs.reason }}" - - name: Set up Python - if: steps.scope.outputs.scope != 'none' - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: '3.11' - - name: Prepare scripts if: steps.scope.outputs.scope != 'none' run: find scripts -type f \( -name '*.sh' -o -name '*.py' \) -exec chmod +x {} + diff --git a/Makefile b/Makefile index 07ee45f18..c91954e68 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,5 @@ SHELL := /usr/bin/env bash REQUIRE_SHELLCHECK ?= 0 -BASH_MIN_MAJOR := 5 SHELL_SCRIPTS := $(shell find scripts -type f -name '*.sh' | sort) PYTHON_TOOLS := $(shell find scripts/tools -type f -name '*.py' | sort) @@ -9,7 +8,7 @@ PYTHON_TOOLS := $(shell find scripts/tools -type f -name '*.py' | sort) help: @echo "Available targets:" - @echo " make check-runtime Verify the supported Bash runtime" + @echo " make check-runtime Verify the supported Bash and Python runtimes" @echo " make lint Run shell, Python, and custom rule lint checks" @echo " make test Run all repository test scripts" @echo " make validate Run lint and tests" @@ -19,7 +18,7 @@ help: @echo " make clean Remove generated artifacts and temporary files" check-runtime: - @bash -c 'if (( BASH_VERSINFO[0] < $(BASH_MIN_MAJOR) )); then echo "Bash $(BASH_MIN_MAJOR)+ is required (found $$BASH_VERSION)" >&2; exit 1; fi' + @./scripts/commands/check-runtime.sh lint: check-runtime lint-shell lint-python lint-config lint-rules diff --git a/README.md b/README.md index 8a00b1a06..939c926e6 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/ip/google.mrs ## 本地维护 -本地命令要求 Bash 5+、GNU Make、Git 和 Python 3。macOS 可使用 Homebrew Bash 运行检查和文本构建;需要下载 sing-box 或 mihomo 的二进制构建只支持 lock 文件声明的 Linux 平台。 +本地命令要求 Bash 5+、Python 3.11+、GNU Make 和 Git。macOS 可使用 Homebrew Bash 与 Python 运行检查和文本构建;需要下载 sing-box 或 mihomo 的二进制构建只支持 lock 文件声明的 Linux 平台。 ```bash make check-runtime diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index eaf5009ec..f2c8548a8 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -4,7 +4,14 @@ ## 环境支持 -CI 的受支持基准是 GitHub Actions `ubuntu-latest`、Bash 5+ 和 Python 3.11。本地验证与文本构建支持安装了 Bash 5+、GNU Make、Git、Python 3、curl、tar、gzip、find 的 Linux、WSL 和 macOS;macOS 应使用 Homebrew Bash,并确保 `/opt/homebrew/bin` 或 `/usr/local/bin` 位于 `/bin` 之前。`make check-runtime` 会在构建前拒绝旧版 Bash。 +CI 的受支持基准是 GitHub Actions `ubuntu-latest`、Bash 5+ 和 Python 3.11。本地验证与文本构建支持安装了 Bash 5+、Python 3.11+、GNU Make、Git、curl、tar、gzip、find 的 Linux、WSL 和 macOS;macOS 应使用 Homebrew Bash 与 Python,并确保 `/opt/homebrew/bin` 或 `/usr/local/bin` 位于系统路径之前。`make check-runtime` 会在构建前拒绝旧版 Bash 或 Python。 + +Homebrew 的版本化 Python 将通用的 `python3` 链接放在 `libexec/bin`。macOS 可用以下配置验证当前终端,而无需为本地系统增加二进制规则编译支持: + +```bash +export PATH="$(brew --prefix python@3.11)/libexec/bin:$(brew --prefix)/bin:$PATH" +make check-runtime +``` 二进制工具下载逻辑仅支持 Linux 的 `amd64`、`arm64` 锁定资产。原生 Windows 在需要下载 sing-box/mihomo 时会被 `require_non_windows_shell` 明确拒绝;macOS 也没有对应 lock 平台。请使用 Linux、WSL 或 GitHub Actions 完成二进制构建。 @@ -19,22 +26,20 @@ make test make validate make build-custom-text make build-custom -ARTIFACT_GENERATION_ID=local-1 ARTIFACT_BUILD_SCOPE=full ./scripts/commands/generate-artifact-manifest.sh -./scripts/commands/verify-artifact-manifest.sh make preflight make clean ``` - `make validate`:Shell 语法、可用时的 ShellCheck、Python 编译、配置、自定义规则和测试。 -- `make check-runtime`:验证当前 `PATH` 解析到 Bash 5+。 +- `make check-runtime`:验证当前 `PATH` 解析到 Bash 5+ 和 Python 3.11+。 - `make build-custom-text`:只生成自定义文本产物,不下载二进制编译器。 - `make build-custom`:生成自定义文本和二进制产物。 - `make preflight`:`make validate` 加文本自定义构建;不执行完整同步、产物守卫(artifact guard)或发布。 -- `build-artifacts-transaction.sh`:CI 的完整入口。它在 `.tmp/` 中以事务自有的 `RULES_ARTIFACT_ROOT` 组合上游同步或已发布分支恢复、自定义构建、守卫、摘要、manifest 生成与验证;调用方提供 `RULES_ARTIFACT_ROOT` 会被拒绝,测试或运维如需改变最终提升位置应使用 `RULES_LIVE_ARTIFACT_ROOT`。全部成功后才以目录替换提升为 `.output/`(或该显式 live root)。失败诊断写入非发布目录 `.artifacts/diagnostics/`,旧 live root 保持不变。`config/upstreams.json` 为每个源声明 parser、required/optional、原始字节、规范条目、地址族和 fallback policy;required 源在主 URL 与允许的回退均失败,或出现语义健康回归时阻断提升。每个 RIPE Stat ASN 响应和合并分组都使用 `ripe-stat` health policy,过小或无效响应会先写入诊断摘要再阻断事务。自定义恢复要求五个发布分支都存在且具有相同 generation/source 身份,并把分支 commit 写入 manifest restoration metadata;缺失或身份分裂时失败关闭,应执行 full 构建恢复发布 cohort。 -- `generate-artifact-manifest.sh`:在构建与守卫完成后、写 manifest 前,按能力配置中的 `verifier` 分派器验证每个产物;缺失或未验证的二进制会阻断生成。默认位于 `.output/`,事务内服从 `RULES_ARTIFACT_ROOT`。调用方应明确提供 generation id、build scope,CI 还将 source SHA 绑定到实际 checkout 的 `github.sha`:PR 验证记录被测试的合并提交,正式发布记录 `main` 提交。 -- `verify-artifact-manifest.sh`:严格重算所选 artifact root 内的可发布文件集合、路径、大小和 SHA-256,并重新执行产物验证、核对能力/lock 与可选 source SHA;发布 job 在恢复或安装锁定工具后强制执行同一验证。 +- `build-artifacts-transaction.sh`:CI 的完整入口。它在 `.tmp/` 中以事务自有的 `RULES_ARTIFACT_ROOT` 组合上游同步或已发布分支恢复、自定义构建、守卫、摘要、manifest 生成与验证;调用方提供 `RULES_ARTIFACT_ROOT` 会被拒绝,测试或运维如需改变最终提升位置应使用 `RULES_LIVE_ARTIFACT_ROOT`。该 live root 必须与仓库 `.tmp/` 位于同一文件系统,跨设备目标会在构建前被拒绝;最终 backup、promotion 和 rollback 均使用不允许复制回退的严格目录 rename,因此检查后的设备变化也会以 EXDEV 失败。全部成功后才提升为 `.output/`(或该显式 live root)。backup 后收到 HUP、INT 或 TERM,以及 promotion rename 失败时,都会通过同一幂等回滚恢复旧目录;恢复本身失败时事务目录保留唯一备份以供人工处理。失败诊断写入非发布目录 `.artifacts/diagnostics/`,并记录 failure reason、promotion state、rollback status 与可用的 signal。`config/upstreams.json` 为每个源声明 parser、required/optional、原始字节、规范条目、地址族和 fallback policy;required 源在主 URL 与允许的回退均失败,或出现语义健康回归时阻断提升。每个 RIPE Stat ASN 响应和合并分组都使用 `ripe-stat` health policy,过小或无效响应会先写入诊断摘要再阻断事务。自定义恢复要求五个发布分支都存在且具有相同 generation/source 身份,并把分支 commit 写入 manifest restoration metadata;缺失或身份分裂时失败关闭,应执行 full 构建恢复发布 cohort。 +- `generate-artifact-manifest.sh`:完整构建事务的内部阶段,不是独立的日常构建入口。它要求当前 artifact root 已有 canonical 输入、摘要、来源记录和发布基线;按能力配置验证五个平台后才写入 schema v4 manifest。缺失或未验证的产物会阻断生成。CI 将 source SHA 绑定到实际 checkout 的 `github.sha`:PR 验证记录被测试的合并提交,正式发布记录 `main` 提交。 +- `verify-artifact-manifest.sh`:严格重算所选 artifact root 内的可发布文件集合、路径、大小和 SHA-256,并重新执行五平台 canonical 验证、核对发布基线、能力/lock 与可选 source SHA;发布 job 在恢复或安装锁定工具后强制执行同一验证。需要单独调试时,应先保留完整事务生成的 artifact root,不要手工拼装 manifest 参数。 -二进制验证使用固定工具的真实读回接口:`.srs` 执行 `sing-box rule-set decompile` 并解析 JSON;`.mrs` 执行 `mihomo convert-ruleset mrs INPUT OUTPUT`。读回结果与同名 custom 源或同一事务的 Egern/Surge 文本产物比较规范化语义集合:域名消除已被更宽后缀覆盖的冗余项,IP 合并为等价 CIDR 并集;值替换、范围扩大或范围丢失都会失败。manifest 记录验证方法、原始计数、读回语义 SHA-256,以及规范输入的语义 SHA-256。 +五平台验证以 `.output/.canonical/{domain,ip}/` 为共同基准。Surge、Quantumult X、Egern 解析各自文本/YAML,`.srs` 使用固定的 `sing-box rule-set decompile` 读回 JSON,`.mrs` 使用固定的 `mihomo convert-ruleset mrs INPUT OUTPUT` 读回。每个平台先按能力矩阵过滤不支持的类型,再比较规范化语义集合:域名消除已被更宽后缀覆盖的冗余项,IP 合并为等价 CIDR 并集;值替换、范围扩大、范围丢失、缺少副本和没有 canonical 来源的额外文件都会失败。manifest 记录验证方法、原始计数、读回语义 SHA-256,以及规范输入的语义 SHA-256。 - `make clean`:删除 `.tmp/`、`.output/`、`.artifacts/`、Python `__pycache__` 和未完成的 `.bin/*.new*`;保留已安装的 `.bin/sing-box`、`.bin/mihomo` 及 provenance sidecar。 CI 设置 `REQUIRE_SHELLCHECK=1`,本地缺少 ShellCheck 时的跳过不代表 CI 会通过。 diff --git a/docs/STRUCTURE.md b/docs/STRUCTURE.md index 742b1d036..040bd251f 100644 --- a/docs/STRUCTURE.md +++ b/docs/STRUCTURE.md @@ -14,13 +14,14 @@ | `scripts/tests/`、`tests/fixtures/` | 自动测试与稳定夹具 | | `templates/branch-readmes/` | 发布分支 README 模板及随发布树携带的 v2fly MIT 充分通知 | | `.output/` | 构建产物及部分审计摘要 | +| `.output/.canonical/` | 事务内保留的 domain/IP 规范编译输入,只用于跨平台语义校验,不进入发布分支 | | `.tmp/` | 可清理临时工作区 | | `.bin/` | 外部工具及版本缓存 | | `.artifacts/` | 失败构建保留的诊断摘要;属于可清理的本地/CI 数据,不进入发布分支 | ## 构建范围 -完整范围依次同步主上游、构建自定义规则、执行产物守卫(artifact guard),再上传并发布。自定义范围从五个发布分支恢复既有产物,然后重建自定义规则。工作流没有独立的 Fake-IP 同步步骤。 +完整范围依次同步主上游、写入 `.output/.canonical/`、构建自定义规则、执行产物守卫(artifact guard),再上传并发布。自定义范围从五个发布分支恢复同一发布 cohort;旧 cohort 没有 canonical 状态时先从 Egern 文本产物重建,再用本次自定义源覆盖相应规范输入并重建产物。工作流没有独立的 Fake-IP 同步步骤。 `fakeip-filter` 当前源为本仓库维护的 `sources/custom/domain/fakeip-filter.list`,由 `build-custom.sh` 与其他自定义规则一起生成五平台形式,不从网络下载预编译文件。`config/upstreams.json` 覆盖主上游网络输入;工具资产下载另由工具 lock 控制。 @@ -31,11 +32,14 @@ - `.output/upstream-summary.json`:主同步在健康检查后记录名称、实际 URL、状态、回退、临时路径、原始与规范化内容的字节数、条目数和 SHA-256;DLC 另含检出的 commit。 - `.output/domain/rule-manifest.json`:域名列表、区域集合和属性派生结构。 - `.output/build-summary.json`:GitHub Actions 在产物守卫之后、规范发布清单之前扫描 `.output/` 生成。 -- `.output/artifact-manifest.json`:规范发布清单,包含 schema/generation/build/source/build scope、能力与工具 lock 摘要、工具 provenance metadata、可用的上游/构建摘要,以及每个可发布 domain/ip 文件的平台、类型、扩展名、字节数、SHA-256 和可判定来源。JSON 键与产物按稳定顺序输出;generation/build id 由调用方提供,因此相同输入和 id 可复现相同内容。 +- `.output/.canonical/{domain,ip}/`:规范化、去重后的平台无关规则集合。它是五个平台语义验证的唯一比较基准;manifest 记录验证结果,但不会将该目录列为发布产物。 +- `.output/artifact-manifest.json`:schema v4 规范发布清单,包含 generation/build/source/build scope、完整发布基线(整体状态以及各分支 commit/generation/source)、能力与工具 lock 摘要、工具 provenance metadata、可用的上游/构建摘要,以及每个可发布 domain/ip 文件的平台、类型、扩展名、字节数、SHA-256 和可判定来源。JSON 键与产物按稳定顺序输出;generation/build id 由调用方提供,因此相同输入和 id 可复现相同内容。 - `.tmp/**/normalize-tasks.json`:批处理任务描述,属于临时数据。 - `.artifacts/diagnostics//`:失败事务保留的 `transaction-health.json` 及可用的构建/上游摘要;CI 日志只展示白名单内且大小受限的 JSON,完整诊断作为短期 Actions artifact 上传。 -`verify-artifact-manifest.sh` 严格重算能力矩阵允许的完整文件集合、路径层级与安全性、非零大小、字节数和 SHA-256,并核对能力/lock 摘要及可选的预期 source SHA。二进制读回规则与同名 custom 源或同一事务的文本产物比较等价语义集合:域名后缀覆盖和 CIDR 并集合并允许编译器消除冗余,但值替换、扩大或丢失范围会失败;清单同时记录语义 SHA-256。发布作业下载后再次验证;`publish-branches.sh` 自身也必须先验证清单,拒绝缺失、额外或被修改的产物。清单只作为流水线审计输入,不复制到发布分支。一次发布中五个分支提交携带共同 generation id 和 source SHA;任一平台 tree 改变时完整 cohort 原子推进并保留各分支父历史,全部 tree 不变时整体跳过。 +`verify-artifact-manifest.sh` 严格重算能力矩阵允许的完整文件集合、路径层级与安全性、非零大小、字节数和 SHA-256,并核对能力/lock 摘要及可选的预期 source SHA。Surge、Quantumult X、Egern、sing-box 和 Mihomo 都会按各自能力过滤规则类型,再与 `.output/.canonical/` 比较等价语义集合;域名后缀覆盖和 CIDR 并集合并允许编译器消除冗余,但值替换、扩大或丢失范围会失败。缺少任一应有平台副本、存在没有 canonical 来源的额外产物,或为只含该平台不支持规则的空集合生成文件,同样失败。清单同时记录读回方法、计数和语义 SHA-256。 + +发布作业下载后再次验证;`publish-branches.sh` 自身也必须先验证清单,拒绝缺失、额外、被修改或重放的产物。清单只作为流水线审计输入,不复制到发布分支。一次发布中五个分支提交携带共同 generation id 和 source SHA;任一平台 tree 改变时完整 cohort 原子推进并保留各分支父历史,全部 tree 不变时在重新读取远端 cohort 后整体跳过。自定义范围以最近一次一致发布 cohort 的 source 为累计比较基准,而不是只比较 `HEAD^`,因此连续的文档提交不会掩盖此前未发布的规则改动。候选 source/generation 不得早于远端基线;准备前、推送前和推送后均检查远端 `main`,五个产物 ref 使用带预期 SHA lease 的原子推送。Git 协议无法把未变化的 `main` ref 纳入同一次 compare-and-swap;若推送产物后发现 `main` 已前进,本次运行会明确失败,由当前 `main` 的排队运行继续推进。 `scripts/tools/artifact_origins.py` 是 `artifact-origins.json` 的唯一写入口:完整同步重置为 `generated-upstream`,发布分支恢复重置为 `restored-published-branch`,自定义构建只重标本次控制且实际存在的目标,并清除对应平台已经删除或降级省略的旧记录。 @@ -62,7 +66,7 @@ - 两个平台部分内置 IP 集的总数与 IPv4/IPv6 最低值; - Surge 上部分内置 IP 集相对基线的增长或删除检查。 -artifact guard 本身不解析 `.srs` / `.mrs`,二进制读回与精确语义关联由随后生成和复验 manifest 的阶段执行;两者都不审查许可。发布脚本另行检查发布树、扩展名和本地产物完整性。 +artifact guard 本身不执行五平台 canonical 语义比较;该检查由随后生成和复验 manifest 的阶段执行,其中 `.srs` / `.mrs` 使用锁定工具真实读回。两者都不审查许可。发布脚本另行检查发布树、扩展名和本地产物完整性。 ## 工具缓存与清理 diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 5f2d77a7e..5af7d6e9d 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -5,10 +5,16 @@ ## `make validate` 失败 - `shellcheck not found`:本地默认可跳过,CI 强制要求;安装后重试。 -- Python 编译失败:确认 Python 3 环境并修复首个语法错误。 +- Python 编译失败:确认 `python3 --version` 为 3.11 或更高版本,再修复首个语法错误。 - 配置失败:检查 HTTPS URL、正整数阈值、必需项和支持的枚举。 - 自定义规则失败:处理非 canonical 类型/值、跨文件或同文件的 domain 精确项/后缀覆盖、IP 重复/包含、正则或 CIDR 规范问题。若涉及豁免,只能在 `config/custom-rule-conflicts.json` 逐条记录真实关系;不要添加文件对级宽泛关系,失效和重复关系也会失败。 -- 测试失败:直接运行对应 `scripts/tests/test-*.sh`,再检查夹具是否应有意更新。 +- 测试失败:运行 `bash scripts/tests/test-*.sh` 中对应的脚本,再检查夹具是否应有意更新。 + +## `make check-runtime` 失败 + +本地命令要求 `PATH` 解析到 Bash 5+ 和 Python 3.11+。macOS 应优先使用 Homebrew 安装的版本;若 `python@3.11` 已安装但仍解析到系统 Python,把 `$(brew --prefix python@3.11)/libexec/bin` 放到 `PATH` 前部。分别运行 `bash --version`、`python3 --version` 和 `command -v bash python3`,确认没有落回系统旧版本。 + +若错误提示 `cross-device artifact promotion refused` 或 `cross-device rename refused`,应把 `RULES_LIVE_ARTIFACT_ROOT` 改到仓库所在文件系统。事务的预检与最终严格 rename 都不会通过跨设备复制模拟原子目录提升。若日志提示 `transaction recovery data preserved`,不要删除所示目录;其中的 `previous-output` 是自动恢复失败后保留的旧 live tree。 ## 原生 Windows 构建失败 @@ -55,7 +61,7 @@ CI 的失败事务会把 `.artifacts/diagnostics/` 上传为保留 7 天的 diag ## 产物守卫(artifact guard)阻断 -检查最低文件数、冗余派生名、文本域名下降、Surge/Quantumult X 文本 IP 合法性和部分内置 IP 基线。随后 manifest 阶段会真实读回二进制并精确比较可关联的规范规则集合。只有在来源证据、测试和评审说明齐全时才调整阈值。 +检查最低文件数、冗余派生名、文本域名下降、Surge/Quantumult X 文本 IP 合法性和部分内置 IP 基线。随后 manifest 阶段会解析或真实读回五个平台产物,并与 `.output/.canonical/` 中的平台无关规范规则集合精确比较。只有在来源证据、测试和评审说明齐全时才调整阈值。 ## 许可状态不明 diff --git a/scripts/commands/build-artifacts-transaction.sh b/scripts/commands/build-artifacts-transaction.sh index 628f71b1c..f4efba244 100755 --- a/scripts/commands/build-artifacts-transaction.sh +++ b/scripts/commands/build-artifacts-transaction.sh @@ -4,6 +4,9 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" cd "$ROOT" +# shellcheck source=/dev/null +source "$ROOT/scripts/commands/check-runtime.sh" + if [ -n "${RULES_ARTIFACT_ROOT:-}" ]; then echo "RULES_ARTIFACT_ROOT is transaction-owned; use RULES_LIVE_ARTIFACT_ROOT to override the promotion destination" >&2 exit 2 @@ -13,28 +16,231 @@ SCOPE="${RULES_BUILD_SCOPE:-full}" LIVE_ROOT="${RULES_LIVE_ARTIFACT_ROOT:-$ROOT/.output}" TMP_PARENT="$ROOT/.tmp" DIAGNOSTICS_ROOT="${RULES_ARTIFACT_DIAGNOSTICS_ROOT:-$ROOT/.artifacts/diagnostics}" -mkdir -p "$TMP_PARENT" +LIVE_PARENT="$(dirname "$LIVE_ROOT")" +mkdir -p "$TMP_PARENT" "$LIVE_PARENT" TRANSACTION_ROOT="$(mktemp -d "$TMP_PARENT/artifacts.XXXXXX")" STAGE_ROOT="$TRANSACTION_ROOT/output" BACKUP_ROOT="$TRANSACTION_ROOT/previous-output" +FAILED_ROOT="$TRANSACTION_ROOT/failed-output" +PRESERVE_TRANSACTION_ROOT=0 +HAD_LIVE_ROOT=0 +PROMOTION_STATE=building +ROLLBACK_STATUS=not-required +TRANSACTION_FAILURE_REASON=command-failure +TRANSACTION_SIGNAL="" mkdir -p "$STAGE_ROOT" +path_exists() { + [ -e "$1" ] || [ -L "$1" ] +} + +strict_rename() { + local operation="$1" + local source="$2" + local target="$3" + + python3 - "$operation" "$source" "$target" <<'PY' +import errno +import os +import sys + +operation, source, target = sys.argv[1:] +try: + for specification in filter( + None, os.environ.get("RULES_TRANSACTION_TEST_RENAME_FAILURES", "").split(",") + ): + name, separator, error_name = specification.partition(":") + if name != operation: + continue + error_name = error_name if separator else "EIO" + error_number = getattr(errno, error_name, None) + if not isinstance(error_number, int): + raise ValueError(f"unsupported injected errno: {error_name}") + raise OSError(error_number, f"injected {error_name}") + + if os.path.lexists(target): + raise FileExistsError(errno.EEXIST, "rename target already exists", target) + os.replace(source, target) +except (OSError, ValueError) as exc: + if isinstance(exc, OSError) and exc.errno == errno.EXDEV: + category = "cross-device rename refused" + else: + category = "strict rename refused" + print( + f"artifact transaction {operation} rename failed ({category}): " + f"{source} -> {target}: {exc}", + file=sys.stderr, + ) + raise SystemExit(1) +PY +} + +assert_promotion_filesystem() { + python3 - "$TRANSACTION_ROOT" "$LIVE_PARENT" "$LIVE_ROOT" <<'PY' +import os +import sys + +transaction_root, live_parent, live_root = map(os.path.abspath, sys.argv[1:]) +transaction_device = os.stat(transaction_root).st_dev +parent_device = os.stat(live_parent).st_dev + +if os.environ.get("RULES_TRANSACTION_TEST_DEVICE_MISMATCH") == "1": + parent_device = transaction_device + 1 + +if transaction_device != parent_device: + raise SystemExit( + "cross-device artifact promotion refused before build: " + f"staging device {transaction_device}, live parent device {parent_device}" + ) + +if os.path.lexists(live_root): + live_device = os.lstat(live_root).st_dev + if transaction_device != live_device: + raise SystemExit( + "cross-device artifact promotion refused before build: " + f"staging device {transaction_device}, existing live device {live_device}" + ) +PY +} + +rollback_promotion() { + case "$PROMOTION_STATE" in + building|committed) + return 0 + ;; + rolled-back) + ROLLBACK_STATUS=succeeded + return 0 + ;; + backup-pending|backed-up|promotion-pending) ;; + *) + echo "unknown artifact promotion state during rollback: $PROMOTION_STATE" >&2 + ROLLBACK_STATUS=failed + PRESERVE_TRANSACTION_ROOT=1 + return 1 + ;; + esac + + ROLLBACK_STATUS=in-progress + if [ "$HAD_LIVE_ROOT" -eq 1 ]; then + if path_exists "$BACKUP_ROOT"; then + if path_exists "$LIVE_ROOT" \ + && ! strict_rename rollback-discard "$LIVE_ROOT" "$FAILED_ROOT"; then + echo "failed to move incomplete live root aside: $LIVE_ROOT" >&2 + ROLLBACK_STATUS=failed + PRESERVE_TRANSACTION_ROOT=1 + return 1 + fi + if ! strict_rename rollback-restore "$BACKUP_ROOT" "$LIVE_ROOT"; then + echo "failed to restore previous live root; recover it from $BACKUP_ROOT" >&2 + ROLLBACK_STATUS=failed + PRESERVE_TRANSACTION_ROOT=1 + return 1 + fi + elif ! path_exists "$LIVE_ROOT"; then + echo "previous live root is missing from both live and backup paths" >&2 + ROLLBACK_STATUS=failed + PRESERVE_TRANSACTION_ROOT=1 + return 1 + fi + elif path_exists "$LIVE_ROOT"; then + if ! strict_rename rollback-discard "$LIVE_ROOT" "$FAILED_ROOT"; then + echo "failed to remove incomplete first-time live root: $LIVE_ROOT" >&2 + ROLLBACK_STATUS=failed + PRESERVE_TRANSACTION_ROOT=1 + return 1 + fi + fi + + PROMOTION_STATE=rolled-back + ROLLBACK_STATUS=succeeded +} + +write_transaction_health() { + local output_file="$1" + python3 - \ + "$output_file" \ + "$SCOPE" \ + "$TRANSACTION_FAILURE_REASON" \ + "$PROMOTION_STATE" \ + "$ROLLBACK_STATUS" \ + "$TRANSACTION_SIGNAL" <<'PY' +import json +import sys + +output, scope, reason, promotion_state, rollback_status, signal = sys.argv[1:] +payload = { + "failure_reason": reason, + "promotion_state": promotion_state, + "rollback_status": rollback_status, + "scope": scope, + "status": "failed", +} +if signal: + payload["signal"] = signal +with open(output, "w", encoding="utf-8", newline="\n") as handle: + json.dump(payload, handle, separators=(",", ":"), sort_keys=True) + handle.write("\n") +PY +} + preserve_failure_diagnostics() { local status=$? + trap - EXIT HUP INT TERM + set +e if [ "$status" -ne 0 ]; then + rollback_promotion local failure_dir failure_dir="$DIAGNOSTICS_ROOT/${ARTIFACT_GENERATION_ID:-local}-$(date -u +%Y%m%dT%H%M%SZ)" mkdir -p "$failure_dir" [ ! -f "$STAGE_ROOT/upstream-summary.json" ] || cp "$STAGE_ROOT/upstream-summary.json" "$failure_dir/" [ ! -f "$STAGE_ROOT/build-summary.json" ] || cp "$STAGE_ROOT/build-summary.json" "$failure_dir/" [ ! -f "$DIAGNOSTICS_ROOT/upstream-summary.jsonl" ] || mv "$DIAGNOSTICS_ROOT/upstream-summary.jsonl" "$failure_dir/" - printf '{"scope":"%s","status":"failed"}\n' "$SCOPE" > "$failure_dir/transaction-health.json" + write_transaction_health "$failure_dir/transaction-health.json" echo "failure diagnostics preserved in $failure_dir" >&2 fi - rm -rf "$TRANSACTION_ROOT" + if [ "$PRESERVE_TRANSACTION_ROOT" -eq 1 ]; then + echo "transaction recovery data preserved in $TRANSACTION_ROOT" >&2 + else + rm -rf "$TRANSACTION_ROOT" + fi return "$status" } + +handle_transaction_signal() { + local signal="$1" + local status="$2" + + trap - HUP INT TERM + TRANSACTION_FAILURE_REASON=signal + TRANSACTION_SIGNAL="$signal" + echo "artifact transaction received $signal signal" >&2 + rollback_promotion || true + exit "$status" +} + +inject_signal_after_backup() { + local signal="${RULES_TRANSACTION_TEST_SIGNAL_AFTER_BACKUP:-}" + + case "$signal" in + '') return 0 ;; + HUP|INT|TERM) kill -s "$signal" "$$" ;; + *) + echo "unsupported injected transaction signal: $signal" >&2 + return 2 + ;; + esac +} + trap preserve_failure_diagnostics EXIT +trap 'handle_transaction_signal HUP 129' HUP +trap 'handle_transaction_signal INT 130' INT +trap 'handle_transaction_signal TERM 143' TERM + +if ! assert_promotion_filesystem; then + TRANSACTION_FAILURE_REASON=filesystem-preflight-failed + exit 2 +fi export RULES_ARTIFACT_ROOT="$STAGE_ROOT" export RULES_ARTIFACT_DIAGNOSTICS_ROOT="$DIAGNOSTICS_ROOT" @@ -51,13 +257,27 @@ python3 "$ROOT/scripts/tools/summarize-artifacts.py" "$STAGE_ROOT" --output "$ST ARTIFACT_BUILD_SCOPE="$SCOPE" "$ROOT/scripts/commands/generate-artifact-manifest.sh" "$ROOT/scripts/commands/verify-artifact-manifest.sh" -mkdir -p "$(dirname "$LIVE_ROOT")" -if [ -e "$LIVE_ROOT" ]; then - mv "$LIVE_ROOT" "$BACKUP_ROOT" +if path_exists "$LIVE_ROOT"; then + HAD_LIVE_ROOT=1 + PROMOTION_STATE=backup-pending + if ! strict_rename backup "$LIVE_ROOT" "$BACKUP_ROOT"; then + TRANSACTION_FAILURE_REASON=backup-rename-failed + rollback_promotion || true + exit 1 + fi + PROMOTION_STATE=backed-up + inject_signal_after_backup fi -if ! mv "$STAGE_ROOT" "$LIVE_ROOT"; then - [ ! -e "$BACKUP_ROOT" ] || mv "$BACKUP_ROOT" "$LIVE_ROOT" + +PROMOTION_STATE=promotion-pending +if ! strict_rename promote "$STAGE_ROOT" "$LIVE_ROOT"; then + TRANSACTION_FAILURE_REASON=promotion-rename-failed + rollback_promotion || true exit 1 fi -rm -rf "$BACKUP_ROOT" +PROMOTION_STATE=committed + +if ! rm -rf "$BACKUP_ROOT"; then + echo "warning: promoted artifacts but could not remove backup: $BACKUP_ROOT" >&2 +fi echo "artifact transaction promoted: $LIVE_ROOT" diff --git a/scripts/commands/build-custom.sh b/scripts/commands/build-custom.sh index a17c9b0e1..8337d5adf 100755 --- a/scripts/commands/build-custom.sh +++ b/scripts/commands/build-custom.sh @@ -4,6 +4,9 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" cd "$ROOT" +# shellcheck source=/dev/null +source "$ROOT/scripts/commands/check-runtime.sh" + TEXT_ONLY_MODE="${RULES_BUILD_CUSTOM_TEXT_ONLY:-0}" # Reject malformed or globally conflicting custom sources before creating build @@ -18,6 +21,7 @@ TMP_DIR="$(mktemp -d "$TMP_PARENT_DIR/custom.XXXXXX")" TMP_DOMAIN_DIR="$TMP_DIR/domain" TMP_IP_DIR="$TMP_DIR/ip" STAGE_ROOT="$TMP_DIR/output" +CANONICAL_STAGE_ROOT="$TMP_DIR/canonical" BIN_DIR="$ROOT/.bin" ARTIFACT_ROOT="${RULES_ARTIFACT_ROOT:-$ROOT/.output}" DOMAIN_SURGE_DIR="$STAGE_ROOT/domain/surge" @@ -31,9 +35,9 @@ IP_EGERN_DIR="$STAGE_ROOT/ip/egern" IP_SINGBOX_DIR="$STAGE_ROOT/ip/sing-box" IP_MIHOMO_DIR="$STAGE_ROOT/ip/mihomo" -# shellcheck source=scripts/lib/common.sh +# shellcheck source=/dev/null source "$ROOT/scripts/lib/common.sh" -# shellcheck source=scripts/lib/rules.sh +# shellcheck source=/dev/null source "$ROOT/scripts/lib/rules.sh" setup_tool_cache @@ -55,6 +59,22 @@ mkdir -p \ mkdir -p "$TMP_DOMAIN_DIR" "$TMP_IP_DIR" trap 'rm -rf "$TMP_DIR"' EXIT +prepare_canonical_stage() { + if [ -d "$ARTIFACT_ROOT/.canonical" ]; then + mkdir -p "$CANONICAL_STAGE_ROOT" + cp -R "$ARTIFACT_ROOT/.canonical/." "$CANONICAL_STAGE_ROOT/" + elif compgen -G "$ARTIFACT_ROOT/domain/egern/*.yaml" >/dev/null \ + || compgen -G "$ARTIFACT_ROOT/ip/egern/*.yaml" >/dev/null; then + python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$ROOT" \ + --seed-canonical-from "$ARTIFACT_ROOT" \ + --canonical-output "$CANONICAL_STAGE_ROOT" + else + mkdir -p "$CANONICAL_STAGE_ROOT" + fi + mkdir -p "$CANONICAL_STAGE_ROOT/domain" "$CANONICAL_STAGE_ROOT/ip" +} + has_custom_domain=0 has_custom_ip=0 MIHOMO_READY=0 @@ -70,6 +90,8 @@ if [ "$has_custom_domain" -eq 0 ] && [ "$has_custom_ip" -eq 0 ]; then exit 0 fi +prepare_canonical_stage + ensure_mihomo_once() { if [ "$MIHOMO_READY" -eq 0 ]; then ensure_mihomo @@ -162,6 +184,7 @@ build_domain_plain_and_surge() { egern_tmp="$TMP_DOMAIN_DIR/$base.egern.tmp" normalize_custom_domain_source "$list_file" "$plain_out" + cp "$plain_out" "$CANONICAL_STAGE_ROOT/domain/$base.list" render_surge_domain_ruleset_from_rules "$plain_out" "$surge_tmp" render_quanx_domain_ruleset_from_rules "$plain_out" "$quanx_tmp" "$base" render_egern_domain_ruleset_from_rules "$plain_out" "$egern_tmp" @@ -221,6 +244,7 @@ build_ip_plain_and_surge() { write_if_nonempty_or_remove "$quanx_tmp" "$quanx_out" write_if_nonempty_or_remove "$egern_tmp" "$egern_out" mv "$plain_tmp" "$plain_out" + render_ip_plain_to_canonical_list "$plain_out" "$CANONICAL_STAGE_ROOT/ip/$base.list" rm -f "$surge_tmp" "$quanx_tmp" "$egern_tmp" } @@ -307,70 +331,163 @@ if [ "$TEXT_ONLY_MODE" -ne 1 ]; then done fi -commit_staged_custom_artifacts() { - local list_file base relative staged target target_dir - local controlled=() - +controlled_artifact_paths() { + local list_file base while IFS= read -r list_file; do [ -n "$list_file" ] || continue base="$(basename "$list_file" .list)" - controlled+=( - "domain/surge/$base.list" - "domain/quanx/$base.list" + printf '%s\n' \ + "domain/surge/$base.list" \ + "domain/quanx/$base.list" \ "domain/egern/$base.yaml" - ) if [ "$TEXT_ONLY_MODE" -ne 1 ]; then - controlled+=( - "domain/sing-box/$base.srs" - "domain/mihomo/$base.list" + printf '%s\n' \ + "domain/sing-box/$base.srs" \ + "domain/mihomo/$base.list" \ "domain/mihomo/$base.mrs" - ) fi done <<< "$DOMAIN_RULE_FILES" while IFS= read -r list_file; do [ -n "$list_file" ] || continue base="$(basename "$list_file" .list)" - controlled+=( - "ip/surge/$base.list" - "ip/quanx/$base.list" + printf '%s\n' \ + "ip/surge/$base.list" \ + "ip/quanx/$base.list" \ "ip/egern/$base.yaml" - ) if [ "$TEXT_ONLY_MODE" -ne 1 ]; then - controlled+=( - "ip/sing-box/$base.srs" + printf '%s\n' \ + "ip/sing-box/$base.srs" \ "ip/mihomo/$base.mrs" - ) fi done <<< "$IP_RULE_FILES" +} + +commit_staged_custom_artifacts() { + local relative staged target target_dir # Only the paths derived from current custom sources are controlled here. # Restored/upstream artifacts and summaries elsewhere in .output are untouched. - for relative in "${controlled[@]}"; do + for relative in "${CONTROLLED_ARTIFACTS[@]}"; do staged="$STAGE_ROOT/$relative" target="$ARTIFACT_ROOT/$relative" target_dir="$(dirname "$target")" - mkdir -p "$target_dir" + mkdir -p "$target_dir" || return 1 if [ -f "$staged" ]; then - write_if_changed "$staged" "$target" + write_if_changed "$staged" "$target" || return 1 else # A platform-specific skip is committed as deletion only after every # render and binary compile has succeeded. - rm -f "$target" + rm -f "$target" || return 1 + fi + done +} + +COMMIT_BACKUP_ROOT="$TMP_DIR/commit-backup" +CANONICAL_HAD_PREVIOUS=0 +CANONICAL_COMMITTED=0 + +backup_controlled_state() { + local relative source backup + + rm -rf "$COMMIT_BACKUP_ROOT" + mkdir -p "$COMMIT_BACKUP_ROOT/artifacts" + for relative in "${CONTROLLED_ARTIFACTS[@]}"; do + source="$ARTIFACT_ROOT/$relative" + backup="$COMMIT_BACKUP_ROOT/artifacts/$relative" + if [ -f "$source" ]; then + mkdir -p "$(dirname "$backup")" + cp "$source" "$backup" + fi + done + if [ -f "$ARTIFACT_ROOT/artifact-origins.json" ]; then + cp "$ARTIFACT_ROOT/artifact-origins.json" "$COMMIT_BACKUP_ROOT/artifact-origins.json" + fi +} + +rollback_controlled_state() { + local relative target backup + + for relative in "${CONTROLLED_ARTIFACTS[@]}"; do + target="$ARTIFACT_ROOT/$relative" + backup="$COMMIT_BACKUP_ROOT/artifacts/$relative" + rm -f "$target" + if [ -f "$backup" ]; then + mkdir -p "$(dirname "$target")" + cp "$backup" "$target" fi done + if [ -f "$COMMIT_BACKUP_ROOT/artifact-origins.json" ]; then + cp "$COMMIT_BACKUP_ROOT/artifact-origins.json" "$ARTIFACT_ROOT/artifact-origins.json" + else + rm -f "$ARTIFACT_ROOT/artifact-origins.json" + fi +} + +commit_canonical_stage() { + local target="$ARTIFACT_ROOT/.canonical" + local next="$ARTIFACT_ROOT/.canonical.next" + local backup="$ARTIFACT_ROOT/.canonical.backup" + + rm -rf "$next" "$backup" + mv "$CANONICAL_STAGE_ROOT" "$next" || return 1 + if [ -e "$target" ]; then + CANONICAL_HAD_PREVIOUS=1 + if ! mv "$target" "$backup"; then + rm -rf "$next" + return 1 + fi + fi + if ! mv "$next" "$target"; then + [ ! -e "$backup" ] || mv "$backup" "$target" + rm -rf "$next" + return 1 + fi + CANONICAL_COMMITTED=1 +} + +rollback_canonical_stage() { + local target="$ARTIFACT_ROOT/.canonical" + local backup="$ARTIFACT_ROOT/.canonical.backup" + + [ "$CANONICAL_COMMITTED" -eq 1 ] || return 0 + rm -rf "$target" + if [ "$CANONICAL_HAD_PREVIOUS" -eq 1 ] && [ -e "$backup" ]; then + mv "$backup" "$target" + fi + CANONICAL_COMMITTED=0 +} + +finalize_canonical_stage() { + rm -rf "$ARTIFACT_ROOT/.canonical.backup" "$COMMIT_BACKUP_ROOT" } if [ "$TEXT_ONLY_MODE" -ne 1 ]; then # This point is after the last binary compile but before the controlled commit. inject_custom_build_failure late-binary fi -commit_staged_custom_artifacts +mapfile -t CONTROLLED_ARTIFACTS < <(controlled_artifact_paths) origin_args=(mark-custom "$ARTIFACT_ROOT" "$CUSTOM_DOMAIN_DIR" "$CUSTOM_IP_DIR") if [ "$TEXT_ONLY_MODE" -eq 1 ]; then origin_args+=(--text-only) fi -python3 "$ROOT/scripts/tools/artifact_origins.py" "${origin_args[@]}" +backup_controlled_state +commit_failed=0 +if ! commit_staged_custom_artifacts; then + commit_failed=1 +elif ! inject_custom_build_failure canonical-commit; then + commit_failed=1 +elif ! commit_canonical_stage; then + commit_failed=1 +elif ! python3 "$ROOT/scripts/tools/artifact_origins.py" "${origin_args[@]}"; then + commit_failed=1 +fi +if [ "$commit_failed" -ne 0 ]; then + rollback_canonical_stage + rollback_controlled_state + exit 1 +fi +finalize_canonical_stage if [ "$TEXT_ONLY_MODE" -eq 1 ]; then echo "custom build done (text only)" diff --git a/scripts/commands/check-runtime.sh b/scripts/commands/check-runtime.sh new file mode 100755 index 000000000..9b924d838 --- /dev/null +++ b/scripts/commands/check-runtime.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +required_bash_major=5 +required_python_major=3 +required_python_minor=11 + +validate_test_minimum() { + local name="$1" + local value="$2" + + case "$value" in + ''|*[!0-9]*) + echo "$name must be a non-negative integer" >&2 + exit 2 + ;; + esac +} + +if [ -n "${RULES_RUNTIME_TEST_BASH_MIN_MAJOR:-}" ]; then + validate_test_minimum RULES_RUNTIME_TEST_BASH_MIN_MAJOR "$RULES_RUNTIME_TEST_BASH_MIN_MAJOR" + required_bash_major="$RULES_RUNTIME_TEST_BASH_MIN_MAJOR" +fi +if [ -n "${RULES_RUNTIME_TEST_PYTHON_MIN_MAJOR:-}" ]; then + validate_test_minimum RULES_RUNTIME_TEST_PYTHON_MIN_MAJOR "$RULES_RUNTIME_TEST_PYTHON_MIN_MAJOR" + required_python_major="$RULES_RUNTIME_TEST_PYTHON_MIN_MAJOR" +fi +if [ -n "${RULES_RUNTIME_TEST_PYTHON_MIN_MINOR:-}" ]; then + validate_test_minimum RULES_RUNTIME_TEST_PYTHON_MIN_MINOR "$RULES_RUNTIME_TEST_PYTHON_MIN_MINOR" + required_python_minor="$RULES_RUNTIME_TEST_PYTHON_MIN_MINOR" +fi + +if [ -z "${BASH_VERSION:-}" ] || [ "${BASH_VERSINFO[0]:-0}" -lt "$required_bash_major" ]; then + echo "Bash ${required_bash_major}+ is required (found ${BASH_VERSION:-non-Bash shell})" >&2 + exit 1 +fi + +if ! command -v python3 >/dev/null 2>&1; then + echo "Python ${required_python_major}.${required_python_minor}+ is required (python3 not found in PATH)" >&2 + exit 1 +fi + +python3 - "$required_python_major" "$required_python_minor" <<'PY' +import sys + +required = tuple(map(int, sys.argv[1:])) +if sys.version_info[:2] < required: + current = ".".join(map(str, sys.version_info[:3])) + raise SystemExit( + f"Python {required[0]}.{required[1]}+ is required (found {current})" + ) +PY diff --git a/scripts/commands/generate-artifact-manifest.sh b/scripts/commands/generate-artifact-manifest.sh index 3250b06bb..0d0927946 100755 --- a/scripts/commands/generate-artifact-manifest.sh +++ b/scripts/commands/generate-artifact-manifest.sh @@ -2,10 +2,17 @@ set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +# shellcheck source=/dev/null +source "$ROOT/scripts/commands/check-runtime.sh" + : "${ARTIFACT_GENERATION_ID:=${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}}" : "${ARTIFACT_BUILD_ID:=$ARTIFACT_GENERATION_ID}" : "${ARTIFACT_BUILD_SCOPE:=full}" +python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$ROOT" \ + --verify-canonical-inventory "${RULES_ARTIFACT_ROOT:-$ROOT/.output}" + args=(generate --generation-id "$ARTIFACT_GENERATION_ID" --build-id "$ARTIFACT_BUILD_ID" --build-scope "$ARTIFACT_BUILD_SCOPE") if [ -n "${ARTIFACT_SOURCE_SHA:-}" ]; then args+=(--source-sha "$ARTIFACT_SOURCE_SHA") diff --git a/scripts/commands/publish-branches.sh b/scripts/commands/publish-branches.sh index 84f86c20c..e4cbfab6d 100755 --- a/scripts/commands/publish-branches.sh +++ b/scripts/commands/publish-branches.sh @@ -4,20 +4,138 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" cd "$ROOT" +# shellcheck source=/dev/null +source "$ROOT/scripts/commands/check-runtime.sh" + DRY_RUN="${PUBLISH_DRY_RUN:-0}" ARTIFACT_ROOT="${RULES_ARTIFACT_ROOT:-$ROOT/.output}" MANIFEST_FILE="$ARTIFACT_ROOT/artifact-manifest.json" CAPABILITY_REGISTRY="$(python3 "$ROOT/scripts/tools/platform_capabilities.py" shell-registry)" +PUBLISH_BRANCH_NAMES=(surge quanx egern sing-box mihomo) ARTIFACT_SOURCE_SHA="${ARTIFACT_SOURCE_SHA:-}" "$ROOT/scripts/commands/verify-artifact-manifest.sh" -read -r MANIFEST_GENERATION_ID MANIFEST_SOURCE_SHA < <( +read -r MANIFEST_GENERATION_ID MANIFEST_SOURCE_SHA MANIFEST_BASELINE_STATUS MANIFEST_BASELINE_SOURCE < <( python3 - <<'PY' "$MANIFEST_FILE" import json, sys manifest = json.load(open(sys.argv[1], encoding="utf-8")) -print(manifest["generation_id"], manifest["source"]["commit"] or "unknown") +print( + manifest["generation_id"], + manifest["source"]["commit"] or "unknown", + manifest["baseline"]["status"], + manifest["baseline"]["source_commit"] or "-", +) PY ) +refresh_and_validate_remote_baseline() { + local metadata_file branch commit subject generation source + metadata_file="$(mktemp)" + : > "$metadata_file" + + for branch in "${PUBLISH_BRANCH_NAMES[@]}"; do + if ! git -C "$ROOT" fetch --quiet --no-tags --depth=1 origin \ + "+refs/heads/$branch:refs/remotes/origin/$branch"; then + echo "required remote publication branch origin/$branch is unavailable" >&2 + rm -f "$metadata_file" + return 1 + fi + commit="$(git -C "$ROOT" rev-parse --verify "origin/$branch^{commit}")" + subject="$(git -C "$ROOT" log -1 --format=%s "origin/$branch")" + if [[ "$subject" =~ ^chore:\ publish\ ${branch}\ artifacts\ \[generation\ ([0-9]+-[0-9]+)\ source\ ([0-9a-f]{40})\]$ ]]; then + generation="${BASH_REMATCH[1]}" + source="${BASH_REMATCH[2]}" + else + generation="-" + source="-" + fi + printf '%s\t%s\t%s\t%s\n' "$branch" "$commit" "$generation" "$source" >> "$metadata_file" + done + + if ! python3 - "$MANIFEST_FILE" "$metadata_file" <<'PY' +import json +import sys +from pathlib import Path + +manifest = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +baseline = manifest["baseline"] +rows = [line.split("\t") for line in Path(sys.argv[2]).read_text(encoding="utf-8").splitlines() if line] +if len(rows) != 5: + raise SystemExit(f"remote publication cohort is incomplete: branches={len(rows)}") +valid_identities = {(row[2], row[3]) for row in rows if row[2] != "-" and row[3] != "-"} +consistent = len(valid_identities) == 1 and all(row[2] != "-" and row[3] != "-" for row in rows) +generation, source = next(iter(valid_identities)) if consistent else (None, None) +remote = { + "status": "consistent" if consistent else "inconsistent", + "generation_id": generation, + "source_commit": source, + "branches": { + row[0]: { + "commit": row[1], + "generation_id": None if row[2] == "-" else row[2], + "source_commit": None if row[3] == "-" else row[3], + } + for row in rows + }, +} +if remote != baseline: + raise SystemExit( + "publication baseline is stale: the remote cohort changed after this build started" + ) +PY + then + rm -f "$metadata_file" + return 1 + fi + rm -f "$metadata_file" +} + +assert_candidate_advances_baseline() { + if [ "$MANIFEST_BASELINE_STATUS" = "consistent" ] && ! git -C "$ROOT" merge-base --is-ancestor \ + "$MANIFEST_BASELINE_SOURCE" "$MANIFEST_SOURCE_SHA"; then + echo "stale publication source refused: candidate $MANIFEST_SOURCE_SHA does not descend from baseline $MANIFEST_BASELINE_SOURCE" >&2 + return 1 + fi + + python3 - "$MANIFEST_FILE" <<'PY' +import json +import re +import sys + +manifest = json.load(open(sys.argv[1], encoding="utf-8")) +candidate = manifest["generation_id"] +pattern = re.compile(r"([0-9]+)-([0-9]+)") +candidate_match = pattern.fullmatch(candidate) +if candidate_match is None: + raise SystemExit("candidate publication generation must use -") +candidate_order = tuple(map(int, candidate_match.groups())) +for branch, item in manifest["baseline"]["branches"].items(): + baseline = item["generation_id"] + if baseline is None: + continue + baseline_match = pattern.fullmatch(baseline) + if baseline_match is None: + raise SystemExit(f"baseline generation is invalid for {branch}: {baseline}") + if candidate_order <= tuple(map(int, baseline_match.groups())): + raise SystemExit( + f"stale publication generation refused: candidate {candidate} " + f"is not newer than {branch} baseline {baseline}" + ) +PY +} + +assert_remote_main_tip() { + local remote_main + remote_main="$(git -C "$ROOT" ls-remote --exit-code origin refs/heads/main | awk 'NR == 1 {print $1}')" + if [ "$remote_main" != "$MANIFEST_SOURCE_SHA" ]; then + echo "stale publication source refused: remote main is $remote_main, candidate is $MANIFEST_SOURCE_SHA" >&2 + return 1 + fi +} + +assert_remote_main_tip +refresh_and_validate_remote_baseline +assert_candidate_advances_baseline + branch_readme() { local branch="$1" local template="$ROOT/templates/branch-readmes/${branch}.md" @@ -217,6 +335,11 @@ publish_queued_refs() { return 0 fi + # Close the source race after all branch trees have been prepared. The + # expected-SHA leases below independently close artifact-branch races. + assert_remote_main_tip + refresh_and_validate_remote_baseline + if [ "$PUBLISH_COHORT_CHANGED" -eq 0 ]; then echo "all publish branches unchanged, skip push" return 0 @@ -250,6 +373,10 @@ publish_queued_refs() { echo "publishing branches atomically: ${names[*]}" git push --atomic "${leases[@]}" origin "${refspecs[@]}" + if ! assert_remote_main_tip; then + echo "artifact cohort was published while main advanced; this run is failed intentionally and the queued current-main run must roll forward" >&2 + return 1 + fi } PUBLISH_TMPDIR="$(mktemp -d)" @@ -265,7 +392,13 @@ git config user.email "41898282+github-actions[bot]@users.noreply.github.com" mkdir -p .git/objects/info git -C "$ROOT" rev-parse --path-format=absolute --git-path objects > .git/objects/info/alternates -PUBLISH_COHORT_CHANGED=0 +if [ "$MANIFEST_BASELINE_STATUS" = "consistent" ]; then + PUBLISH_COHORT_CHANGED=0 +else + # A full recovery must repair split/invalid cohort metadata even when every + # artifact tree is byte-for-byte unchanged. + PUBLISH_COHORT_CHANGED=1 +fi declare -A PUBLISH_BRANCH PUBLISH_DOMAIN_EXTENSION PUBLISH_IP_EXTENSION while IFS=$'\t' read -r platform _public_name branch section extension _format _empty _compiler _verifier; do diff --git a/scripts/commands/restore-artifacts.sh b/scripts/commands/restore-artifacts.sh index 21638ea25..4a38cfb10 100755 --- a/scripts/commands/restore-artifacts.sh +++ b/scripts/commands/restore-artifacts.sh @@ -11,6 +11,7 @@ fi ARTIFACT_ROOT="${RULES_ARTIFACT_ROOT:-$ROOT/.output}" TMP_ROOT="$ROOT/.tmp/restore-published" RESTORE_METADATA_FILE="$TMP_ROOT/restored-branches.tsv" +BASELINE_FILE="${ARTIFACT_BASELINE_FILE:-$ROOT/.tmp/publication-baseline.json}" inject_restore_failure() { local point="$1" @@ -57,10 +58,11 @@ restore_branch_artifacts() { local commit subject generation source commit="$(git rev-parse "origin/$branch^{commit}")" subject="$(git log -1 --format=%s "origin/$branch")" - generation="$(printf '%s' "$subject" | grep -oE '\[generation [^ ]+' | cut -d' ' -f2 || true)" - source="$(printf '%s' "$subject" | grep -oE 'source [0-9a-f]{40}\]' | cut -d' ' -f2 | tr -d ']' || true)" - if [ -z "$generation" ] || [ -z "$source" ]; then - echo "origin/$branch lacks required generation/source publication metadata" >&2 + if [[ "$subject" =~ ^chore:\ publish\ ${branch}\ artifacts\ \[generation\ ([0-9]+-[0-9]+)\ source\ ([0-9a-f]{40})\]$ ]]; then + generation="${BASH_REMATCH[1]}" + source="${BASH_REMATCH[2]}" + else + echo "origin/$branch lacks valid generation/source publication metadata" >&2 return 1 fi printf '%s\t%s\t%s\t%s\n' "$branch" "$commit" "$generation" "$source" >> "$RESTORE_METADATA_FILE" @@ -86,11 +88,42 @@ if len(rows) != 5: generations = {row[2] for row in rows}; sources = {row[3] for row in rows} if len(generations) != 1 or len(sources) != 1: raise SystemExit(f"restored branches are from inconsistent publications: generations={sorted(generations)}, sources={sorted(sources)}") -payload = {"generation_id": next(iter(generations)), "source_commit": next(iter(sources)), - "branches": {row[0]: {"commit": row[1]} for row in rows}} +generation = next(iter(generations)); source = next(iter(sources)) +payload = { + "status": "consistent", + "generation_id": generation, + "source_commit": source, + "branches": { + row[0]: { + "commit": row[1], + "generation_id": generation, + "source_commit": source, + } + for row in rows + }, +} Path(sys.argv[2]).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") PY +if [ -f "$BASELINE_FILE" ]; then + python3 - "$BASELINE_FILE" "$ARTIFACT_ROOT/restoration-metadata.json" <<'PY' +import json +import sys +from pathlib import Path + +baseline = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +restored = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) +if restored != baseline: + raise SystemExit( + "restored publication cohort differs from the selected build baseline; " + "refusing a mixed custom build" + ) +PY +else + mkdir -p "$(dirname "$BASELINE_FILE")" + cp "$ARTIFACT_ROOT/restoration-metadata.json" "$BASELINE_FILE" +fi + python3 "$ROOT/scripts/tools/artifact_origins.py" reset \ "$ARTIFACT_ROOT" \ restored-published-branch diff --git a/scripts/commands/select-build-scope.sh b/scripts/commands/select-build-scope.sh index b46103e34..db8c36b3c 100755 --- a/scripts/commands/select-build-scope.sh +++ b/scripts/commands/select-build-scope.sh @@ -7,6 +7,10 @@ BEFORE_SHA="${BEFORE_SHA:-}" CURRENT_SHA="${CURRENT_SHA:-HEAD}" CHANGED_FILES_INPUT="${CHANGED_FILES:-}" DELETED_CUSTOM_FILES_INPUT="${DELETED_CUSTOM_FILES:-}" +ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)" +BASELINE_FILE="${ARTIFACT_BASELINE_FILE:-$ROOT/.tmp/publication-baseline.json}" +BASELINE_INPUT_FILE="${RULES_PUBLICATION_BASELINE_INPUT:-}" +PUBLISH_BRANCHES=(surge quanx egern sing-box mihomo) scope="full" reason="scheduled sync refresh" @@ -18,15 +22,165 @@ print_output() { echo "scope=$scope" echo "reason=$reason" echo "base_sha=$base_sha" + echo "baseline_file=$BASELINE_FILE" } >> "$GITHUB_OUTPUT" fi echo "scope=$scope" echo "reason=$reason" echo "base_sha=$base_sha" + echo "baseline_file=$BASELINE_FILE" echo "Build scope: $scope ($reason)" } +validate_and_write_baseline() { + local input_file="$1" + local output_file="$2" + + mkdir -p "$(dirname "$output_file")" + python3 - "$input_file" "$output_file" <<'PY' +import json +import re +import sys +from pathlib import Path + +source_path, output_path = map(Path, sys.argv[1:]) +branches = {"surge", "quanx", "egern", "sing-box", "mihomo"} +sha_re = re.compile(r"[0-9a-f]{40}") +generation_re = re.compile(r"[0-9]+-[0-9]+") + +try: + payload = json.loads(source_path.read_text(encoding="utf-8")) +except (OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"publication baseline unreadable: {exc}") + +if not isinstance(payload, dict) or set(payload) != {"status", "generation_id", "source_commit", "branches"}: + raise SystemExit("publication baseline has an invalid top-level schema") +if payload["status"] not in {"consistent", "inconsistent"}: + raise SystemExit("publication baseline status must be consistent or inconsistent") +if not isinstance(payload["branches"], dict) or set(payload["branches"]) != branches: + raise SystemExit("publication baseline must record exactly the five publish branches") +for branch, item in payload["branches"].items(): + if not isinstance(item, dict) or set(item) != {"commit", "generation_id", "source_commit"}: + raise SystemExit(f"publication baseline branch entry is invalid: {branch}") + if not isinstance(item["commit"], str) or not sha_re.fullmatch(item["commit"]): + raise SystemExit(f"publication baseline branch commit is invalid: {branch}") + if item["generation_id"] is not None and ( + not isinstance(item["generation_id"], str) or not generation_re.fullmatch(item["generation_id"]) + ): + raise SystemExit(f"publication baseline branch generation is invalid: {branch}") + if item["source_commit"] is not None and ( + not isinstance(item["source_commit"], str) or not sha_re.fullmatch(item["source_commit"]) + ): + raise SystemExit(f"publication baseline branch source is invalid: {branch}") + +if payload["status"] == "consistent": + if not isinstance(payload["generation_id"], str) or not generation_re.fullmatch(payload["generation_id"]): + raise SystemExit("consistent publication baseline generation_id is invalid") + if not isinstance(payload["source_commit"], str) or not sha_re.fullmatch(payload["source_commit"]): + raise SystemExit("consistent publication baseline source_commit is invalid") + identities = {(item["generation_id"], item["source_commit"]) for item in payload["branches"].values()} + if identities != {(payload["generation_id"], payload["source_commit"])}: + raise SystemExit("consistent publication baseline branch identities disagree") +elif payload["generation_id"] is not None or payload["source_commit"] is not None: + raise SystemExit("inconsistent publication baseline must use null cohort identity") + +output_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") +print(payload["status"], payload["generation_id"] or "-", payload["source_commit"] or "-") +PY +} + +resolve_remote_baseline() { + local metadata_file branch commit subject generation source + metadata_file="$(mktemp)" + trap 'rm -f "$metadata_file"' RETURN + + : > "$metadata_file" + for branch in "${PUBLISH_BRANCHES[@]}"; do + if ! git fetch --quiet --no-tags --depth=1 origin \ + "+refs/heads/$branch:refs/remotes/origin/$branch"; then + echo "required remote publication branch origin/$branch is unavailable" >&2 + return 1 + fi + commit="$(git rev-parse --verify "origin/$branch^{commit}")" + subject="$(git log -1 --format=%s "origin/$branch")" + if [[ "$subject" =~ ^chore:\ publish\ ${branch}\ artifacts\ \[generation\ ([0-9]+-[0-9]+)\ source\ ([0-9a-f]{40})\]$ ]]; then + generation="${BASH_REMATCH[1]}" + source="${BASH_REMATCH[2]}" + else + generation="-" + source="-" + fi + printf '%s\t%s\t%s\t%s\n' "$branch" "$commit" "$generation" "$source" >> "$metadata_file" + done + + mkdir -p "$(dirname "$BASELINE_FILE")" + python3 - "$metadata_file" "$BASELINE_FILE" <<'PY' +import json +import sys +from pathlib import Path + +rows = [line.split("\t") for line in Path(sys.argv[1]).read_text(encoding="utf-8").splitlines() if line] +if len(rows) != 5: + raise SystemExit(f"publication baseline incomplete: expected 5 branches, got {len(rows)}") +valid_identities = {(row[2], row[3]) for row in rows if row[2] != "-" and row[3] != "-"} +consistent = len(valid_identities) == 1 and all(row[2] != "-" and row[3] != "-" for row in rows) +generation, source = next(iter(valid_identities)) if consistent else (None, None) +payload = { + "status": "consistent" if consistent else "inconsistent", + "generation_id": generation, + "source_commit": source, + "branches": { + row[0]: { + "commit": row[1], + "generation_id": None if row[2] == "-" else row[2], + "source_commit": None if row[3] == "-" else row[3], + } + for row in rows + }, +} +Path(sys.argv[2]).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") +print(payload["status"], payload["generation_id"] or "-", payload["source_commit"] or "-") +PY +} + +if ! git cat-file -e "${CURRENT_SHA}^{commit}" 2>/dev/null; then + echo "current source commit is unavailable: $CURRENT_SHA" >&2 + exit 1 +fi +CURRENT_SHA="$(git rev-parse "${CURRENT_SHA}^{commit}")" + +baseline_loaded=0 +baseline_status="" +baseline_generation="" +baseline_source="" + +load_publication_baseline() { + if [ "$baseline_loaded" -eq 1 ]; then + return 0 + fi + if [ -n "$BASELINE_INPUT_FILE" ]; then + read -r baseline_status baseline_generation baseline_source < <( + validate_and_write_baseline "$BASELINE_INPUT_FILE" "$BASELINE_FILE" + ) + else + read -r baseline_status baseline_generation baseline_source < <(resolve_remote_baseline) + fi + baseline_loaded=1 + echo "Publication baseline: status $baseline_status, generation $baseline_generation, source $baseline_source" + + if [ "$baseline_status" = "consistent" ]; then + if ! git cat-file -e "${baseline_source}^{commit}" 2>/dev/null; then + echo "published source commit is unavailable locally: $baseline_source" >&2 + return 1 + fi + if ! git merge-base --is-ancestor "$baseline_source" "$CURRENT_SHA"; then + echo "stale source refused: published source $baseline_source is not an ancestor of candidate $CURRENT_SHA" >&2 + return 1 + fi + fi +} + collect_changed_files() { local before="$1" local current="$2" @@ -71,13 +225,25 @@ has_only_non_build_changes() { ! grep -Eqv '^(\.github/(CODEOWNERS$|ISSUE_TEMPLATE/|dependabot\.yml$|pull_request_template\.md$)|\.gitignore$|CONTRIBUTING\.md$|LICENSE$|NOTICE$|README\.md$|SECURITY\.md$|THIRD_PARTY_NOTICES\.md$|docs/)' } +if [ "$EVENT_NAME" != "pull_request" ]; then + load_publication_baseline +fi + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then case "$INPUT_SCOPE" in custom) - base_sha="$(git rev-parse HEAD^ 2>/dev/null || true)" - if [ -z "$base_sha" ] || ! changed_files="$(collect_changed_files "$base_sha" "$CURRENT_SHA")"; then + if [ "$baseline_status" != "consistent" ]; then + scope="full" + reason="publication cohort inconsistent; using full sync" + base_sha="" + else + base_sha="$baseline_source" + fi + if [ "$scope" = "full" ] && [ "$reason" = "publication cohort inconsistent; using full sync" ]; then + : + elif [ -z "$base_sha" ] || ! changed_files="$(collect_changed_files "$base_sha" "$CURRENT_SHA")"; then scope="full" - reason="manual custom baseline unavailable; using full sync" + reason="manual custom publication baseline unavailable; using full sync" base_sha="" elif ! deleted_custom_files="$(collect_deleted_custom_files "$base_sha" "$CURRENT_SHA")"; then scope="full" @@ -108,13 +274,25 @@ if [ "$EVENT_NAME" = "workflow_dispatch" ]; then ;; esac elif [ "$EVENT_NAME" = "push" ] || [ "$EVENT_NAME" = "pull_request" ]; then - before="$BEFORE_SHA" - if [ "$EVENT_NAME" = "push" ] && { [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; }; then - before="$(git rev-parse "${CURRENT_SHA}^" 2>/dev/null || true)" + if [ "$EVENT_NAME" = "push" ]; then + if [ "$baseline_status" != "consistent" ]; then + scope="full" + reason="publication cohort inconsistent; using full sync" + base_sha="" + before="" + else + before="$baseline_source" + fi + else + before="$BEFORE_SHA" + fi + if [ "$baseline_status" = "consistent" ] || [ "$EVENT_NAME" = "pull_request" ]; then + base_sha="$before" fi - base_sha="$before" - if [ -z "$CHANGED_FILES_INPUT" ] && { + if [ "$EVENT_NAME" = "push" ] && [ "$baseline_status" != "consistent" ]; then + : + elif [ -z "$CHANGED_FILES_INPUT" ] && { [ -z "$before" ] || ! git cat-file -e "${before}^{commit}" 2>/dev/null || ! git cat-file -e "${CURRENT_SHA}^{commit}" 2>/dev/null @@ -166,4 +344,13 @@ elif [ "$EVENT_NAME" = "push" ] || [ "$EVENT_NAME" = "pull_request" ]; then fi fi +if [ "$EVENT_NAME" = "pull_request" ] && [ "$scope" != "none" ]; then + load_publication_baseline + if [ "$scope" = "custom" ] && [ "$baseline_status" != "consistent" ]; then + scope="full" + reason="publication cohort inconsistent; using full sync" + base_sha="" + fi +fi + print_output diff --git a/scripts/commands/sync-upstream.sh b/scripts/commands/sync-upstream.sh index 374840263..fadf7ddfd 100755 --- a/scripts/commands/sync-upstream.sh +++ b/scripts/commands/sync-upstream.sh @@ -17,8 +17,10 @@ ARTIFACTS_DIR="${RULES_ARTIFACT_ROOT:-$ROOT_DIR/.output}" DIAGNOSTICS_DIR="${RULES_ARTIFACT_DIAGNOSTICS_ROOT:-$ROOT_DIR/.tmp/artifact-diagnostics}" DOMAIN_ARTIFACTS_DIR="$ARTIFACTS_DIR/domain" IP_ARTIFACTS_DIR="$ARTIFACTS_DIR/ip" +CANONICAL_ARTIFACTS_DIR="$ARTIFACTS_DIR/.canonical" DOMAIN_RULE_MANIFEST_FILE="$DOMAIN_ARTIFACTS_DIR/rule-manifest.json" IP_TEXT_ARTIFACTS=(cn private google telegram cloudflare cloudfront aws fastly github apple) +IP_CANONICAL_ARTIFACTS=("${IP_TEXT_ARTIFACTS[@]}" netflix spotify disney) UPSTREAMS_CONFIG_FILE="$ROOT_DIR/config/upstreams.json" UPSTREAM_SUMMARY_FILE="$WORK_TMP_DIR/upstream-summary.jsonl" @@ -95,7 +97,8 @@ preserve_sync_diagnostics() { } trap preserve_sync_diagnostics EXIT -mkdir -p "$DOMAIN_ARTIFACTS_DIR" "$IP_ARTIFACTS_DIR" +rm -rf "$CANONICAL_ARTIFACTS_DIR" +mkdir -p "$DOMAIN_ARTIFACTS_DIR" "$IP_ARTIFACTS_DIR" "$CANONICAL_ARTIFACTS_DIR" : > "$UPSTREAM_SUMMARY_FILE" echo "=== SYNC START ===" @@ -795,6 +798,9 @@ python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" export \ python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" domain-rule-manifest \ "$DOMAIN_RULE_TMP_DIR" \ "$DOMAIN_RULE_MANIFEST_FILE" +stage_domain_canonical_rules \ + "$DOMAIN_RULE_TMP_DIR" \ + "$CANONICAL_ARTIFACTS_DIR/domain" assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE" verify_and_record_upstream_health \ domain \ @@ -896,6 +902,14 @@ sync_asn_ip_list netflix "${NETFLIX_ASNS[@]}" sync_asn_ip_list spotify "${SPOTIFY_ASNS[@]}" sync_asn_ip_list disney "${DISNEY_ASNS[@]}" +rm -rf "$CANONICAL_ARTIFACTS_DIR/ip" +mkdir -p "$CANONICAL_ARTIFACTS_DIR/ip" +for name in "${IP_CANONICAL_ARTIFACTS[@]}"; do + render_ip_plain_to_canonical_list \ + "$IP_BUILD_TMP_DIR/${name}.cidr.txt" \ + "$CANONICAL_ARTIFACTS_DIR/ip/${name}.list" +done + assert_files_present "$IP_ARTIFACTS_DIR/surge" "$IP_ARTIFACTS_DIR/surge/*.list" assert_files_present "$IP_ARTIFACTS_DIR/quanx" "$IP_ARTIFACTS_DIR/quanx/*.list" inject_sync_failure late-ip diff --git a/scripts/commands/verify-artifact-manifest.sh b/scripts/commands/verify-artifact-manifest.sh index a82c9fcc1..3f2541b5e 100755 --- a/scripts/commands/verify-artifact-manifest.sh +++ b/scripts/commands/verify-artifact-manifest.sh @@ -1,6 +1,13 @@ #!/usr/bin/env bash set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +# shellcheck source=/dev/null +source "$ROOT/scripts/commands/check-runtime.sh" + +python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$ROOT" \ + --verify-canonical-inventory "${RULES_ARTIFACT_ROOT:-$ROOT/.output}" args=(verify) if [ -n "${ARTIFACT_SOURCE_SHA:-}" ]; then args+=(--source-sha "$ARTIFACT_SOURCE_SHA") diff --git a/scripts/lib/rules.sh b/scripts/lib/rules.sh index fc5ae210b..1a1db7080 100644 --- a/scripts/lib/rules.sh +++ b/scripts/lib/rules.sh @@ -126,6 +126,23 @@ normalize_custom_domain_source() { python3 "$ROOT/scripts/tools/export-domain-rules.py" normalize-classical "$input_file" "$output_file" } +stage_domain_canonical_rules() { + local rule_dir="$1" + local canonical_dir="$2" + local rule_file + + rm -rf "$canonical_dir" + mkdir -p "$canonical_dir" + for rule_file in "$rule_dir"/*.list; do + [ -f "$rule_file" ] || continue + cp "$rule_file" "$canonical_dir/" + done + if ! compgen -G "$canonical_dir/*.list" >/dev/null; then + echo "canonical domain rule directory is empty: $rule_dir" >&2 + return 1 + fi +} + build_domain_json_from_rules() { local rule_list="$1" local json_out="$2" @@ -323,6 +340,21 @@ render_ip_plain_to_surge_list() { python3 "$ROOT/scripts/tools/normalize-ip-rules.py" "${args[@]}" } +render_ip_plain_to_canonical_list() { + local plain_list="$1" + local canonical_out="$2" + + mkdir -p "$(dirname "$canonical_out")" + awk ' + /^[[:space:]]*$/ || /^[[:space:]]*#/ { next } + { printf "%s,%s\n", ($0 ~ /:/ ? "IP-CIDR6" : "IP-CIDR"), $0 } + ' "$plain_list" > "$canonical_out" + if [ ! -s "$canonical_out" ]; then + echo "canonical IP rule source is empty: $plain_list" >&2 + return 1 + fi +} + render_ip_plain_to_quanx_list() { local plain_list="$1" local quanx_out="$2" diff --git a/scripts/tests/test-artifact-manifest.sh b/scripts/tests/test-artifact-manifest.sh index 0f18e0dd3..0de820196 100644 --- a/scripts/tests/test-artifact-manifest.sh +++ b/scripts/tests/test-artifact-manifest.sh @@ -18,7 +18,10 @@ cat > "$REPO/.bin/sing-box" <<'EOF' set -eu input="$3"; output="$5" grep -q CORRUPT "$input" && exit 1 -if [[ "$input" == */ip/* ]]; then printf '{"version":4,"rules":[{"ip_cidr":["192.0.2.0/24"]}]}\n' > "$output" +if [[ "$input" == */ip/* ]]; then + value=192.0.2.0/24 + grep -q IPDRIFT "$input" && value=198.51.100.0/24 + printf '{"version":4,"rules":[{"ip_cidr":["%s"]}]}\n' "$value" > "$output" else printf '{"version":4,"rules":[{"domain_suffix":["custom.example"]}]}\n' > "$output"; fi EOF cat > "$REPO/.bin/mihomo" <<'EOF' @@ -26,10 +29,18 @@ cat > "$REPO/.bin/mihomo" <<'EOF' set -eu input="$4"; output="$5" grep -q CORRUPT "$input" && exit 1 -if [[ "$input" == */ip/* ]]; then printf '192.0.2.0/24\n' > "$output"; else printf '+.custom.example\n' > "$output"; fi +if [[ "$input" == */ip/* ]]; then + value=192.0.2.0/24 + grep -q IPDRIFT "$input" && value=198.51.100.0/24 + printf '%s\n' "$value" > "$output" +else printf '+.custom.example\n' > "$output"; fi EOF chmod +x "$REPO/.bin/sing-box" "$REPO/.bin/mihomo" -cp "$ROOT/scripts/commands/generate-artifact-manifest.sh" "$ROOT/scripts/commands/verify-artifact-manifest.sh" "$REPO/scripts/commands/" +cp \ + "$ROOT/scripts/commands/check-runtime.sh" \ + "$ROOT/scripts/commands/generate-artifact-manifest.sh" \ + "$ROOT/scripts/commands/verify-artifact-manifest.sh" \ + "$REPO/scripts/commands/" cp "$ROOT/config/domain-platform-capabilities.json" "$ROOT/config/tools-lock.json" "$REPO/config/" printf 'DOMAIN-SUFFIX,custom.example\n' > "$REPO/sources/custom/domain/custom.list" git -C "$REPO" init -q @@ -37,7 +48,12 @@ git -C "$REPO" config user.name test git -C "$REPO" config user.email test@example.com git -C "$REPO" add . git -C "$REPO" commit -m fixture >/dev/null +BASELINE_SOURCE_SHA="$(git -C "$REPO" rev-parse HEAD)" +printf 'candidate\n' > "$REPO/candidate.txt" +git -C "$REPO" add candidate.txt +git -C "$REPO" commit -m candidate >/dev/null SOURCE_SHA="$(git -C "$REPO" rev-parse HEAD)" +BASELINE_FILE="$REPO/publication-baseline.json" python3 - "$REPO" <<'PY' import hashlib, json, sys from pathlib import Path @@ -52,6 +68,17 @@ for tool in ('sing-box','mihomo'): lock_path.write_text(json.dumps(lock,indent=2,sort_keys=True)+'\n') PY +python3 - "$BASELINE_FILE" "$BASELINE_SOURCE_SHA" <<'PY' +import json, sys +from pathlib import Path +path, source = sys.argv[1:] +branches = ('surge','quanx','egern','sing-box','mihomo') +payload = {'status':'consistent','generation_id':'90-1','source_commit':source, + 'branches':{branch:{'commit':f'{index + 1:040x}','generation_id':'90-1','source_commit':source} + for index,branch in enumerate(branches)}} +Path(path).write_text(json.dumps(payload,indent=2,sort_keys=True)+'\n', encoding='utf-8') +PY + make_files() { rm -rf "$REPO/.output" while read -r platform extension; do @@ -81,23 +108,30 @@ egern yaml sing-box srs mihomo mrs EOF - python3 - "$REPO/.output" <<'PY' + python3 "$REPO/scripts/tools/artifact_verifier.py" \ + --root "$REPO" \ + --seed-canonical-from "$REPO/.output" \ + --canonical-output "$REPO/.output/.canonical" + python3 - "$REPO/.output" "$BASELINE_SOURCE_SHA" <<'PY' import json, sys from pathlib import Path root=Path(sys.argv[1]); origins={} -for path in root.glob('*/*/*'): - if path.is_file(): - rel=path.relative_to(root).as_posix() - origins[rel]='generated-custom' if path.stem == 'custom' else 'restored-published-branch' +for section in ('domain', 'ip'): + for path in (root/section).glob('*/*'): + if path.is_file(): + rel=path.relative_to(root).as_posix() + origins[rel]='generated-custom' if path.stem == 'custom' else 'restored-published-branch' (root/'artifact-origins.json').write_text(json.dumps(origins,indent=2,sort_keys=True)+'\n') (root/'build-summary.json').write_text('{}\n') -restoration={'generation_id':'restored-1','source_commit':'0'*40,'branches':{b:{'commit':'1'*40} for b in ('surge','quanx','egern','sing-box','mihomo')}} +restoration={'status':'consistent','generation_id':'90-1','source_commit':sys.argv[2], + 'branches':{b:{'commit':f'{i + 1:040x}','generation_id':'90-1','source_commit':sys.argv[2]} + for i,b in enumerate(('surge','quanx','egern','sing-box','mihomo'))}} (root/'restoration-metadata.json').write_text(json.dumps(restoration,indent=2,sort_keys=True)+'\n') PY } generate() { - ARTIFACT_GENERATION_ID=offline-1 ARTIFACT_BUILD_ID=build-1 ARTIFACT_BUILD_SCOPE=custom ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null + ARTIFACT_BASELINE_FILE="$BASELINE_FILE" ARTIFACT_GENERATION_ID=offline-1 ARTIFACT_BUILD_ID=build-1 ARTIFACT_BUILD_SCOPE=custom ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null } verify() { ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/verify-artifact-manifest.sh" >/dev/null; } rejects() { @@ -125,16 +159,40 @@ origins={a['path']:a['origin'] for a in m['artifacts']} assert origins['domain/surge/custom.list']=='generated-custom' assert origins['domain/quanx/custom.list']=='generated-custom' assert origins['ip/surge/base.list']=='restored-published-branch' +assert m['baseline']==m['restoration'] PY +make_files +printf 'HOST-SUFFIX,drift.example,custom\n' > "$REPO/.output/domain/quanx/custom.list" +rejects 'decoded rule values differ from canonical source' generate + +make_files +rm "$REPO/.output/domain/quanx/custom.list" +rejects 'canonical counterpart missing: domain/quanx/custom.list' generate + +mv "$REPO/sources/custom/domain/custom.list" "$TMP/custom.list" +make_files +printf 'HOST-SUFFIX,drift.example,custom\n' > "$REPO/.output/domain/quanx/custom.list" +rejects 'decoded rule values differ from canonical source' generate +mv "$TMP/custom.list" "$REPO/sources/custom/domain/custom.list" + +make_files +printf 'IP-CIDR,198.51.100.0/24,no-resolve\n' > "$REPO/.output/ip/surge/base.list" +printf "no_resolve: true\nip_cidr_set:\n - '198.51.100.0/24'\n" > "$REPO/.output/ip/egern/base.yaml" +printf 'IPDRIFT\n' > "$REPO/.output/ip/sing-box/base.srs" +printf 'IPDRIFT\n' > "$REPO/.output/ip/mihomo/base.mrs" +rejects 'decoded rule values differ from canonical source' generate + +make_files +generate printf tampered >> "$REPO/.output/domain/surge/custom.list" rejects 'artifact hash mismatch' verify make_files; generate printf extra > "$REPO/.output/domain/surge/extra.list" -rejects 'unmanifested artifact' verify +rejects 'artifact has no canonical audit source: domain/surge/extra.list' verify make_files; generate rm "$REPO/.output/ip/egern/base.yaml" -rejects 'manifest artifact missing' verify +rejects 'canonical counterpart missing: ip/egern/base.yaml' verify make_files; generate mkdir -p "$REPO/.output/domain/surge/nested" printf nested > "$REPO/.output/domain/surge/nested/file.list" @@ -159,4 +217,45 @@ rejects 'artifact binary/readability verification failed' verify make_files; generate rejects 'source commit mismatch' env ARTIFACT_SOURCE_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa "$REPO/scripts/commands/verify-artifact-manifest.sh" +make_files +rm "$REPO/.output/restoration-metadata.json" +ARTIFACT_BASELINE_FILE="$BASELINE_FILE" ARTIFACT_GENERATION_ID=offline-full ARTIFACT_BUILD_ID=build-full ARTIFACT_BUILD_SCOPE=full ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null +verify +python3 - "$REPO/.output/artifact-manifest.json" <<'PY' +import json, sys +manifest=json.load(open(sys.argv[1], encoding='utf-8')) +assert manifest['build_scope']=='full' +assert manifest['baseline']['generation_id']=='90-1' +assert manifest['restoration'] is None +PY + +INCONSISTENT_BASELINE_FILE="$REPO/inconsistent-publication-baseline.json" +python3 - "$BASELINE_FILE" "$INCONSISTENT_BASELINE_FILE" <<'PY' +import json, sys +payload=json.load(open(sys.argv[1], encoding='utf-8')) +payload['status']='inconsistent'; payload['generation_id']=None; payload['source_commit']=None +payload['branches']['surge']['generation_id']=None +payload['branches']['surge']['source_commit']=None +json.dump(payload, open(sys.argv[2], 'w', encoding='utf-8'), indent=2, sort_keys=True) +PY +make_files +rm "$REPO/.output/restoration-metadata.json" +ARTIFACT_BASELINE_FILE="$INCONSISTENT_BASELINE_FILE" ARTIFACT_GENERATION_ID=offline-recovery ARTIFACT_BUILD_ID=build-recovery ARTIFACT_BUILD_SCOPE=full ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null +verify +make_files +rejects 'custom build requires a consistent publication baseline' env ARTIFACT_BASELINE_FILE="$INCONSISTENT_BASELINE_FILE" ARTIFACT_GENERATION_ID=offline-custom ARTIFACT_BUILD_ID=build-custom ARTIFACT_BUILD_SCOPE=custom ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" + +STALE_BASELINE_FILE="$REPO/stale-publication-baseline.json" +python3 - "$BASELINE_FILE" "$STALE_BASELINE_FILE" "$SOURCE_SHA" <<'PY' +import json, sys +source, target, stale_source = sys.argv[1:] +payload=json.load(open(source, encoding='utf-8')) +payload['source_commit']=stale_source +for item in payload['branches'].values(): + item['source_commit']=stale_source +json.dump(payload, open(target, 'w', encoding='utf-8'), indent=2, sort_keys=True) +PY +make_files +rejects 'stale source refused' env ARTIFACT_BASELINE_FILE="$STALE_BASELINE_FILE" ARTIFACT_GENERATION_ID=offline-stale ARTIFACT_BUILD_ID=build-stale ARTIFACT_BUILD_SCOPE=custom ARTIFACT_SOURCE_SHA="$BASELINE_SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" + printf 'artifact manifest tests passed\n' diff --git a/scripts/tests/test-artifact-transaction-atomic.sh b/scripts/tests/test-artifact-transaction-atomic.sh index ee7bc3483..abc72a3d5 100644 --- a/scripts/tests/test-artifact-transaction-atomic.sh +++ b/scripts/tests/test-artifact-transaction-atomic.sh @@ -6,7 +6,10 @@ TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT REPO="$TMP/repo" mkdir -p "$REPO/scripts/commands" "$REPO/scripts/tools" "$REPO/.output" -cp "$ROOT/scripts/commands/build-artifacts-transaction.sh" "$REPO/scripts/commands/" +cp \ + "$ROOT/scripts/commands/build-artifacts-transaction.sh" \ + "$ROOT/scripts/commands/check-runtime.sh" \ + "$REPO/scripts/commands/" printf 'old live tree\n' > "$REPO/.output/sentinel" cat > "$REPO/scripts/commands/sync-upstream.sh" <<'EOF' @@ -36,23 +39,73 @@ open(a.output, 'w').write('{}\n') EOF chmod +x "$REPO/scripts/commands/"*.sh -snapshot() { find "$REPO/.output" -type f -print0 | sort -z | xargs -0 sha256sum; } +snapshot() { + python3 - "$REPO/.output" <<'PY' +import hashlib +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +for path in sorted(item for item in root.rglob("*") if item.is_file()): + digest = hashlib.sha256(path.read_bytes()).hexdigest() + print(f"{path.relative_to(root).as_posix()}\t{digest}") +PY +} assert_preserved() { local label="$1"; shift + local health_file snapshot > "$TMP/before" - if env "$@" "$REPO/scripts/commands/build-artifacts-transaction.sh" >/dev/null 2>"$TMP/$label.err"; then + if env ARTIFACT_GENERATION_ID="$label" "$@" \ + "$REPO/scripts/commands/build-artifacts-transaction.sh" \ + >/dev/null 2>"$TMP/$label.err"; then echo "expected transaction failure: $label" >&2; exit 1 fi snapshot > "$TMP/after" cmp -s "$TMP/before" "$TMP/after" || { echo "$label changed live output" >&2; exit 1; } - grep -R '"status":"failed"' "$REPO/.artifacts/diagnostics" >/dev/null + health_file="$(find "$REPO/.artifacts/diagnostics" -type f -path "*/${label}-*/transaction-health.json" | sort | tail -n 1)" + [ -n "$health_file" ] + python3 - "$health_file" <<'PY' +import json, sys +payload = json.load(open(sys.argv[1], encoding="utf-8")) +assert payload["status"] == "failed" +PY +} + +health_file_for() { + local generation="$1" + find "$REPO/.artifacts/diagnostics" -type f \ + -path "*/${generation}-*/transaction-health.json" | sort | tail -n 1 +} + +assert_health() { + local generation="$1" + local reason="$2" + local promotion_state="$3" + local rollback_status="$4" + local signal="${5:-}" + local health_file + health_file="$(health_file_for "$generation")" + [ -n "$health_file" ] + python3 - "$health_file" "$reason" "$promotion_state" "$rollback_status" "$signal" <<'PY' +import json, sys +path, reason, promotion_state, rollback_status, signal = sys.argv[1:] +payload = json.load(open(path, encoding="utf-8")) +assert payload["status"] == "failed" +assert payload["failure_reason"] == reason +assert payload["promotion_state"] == promotion_state +assert payload["rollback_status"] == rollback_status +if signal: + assert payload["signal"] == signal +else: + assert "signal" not in payload +PY } for point in late-domain late-ip late-compiler; do - assert_preserved "sync-$point" RULES_BUILD_SCOPE=full RULES_SYNC_FAIL_AT="$point" ARTIFACT_GENERATION_ID="$point" + assert_preserved "sync-$point" RULES_BUILD_SCOPE=full RULES_SYNC_FAIL_AT="$point" done for point in late-text late-binary; do - assert_preserved "restore-$point" RULES_BUILD_SCOPE=custom RULES_RESTORE_FAIL_AT="$point" ARTIFACT_GENERATION_ID="$point" + assert_preserved "restore-$point" RULES_BUILD_SCOPE=custom RULES_RESTORE_FAIL_AT="$point" done if RULES_ARTIFACT_ROOT="$TMP/caller-stage" "$REPO/scripts/commands/build-artifacts-transaction.sh" >"$TMP/root.out" 2>"$TMP/root.err"; then @@ -61,6 +114,90 @@ fi grep -F 'RULES_ARTIFACT_ROOT is transaction-owned' "$TMP/root.err" >/dev/null [ ! -e "$TMP/caller-stage" ] +NO_PYTHON_BIN="$TMP/no-python-bin" +mkdir -p "$NO_PYTHON_BIN" +ln -s "$(command -v bash)" "$NO_PYTHON_BIN/bash" +ln -s "$(command -v dirname)" "$NO_PYTHON_BIN/dirname" +if PATH="$NO_PYTHON_BIN" \ + "$REPO/scripts/commands/build-artifacts-transaction.sh" \ + >"$TMP/missing-python.out" 2>"$TMP/missing-python.err"; then + echo "transaction accepted a missing Python runtime" >&2 + exit 1 +fi +grep -F 'Python 3.11+ is required (python3 not found in PATH)' "$TMP/missing-python.err" >/dev/null +if grep -F 'cross-device' "$TMP/missing-python.err" >/dev/null; then + echo "missing Python runtime was misreported as a filesystem error" >&2 + exit 1 +fi + +assert_preserved \ + filesystem-preflight \ + RULES_TRANSACTION_TEST_DEVICE_MISMATCH=1 +assert_health filesystem-preflight filesystem-preflight-failed building not-required +grep -F 'cross-device artifact promotion refused before build' \ + "$TMP/filesystem-preflight.err" >/dev/null + +assert_preserved \ + backup-rename-failure \ + RULES_TRANSACTION_TEST_RENAME_FAILURES=backup:EIO +assert_health backup-rename-failure backup-rename-failed rolled-back succeeded +grep -F 'artifact transaction backup rename failed (strict rename refused)' \ + "$TMP/backup-rename-failure.err" >/dev/null + +assert_preserved \ + promotion-exdev \ + RULES_TRANSACTION_TEST_RENAME_FAILURES=promote:EXDEV +assert_health promotion-exdev promotion-rename-failed rolled-back succeeded +grep -F 'artifact transaction promote rename failed (cross-device rename refused)' \ + "$TMP/promotion-exdev.err" >/dev/null + +for signal in HUP INT TERM; do + label="signal-${signal,,}" + case "$signal" in + HUP) expected_status=129 ;; + INT) expected_status=130 ;; + TERM) expected_status=143 ;; + esac + snapshot > "$TMP/$label.before" + if env \ + ARTIFACT_GENERATION_ID="$label" \ + RULES_TRANSACTION_TEST_SIGNAL_AFTER_BACKUP="$signal" \ + "$REPO/scripts/commands/build-artifacts-transaction.sh" \ + >"$TMP/$label.out" 2>"$TMP/$label.err"; then + echo "expected transaction signal failure: $signal" >&2 + exit 1 + else + actual_status=$? + fi + [ "$actual_status" -eq "$expected_status" ] + snapshot > "$TMP/$label.after" + cmp -s "$TMP/$label.before" "$TMP/$label.after" + assert_health "$label" signal rolled-back succeeded "$signal" + grep -F "artifact transaction received $signal signal" "$TMP/$label.err" >/dev/null +done + +snapshot > "$TMP/rollback-restore.before" +if env \ + ARTIFACT_GENERATION_ID=rollback-restore-failure \ + RULES_TRANSACTION_TEST_RENAME_FAILURES=promote:EIO,rollback-restore:EIO \ + "$REPO/scripts/commands/build-artifacts-transaction.sh" \ + >"$TMP/rollback-restore.out" 2>"$TMP/rollback-restore.err"; then + echo "expected rollback restore failure" >&2 + exit 1 +fi +[ ! -e "$REPO/.output" ] +assert_health rollback-restore-failure promotion-rename-failed promotion-pending failed +recovery_root="$(sed -n 's/^transaction recovery data preserved in //p' "$TMP/rollback-restore.err" | tail -n 1)" +[ -n "$recovery_root" ] +grep -Fx 'old live tree' "$recovery_root/previous-output/sentinel" >/dev/null +python3 - "$recovery_root/previous-output" "$REPO/.output" <<'PY' +import os, sys +os.replace(sys.argv[1], sys.argv[2]) +PY +rm -rf "$recovery_root" +snapshot > "$TMP/rollback-restore.after" +cmp -s "$TMP/rollback-restore.before" "$TMP/rollback-restore.after" + LIVE_OVERRIDE="$TMP/live-override" RULES_BUILD_SCOPE=full RULES_LIVE_ARTIFACT_ROOT="$LIVE_OVERRIDE" "$REPO/scripts/commands/build-artifacts-transaction.sh" >/dev/null [ -f "$LIVE_OVERRIDE/new" ] diff --git a/scripts/tests/test-binary-artifact-verifier.sh b/scripts/tests/test-binary-artifact-verifier.sh index e479d7a4c..b8fdeafb2 100644 --- a/scripts/tests/test-binary-artifact-verifier.sh +++ b/scripts/tests/test-binary-artifact-verifier.sh @@ -8,7 +8,7 @@ python3 -m py_compile "$ROOT/scripts/tools/artifact_verifier.py" mkdir -p "$TMP/text-repo/config" cp "$ROOT/config/domain-platform-capabilities.json" "$TMP/text-repo/config/" printf 'DOMAIN,example.com\n' > "$TMP/good.list" -PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/good.list" --type domain --platform surge >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/good.list" --type domain --platform surge --syntax-only >/dev/null printf 'UNKNOWN,example.com\n' > "$TMP/bad-kind.list" printf 'DOMAIN,example.com,extra\n' > "$TMP/bad-shape.list" printf "domain_set:\n - 'example.com'\n" > "$TMP/good.yaml" @@ -17,9 +17,9 @@ printf 'HOST-SUFFIX,example.com,quanx\n' > "$TMP/quanx.list" printf 'HOST-SUFFIX,example.com,reject\n' > "$TMP/quanx-bad.list" printf 'IP-CIDR,192.0.2.0/24,quanx-ip\n' > "$TMP/quanx-ip.list" printf 'IP-CIDR,192.0.2.0/24,reject\n' > "$TMP/quanx-ip-bad.list" -PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/good.yaml" --type domain --platform egern >/dev/null -PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/quanx.list" --type domain --platform quanx >/dev/null -PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/quanx-ip.list" --type ip --platform quanx >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/good.yaml" --type domain --platform egern --syntax-only >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/quanx.list" --type domain --platform quanx --syntax-only >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/quanx-ip.list" --type ip --platform quanx --syntax-only >/dev/null for fixture in bad-kind.list bad-shape.list; do if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/$fixture" --type domain --platform surge >/dev/null 2>&1; then echo "text verifier accepted $fixture" >&2; exit 1 @@ -32,7 +32,7 @@ for fixture in quanx-bad.list quanx-ip-bad.list; do artifact_type=domain [[ "$fixture" == quanx-ip-* ]] && artifact_type=ip if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ - --root "$TMP/text-repo" --path "$TMP/$fixture" --type "$artifact_type" --platform quanx >/dev/null 2>&1; then + --root "$TMP/text-repo" --path "$TMP/$fixture" --type "$artifact_type" --platform quanx --syntax-only >/dev/null 2>&1; then echo "Quantumult X verifier accepted wrong policy in $fixture" >&2 exit 1 fi @@ -85,9 +85,11 @@ mkdir -p \ "$TMP/exact-repo/.bin" \ "$TMP/exact-repo/.output/domain/sing-box" \ "$TMP/exact-repo/.output/domain/mihomo" \ + "$TMP/exact-repo/.output/.canonical/domain" \ "$TMP/exact-repo/sources/custom/domain" cp "$ROOT/config/domain-platform-capabilities.json" "$TMP/exact-repo/config/" printf 'DOMAIN,fixture.example\nDOMAIN-SUFFIX,example.org\n' > "$TMP/exact-repo/sources/custom/domain/fixture.list" +cp "$TMP/exact-repo/sources/custom/domain/fixture.list" "$TMP/exact-repo/.output/.canonical/domain/fixture.list" printf 'fixture\n' > "$TMP/exact-repo/.output/domain/sing-box/fixture.srs" printf 'fixture\n' > "$TMP/exact-repo/.output/domain/mihomo/fixture.mrs" cat > "$TMP/exact-repo/.bin/sing-box" <<'EOF' @@ -139,6 +141,7 @@ fi printf 'DOMAIN-SUFFIX,example.com\nDOMAIN-SUFFIX,child.example.com\nDOMAIN,api.example.com\n' \ > "$TMP/exact-repo/sources/custom/domain/reduction.list" +cp "$TMP/exact-repo/sources/custom/domain/reduction.list" "$TMP/exact-repo/.output/.canonical/domain/reduction.list" printf 'reduction\n' > "$TMP/exact-repo/.output/domain/sing-box/reduction.srs" cat > "$TMP/exact-repo/.output/domain/sing-box/reduction.srs.json" <<'EOF' {"rules":[{"domain_suffix":["example.com"]}]} @@ -180,16 +183,25 @@ sing-box rule-set compile "$TMP/domain.json" --output "$TMP/domain.srs" printf 'fixture.example\n+.example.org\n' > "$TMP/domain.txt" mihomo convert-ruleset domain text "$TMP/domain.txt" "$TMP/domain.mrs" >/dev/null -mkdir -p "$TMP/repo/config" "$TMP/repo/.bin" "$TMP/repo/sources/custom/domain" +mkdir -p \ + "$TMP/repo/config" \ + "$TMP/repo/.bin" \ + "$TMP/repo/sources/custom/domain" \ + "$TMP/repo/.output/.canonical/domain" \ + "$TMP/repo/.output/domain/sing-box" \ + "$TMP/repo/.output/domain/mihomo" cp "$ROOT/config/domain-platform-capabilities.json" "$TMP/repo/config/" cp "$TMP/domain.list" "$TMP/repo/sources/custom/domain/fixture.list" +cp "$TMP/domain.list" "$TMP/repo/.output/.canonical/domain/fixture.list" +cp "$TMP/domain.srs" "$TMP/repo/.output/domain/sing-box/fixture.srs" +cp "$TMP/domain.mrs" "$TMP/repo/.output/domain/mihomo/fixture.mrs" ln -s "$(command -v sing-box)" "$TMP/repo/.bin/sing-box" ln -s "$(command -v mihomo)" "$TMP/repo/.bin/mihomo" -PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/repo" --path "$TMP/domain.srs" --type domain --platform sing-box | grep -F '"status": "verified"' >/dev/null -PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/repo" --path "$TMP/domain.mrs" --type domain --platform mihomo | grep -F '"status": "verified"' >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/repo" --path "$TMP/repo/.output/domain/sing-box/fixture.srs" --type domain --platform sing-box | grep -F '"status": "verified"' >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/repo" --path "$TMP/repo/.output/domain/mihomo/fixture.mrs" --type domain --platform mihomo | grep -F '"status": "verified"' >/dev/null printf CORRUPT > "$TMP/corrupt.srs" -if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/repo" --path "$TMP/corrupt.srs" --type domain --platform sing-box >/dev/null 2>&1; then +if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/repo" --path "$TMP/corrupt.srs" --type domain --platform sing-box --syntax-only >/dev/null 2>&1; then echo "sing-box accepted corrupted fixture" >&2; exit 1 fi printf 'binary artifact verifier real-tool fixtures passed\n' diff --git a/scripts/tests/test-build-scope.sh b/scripts/tests/test-build-scope.sh index 5f8f8cb7a..8059428f9 100755 --- a/scripts/tests/test-build-scope.sh +++ b/scripts/tests/test-build-scope.sh @@ -3,6 +3,67 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" SCRIPT="$ROOT/scripts/commands/select-build-scope.sh" +TEST_TMP="$(mktemp -d)" +trap 'rm -rf "$TEST_TMP"' EXIT +SCOPE_REPO="$TEST_TMP/scope-repo" +BASELINE_INPUT="$TEST_TMP/publication-baseline.json" +BASELINE_AT_MIDDLE_INPUT="$TEST_TMP/publication-baseline-at-middle.json" +INCONSISTENT_BASELINE_INPUT="$TEST_TMP/publication-baseline-inconsistent.json" +BASELINE_OUTPUT="$TEST_TMP/selected-publication-baseline.json" + +write_baseline() { + local path="$1" + local source="$2" + python3 - "$path" "$source" <<'PY' +import json +import sys +from pathlib import Path + +path, source = sys.argv[1:] +branches = ("surge", "quanx", "egern", "sing-box", "mihomo") +payload = { + "status": "consistent", + "generation_id": "10-1", + "source_commit": source, + "branches": { + branch: { + "commit": f"{index + 1:040x}", + "generation_id": "10-1", + "source_commit": source, + } + for index, branch in enumerate(branches) + }, +} +Path(path).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY +} + +mkdir -p "$SCOPE_REPO" +git -C "$SCOPE_REPO" init -q +git -C "$SCOPE_REPO" config user.name test +git -C "$SCOPE_REPO" config user.email test@example.com +printf 'base\n' > "$SCOPE_REPO/README.md" +git -C "$SCOPE_REPO" add README.md +git -C "$SCOPE_REPO" commit -m baseline >/dev/null +PUBLISHED_SOURCE="$(git -C "$SCOPE_REPO" rev-parse HEAD)" +write_baseline "$BASELINE_INPUT" "$PUBLISHED_SOURCE" +mkdir -p "$SCOPE_REPO/scripts/commands" +printf '# change requiring full\n' > "$SCOPE_REPO/scripts/commands/example.sh" +git -C "$SCOPE_REPO" add scripts/commands/example.sh +git -C "$SCOPE_REPO" commit -m full-change >/dev/null +MIDDLE_SOURCE="$(git -C "$SCOPE_REPO" rev-parse HEAD)" +write_baseline "$BASELINE_AT_MIDDLE_INPUT" "$MIDDLE_SOURCE" +mkdir -p "$SCOPE_REPO/sources/custom/domain" +printf 'DOMAIN,custom.example\n' > "$SCOPE_REPO/sources/custom/domain/custom.list" +git -C "$SCOPE_REPO" add sources/custom/domain/custom.list +git -C "$SCOPE_REPO" commit -m custom-change >/dev/null +python3 - "$BASELINE_AT_MIDDLE_INPUT" "$INCONSISTENT_BASELINE_INPUT" <<'PY' +import json, sys +payload=json.load(open(sys.argv[1], encoding='utf-8')) +payload['status']='inconsistent'; payload['generation_id']=None; payload['source_commit']=None +payload['branches']['surge']['generation_id']='9-1' +json.dump(payload, open(sys.argv[2], 'w', encoding='utf-8'), indent=2, sort_keys=True) +PY assert_scope() { local expected_scope="$1" @@ -10,7 +71,14 @@ assert_scope() { shift 2 local output - output="$(env "$@" "$SCRIPT")" + output="$( + cd "$SCOPE_REPO" + env \ + ARTIFACT_BASELINE_FILE="$BASELINE_OUTPUT" \ + RULES_PUBLICATION_BASELINE_INPUT="$BASELINE_INPUT" \ + "$@" \ + "$SCRIPT" + )" grep -Fx "scope=$expected_scope" <<< "$output" >/dev/null || { echo "expected scope=$expected_scope" >&2 echo "$output" >&2 @@ -48,8 +116,30 @@ assert_scope full "custom deletions require full sync" \ CHANGED_FILES=$'sources/custom/domain/emby.list' \ DELETED_CUSTOM_FILES=$'sources/custom/domain/old.list' -assert_scope full "push base unavailable; using full sync" \ - EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA=missing-build-scope-base +assert_scope full "push includes changes outside custom sources" \ + EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA="$MIDDLE_SOURCE" + +assert_scope custom "push only updates custom sources" \ + RULES_PUBLICATION_BASELINE_INPUT="$BASELINE_AT_MIDDLE_INPUT" \ + EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA="$MIDDLE_SOURCE" + +assert_scope full "publication cohort inconsistent; using full sync" \ + RULES_PUBLICATION_BASELINE_INPUT="$INCONSISTENT_BASELINE_INPUT" \ + EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA="$MIDDLE_SOURCE" \ + CHANGED_FILES=$'sources/custom/domain/custom.list' + +set +e +manual_cumulative_output="$( + cd "$SCOPE_REPO" + env ARTIFACT_BASELINE_FILE="$BASELINE_OUTPUT" \ + RULES_PUBLICATION_BASELINE_INPUT="$BASELINE_INPUT" \ + EVENT_NAME=workflow_dispatch INPUT_SCOPE=custom CURRENT_SHA=HEAD \ + "$SCRIPT" 2>&1 +)" +manual_cumulative_status=$? +set -e +[ "$manual_cumulative_status" -ne 0 ] +grep -F "manual custom scope refused: delta contains non-custom paths" <<< "$manual_cumulative_output" >/dev/null assert_scope none "pull_request has no build-relevant changes" \ EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ @@ -79,8 +169,6 @@ assert_scope full "pull_request includes changes outside custom sources" \ assert_scope full "pull_request base unavailable; using full sync" \ EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=missing-build-scope-base -TEST_TMP="$(mktemp -d)" -trap 'rm -rf "$TEST_TMP"' EXIT ROOT_COMMIT_REPO="$TEST_TMP/root-commit" mkdir -p "$ROOT_COMMIT_REPO" git -C "$ROOT_COMMIT_REPO" init -q @@ -89,10 +177,15 @@ git -C "$ROOT_COMMIT_REPO" config user.email test@example.com printf 'root\n' > "$ROOT_COMMIT_REPO/root.txt" git -C "$ROOT_COMMIT_REPO" add root.txt git -C "$ROOT_COMMIT_REPO" commit -m root >/dev/null -root_output="$(cd "$ROOT_COMMIT_REPO" && env EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA=0000000000000000000000000000000000000000 "$SCRIPT")" -grep -Fx "scope=full" <<< "$root_output" >/dev/null -grep -Fx "reason=push base unavailable; using full sync" <<< "$root_output" >/dev/null -grep -Fx "base_sha=" <<< "$root_output" >/dev/null +pr_no_remote_output="$(cd "$ROOT_COMMIT_REPO" && env EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base CHANGED_FILES=README.md "$SCRIPT")" +grep -Fx "scope=none" <<< "$pr_no_remote_output" >/dev/null +grep -Fx "reason=pull_request has no build-relevant changes" <<< "$pr_no_remote_output" >/dev/null +set +e +root_output="$(cd "$ROOT_COMMIT_REPO" && env EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA=0000000000000000000000000000000000000000 "$SCRIPT" 2>&1)" +root_status=$? +set -e +[ "$root_status" -ne 0 ] +grep -F "required remote publication branch origin/surge is unavailable" <<< "$root_output" >/dev/null RENAME_REPO="$TEST_TMP/rename" mkdir -p "$RENAME_REPO/sources/custom/domain" @@ -103,9 +196,11 @@ printf 'DOMAIN,old.example\n' > "$RENAME_REPO/sources/custom/domain/old.list" git -C "$RENAME_REPO" add sources/custom/domain/old.list git -C "$RENAME_REPO" commit -m base >/dev/null rename_base="$(git -C "$RENAME_REPO" rev-parse HEAD)" +RENAME_BASELINE_INPUT="$TEST_TMP/rename-publication-baseline.json" +write_baseline "$RENAME_BASELINE_INPUT" "$rename_base" git -C "$RENAME_REPO" mv sources/custom/domain/old.list sources/custom/domain/new.list git -C "$RENAME_REPO" commit -m rename >/dev/null -rename_output="$(cd "$RENAME_REPO" && env EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA="$rename_base" "$SCRIPT")" +rename_output="$(cd "$RENAME_REPO" && env ARTIFACT_BASELINE_FILE="$BASELINE_OUTPUT" RULES_PUBLICATION_BASELINE_INPUT="$RENAME_BASELINE_INPUT" EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA="$rename_base" "$SCRIPT")" grep -Fx "scope=full" <<< "$rename_output" >/dev/null grep -Fx "reason=custom deletions require full sync" <<< "$rename_output" >/dev/null diff --git a/scripts/tests/test-custom-build-atomic.sh b/scripts/tests/test-custom-build-atomic.sh index b523c113f..09d35623d 100644 --- a/scripts/tests/test-custom-build-atomic.sh +++ b/scripts/tests/test-custom-build-atomic.sh @@ -94,6 +94,7 @@ printf 'fake-mrs\n' > "$out" EOF chmod +x "$REPO/.bin/sing-box" "$REPO/.bin/mihomo" assert_injected_failure_preserves_output late-binary +assert_injected_failure_preserves_output canonical-commit # A successful text-only commit updates only controlled text targets. Binary # targets and unrelated restored/upstream files remain untouched. diff --git a/scripts/tests/test-publish-branches.sh b/scripts/tests/test-publish-branches.sh index 3c311739b..28fd980fc 100755 --- a/scripts/tests/test-publish-branches.sh +++ b/scripts/tests/test-publish-branches.sh @@ -85,6 +85,11 @@ printf 'srs-ip\n' > "$REPO/.output/ip/sing-box/test.srs" printf 'mrs-domain\n' > "$REPO/.output/domain/mihomo/test.mrs" printf 'mrs-ip\n' > "$REPO/.output/ip/mihomo/test.mrs" +python3 "$REPO/scripts/tools/artifact_verifier.py" \ + --root "$REPO" \ + --seed-canonical-from "$REPO/.output" \ + --canonical-output "$REPO/.output/.canonical" + cat > "$REPO/.bin/sing-box" <<'EOF' #!/usr/bin/env bash set -eu @@ -146,9 +151,10 @@ for tool in ("sing-box", "mihomo"): lock_path.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8") origins = {} -for path in (root / ".output").glob("*/*/*"): - if path.is_file(): - origins[path.relative_to(root / ".output").as_posix()] = "generated-upstream" +for section in ("domain", "ip"): + for path in (root / ".output" / section).glob("*/*"): + if path.is_file(): + origins[path.relative_to(root / ".output").as_posix()] = "generated-upstream" (root / ".output/artifact-origins.json").write_text( json.dumps(origins, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) @@ -159,7 +165,67 @@ git -C "$REPO" init -q git -C "$REPO" remote add origin "$REMOTE" git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 SOURCE_SHA="$(git -C "$SEED" rev-parse HEAD)" -ARTIFACT_GENERATION_ID=test-generation ARTIFACT_BUILD_SCOPE=full ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null +BASELINE_FILE="$REPO/.tmp/publication-baseline.json" +BRANCH_SEED="$TMP_DIR/branch-seed" +git clone -q "$REMOTE" "$BRANCH_SEED" +git -C "$BRANCH_SEED" config user.name test +git -C "$BRANCH_SEED" config user.email test@example.com + +for branch in "${BRANCHES[@]}"; do + git -C "$BRANCH_SEED" checkout --orphan "$branch" >/dev/null 2>&1 + git -C "$BRANCH_SEED" rm -rf . >/dev/null 2>&1 || true + find "$BRANCH_SEED" -mindepth 1 -maxdepth 1 ! -name .git -exec rm -rf {} + + mkdir -p "$BRANCH_SEED/domain" "$BRANCH_SEED/ip" + cp -R "$REPO/.output/domain/$branch/." "$BRANCH_SEED/domain/" + cp -R "$REPO/.output/ip/$branch/." "$BRANCH_SEED/ip/" + printf 'seed %s\n' "$branch" > "$BRANCH_SEED/README.md" + git -C "$BRANCH_SEED" add README.md domain ip + git -C "$BRANCH_SEED" commit -m "chore: publish ${branch} artifacts [generation 100-1 source ${SOURCE_SHA}]" >/dev/null + git -C "$BRANCH_SEED" push origin "$branch" >/dev/null 2>&1 +done + +write_remote_baseline() { + mkdir -p "$(dirname "$BASELINE_FILE")" + python3 - "$REMOTE" "$BASELINE_FILE" <<'PY' +import json +import re +import subprocess +import sys +from pathlib import Path + +remote, output = sys.argv[1:] +branches = ("surge", "quanx", "egern", "sing-box", "mihomo") +pattern = re.compile(r"^chore: publish ([^ ]+) artifacts \[generation ([0-9]+-[0-9]+) source ([0-9a-f]{40})\]$") +items = {} +identities = set() +for branch in branches: + commit = subprocess.check_output(["git", f"--git-dir={remote}", "rev-parse", branch], text=True).strip() + subject = subprocess.check_output(["git", f"--git-dir={remote}", "log", "-1", "--format=%s", branch], text=True).strip() + match = pattern.fullmatch(subject) + generation = match.group(2) if match and match.group(1) == branch else None + source = match.group(3) if match and match.group(1) == branch else None + if generation and source: + identities.add((generation, source)) + items[branch] = {"commit": commit, "generation_id": generation, "source_commit": source} +consistent = len(identities) == 1 and all(item["generation_id"] for item in items.values()) +generation, source = next(iter(identities)) if consistent else (None, None) +payload = {"status": "consistent" if consistent else "inconsistent", + "generation_id": generation, "source_commit": source, "branches": items} +Path(output).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") +PY +} + +generate_manifest() { + local generation="$1" + git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 + write_remote_baseline + ARTIFACT_BASELINE_FILE="$BASELINE_FILE" ARTIFACT_GENERATION_ID="$generation" \ + ARTIFACT_BUILD_SCOPE=full ARTIFACT_SOURCE_SHA="$SOURCE_SHA" \ + "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null +} + +generate_manifest 101-1 +cp "$REPO/.output/artifact-manifest.json" "$TMP_DIR/old-artifact-manifest.json" first_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" grep -F "publishing branches atomically: surge quanx egern sing-box mihomo" <<< "$first_output" >/dev/null @@ -199,12 +265,46 @@ git --git-dir="$REMOTE" show surge:domain/test.list | grep -Fx 'DOMAIN-SUFFIX,ex git --git-dir="$REMOTE" show quanx:ip/test.list | grep -Fx 'IP-CIDR,192.0.2.0/24,test' >/dev/null before_idempotent="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" -git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 +generate_manifest 102-1 second_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" after_idempotent="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" [ "$before_idempotent" = "$after_idempotent" ] grep -F "all publish branches unchanged, skip push" <<< "$second_output" >/dev/null +# Race an artifact ref during the second cohort refresh on the all-unchanged +# path. The script must reject the stale baseline instead of returning success. +generate_manifest 102-2 +mkdir -p "$TMP_DIR/skip-race-bin" +cat > "$TMP_DIR/skip-race-bin/git" <<'EOF' +#!/usr/bin/env bash +set -e +if [[ " $* " == *" fetch "* && " $* " == *"+refs/heads/surge:refs/remotes/origin/surge"* ]]; then + count=0 + [ ! -f "$SKIP_RACE_COUNT" ] || count="$(cat "$SKIP_RACE_COUNT")" + count=$((count + 1)) + printf '%s\n' "$count" > "$SKIP_RACE_COUNT" + if [ "$count" -eq 2 ]; then + "$REAL_GIT" clone -q --branch surge "$RACE_REMOTE" "$SKIP_RACE_REPO" + "$REAL_GIT" -C "$SKIP_RACE_REPO" config user.name racer + "$REAL_GIT" -C "$SKIP_RACE_REPO" config user.email racer@example.com + "$REAL_GIT" -C "$SKIP_RACE_REPO" commit --allow-empty -m skip-race >/dev/null + "$REAL_GIT" -C "$SKIP_RACE_REPO" push origin surge >/dev/null 2>&1 + fi +fi +exec "$REAL_GIT" "$@" +EOF +chmod +x "$TMP_DIR/skip-race-bin/git" +before_skip_race="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" +set +e +skip_race_output="$(PATH="$TMP_DIR/skip-race-bin:$PATH" REAL_GIT="$REAL_GIT" SKIP_RACE_COUNT="$TMP_DIR/skip-race-count" RACE_REMOTE="$REMOTE" SKIP_RACE_REPO="$TMP_DIR/skip-racer" ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +skip_race_status=$? +set -e +[ "$skip_race_status" -ne 0 ] +grep -F 'publication baseline is stale' <<< "$skip_race_output" >/dev/null +after_skip_race="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" +[ "$(printf '%s\n' "$before_skip_race" | tail -n +2)" = "$(printf '%s\n' "$after_skip_race" | tail -n +2)" ] +[ "$(printf '%s\n' "$before_skip_race" | head -n 1)" != "$(printf '%s\n' "$after_skip_race" | head -n 1)" ] + # A change to one platform must advance the complete publication cohort. The # four unchanged trees receive metadata-only commits so a later custom restore # observes one generation/source identity across all five branches. @@ -215,8 +315,7 @@ for branch in "${BRANCHES[@]}"; do done cp "$REPO/templates/branch-readmes/surge.md" "$REPO/templates/branch-readmes/surge.md.partial-original" printf '\npartial cohort fixture\n' >> "$REPO/templates/branch-readmes/surge.md" -git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 -ARTIFACT_GENERATION_ID=test-generation-partial ARTIFACT_BUILD_SCOPE=full ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null +generate_manifest 103-1 partial_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" grep -F "publishing branches atomically: surge quanx egern sing-box mihomo" <<< "$partial_output" >/dev/null @@ -233,7 +332,7 @@ for branch in "${BRANCHES[@]}"; do echo "test failed: $branch publication commit did not preserve parent history" >&2 exit 1 } - [ "$subject" = "chore: publish ${branch} artifacts [generation test-generation-partial source ${SOURCE_SHA}]" ] || { + [ "$subject" = "chore: publish ${branch} artifacts [generation 103-1 source ${SOURCE_SHA}]" ] || { echo "test failed: $branch publication identity differs from partial cohort" >&2 exit 1 } @@ -251,7 +350,28 @@ for branch in "${BRANCHES[@]}"; do done mv "$REPO/templates/branch-readmes/surge.md.partial-original" "$REPO/templates/branch-readmes/surge.md" +# Replaying an artifact manifest built against an older cohort must fail even +# though every new commit would otherwise be a fast-forward artifact update. +cp "$TMP_DIR/old-artifact-manifest.json" "$REPO/.output/artifact-manifest.json" +set +e +replay_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +replay_status=$? +set -e +[ "$replay_status" -ne 0 ] +grep -F 'publication baseline is stale' <<< "$replay_output" >/dev/null + +# A fresh build may use the current cohort as its baseline, but its generation +# still has to advance monotonically for the same source commit. +generate_manifest 103-1 +set +e +stale_generation_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +stale_generation_status=$? +set -e +[ "$stale_generation_status" -ne 0 ] +grep -F 'stale publication generation refused' <<< "$stale_generation_output" >/dev/null + # A missing template must fail before the queued batch is pushed. +generate_manifest 104-1 before_missing_template="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" cp "$REPO/templates/branch-readmes/surge.md" "$REPO/templates/branch-readmes/surge.md.original" printf '\nqueued template change\n' >> "$REPO/templates/branch-readmes/surge.md" @@ -310,8 +430,65 @@ set -e } grep -F 'artifact hash mismatch' <<< "$unverified_publish_output" >/dev/null -git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 -ARTIFACT_GENERATION_ID=test-generation-2 ARTIFACT_BUILD_SCOPE=full ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null +rm -rf "$REPO/.output/.canonical" +python3 "$REPO/scripts/tools/artifact_verifier.py" \ + --root "$REPO" \ + --seed-canonical-from "$REPO/.output" \ + --canonical-output "$REPO/.output/.canonical" +generate_manifest 105-1 +printf 'new main tip\n' >> "$SEED/README.md" +git -C "$SEED" add README.md +git -C "$SEED" commit -m main-advanced >/dev/null +git -C "$SEED" push origin main >/dev/null 2>&1 +NEW_SOURCE_SHA="$(git -C "$SEED" rev-parse HEAD)" +set +e +stale_main_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +stale_main_status=$? +set -e +[ "$stale_main_status" -ne 0 ] +grep -F 'stale publication source refused: remote main is' <<< "$stale_main_output" >/dev/null +SOURCE_SHA="$NEW_SOURCE_SHA" +generate_manifest 106-1 + +# Move main from inside the git-push wrapper, after both pre-push checks but +# before the real atomic artifact push. The push succeeds, then the mandatory +# post-push source check fails the run and requests a queued roll-forward. +mkdir -p "$TMP_DIR/main-race-bin" +cat > "$TMP_DIR/main-race-bin/git" <<'EOF' +#!/usr/bin/env bash +set -e +if [ "${1:-}" = push ] && [ "${2:-}" = --atomic ] && [ ! -e "$MAIN_RACE_MARKER" ]; then + : > "$MAIN_RACE_MARKER" + "$REAL_GIT" clone -q --branch main "$RACE_REMOTE" "$MAIN_RACE_REPO" + "$REAL_GIT" -C "$MAIN_RACE_REPO" config user.name racer + "$REAL_GIT" -C "$MAIN_RACE_REPO" config user.email racer@example.com + printf '\npost-push main race\n' >> "$MAIN_RACE_REPO/README.md" + "$REAL_GIT" -C "$MAIN_RACE_REPO" add README.md + "$REAL_GIT" -C "$MAIN_RACE_REPO" commit -m main-race >/dev/null + "$REAL_GIT" -C "$MAIN_RACE_REPO" push origin main >/dev/null 2>&1 +fi +exec "$REAL_GIT" "$@" +EOF +chmod +x "$TMP_DIR/main-race-bin/git" +before_post_push_race="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" +set +e +post_push_race_output="$(PATH="$TMP_DIR/main-race-bin:$PATH" REAL_GIT="$REAL_GIT" MAIN_RACE_MARKER="$TMP_DIR/main-raced" RACE_REMOTE="$REMOTE" MAIN_RACE_REPO="$TMP_DIR/main-racer" ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +post_push_race_status=$? +set -e +[ "$post_push_race_status" -ne 0 ] +[ -e "$TMP_DIR/main-raced" ] +grep -F 'artifact cohort was published while main advanced' <<< "$post_push_race_output" >/dev/null +after_post_push_race="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" +[ "$before_post_push_race" != "$after_post_push_race" ] +for branch in "${BRANCHES[@]}"; do + subject="$(git --git-dir="$REMOTE" log -1 --format=%s "$branch")" + [ "$subject" = "chore: publish ${branch} artifacts [generation 106-1 source ${SOURCE_SHA}]" ] +done + +SOURCE_SHA="$(git --git-dir="$REMOTE" rev-parse refs/heads/main)" +generate_manifest 107-1 +cp "$REPO/templates/branch-readmes/surge.md" "$REPO/templates/branch-readmes/surge.md.race-original" +printf '\nlease race fixture\n' >> "$REPO/templates/branch-readmes/surge.md" before_race="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" mkdir -p "$TMP_DIR/bin" cat > "$TMP_DIR/bin/git" <<'EOF' @@ -350,5 +527,47 @@ after_race_without_surge="$(printf '%s\n' "$after_race" | tail -n +2)" } [ "$(printf '%s\n' "$after_race" | head -n 1)" != "$(printf '%s\n' "$before_race" | head -n 1)" ] git --git-dir="$REMOTE" show surge:README.md | grep -F 'racing update' >/dev/null +mv "$REPO/templates/branch-readmes/surge.md.race-original" "$REPO/templates/branch-readmes/surge.md" + +# A full build can bind the exact five current commits even when their +# generation/source metadata is split, then repair the cohort atomically. +generate_manifest 108-1 +python3 - "$REPO/.output/artifact-manifest.json" <<'PY' +import json, sys +manifest=json.load(open(sys.argv[1], encoding='utf-8')) +assert manifest['baseline']['status']=='inconsistent' +PY +recovery_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +grep -F "publishing branches atomically: surge quanx egern sing-box mihomo" <<< "$recovery_output" >/dev/null +for branch in "${BRANCHES[@]}"; do + subject="$(git --git-dir="$REMOTE" log -1 --format=%s "$branch")" + [ "$subject" = "chore: publish ${branch} artifacts [generation 108-1 source ${SOURCE_SHA}]" ] || { + echo "test failed: full recovery did not repair $branch metadata" >&2 + exit 1 + } +done + +# Invalid metadata with unchanged artifact trees must still force a metadata +# cohort publication; otherwise an all-unchanged full build could never heal it. +surge_parent="$(git --git-dir="$REMOTE" rev-parse surge)" +surge_tree="$(git --git-dir="$REMOTE" rev-parse 'surge^{tree}')" +surge_invalid="$(printf 'manual metadata drift\n' | \ + GIT_AUTHOR_NAME=test GIT_AUTHOR_EMAIL=test@example.com \ + GIT_COMMITTER_NAME=test GIT_COMMITTER_EMAIL=test@example.com \ + git --git-dir="$REMOTE" commit-tree "$surge_tree" -p "$surge_parent")" +git --git-dir="$REMOTE" update-ref refs/heads/surge "$surge_invalid" "$surge_parent" +before_metadata_recovery="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" +generate_manifest 109-1 +metadata_recovery_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +grep -F "publishing branches atomically: surge quanx egern sing-box mihomo" <<< "$metadata_recovery_output" >/dev/null +after_metadata_recovery="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" +[ "$before_metadata_recovery" != "$after_metadata_recovery" ] +for branch in "${BRANCHES[@]}"; do + subject="$(git --git-dir="$REMOTE" log -1 --format=%s "$branch")" + [ "$subject" = "chore: publish ${branch} artifacts [generation 109-1 source ${SOURCE_SHA}]" ] || { + echo "test failed: unchanged-tree recovery did not advance $branch metadata" >&2 + exit 1 + } +done printf 'publish branch tests passed\n' diff --git a/scripts/tests/test-runtime.sh b/scripts/tests/test-runtime.sh index a3467544f..fd15b09cf 100644 --- a/scripts/tests/test-runtime.sh +++ b/scripts/tests/test-runtime.sh @@ -4,14 +4,47 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" TMP_DIR="$(mktemp -d)" trap 'rm -rf "$TMP_DIR"' EXIT +CHECK_RUNTIME="$ROOT/scripts/commands/check-runtime.sh" -make -s -C "$ROOT" check-runtime BASH_MIN_MAJOR=1 +make -s -C "$ROOT" check-runtime -if make -s -C "$ROOT" check-runtime BASH_MIN_MAJOR=999 \ +if RULES_RUNTIME_TEST_BASH_MIN_MAJOR=999 "$CHECK_RUNTIME" \ >"$TMP_DIR/unsupported.stdout" 2>"$TMP_DIR/unsupported.stderr"; then echo "test failed: unsupported Bash runtime was accepted" >&2 exit 1 fi grep -F 'Bash 999+ is required' "$TMP_DIR/unsupported.stderr" >/dev/null +if RULES_RUNTIME_TEST_PYTHON_MIN_MAJOR=999 "$CHECK_RUNTIME" \ + >"$TMP_DIR/unsupported-python.stdout" 2>"$TMP_DIR/unsupported-python.stderr"; then + echo "test failed: unsupported Python runtime was accepted" >&2 + exit 1 +fi +grep -F 'Python 999.11+ is required' "$TMP_DIR/unsupported-python.stderr" >/dev/null + +mkdir -p "$TMP_DIR/no-python" +if PATH="$TMP_DIR/no-python" "$(command -v bash)" "$CHECK_RUNTIME" \ + >"$TMP_DIR/missing-python.stdout" 2>"$TMP_DIR/missing-python.stderr"; then + echo "test failed: missing Python runtime was accepted" >&2 + exit 1 +fi +grep -F 'Python 3.11+ is required (python3 not found in PATH)' "$TMP_DIR/missing-python.stderr" >/dev/null + +ln -s "$(command -v bash)" "$TMP_DIR/no-python/bash" +ln -s "$(command -v dirname)" "$TMP_DIR/no-python/dirname" +for entrypoint in \ + build-artifacts-transaction.sh \ + build-custom.sh \ + generate-artifact-manifest.sh \ + verify-artifact-manifest.sh \ + publish-branches.sh; do + if PATH="$TMP_DIR/no-python" "$ROOT/scripts/commands/$entrypoint" \ + >"$TMP_DIR/$entrypoint.stdout" 2>"$TMP_DIR/$entrypoint.stderr"; then + echo "test failed: $entrypoint accepted a missing Python runtime" >&2 + exit 1 + fi + grep -F 'Python 3.11+ is required (python3 not found in PATH)' \ + "$TMP_DIR/$entrypoint.stderr" >/dev/null +done + echo "runtime requirement tests passed" diff --git a/scripts/tests/test-shell-utils.sh b/scripts/tests/test-shell-utils.sh index abe3a2910..a92c19669 100755 --- a/scripts/tests/test-shell-utils.sh +++ b/scripts/tests/test-shell-utils.sh @@ -132,6 +132,22 @@ test_setup_tool_cache_creates_bin_and_updates_path_once() { path=present" "$probe_output" "setup_tool_cache is explicit and idempotent" } +test_sha256_file_uses_python_fallback_without_sha256sum() { + local fallback_path="$TMP_DIR/no-sha256sum" + local input_file="$TMP_DIR/empty" + local actual + + mkdir -p "$fallback_path" + ln -s "$(command -v python3)" "$fallback_path/python3" + : > "$input_file" + + actual="$(PATH="$fallback_path" sha256_file "$input_file")" + assert_equals \ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" \ + "$actual" \ + "sha256_file Python fallback" +} + test_list_rule_files_sorts_lists_only test_list_rule_files_missing_dir_is_empty test_write_if_changed_replaces_different_file @@ -140,5 +156,6 @@ test_write_if_nonempty_or_remove_moves_nonempty_source test_write_if_nonempty_or_remove_deletes_empty_output_and_stale_target test_common_source_has_no_tool_cache_side_effects test_setup_tool_cache_creates_bin_and_updates_path_once +test_sha256_file_uses_python_fallback_without_sha256sum echo "shell utility tests passed" diff --git a/scripts/tools/artifact_manifest.py b/scripts/tools/artifact_manifest.py index c07a04189..c7171abb5 100644 --- a/scripts/tools/artifact_manifest.py +++ b/scripts/tools/artifact_manifest.py @@ -16,18 +16,19 @@ from artifact_verifier import verify_one from platform_capabilities import load_platform_capabilities -SCHEMA_VERSION = 3 +SCHEMA_VERSION = 4 SHA256_RE = re.compile(r"^[0-9a-f]{64}$") GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") -TOP_KEYS = {"schema_version", "generation_id", "build_id", "build_scope", "source", "inputs", "tools", "summaries", "artifacts", "restoration"} +GENERATION_RE = re.compile(r"^[0-9]+-[0-9]+$") +TOP_KEYS = {"schema_version", "generation_id", "build_id", "build_scope", "source", "baseline", "inputs", "tools", "summaries", "artifacts", "restoration"} SOURCE_KEYS = {"commit", "tree"} INPUT_KEYS = {"capabilities", "tool_lock", "artifact_origins"} INPUT_VALUE_KEYS = {"path", "sha256"} TOOLS_KEYS = {"lock", "installed"} SUMMARY_KEYS = {"path", "sha256", "content"} ARTIFACT_KEYS = {"path", "platform", "type", "extension", "bytes", "sha256", "origin", "verification"} -RESTORATION_KEYS = {"generation_id", "source_commit", "branches"} -BRANCH_KEYS = {"commit"} +RESTORATION_KEYS = {"status", "generation_id", "source_commit", "branches"} +BRANCH_KEYS = {"commit", "generation_id", "source_commit"} PUBLISH_BRANCHES = {"surge", "quanx", "egern", "sing-box", "mihomo"} INTERNAL_ARTIFACT_FILES = {"domain/rule-manifest.json"} @@ -170,6 +171,63 @@ def validate_tool_provenance(root: Path, tools: Any, lock: dict[str, Any], error errors.append(f"installed tool binary is not authenticated: {name}") +def validate_publication_cohort(value: Any, location: str, errors: list[str]) -> bool: + if not require_exact(value, RESTORATION_KEYS, location, errors): + return False + status = value["status"] + if status not in {"consistent", "inconsistent"}: + errors.append(f"{location} status must be consistent or inconsistent") + branches = value["branches"] + if not isinstance(branches, dict) or set(branches) != PUBLISH_BRANCHES: + errors.append(f"{location} must record all publish branches") + else: + for branch, item in branches.items(): + if ( + not require_exact(item, BRANCH_KEYS, f"{location}.branches.{branch}", errors) + or not isinstance(item.get("commit"), str) + or not GIT_SHA_RE.fullmatch(item.get("commit", "")) + ): + errors.append(f"{location} branch commit invalid: {branch}") + continue + branch_generation = item["generation_id"] + branch_source = item["source_commit"] + if branch_generation is not None and ( + not isinstance(branch_generation, str) or not GENERATION_RE.fullmatch(branch_generation) + ): + errors.append(f"{location} branch generation invalid: {branch}") + if branch_source is not None and ( + not isinstance(branch_source, str) or not GIT_SHA_RE.fullmatch(branch_source) + ): + errors.append(f"{location} branch source invalid: {branch}") + if status == "consistent": + generation = value["generation_id"] + source = value["source_commit"] + if not isinstance(generation, str) or not GENERATION_RE.fullmatch(generation): + errors.append(f"{location} generation_id must use -") + if not isinstance(source, str) or not GIT_SHA_RE.fullmatch(source): + errors.append(f"{location} source_commit must be a Git commit") + if isinstance(branches, dict): + identities = { + (item.get("generation_id"), item.get("source_commit")) + for item in branches.values() + if isinstance(item, dict) + } + if identities != {(generation, source)}: + errors.append(f"{location} consistent branch identities disagree") + elif value["generation_id"] is not None or value["source_commit"] is not None: + errors.append(f"{location} inconsistent identity must be null") + return True + + +def git_is_ancestor(root: Path, ancestor: str, descendant: str) -> bool: + return subprocess.run( + ["git", "-C", str(root), "merge-base", "--is-ancestor", ancestor, descendant], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ).returncode == 0 + + def generate(args: argparse.Namespace) -> None: root, output = args.root.resolve(), artifact_output(args.root.resolve()) output.mkdir(parents=True, exist_ok=True) @@ -185,10 +243,32 @@ def generate(args: argparse.Namespace) -> None: path = output / name if path.is_file(): summaries[name.removesuffix(".json").replace("-", "_")] = {"path": name, "sha256": digest(path), "content": load_json(path)} + baseline_path = Path(os.environ.get("ARTIFACT_BASELINE_FILE", root / ".tmp" / "publication-baseline.json")) + try: + baseline = load_json(baseline_path) + except (OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"publication baseline unreadable: {exc}") from exc + baseline_errors: list[str] = [] + validate_publication_cohort(baseline, "publication baseline", baseline_errors) + if baseline_errors: + raise SystemExit("invalid publication baseline: " + "; ".join(baseline_errors)) + if baseline["status"] == "consistent" and source_commit and not git_is_ancestor(root, baseline["source_commit"], source_commit): + raise SystemExit( + f"stale source refused: publication baseline {baseline['source_commit']} " + f"is not an ancestor of candidate {source_commit}" + ) restoration = load_json(output / "restoration-metadata.json") if (output / "restoration-metadata.json").is_file() else None + if args.build_scope == "custom": + if baseline["status"] != "consistent": + raise SystemExit("custom build requires a consistent publication baseline") + if restoration != baseline: + raise SystemExit("custom restoration metadata does not match the selected publication baseline") + if args.build_scope == "full" and restoration is not None: + raise SystemExit("full build must not contain restoration metadata") manifest = {"schema_version": SCHEMA_VERSION, "generation_id": args.generation_id, "build_id": args.build_id or args.generation_id, "build_scope": args.build_scope, "source": {"commit": source_commit, "tree": source_tree}, + "baseline": baseline, "inputs": {"capabilities": {"path": "config/domain-platform-capabilities.json", "sha256": digest(capabilities_path)}, "tool_lock": {"path": "config/tools-lock.json", "sha256": digest(lock_path)}, "artifact_origins": {"path": "artifact-origins.json", "sha256": digest(origins_path)}}, @@ -226,6 +306,23 @@ def verify(args: argparse.Namespace) -> None: if actual_tree is None: errors.append(f"source SHA is not a locally resolvable Git commit: {args.source_sha}") if source["commit"] != args.source_sha: errors.append(f"source commit mismatch: expected {args.source_sha}, got {source['commit']}") if actual_tree and source["tree"] != actual_tree: errors.append(f"source tree mismatch: expected {actual_tree}, got {source['tree']}") + baseline = manifest.get("baseline") + if validate_publication_cohort(baseline, "manifest baseline", errors): + source_commit = source.get("commit") if isinstance(source, dict) else None + baseline_source = baseline.get("source_commit") + if ( + baseline.get("status") == "consistent" + and + isinstance(source_commit, str) + and GIT_SHA_RE.fullmatch(source_commit) + and isinstance(baseline_source, str) + and GIT_SHA_RE.fullmatch(baseline_source) + and not git_is_ancestor(root, baseline_source, source_commit) + ): + errors.append( + f"stale source refused: manifest baseline {baseline_source} " + f"is not an ancestor of candidate {source_commit}" + ) capabilities_path, lock_path, origins_path = root / "config/domain-platform-capabilities.json", root / "config/tools-lock.json", output / "artifact-origins.json" try: matrix, lock, origins = capability_matrix(root), load_json(lock_path), load_origin_file(origins_path, require_nonempty=True) @@ -254,14 +351,11 @@ def verify(args: argparse.Namespace) -> None: if item["content"] != content: errors.append(f"summary embedded content mismatch: {expected_path}") restoration = manifest.get("restoration") if manifest.get("build_scope") == "custom": - if require_exact(restoration, RESTORATION_KEYS, "manifest restoration", errors): - if not isinstance(restoration["generation_id"], str) or not restoration["generation_id"]: errors.append("restoration generation_id must be non-empty") - if not isinstance(restoration["source_commit"], str) or not GIT_SHA_RE.fullmatch(restoration["source_commit"]): errors.append("restoration source_commit must be a Git commit") - branches = restoration["branches"] - if not isinstance(branches, dict) or set(branches) != PUBLISH_BRANCHES: errors.append("restoration metadata must record all publish branches") - else: - for branch, item in branches.items(): - if not require_exact(item, BRANCH_KEYS, f"manifest restoration.branches.{branch}", errors) or not isinstance(item.get("commit"), str) or not GIT_SHA_RE.fullmatch(item.get("commit", "")): errors.append(f"restoration branch commit invalid: {branch}") + if validate_publication_cohort(restoration, "manifest restoration", errors): + if isinstance(baseline, dict) and baseline.get("status") != "consistent": + errors.append("custom manifest baseline must be consistent") + if restoration != baseline: + errors.append("manifest restoration must match the publication baseline") elif restoration is not None: errors.append("full manifest restoration must be null") entries = manifest.get("artifacts") if not isinstance(entries, list) or not entries: errors.append("manifest artifacts must be a non-empty array"); entries = [] diff --git a/scripts/tools/artifact_verifier.py b/scripts/tools/artifact_verifier.py index d3e648e84..c998e88cc 100644 --- a/scripts/tools/artifact_verifier.py +++ b/scripts/tools/artifact_verifier.py @@ -78,32 +78,34 @@ def semantic_digest(entries: Counter[RuleEntry]) -> str: return hashlib.sha256(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() -def canonical_source_entries( - root: Path, - kind: str, - stem: str, - supported_kinds: set[str], -) -> Counter[RuleEntry] | None: - source = root / "sources" / "custom" / kind / f"{stem}.list" - if not source.is_file(): - return None +def canonical_file_entries(path: Path, kind: str) -> Counter[RuleEntry]: + if not path.is_file(): + raise ValueError(f"canonical rule source is missing: {path}") result: Counter[RuleEntry] = Counter() if kind == "domain": rules, errors = parse_classical_domain_file( - source, + path, require_canonical=True, allow_single_label_suffix=True, ) else: - rules, errors = parse_classical_ip_file(source, require_canonical=True) + rules, errors = parse_classical_ip_file(path, require_canonical=True) if errors: - raise ValueError(f"canonical custom source is invalid: {'; '.join(errors)}") + raise ValueError(f"canonical rule source is invalid: {'; '.join(errors)}") for rule in rules: - if rule.kind in supported_kinds: - result[(rule.kind, rule.value)] += 1 + result[(rule.kind, rule.value)] += 1 + if not result: + raise ValueError(f"canonical rule source is empty: {path}") return result +def canonical_source_entries(root: Path, kind: str, stem: str) -> Counter[RuleEntry] | None: + source = root / "sources" / "custom" / kind / f"{stem}.list" + if not source.is_file(): + return None + return canonical_file_entries(source, kind) + + def singbox_entries(data: dict[str, Any], kind: str) -> Counter[RuleEntry]: mappings = { "domain": {"domain": "DOMAIN", "domain_suffix": "DOMAIN-SUFFIX", "domain_keyword": "DOMAIN-KEYWORD", "domain_regex": "DOMAIN-REGEX"}, @@ -292,38 +294,148 @@ def parse_egern_yaml(path: Path, capability: Any, artifact_type: str) -> Counter return entries -def canonical_binary_entries( +def artifact_root_for(path: Path, artifact_type: str, platform: str) -> Path: + if path.parent.name != platform or path.parent.parent.name != artifact_type: + raise ValueError( + f"artifact path is outside the expected {artifact_type}/{platform} layout: {path}" + ) + return path.parent.parent.parent + + +def canonical_artifact_entries( root: Path, path: Path, artifact_type: str, platform: str, capabilities: Any, -) -> tuple[Counter[RuleEntry] | None, str | None]: +) -> tuple[Counter[RuleEntry], str]: target = getattr(capabilities.platforms[platform], artifact_type) supported_kinds = set(target.rule_mappings) - custom = canonical_source_entries(root, artifact_type, path.stem, supported_kinds) - if custom is not None: - return custom, f"sources/custom/{artifact_type}/{path.stem}.list" - - artifact_root = path.parents[2] - reference_platform = "surge" - if artifact_type == "domain" and platform == "sing-box": - reference_platform = "egern" - reference_capability = getattr(capabilities.platforms[reference_platform], artifact_type) - reference = artifact_root / artifact_type / reference_platform / f"{path.stem}.{reference_capability.extension}" - if not reference.is_file(): - return None, None - if reference_capability.verifier.startswith("classical-"): - entries = verify_classical(reference, reference_capability, reference_platform, artifact_type) - elif reference_capability.verifier == "egern-yaml": - entries = parse_egern_yaml(reference, reference_capability, artifact_type) - else: - raise ValueError(f"unsupported canonical reference verifier: {reference_capability.verifier}") - filtered = Counter({entry: count for entry, count in entries.items() if entry[0] in supported_kinds}) - return filtered, reference.relative_to(artifact_root).as_posix() + artifact_root = artifact_root_for(path, artifact_type, platform) + canonical_path = artifact_root / ".canonical" / artifact_type / f"{path.stem}.list" + canonical = canonical_file_entries(canonical_path, artifact_type) + + custom = canonical_source_entries(root, artifact_type, path.stem) + if custom is not None and normalized_semantic_entries(custom) != normalized_semantic_entries(canonical): + raise ValueError( + "internal canonical rules differ from custom source: " + f"canonical_sha256={semantic_digest(canonical)}, custom_sha256={semantic_digest(custom)}" + ) + + filtered = Counter( + {entry: count for entry, count in canonical.items() if entry[0] in supported_kinds} + ) + if not filtered: + raise ValueError( + f"artifact exists although canonical rules contain no kinds supported by {platform}" + ) + return filtered, canonical_path.relative_to(artifact_root).as_posix() + + +def write_canonical_entries(path: Path, artifact_type: str, entries: Counter[RuleEntry]) -> None: + kind_order = { + "domain": ("DOMAIN", "DOMAIN-SUFFIX", "DOMAIN-KEYWORD", "DOMAIN-REGEX"), + "ip": ("IP-CIDR", "IP-CIDR6"), + }[artifact_type] + order = {kind: index for index, kind in enumerate(kind_order)} + lines: list[str] = [] + for (kind, value), count in sorted(entries.items(), key=lambda item: (order[item[0][0]], item[0][1])): + if count != 1: + raise ValueError(f"canonical seed contains duplicate rule: {kind},{value}") + lines.append(f"{kind},{value}") + if not lines: + raise ValueError(f"refusing to write empty canonical rule source: {path}") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def seed_canonical_from_egern(root: Path, artifact_root: Path, output: Path) -> None: + capabilities = load_platform_capabilities(root / "config" / "domain-platform-capabilities.json") + if output.exists(): + raise ValueError(f"canonical seed output already exists: {output}") + seeded = 0 + for artifact_type in ("domain", "ip"): + capability = getattr(capabilities.platforms["egern"], artifact_type) + source_dir = artifact_root / artifact_type / "egern" + if not source_dir.is_dir(): + continue + for path in sorted(source_dir.glob(f"*.{capability.extension}")): + entries = parse_egern_yaml(path, capability, artifact_type) + write_canonical_entries(output / artifact_type / f"{path.stem}.list", artifact_type, entries) + seeded += 1 + if seeded == 0: + raise ValueError(f"no Egern artifacts are available to seed canonical audit rules: {artifact_root}") -def verify_one(root: Path, path: Path, artifact_type: str, platform: str) -> dict[str, Any]: +def verify_canonical_inventory(root: Path, artifact_root: Path) -> None: + capabilities = load_platform_capabilities(root / "config" / "domain-platform-capabilities.json") + errors: list[str] = [] + for artifact_type in ("domain", "ip"): + canonical_dir = artifact_root / ".canonical" / artifact_type + if not canonical_dir.is_dir(): + errors.append(f"canonical {artifact_type} directory is missing: {canonical_dir}") + continue + canonical_paths = sorted(canonical_dir.glob("*.list")) + if not canonical_paths: + errors.append(f"canonical {artifact_type} directory is empty: {canonical_dir}") + continue + unexpected_canonical = sorted( + path.relative_to(canonical_dir).as_posix() + for path in canonical_dir.rglob("*") + if path.is_file() and (path.parent != canonical_dir or path.suffix != ".list") + ) + if unexpected_canonical: + errors.extend( + f"unexpected canonical audit file: {artifact_type}/{relative}" + for relative in unexpected_canonical + ) + + canonical: dict[str, Counter[RuleEntry]] = {} + for path in canonical_paths: + try: + canonical[path.stem] = canonical_file_entries(path, artifact_type) + except ValueError as exc: + errors.append(str(exc)) + + for platform, details in capabilities.platforms.items(): + capability = getattr(details, artifact_type) + supported_kinds = set(capability.rule_mappings) + platform_dir = artifact_root / artifact_type / platform + for stem, entries in canonical.items(): + supported = Counter( + {entry: count for entry, count in entries.items() if entry[0] in supported_kinds} + ) + expected = bool(normalized_semantic_entries(supported)) + artifact = platform_dir / f"{stem}.{capability.extension}" + if expected and not artifact.is_file(): + errors.append( + f"canonical counterpart missing: {artifact_type}/{platform}/{artifact.name}" + ) + elif not expected and artifact.exists(): + errors.append( + f"unsupported-only canonical rules must not publish: " + f"{artifact_type}/{platform}/{artifact.name}" + ) + + if platform_dir.is_dir(): + for artifact in sorted(platform_dir.glob(f"*.{capability.extension}")): + if artifact.stem not in canonical: + errors.append( + f"artifact has no canonical audit source: " + f"{artifact_type}/{platform}/{artifact.name}" + ) + if errors: + raise ValueError("canonical artifact inventory failed: " + "; ".join(errors)) + + +def verify_one( + root: Path, + path: Path, + artifact_type: str, + platform: str, + *, + require_canonical_linkage: bool = True, +) -> dict[str, Any]: capabilities = load_platform_capabilities(root / "config" / "domain-platform-capabilities.json") capability = getattr(capabilities.platforms[platform], artifact_type) verifier = capability.verifier @@ -339,17 +451,18 @@ def verify_one(root: Path, path: Path, artifact_type: str, platform: str) -> dic raise ValueError(f"unsupported verifier implementation: {verifier}") canonical: Counter[RuleEntry] | None = None canonical_source: str | None = None - if verifier in {"sing-box", "mihomo"}: - canonical, canonical_source = canonical_binary_entries( + if require_canonical_linkage: + canonical, canonical_source = canonical_artifact_entries( root, path, artifact_type, platform, capabilities, ) - if canonical is None or canonical_source is None: - raise ValueError("binary artifact has no canonical custom source or same-build text counterpart") - linkage: dict[str, Any] = {"status": "unavailable", "reason": "canonical binary compiler input is not retained or not applicable"} + linkage: dict[str, Any] = { + "status": "unavailable", + "reason": "syntax-only verification does not require canonical linkage", + } if canonical is not None: if normalized_semantic_entries(decoded) != normalized_semantic_entries(canonical): raise ValueError( @@ -376,14 +489,52 @@ def verify_one(root: Path, path: Path, artifact_type: str, platform: str) -> dic def main() -> None: parser = argparse.ArgumentParser() parser.add_argument("--root", type=Path, required=True) - parser.add_argument("--path", type=Path, required=True) - parser.add_argument("--type", choices=("domain", "ip"), required=True) - parser.add_argument("--platform", required=True) + parser.add_argument("--path", type=Path) + parser.add_argument("--type", choices=("domain", "ip")) + parser.add_argument("--platform") + parser.add_argument("--syntax-only", action="store_true") + parser.add_argument("--seed-canonical-from", type=Path) + parser.add_argument("--canonical-output", type=Path) + parser.add_argument("--verify-canonical-inventory", type=Path) args = parser.parse_args() try: - result = verify_one(args.root.resolve(), args.path.resolve(), args.type, args.platform) + if args.verify_canonical_inventory is not None: + if any( + value is not None + for value in ( + args.path, + args.type, + args.platform, + args.seed_canonical_from, + args.canonical_output, + ) + ) or args.syntax_only: + parser.error("canonical inventory verification cannot be combined with other operations") + verify_canonical_inventory(args.root.resolve(), args.verify_canonical_inventory.resolve()) + return + if args.seed_canonical_from is not None or args.canonical_output is not None: + if args.seed_canonical_from is None or args.canonical_output is None: + parser.error("--seed-canonical-from and --canonical-output must be used together") + if args.path is not None or args.type is not None or args.platform is not None or args.syntax_only: + parser.error("canonical seeding cannot be combined with artifact verification options") + seed_canonical_from_egern( + args.root.resolve(), + args.seed_canonical_from.resolve(), + args.canonical_output.resolve(), + ) + return + if args.path is None or args.type is None or args.platform is None: + parser.error("--path, --type, and --platform are required for artifact verification") + result = verify_one( + args.root.resolve(), + args.path.resolve(), + args.type, + args.platform, + require_canonical_linkage=not args.syntax_only, + ) except (OSError, ValueError, json.JSONDecodeError, subprocess.CalledProcessError) as exc: - raise SystemExit(f"artifact verification failed for {args.path}: {exc}") + operation = args.path or args.seed_canonical_from or args.verify_canonical_inventory + raise SystemExit(f"artifact verification failed for {operation}: {exc}") print(json.dumps(result, sort_keys=True))