From 10e844574095cde1b904ea90bc75cd80e6b4fdb1 Mon Sep 17 00:00:00 2001 From: KuGouGo <62388728+KuGouGo@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:18:45 +0800 Subject: [PATCH 1/3] Harden scoped validation and artifact publishing --- .github/dependabot.yml | 10 +- .github/workflows/build.yml | 36 +++ .github/workflows/pull-request.yml | 34 ++- .gitignore | 1 + CONTRIBUTING.md | 4 +- README.md | 4 +- THIRD_PARTY_NOTICES.md | 10 +- config/upstreams.json | 4 + docs/DEVELOPMENT.md | 12 +- docs/STRUCTURE.md | 12 +- docs/TROUBLESHOOTING.md | 8 +- scripts/commands/build-custom.sh | 24 +- scripts/commands/clean.sh | 2 +- scripts/commands/publish-branches.sh | 108 ++++----- scripts/commands/restore-artifacts.sh | 92 ------- scripts/commands/select-build-scope.sh | 44 +++- scripts/commands/sync-upstream.sh | 174 ++++++++++--- scripts/tests/test-artifact-manifest.sh | 2 + .../tests/test-binary-artifact-verifier.sh | 107 +++++++- scripts/tests/test-build-scope.sh | 51 +++- ...eip-migration.sh => test-fakeip-filter.sh} | 56 ++--- scripts/tests/test-publish-branches.sh | 65 ++++- .../tests/test-sync-upstream-render-matrix.sh | 7 +- scripts/tests/test-upstream-config.sh | 48 ++++ scripts/tests/test-upstream-health-summary.sh | 52 ++++ scripts/tests/test-workflow-action-pinning.sh | 53 ++++ scripts/tools/artifact_manifest.py | 8 +- scripts/tools/artifact_verifier.py | 229 +++++++++++++++--- scripts/tools/lint-config.py | 61 +++-- scripts/tools/normalize-ip-rules.py | 12 +- templates/branch-readmes/mihomo.md | 2 +- 31 files changed, 942 insertions(+), 390 deletions(-) rename scripts/tests/{test-fakeip-migration.sh => test-fakeip-filter.sh} (55%) create mode 100644 scripts/tests/test-upstream-health-summary.sh create mode 100644 scripts/tests/test-workflow-action-pinning.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ea7289db4..9e8432814 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,19 +4,15 @@ updates: directory: / target-branch: main schedule: - interval: monthly - open-pull-requests-limit: 1 + interval: weekly + open-pull-requests-limit: 5 groups: - github-actions: + github-actions-minor-patch: patterns: - '*' update-types: - minor - patch - ignore: - - dependency-name: '*' - update-types: - - version-update:semver-major commit-message: prefix: deps(actions) rebase-strategy: auto diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ea6f7dea..671bcafd0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -99,9 +99,44 @@ jobs: cat .output/upstream-summary.json fi fi + if [ -d .artifacts/diagnostics ]; then + python3 - <<'PY' + import json + from pathlib import Path + + diagnostics = Path(".artifacts/diagnostics") + allowed_names = { + "build-summary.json", + "transaction-health.json", + "upstream-summary.json", + } + max_bytes = 256 * 1024 + + for path in sorted(diagnostics.rglob("*.json")): + if path.name not in allowed_names or path.stat().st_size > max_bytes: + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + print(f"diagnostic summary unreadable: {path}: {exc}") + continue + print(f"Failure diagnostic summary: {path}") + print(json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True)) + PY + fi echo "Changed files summary:" git status --short + - name: Upload failure diagnostics + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: build-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: .artifacts/diagnostics/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 7 + - name: Upload publish artifacts if: success() uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 @@ -114,6 +149,7 @@ jobs: publish: needs: build + if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch) runs-on: ubuntu-latest timeout-minutes: 15 permissions: diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 7efcc902d..1e44e49bf 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -42,33 +42,63 @@ jobs: fetch-depth: 0 persist-credentials: false + - name: Select release candidate scope + id: scope + env: + EVENT_NAME: pull_request + BEFORE_SHA: ${{ github.event.pull_request.base.sha }} + CURRENT_SHA: ${{ github.sha }} + run: ./scripts/commands/select-build-scope.sh + + - name: Show selected release candidate scope + run: echo "${{ steps.scope.outputs.scope }} - ${{ steps.scope.outputs.reason }}" + - name: Set up Python + if: steps.scope.outputs.scope != 'none' uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: '3.11' - name: Prepare scripts + if: steps.scope.outputs.scope != 'none' run: find scripts -type f \( -name '*.sh' -o -name '*.py' \) -exec chmod +x {} + - name: Resolve tool versions + if: steps.scope.outputs.scope != 'none' id: core_versions run: bash ./scripts/commands/resolve-tool-versions.sh - name: Restore verified tool cache + if: steps.scope.outputs.scope != 'none' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 with: path: .bin key: rules-tools-v3-${{ steps.core_versions.outputs.sing_box_version }}-${{ steps.core_versions.outputs.mihomo_version }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('config/tools-lock.json') }} - name: Build and verify release candidate + if: steps.scope.outputs.scope != 'none' env: - RULES_BUILD_SCOPE: full + RULES_BUILD_SCOPE: ${{ steps.scope.outputs.scope }} + RULES_CONFLICT_BASE_SHA: ${{ steps.scope.outputs.base_sha }} ARTIFACT_GENERATION_ID: pr-${{ github.event.pull_request.number }}-${{ github.run_attempt }} ARTIFACT_BUILD_ID: ${{ github.run_id }} ARTIFACT_SOURCE_SHA: ${{ github.sha }} run: ./scripts/commands/build-artifacts-transaction.sh - name: Show release candidate summary + if: steps.scope.outputs.scope != 'none' run: | cat .output/build-summary.json - cat .output/upstream-summary.json + if [ -f .output/upstream-summary.json ]; then + cat .output/upstream-summary.json + fi + + - name: Upload failure diagnostics + if: failure() && steps.scope.outputs.scope != 'none' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: release-candidate-diagnostics-${{ github.run_id }}-${{ github.run_attempt }} + path: .artifacts/diagnostics/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 7 diff --git a/.gitignore b/.gitignore index 7fdf1911b..cbdba46a1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .output/ .tmp/ .bin/ +.artifacts/ # Python __pycache__/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c717cab48..c85b0c640 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,7 @@ IP 目录可不存在或为空。文件名只使用小写字母、数字和连 ## 审计信息的使用 -完整主上游同步生成 `.output/upstream-summary.json` 和域名 `rule-manifest.json`。前者是部分采集摘要，不覆盖本仓库维护的自定义源（包括 `sources/custom/domain/fakeip-filter.list`）、完整转换链、提交身份或内容校验和，不能称为完整来源追溯记录。`fakeip-filter` 与其他自定义规则一起生成各平台产物，不得重新引入第三方预编译文件或下载步骤；过去采用的 `wwqgtxx/clash-rules` 二进制仅属历史。 +完整主上游同步生成 `.output/upstream-summary.json` 和域名 `rule-manifest.json`。前者记录健康结果与输入摘要，但不覆盖本仓库维护的自定义源（包括 `sources/custom/domain/fakeip-filter.list`）、完整转换链或 HTTP 响应身份，不能称为完整来源追溯记录。`fakeip-filter` 与其他自定义规则一起生成各平台产物，不得引入独立预编译文件或下载步骤。 成功的 `build-artifacts-transaction.sh` 在产物守卫之后、manifest 之前生成 `.output/build-summary.json`；manifest 绑定其文件摘要与嵌入内容。独立运行 `make build-custom*` 不生成该文件。所有摘要都不是许可证明。 @@ -61,7 +61,7 @@ make build-custom `make preflight` 只组合 `make validate` 与文本自定义构建，不执行完整同步、产物守卫（artifact guard）或发布。本地缺少 ShellCheck 时可能跳过，CI 则设置 `REQUIRE_SHELLCHECK=1`。 -完整支持环境与工具下载边界见 [开发指南](docs/DEVELOPMENT.md)。不要提交 `.output/`、`.tmp/`、`.bin/`、凭据或本机缓存。 +完整支持环境与工具下载边界见 [开发指南](docs/DEVELOPMENT.md)。不要提交 `.output/`、`.tmp/`、`.artifacts/`、`.bin/`、凭据或本机缓存。 ## 修改实现 diff --git a/README.md b/README.md index 0cd94bbdb..8a00b1a06 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,9 @@ make clean ## 仓库边界 - 长期分支只保留 `main` 与 `surge`、`quanx`、`egern`、`sing-box`、`mihomo` 五个产物分支。 -- 变更通过临时分支向 `main` 提交 Pull Request；PR 运行静态检查和不发布的完整候选构建，临时分支在合并后删除。 +- 变更通过临时分支向 `main` 提交 Pull Request；PR 始终运行静态检查，并按改动路径选择跳过、custom 或 full 候选构建，临时分支在合并后删除。 - `main` 在构建相关路径变化、定时任务运行或人工触发时执行发布工作流；产物内容没有变化时不会创建新的产物分支提交。 -- Dependabot 每月向 `main` 集中提交一个 GitHub Actions minor/patch 更新 PR；major 与安全告警单独人工评估。 +- Dependabot 每周向 `main` 集中提交一个 GitHub Actions minor/patch 更新 PR；major 与安全更新保持独立，逐项评估。 - `fakeip-filter` 是本仓库维护的文本源，不下载第三方预编译文件。 - 构建摘要、manifest 和 CI 通过都不是第三方许可证明。 - 规则按现状提供。使用者需自行判断策略、顺序和更新带来的影响，并保留可回退版本。 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index d9d62e1dc..8dab320d5 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -1,6 +1,6 @@ # 第三方声明 -本文件记录当前已知第三方输入及截至 **2026-07-14** 的人工核对状态。它不是自动生成清单，也不是法律意见。`trust` 只表示采集来源分类；公开可访问、官方托管、注册机构提供或 CI 成功都不等于获得复制、修改或再分发授权。 +本文件记录当前已知第三方输入及截至 **2026-07-15** 的人工核对状态。它不是自动生成清单，也不是法律意见。`trust` 只表示采集来源分类；公开可访问、官方托管、注册机构提供或 CI 成功都不等于获得复制、修改或再分发授权。 标准 MIT License 位于 [`LICENSE`](LICENSE)，适用范围见 [`NOTICE`](NOTICE)。第三方材料不会因下载、规范化、合并、转换或编译而变为本仓库的 MIT 内容。 @@ -24,7 +24,7 @@ | `ip.google` | | `official` | 未知；响应中未见独立数据许可证。 | | `ip.telegram` | | `official` | 未知；响应中未见独立数据许可证。 | | `ip.cloudflare-ipv4`、`ip.cloudflare-ipv6` | 、 | `official` | 未知；未确认独立数据许可证。 | -| `ip.aws`（也生成 `cloudfront`） | | `official` | 未知；响应中未见独立数据许可证。 | +| `ip.aws`、`ip.cloudfront` | | `official` | 未知；两个解析器复用同一官方响应，未见独立数据许可证。 | | `ip.fastly` | | `official` | 未知；响应中未见独立数据许可证。 | | `ip.github` | | `official` | 未知；未确认 Meta API 数据的独立再分发许可证。 | | `ip.apple` | （回退：） | `official` | 未知；未确认页面中网络范围数据的独立再分发许可证。 | @@ -32,16 +32,14 @@ `asn_groups.telegram`、`asn_groups.netflix`、`asn_groups.spotify` 和 `asn_groups.disney` 只是 RIPEstat 查询参数分组，不是独立许可声明；所得结果沿用 `ip.ripe-stat` 的“未知”状态。组织名称和商标归各自权利人所有。 -## Fake-IP 迁移说明 +## Fake-IP 当前 `fakeip-filter` 是 KuGouGo 在 `sources/custom/domain/fakeip-filter.list` 维护的仓库源码，由自定义构建生成各平台文本产物以及 sing-box、mihomo 二进制产物；它不属于第三方输入，也不下载第三方预编译文件。 -本仓库过去曾直接采用 `wwqgtxx/clash-rules` 的预编译 `release/fakeip-filter.mrs`。该来源仅记录历史迁移背景，不是当前网络输入、构建步骤或发布材料；当前产物不得据此归因于该项目。 - ## 审计与评审边界 - `config/upstreams.json` 覆盖主上游规则网络输入；工具资产下载另由工具 lock 控制。当前没有 Fake-IP 网络输入或独立同步步骤。 -- `.output/upstream-summary.json` 只覆盖主同步记录，不覆盖本仓库自定义源（包括 `fakeip-filter`）、完整转换链、提交身份或内容校验和，因此不是完整来源追溯记录。 +- `.output/upstream-summary.json` 只覆盖主同步记录，不覆盖本仓库自定义源（包括 `fakeip-filter`）、完整转换链或 HTTP 响应身份，因此不是完整来源追溯记录。 - `.output/build-summary.json` 由成功构建事务在产物守卫后生成并受 manifest 摘要绑定；它只说明事务产物统计，不证明来源或授权。 - 当前自动化不解析 `NOTICE` 或本文件，不验证第三方许可，也不会因“未知”状态自动失败。 - 新增、更换或改变第三方输入时，应同步更新本文件并提供可核验依据；无法确认时保持“未知”，由维护者人工决定停止、替换、取得授权或满足适用义务后再发布。 diff --git a/config/upstreams.json b/config/upstreams.json index 1a203f0f7..083fb579d 100644 --- a/config/upstreams.json +++ b/config/upstreams.json @@ -45,6 +45,10 @@ "kind": "json", "trust": "official", "parser": "aws-json", "url": "https://ip-ranges.amazonaws.com/ip-ranges.json", "health": { "requirement": "required", "min_raw_bytes": 1000, "min_entries": 100, "family": "dual", "fallback_policy": "none" } }, + "cloudfront": { + "kind": "json", "trust": "official", "parser": "aws-cloudfront-json", "url": "https://ip-ranges.amazonaws.com/ip-ranges.json", + "health": { "requirement": "required", "min_raw_bytes": 1000, "min_entries": 150, "family": "dual", "fallback_policy": "none" } + }, "fastly": { "kind": "json", "trust": "official", "parser": "fastly-json", "url": "https://api.fastly.com/public-ip-list", "health": { "requirement": "required", "min_raw_bytes": 50, "min_entries": 12, "family": "any", "fallback_policy": "none" } diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index de03d9253..d663065c5 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -30,16 +30,16 @@ make clean - `make build-custom-text`：只生成自定义文本产物，不下载二进制编译器。 - `make build-custom`：生成自定义文本和二进制产物。 - `make preflight`：`make validate` 加文本自定义构建；不执行完整同步、产物守卫（artifact guard）或发布。 -- `build-artifacts-transaction.sh`：CI 的完整入口。它在 `.tmp/` 中以事务自有的 `RULES_ARTIFACT_ROOT` 组合上游同步或已发布分支恢复、自定义构建、守卫、摘要、manifest 生成与验证；调用方提供 `RULES_ARTIFACT_ROOT` 会被拒绝，测试或运维如需改变最终提升位置应使用 `RULES_LIVE_ARTIFACT_ROOT`。全部成功后才以目录替换提升为 `.output/`（或该显式 live root）。失败诊断写入非发布目录 `.artifacts/diagnostics/`，旧 live root 保持不变。`config/upstreams.json` 为每个源声明 parser、required/optional、原始字节、规范条目、地址族和 fallback policy；required 源的传输、fallback 或语义健康回归都会在提升前失败。每个 RIPE Stat ASN 响应和合并分组都使用 `ripe-stat` health policy，过小或无效响应会先写入诊断摘要再阻断事务。自定义恢复要求五个发布分支具有相同 generation/source 身份，并把分支 commit 写入 manifest restoration metadata。缺失分支的跨平台有损转换默认禁用，仅可用 `RULES_ALLOW_LOSSY_RESTORE_FALLBACK=1` 显式开启。 +- `build-artifacts-transaction.sh`：CI 的完整入口。它在 `.tmp/` 中以事务自有的 `RULES_ARTIFACT_ROOT` 组合上游同步或已发布分支恢复、自定义构建、守卫、摘要、manifest 生成与验证；调用方提供 `RULES_ARTIFACT_ROOT` 会被拒绝，测试或运维如需改变最终提升位置应使用 `RULES_LIVE_ARTIFACT_ROOT`。全部成功后才以目录替换提升为 `.output/`（或该显式 live root）。失败诊断写入非发布目录 `.artifacts/diagnostics/`，旧 live root 保持不变。`config/upstreams.json` 为每个源声明 parser、required/optional、原始字节、规范条目、地址族和 fallback policy；required 源在主 URL 与允许的回退均失败，或出现语义健康回归时阻断提升。每个 RIPE Stat ASN 响应和合并分组都使用 `ripe-stat` health policy，过小或无效响应会先写入诊断摘要再阻断事务。自定义恢复要求五个发布分支都存在且具有相同 generation/source 身份，并把分支 commit 写入 manifest restoration metadata；缺失或身份分裂时失败关闭，应执行 full 构建恢复发布 cohort。 - `generate-artifact-manifest.sh`：在构建与守卫完成后、写 manifest 前，按能力配置中的 `verifier` 分派器验证每个产物；缺失或未验证的二进制会阻断生成。默认位于 `.output/`，事务内服从 `RULES_ARTIFACT_ROOT`。调用方应明确提供 generation id、build scope，CI 还将 source SHA 绑定到实际 checkout 的 `github.sha`：PR 验证记录被测试的合并提交，正式发布记录 `main` 提交。 - `verify-artifact-manifest.sh`：严格重算所选 artifact root 内的可发布文件集合、路径、大小和 SHA-256，并重新执行产物验证、核对能力/lock 与可选 source SHA；发布 job 在恢复或安装锁定工具后强制执行同一验证。 -二进制验证使用固定工具的真实读回接口：`.srs` 执行 `sing-box rule-set decompile` 并解析 JSON；`.mrs` 执行 `mihomo convert-ruleset mrs INPUT OUTPUT`。对仍有规范自定义源的二进制，读回规则总数和类型计数必须匹配 compiler 输入。恢复分支和上游二进制没有保留逐产物规范 compiler 输入，因此仅记录真实读回计数及 `canonical_linkage.status=unavailable`，不宣称语义 round-trip。manifest 每个产物记录验证状态、方法、读回计数和规范计数关联。 -- `make clean`：删除 `.tmp/`、`.output/`、Python `__pycache__` 和未完成的 `.bin/*.new*`；保留已安装的 `.bin/sing-box`、`.bin/mihomo` 及 provenance sidecar。 +二进制验证使用固定工具的真实读回接口：`.srs` 执行 `sing-box rule-set decompile` 并解析 JSON；`.mrs` 执行 `mihomo convert-ruleset mrs INPUT OUTPUT`。读回结果规范化为 `(规则类型, 规则值)` 多重集合，并与同名 custom 源或同一事务的 Egern/Surge 文本产物精确比较；类型计数相同但值不同也会失败。manifest 记录验证方法、计数、读回语义 SHA-256，以及可关联规范输入时的语义 SHA-256。 +- `make clean`：删除 `.tmp/`、`.output/`、`.artifacts/`、Python `__pycache__` 和未完成的 `.bin/*.new*`；保留已安装的 `.bin/sing-box`、`.bin/mihomo` 及 provenance sidecar。 CI 设置 `REQUIRE_SHELLCHECK=1`，本地缺少 ShellCheck 时的跳过不代表 CI 会通过。 -GitHub Actions 使用完整 commit SHA 固定版本。Dependabot 每月把 GitHub Actions 的 minor/patch 更新组合为一个以 `main` 为目标的 PR，减少临时分支和重复 CI；major 更新单独评估，避免阻塞常规更新。GitHub 漏洞告警保持启用，自动安全修复分支关闭；安全更新由维护者确认影响后通过临时分支提交。合并后的临时分支由 GitHub 自动删除。 +GitHub Actions 使用完整 commit SHA 固定版本，仓库测试拒绝 tag 或非完整 SHA 的 `uses:`。Dependabot 每周把 GitHub Actions 的 minor/patch 更新组合为一个以 `main` 为目标的 PR；major 与安全更新保持独立并逐项评估。合并后的临时分支由 GitHub 自动删除。 ## 开发流程 @@ -47,7 +47,7 @@ GitHub Actions 使用完整 commit SHA 固定版本。Dependabot 每月把 GitHu 2. 修改自定义源、配置、实现或测试夹具。 3. 运行 `make preflight` 和适用的完整构建命令。 4. 检查差异中没有 `.output/`、`.tmp/`、`.bin/`、凭据或无关格式化。 -5. 通过 Pull Request 合并到 `main`；PR 必须完成预检和不发布的完整构建，合并后由 `main` 工作流更新五个平台分支。 +5. 通过 Pull Request 合并到 `main`；PR 必须完成预检。候选构建按路径选择范围：仅文档和治理文件为 `none`，仅修改且不删除自定义源为 `custom`，构建脚本、配置、模板、测试或自定义源删除为 `full`。合并后的构建相关变更由 `main` 工作流更新五个平台分支。 6. 按 [贡献指南](../CONTRIBUTING.md) 说明来源、人工许可评审状态、测试和产物影响。 ## 自定义规则与名称 @@ -60,7 +60,7 @@ GitHub Actions 使用完整 commit SHA 固定版本。Dependabot 每月把 GitHu ## 摘要与许可评审 -主上游完整同步生成的 `upstream-summary.json` 是部分采集摘要，不包含本仓库维护的自定义源（包括 `sources/custom/domain/fakeip-filter.list`），也不记录完整转换链、提交身份或内容校验和。`fakeip-filter` 与其他自定义规则一起构建为各平台文本和二进制产物；过去采用的 `wwqgtxx/clash-rules` 预编译文件仅属历史，当前没有独立下载步骤。 +主上游完整同步生成的 `upstream-summary.json` 记录健康检查后的状态、实际 URL、回退、原始与规范化输入的字节数、条目数和 SHA-256；DLC 另记录检出的 commit。它不包含本仓库维护的自定义源（包括 `sources/custom/domain/fakeip-filter.list`），也不覆盖完整转换链或 HTTP 响应身份，因此仍不是完整来源证明。`fakeip-filter` 与其他自定义规则共用同一构建入口，没有独立下载步骤。 `build-summary.json` 是成功构建事务的固有输出：产物守卫通过后、manifest 生成前，由 `build-artifacts-transaction.sh` 在事务 artifact root 中生成。manifest 同时记录摘要文件 SHA-256 与解析后的嵌入内容，发布前验证两者。工作流的 `Show summary` 步骤只负责显示成功输出，或在失败诊断场景中临时生成可读摘要，不定义成功产物。 diff --git a/docs/STRUCTURE.md b/docs/STRUCTURE.md index b6606e367..05b38bab0 100644 --- a/docs/STRUCTURE.md +++ b/docs/STRUCTURE.md @@ -16,30 +16,32 @@ | `.output/` | 构建产物及部分审计摘要 | | `.tmp/` | 可清理临时工作区 | | `.bin/` | 外部工具及版本缓存 | +| `.artifacts/` | 失败构建保留的诊断摘要；属于可清理的本地/CI 数据，不进入发布分支 | ## 构建范围 完整范围依次同步主上游、构建自定义规则、执行产物守卫（artifact guard），再上传并发布。自定义范围从五个发布分支恢复既有产物，然后重建自定义规则。工作流没有独立的 Fake-IP 同步步骤。 -`fakeip-filter` 当前源为本仓库维护的 `sources/custom/domain/fakeip-filter.list`，由 `build-custom.sh` 与其他自定义规则一起生成五平台形式，不从网络下载预编译文件。过去采用 `wwqgtxx/clash-rules` 的 `fakeip-filter.mrs` 仅是历史迁移背景，不属于当前输入。`config/upstreams.json` 覆盖主上游网络输入；工具资产下载另由工具 lock 控制。 +`fakeip-filter` 当前源为本仓库维护的 `sources/custom/domain/fakeip-filter.list`，由 `build-custom.sh` 与其他自定义规则一起生成五平台形式，不从网络下载预编译文件。`config/upstreams.json` 覆盖主上游网络输入；工具资产下载另由工具 lock 控制。 `main` 是唯一长期源码与发布源分支，开发使用合并后删除的临时分支。发布分支为 `surge`、`quanx`、`egern`、`sing-box`、`mihomo`，只允许生成的 `README.md`、`domain/`、`ip/` 及平台对应扩展名。各分支 `README.md` 由 `templates/branch-readmes/` 生成，并直接包含 v2fly/domain-list-community 的完整 MIT 版权与许可通知；因此发布树无需新增独立许可证文件。模板变更属于构建触发路径。 ## 审计文件 -- `.output/upstream-summary.json`：主同步记录的名称、URL、状态、回退、临时路径、字节数和按非注释行估算的条目数。 +- `.output/upstream-summary.json`：主同步在健康检查后记录名称、实际 URL、状态、回退、临时路径、原始与规范化内容的字节数、条目数和 SHA-256；DLC 另含检出的 commit。 - `.output/domain/rule-manifest.json`：域名列表、区域集合和属性派生结构。 - `.output/build-summary.json`：GitHub Actions 在产物守卫之后、规范发布清单之前扫描 `.output/` 生成。 - `.output/artifact-manifest.json`：规范发布清单，包含 schema/generation/build/source/build scope、能力与工具 lock 摘要、工具 provenance metadata、可用的上游/构建摘要，以及每个可发布 domain/ip 文件的平台、类型、扩展名、字节数、SHA-256 和可判定来源。JSON 键与产物按稳定顺序输出；generation/build id 由调用方提供，因此相同输入和 id 可复现相同内容。 - `.tmp/**/normalize-tasks.json`：批处理任务描述，属于临时数据。 +- `.artifacts/diagnostics//`：失败事务保留的 `transaction-health.json` 及可用的构建/上游摘要；CI 日志只展示白名单内且大小受限的 JSON，完整诊断作为短期 Actions artifact 上传。 -`verify-artifact-manifest.sh` 严格重算能力矩阵允许的完整文件集合、路径层级与安全性、非零大小、字节数和 SHA-256，并核对能力/lock 摘要及可选的预期 source SHA。发布作业下载后再次验证；`publish-branches.sh` 自身也必须先验证清单，拒绝缺失、额外或被修改的产物。清单只作为流水线审计输入，不复制到发布分支。分支提交消息携带共同 generation id 和 source SHA。 +`verify-artifact-manifest.sh` 严格重算能力矩阵允许的完整文件集合、路径层级与安全性、非零大小、字节数和 SHA-256，并核对能力/lock 摘要及可选的预期 source SHA。二进制读回规则与同名 custom 源或同一事务的文本产物按类型和值精确比较，并记录语义 SHA-256。发布作业下载后再次验证；`publish-branches.sh` 自身也必须先验证清单，拒绝缺失、额外或被修改的产物。清单只作为流水线审计输入，不复制到发布分支。一次发布中五个分支提交携带共同 generation id 和 source SHA；任一平台 tree 改变时完整 cohort 原子推进并保留各分支父历史，全部 tree 不变时整体跳过。 `scripts/tools/artifact_origins.py` 是 `artifact-origins.json` 的唯一写入口：完整同步重置为 `generated-upstream`，发布分支恢复重置为 `restored-published-branch`，自定义构建只重标本次控制且实际存在的目标，并清除对应平台已经删除或降级省略的旧记录。 `config/domain-platform-capabilities.json` 是平台 branch、extension、format、rule mapping、empty policy、compiler 与 verifier 的唯一结构化来源。`scripts/tools/platform_capabilities.py` 严格加载并校验实现标识，同时向 Python 消费者提供查询对象、向 shell 消费者生成稳定的 tab-separated registry。IP 渲染、构建/守卫的安全循环、摘要和发布均查询该 registry；声明未知实现时构建会 fail closed。公开分支名仍由能力文件中的 `branch` 字段固定。 -`upstream-summary.json` 不是完整来源追溯记录：它不覆盖自定义源（包括 `fakeip-filter`）、全部转换步骤、上游提交身份或内容校验和。上述文件也都不是许可证证明。 +`upstream-summary.json` 不是完整来源追溯记录：它不覆盖自定义源（包括 `fakeip-filter`）、全部转换步骤或 HTTP 响应身份。上述文件也都不是许可证证明。 ## 名称冲突检查 @@ -60,7 +62,7 @@ - 两个平台部分内置 IP 集的总数与 IPv4/IPv6 最低值； - Surge 上部分内置 IP 集相对基线的增长或删除检查。 -它不对所有 `.srs` / `.mrs` 执行等价语义检查，也不审查许可。发布脚本另行检查发布树、扩展名和本地产物完整性。 +artifact guard 本身不解析 `.srs` / `.mrs`，二进制读回与精确语义关联由随后生成和复验 manifest 的阶段执行；两者都不审查许可。发布脚本另行检查发布树、扩展名和本地产物完整性。 ## 工具缓存与清理 diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index 5e7ef4bc6..5f2d77a7e 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -26,7 +26,7 @@ make build-custom ## `make clean` 后工具仍存在 -这是预期行为。清理会删除 `.tmp/`、`.output/`、Python 缓存及 `.bin/*.new*`，但保留已缓存的 sing-box、mihomo 和 provenance sidecar。若需要强制重新下载，应人工删除对应二进制及其 `.provenance.json`；不要提交 `.bin/`。 +这是预期行为。清理会删除 `.tmp/`、`.output/`、`.artifacts/`、Python 缓存及 `.bin/*.new*`，但保留已缓存的 sing-box、mihomo 和 provenance sidecar。若需要强制重新下载，应人工删除对应二进制及其 `.provenance.json`；不要提交 `.bin/`。 ## 自定义名称冲突 @@ -45,7 +45,9 @@ make build-custom 3. 成功后查看 `.output/domain/rule-manifest.json` 定位属性派生。 4. 对照基线和发布分支判断变化是否预期。 -该摘要不包括本仓库维护的自定义源（包括 `sources/custom/domain/fakeip-filter.list`），也没有完整转换链、提交身份或内容校验和，不能单独作为完整来源追溯记录或许可依据。`fakeip-filter` 应由 `build-custom.sh` 生成；若日志出现第三方 URL、独立同步或预编译下载，应视为迁移回归。过去的 `wwqgtxx/clash-rules` 二进制仅属历史。 +CI 的失败事务会把 `.artifacts/diagnostics/` 上传为保留 7 天的 diagnostics artifact。若日志被截断，优先下载对应 artifact；仅文档或治理文件的 PR 不运行外部上游同步。 + +该摘要不包括本仓库维护的自定义源（包括 `sources/custom/domain/fakeip-filter.list`），也没有完整转换链或 HTTP 响应身份，不能单独作为完整来源追溯记录或许可依据。`fakeip-filter` 应由 `build-custom.sh` 生成；若日志出现独立同步或预编译下载，应视为回归。 ## 找不到 `build-summary.json` @@ -53,7 +55,7 @@ make build-custom ## 产物守卫（artifact guard）阻断 -检查最低文件数、冗余派生名、文本域名下降、Surge/Quantumult X 文本 IP 合法性和部分内置 IP 基线。二进制格式没有全部执行等价语义检查。只有在来源证据、测试和评审说明齐全时才调整阈值。 +检查最低文件数、冗余派生名、文本域名下降、Surge/Quantumult X 文本 IP 合法性和部分内置 IP 基线。随后 manifest 阶段会真实读回二进制并精确比较可关联的规范规则集合。只有在来源证据、测试和评审说明齐全时才调整阈值。 ## 许可状态不明 diff --git a/scripts/commands/build-custom.sh b/scripts/commands/build-custom.sh index 2f3662649..a17c9b0e1 100755 --- a/scripts/commands/build-custom.sh +++ b/scripts/commands/build-custom.sh @@ -118,27 +118,12 @@ artifact_was_generated_custom() { printf '%s\n' "$GENERATED_CUSTOM_ARTIFACTS" | grep -Fxq "$relative_path" } -historical_fakeip_migration_collision() { - local base_ref="$1" - local custom_rel_path="$2" - local tracked_path="$3" - - # This exception is available only while migrating a base commit that - # actually tracked the old binary but did not yet track the maintained source. - # Once the source exists in the base, the normal existing-source path applies. - [ -n "$base_ref" ] \ - && [ "$custom_rel_path" = "sources/custom/domain/fakeip-filter.list" ] \ - && [ "$tracked_path" = ".output/domain/mihomo/fakeip-filter.mrs" ] \ - && git cat-file -e "$base_ref:$tracked_path" 2>/dev/null -} - assert_no_name_conflict() { local base="$1" local custom_rel_path="$2" local custom_dir="$3" - local base_ref="$4" - local base_custom_sources="$5" - shift 5 + local base_custom_sources="$4" + shift 4 local conflicts=() local tracked_path @@ -147,9 +132,6 @@ assert_no_name_conflict() { fi for tracked_path in "$@"; do - if historical_fakeip_migration_collision "$base_ref" "$custom_rel_path" "$tracked_path"; then - continue - fi if [ -e "$ARTIFACT_ROOT/${tracked_path#.output/}" ] \ && ! artifact_was_generated_custom "${tracked_path#.output/}"; then conflicts+=("$tracked_path") @@ -267,7 +249,6 @@ while IFS= read -r list_file; do "$base" \ "sources/custom/domain/$base.list" \ "$CUSTOM_DOMAIN_DIR" \ - "$CONFLICT_BASE_REF" \ "$BASE_CUSTOM_SOURCES" \ ".output/domain/surge/$base.list" \ ".output/domain/quanx/$base.list" \ @@ -284,7 +265,6 @@ while IFS= read -r list_file; do "$base" \ "sources/custom/ip/$base.list" \ "$CUSTOM_IP_DIR" \ - "$CONFLICT_BASE_REF" \ "$BASE_CUSTOM_SOURCES" \ ".output/ip/surge/$base.list" \ ".output/ip/quanx/$base.list" \ diff --git a/scripts/commands/clean.sh b/scripts/commands/clean.sh index e785505e8..3022547dc 100755 --- a/scripts/commands/clean.sh +++ b/scripts/commands/clean.sh @@ -4,7 +4,7 @@ set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" cd "$ROOT" -rm -rf .tmp .output +rm -rf .tmp .output .artifacts rm -f .bin/*.new .bin/*.new.* .bin/.*.provenance.* .bin/sing-box.new.tar.gz .bin/mihomo.new.gz find scripts -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/scripts/commands/publish-branches.sh b/scripts/commands/publish-branches.sh index 9528d7644..84f86c20c 100755 --- a/scripts/commands/publish-branches.sh +++ b/scripts/commands/publish-branches.sh @@ -5,7 +5,6 @@ ROOT="$(cd "$(dirname "$0")/../.." && pwd)" cd "$ROOT" DRY_RUN="${PUBLISH_DRY_RUN:-0}" -ALLOW_REMOTE_FALLBACK="${PUBLISH_ALLOW_REMOTE_FALLBACK:-0}" ARTIFACT_ROOT="${RULES_ARTIFACT_ROOT:-$ROOT/.output}" MANIFEST_FILE="$ARTIFACT_ROOT/artifact-manifest.json" CAPABILITY_REGISTRY="$(python3 "$ROOT/scripts/tools/platform_capabilities.py" shell-registry)" @@ -72,70 +71,18 @@ copy_artifacts() { fi } -restore_remote_publish_side() { - local branch="$1" - local side="$2" - local extensions_csv="$3" - local dest_dir="$4" - local tree_path="$side" - local -a extensions=() - local extension restored=0 - local file_path rel_name - - IFS=',' read -r -a extensions <<< "$extensions_csv" - mkdir -p "$dest_dir" - - if ! git -C "$ROOT" rev-parse --verify "origin/$branch^{commit}" >/dev/null 2>&1; then - return 1 - fi - - while IFS= read -r file_path; do - [ -n "$file_path" ] || continue - rel_name="${file_path#"${tree_path}"/}" - mkdir -p "$dest_dir/$(dirname "$rel_name")" - if git -C "$ROOT" show "origin/$branch:$file_path" > "$dest_dir/$rel_name"; then - restored=1 - else - rm -f "$dest_dir/$rel_name" - return 1 - fi - done < <( - for extension in "${extensions[@]}"; do - git -C "$ROOT" ls-tree -r --name-only "origin/$branch" -- "$tree_path" 2>/dev/null | grep -E "\\.${extension}$" || true - done - ) - - [ "$restored" -eq 1 ] -} - prepare_publish_side() { - local branch="$1" - local src_dir="$2" - local side="$3" - local extensions_csv="$4" - local dest_dir="$5" + local src_dir="$1" + local extensions_csv="$2" + local dest_dir="$3" if has_publish_source_artifacts "$src_dir" "$extensions_csv"; then copy_artifacts "$src_dir" "$dest_dir" "$extensions_csv" return 0 fi - if [ "$ALLOW_REMOTE_FALLBACK" != "1" ]; then - echo "publish source missing artifacts: $ARTIFACT_ROOT/$src_dir (expected: $extensions_csv)" >&2 - echo "hint: run the build pipeline first to populate .output before publishing" >&2 - echo "hint: set PUBLISH_ALLOW_REMOTE_FALLBACK=1 to explicitly allow origin/$branch:$side fallback" >&2 - exit 1 - fi - - echo "publish source missing local artifacts for $src_dir; attempting fallback from origin/$branch:$side" >&2 - if restore_remote_publish_side "$branch" "$side" "$extensions_csv" "$dest_dir"; then - echo "restored $side artifacts from origin/$branch baseline" >&2 - return 0 - fi - echo "publish source missing artifacts: $ARTIFACT_ROOT/$src_dir (expected: $extensions_csv)" >&2 echo "hint: run the build pipeline first to populate .output before publishing" >&2 - echo "hint: or ensure origin/$branch contains baseline $side artifacts for fallback restore" >&2 exit 1 } @@ -205,6 +152,24 @@ queue_publish_ref() { printf '%s\t%s\t%s\n' "$branch" "$commit" "$remote_commit" >> "$PUBLISH_QUEUE_FILE" } +create_publish_commit() { + local branch="$1" + local tree="$2" + local remote_commit="$3" + local message commit + local -a parent_args=() + + message="chore: publish ${branch} artifacts [generation ${MANIFEST_GENERATION_ID} source ${MANIFEST_SOURCE_SHA}]" + if [ -n "$remote_commit" ]; then + git cat-file -e "${remote_commit}^{commit}" + parent_args=(-p "$remote_commit") + fi + + commit="$(git commit-tree "$tree" "${parent_args[@]}" -m "$message")" + git update-ref "refs/heads/$branch" "$commit" + printf '%s' "$commit" +} + prepare_branch() { local branch="$1" local domain_dir="$2" @@ -214,23 +179,25 @@ prepare_branch() { local local_tree remote_tree remote_commit commit reset_publish_worktree "$branch" - prepare_publish_side "$branch" "$domain_dir" domain "$domain_extensions" domain - prepare_publish_side "$branch" "$ip_dir" ip "$ip_extensions" ip + prepare_publish_side "$domain_dir" "$domain_extensions" domain + prepare_publish_side "$ip_dir" "$ip_extensions" ip branch_readme "$branch" assert_branch_layout "$domain_extensions" "$ip_extensions" git add README.md domain ip local_tree="$(git write-tree)" remote_tree="$(git -C "$ROOT" rev-parse --verify "origin/$branch^{tree}" 2>/dev/null || true)" + remote_commit="$(git -C "$ROOT" rev-parse --verify "origin/$branch^{commit}" 2>/dev/null || true)" - if [ -n "$remote_tree" ] && [ "$local_tree" = "$remote_tree" ]; then - echo "$branch artifacts unchanged, skip publish" - return 0 + if [ -z "$remote_tree" ] || [ "$local_tree" != "$remote_tree" ]; then + PUBLISH_COHORT_CHANGED=1 + echo "$branch artifact tree changed" + else + echo "$branch artifact tree unchanged; cohort metadata commit prepared" fi - git commit -m "chore: publish ${branch} artifacts [generation ${MANIFEST_GENERATION_ID} source ${MANIFEST_SOURCE_SHA}]" >/dev/null - commit="$(git rev-parse HEAD)" - remote_commit="$(git -C "$ROOT" rev-parse --verify "origin/$branch^{commit}" 2>/dev/null || true)" + commit="$(create_publish_commit "$branch" "$local_tree" "$remote_commit")" + queue_publish_ref "$branch" "$commit" "$remote_commit" if [ "$DRY_RUN" = "1" ]; then echo "=== ${branch} publish dry-run ===" @@ -238,8 +205,6 @@ prepare_branch() { echo "ip files: $(find ip -maxdepth 1 -type f | wc -l | tr -d ' ')" return 0 fi - - queue_publish_ref "$branch" "$commit" "$remote_commit" } publish_queued_refs() { @@ -252,11 +217,16 @@ publish_queued_refs() { return 0 fi - if [ ! -s "$PUBLISH_QUEUE_FILE" ]; then + if [ "$PUBLISH_COHORT_CHANGED" -eq 0 ]; then echo "all publish branches unchanged, skip push" return 0 fi + if [ ! -s "$PUBLISH_QUEUE_FILE" ]; then + echo "publish cohort changed but no refs were prepared" >&2 + return 1 + fi + remote_url="$(git -C "$ROOT" remote get-url origin)" if [[ "$remote_url" == https://github.com/* ]] && [ -n "${GITHUB_TOKEN:-}" ]; then remote_url="https://x-access-token:${GITHUB_TOKEN}@${remote_url#https://}" @@ -292,6 +262,10 @@ cd "$PUBLISH_WORKTREE" git init -q git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" +mkdir -p .git/objects/info +git -C "$ROOT" rev-parse --path-format=absolute --git-path objects > .git/objects/info/alternates + +PUBLISH_COHORT_CHANGED=0 declare -A PUBLISH_BRANCH PUBLISH_DOMAIN_EXTENSION PUBLISH_IP_EXTENSION while IFS=$'\t' read -r platform _public_name branch section extension _format _empty _compiler _verifier; do diff --git a/scripts/commands/restore-artifacts.sh b/scripts/commands/restore-artifacts.sh index 716f3fad6..21638ea25 100755 --- a/scripts/commands/restore-artifacts.sh +++ b/scripts/commands/restore-artifacts.sh @@ -11,7 +11,6 @@ fi ARTIFACT_ROOT="${RULES_ARTIFACT_ROOT:-$ROOT/.output}" TMP_ROOT="$ROOT/.tmp/restore-published" RESTORE_METADATA_FILE="$TMP_ROOT/restored-branches.tsv" -ALLOW_LOSSY_FALLBACK="${RULES_ALLOW_LOSSY_RESTORE_FALLBACK:-0}" inject_restore_failure() { local point="$1" @@ -21,9 +20,6 @@ inject_restore_failure() { fi } -# shellcheck source=scripts/lib/rules.sh -source "$ROOT/scripts/lib/rules.sh" - rm -rf "$TMP_ROOT" mkdir -p "$TMP_ROOT" : > "$RESTORE_METADATA_FILE" @@ -39,99 +35,11 @@ fetch_branch_ref() { git fetch --depth=1 origin "+${branch}:refs/remotes/origin/${branch}" >/dev/null 2>&1 } -generate_quanx_from_restored_surge() { - local domain_src="$ARTIFACT_ROOT/domain/surge" - local ip_src="$ARTIFACT_ROOT/ip/surge" - local domain_dst="$ARTIFACT_ROOT/domain/quanx" - local ip_dst="$ARTIFACT_ROOT/ip/quanx" - local tmpdir="$TMP_ROOT/quanx-fallback" - local list base domain_tmp plain_tmp ip_tmp - - if [ ! -d "$domain_src" ] || [ ! -d "$ip_src" ]; then - echo "cannot build quanx fallback without restored surge artifacts" >&2 - return 1 - fi - - rm -rf "$domain_dst" "$ip_dst" "$tmpdir" - mkdir -p "$domain_dst" "$ip_dst" "$tmpdir" - - for list in "$domain_src"/*.list; do - [ -f "$list" ] || continue - base="$(basename "$list" .list)" - domain_tmp="$tmpdir/$base.domain.tmp" - render_quanx_domain_ruleset_from_rules "$list" "$domain_tmp" "$base" - mv "$domain_tmp" "$domain_dst/$base.list" - done - - for list in "$ip_src"/*.list; do - [ -f "$list" ] || continue - base="$(basename "$list" .list)" - plain_tmp="$tmpdir/$base.ip.plain.tmp" - ip_tmp="$tmpdir/$base.ip.tmp" - normalize_ip_surge_list_to_plain "$list" "$plain_tmp" - render_ip_plain_to_quanx_list "$plain_tmp" "$ip_tmp" "$base" - mv "$ip_tmp" "$ip_dst/$base.list" - done - - echo "generated quanx fallback artifacts from surge baseline" -} - -generate_egern_from_restored_surge() { - local domain_src="$ARTIFACT_ROOT/domain/surge" - local ip_src="$ARTIFACT_ROOT/ip/surge" - local domain_dst="$ARTIFACT_ROOT/domain/egern" - local ip_dst="$ARTIFACT_ROOT/ip/egern" - local tmpdir="$TMP_ROOT/egern-fallback" - local list base domain_tmp plain_tmp ip_tmp - - if [ ! -d "$domain_src" ] || [ ! -d "$ip_src" ]; then - echo "cannot build egern fallback without restored surge artifacts" >&2 - return 1 - fi - - rm -rf "$domain_dst" "$ip_dst" "$tmpdir" - mkdir -p "$domain_dst" "$ip_dst" "$tmpdir" - - for list in "$domain_src"/*.list; do - [ -f "$list" ] || continue - base="$(basename "$list" .list)" - domain_tmp="$tmpdir/$base.domain.tmp" - render_egern_domain_ruleset_from_rules "$list" "$domain_tmp" - mv "$domain_tmp" "$domain_dst/$base.yaml" - done - - for list in "$ip_src"/*.list; do - [ -f "$list" ] || continue - base="$(basename "$list" .list)" - plain_tmp="$tmpdir/$base.ip.plain.tmp" - ip_tmp="$tmpdir/$base.ip.tmp" - normalize_ip_surge_list_to_plain "$list" "$plain_tmp" - render_ip_plain_to_egern_yaml "$plain_tmp" "$ip_tmp" - mv "$ip_tmp" "$ip_dst/$base.yaml" - done - - echo "generated egern fallback artifacts from surge baseline" -} - restore_branch_artifacts() { local branch="$1" local tmpdir="$TMP_ROOT/$branch" if ! remote_branch_exists "$branch"; then - if [ "$ALLOW_LOSSY_FALLBACK" != "1" ]; then - echo "required remote branch origin/$branch not found; lossy cross-platform fallback is disabled" >&2 - return 1 - fi - if [ "$branch" = "quanx" ]; then - echo "origin/quanx not found, building quanx baseline from restored surge artifacts" - generate_quanx_from_restored_surge - return 0 - fi - if [ "$branch" = "egern" ]; then - echo "origin/egern not found, building egern baseline from restored surge artifacts" - generate_egern_from_restored_surge - return 0 - fi echo "required remote branch origin/$branch not found" >&2 return 1 fi diff --git a/scripts/commands/select-build-scope.sh b/scripts/commands/select-build-scope.sh index 5ed6d982c..b46103e34 100755 --- a/scripts/commands/select-build-scope.sh +++ b/scripts/commands/select-build-scope.sh @@ -37,9 +37,9 @@ collect_changed_files() { fi if [ -n "$before" ]; then - git diff --name-only "$before" "$current" + git diff --no-renames --name-only "$before" "$current" else - git diff-tree --no-commit-id --name-only -r "$current" + git diff-tree --no-renames --no-commit-id --name-only -r "$current" fi } @@ -57,12 +57,20 @@ collect_deleted_custom_files() { fi if [ -n "$before" ]; then - git diff --name-only --diff-filter=D "$before" "$current" -- 'sources/custom/**' + git diff --no-renames --name-only --diff-filter=D "$before" "$current" -- 'sources/custom/**' else - git diff-tree --no-commit-id --name-only --diff-filter=D -r "$current" -- 'sources/custom/**' + git diff-tree --no-renames --no-commit-id --name-only --diff-filter=D -r "$current" -- 'sources/custom/**' fi } +has_build_relevant_changes() { + grep -Eq '^(\.github/workflows/|Makefile$|config/|scripts/|sources/custom/|templates/|tests/)' +} + +has_only_non_build_changes() { + ! grep -Eqv '^(\.github/(CODEOWNERS$|ISSUE_TEMPLATE/|dependabot\.yml$|pull_request_template\.md$)|\.gitignore$|CONTRIBUTING\.md$|LICENSE$|NOTICE$|README\.md$|SECURITY\.md$|THIRD_PARTY_NOTICES\.md$|docs/)' +} + if [ "$EVENT_NAME" = "workflow_dispatch" ]; then case "$INPUT_SCOPE" in custom) @@ -99,9 +107,9 @@ if [ "$EVENT_NAME" = "workflow_dispatch" ]; then exit 1 ;; esac -elif [ "$EVENT_NAME" = "push" ]; then +elif [ "$EVENT_NAME" = "push" ] || [ "$EVENT_NAME" = "pull_request" ]; then before="$BEFORE_SHA" - if [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; then + if [ "$EVENT_NAME" = "push" ] && { [ -z "$before" ] || [ "$before" = "0000000000000000000000000000000000000000" ]; }; then before="$(git rev-parse "${CURRENT_SHA}^" 2>/dev/null || true)" fi base_sha="$before" @@ -112,17 +120,17 @@ elif [ "$EVENT_NAME" = "push" ]; then ! git cat-file -e "${CURRENT_SHA}^{commit}" 2>/dev/null }; then scope="full" - reason="push base unavailable; using full sync" + reason="$EVENT_NAME base unavailable; using full sync" base_sha="" else if ! changed_files="$(collect_changed_files "$before" "$CURRENT_SHA")"; then scope="full" - reason="push diff unavailable; using full sync" + reason="$EVENT_NAME diff unavailable; using full sync" base_sha="" changed_files="" elif ! deleted_custom_files="$(collect_deleted_custom_files "$before" "$CURRENT_SHA")"; then scope="full" - reason="push diff unavailable; using full sync" + reason="$EVENT_NAME diff unavailable; using full sync" base_sha="" deleted_custom_files="" else @@ -131,16 +139,28 @@ elif [ "$EVENT_NAME" = "push" ]; then if [ -z "$changed_files" ]; then scope="full" - reason="push diff empty; using full sync" + reason="$EVENT_NAME diff empty; using full sync" elif [ -n "$deleted_custom_files" ]; then scope="full" reason="custom deletions require full sync" + elif ! printf '%s\n' "$changed_files" | has_build_relevant_changes; then + if [ "$EVENT_NAME" = "pull_request" ] \ + && printf '%s\n' "$changed_files" | has_only_non_build_changes; then + scope="none" + reason="pull_request has no build-relevant changes" + elif [ "$EVENT_NAME" = "pull_request" ]; then + scope="full" + reason="pull_request includes unclassified changes" + else + scope="full" + reason="push includes non-custom changes" + fi elif printf '%s\n' "$changed_files" | grep -Eqv '^sources/custom/'; then scope="full" - reason="push includes non-custom changes" + reason="$EVENT_NAME includes changes outside custom sources" else scope="custom" - reason="push only updates custom sources" + reason="$EVENT_NAME only updates custom sources" fi fi fi diff --git a/scripts/commands/sync-upstream.sh b/scripts/commands/sync-upstream.sh index 8e937facf..374840263 100755 --- a/scripts/commands/sync-upstream.sh +++ b/scripts/commands/sync-upstream.sh @@ -59,6 +59,7 @@ TELEGRAM_IP_SOURCE_URL="$(upstream_value ip telegram url)" CLOUDFLARE_IPV4_SOURCE_URL="$(upstream_value ip cloudflare-ipv4 url)" CLOUDFLARE_IPV6_SOURCE_URL="$(upstream_value ip cloudflare-ipv6 url)" AWS_IP_SOURCE_URL="$(upstream_value ip aws url)" +CLOUDFRONT_IP_SOURCE_URL="$(upstream_value ip cloudfront url)" FASTLY_IP_SOURCE_URL="$(upstream_value ip fastly url)" GITHUB_IP_SOURCE_URL="$(upstream_value ip github url)" APPLE_IP_SOURCE_URL="$(upstream_value ip apple url)" @@ -118,7 +119,7 @@ record_upstream_summary() { local detail="${8:-}" python3 - <<'PY' "$UPSTREAMS_CONFIG_FILE" "$UPSTREAM_SUMMARY_FILE" "$category" "$name" "$status" "$url" "$raw_file" "$normalized_file" "$fallback_used" "$detail" -import json, sys +import hashlib, json, sys from pathlib import Path config_file, summary_file, category, name, status, url, raw_file, normalized_file, fallback_used, detail = sys.argv[1:] config = json.load(open(config_file, encoding="utf-8")) @@ -139,11 +140,24 @@ for key, file_name in (("raw", raw_file), ("normalized", normalized_file)): continue path = Path(file_name) info = {"path": str(path)} - if path.exists(): - info["bytes"] = path.stat().st_size - if path.is_file(): - lines = [line for line in path.read_text(encoding="utf-8", errors="ignore").splitlines() if line.strip() and not line.lstrip().startswith("#")] - info["entries"] = len(lines) + if path.is_file(): + content = path.read_bytes() + info["bytes"] = len(content) + info["sha256"] = hashlib.sha256(content).hexdigest() + lines = [line for line in content.decode("utf-8", errors="ignore").splitlines() if line.strip() and not line.lstrip().startswith("#")] + info["entries"] = len(lines) + elif path.is_dir(): + files = sorted(candidate for candidate in path.rglob("*") if candidate.is_file()) + digest = hashlib.sha256() + total_bytes = 0 + for candidate in files: + relative = candidate.relative_to(path).as_posix().encode() + content = candidate.read_bytes() + digest.update(relative + b"\0" + content + b"\0") + total_bytes += len(content) + info["bytes"] = total_bytes + info["entries"] = len(files) + info["sha256"] = digest.hexdigest() payload[key] = info Path(summary_file).parent.mkdir(parents=True, exist_ok=True) with open(summary_file, "a", encoding="utf-8") as fh: @@ -151,6 +165,48 @@ with open(summary_file, "a", encoding="utf-8") as fh: PY } +verify_and_record_upstream_health() { + local category="$1" + local name="$2" + local url="$3" + local raw_file="$4" + local normalized_file="$5" + local fallback_used="${6:-0}" + local context="${7:-}" + local health_json status detail health_detail verifier_failed=0 + + if health_json="$(python3 "$ROOT_DIR/scripts/tools/verify-upstream-health.py" \ + "$UPSTREAMS_CONFIG_FILE" "$category" "$name" "$raw_file" "$normalized_file")"; then + : + else + verifier_failed=1 + fi + if ! status="$(printf '%s' "$health_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])' 2>/dev/null)"; then + status=semantic_regression + verifier_failed=1 + fi + case "$status" in + ok|semantic_regression) ;; + *) status=semantic_regression; verifier_failed=1 ;; + esac + if [ "$status" = "semantic_regression" ] && [ "$verifier_failed" -eq 0 ]; then + detail="optional source failed health policy" + else + detail="" + fi + health_detail="$(printf '%s' "$health_json" | python3 -c 'import json,sys; print("; ".join(json.load(sys.stdin).get("errors", [])))' 2>/dev/null || printf 'health verifier failed')" + detail="${health_detail:-$detail}" + if [ -n "$context" ]; then + detail="${context}${detail:+; $detail}" + fi + record_upstream_summary \ + "$category" "$name" "$status" "$url" "$raw_file" "$normalized_file" "$fallback_used" "$detail" + if [ "$verifier_failed" -ne 0 ]; then + echo "required upstream $name failed configured health policy: $detail" >&2 + return 1 + fi +} + write_upstream_summary_json() { local json_file="$ARTIFACTS_DIR/upstream-summary.json" python3 - <<'PY' "$UPSTREAM_SUMMARY_FILE" "$json_file" @@ -495,6 +551,24 @@ first_batch_source_type() { esac } +first_batch_config_name() { + case "$1" in + google-json) printf '%s' google ;; + github-json) printf '%s' github ;; + telegram) printf '%s' telegram ;; + *) echo "unsupported first-batch source: $1" >&2; return 1 ;; + esac +} + +first_batch_source_url() { + case "$1" in + google-json) printf '%s' "$GOOGLE_IP_SOURCE_URL" ;; + github-json) printf '%s' "$GITHUB_IP_SOURCE_URL" ;; + telegram) printf '%s' "$TELEGRAM_IP_SOURCE_URL" ;; + *) echo "unsupported first-batch source: $1" >&2; return 1 ;; + esac +} + classify_first_batch_source() { local source="$1" local raw_file result_json status reason @@ -511,7 +585,17 @@ classify_first_batch_source() { reason="$(printf '%s' "$result_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["reason"])')" set_first_batch_result "$source" "$status" "$reason" - record_upstream_summary ip "$source" "$status" "" "$raw_file" "" 0 "$reason" + if [ "$status" != "ok" ]; then + record_upstream_summary \ + ip \ + "$(first_batch_config_name "$source")" \ + "$status" \ + "$(first_batch_source_url "$source")" \ + "$raw_file" \ + "" \ + 0 \ + "$reason" + fi } download_and_classify_first_batch_source() { @@ -548,7 +632,18 @@ normalize_first_batch_source() { ;; esac - python3 "$ROOT_DIR/scripts/tools/normalize-ip-rules.py" single "$source_type" "$raw_file" "$output_file" + if ! python3 "$ROOT_DIR/scripts/tools/normalize-ip-rules.py" single "$source_type" "$raw_file" "$output_file"; then + record_upstream_summary \ + ip \ + "$(first_batch_config_name "$source")" \ + semantic_regression \ + "$(first_batch_source_url "$source")" \ + "$raw_file" \ + "$output_file" \ + 0 \ + "normalization failed" + return 1 + fi } summarize_first_batch_checks() { @@ -697,13 +792,18 @@ clone_repository_shallow "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-co python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" export \ "$WORK_TMP_DIR/domain-list-community/data" \ "$DOMAIN_RULE_TMP_DIR" -record_upstream_summary domain dlc ok "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-community/data" "$DOMAIN_RULE_TMP_DIR" python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" domain-rule-manifest \ "$DOMAIN_RULE_TMP_DIR" \ "$DOMAIN_RULE_MANIFEST_FILE" assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE" -python3 "$ROOT_DIR/scripts/tools/verify-upstream-health.py" "$UPSTREAMS_CONFIG_FILE" domain dlc \ - "$WORK_TMP_DIR/domain-list-community/data" "$DOMAIN_RULE_TMP_DIR" +verify_and_record_upstream_health \ + domain \ + dlc \ + "$DOMAIN_SOURCE_REPO_URL" \ + "$WORK_TMP_DIR/domain-list-community/data" \ + "$DOMAIN_RULE_TMP_DIR" \ + 0 \ + "commit=$(git -C "$WORK_TMP_DIR/domain-list-community" rev-parse HEAD)" assert_files_present "$DOMAIN_RULE_TMP_DIR" "$DOMAIN_RULE_TMP_DIR/*.list" render_domain_rule_dir_to_text_platform_dirs \ "$DOMAIN_RULE_TMP_DIR" \ @@ -743,42 +843,40 @@ download_file_with_fallback \ "$IP_BUILD_TMP_DIR/apple.raw.html" \ "$APPLE_IP_SOURCE_URL" \ "$APPLE_IP_SOURCE_FALLBACK_URL" +APPLE_RESOLVED_URL="${UPSTREAM_LAST_URL:-$APPLE_IP_SOURCE_URL}" +APPLE_FALLBACK_USED="${UPSTREAM_LAST_FALLBACK_USED:-0}" IP_NORMALIZE_MANIFEST="$IP_BUILD_TMP_DIR/normalize-tasks.json" generate_ip_normalize_manifest "$IP_NORMALIZE_MANIFEST" python3 "$ROOT_DIR/scripts/tools/normalize-ip-rules.py" batch "$IP_NORMALIZE_MANIFEST" -record_upstream_summary ip cn-ipv46 ok "$CN_IPV46_SOURCE_URL" "$IP_BUILD_TMP_DIR/cn_ipv46.raw.txt" "$IP_BUILD_TMP_DIR/cn_ipv46.cidr.txt" -record_upstream_summary ip cn-ipv46-apnic ok "$CN_IPV46_APNIC_SOURCE_URL" "$IP_BUILD_TMP_DIR/cn_ipv46_apnic.raw.txt" "$IP_BUILD_TMP_DIR/cn_ipv46_apnic.cidr.txt" -record_upstream_summary ip loyalsoldier-geoip-cn ok "$LOYALSOLDIER_GEOIP_CN_SOURCE_URL" "$IP_BUILD_TMP_DIR/loyalsoldier_geoip_cn.raw.txt" "$IP_BUILD_TMP_DIR/loyalsoldier_geoip_cn.cidr.txt" -record_upstream_summary ip loyalsoldier-geoip-private ok "$LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL" "$IP_BUILD_TMP_DIR/private.raw.txt" "$IP_BUILD_TMP_DIR/private.cidr.txt" -record_upstream_summary ip cloudflare-ipv4 ok "$CLOUDFLARE_IPV4_SOURCE_URL" "$IP_BUILD_TMP_DIR/cloudflare_ipv4.raw.txt" "$IP_BUILD_TMP_DIR/cloudflare_ipv4.cidr.txt" -record_upstream_summary ip cloudflare-ipv6 ok "$CLOUDFLARE_IPV6_SOURCE_URL" "$IP_BUILD_TMP_DIR/cloudflare_ipv6.raw.txt" "$IP_BUILD_TMP_DIR/cloudflare_ipv6.cidr.txt" -record_upstream_summary ip aws ok "$AWS_IP_SOURCE_URL" "$IP_BUILD_TMP_DIR/aws.raw.json" "$IP_BUILD_TMP_DIR/aws.cidr.txt" -record_upstream_summary ip cloudfront ok "$AWS_IP_SOURCE_URL" "$IP_BUILD_TMP_DIR/aws.raw.json" "$IP_BUILD_TMP_DIR/cloudfront.cidr.txt" -record_upstream_summary ip fastly ok "$FASTLY_IP_SOURCE_URL" "$IP_BUILD_TMP_DIR/fastly.raw.json" "$IP_BUILD_TMP_DIR/fastly.cidr.txt" -record_upstream_summary ip apple ok "${UPSTREAM_LAST_URL:-$APPLE_IP_SOURCE_URL}" "$IP_BUILD_TMP_DIR/apple.raw.html" "$IP_BUILD_TMP_DIR/apple.cidr.txt" "${UPSTREAM_LAST_FALLBACK_USED:-0}" summarize_first_batch_checks normalize_first_batch_source "google-json" normalize_first_batch_source "github-json" normalize_first_batch_source "telegram" -for health_spec in \ - "cn-ipv46 cn_ipv46.raw.txt cn_ipv46.cidr.txt" \ - "cn-ipv46-apnic cn_ipv46_apnic.raw.txt cn_ipv46_apnic.cidr.txt" \ - "loyalsoldier-geoip-cn loyalsoldier_geoip_cn.raw.txt loyalsoldier_geoip_cn.cidr.txt" \ - "loyalsoldier-geoip-private private.raw.txt private.cidr.txt" \ - "google google.raw.json google.cidr.txt" \ - "telegram telegram.raw.txt telegram.cidr.txt" \ - "cloudflare-ipv4 cloudflare_ipv4.raw.txt cloudflare_ipv4.cidr.txt" \ - "cloudflare-ipv6 cloudflare_ipv6.raw.txt cloudflare_ipv6.cidr.txt" \ - "aws aws.raw.json aws.cidr.txt" \ - "fastly fastly.raw.json fastly.cidr.txt" \ - "github github.raw.json github.cidr.txt" \ - "apple apple.raw.html apple.cidr.txt"; do - read -r health_name health_raw health_normalized <<< "$health_spec" - python3 "$ROOT_DIR/scripts/tools/verify-upstream-health.py" "$UPSTREAMS_CONFIG_FILE" ip "$health_name" \ - "$IP_BUILD_TMP_DIR/$health_raw" "$IP_BUILD_TMP_DIR/$health_normalized" -done +while IFS='|' read -r health_name health_url health_raw health_normalized fallback_used; do + verify_and_record_upstream_health \ + ip \ + "$health_name" \ + "$health_url" \ + "$IP_BUILD_TMP_DIR/$health_raw" \ + "$IP_BUILD_TMP_DIR/$health_normalized" \ + "$fallback_used" +done < "$REPO/.bin/sing-box" <<'EOF' diff --git a/scripts/tests/test-binary-artifact-verifier.sh b/scripts/tests/test-binary-artifact-verifier.sh index dc0277010..86b00893f 100644 --- a/scripts/tests/test-binary-artifact-verifier.sh +++ b/scripts/tests/test-binary-artifact-verifier.sh @@ -13,7 +13,13 @@ printf 'UNKNOWN,example.com\n' > "$TMP/bad-kind.list" printf 'DOMAIN,example.com,extra\n' > "$TMP/bad-shape.list" printf "domain_set:\n - 'example.com'\n" > "$TMP/good.yaml" printf "unknown_set:\n - 'example.com'\n" > "$TMP/bad.yaml" +printf 'HOST-SUFFIX,example.com,quanx\n' > "$TMP/quanx.list" +printf 'HOST-SUFFIX,example.com,reject\n' > "$TMP/quanx-bad.list" +printf 'IP-CIDR,192.0.2.0/24,quanx-ip\n' > "$TMP/quanx-ip.list" +printf 'IP-CIDR,192.0.2.0/24,reject\n' > "$TMP/quanx-ip-bad.list" PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/good.yaml" --type domain --platform egern >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/quanx.list" --type domain --platform quanx >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/quanx-ip.list" --type ip --platform quanx >/dev/null for fixture in bad-kind.list bad-shape.list; do if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/$fixture" --type domain --platform surge >/dev/null 2>&1; then echo "text verifier accepted $fixture" >&2; exit 1 @@ -22,17 +28,116 @@ done if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" --root "$TMP/text-repo" --path "$TMP/bad.yaml" --type domain --platform egern >/dev/null 2>&1; then echo "Egern verifier accepted unknown YAML field" >&2; exit 1 fi +for fixture in quanx-bad.list quanx-ip-bad.list; do + artifact_type=domain + [[ "$fixture" == quanx-ip-* ]] && artifact_type=ip + if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/text-repo" --path "$TMP/$fixture" --type "$artifact_type" --platform quanx >/dev/null 2>&1; then + echo "Quantumult X verifier accepted wrong policy in $fixture" >&2 + exit 1 + fi +done PYTHONPATH="$ROOT/scripts/tools" python3 - <<'PY' from collections import Counter -from artifact_verifier import singbox_counts +from artifact_verifier import semantic_digest, singbox_counts, singbox_entries domain = {"rules": [{"domain": "one.example", "domain_suffix": ["a.example", "b.example"], "domain_keyword": "emby"}]} assert singbox_counts(domain, "domain") == Counter({"DOMAIN-SUFFIX": 2, "DOMAIN": 1, "DOMAIN-KEYWORD": 1}) +assert singbox_entries(domain, "domain") == Counter({ + ("DOMAIN", "one.example"): 1, + ("DOMAIN-SUFFIX", "a.example"): 1, + ("DOMAIN-SUFFIX", "b.example"): 1, + ("DOMAIN-KEYWORD", "emby"): 1, +}) ip = {"rules": [{"ip_cidr": "192.0.2.0/24"}, {"ip_cidr": ["2001:db8::/32"]}]} assert singbox_counts(ip, "ip") == Counter({"IP-CIDR": 1, "IP-CIDR6": 1}) +assert semantic_digest(Counter({("DOMAIN", "one.example"): 1})) != semantic_digest( + Counter({("DOMAIN", "two.example"): 1}) +) +for invalid in ( + {"rules": [{"domain": ["one.example"], "port": [443]}]}, + {"rules": [{"ip_cidr": ["192.0.2.0/24"]}]}, + {"rules": [{}]}, +): + try: + singbox_entries(invalid, "domain") + except ValueError: + pass + else: + raise AssertionError(f"sing-box verifier accepted unsupported rule object: {invalid!r}") PY +mkdir -p \ + "$TMP/exact-repo/config" \ + "$TMP/exact-repo/.bin" \ + "$TMP/exact-repo/.output/domain/sing-box" \ + "$TMP/exact-repo/.output/domain/mihomo" \ + "$TMP/exact-repo/sources/custom/domain" +cp "$ROOT/config/domain-platform-capabilities.json" "$TMP/exact-repo/config/" +printf 'DOMAIN,fixture.example\nDOMAIN-SUFFIX,example.org\n' > "$TMP/exact-repo/sources/custom/domain/fixture.list" +printf 'fixture\n' > "$TMP/exact-repo/.output/domain/sing-box/fixture.srs" +printf 'fixture\n' > "$TMP/exact-repo/.output/domain/mihomo/fixture.mrs" +cat > "$TMP/exact-repo/.bin/sing-box" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +cp "${3}.json" "$5" +EOF +cat > "$TMP/exact-repo/.bin/mihomo" <<'EOF' +#!/usr/bin/env bash +set -euo pipefail +cp "${4}.txt" "$5" +EOF +chmod +x "$TMP/exact-repo/.bin/sing-box" "$TMP/exact-repo/.bin/mihomo" +cat > "$TMP/exact-repo/.output/domain/sing-box/fixture.srs.json" <<'EOF' +{"rules":[{"domain":["fixture.example"],"domain_suffix":["example.org"]}]} +EOF +printf 'fixture.example\n+.example.org\n' > "$TMP/exact-repo/.output/domain/mihomo/fixture.mrs.txt" +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/exact-repo" \ + --path "$TMP/exact-repo/.output/domain/sing-box/fixture.srs" \ + --type domain \ + --platform sing-box | grep -F '"canonical_linkage": {"counts"' >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/exact-repo" \ + --path "$TMP/exact-repo/.output/domain/mihomo/fixture.mrs" \ + --type domain \ + --platform mihomo | grep -F '"canonical_linkage": {"counts"' >/dev/null + +cat > "$TMP/exact-repo/.output/domain/sing-box/fixture.srs.json" <<'EOF' +{"rules":[{"domain":["replaced.example"],"domain_suffix":["example.org"]}]} +EOF +if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/exact-repo" \ + --path "$TMP/exact-repo/.output/domain/sing-box/fixture.srs" \ + --type domain \ + --platform sing-box >/dev/null 2>&1; then + echo "sing-box verifier accepted equal-count rule substitution" >&2 + exit 1 +fi +printf 'replaced.example\n+.example.org\n' > "$TMP/exact-repo/.output/domain/mihomo/fixture.mrs.txt" +if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/exact-repo" \ + --path "$TMP/exact-repo/.output/domain/mihomo/fixture.mrs" \ + --type domain \ + --platform mihomo >/dev/null 2>&1; then + echo "mihomo verifier accepted equal-count rule substitution" >&2 + exit 1 +fi + +printf 'orphan\n' > "$TMP/exact-repo/.output/domain/sing-box/orphan.srs" +cat > "$TMP/exact-repo/.output/domain/sing-box/orphan.srs.json" <<'EOF' +{"rules":[{"domain":["orphan.example"]}]} +EOF +if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/exact-repo" \ + --path "$TMP/exact-repo/.output/domain/sing-box/orphan.srs" \ + --type domain \ + --platform sing-box >/dev/null 2>&1; then + echo "binary verifier accepted artifact without canonical counterpart" >&2 + exit 1 +fi + if ! command -v sing-box >/dev/null 2>&1 || ! command -v mihomo >/dev/null 2>&1; then echo "binary verifier real-tool fixture skipped: sing-box and mihomo are not both available" exit 0 diff --git a/scripts/tests/test-build-scope.sh b/scripts/tests/test-build-scope.sh index c75464c57..5f8f8cb7a 100755 --- a/scripts/tests/test-build-scope.sh +++ b/scripts/tests/test-build-scope.sh @@ -39,7 +39,7 @@ assert_scope custom "push only updates custom sources" \ EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA=base \ CHANGED_FILES=$'sources/custom/domain/emby.list\nsources/custom/ip/example.list' -assert_scope full "push includes non-custom changes" \ +assert_scope full "push includes changes outside custom sources" \ EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA=base \ CHANGED_FILES=$'sources/custom/domain/emby.list\nscripts/commands/build-custom.sh' @@ -51,8 +51,38 @@ assert_scope full "custom deletions require full sync" \ assert_scope full "push base unavailable; using full sync" \ EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA=missing-build-scope-base -ROOT_COMMIT_REPO="$(mktemp -d)" -trap 'rm -rf "$ROOT_COMMIT_REPO"' EXIT +assert_scope none "pull_request has no build-relevant changes" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ + CHANGED_FILES=$'README.md\ndocs/DEVELOPMENT.md' + +assert_scope custom "pull_request only updates custom sources" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ + CHANGED_FILES=$'sources/custom/domain/fakeip-filter.list' + +assert_scope full "custom deletions require full sync" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ + CHANGED_FILES=$'sources/custom/domain/old.list' \ + DELETED_CUSTOM_FILES=$'sources/custom/domain/old.list' + +assert_scope full "pull_request includes changes outside custom sources" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ + CHANGED_FILES=$'README.md\nscripts/commands/build-custom.sh' + +assert_scope full "pull_request includes unclassified changes" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ + CHANGED_FILES=$'pyproject.toml' + +assert_scope full "pull_request includes changes outside custom sources" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ + CHANGED_FILES=$'sources/custom/domain/fakeip-filter.list\ntools/generate.py' + +assert_scope full "pull_request base unavailable; using full sync" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=missing-build-scope-base + +TEST_TMP="$(mktemp -d)" +trap 'rm -rf "$TEST_TMP"' EXIT +ROOT_COMMIT_REPO="$TEST_TMP/root-commit" +mkdir -p "$ROOT_COMMIT_REPO" git -C "$ROOT_COMMIT_REPO" init -q git -C "$ROOT_COMMIT_REPO" config user.name test git -C "$ROOT_COMMIT_REPO" config user.email test@example.com @@ -64,4 +94,19 @@ grep -Fx "scope=full" <<< "$root_output" >/dev/null grep -Fx "reason=push base unavailable; using full sync" <<< "$root_output" >/dev/null grep -Fx "base_sha=" <<< "$root_output" >/dev/null +RENAME_REPO="$TEST_TMP/rename" +mkdir -p "$RENAME_REPO/sources/custom/domain" +git -C "$RENAME_REPO" init -q +git -C "$RENAME_REPO" config user.name test +git -C "$RENAME_REPO" config user.email test@example.com +printf 'DOMAIN,old.example\n' > "$RENAME_REPO/sources/custom/domain/old.list" +git -C "$RENAME_REPO" add sources/custom/domain/old.list +git -C "$RENAME_REPO" commit -m base >/dev/null +rename_base="$(git -C "$RENAME_REPO" rev-parse HEAD)" +git -C "$RENAME_REPO" mv sources/custom/domain/old.list sources/custom/domain/new.list +git -C "$RENAME_REPO" commit -m rename >/dev/null +rename_output="$(cd "$RENAME_REPO" && env EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA="$rename_base" "$SCRIPT")" +grep -Fx "scope=full" <<< "$rename_output" >/dev/null +grep -Fx "reason=custom deletions require full sync" <<< "$rename_output" >/dev/null + echo "build scope tests passed" diff --git a/scripts/tests/test-fakeip-migration.sh b/scripts/tests/test-fakeip-filter.sh similarity index 55% rename from scripts/tests/test-fakeip-migration.sh rename to scripts/tests/test-fakeip-filter.sh index da2cb2e99..708ac97da 100644 --- a/scripts/tests/test-fakeip-migration.sh +++ b/scripts/tests/test-fakeip-filter.sh @@ -5,8 +5,8 @@ ROOT="$(cd "$(dirname "$0")/../.." && pwd)" TMP_DIR="$(mktemp -d)" trap 'rm -rf "$TMP_DIR"' EXIT -# Active build/configuration paths must not retain the historical provider, -# a Fake-IP URL, a download command, or an independent workflow sync step. +# Fake-IP is a maintained custom source and must not regain an independent +# provider, download, wrapper, or workflow path. if grep -RInE 'wwqgtxx|https?://[^[:space:]]*fakeip|sync-fakeip-filter|curl[^\n]*fakeip|wget[^\n]*fakeip' \ "$ROOT/scripts/commands" "$ROOT/scripts/lib" "$ROOT/scripts/tools" \ "$ROOT/config" "$ROOT/.github/workflows" "$ROOT/Makefile"; then @@ -15,26 +15,20 @@ if grep -RInE 'wwqgtxx|https?://[^[:space:]]*fakeip|sync-fakeip-filter|curl[^\n] fi [ ! -e "$ROOT/scripts/commands/sync-fakeip-filter.sh" ] || { - echo "test failed: unused legacy Fake-IP sync wrapper still exists" >&2 + echo "test failed: unused Fake-IP sync wrapper still exists" >&2 exit 1 } grep -F 'KuGouGo-maintained' "$ROOT/sources/custom/domain/fakeip-filter.list" >/dev/null RENDER_REPO="$TMP_DIR/render" -FAKEIP_SOURCE="$TMP_DIR/fakeip-filter.list" -cp "$ROOT/sources/custom/domain/fakeip-filter.list" "$FAKEIP_SOURCE" mkdir -p "$RENDER_REPO" cp -R "$ROOT/scripts" "$ROOT/config" "$ROOT/sources" "$RENDER_REPO/" -rm "$RENDER_REPO/sources/custom/domain/fakeip-filter.list" git -C "$RENDER_REPO" init -q git -C "$RENDER_REPO" config user.name test git -C "$RENDER_REPO" config user.email test@example.com git -C "$RENDER_REPO" add scripts config sources git -C "$RENDER_REPO" commit -m base >/dev/null -cp "$FAKEIP_SOURCE" "$RENDER_REPO/sources/custom/domain/fakeip-filter.list" -git -C "$RENDER_REPO" add sources/custom/domain/fakeip-filter.list -git -C "$RENDER_REPO" commit -m add-fakeip >/dev/null RULES_BUILD_CUSTOM_TEXT_ONLY=1 "$RENDER_REPO/scripts/commands/build-custom.sh" >/dev/null RULES_BUILD_CUSTOM_TEXT_ONLY=1 "$RENDER_REPO/scripts/commands/build-custom.sh" >/dev/null @@ -52,48 +46,30 @@ grep -Fx 'HOST-SUFFIX,lan,fakeip-filter' "$RENDER_REPO/.output/domain/quanx/fake grep -Fx 'HOST,proxy.golang.org,fakeip-filter' "$RENDER_REPO/.output/domain/quanx/fakeip-filter.list" >/dev/null grep -Fx " - 'lan'" "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null grep -Fx " - 'proxy.golang.org'" "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null -grep -Fx " - '^time\\.[^.]+\\.com$'" "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null +grep -Fx " - '^time\.[^.]+\.com$'" "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null -prepare_collision_repo() { - local repo="$1" - mkdir -p "$repo" - cp -R "$ROOT/scripts" "$ROOT/config" "$ROOT/sources" "$repo/" - rm "$repo/sources/custom/domain/fakeip-filter.list" - git -C "$repo" init -q - git -C "$repo" config user.name test - git -C "$repo" config user.email test@example.com - git -C "$repo" add scripts config sources -} - -# The one-time exception is valid only when the selected base really contains -# the historical artifact and lacks the maintained source. -MIGRATION_REPO="$TMP_DIR/migration" -prepare_collision_repo "$MIGRATION_REPO" -mkdir -p "$MIGRATION_REPO/.output/domain/mihomo" -printf 'historical binary\n' > "$MIGRATION_REPO/.output/domain/mihomo/fakeip-filter.mrs" -git -C "$MIGRATION_REPO" add -f .output/domain/mihomo/fakeip-filter.mrs -git -C "$MIGRATION_REPO" commit -m historical >/dev/null -MIGRATION_BASE="$(git -C "$MIGRATION_REPO" rev-parse HEAD)" -cp "$ROOT/sources/custom/domain/fakeip-filter.list" "$MIGRATION_REPO/sources/custom/domain/" -RULES_CONFLICT_BASE_SHA="$MIGRATION_BASE" RULES_BUILD_CUSTOM_TEXT_ONLY=1 \ - "$MIGRATION_REPO/scripts/commands/build-custom.sh" >/dev/null - -# Merely placing a same-named output in the working tree must not activate a -# permanent broad bypass when the base never tracked that historical binary. +# A new custom source may not overwrite an existing published artifact with the +# same name. Fake-IP follows the same conflict rule as every other custom list. COLLISION_REPO="$TMP_DIR/collision" -prepare_collision_repo "$COLLISION_REPO" +mkdir -p "$COLLISION_REPO" +cp -R "$ROOT/scripts" "$ROOT/config" "$ROOT/sources" "$COLLISION_REPO/" +rm "$COLLISION_REPO/sources/custom/domain/fakeip-filter.list" +git -C "$COLLISION_REPO" init -q +git -C "$COLLISION_REPO" config user.name test +git -C "$COLLISION_REPO" config user.email test@example.com +git -C "$COLLISION_REPO" add scripts config sources git -C "$COLLISION_REPO" commit -m base >/dev/null COLLISION_BASE="$(git -C "$COLLISION_REPO" rev-parse HEAD)" mkdir -p "$COLLISION_REPO/.output/domain/mihomo" -printf 'unrelated existing output\n' > "$COLLISION_REPO/.output/domain/mihomo/fakeip-filter.mrs" +printf 'existing output\n' > "$COLLISION_REPO/.output/domain/mihomo/fakeip-filter.mrs" cp "$ROOT/sources/custom/domain/fakeip-filter.list" "$COLLISION_REPO/sources/custom/domain/" if RULES_CONFLICT_BASE_SHA="$COLLISION_BASE" RULES_BUILD_CUSTOM_TEXT_ONLY=1 \ "$COLLISION_REPO/scripts/commands/build-custom.sh" \ >"$TMP_DIR/collision.stdout" 2>"$TMP_DIR/collision.stderr"; then - echo "test failed: Fake-IP collision bypass applied without historical base artifact" >&2 + echo "test failed: Fake-IP source overwrote an existing same-named artifact" >&2 exit 1 fi grep -F "custom rule name conflict detected for base 'fakeip-filter'" \ "$TMP_DIR/collision.stderr" >/dev/null -echo "Fake-IP migration tests passed" +echo "Fake-IP filter tests passed" diff --git a/scripts/tests/test-publish-branches.sh b/scripts/tests/test-publish-branches.sh index 4e1970d67..3c311739b 100755 --- a/scripts/tests/test-publish-branches.sh +++ b/scripts/tests/test-publish-branches.sh @@ -76,8 +76,8 @@ mkdir -p \ printf 'DOMAIN-SUFFIX,example.com\n' > "$REPO/.output/domain/surge/test.list" printf 'IP-CIDR,192.0.2.0/24,no-resolve\n' > "$REPO/.output/ip/surge/test.list" -printf 'HOST-SUFFIX,example.com,direct\n' > "$REPO/.output/domain/quanx/test.list" -printf 'IP-CIDR,192.0.2.0/24,direct\n' > "$REPO/.output/ip/quanx/test.list" +printf 'HOST-SUFFIX,example.com,test\n' > "$REPO/.output/domain/quanx/test.list" +printf 'IP-CIDR,192.0.2.0/24,test\n' > "$REPO/.output/ip/quanx/test.list" printf "domain_suffix_set:\n - 'example.com'\n" > "$REPO/.output/domain/egern/test.yaml" printf "no_resolve: true\nip_cidr_set:\n - '192.0.2.0/24'\n" > "$REPO/.output/ip/egern/test.yaml" printf 'srs-domain\n' > "$REPO/.output/domain/sing-box/test.srs" @@ -93,18 +93,23 @@ output="$5" if [[ "$input" == */ip/* ]]; then printf '{"version":4,"rules":[{"ip_cidr":["192.0.2.0/24"]}]}\n' > "$output" else - printf '{"version":4,"rules":[{"domain_suffix":["example.com"]}]}\n' > "$output" + value=example.com + grep -q '^updated-' "$input" && value=updated.example + printf '{"version":4,"rules":[{"domain_suffix":["%s"]}]}\n' "$value" > "$output" fi EOF cat > "$REPO/.bin/mihomo" <<'EOF' #!/usr/bin/env bash set -eu +behavior="$2" input="$4" output="$5" -if [[ "$input" == */ip/* ]]; then +if [ "$behavior" = ipcidr ]; then printf '192.0.2.0/24\n' > "$output" else - printf '+.example.com\n' > "$output" + value=example.com + grep -q '^updated-' "$input" && value=updated.example + printf '+.%s\n' "$value" > "$output" fi EOF chmod +x "$REPO/.bin/sing-box" "$REPO/.bin/mihomo" @@ -191,7 +196,7 @@ if git --git-dir="$REMOTE" show quanx:README.md | grep -F 'QuanX' >/dev/null; th fi git --git-dir="$REMOTE" show surge:domain/test.list | grep -Fx 'DOMAIN-SUFFIX,example.com' >/dev/null -git --git-dir="$REMOTE" show quanx:ip/test.list | grep -Fx 'IP-CIDR,192.0.2.0/24,direct' >/dev/null +git --git-dir="$REMOTE" show quanx:ip/test.list | grep -Fx 'IP-CIDR,192.0.2.0/24,test' >/dev/null before_idempotent="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 @@ -200,6 +205,52 @@ after_idempotent="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" r [ "$before_idempotent" = "$after_idempotent" ] grep -F "all publish branches unchanged, skip push" <<< "$second_output" >/dev/null +# A change to one platform must advance the complete publication cohort. The +# four unchanged trees receive metadata-only commits so a later custom restore +# observes one generation/source identity across all five branches. +declare -A before_partial_commit before_partial_tree +for branch in "${BRANCHES[@]}"; do + before_partial_commit["$branch"]="$(git --git-dir="$REMOTE" rev-parse "$branch")" + before_partial_tree["$branch"]="$(git --git-dir="$REMOTE" rev-parse "$branch^{tree}")" +done +cp "$REPO/templates/branch-readmes/surge.md" "$REPO/templates/branch-readmes/surge.md.partial-original" +printf '\npartial cohort fixture\n' >> "$REPO/templates/branch-readmes/surge.md" +git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 +ARTIFACT_GENERATION_ID=test-generation-partial ARTIFACT_BUILD_SCOPE=full ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null +partial_output="$(ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/publish-branches.sh" 2>&1)" +grep -F "publishing branches atomically: surge quanx egern sing-box mihomo" <<< "$partial_output" >/dev/null + +for branch in "${BRANCHES[@]}"; do + after_commit="$(git --git-dir="$REMOTE" rev-parse "$branch")" + after_tree="$(git --git-dir="$REMOTE" rev-parse "$branch^{tree}")" + parent_commit="$(git --git-dir="$REMOTE" rev-parse "$branch^")" + subject="$(git --git-dir="$REMOTE" log -1 --format=%s "$branch")" + [ "$after_commit" != "${before_partial_commit[$branch]}" ] || { + echo "test failed: partial cohort did not advance $branch" >&2 + exit 1 + } + [ "$parent_commit" = "${before_partial_commit[$branch]}" ] || { + echo "test failed: $branch publication commit did not preserve parent history" >&2 + exit 1 + } + [ "$subject" = "chore: publish ${branch} artifacts [generation test-generation-partial source ${SOURCE_SHA}]" ] || { + echo "test failed: $branch publication identity differs from partial cohort" >&2 + exit 1 + } + if [ "$branch" = surge ]; then + [ "$after_tree" != "${before_partial_tree[$branch]}" ] || { + echo "test failed: surge fixture did not change its tree" >&2 + exit 1 + } + else + [ "$after_tree" = "${before_partial_tree[$branch]}" ] || { + echo "test failed: metadata-only cohort commit changed $branch tree" >&2 + exit 1 + } + fi +done +mv "$REPO/templates/branch-readmes/surge.md.partial-original" "$REPO/templates/branch-readmes/surge.md" + # A missing template must fail before the queued batch is pushed. before_missing_template="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REMOTE" rev-parse "$branch"; done)" cp "$REPO/templates/branch-readmes/surge.md" "$REPO/templates/branch-readmes/surge.md.original" @@ -232,7 +283,7 @@ for branch in "${BRANCHES[@]}"; do ;; quanx) extension=list - printf 'HOST-SUFFIX,updated.example,direct\n' > "$REPO/.output/domain/$branch/test.$extension" + printf 'HOST-SUFFIX,updated.example,test\n' > "$REPO/.output/domain/$branch/test.$extension" ;; egern) extension=yaml diff --git a/scripts/tests/test-sync-upstream-render-matrix.sh b/scripts/tests/test-sync-upstream-render-matrix.sh index 51b99ffde..14bd774d4 100755 --- a/scripts/tests/test-sync-upstream-render-matrix.sh +++ b/scripts/tests/test-sync-upstream-render-matrix.sh @@ -57,10 +57,9 @@ loyalsoldier_required_snippets = [ 'download_file "$LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL" "$IP_BUILD_TMP_DIR/private.raw.txt"', 'text "$tmp_dir/loyalsoldier_geoip_cn.raw.txt" "$tmp_dir/loyalsoldier_geoip_cn.cidr.txt"', 'text "$tmp_dir/private.raw.txt" "$tmp_dir/private.cidr.txt"', - 'record_upstream_summary ip loyalsoldier-geoip-cn ok "$LOYALSOLDIER_GEOIP_CN_SOURCE_URL"', - 'record_upstream_summary ip loyalsoldier-geoip-private ok "$LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL"', - '"loyalsoldier-geoip-cn loyalsoldier_geoip_cn.raw.txt loyalsoldier_geoip_cn.cidr.txt"', - '"loyalsoldier-geoip-private private.raw.txt private.cidr.txt"', + 'verify_and_record_upstream_health', + 'loyalsoldier-geoip-cn|$LOYALSOLDIER_GEOIP_CN_SOURCE_URL|loyalsoldier_geoip_cn.raw.txt|loyalsoldier_geoip_cn.cidr.txt|0', + 'loyalsoldier-geoip-private|$LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL|private.raw.txt|private.cidr.txt|0', '"$IP_BUILD_TMP_DIR/loyalsoldier_geoip_cn.cidr.txt"', ] for snippet in loyalsoldier_required_snippets: diff --git a/scripts/tests/test-upstream-config.sh b/scripts/tests/test-upstream-config.sh index 5d8377529..bed8bfda7 100755 --- a/scripts/tests/test-upstream-config.sh +++ b/scripts/tests/test-upstream-config.sh @@ -78,6 +78,54 @@ assert_lint_fails_with \ "upstreams.ip.github.parser: unsupported or missing parser 'unknown-json'" \ --upstreams "$TMP_DIR/upstreams.invalid-parser.json" +cp config/upstreams.json "$TMP_DIR/upstreams.wrong-parser.json" +python3 - <<'PY' "$TMP_DIR/upstreams.wrong-parser.json" +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +data = json.loads(path.read_text(encoding="utf-8")) +data["ip"]["github"]["parser"] = "google-json" +path.write_text(json.dumps(data), encoding="utf-8") +PY +assert_lint_fails_with \ + "wrong-parser" \ + "upstreams.ip.github.parser: must equal 'github-json' for source 'github', got 'google-json'" \ + --upstreams "$TMP_DIR/upstreams.wrong-parser.json" + +cp config/upstreams.json "$TMP_DIR/upstreams.wrong-kind.json" +python3 - <<'PY' "$TMP_DIR/upstreams.wrong-kind.json" +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +data = json.loads(path.read_text(encoding="utf-8")) +data["ip"]["github"]["kind"] = "text" +path.write_text(json.dumps(data), encoding="utf-8") +PY +assert_lint_fails_with \ + "wrong-kind" \ + "upstreams.ip.github.kind: must equal 'json' for source 'github', got 'text'" \ + --upstreams "$TMP_DIR/upstreams.wrong-kind.json" + +cp config/upstreams.json "$TMP_DIR/upstreams.cloudfront-url.json" +python3 - <<'PY' "$TMP_DIR/upstreams.cloudfront-url.json" +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +data = json.loads(path.read_text(encoding="utf-8")) +data["ip"]["cloudfront"]["url"] = "https://example.com/ip-ranges.json" +path.write_text(json.dumps(data), encoding="utf-8") +PY +assert_lint_fails_with \ + "cloudfront-url" \ + "upstreams.ip.cloudfront.url: must equal upstreams.ip.aws.url" \ + --upstreams "$TMP_DIR/upstreams.cloudfront-url.json" + cp config/upstream-first-batch-baselines.json "$TMP_DIR/baselines.invalid.json" python3 - <<'PY' "$TMP_DIR/baselines.invalid.json" import json diff --git a/scripts/tests/test-upstream-health-summary.sh b/scripts/tests/test-upstream-health-summary.sh new file mode 100644 index 000000000..96da4f6c5 --- /dev/null +++ b/scripts/tests/test-upstream-health-summary.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helper="$({ + awk ' + /^verify_and_record_upstream_health\(\) \{/ { capture=1 } + capture { print } + capture && /^}$/ { exit } + ' "$ROOT/scripts/commands/sync-upstream.sh" +})" +[ -n "$helper" ] || { + echo "test failed: verify_and_record_upstream_health helper not found" >&2 + exit 1 +} +eval "$helper" + +record_upstream_summary() { + printf '%s\n' "$3" > "$TMP_DIR/recorded-status" +} + +write_config() { + local requirement="$1" + cat > "$TMP_DIR/upstreams.json" < "$TMP_DIR/raw" +: > "$TMP_DIR/normalized" + +write_config optional +verify_and_record_upstream_health \ + ip fixture https://example.com "$TMP_DIR/raw" "$TMP_DIR/normalized" 0 >/dev/null +grep -Fx semantic_regression "$TMP_DIR/recorded-status" >/dev/null + +write_config required +if verify_and_record_upstream_health \ + ip fixture https://example.com "$TMP_DIR/raw" "$TMP_DIR/normalized" 0 \ + >"$TMP_DIR/required.stdout" 2>"$TMP_DIR/required.stderr"; then + echo "test failed: required semantic regression did not block" >&2 + exit 1 +fi +grep -Fx semantic_regression "$TMP_DIR/recorded-status" >/dev/null +grep -F 'required upstream fixture failed configured health policy' "$TMP_DIR/required.stderr" >/dev/null + +echo "upstream health summary tests passed" diff --git a/scripts/tests/test-workflow-action-pinning.sh b/scripts/tests/test-workflow-action-pinning.sh new file mode 100644 index 000000000..5f7500e88 --- /dev/null +++ b/scripts/tests/test-workflow-action-pinning.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +WORKFLOW_ROOT="${WORKFLOW_ROOT:-$ROOT/.github/workflows}" + +python3 - "$WORKFLOW_ROOT" <<'PY' +import re +import sys +from pathlib import Path + +workflow_root = Path(sys.argv[1]) +workflow_paths = sorted( + [*workflow_root.rglob("*.yml"), *workflow_root.rglob("*.yaml")] +) +uses_key = re.compile(r"^\s*(?:-\s*)?uses\s*:") +uses_value = re.compile( + r'''^\s*(?:-\s*)?uses\s*:\s*(?:"([^"]+)"|'([^']+)'|([^\s#]+))\s*(?:#.*)?$''' +) +pinned_ref = re.compile(r"^[^@\s]+@[0-9a-fA-F]{40}$") +errors = [] +uses_count = 0 + +if not workflow_paths: + errors.append(f"no workflow files found under {workflow_root}") + +for workflow_path in workflow_paths: + for line_number, line in enumerate( + workflow_path.read_text(encoding="utf-8").splitlines(), start=1 + ): + if not uses_key.match(line): + continue + uses_count += 1 + match = uses_value.match(line) + reference = next((value for value in match.groups() if value), None) if match else None + if reference is None or not pinned_ref.fullmatch(reference): + shown = reference if reference is not None else line.strip() + errors.append( + f"{workflow_path}:{line_number}: uses reference must end in a full " + f"40-character commit SHA: {shown}" + ) + +if uses_count == 0: + errors.append(f"no uses references found under {workflow_root}") + +if errors: + print("workflow action pinning validation failed:", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) + raise SystemExit(1) + +print(f"workflow action pinning validated: {uses_count} references") +PY diff --git a/scripts/tools/artifact_manifest.py b/scripts/tools/artifact_manifest.py index 927143d0c..c07a04189 100644 --- a/scripts/tools/artifact_manifest.py +++ b/scripts/tools/artifact_manifest.py @@ -16,7 +16,7 @@ from artifact_verifier import verify_one from platform_capabilities import load_platform_capabilities -SCHEMA_VERSION = 2 +SCHEMA_VERSION = 3 SHA256_RE = re.compile(r"^[0-9a-f]{64}$") GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") TOP_KEYS = {"schema_version", "generation_id", "build_id", "build_scope", "source", "inputs", "tools", "summaries", "artifacts", "restoration"} @@ -112,7 +112,7 @@ def provenance_metadata(root: Path, lock: dict[str, Any]) -> dict[str, Any]: def validate_verification(value: Any, location: str, errors: list[str]) -> None: - keys = {"status", "method", "decoded_counts", "decoded_count", "canonical_linkage"} + keys = {"status", "method", "decoded_counts", "decoded_count", "decoded_semantic_sha256", "canonical_linkage"} if not require_exact(value, keys, location, errors): return if value["status"] != "verified" or not isinstance(value["method"], str) or not value["method"]: @@ -122,11 +122,13 @@ def validate_verification(value: Any, location: str, errors: list[str]) -> None: errors.append(f"{location}.decoded_counts must map strings to non-negative integers") elif type(value["decoded_count"]) is not int or value["decoded_count"] != sum(counts.values()) or value["decoded_count"] <= 0: errors.append(f"{location}.decoded_count is invalid") + if not isinstance(value["decoded_semantic_sha256"], str) or not SHA256_RE.fullmatch(value["decoded_semantic_sha256"]): + errors.append(f"{location}.decoded_semantic_sha256 is invalid") linkage = value["canonical_linkage"] if not isinstance(linkage, dict) or linkage.get("status") not in {"matched", "unavailable"}: errors.append(f"{location}.canonical_linkage is invalid") elif linkage["status"] == "matched": - if set(linkage) != {"status", "source", "counts"} or not isinstance(linkage.get("source"), str) or not isinstance(linkage.get("counts"), dict): + if set(linkage) != {"status", "source", "counts", "semantic_sha256"} or not isinstance(linkage.get("source"), str) or not isinstance(linkage.get("counts"), dict) or not isinstance(linkage.get("semantic_sha256"), str) or not SHA256_RE.fullmatch(linkage["semantic_sha256"]): errors.append(f"{location}.canonical_linkage matched schema is invalid") elif set(linkage) != {"status", "reason"} or not isinstance(linkage.get("reason"), str) or not linkage["reason"]: errors.append(f"{location}.canonical_linkage unavailable schema is invalid") diff --git a/scripts/tools/artifact_verifier.py b/scripts/tools/artifact_verifier.py index 90737f100..8a364facd 100644 --- a/scripts/tools/artifact_verifier.py +++ b/scripts/tools/artifact_verifier.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse +import hashlib +import ipaddress import json import subprocess import tempfile @@ -10,59 +12,99 @@ from pathlib import Path from typing import Any +from domain_rules import domain_value_errors, parse_classical_domain_file +from ip_rules import parse_classical_ip_file from platform_capabilities import load_platform_capabilities +RuleEntry = tuple[str, str] + + def noncomment_lines(path: Path) -> list[str]: return [line.strip() for line in path.read_text(encoding="utf-8").splitlines() if line.strip() and not line.lstrip().startswith("#")] -def canonical_counts(root: Path, kind: str, stem: str, platform: str) -> Counter[str] | None: +def summarize_entries(entries: Counter[RuleEntry]) -> Counter[str]: + return Counter({kind: sum(count for (entry_kind, _), count in entries.items() if entry_kind == kind) + for kind in sorted({entry_kind for entry_kind, _ in entries})}) + + +def semantic_digest(entries: Counter[RuleEntry]) -> str: + payload = [[kind, value, count] for (kind, value), count in sorted(entries.items())] + return hashlib.sha256(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() + + +def canonical_source_entries( + root: Path, + kind: str, + stem: str, + supported_kinds: set[str], +) -> Counter[RuleEntry] | None: source = root / "sources" / "custom" / kind / f"{stem}.list" if not source.is_file(): return None - result: Counter[str] = Counter() + result: Counter[RuleEntry] = Counter() if kind == "domain": - for line in noncomment_lines(source): - parts = [part.strip() for part in line.split(",")] - if len(parts) >= 2: - rule_kind = parts[0] - if platform != "mihomo" or rule_kind in {"DOMAIN", "DOMAIN-SUFFIX"}: - result[rule_kind] += 1 + rules, errors = parse_classical_domain_file( + source, + require_canonical=True, + allow_single_label_suffix=True, + ) else: - for line in noncomment_lines(source): - value = line.split(",", 2) - if value[0] in {"IP-CIDR", "IP-CIDR6"} and len(value) > 1: - result[value[0]] += 1 + rules, errors = parse_classical_ip_file(source, require_canonical=True) + if errors: + raise ValueError(f"canonical custom source is invalid: {'; '.join(errors)}") + for rule in rules: + if rule.kind in supported_kinds: + result[(rule.kind, rule.value)] += 1 return result -def singbox_counts(data: dict[str, Any], kind: str) -> Counter[str]: +def singbox_entries(data: dict[str, Any], kind: str) -> Counter[RuleEntry]: mappings = { "domain": {"domain": "DOMAIN", "domain_suffix": "DOMAIN-SUFFIX", "domain_keyword": "DOMAIN-KEYWORD", "domain_regex": "DOMAIN-REGEX"}, "ip": {"ip_cidr": "IP-CIDR"}, }[kind] - counts: Counter[str] = Counter() - for rule in data.get("rules", []): + entries: Counter[RuleEntry] = Counter() + allowed_fields = set(mappings) + for index, rule in enumerate(data.get("rules", []), 1): if not isinstance(rule, dict): - continue + raise ValueError(f"sing-box decompile returned non-object rule #{index}") + unknown_fields = set(rule) - allowed_fields + if unknown_fields: + raise ValueError( + f"sing-box decompile rule #{index} contains unsupported fields: {sorted(unknown_fields)}" + ) + matched_field = False for field, canonical in mappings.items(): raw_values = rule.get(field) if raw_values is None: continue + matched_field = True values = raw_values if isinstance(raw_values, list) else [raw_values] if not values or any(not isinstance(value, str) for value in values): raise ValueError(f"sing-box decompile returned invalid {field} values") if kind == "ip" and field == "ip_cidr": for value in values: - counts["IP-CIDR6" if ":" in value else "IP-CIDR"] += 1 + try: + network = ipaddress.ip_network(value, strict=False) + except ValueError as exc: + raise ValueError(f"sing-box decompile returned invalid CIDR {value!r}: {exc}") from exc + entries[("IP-CIDR6" if network.version == 6 else "IP-CIDR", str(network))] += 1 else: - counts[canonical] += len(values) - return counts + for value in values: + entries[(canonical, value)] += 1 + if not matched_field: + raise ValueError(f"sing-box decompile rule #{index} contains no {kind} values") + return entries -def verify_singbox(path: Path, kind: str, tool: Path) -> tuple[str, Counter[str]]: +def singbox_counts(data: dict[str, Any], kind: str) -> Counter[str]: + return summarize_entries(singbox_entries(data, kind)) + + +def verify_singbox(path: Path, kind: str, tool: Path) -> tuple[str, Counter[RuleEntry]]: with tempfile.TemporaryDirectory() as temporary: decoded = Path(temporary) / "decoded.json" subprocess.run([str(tool), "rule-set", "decompile", str(path), "--output", str(decoded)], @@ -70,10 +112,10 @@ def verify_singbox(path: Path, kind: str, tool: Path) -> tuple[str, Counter[str] data = json.loads(decoded.read_text(encoding="utf-8")) if not isinstance(data, dict) or not isinstance(data.get("rules"), list): raise ValueError("sing-box decompile did not produce a rule-set object") - return "sing-box-rule-set-decompile", singbox_counts(data, kind) + return "sing-box-rule-set-decompile", singbox_entries(data, kind) -def verify_mihomo(path: Path, kind: str, tool: Path) -> tuple[str, Counter[str]]: +def verify_mihomo(path: Path, kind: str, tool: Path) -> tuple[str, Counter[RuleEntry]]: behavior = "domain" if kind == "domain" else "ipcidr" with tempfile.TemporaryDirectory() as temporary: decoded = Path(temporary) / "decoded.txt" @@ -82,22 +124,31 @@ def verify_mihomo(path: Path, kind: str, tool: Path) -> tuple[str, Counter[str]] values = noncomment_lines(decoded) if not values: raise ValueError("mihomo MRS readback produced no rules") - counts: Counter[str] = Counter() + entries: Counter[RuleEntry] = Counter() if kind == "domain": for value in values: - counts["DOMAIN-SUFFIX" if value.startswith("+.") or value.startswith(".") else "DOMAIN"] += 1 + if value.startswith("+."): + entries[("DOMAIN-SUFFIX", value[2:])] += 1 + elif value.startswith("."): + entries[("DOMAIN-SUFFIX", value[1:])] += 1 + else: + entries[("DOMAIN", value)] += 1 else: for value in values: - counts["IP-CIDR6" if ":" in value else "IP-CIDR"] += 1 - return "mihomo-convert-ruleset-mrs-to-text", counts + try: + network = ipaddress.ip_network(value, strict=False) + except ValueError as exc: + raise ValueError(f"mihomo MRS readback returned invalid CIDR {value!r}: {exc}") from exc + entries[("IP-CIDR6" if network.version == 6 else "IP-CIDR", str(network))] += 1 + return "mihomo-convert-ruleset-mrs-to-text", entries -def verify_classical(path: Path, capability: Any, platform: str, artifact_type: str) -> Counter[str]: +def verify_classical(path: Path, capability: Any, platform: str, artifact_type: str) -> Counter[RuleEntry]: lines = noncomment_lines(path) if not lines: raise ValueError("text artifact contains no rules") targets = {target: kind for kind, target in capability.rule_mappings.items()} - counts: Counter[str] = Counter() + entries: Counter[RuleEntry] = Counter() for line_number, line in enumerate(lines, 1): fields = [field.strip() for field in line.split(",")] if any(not field for field in fields): @@ -110,13 +161,35 @@ def verify_classical(path: Path, capability: Any, platform: str, artifact_type: raise ValueError(f"line {line_number} has {len(fields)} fields; expected {expected_fields}") if platform == "surge" and artifact_type == "ip" and fields[2] != "no-resolve": raise ValueError(f"line {line_number} must end with no-resolve") - counts[targets[target]] += 1 - return counts + if platform == "quanx" and fields[2] != path.stem: + raise ValueError(f"line {line_number} policy must match artifact name {path.stem!r}") + canonical_kind = targets[target] + value = fields[1] + if artifact_type == "domain": + errors = domain_value_errors( + canonical_kind, + value, + require_canonical=True, + allow_single_label_suffix=True, + ) + if errors: + raise ValueError(f"line {line_number} {errors[0]}") + else: + try: + network = ipaddress.ip_network(value, strict=False) + except ValueError as exc: + raise ValueError(f"line {line_number} has invalid CIDR {value!r}: {exc}") from exc + value = str(network) + expected_kind = "IP-CIDR6" if network.version == 6 else "IP-CIDR" + if canonical_kind != expected_kind or fields[1] != value: + raise ValueError(f"line {line_number} has non-canonical CIDR kind or value") + entries[(canonical_kind, value)] += 1 + return entries -def parse_egern_yaml(path: Path, capability: Any, artifact_type: str) -> Counter[str]: +def parse_egern_yaml(path: Path, capability: Any, artifact_type: str) -> Counter[RuleEntry]: allowed = {target: kind for kind, target in capability.rule_mappings.items()} - counts: Counter[str] = Counter() + entries: Counter[RuleEntry] = Counter() current: str | None = None seen: set[str] = set() for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): @@ -147,14 +220,64 @@ def parse_egern_yaml(path: Path, capability: Any, artifact_type: str) -> Counter value = quoted.replace("''", "'") if not value: raise ValueError(f"line {line_number} contains an empty value") - counts[allowed[current]] += 1 + canonical_kind = allowed[current] + if artifact_type == "domain": + errors = domain_value_errors( + canonical_kind, + value, + require_canonical=True, + allow_single_label_suffix=True, + ) + if errors: + raise ValueError(f"line {line_number} {errors[0]}") + else: + try: + network = ipaddress.ip_network(value, strict=False) + except ValueError as exc: + raise ValueError(f"line {line_number} has invalid CIDR {value!r}: {exc}") from exc + value = str(network) + expected_kind = "IP-CIDR6" if network.version == 6 else "IP-CIDR" + if canonical_kind != expected_kind or quoted.replace("''", "'") != value: + raise ValueError(f"line {line_number} has non-canonical CIDR kind or value") + entries[(canonical_kind, value)] += 1 continue raise ValueError(f"line {line_number} has unsupported YAML indentation or structure") - if not counts: + if not entries: raise ValueError("Egern YAML artifact contains no rules") if artifact_type == "ip" and "no_resolve" not in seen: raise ValueError("Egern IP YAML must declare no_resolve: true") - return counts + return entries + + +def canonical_binary_entries( + root: Path, + path: Path, + artifact_type: str, + platform: str, + capabilities: Any, +) -> tuple[Counter[RuleEntry] | None, str | None]: + target = getattr(capabilities.platforms[platform], artifact_type) + supported_kinds = set(target.rule_mappings) + custom = canonical_source_entries(root, artifact_type, path.stem, supported_kinds) + if custom is not None: + return custom, f"sources/custom/{artifact_type}/{path.stem}.list" + + artifact_root = path.parents[2] + reference_platform = "surge" + if artifact_type == "domain" and platform == "sing-box": + reference_platform = "egern" + reference_capability = getattr(capabilities.platforms[reference_platform], artifact_type) + reference = artifact_root / artifact_type / reference_platform / f"{path.stem}.{reference_capability.extension}" + if not reference.is_file(): + return None, None + if reference_capability.verifier.startswith("classical-"): + entries = verify_classical(reference, reference_capability, reference_platform, artifact_type) + elif reference_capability.verifier == "egern-yaml": + entries = parse_egern_yaml(reference, reference_capability, artifact_type) + else: + raise ValueError(f"unsupported canonical reference verifier: {reference_capability.verifier}") + filtered = Counter({entry: count for entry, count in entries.items() if entry[0] in supported_kinds}) + return filtered, reference.relative_to(artifact_root).as_posix() def verify_one(root: Path, path: Path, artifact_type: str, platform: str) -> dict[str, Any]: @@ -171,14 +294,40 @@ def verify_one(root: Path, path: Path, artifact_type: str, platform: str) -> dic method, decoded = verifier, parse_egern_yaml(path, capability, artifact_type) else: raise ValueError(f"unsupported verifier implementation: {verifier}") - canonical = canonical_counts(root, artifact_type, path.stem, platform) if verifier in {"sing-box", "mihomo"} else None + canonical: Counter[RuleEntry] | None = None + canonical_source: str | None = None + if verifier in {"sing-box", "mihomo"}: + canonical, canonical_source = canonical_binary_entries( + root, + path, + artifact_type, + platform, + capabilities, + ) + if canonical is None or canonical_source is None: + raise ValueError("binary artifact has no canonical custom source or same-build text counterpart") linkage: dict[str, Any] = {"status": "unavailable", "reason": "canonical binary compiler input is not retained or not applicable"} if canonical is not None: if decoded != canonical: - raise ValueError(f"decoded rule counts differ from canonical custom source: decoded={dict(decoded)}, canonical={dict(canonical)}") - linkage = {"status": "matched", "source": f"sources/custom/{artifact_type}/{path.stem}.list", "counts": dict(sorted(canonical.items()))} - return {"status": "verified", "method": method, "decoded_counts": dict(sorted(decoded.items())), - "decoded_count": sum(decoded.values()), "canonical_linkage": linkage} + raise ValueError( + "decoded rule values differ from canonical source: " + f"decoded_sha256={semantic_digest(decoded)}, canonical_sha256={semantic_digest(canonical)}" + ) + linkage = { + "status": "matched", + "source": canonical_source, + "counts": dict(sorted(summarize_entries(canonical).items())), + "semantic_sha256": semantic_digest(canonical), + } + decoded_counts = summarize_entries(decoded) + return { + "status": "verified", + "method": method, + "decoded_counts": dict(sorted(decoded_counts.items())), + "decoded_count": sum(decoded_counts.values()), + "decoded_semantic_sha256": semantic_digest(decoded), + "canonical_linkage": linkage, + } def main() -> None: diff --git a/scripts/tools/lint-config.py b/scripts/tools/lint-config.py index e3598b6c6..ca144bd7b 100644 --- a/scripts/tools/lint-config.py +++ b/scripts/tools/lint-config.py @@ -13,27 +13,34 @@ ROOT = Path(__file__).resolve().parents[2] -REQUIRED_DOMAIN_SOURCES = {"dlc"} -REQUIRED_IP_SOURCES = { - "cn-ipv46", - "cn-ipv46-apnic", - "loyalsoldier-geoip-cn", - "loyalsoldier-geoip-private", - "google", - "telegram", - "cloudflare-ipv4", - "cloudflare-ipv6", - "aws", - "fastly", - "github", - "apple", - "ripe-stat", +SOURCE_IMPLEMENTATIONS = { + "domain": { + "dlc": ("git", "git-tree"), + }, + "ip": { + "cn-ipv46": ("text", "cidr-text"), + "cn-ipv46-apnic": ("text", "cidr-text"), + "loyalsoldier-geoip-cn": ("text", "cidr-text"), + "loyalsoldier-geoip-private": ("text", "cidr-text"), + "google": ("json", "google-json"), + "telegram": ("text", "telegram"), + "cloudflare-ipv4": ("text", "cidr-text"), + "cloudflare-ipv6": ("text", "cidr-text"), + "aws": ("json", "aws-json"), + "cloudfront": ("json", "aws-cloudfront-json"), + "fastly": ("json", "fastly-json"), + "github": ("json", "github-json"), + "apple": ("html", "html-cidr"), + "ripe-stat": ("json-api", "ripe-stat-json"), + }, } +REQUIRED_DOMAIN_SOURCES = set(SOURCE_IMPLEMENTATIONS["domain"]) +REQUIRED_IP_SOURCES = set(SOURCE_IMPLEMENTATIONS["ip"]) REQUIRED_ASN_GROUPS = {"telegram", "netflix", "spotify", "disney"} REQUIRED_FIRST_BATCH_SOURCES = {"google-json", "github-json", "telegram"} SUPPORTED_PARSERS = { "git-tree", "cidr-text", "google-json", "github-json", "telegram", - "aws-json", "fastly-json", "html-cidr", "ripe-stat-json", + "aws-json", "aws-cloudfront-json", "fastly-json", "html-cidr", "ripe-stat-json", } ALLOWED_REQUIREMENTS = {"required", "optional"} ALLOWED_FAMILIES = {"any", "ipv4", "ipv6", "dual"} @@ -141,6 +148,20 @@ def validate_source(section: str, name: str, item: object, reporter: Reporter) - if parser not in SUPPORTED_PARSERS: reporter.error(f"{location}.parser", f"unsupported or missing parser {parser!r}") + expected_implementation = SOURCE_IMPLEMENTATIONS.get(section, {}).get(name) + if expected_implementation is not None: + expected_kind, expected_parser = expected_implementation + if kind in ALLOWED_KINDS[section] and kind != expected_kind: + reporter.error( + f"{location}.kind", + f"must equal {expected_kind!r} for source {name!r}, got {kind!r}", + ) + if parser in SUPPORTED_PARSERS and parser != expected_parser: + reporter.error( + f"{location}.parser", + f"must equal {expected_parser!r} for source {name!r}, got {parser!r}", + ) + health = item.get("health") if not isinstance(health, dict): reporter.error(f"{location}.health", "must be an object") @@ -199,6 +220,14 @@ def validate_upstreams(data: dict, reporter: Reporter) -> None: for name, item in sorted(ip.items()): validate_source("ip", name, item, reporter) + aws = ip.get("aws") + cloudfront = ip.get("cloudfront") + if isinstance(aws, dict) and isinstance(cloudfront, dict) and aws.get("url") != cloudfront.get("url"): + reporter.error( + "upstreams.ip.cloudfront.url", + "must equal upstreams.ip.aws.url because both parsers consume one downloaded AWS payload", + ) + for name, values in sorted(asn_groups.items()): location = f"upstreams.asn_groups.{name}" if not isinstance(values, list) or not values: diff --git a/scripts/tools/normalize-ip-rules.py b/scripts/tools/normalize-ip-rules.py index 75f19080b..df2c9ca3c 100644 --- a/scripts/tools/normalize-ip-rules.py +++ b/scripts/tools/normalize-ip-rules.py @@ -328,7 +328,7 @@ def run_batch_tasks(manifest_file: Path) -> None: def main() -> int: - legacy_source_types = { + source_types = { "text", "google-json", "aws-cloudfront-json", @@ -339,21 +339,13 @@ def main() -> int: "html", } - if len(sys.argv) == 4 and sys.argv[1] in legacy_source_types: - try: - run_single_task(sys.argv[1], Path(sys.argv[2]), Path(sys.argv[3])) - return 0 - except Exception as exc: # pragma: no cover - surfaced to shell - print(str(exc), file=sys.stderr) - return 1 - parser = argparse.ArgumentParser() subparsers = parser.add_subparsers(dest="command", required=True) single_parser = subparsers.add_parser("single") single_parser.add_argument( "source_type", - choices=tuple(sorted(legacy_source_types)), + choices=tuple(sorted(source_types)), ) single_parser.add_argument("input_file") single_parser.add_argument("output_file") diff --git a/templates/branch-readmes/mihomo.md b/templates/branch-readmes/mihomo.md index 0cad03fe5..1527f1de9 100644 --- a/templates/branch-readmes/mihomo.md +++ b/templates/branch-readmes/mihomo.md @@ -14,7 +14,7 @@ - `domain/` 与 `ip/` 均使用二进制 `.mrs` 扩展名。 - 域名产物仅保留精确域名和域名后缀；域名关键词与正则会降级丢弃，仅含这些类型的列表不会发布。 - IP CIDR 保留；`.mrs` 仅供 mihomo 使用，不能假定与其他客户端的二进制规则格式兼容。 -- `domain/fakeip-filter.mrs` 由 KuGouGo 在主分支维护的 `sources/custom/domain/fakeip-filter.list` 编译生成；它不是第三方预编译下载。过去采用 `wwqgtxx/clash-rules` 二进制仅属历史。 +- `domain/fakeip-filter.mrs` 由 KuGouGo 在主分支维护的 `sources/custom/domain/fakeip-filter.list` 编译生成，不是第三方预编译下载。 ## 最小示例 From 684c0bbcbbc562399a9ff5dbeb24943559e7c7f2 Mon Sep 17 00:00:00 2001 From: KuGouGo <62388728+KuGouGo@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:25:13 +0800 Subject: [PATCH 2/3] Compare normalized binary rule semantics --- docs/DEVELOPMENT.md | 2 +- docs/STRUCTURE.md | 2 +- .../tests/test-binary-artifact-verifier.sh | 33 ++++++++++++++- scripts/tools/artifact_verifier.py | 40 ++++++++++++++++++- 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index d663065c5..eaf5009ec 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -34,7 +34,7 @@ make clean - `generate-artifact-manifest.sh`：在构建与守卫完成后、写 manifest 前，按能力配置中的 `verifier` 分派器验证每个产物；缺失或未验证的二进制会阻断生成。默认位于 `.output/`，事务内服从 `RULES_ARTIFACT_ROOT`。调用方应明确提供 generation id、build scope，CI 还将 source SHA 绑定到实际 checkout 的 `github.sha`：PR 验证记录被测试的合并提交，正式发布记录 `main` 提交。 - `verify-artifact-manifest.sh`：严格重算所选 artifact root 内的可发布文件集合、路径、大小和 SHA-256，并重新执行产物验证、核对能力/lock 与可选 source SHA；发布 job 在恢复或安装锁定工具后强制执行同一验证。 -二进制验证使用固定工具的真实读回接口：`.srs` 执行 `sing-box rule-set decompile` 并解析 JSON；`.mrs` 执行 `mihomo convert-ruleset mrs INPUT OUTPUT`。读回结果规范化为 `(规则类型, 规则值)` 多重集合，并与同名 custom 源或同一事务的 Egern/Surge 文本产物精确比较；类型计数相同但值不同也会失败。manifest 记录验证方法、计数、读回语义 SHA-256，以及可关联规范输入时的语义 SHA-256。 +二进制验证使用固定工具的真实读回接口：`.srs` 执行 `sing-box rule-set decompile` 并解析 JSON；`.mrs` 执行 `mihomo convert-ruleset mrs INPUT OUTPUT`。读回结果与同名 custom 源或同一事务的 Egern/Surge 文本产物比较规范化语义集合：域名消除已被更宽后缀覆盖的冗余项，IP 合并为等价 CIDR 并集；值替换、范围扩大或范围丢失都会失败。manifest 记录验证方法、原始计数、读回语义 SHA-256，以及规范输入的语义 SHA-256。 - `make clean`：删除 `.tmp/`、`.output/`、`.artifacts/`、Python `__pycache__` 和未完成的 `.bin/*.new*`；保留已安装的 `.bin/sing-box`、`.bin/mihomo` 及 provenance sidecar。 CI 设置 `REQUIRE_SHELLCHECK=1`，本地缺少 ShellCheck 时的跳过不代表 CI 会通过。 diff --git a/docs/STRUCTURE.md b/docs/STRUCTURE.md index 05b38bab0..742b1d036 100644 --- a/docs/STRUCTURE.md +++ b/docs/STRUCTURE.md @@ -35,7 +35,7 @@ - `.tmp/**/normalize-tasks.json`：批处理任务描述，属于临时数据。 - `.artifacts/diagnostics//`：失败事务保留的 `transaction-health.json` 及可用的构建/上游摘要；CI 日志只展示白名单内且大小受限的 JSON，完整诊断作为短期 Actions artifact 上传。 -`verify-artifact-manifest.sh` 严格重算能力矩阵允许的完整文件集合、路径层级与安全性、非零大小、字节数和 SHA-256，并核对能力/lock 摘要及可选的预期 source SHA。二进制读回规则与同名 custom 源或同一事务的文本产物按类型和值精确比较，并记录语义 SHA-256。发布作业下载后再次验证；`publish-branches.sh` 自身也必须先验证清单，拒绝缺失、额外或被修改的产物。清单只作为流水线审计输入，不复制到发布分支。一次发布中五个分支提交携带共同 generation id 和 source SHA；任一平台 tree 改变时完整 cohort 原子推进并保留各分支父历史，全部 tree 不变时整体跳过。 +`verify-artifact-manifest.sh` 严格重算能力矩阵允许的完整文件集合、路径层级与安全性、非零大小、字节数和 SHA-256，并核对能力/lock 摘要及可选的预期 source SHA。二进制读回规则与同名 custom 源或同一事务的文本产物比较等价语义集合：域名后缀覆盖和 CIDR 并集合并允许编译器消除冗余，但值替换、扩大或丢失范围会失败；清单同时记录语义 SHA-256。发布作业下载后再次验证；`publish-branches.sh` 自身也必须先验证清单，拒绝缺失、额外或被修改的产物。清单只作为流水线审计输入，不复制到发布分支。一次发布中五个分支提交携带共同 generation id 和 source SHA；任一平台 tree 改变时完整 cohort 原子推进并保留各分支父历史，全部 tree 不变时整体跳过。 `scripts/tools/artifact_origins.py` 是 `artifact-origins.json` 的唯一写入口：完整同步重置为 `generated-upstream`，发布分支恢复重置为 `restored-published-branch`，自定义构建只重标本次控制且实际存在的目标，并清除对应平台已经删除或降级省略的旧记录。 diff --git a/scripts/tests/test-binary-artifact-verifier.sh b/scripts/tests/test-binary-artifact-verifier.sh index 86b00893f..e479d7a4c 100644 --- a/scripts/tests/test-binary-artifact-verifier.sh +++ b/scripts/tests/test-binary-artifact-verifier.sh @@ -40,7 +40,7 @@ done PYTHONPATH="$ROOT/scripts/tools" python3 - <<'PY' from collections import Counter -from artifact_verifier import semantic_digest, singbox_counts, singbox_entries +from artifact_verifier import normalized_semantic_entries, semantic_digest, singbox_counts, singbox_entries domain = {"rules": [{"domain": "one.example", "domain_suffix": ["a.example", "b.example"], "domain_keyword": "emby"}]} assert singbox_counts(domain, "domain") == Counter({"DOMAIN-SUFFIX": 2, "DOMAIN": 1, "DOMAIN-KEYWORD": 1}) @@ -55,6 +55,18 @@ assert singbox_counts(ip, "ip") == Counter({"IP-CIDR": 1, "IP-CIDR6": 1}) assert semantic_digest(Counter({("DOMAIN", "one.example"): 1})) != semantic_digest( Counter({("DOMAIN", "two.example"): 1}) ) +assert normalized_semantic_entries(Counter({ + ("DOMAIN-SUFFIX", "example.com"): 1, + ("DOMAIN-SUFFIX", "child.example.com"): 1, + ("DOMAIN", "api.example.com"): 1, +})) == Counter({("DOMAIN-SUFFIX", "example.com"): 1}) +assert normalized_semantic_entries(Counter({ + ("DOMAIN-SUFFIX", "child.example.com"): 1, +})) != Counter({("DOMAIN-SUFFIX", "example.com"): 1}) +assert normalized_semantic_entries(Counter({ + ("IP-CIDR", "192.0.2.0/25"): 1, + ("IP-CIDR", "192.0.2.128/25"): 1, +})) == Counter({("IP-CIDR", "192.0.2.0/24"): 1}) for invalid in ( {"rules": [{"domain": ["one.example"], "port": [443]}]}, {"rules": [{"ip_cidr": ["192.0.2.0/24"]}]}, @@ -125,6 +137,25 @@ if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifi exit 1 fi +printf 'DOMAIN-SUFFIX,example.com\nDOMAIN-SUFFIX,child.example.com\nDOMAIN,api.example.com\n' \ + > "$TMP/exact-repo/sources/custom/domain/reduction.list" +printf 'reduction\n' > "$TMP/exact-repo/.output/domain/sing-box/reduction.srs" +cat > "$TMP/exact-repo/.output/domain/sing-box/reduction.srs.json" <<'EOF' +{"rules":[{"domain_suffix":["example.com"]}]} +EOF +printf 'reduction\n' > "$TMP/exact-repo/.output/domain/mihomo/reduction.mrs" +printf '+.example.com\n' > "$TMP/exact-repo/.output/domain/mihomo/reduction.mrs.txt" +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/exact-repo" \ + --path "$TMP/exact-repo/.output/domain/sing-box/reduction.srs" \ + --type domain \ + --platform sing-box >/dev/null +PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifier.py" \ + --root "$TMP/exact-repo" \ + --path "$TMP/exact-repo/.output/domain/mihomo/reduction.mrs" \ + --type domain \ + --platform mihomo >/dev/null + printf 'orphan\n' > "$TMP/exact-repo/.output/domain/sing-box/orphan.srs" cat > "$TMP/exact-repo/.output/domain/sing-box/orphan.srs.json" <<'EOF' {"rules":[{"domain":["orphan.example"]}]} diff --git a/scripts/tools/artifact_verifier.py b/scripts/tools/artifact_verifier.py index 8a364facd..60682a915 100644 --- a/scripts/tools/artifact_verifier.py +++ b/scripts/tools/artifact_verifier.py @@ -30,8 +30,44 @@ def summarize_entries(entries: Counter[RuleEntry]) -> Counter[str]: for kind in sorted({entry_kind for entry_kind, _ in entries})}) +def normalized_semantic_entries(entries: Counter[RuleEntry]) -> Counter[RuleEntry]: + kinds = {kind for kind, _ in entries} + if kinds and kinds <= {"IP-CIDR", "IP-CIDR6"}: + result: Counter[RuleEntry] = Counter() + for version in (4, 6): + networks = [ + ipaddress.ip_network(value) + for (kind, value) in entries + if (kind == "IP-CIDR6") == (version == 6) + ] + for network in ipaddress.collapse_addresses(networks): + kind = "IP-CIDR6" if network.version == 6 else "IP-CIDR" + result[(kind, str(network))] = 1 + return result + + suffixes = {value for (kind, value) in entries if kind == "DOMAIN-SUFFIX"} + + def covered_by_suffix(value: str, candidates: set[str]) -> bool: + return any(value == suffix or value.endswith("." + suffix) for suffix in candidates) + + minimal_suffixes = { + suffix + for suffix in suffixes + if not covered_by_suffix(suffix, suffixes - {suffix}) + } + result = Counter({("DOMAIN-SUFFIX", suffix): 1 for suffix in minimal_suffixes}) + for (kind, value), count in entries.items(): + if kind == "DOMAIN-SUFFIX": + continue + if kind == "DOMAIN" and covered_by_suffix(value, minimal_suffixes): + continue + result[(kind, value)] = 1 if count else 0 + return +result + + def semantic_digest(entries: Counter[RuleEntry]) -> str: - payload = [[kind, value, count] for (kind, value), count in sorted(entries.items())] + normalized = normalized_semantic_entries(entries) + payload = [[kind, value, count] for (kind, value), count in sorted(normalized.items())] return hashlib.sha256(json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode()).hexdigest() @@ -308,7 +344,7 @@ def verify_one(root: Path, path: Path, artifact_type: str, platform: str) -> dic raise ValueError("binary artifact has no canonical custom source or same-build text counterpart") linkage: dict[str, Any] = {"status": "unavailable", "reason": "canonical binary compiler input is not retained or not applicable"} if canonical is not None: - if decoded != canonical: + if normalized_semantic_entries(decoded) != normalized_semantic_entries(canonical): raise ValueError( "decoded rule values differ from canonical source: " f"decoded_sha256={semantic_digest(decoded)}, canonical_sha256={semantic_digest(canonical)}" From 0a541a0f4c2a1d741be6728577de0a7a027a90e5 Mon Sep 17 00:00:00 2001 From: KuGouGo <62388728+KuGouGo@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:34:51 +0800 Subject: [PATCH 3/3] Optimize domain semantic normalization --- scripts/tools/artifact_verifier.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/tools/artifact_verifier.py b/scripts/tools/artifact_verifier.py index 60682a915..d3e648e84 100644 --- a/scripts/tools/artifact_verifier.py +++ b/scripts/tools/artifact_verifier.py @@ -47,13 +47,20 @@ def normalized_semantic_entries(entries: Counter[RuleEntry]) -> Counter[RuleEntr suffixes = {value for (kind, value) in entries if kind == "DOMAIN-SUFFIX"} + def has_suffix_ancestor(value: str, candidates: set[str]) -> bool: + while "." in value: + value = value.split(".", 1)[1] + if value in candidates: + return True + return False + def covered_by_suffix(value: str, candidates: set[str]) -> bool: - return any(value == suffix or value.endswith("." + suffix) for suffix in candidates) + return value in candidates or has_suffix_ancestor(value, candidates) minimal_suffixes = { suffix for suffix in suffixes - if not covered_by_suffix(suffix, suffixes - {suffix}) + if not has_suffix_ancestor(suffix, suffixes) } result = Counter({("DOMAIN-SUFFIX", suffix): 1 for suffix in minimal_suffixes}) for (kind, value), count in entries.items():