diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 800498342..ea7289db4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -2,6 +2,21 @@ version: 2 updates: - package-ecosystem: github-actions directory: / + target-branch: main schedule: - interval: weekly - open-pull-requests-limit: 5 + interval: monthly + open-pull-requests-limit: 1 + groups: + github-actions: + patterns: + - '*' + update-types: + - minor + - patch + ignore: + - dependency-name: '*' + update-types: + - version-update:semver-major + commit-message: + prefix: deps(actions) + rebase-strategy: auto diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index ba1f6c36d..d2dfd579e 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -28,5 +28,47 @@ jobs: - name: Prepare scripts run: find scripts -type f \( -name '*.sh' -o -name '*.py' \) -exec chmod +x {} + - - name: Validate scripts and fixtures - run: REQUIRE_SHELLCHECK=1 make validate + - name: Run preflight validation and text build + run: REQUIRE_SHELLCHECK=1 make preflight + + release-candidate: + if: github.base_ref == 'main' + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.11' + + - name: Prepare scripts + run: find scripts -type f \( -name '*.sh' -o -name '*.py' \) -exec chmod +x {} + + + - name: Resolve tool versions + id: core_versions + run: bash ./scripts/commands/resolve-tool-versions.sh + + - name: Restore verified tool cache + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: .bin + key: rules-tools-v3-${{ steps.core_versions.outputs.sing_box_version }}-${{ steps.core_versions.outputs.mihomo_version }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('config/tools-lock.json') }} + + - name: Build and verify release candidate + env: + RULES_BUILD_SCOPE: full + ARTIFACT_GENERATION_ID: pr-${{ github.event.pull_request.number }}-${{ github.run_attempt }} + ARTIFACT_BUILD_ID: ${{ github.run_id }} + ARTIFACT_SOURCE_SHA: ${{ github.event.pull_request.head.sha }} + run: ./scripts/commands/build-artifacts-transaction.sh + + - name: Show release candidate summary + run: | + cat .output/build-summary.json + cat .output/upstream-summary.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 638400f66..c717cab48 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,7 @@ ## 开始之前 -从最新 `main` 创建范围单一的分支。提交第三方规则、数据或派生内容时,提供原始来源及可核验的许可、条款或授权;无法确认时明确写“未知”,并由维护者在合并和发布前人工决定处置。 +日常变更从临时分支向 `main` 提交 Pull Request,验证通过并合并后发布。提交第三方规则、数据或派生内容时,提供原始来源及可核验的许可、条款或授权;无法确认时明确写“未知”,并由维护者在合并和发布前人工决定处置。 许可审查不是自动检查。CI 不解析 `NOTICE` 或 `THIRD_PARTY_NOTICES.md`,也不会因“未知”状态自动失败。公开 URL、官方来源、`trust` 分类和 CI 通过都不能替代人工许可评审。 diff --git a/Makefile b/Makefile index 7983a7d8a..07ee45f18 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,15 @@ SHELL := /usr/bin/env bash REQUIRE_SHELLCHECK ?= 0 +BASH_MIN_MAJOR := 5 SHELL_SCRIPTS := $(shell find scripts -type f -name '*.sh' | sort) PYTHON_TOOLS := $(shell find scripts/tools -type f -name '*.py' | sort) -.PHONY: help lint lint-shell lint-python lint-config lint-rules test validate preflight build-custom build-custom-text clean +.PHONY: help check-runtime lint lint-shell lint-python lint-config lint-rules test validate preflight build-custom build-custom-text clean help: @echo "Available targets:" + @echo " make check-runtime Verify the supported Bash runtime" @echo " make lint Run shell, Python, and custom rule lint checks" @echo " make test Run all repository test scripts" @echo " make validate Run lint and tests" @@ -16,7 +18,10 @@ help: @echo " make build-custom-text Build custom text artifacts without downloading binary compilers" @echo " make clean Remove generated artifacts and temporary files" -lint: lint-shell lint-python lint-config lint-rules +check-runtime: + @bash -c 'if (( BASH_VERSINFO[0] < $(BASH_MIN_MAJOR) )); then echo "Bash $(BASH_MIN_MAJOR)+ is required (found $$BASH_VERSION)" >&2; exit 1; fi' + +lint: check-runtime lint-shell lint-python lint-config lint-rules lint-shell: bash -n $(SHELL_SCRIPTS) @@ -38,17 +43,17 @@ lint-config: lint-rules: ./scripts/commands/lint-custom-rules.sh -test: +test: check-runtime ./scripts/tests/run.sh validate: lint test preflight: validate build-custom-text -build-custom: +build-custom: check-runtime ./scripts/commands/build-custom.sh -build-custom-text: +build-custom-text: check-runtime RULES_BUILD_CUSTOM_TEXT_ONLY=1 ./scripts/commands/build-custom.sh clean: diff --git a/README.md b/README.md index f55ed233e..70b9fbb64 100644 --- a/README.md +++ b/README.md @@ -7,39 +7,28 @@ License

-面向 **Surge、Quantumult X、Egern、sing-box 和 mihomo** 的多平台规则构建仓库。仓库维护少量原创规则,并同步第三方数据,经规范化、转换和编译后发布到五个平台分支。 +面向 Surge、Quantumult X、Egern、sing-box 和 mihomo 的规则构建仓库。`main` 保存稳定源码并触发发布,客户端产物位于对应平台分支。 > [!IMPORTANT] -> `main` 保存源码和构建逻辑,不是规则下载分支。客户端应引用 `surge`、`quanx`、`egern`、`sing-box` 或 `mihomo`。MIT 许可的适用范围及第三方边界见 [`NOTICE`](NOTICE) 和 [`THIRD_PARTY_NOTICES.md`](THIRD_PARTY_NOTICES.md)。 +> 客户端不要引用 `main`。第三方规则不因格式转换而自动适用 MIT,来源和许可边界见 [`NOTICE`](NOTICE) 与 [`THIRD_PARTY_NOTICES.md`](THIRD_PARTY_NOTICES.md)。 -## 快速选择 +## 使用规则 -| 需求 | 规则 | 类型 | 注意事项 | -| --- | --- | --- | --- | -| 中国大陆域名 | `cn` 或 `geolocation-cn` | domain | 两者沿用不同上游语义,不是同义别名 | -| 非中国大陆域名 | `geolocation-!cn` | domain | `!cn` 是上游名称的一部分 | -| 中国大陆 IP | `cn` | ip | 合并多个已配置来源并规范化为 CIDR | -| Google / Telegram IP | `google` / `telegram` | ip | Telegram 还会合并配置的 ASN 前缀 | -| Emby 中国大陆细分 | `emby-cn` | domain | 必须放在 `emby` 前面 | -| 通用 Emby | `emby` | domain | 包含较宽泛的后缀和关键词 | - -上游列表名称会随上游演进。使用前请在目标发布分支的 `domain/` 或 `ip/` 目录确认文件存在;构建摘要和结构清单不随发布分支分发。 - -## 稳定 URL 模板 +稳定 URL 格式: ```text https://raw.githubusercontent.com/KuGouGo/Rules/{branch}/{type}/{name}.{extension} ``` -| 分支 | domain | ip | -| --- | --- | --- | -| `surge` | `{name}.list` | `{name}.list` | -| `quanx` | `{name}.list` | `{name}.list` | -| `egern` | `{name}.yaml` | `{name}.yaml` | -| `sing-box` | `{name}.srs` | `{name}.srs` | -| `mihomo` | `{name}.mrs` | `{name}.mrs` | +| 客户端 | 分支 | domain / ip 扩展名 | 分支说明 | +| --- | --- | --- | --- | +| Surge | `surge` | `.list` | [查看产物](https://github.com/KuGouGo/Rules/tree/surge) | +| Quantumult X | `quanx` | `.list` | [查看产物](https://github.com/KuGouGo/Rules/tree/quanx) | +| Egern | `egern` | `.yaml` | [查看产物](https://github.com/KuGouGo/Rules/tree/egern) | +| sing-box | `sing-box` | `.srs` | [查看产物](https://github.com/KuGouGo/Rules/tree/sing-box) | +| mihomo | `mihomo` | `.mrs` | [查看产物](https://github.com/KuGouGo/Rules/tree/mihomo) | -五个平台的可用 URL 示例: +示例: ```text https://raw.githubusercontent.com/KuGouGo/Rules/surge/domain/cn.list @@ -49,188 +38,75 @@ https://raw.githubusercontent.com/KuGouGo/Rules/sing-box/domain/cn.srs https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/ip/google.mrs ``` -发布分支 URL 会随更新改变内容,不是固定快照。需要可复现内容时请把 URL 中的分支替换为具体提交 SHA。 - -## 五平台接入示例 - -示例策略仅用于展示,请替换为客户端中真实存在的策略或出站标签,并保持规则顺序。 - -### Surge - -Surge 产物为 classical rule-set;IP 规则默认带 `no-resolve`。 - -```ini -[Rule] -RULE-SET,https://raw.githubusercontent.com/KuGouGo/Rules/surge/domain/emby-cn.list,DIRECT -RULE-SET,https://raw.githubusercontent.com/KuGouGo/Rules/surge/domain/emby.list,Emby -RULE-SET,https://raw.githubusercontent.com/KuGouGo/Rules/surge/ip/cn.list,DIRECT -``` - -### Quantumult X - -Quantumult X 产物使用 `HOST`、`HOST-SUFFIX`、`HOST-KEYWORD`、`IP-CIDR` 和 `IP6-CIDR`。构建器把规则名写入第三字段;使用 `filter_remote` 时应通过 `force-policy` 绑定本地策略。 - -```ini -[filter_remote] -https://raw.githubusercontent.com/KuGouGo/Rules/quanx/domain/emby-cn.list, tag=Emby-CN, force-policy=direct, enabled=true -https://raw.githubusercontent.com/KuGouGo/Rules/quanx/domain/emby.list, tag=Emby, force-policy=Emby, enabled=true -https://raw.githubusercontent.com/KuGouGo/Rules/quanx/ip/cn.list, tag=CN-IP, force-policy=direct, enabled=true -``` - -### Egern +各平台的接入方式和最小配置示例位于对应产物分支的 `README.md`。分支 URL 始终指向最新产物;需要固定内容时,将分支名替换为具体提交 SHA。 -Egern 产物为 YAML 规则集。域名按内容写入对应集合,IP 写入 IPv4/IPv6 CIDR 集合并设置 `no_resolve: true`。 +## 常用规则 -```text -https://raw.githubusercontent.com/KuGouGo/Rules/egern/domain/emby-cn.yaml -https://raw.githubusercontent.com/KuGouGo/Rules/egern/domain/emby.yaml -https://raw.githubusercontent.com/KuGouGo/Rules/egern/ip/cn.yaml -``` - -在 Egern 中添加远程规则集,再绑定本地策略。仓库只保证生成的数据结构,不提供完整客户端配置。 - -### sing-box - -sing-box 产物是 `sing-box rule-set compile` 生成的二进制 `.srs`。以下仅为展示远程规则集引用关系的概念片段,不是可直接使用的完整配置。 - -```json -{ - "route": { - "rules": [ - { "rule_set": ["emby-cn"], "outbound": "direct" }, - { "rule_set": ["emby"], "outbound": "proxy" }, - { "rule_set": ["cn-ip"], "outbound": "direct" } - ], - "rule_set": [ - { - "tag": "emby-cn", - "type": "remote", - "format": "binary", - "url": "https://raw.githubusercontent.com/KuGouGo/Rules/sing-box/domain/emby-cn.srs" - }, - { - "tag": "emby", - "type": "remote", - "format": "binary", - "url": "https://raw.githubusercontent.com/KuGouGo/Rules/sing-box/domain/emby.srs" - }, - { - "tag": "cn-ip", - "type": "remote", - "format": "binary", - "url": "https://raw.githubusercontent.com/KuGouGo/Rules/sing-box/ip/cn.srs" - } - ] - } -} -``` - -字段可能随 sing-box 版本演进,请以所用版本的官方 schema 为准。 - -### mihomo - -mihomo 只发布二进制 `.mrs`。domain provider 使用 `behavior: domain`,IP provider 使用 `behavior: ipcidr`。 - -```yaml -rule-providers: - emby-cn: - type: http - behavior: domain - format: mrs - url: https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/domain/emby-cn.mrs - path: ./ruleset/emby-cn.mrs - interval: 86400 - emby: - type: http - behavior: domain - format: mrs - url: https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/domain/emby.mrs - path: ./ruleset/emby.mrs - interval: 86400 - cn-ip: - type: http - behavior: ipcidr - format: mrs - url: https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/ip/cn.mrs - path: ./ruleset/cn-ip.mrs - interval: 86400 -rules: - - RULE-SET,emby-cn,DIRECT - - RULE-SET,emby,Emby - - RULE-SET,cn-ip,DIRECT,no-resolve -``` - -## 平台能力与降级 - -| 平台 | 精确域名 | 后缀 | 关键词 | 正则 | IP CIDR | 当前转换边界 | -| --- | :---: | :---: | :---: | :---: | :---: | --- | -| Surge | ✓ | ✓ | ✓ | — | ✓ | 不写入域名正则 | -| Quantumult X | ✓ | ✓ | ✓ | — | ✓ | 转换为 `HOST*`;不写入域名正则 | -| Egern | ✓ | ✓ | ✓ | ✓ | ✓ | 保留四类域名规则 | -| sing-box | ✓ | ✓ | ✓ | ✓ | ✓ | 编译为 `.srs` | -| mihomo | ✓ | ✓ | — | — | ✓ | domain `.mrs` 仅保留精确域名与后缀 | - -`—` 只表示当前转换链路未保留,不表示客户端本身不支持。若源列表仅含目标平台不保留的类型,该平台可能不发布对应空产物。 - -## 规则语义 - -- `name@cn`、`name@!cn`、`name@ads` 是按上游属性派生的集合;`@!cn` 不是布尔取反。 -- `*-cn` 和 `*-!cn` 保留上游区域列表命名;明显冗余的属性派生不会发布。 -- `official`、`registry`、`community` 只是采集来源分类,不是质量等级或许可结论。 -- `emby-cn` 与 `emby` 当前有三条经审核并逐条锁定的覆盖关系。采用首条命中时必须先加载 `emby-cn`,再加载 `emby`。 - -## 更新、构建与审计边界 +| 名称 | 类型 | 用途 | +| --- | --- | --- | +| `cn` | domain / ip | 中国大陆域名或合并后的中国大陆 CIDR | +| `geolocation-cn` | domain | 上游定义的中国大陆区域域名 | +| `geolocation-!cn` | domain | 上游定义的非中国大陆区域域名 | +| `google` / `telegram` | ip | 对应服务的 CIDR;Telegram 额外合并 ASN 前缀 | +| `emby-cn` | domain | Emby 中国大陆细分规则 | +| `emby` | domain | 通用 Emby 规则 | +| `fakeip-filter` | domain | 本仓库维护的 Fake-IP 排除规则 | -- `Sync Rules` 每天 08:00 UTC 完整同步;也支持手动选择构建范围。 -- `main` 的相关实现、配置或自定义源变更会触发构建;纯文档变更不在 `push.paths` 中。 -- 自定义范围会恢复发布分支产物后重建自定义规则;完整范围会同步全部已配置上游。 -- `fakeip-filter` 当前由本仓库在 `sources/custom/domain/fakeip-filter.list` 维护,并与其他自定义规则一起生成 Surge、Quantumult X、Egern 文本产物及 sing-box、mihomo 二进制产物;构建不再下载第三方预编译文件。 -- 过去曾直接采用 `wwqgtxx/clash-rules` 的预编译 `fakeip-filter.mrs`;该路径仅属历史,不是当前输入或构建步骤。 -- `.output/upstream-summary.json` 由主上游同步生成,只是部分采集摘要;本仓库维护的自定义源(包括 `fakeip-filter`)、完整转换链、提交身份和内容校验和不在其中,因此它不是完整来源追溯记录。 -- `.output/domain/rule-manifest.json` 描述域名列表和属性派生结构。 -- `.output/build-summary.json` 由成功的构建事务在产物守卫之后、manifest 之前生成,并由 manifest 绑定文件摘要与嵌入内容;独立运行 `make build-custom*` 不会生成它。 -- 摘要和结构清单都不是许可证证明,也不进入客户端发布分支。 +`emby-cn` 必须放在 `emby` 前面。规则名称来自上游,不同名称即使内容相近也不视为别名;使用前请在目标分支确认文件存在。 -## 校验的实际范围 +## 平台能力 -`make validate` 执行 Shell 语法、可用时的 ShellCheck、Python 编译、配置、自定义规则和测试检查。CI 强制要求 ShellCheck。 +| 平台 | 精确域名 | 后缀 | 关键词 | 正则 | IP CIDR | +| --- | :---: | :---: | :---: | :---: | :---: | +| Surge | ✓ | ✓ | ✓ | — | ✓ | +| Quantumult X | ✓ | ✓ | ✓ | — | ✓ | +| Egern | ✓ | ✓ | ✓ | ✓ | ✓ | +| sing-box | ✓ | ✓ | ✓ | ✓ | ✓ | +| mihomo | ✓ | ✓ | — | — | ✓ | -产物守卫(artifact guard)检查产物最低数量、域名派生形状、部分文本域名产物的下降、Surge/Quantumult X 文本 IP 的 CIDR 与非公网地址、部分内置 IP 集最低数量及其 Surge 基线波动。它不对所有二进制内容执行等价语义检查。 +`—` 表示当前转换链不保留该类型,不代表客户端本身不支持。列表只包含不受支持的规则类型时,对应平台不会发布空产物。 -自定义名称冲突检查发生在构建阶段,仅针对相对基准提交**新加入**的自定义源,并在同一 `domain` 或 `ip` 类型内检查五个平台当前 `.output/` 路径;既有自定义源的修改不会经过同一“新增名称”判断,domain 与 ip 同名也不互相冲突。 +属性与区域名称沿用上游语义: -许可核验完全依赖维护者人工评审。脚本不会解析 `NOTICE` 或 `THIRD_PARTY_NOTICES.md`,未知许可也不会被 CI 自动识别或阻断。CI 通过不能视为已取得授权。 +- `name@cn`、`name@!cn`、`name@ads` 是属性派生集合,`@!cn` 不是布尔取反。 +- `*-cn`、`*-!cn` 是区域列表名称;明显冗余的属性派生不会发布。 +- `official`、`registry`、`community` 仅表示采集来源分类,不代表质量或许可等级。 -## 本地开发 +## 本地维护 -支持目标是 GitHub Actions 的 Ubuntu 环境,以及具备 Bash、GNU Make、Git、Python 3、curl 和常用 GNU 工具的 Linux/WSL。脚本可识别非 Windows 的 `amd64` 与 `arm64` 二进制工具资产;原生 Windows 在需要下载新工具时会被明确拒绝,完整二进制构建不属于支持组合,其他环境组合也不作保证。 +要求 Bash 5+、GNU Make、Git 和 Python 3。macOS 可使用 Homebrew Bash 完成验证与文本构建;sing-box 和 mihomo 的完整二进制构建仅支持 lock 文件声明的 Linux 平台。 ```bash -make help +make check-runtime make validate make build-custom-text -make build-custom make preflight make clean ``` -`make build-custom-text` 不下载二进制编译器;`make preflight` 也不执行完整上游同步、产物守卫(artifact guard)或发布。完整构建只接受 `config/tools-lock.json` 固定的版本、tag commit、Linux 资产名和 SHA-256;归档在解包前校验,`.bin/` 命中还会复核原子写入的 provenance、二进制 SHA-256 与版本探针。`make clean` 删除 `.tmp/`、`.output/`、Python 缓存和未完成的工具临时文件,但保留已验证的工具缓存。 - -自定义域名源位于 `sources/custom/domain/*.list`,支持 `DOMAIN`、`DOMAIN-SUFFIX`、`DOMAIN-KEYWORD` 和 `DOMAIN-REGEX`。可选 IP 源位于 `sources/custom/ip/*.list`,支持 `IP-CIDR` 和 `IP-CIDR6`;目录可不存在。文件名只使用小写字母、数字和连字符。 +- `make validate`:运行 Shell、Python、配置、规则质量和测试检查。 +- `make build-custom-text`:生成自定义文本产物,不下载二进制工具。 +- `make preflight`:执行完整验证和文本构建,适合提交前检查。 +- `make clean`:删除生成产物和临时文件,保留可信工具缓存。 -## 许可与风险 +完整环境、构建事务和调试命令见 [开发指南](docs/DEVELOPMENT.md)。 -标准 MIT License 位于 [`LICENSE`](LICENSE),适用范围说明位于 [`NOTICE`](NOTICE)。第三方来源及人工核对状态见 [`THIRD_PARTY_NOTICES.md`](THIRD_PARTY_NOTICES.md)。格式转换、合并或编译不会把第三方材料重新许可为 MIT;“官方来源”、公开 URL、摘要记录或构建成功也不代表获得再分发授权。 +## 仓库边界 -规则按现状提供,不保证完整、实时、无误或适合特定用途。部署前请审查规则并保留回滚方案。本说明不是法律意见。 +- 长期分支只保留 `main` 与 `surge`、`quanx`、`egern`、`sing-box`、`mihomo` 五个产物分支。 +- 代码、文档和依赖更新通过临时分支向 `main` 提交 Pull Request;验证通过并合并后才构建和发布,临时分支自动删除。 +- Dependabot 每月向 `main` 集中提交一个 GitHub Actions minor/patch 更新 PR;major 与安全告警单独人工评估。 +- `fakeip-filter` 是本仓库维护的文本源,不下载第三方预编译文件。 +- 构建摘要、manifest 和 CI 通过都不是第三方许可证明。 +- 规则按现状提供,部署前应检查内容并保留回滚方案。 -## 文档导航 +## 文档 -- [`README.md`](README.md):用户入口、平台示例和关键边界 -- [`CONTRIBUTING.md`](CONTRIBUTING.md):贡献规则与人工评审清单 -- [`docs/README.md`](docs/README.md):文档职责与阅读路径 -- [`docs/DEVELOPMENT.md`](docs/DEVELOPMENT.md):环境、命令和开发流程 -- [`docs/STRUCTURE.md`](docs/STRUCTURE.md):构建、产物、守卫和发布结构 -- [`docs/TROUBLESHOOTING.md`](docs/TROUBLESHOOTING.md):常见失败与定位步骤 -- [`SECURITY.md`](SECURITY.md):安全支持范围和私密报告 -- [`NOTICE`](NOTICE) / [`THIRD_PARTY_NOTICES.md`](THIRD_PARTY_NOTICES.md):许可范围与第三方状态 +| 文档 | 内容 | +| --- | --- | +| [贡献指南](CONTRIBUTING.md) | 规则格式、来源要求和评审清单 | +| [开发指南](docs/DEVELOPMENT.md) | 环境、命令和开发流程 | +| [仓库结构](docs/STRUCTURE.md) | 同步、构建、manifest、守卫和发布架构 | +| [故障排查](docs/TROUBLESHOOTING.md) | 常见构建和产物问题 | +| [安全政策](SECURITY.md) | 安全问题报告范围与方式 | +| [第三方声明](THIRD_PARTY_NOTICES.md) | 上游来源和人工许可核对状态 | diff --git a/SECURITY.md b/SECURITY.md index 3fc6d8a19..e2534bfcd 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,7 +8,7 @@ 可能导致凭据泄露、工作流或发布分支被接管、任意代码执行、供应链污染或绕过发布守卫的问题属于安全问题。规则误分流、上游失效、普通构建错误和许可状态未知通常不是安全漏洞,请使用普通 Issue 或 Pull Request,但不要公开敏感信息。 -工具下载当前依赖 GitHub Release 的 HTTPS 资产和版本匹配,不验证校验和。若发现资产替换、下载链路劫持或缓存投毒证据,请按安全问题私密报告。 +构建工具固定到 `config/tools-lock.json` 声明的 GitHub Release 资产,并校验归档与解包后二进制的 SHA-256;缓存还会复核来源 metadata、摘要和版本探针。该机制不等同于发布者签名,若发现资产替换、下载链路劫持或缓存投毒证据,请按安全问题私密报告。 ## 私密报告 diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index fab5d0e9e..d541615e9 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -4,7 +4,7 @@ ## 环境支持 -CI 的受支持基准是 GitHub Actions `ubuntu-latest` 和 Python 3.11。本地建议使用具备 Bash、GNU Make、Git、Python 3、curl、tar、gzip、find 等常用 GNU 工具的 Linux 或 WSL。 +CI 的受支持基准是 GitHub Actions `ubuntu-latest`、Bash 5+ 和 Python 3.11。本地验证与文本构建支持安装了 Bash 5+、GNU Make、Git、Python 3、curl、tar、gzip、find 的 Linux、WSL 和 macOS;macOS 应使用 Homebrew Bash,并确保 `/opt/homebrew/bin` 或 `/usr/local/bin` 位于 `/bin` 之前。`make check-runtime` 会在构建前拒绝旧版 Bash。 二进制工具下载逻辑仅支持 Linux 的 `amd64`、`arm64` 锁定资产。原生 Windows 在需要下载 sing-box/mihomo 时会被 `require_non_windows_shell` 明确拒绝;macOS 也没有对应 lock 平台。请使用 Linux、WSL 或 GitHub Actions 完成二进制构建。 @@ -26,6 +26,7 @@ make clean ``` - `make validate`:Shell 语法、可用时的 ShellCheck、Python 编译、配置、自定义规则和测试。 +- `make check-runtime`:验证当前 `PATH` 解析到 Bash 5+。 - `make build-custom-text`:只生成自定义文本产物,不下载二进制编译器。 - `make build-custom`:生成自定义文本和二进制产物。 - `make preflight`:`make validate` 加文本自定义构建;不执行完整同步、产物守卫(artifact guard)或发布。 @@ -38,13 +39,16 @@ make clean CI 设置 `REQUIRE_SHELLCHECK=1`,本地缺少 ShellCheck 时的跳过不代表 CI 会通过。 +GitHub Actions 使用完整 commit SHA 固定版本。Dependabot 每月把 GitHub Actions 的 minor/patch 更新组合为一个以 `main` 为目标的 PR,减少临时分支和重复 CI;major 更新单独评估,避免阻塞常规更新。GitHub 漏洞告警保持启用,自动安全修复分支关闭;安全更新由维护者确认影响后通过临时分支提交。合并后的临时分支由 GitHub 自动删除。 + ## 开发流程 -1. 从最新 `main` 创建分支,不手工编辑生成目录。 +1. 从 `main` 创建临时分支,不手工编辑生成目录。 2. 修改自定义源、配置、实现或测试夹具。 -3. 运行 `make validate` 和适用的自定义构建命令。 +3. 运行 `make preflight` 和适用的完整构建命令。 4. 检查差异中没有 `.output/`、`.tmp/`、`.bin/`、凭据或无关格式化。 -5. 按 [贡献指南](../CONTRIBUTING.md) 说明来源、人工许可评审状态、测试和产物影响。 +5. 通过 Pull Request 合并到 `main`;PR 必须完成预检和不发布的完整构建,合并后由 `main` 工作流更新五个平台分支。 +6. 按 [贡献指南](../CONTRIBUTING.md) 说明来源、人工许可评审状态、测试和产物影响。 ## 自定义规则与名称 diff --git a/docs/STRUCTURE.md b/docs/STRUCTURE.md index 46c62a90f..b6606e367 100644 --- a/docs/STRUCTURE.md +++ b/docs/STRUCTURE.md @@ -23,7 +23,7 @@ `fakeip-filter` 当前源为本仓库维护的 `sources/custom/domain/fakeip-filter.list`,由 `build-custom.sh` 与其他自定义规则一起生成五平台形式,不从网络下载预编译文件。过去采用 `wwqgtxx/clash-rules` 的 `fakeip-filter.mrs` 仅是历史迁移背景,不属于当前输入。`config/upstreams.json` 覆盖主上游网络输入;工具资产下载另由工具 lock 控制。 -发布分支为 `surge`、`quanx`、`egern`、`sing-box`、`mihomo`,只允许生成的 `README.md`、`domain/`、`ip/` 及平台对应扩展名。各分支 `README.md` 由 `templates/branch-readmes/` 生成,并直接包含 v2fly/domain-list-community 的完整 MIT 版权与许可通知;因此发布树无需新增独立许可证文件。模板变更属于构建触发路径。 +`main` 是唯一长期源码与发布源分支,开发使用合并后删除的临时分支。发布分支为 `surge`、`quanx`、`egern`、`sing-box`、`mihomo`,只允许生成的 `README.md`、`domain/`、`ip/` 及平台对应扩展名。各分支 `README.md` 由 `templates/branch-readmes/` 生成,并直接包含 v2fly/domain-list-community 的完整 MIT 版权与许可通知;因此发布树无需新增独立许可证文件。模板变更属于构建触发路径。 ## 审计文件 @@ -35,6 +35,8 @@ `verify-artifact-manifest.sh` 严格重算能力矩阵允许的完整文件集合、路径层级与安全性、非零大小、字节数和 SHA-256,并核对能力/lock 摘要及可选的预期 source SHA。发布作业下载后再次验证;`publish-branches.sh` 自身也必须先验证清单,拒绝缺失、额外或被修改的产物。清单只作为流水线审计输入,不复制到发布分支。分支提交消息携带共同 generation id 和 source SHA。 +`scripts/tools/artifact_origins.py` 是 `artifact-origins.json` 的唯一写入口:完整同步重置为 `generated-upstream`,发布分支恢复重置为 `restored-published-branch`,自定义构建只重标本次控制且实际存在的目标,并清除对应平台已经删除或降级省略的旧记录。 + `config/domain-platform-capabilities.json` 是平台 branch、extension、format、rule mapping、empty policy、compiler 与 verifier 的唯一结构化来源。`scripts/tools/platform_capabilities.py` 严格加载并校验实现标识,同时向 Python 消费者提供查询对象、向 shell 消费者生成稳定的 tab-separated registry。IP 渲染、构建/守卫的安全循环、摘要和发布均查询该 registry;声明未知实现时构建会 fail closed。公开分支名仍由能力文件中的 `branch` 字段固定。 `upstream-summary.json` 不是完整来源追溯记录:它不覆盖自定义源(包括 `fakeip-filter`)、全部转换步骤、上游提交身份或内容校验和。上述文件也都不是许可证证明。 diff --git a/scripts/commands/build-artifacts-transaction.sh b/scripts/commands/build-artifacts-transaction.sh old mode 100644 new mode 100755 index 86d20e7bd..628f71b1c --- a/scripts/commands/build-artifacts-transaction.sh +++ b/scripts/commands/build-artifacts-transaction.sh @@ -22,7 +22,8 @@ mkdir -p "$STAGE_ROOT" preserve_failure_diagnostics() { local status=$? if [ "$status" -ne 0 ]; then - local failure_dir="$DIAGNOSTICS_ROOT/${ARTIFACT_GENERATION_ID:-local}-$(date -u +%Y%m%dT%H%M%SZ)" + local failure_dir + failure_dir="$DIAGNOSTICS_ROOT/${ARTIFACT_GENERATION_ID:-local}-$(date -u +%Y%m%dT%H%M%SZ)" mkdir -p "$failure_dir" [ ! -f "$STAGE_ROOT/upstream-summary.json" ] || cp "$STAGE_ROOT/upstream-summary.json" "$failure_dir/" [ ! -f "$STAGE_ROOT/build-summary.json" ] || cp "$STAGE_ROOT/build-summary.json" "$failure_dir/" diff --git a/scripts/commands/build-custom.sh b/scripts/commands/build-custom.sh index 26c1fc746..2f3662649 100755 --- a/scripts/commands/build-custom.sh +++ b/scripts/commands/build-custom.sh @@ -106,6 +106,18 @@ custom_source_existed_in_base() { printf '%s\n' "$base_custom_sources" | grep -Fxq "$rel_path" } +collect_generated_custom_artifacts() { + python3 "$ROOT/scripts/tools/artifact_origins.py" list \ + "$ARTIFACT_ROOT" \ + --origin generated-custom +} + +artifact_was_generated_custom() { + local relative_path="$1" + + printf '%s\n' "$GENERATED_CUSTOM_ARTIFACTS" | grep -Fxq "$relative_path" +} + historical_fakeip_migration_collision() { local base_ref="$1" local custom_rel_path="$2" @@ -138,7 +150,8 @@ assert_no_name_conflict() { if historical_fakeip_migration_collision "$base_ref" "$custom_rel_path" "$tracked_path"; then continue fi - if [ -e "$ARTIFACT_ROOT/${tracked_path#.output/}" ]; then + if [ -e "$ARTIFACT_ROOT/${tracked_path#.output/}" ] \ + && ! artifact_was_generated_custom "${tracked_path#.output/}"; then conflicts+=("$tracked_path") fi done @@ -152,6 +165,8 @@ assert_no_name_conflict() { fi } +GENERATED_CUSTOM_ARTIFACTS="$(collect_generated_custom_artifacts)" + build_domain_plain_and_surge() { local list_file="$1" local base surge_out quanx_out egern_out plain_out surge_tmp quanx_tmp egern_tmp @@ -371,21 +386,11 @@ if [ "$TEXT_ONLY_MODE" -ne 1 ]; then inject_custom_build_failure late-binary fi commit_staged_custom_artifacts -python3 - <<'PY' "$ARTIFACT_ROOT" "$CUSTOM_DOMAIN_DIR" "$CUSTOM_IP_DIR" -import json, sys -from pathlib import Path -root, domain_sources, ip_sources = map(Path, sys.argv[1:]) -target = root / "artifact-origins.json" -origins = json.loads(target.read_text(encoding="utf-8")) if target.is_file() else {} -for section, sources in (("domain", domain_sources), ("ip", ip_sources)): - if not sources.is_dir(): - continue - for source in sources.glob("*.list"): - for path in (root / section).glob(f"*/{source.stem}.*"): - if path.is_file(): - origins[path.relative_to(root).as_posix()] = "generated-custom" -target.write_text(json.dumps(origins, indent=2, sort_keys=True) + "\n", encoding="utf-8") -PY +origin_args=(mark-custom "$ARTIFACT_ROOT" "$CUSTOM_DOMAIN_DIR" "$CUSTOM_IP_DIR") +if [ "$TEXT_ONLY_MODE" -eq 1 ]; then + origin_args+=(--text-only) +fi +python3 "$ROOT/scripts/tools/artifact_origins.py" "${origin_args[@]}" if [ "$TEXT_ONLY_MODE" -eq 1 ]; then echo "custom build done (text only)" diff --git a/scripts/commands/generate-artifact-manifest.sh b/scripts/commands/generate-artifact-manifest.sh old mode 100644 new mode 100755 diff --git a/scripts/commands/resolve-tool-versions.sh b/scripts/commands/resolve-tool-versions.sh old mode 100644 new mode 100755 diff --git a/scripts/commands/restore-artifacts.sh b/scripts/commands/restore-artifacts.sh index 58c603cba..716f3fad6 100755 --- a/scripts/commands/restore-artifacts.sh +++ b/scripts/commands/restore-artifacts.sh @@ -151,10 +151,10 @@ restore_branch_artifacts() { subject="$(git log -1 --format=%s "origin/$branch")" generation="$(printf '%s' "$subject" | grep -oE '\[generation [^ ]+' | cut -d' ' -f2 || true)" source="$(printf '%s' "$subject" | grep -oE 'source [0-9a-f]{40}\]' | cut -d' ' -f2 | tr -d ']' || true)" - [ -n "$generation" ] && [ -n "$source" ] || { + if [ -z "$generation" ] || [ -z "$source" ]; then echo "origin/$branch lacks required generation/source publication metadata" >&2 return 1 - } + fi printf '%s\t%s\t%s\t%s\n' "$branch" "$commit" "$generation" "$source" >> "$RESTORE_METADATA_FILE" echo "restored $branch artifacts at $commit (generation $generation, source $source)" } @@ -183,18 +183,8 @@ payload = {"generation_id": next(iter(generations)), "source_commit": next(iter( Path(sys.argv[2]).write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") PY -python3 - <<'PY' "$ARTIFACT_ROOT" "$ARTIFACT_ROOT/artifact-origins.json" -import json, sys -from pathlib import Path -root, target = map(Path, sys.argv[1:]) -origins = {} -for section in ("domain", "ip"): - base = root / section - if base.is_dir(): - for path in base.glob("*/*"): - if path.is_file(): - origins[path.relative_to(root).as_posix()] = "restored-published-branch" -target.write_text(json.dumps(origins, indent=2, sort_keys=True) + "\n", encoding="utf-8") -PY +python3 "$ROOT/scripts/tools/artifact_origins.py" reset \ + "$ARTIFACT_ROOT" \ + restored-published-branch echo "published artifact restore done" diff --git a/scripts/commands/select-build-scope.sh b/scripts/commands/select-build-scope.sh index 067a99ddf..5ed6d982c 100755 --- a/scripts/commands/select-build-scope.sh +++ b/scripts/commands/select-build-scope.sh @@ -71,9 +71,16 @@ if [ "$EVENT_NAME" = "workflow_dispatch" ]; then scope="full" reason="manual custom baseline unavailable; using full sync" base_sha="" + elif ! deleted_custom_files="$(collect_deleted_custom_files "$base_sha" "$CURRENT_SHA")"; then + scope="full" + reason="manual custom diff unavailable; using full sync" + base_sha="" elif [ -z "$changed_files" ]; then scope="full" reason="manual custom delta empty; using full sync" + elif [ -n "$deleted_custom_files" ]; then + scope="full" + reason="custom deletions require full sync" elif printf '%s\n' "$changed_files" | grep -Eqv '^sources/custom/'; then echo "manual custom scope refused: delta contains non-custom paths" >&2 printf '%s\n' "$changed_files" >&2 diff --git a/scripts/commands/sync-fakeip-filter.sh b/scripts/commands/sync-fakeip-filter.sh deleted file mode 100755 index 9998d454d..000000000 --- a/scripts/commands/sync-fakeip-filter.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT="$(cd "$(dirname "$0")/../.." && pwd)" -cd "$ROOT" - -URL="https://raw.githubusercontent.com/wwqgtxx/clash-rules/release/fakeip-filter.mrs" -OUT=".output/domain/mihomo/fakeip-filter.mrs" -TMP="${OUT}.tmp" - -mkdir -p "$(dirname "$OUT")" - -curl --retry 3 --retry-all-errors --connect-timeout 20 --max-time 120 -fL "$URL" -o "$TMP" - -if [ -f "$OUT" ] && cmp -s "$TMP" "$OUT"; then - rm -f "$TMP" - echo "fakeip-filter.mrs unchanged" - exit 0 -fi - -mv "$TMP" "$OUT" -echo "updated $OUT" diff --git a/scripts/commands/sync-upstream.sh b/scripts/commands/sync-upstream.sh index 263c1e9b6..8e937facf 100755 --- a/scripts/commands/sync-upstream.sh +++ b/scripts/commands/sync-upstream.sh @@ -815,18 +815,8 @@ assert_files_present "$IP_ARTIFACTS_DIR/mihomo" "$IP_ARTIFACTS_DIR/mihomo/*.mrs" assert_files_present "$IP_ARTIFACTS_DIR/egern" "$IP_ARTIFACTS_DIR/egern/*.yaml" inject_sync_failure late-compiler write_upstream_summary_json -python3 - <<'PYORIGINS' "$ARTIFACTS_DIR" "$ARTIFACTS_DIR/artifact-origins.json" -import json, sys -from pathlib import Path -root, target = map(Path, sys.argv[1:]) -origins = {} -for section in ("domain", "ip"): - base = root / section - if base.is_dir(): - for path in base.glob("*/*"): - if path.is_file(): - origins[path.relative_to(root).as_posix()] = "generated-upstream" -target.write_text(json.dumps(origins, indent=2, sort_keys=True) + "\n", encoding="utf-8") -PYORIGINS +python3 "$ROOT_DIR/scripts/tools/artifact_origins.py" reset \ + "$ARTIFACTS_DIR" \ + generated-upstream echo "=== SYNC DONE ===" diff --git a/scripts/commands/verify-artifact-manifest.sh b/scripts/commands/verify-artifact-manifest.sh old mode 100644 new mode 100755 diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh index 44342a88e..9b06c632e 100755 --- a/scripts/tests/run.sh +++ b/scripts/tests/run.sh @@ -2,7 +2,7 @@ set -uo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -cd "$ROOT" +cd "$ROOT" || exit TEST_DIR="${TEST_DIR:-$ROOT/scripts/tests}" TEST_FILTER="${TEST_FILTER:-}" diff --git a/scripts/tests/test-artifact-manifest.sh b/scripts/tests/test-artifact-manifest.sh index 7edbdd4e0..b3f6988a6 100644 --- a/scripts/tests/test-artifact-manifest.sh +++ b/scripts/tests/test-artifact-manifest.sh @@ -5,7 +5,12 @@ TMP="$(mktemp -d)" trap 'rm -rf "$TMP"' EXIT REPO="$TMP/repo" mkdir -p "$REPO/scripts/tools" "$REPO/scripts/commands" "$REPO/config" "$REPO/sources/custom/domain" "$REPO/.bin" -cp "$ROOT/scripts/tools/artifact_manifest.py" "$ROOT/scripts/tools/artifact_verifier.py" "$ROOT/scripts/tools/platform_capabilities.py" "$REPO/scripts/tools/" +cp \ + "$ROOT/scripts/tools/artifact_manifest.py" \ + "$ROOT/scripts/tools/artifact_origins.py" \ + "$ROOT/scripts/tools/artifact_verifier.py" \ + "$ROOT/scripts/tools/platform_capabilities.py" \ + "$REPO/scripts/tools/" cat > "$REPO/.bin/sing-box" <<'EOF' #!/usr/bin/env bash set -eu @@ -49,8 +54,24 @@ make_files() { rm -rf "$REPO/.output" while read -r platform extension; do mkdir -p "$REPO/.output/domain/$platform" "$REPO/.output/ip/$platform" - printf 'domain-%s\n' "$platform" > "$REPO/.output/domain/$platform/custom.$extension" - printf 'ip-%s\n' "$platform" > "$REPO/.output/ip/$platform/base.$extension" + case "$platform" in + surge) + printf 'DOMAIN-SUFFIX,custom.example\n' > "$REPO/.output/domain/$platform/custom.$extension" + printf 'IP-CIDR,192.0.2.0/24,no-resolve\n' > "$REPO/.output/ip/$platform/base.$extension" + ;; + quanx) + printf 'HOST-SUFFIX,custom.example,custom\n' > "$REPO/.output/domain/$platform/custom.$extension" + printf 'IP-CIDR,192.0.2.0/24,base\n' > "$REPO/.output/ip/$platform/base.$extension" + ;; + egern) + printf "domain_suffix_set:\n - 'custom.example'\n" > "$REPO/.output/domain/$platform/custom.$extension" + printf "no_resolve: true\nip_cidr_set:\n - '192.0.2.0/24'\n" > "$REPO/.output/ip/$platform/base.$extension" + ;; + sing-box|mihomo) + printf 'domain-%s\n' "$platform" > "$REPO/.output/domain/$platform/custom.$extension" + printf 'ip-%s\n' "$platform" > "$REPO/.output/ip/$platform/base.$extension" + ;; + esac done <<'EOF' surge list quanx list @@ -74,7 +95,7 @@ PY } generate() { - ARTIFACT_GENERATION_ID=offline-1 ARTIFACT_BUILD_ID=build-1 ARTIFACT_BUILD_SCOPE="${1:-custom}" ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null + ARTIFACT_GENERATION_ID=offline-1 ARTIFACT_BUILD_ID=build-1 ARTIFACT_BUILD_SCOPE=custom ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/generate-artifact-manifest.sh" >/dev/null } verify() { ARTIFACT_SOURCE_SHA="$SOURCE_SHA" "$REPO/scripts/commands/verify-artifact-manifest.sh" >/dev/null; } rejects() { @@ -89,6 +110,9 @@ rejects() { make_files generate verify +mkdir -p "$REPO/.output/domain" +printf '{}\n' > "$REPO/.output/domain/rule-manifest.json" +verify cp "$REPO/.output/artifact-manifest.json" "$TMP/first.json" generate cmp "$TMP/first.json" "$REPO/.output/artifact-manifest.json" diff --git a/scripts/tests/test-artifact-origins.sh b/scripts/tests/test-artifact-origins.sh new file mode 100644 index 000000000..e230a3b8b --- /dev/null +++ b/scripts/tests/test-artifact-origins.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +ARTIFACT_ROOT="$TMP_DIR/output" +DOMAIN_SOURCES="$TMP_DIR/sources/domain" +IP_SOURCES="$TMP_DIR/sources/ip" +TOOL="$ROOT/scripts/tools/artifact_origins.py" + +mkdir -p "$DOMAIN_SOURCES" "$IP_SOURCES" +printf 'DOMAIN-REGEX,^example$\n' > "$DOMAIN_SOURCES/sample.list" + +create_domain_artifacts() { + local platform extension + while read -r platform extension; do + mkdir -p "$ARTIFACT_ROOT/domain/$platform" + printf 'fixture\n' > "$ARTIFACT_ROOT/domain/$platform/sample.$extension" + printf 'upstream\n' > "$ARTIFACT_ROOT/domain/$platform/upstream.$extension" + done <<'EOF' +surge list +quanx list +egern yaml +sing-box srs +mihomo mrs +EOF +} + +assert_origin_state() { + local expression="$1" + python3 - "$ARTIFACT_ROOT/artifact-origins.json" "$expression" <<'PY' +import json +import sys + +origins = json.load(open(sys.argv[1], encoding="utf-8")) +if not eval(sys.argv[2], {"origins": origins}): + raise SystemExit(f"origin assertion failed: {sys.argv[2]}\n{origins}") +PY +} + +create_domain_artifacts +python3 "$TOOL" reset "$ARTIFACT_ROOT" restored-published-branch +rm \ + "$ARTIFACT_ROOT/domain/surge/sample.list" \ + "$ARTIFACT_ROOT/domain/quanx/sample.list" \ + "$ARTIFACT_ROOT/domain/mihomo/sample.mrs" +python3 "$TOOL" mark-custom "$ARTIFACT_ROOT" "$DOMAIN_SOURCES" "$IP_SOURCES" + +assert_origin_state \ + 'origins["domain/egern/sample.yaml"] == "generated-custom" and origins["domain/sing-box/sample.srs"] == "generated-custom"' +assert_origin_state \ + 'all(f"domain/{platform}/sample.{extension}" not in origins for platform, extension in (("surge", "list"), ("quanx", "list"), ("mihomo", "mrs")))' +assert_origin_state \ + 'all(value == "restored-published-branch" for key, value in origins.items() if "/upstream." in key)' + +actual_custom="$(python3 "$TOOL" list "$ARTIFACT_ROOT" --origin generated-custom)" +expected_custom="domain/egern/sample.yaml +domain/sing-box/sample.srs" +[ "$actual_custom" = "$expected_custom" ] || { + echo "test failed: custom origin listing mismatch" >&2 + printf 'expected:\n%s\nactual:\n%s\n' "$expected_custom" "$actual_custom" >&2 + exit 1 +} + +create_domain_artifacts +python3 "$TOOL" reset "$ARTIFACT_ROOT" restored-published-branch +rm "$ARTIFACT_ROOT/domain/surge/sample.list" +python3 "$TOOL" mark-custom \ + "$ARTIFACT_ROOT" "$DOMAIN_SOURCES" "$IP_SOURCES" --text-only + +assert_origin_state \ + '"domain/surge/sample.list" not in origins and origins["domain/quanx/sample.list"] == "generated-custom" and origins["domain/egern/sample.yaml"] == "generated-custom"' +assert_origin_state \ + 'origins["domain/sing-box/sample.srs"] == "restored-published-branch" and origins["domain/mihomo/sample.mrs"] == "restored-published-branch"' + +printf '[]\n' > "$ARTIFACT_ROOT/artifact-origins.json" +if python3 "$TOOL" list "$ARTIFACT_ROOT" >"$TMP_DIR/invalid.stdout" 2>"$TMP_DIR/invalid.stderr"; then + echo "test failed: invalid origin map was accepted" >&2 + exit 1 +fi +grep -F 'invalid artifact origin map' "$TMP_DIR/invalid.stderr" >/dev/null + +echo "artifact origin lifecycle tests passed" diff --git a/scripts/tests/test-artifact-summary.sh b/scripts/tests/test-artifact-summary.sh index 2bb72c48a..1d1c7c37c 100755 --- a/scripts/tests/test-artifact-summary.sh +++ b/scripts/tests/test-artifact-summary.sh @@ -49,9 +49,9 @@ summary = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) assert summary["domain"]["surge"]["files"] == 1 assert summary["domain"]["surge"]["rules"] == 3 assert summary["domain"]["surge"]["by_kind"]["DOMAIN-KEYWORD"] == 1 -assert summary["domain"]["quanx"]["rules"] == 4 +assert summary["domain"]["quanx"]["rules"] == 3 assert summary["domain"]["quanx"]["by_kind"]["HOST-KEYWORD"] == 1 -assert summary["domain"]["quanx"]["by_kind"]["HOST-REGEX"] == 1 +assert "HOST-REGEX" not in summary["domain"]["quanx"]["by_kind"] assert summary["domain"]["egern"]["rules"] == 2 assert summary["domain"]["egern"]["by_kind"]["YAML-ENTRY"] == 2 assert summary["ip"]["surge"]["rules"] == 2 diff --git a/scripts/tests/test-binary-artifact-verifier.sh b/scripts/tests/test-binary-artifact-verifier.sh index adc1a29c9..dc0277010 100644 --- a/scripts/tests/test-binary-artifact-verifier.sh +++ b/scripts/tests/test-binary-artifact-verifier.sh @@ -23,6 +23,16 @@ if PYTHONPATH="$ROOT/scripts/tools" python3 "$ROOT/scripts/tools/artifact_verifi echo "Egern verifier accepted unknown YAML field" >&2; exit 1 fi +PYTHONPATH="$ROOT/scripts/tools" python3 - <<'PY' +from collections import Counter +from artifact_verifier import singbox_counts + +domain = {"rules": [{"domain": "one.example", "domain_suffix": ["a.example", "b.example"], "domain_keyword": "emby"}]} +assert singbox_counts(domain, "domain") == Counter({"DOMAIN-SUFFIX": 2, "DOMAIN": 1, "DOMAIN-KEYWORD": 1}) +ip = {"rules": [{"ip_cidr": "192.0.2.0/24"}, {"ip_cidr": ["2001:db8::/32"]}]} +assert singbox_counts(ip, "ip") == Counter({"IP-CIDR": 1, "IP-CIDR6": 1}) +PY + if ! command -v sing-box >/dev/null 2>&1 || ! command -v mihomo >/dev/null 2>&1; then echo "binary verifier real-tool fixture skipped: sing-box and mihomo are not both available" exit 0 diff --git a/scripts/tests/test-build-scope.sh b/scripts/tests/test-build-scope.sh index b62f9f26b..c75464c57 100755 --- a/scripts/tests/test-build-scope.sh +++ b/scripts/tests/test-build-scope.sh @@ -30,6 +30,11 @@ assert_scope custom "manual custom-only publish" \ EVENT_NAME=workflow_dispatch INPUT_SCOPE=custom CURRENT_SHA=HEAD \ CHANGED_FILES=$'sources/custom/domain/emby.list' +assert_scope full "custom deletions require full sync" \ + EVENT_NAME=workflow_dispatch INPUT_SCOPE=custom CURRENT_SHA=HEAD \ + CHANGED_FILES=$'sources/custom/domain/old.list' \ + DELETED_CUSTOM_FILES=$'sources/custom/domain/old.list' + assert_scope custom "push only updates custom sources" \ EVENT_NAME=push CURRENT_SHA=HEAD BEFORE_SHA=base \ CHANGED_FILES=$'sources/custom/domain/emby.list\nsources/custom/ip/example.list' diff --git a/scripts/tests/test-custom-build-atomic.sh b/scripts/tests/test-custom-build-atomic.sh index 6d0133361..3694a66e9 100644 --- a/scripts/tests/test-custom-build-atomic.sh +++ b/scripts/tests/test-custom-build-atomic.sh @@ -17,6 +17,7 @@ git -C "$REPO" config user.name test git -C "$REPO" config user.email test@example.com git -C "$REPO" add scripts config sources git -C "$REPO" commit -m base >/dev/null +BASE_SHA="$(git -C "$REPO" rev-parse HEAD)" snapshot_output() { local destination="$1" @@ -40,13 +41,17 @@ assert_injected_failure_preserves_output() { local after="$TMP_DIR/$point.after" snapshot_output "$before" - if env RULES_BUILD_CUSTOM_FAIL_AT="$point" "$@" \ + if env RULES_CONFLICT_BASE_SHA="$BASE_SHA" RULES_BUILD_CUSTOM_FAIL_AT="$point" "$@" \ "$REPO/scripts/commands/build-custom.sh" \ >"$TMP_DIR/$point.stdout" 2>"$TMP_DIR/$point.stderr"; then echo "test failed: expected injected $point failure" >&2 exit 1 fi - grep -Fx "injected custom build failure at $point" "$TMP_DIR/$point.stderr" >/dev/null + grep -Fx "injected custom build failure at $point" "$TMP_DIR/$point.stderr" >/dev/null || { + echo "test failed: missing injected failure marker for $point" >&2 + cat "$TMP_DIR/$point.stderr" >&2 + exit 1 + } snapshot_output "$after" if ! cmp -s "$before" "$after"; then echo "test failed: $point failure changed .output" >&2 @@ -93,7 +98,8 @@ assert_injected_failure_preserves_output late-binary # A successful text-only commit updates only controlled text targets. Binary # targets and unrelated restored/upstream files remain untouched. printf 'restored binary\n' > "$REPO/.output/domain/surge/unrelated.list" -RULES_BUILD_CUSTOM_TEXT_ONLY=1 "$REPO/scripts/commands/build-custom.sh" >/dev/null +RULES_CONFLICT_BASE_SHA="$BASE_SHA" RULES_BUILD_CUSTOM_TEXT_ONLY=1 \ + "$REPO/scripts/commands/build-custom.sh" >/dev/null grep -Fx 'DOMAIN-SUFFIX,atomic-stage.example' "$REPO/.output/domain/surge/emby.list" >/dev/null grep -Fx 'restored upstream' "$REPO/.output/unrelated/upstream.txt" >/dev/null grep -Fx 'restored binary' "$REPO/.output/domain/surge/unrelated.list" >/dev/null diff --git a/scripts/tests/test-domain-entrypoint-guard.sh b/scripts/tests/test-domain-entrypoint-guard.sh index 1c7c4f179..b9c4b4f62 100755 --- a/scripts/tests/test-domain-entrypoint-guard.sh +++ b/scripts/tests/test-domain-entrypoint-guard.sh @@ -70,7 +70,10 @@ assert_wrapper_calls_cli() { } RULES_TRACE_DOMAIN_CLI_FILE="$TRACE_TEST" ./scripts/tests/test-domain-parsing.sh >/dev/null -RULES_TRACE_DOMAIN_CLI_FILE="$TRACE_BUILD" RULES_BUILD_CUSTOM_TEXT_ONLY=1 ./scripts/commands/build-custom.sh >/dev/null +RULES_TRACE_DOMAIN_CLI_FILE="$TRACE_BUILD" \ + RULES_ARTIFACT_ROOT="$TMP_DIR/output" \ + RULES_BUILD_CUSTOM_TEXT_ONLY=1 \ + ./scripts/commands/build-custom.sh >/dev/null for command in "${required_text_commands[@]}"; do assert_trace_contains "$TRACE_TEST" "$command" "test-domain-parsing" @@ -93,8 +96,10 @@ assert_wrapper_calls_cli "render_quanx_domain_ruleset_from_rules" "quanx-list" assert_wrapper_calls_cli "render_egern_domain_ruleset_from_rules" "egern-yaml" assert_wrapper_calls_cli "render_domain_rule_dir_to_text_platform_dirs" "text-platform-dirs" -lint_line="$(grep -nF '"$ROOT/scripts/commands/lint-custom-rules.sh"' "$ROOT/scripts/commands/build-custom.sh" | cut -d: -f1)" -tmp_line="$(grep -nF 'mkdir -p "$TMP_PARENT_DIR"' "$ROOT/scripts/commands/build-custom.sh" | cut -d: -f1)" +lint_invocation="\"\$ROOT/scripts/commands/lint-custom-rules.sh\"" +tmp_setup="mkdir -p \"\$TMP_PARENT_DIR\"" +lint_line="$(grep -nF "$lint_invocation" "$ROOT/scripts/commands/build-custom.sh" | cut -d: -f1)" +tmp_line="$(grep -nF "$tmp_setup" "$ROOT/scripts/commands/build-custom.sh" | cut -d: -f1)" setup_line="$(grep -nF 'setup_tool_cache' "$ROOT/scripts/commands/build-custom.sh" | cut -d: -f1)" if [ -z "$lint_line" ] || [ -z "$tmp_line" ] || [ -z "$setup_line" ] \ || [ "$lint_line" -ge "$tmp_line" ] || [ "$lint_line" -ge "$setup_line" ]; then diff --git a/scripts/tests/test-domain-parsing.sh b/scripts/tests/test-domain-parsing.sh index 1d312b3f4..aea96fdc2 100755 --- a/scripts/tests/test-domain-parsing.sh +++ b/scripts/tests/test-domain-parsing.sh @@ -159,6 +159,34 @@ EOF fi } +test_upstream_single_label_suffix() { + mkdir -p "$TMP_DIR/tld_suffix/data" "$TMP_DIR/tld_suffix/out" + printf '%s\n' 'alibaba' > "$TMP_DIR/tld_suffix/data/brand-tld" + + python3 "$ROOT/scripts/tools/export-domain-rules.py" export \ + "$TMP_DIR/tld_suffix/data" \ + "$TMP_DIR/tld_suffix/out" + python3 "$ROOT/scripts/tools/export-domain-rules.py" surge-list \ + "$TMP_DIR/tld_suffix/out/brand-tld.list" \ + "$TMP_DIR/tld_suffix/brand-tld.surge.list" + + grep -Fx 'DOMAIN-SUFFIX,alibaba' "$TMP_DIR/tld_suffix/brand-tld.surge.list" >/dev/null || { + echo "test failed: upstream brand TLD suffix was not rendered" >&2 + exit 1 + } + + python3 - "$ROOT" "$TMP_DIR/tld_suffix/out/brand-tld.list" <<'PY' +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(sys.argv[1]) / "scripts" / "tools")) +from domain_rules import parse_classical_domain_file + +_, errors = parse_classical_domain_file(Path(sys.argv[2])) +assert errors and "too broad" in errors[0], errors +PY +} + test_export_plain_yaml_artifact() { mkdir -p "$TMP_DIR/export_plain_yaml/out" cat > "$TMP_DIR/export_plain_yaml/dlc.dat_plain.yml" <<'EOF' @@ -627,7 +655,7 @@ EOF >"$TMP_DIR/capability_summary/stdout" \ 2>"$TMP_DIR/capability_summary/stderr" - grep -Fx "domain summary: base skips unsupported rules for mihomo-mrs: DOMAIN-KEYWORD=1, DOMAIN-REGEX=1" \ + grep -Fx "domain summary: base skips unsupported rules for mihomo: DOMAIN-KEYWORD=1, DOMAIN-REGEX=1" \ "$TMP_DIR/capability_summary/stderr" >/dev/null || { echo "test failed: missing mihomo-mrs capability summary" >&2 cat "$TMP_DIR/capability_summary/stderr" >&2 @@ -736,6 +764,7 @@ EOF test_compile_jobs_override_validation test_export_alias_prefixes test_export_unknown_prefix_fails +test_upstream_single_label_suffix test_export_plain_yaml_artifact test_classical_domain_fixture_outputs test_batch_domain_dir_outputs diff --git a/scripts/tests/test-fakeip-migration.sh b/scripts/tests/test-fakeip-migration.sh index 162f51de1..c68363588 100644 --- a/scripts/tests/test-fakeip-migration.sh +++ b/scripts/tests/test-fakeip-migration.sh @@ -22,21 +22,28 @@ fi grep -F 'KuGouGo-maintained' "$ROOT/sources/custom/domain/fakeip-filter.list" >/dev/null RENDER_REPO="$TMP_DIR/render" +FAKEIP_SOURCE="$TMP_DIR/fakeip-filter.list" +cp "$ROOT/sources/custom/domain/fakeip-filter.list" "$FAKEIP_SOURCE" mkdir -p "$RENDER_REPO" cp -R "$ROOT/scripts" "$ROOT/config" "$ROOT/sources" "$RENDER_REPO/" +rm "$RENDER_REPO/sources/custom/domain/fakeip-filter.list" git -C "$RENDER_REPO" init -q git -C "$RENDER_REPO" config user.name test git -C "$RENDER_REPO" config user.email test@example.com git -C "$RENDER_REPO" add scripts config sources git -C "$RENDER_REPO" commit -m base >/dev/null +cp "$FAKEIP_SOURCE" "$RENDER_REPO/sources/custom/domain/fakeip-filter.list" +git -C "$RENDER_REPO" add sources/custom/domain/fakeip-filter.list +git -C "$RENDER_REPO" commit -m add-fakeip >/dev/null +RULES_BUILD_CUSTOM_TEXT_ONLY=1 "$RENDER_REPO/scripts/commands/build-custom.sh" >/dev/null RULES_BUILD_CUSTOM_TEXT_ONLY=1 "$RENDER_REPO/scripts/commands/build-custom.sh" >/dev/null grep -Fx 'DOMAIN-SUFFIX,in-addr.arpa' "$RENDER_REPO/.output/domain/surge/fakeip-filter.list" >/dev/null grep -Fx 'DOMAIN,localhost.localdomain' "$RENDER_REPO/.output/domain/surge/fakeip-filter.list" >/dev/null grep -Fx 'HOST-SUFFIX,in-addr.arpa,fakeip-filter' "$RENDER_REPO/.output/domain/quanx/fakeip-filter.list" >/dev/null grep -Fx 'HOST,localhost.localdomain,fakeip-filter' "$RENDER_REPO/.output/domain/quanx/fakeip-filter.list" >/dev/null -grep -Fx ' - in-addr.arpa' "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null -grep -Fx ' - localhost.localdomain' "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null +grep -Fx " - 'in-addr.arpa'" "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null +grep -Fx " - 'localhost.localdomain'" "$RENDER_REPO/.output/domain/egern/fakeip-filter.yaml" >/dev/null prepare_collision_repo() { local repo="$1" diff --git a/scripts/tests/test-guard-artifacts.sh b/scripts/tests/test-guard-artifacts.sh index ab0c97d33..462e426e0 100755 --- a/scripts/tests/test-guard-artifacts.sh +++ b/scripts/tests/test-guard-artifacts.sh @@ -158,7 +158,7 @@ cat > "$TMP_DIR/plain-summary/.output/upstream-summary.json" <<'JSON' ] JSON -if ! ( cd "$TMP_DIR/plain-summary" && uses_dlc_plain_yaml_artifact ); then +if ! ( ARTIFACT_ROOT="$TMP_DIR/plain-summary/.output"; uses_dlc_plain_yaml_artifact ); then echo "test failed: DLC plain YAML summary should be detected" >&2 exit 1 fi @@ -174,7 +174,7 @@ cat > "$TMP_DIR/git-summary/.output/upstream-summary.json" <<'JSON' ] JSON -if ( cd "$TMP_DIR/git-summary" && uses_dlc_plain_yaml_artifact ); then +if ( ARTIFACT_ROOT="$TMP_DIR/git-summary/.output"; uses_dlc_plain_yaml_artifact ); then echo "test failed: git DLC summary should not be treated as plain YAML" >&2 exit 1 fi diff --git a/scripts/tests/test-publish-branches.sh b/scripts/tests/test-publish-branches.sh index 47550c833..4e1970d67 100755 --- a/scripts/tests/test-publish-branches.sh +++ b/scripts/tests/test-publish-branches.sh @@ -13,6 +13,7 @@ SEED="$TMP_DIR/seed" REAL_GIT="$(command -v git)" BRANCHES=(surge quanx egern sing-box mihomo) TEMPLATE_DIR="$ROOT/templates/branch-readmes" +published_tree_description="仅保留 \`README.md\`、\`domain/\` 和 \`ip/\`" grep -Fq -- "- 'templates/**'" "$ROOT/.github/workflows/build.yml" || { echo "test failed: build workflow push.paths must include templates/**" >&2 @@ -26,7 +27,7 @@ for branch in "${BRANCHES[@]}"; do exit 1 } grep -F '滚动构建产物' "$template" >/dev/null - grep -F '仅保留 `README.md`、`domain/` 和 `ip/`' "$template" >/dev/null + grep -F "$published_tree_description" "$template" >/dev/null grep -F '## 产物格式与能力降级' "$template" >/dev/null grep -F '## 最小示例' "$template" >/dev/null case "$branch" in @@ -65,6 +66,7 @@ git --git-dir="$REMOTE" symbolic-ref HEAD refs/heads/main mkdir -p "$REPO" cp -R scripts templates config "$REPO/" +mkdir -p "$REPO/.bin" mkdir -p \ "$REPO/.output/domain/surge" "$REPO/.output/ip/surge" \ "$REPO/.output/domain/quanx" "$REPO/.output/ip/quanx" \ @@ -76,13 +78,78 @@ printf 'DOMAIN-SUFFIX,example.com\n' > "$REPO/.output/domain/surge/test.list" printf 'IP-CIDR,192.0.2.0/24,no-resolve\n' > "$REPO/.output/ip/surge/test.list" printf 'HOST-SUFFIX,example.com,direct\n' > "$REPO/.output/domain/quanx/test.list" printf 'IP-CIDR,192.0.2.0/24,direct\n' > "$REPO/.output/ip/quanx/test.list" -printf 'domain_suffix_set:\n - example.com\n' > "$REPO/.output/domain/egern/test.yaml" -printf 'ip_cidr_set:\n - 192.0.2.0/24\n' > "$REPO/.output/ip/egern/test.yaml" +printf "domain_suffix_set:\n - 'example.com'\n" > "$REPO/.output/domain/egern/test.yaml" +printf "no_resolve: true\nip_cidr_set:\n - '192.0.2.0/24'\n" > "$REPO/.output/ip/egern/test.yaml" printf 'srs-domain\n' > "$REPO/.output/domain/sing-box/test.srs" printf 'srs-ip\n' > "$REPO/.output/ip/sing-box/test.srs" printf 'mrs-domain\n' > "$REPO/.output/domain/mihomo/test.mrs" printf 'mrs-ip\n' > "$REPO/.output/ip/mihomo/test.mrs" +cat > "$REPO/.bin/sing-box" <<'EOF' +#!/usr/bin/env bash +set -eu +input="$3" +output="$5" +if [[ "$input" == */ip/* ]]; then + printf '{"version":4,"rules":[{"ip_cidr":["192.0.2.0/24"]}]}\n' > "$output" +else + printf '{"version":4,"rules":[{"domain_suffix":["example.com"]}]}\n' > "$output" +fi +EOF +cat > "$REPO/.bin/mihomo" <<'EOF' +#!/usr/bin/env bash +set -eu +input="$4" +output="$5" +if [[ "$input" == */ip/* ]]; then + printf '192.0.2.0/24\n' > "$output" +else + printf '+.example.com\n' > "$output" +fi +EOF +chmod +x "$REPO/.bin/sing-box" "$REPO/.bin/mihomo" + +python3 - "$REPO" <<'PY' +import hashlib +import json +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +lock_path = root / "config/tools-lock.json" +lock = json.loads(lock_path.read_text(encoding="utf-8")) +for tool in ("sing-box", "mihomo"): + binary = root / ".bin" / tool + digest = hashlib.sha256(binary.read_bytes()).hexdigest() + item = lock["tools"][tool] + platform = "linux-amd64" + item["platforms"][platform]["binary_sha256"] = digest + sidecar = { + "schema_version": 1, + "tool": tool, + "version": item["version"], + "tag_commit": item["tag_commit"], + "platform": platform, + "asset": item["platforms"][platform]["asset"], + "archive_sha256": item["platforms"][platform]["sha256"], + "binary_sha256": digest, + "version_probe": "fixture", + } + (root / ".bin" / f"{tool}.provenance.json").write_text( + json.dumps(sidecar, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) +lock_path.write_text(json.dumps(lock, indent=2, sort_keys=True) + "\n", encoding="utf-8") + +origins = {} +for path in (root / ".output").glob("*/*/*"): + if path.is_file(): + origins[path.relative_to(root / ".output").as_posix()] = "generated-upstream" +(root / ".output/artifact-origins.json").write_text( + json.dumps(origins, indent=2, sort_keys=True) + "\n", encoding="utf-8" +) +(root / ".output/build-summary.json").write_text("{}\n", encoding="utf-8") +PY + git -C "$REPO" init -q git -C "$REPO" remote add origin "$REMOTE" git -C "$REPO" fetch origin '+refs/heads/*:refs/remotes/origin/*' >/dev/null 2>&1 @@ -159,12 +226,27 @@ after_missing_template="$(for branch in "${BRANCHES[@]}"; do git --git-dir="$REM # update must reject the complete batch, leaving every other branch untouched. for branch in "${BRANCHES[@]}"; do case "$branch" in - surge|quanx) extension=list ;; - egern) extension=yaml ;; - sing-box) extension=srs ;; - mihomo) extension=mrs ;; + surge) + extension=list + printf 'DOMAIN-SUFFIX,updated.example\n' > "$REPO/.output/domain/$branch/test.$extension" + ;; + quanx) + extension=list + printf 'HOST-SUFFIX,updated.example,direct\n' > "$REPO/.output/domain/$branch/test.$extension" + ;; + egern) + extension=yaml + printf "domain_suffix_set:\n - 'updated.example'\n" > "$REPO/.output/domain/$branch/test.$extension" + ;; + sing-box) + extension=srs + printf 'updated-%s\n' "$branch" > "$REPO/.output/domain/$branch/test.$extension" + ;; + mihomo) + extension=mrs + printf 'updated-%s\n' "$branch" > "$REPO/.output/domain/$branch/test.$extension" + ;; esac - printf 'updated-%s\n' "$branch" > "$REPO/.output/domain/$branch/test.$extension" done set +e diff --git a/scripts/tests/test-runtime.sh b/scripts/tests/test-runtime.sh new file mode 100644 index 000000000..a3467544f --- /dev/null +++ b/scripts/tests/test-runtime.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +make -s -C "$ROOT" check-runtime BASH_MIN_MAJOR=1 + +if make -s -C "$ROOT" check-runtime BASH_MIN_MAJOR=999 \ + >"$TMP_DIR/unsupported.stdout" 2>"$TMP_DIR/unsupported.stderr"; then + echo "test failed: unsupported Bash runtime was accepted" >&2 + exit 1 +fi +grep -F 'Bash 999+ is required' "$TMP_DIR/unsupported.stderr" >/dev/null + +echo "runtime requirement tests passed" diff --git a/scripts/tests/test-sync-upstream-render-matrix.sh b/scripts/tests/test-sync-upstream-render-matrix.sh index 4b4820341..51b99ffde 100755 --- a/scripts/tests/test-sync-upstream-render-matrix.sh +++ b/scripts/tests/test-sync-upstream-render-matrix.sh @@ -59,13 +59,13 @@ loyalsoldier_required_snippets = [ 'text "$tmp_dir/private.raw.txt" "$tmp_dir/private.cidr.txt"', 'record_upstream_summary ip loyalsoldier-geoip-cn ok "$LOYALSOLDIER_GEOIP_CN_SOURCE_URL"', 'record_upstream_summary ip loyalsoldier-geoip-private ok "$LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL"', - 'assert_min_cidrs loyalsoldier-geoip-cn "$IP_BUILD_TMP_DIR/loyalsoldier_geoip_cn.cidr.txt"', - 'assert_min_cidrs loyalsoldier-geoip-private "$IP_BUILD_TMP_DIR/private.cidr.txt"', + '"loyalsoldier-geoip-cn loyalsoldier_geoip_cn.raw.txt loyalsoldier_geoip_cn.cidr.txt"', + '"loyalsoldier-geoip-private private.raw.txt private.cidr.txt"', '"$IP_BUILD_TMP_DIR/loyalsoldier_geoip_cn.cidr.txt"', ] for snippet in loyalsoldier_required_snippets: if snippet not in script: - raise SystemExit(f"test failed: sync-upstream missing Loyalsoldier CN IP snippet {snippet!r}") + raise SystemExit(f"test failed: sync-upstream missing Loyalsoldier IP snippet {snippet!r}") domain_required_snippets = [ 'clone_repository_shallow "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-community"', diff --git a/scripts/tests/test-tool-lock.sh b/scripts/tests/test-tool-lock.sh index 605e7983a..e649d2926 100644 --- a/scripts/tests/test-tool-lock.sh +++ b/scripts/tests/test-tool-lock.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# Calls use the sourced implementation until the deliberate final failure override. +# shellcheck disable=SC2218 set -euo pipefail ROOT="$(cd "$(dirname "$0")/../.." && pwd)" @@ -15,8 +17,8 @@ fail() { exit 1 } -SING_BOX_VERSION=latest -MIHOMO_VERSION=latest +export SING_BOX_VERSION=latest +export MIHOMO_VERSION=latest [ "$(resolve_sing_box_version)" = "1.13.14" ] || fail "sing-box version must come only from lock" [ "$(resolve_mihomo_version)" = "1.19.28" ] || fail "mihomo version must come only from lock" [ "$(tool_lock_value sing-box tag_commit)" = "25a600db24f7680ad9806ce5427bd0ab8afe1114" ] || fail "sing-box tag commit is not locked" diff --git a/scripts/tests/test-upstream-config.sh b/scripts/tests/test-upstream-config.sh index 2400f5fc2..5d8377529 100755 --- a/scripts/tests/test-upstream-config.sh +++ b/scripts/tests/test-upstream-config.sh @@ -75,7 +75,7 @@ path.write_text(json.dumps(data), encoding="utf-8") PY assert_lint_fails_with \ "invalid-parser" \ - "upstreams.ip.github.parser: unsupported parser" \ + "upstreams.ip.github.parser: unsupported or missing parser 'unknown-json'" \ --upstreams "$TMP_DIR/upstreams.invalid-parser.json" cp config/upstream-first-batch-baselines.json "$TMP_DIR/baselines.invalid.json" diff --git a/scripts/tools/artifact_manifest.py b/scripts/tools/artifact_manifest.py index 30106e1b8..927143d0c 100644 --- a/scripts/tools/artifact_manifest.py +++ b/scripts/tools/artifact_manifest.py @@ -12,13 +12,13 @@ from pathlib import Path, PurePosixPath from typing import Any +from artifact_origins import ALLOWED_ORIGINS, load_origin_file from artifact_verifier import verify_one from platform_capabilities import load_platform_capabilities SCHEMA_VERSION = 2 SHA256_RE = re.compile(r"^[0-9a-f]{64}$") GIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") -ORIGINS = {"generated-custom", "generated-upstream", "restored-published-branch"} TOP_KEYS = {"schema_version", "generation_id", "build_id", "build_scope", "source", "inputs", "tools", "summaries", "artifacts", "restoration"} SOURCE_KEYS = {"commit", "tree"} INPUT_KEYS = {"capabilities", "tool_lock", "artifact_origins"} @@ -29,6 +29,7 @@ RESTORATION_KEYS = {"generation_id", "source_commit", "branches"} BRANCH_KEYS = {"commit"} PUBLISH_BRANCHES = {"surge", "quanx", "egern", "sing-box", "mihomo"} +INTERNAL_ARTIFACT_FILES = {"domain/rule-manifest.json"} def digest(path: Path) -> str: @@ -72,13 +73,6 @@ def require_exact(value: Any, keys: set[str], location: str, errors: list[str]) return True -def load_origins(path: Path) -> dict[str, str]: - data = load_json(path) - if not isinstance(data, dict) or not data or any(not isinstance(key, str) or value not in ORIGINS for key, value in data.items()): - raise SystemExit("artifact origin provenance map must be a non-empty path-to-origin object") - return data - - def collect_artifacts(root: Path, origins: dict[str, str]) -> list[dict[str, Any]]: output, matrix = artifact_output(root), capability_matrix(root) artifacts: list[dict[str, Any]] = [] @@ -179,7 +173,7 @@ def generate(args: argparse.Namespace) -> None: output.mkdir(parents=True, exist_ok=True) capabilities_path, lock_path = root / "config/domain-platform-capabilities.json", root / "config/tools-lock.json" origins_path = output / "artifact-origins.json" - lock, origins = load_json(lock_path), load_origins(origins_path) + lock, origins = load_json(lock_path), load_origin_file(origins_path, require_nonempty=True) source_commit = args.source_sha or git_value(root, "rev-parse", "HEAD") source_tree = git_value(root, "rev-parse", f"{source_commit}^{{tree}}") if source_commit else None if args.source_sha and (not GIT_SHA_RE.fullmatch(args.source_sha) or source_tree is None): @@ -232,7 +226,7 @@ def verify(args: argparse.Namespace) -> None: if actual_tree and source["tree"] != actual_tree: errors.append(f"source tree mismatch: expected {actual_tree}, got {source['tree']}") capabilities_path, lock_path, origins_path = root / "config/domain-platform-capabilities.json", root / "config/tools-lock.json", output / "artifact-origins.json" try: - matrix, lock, origins = capability_matrix(root), load_json(lock_path), load_origins(origins_path) + matrix, lock, origins = capability_matrix(root), load_json(lock_path), load_origin_file(origins_path, require_nonempty=True) except (OSError, ValueError, json.JSONDecodeError, SystemExit) as exc: errors.append(f"manifest verification input invalid: {exc}"); matrix, lock, origins = {}, {}, {} inputs = manifest.get("inputs") @@ -278,7 +272,7 @@ def verify(args: argparse.Namespace) -> None: recorded.add(rel); artifact_type, platform, filename = PurePosixPath(rel).parts; expected = matrix.get((artifact_type, platform)); extension = filename.rsplit(".", 1)[-1] if "." in filename else "" if not expected: errors.append(f"artifact capability missing for {rel}") elif entry["type"] != artifact_type or entry["platform"] != platform or entry["extension"] != extension or extension != expected["extension"]: errors.append(f"artifact metadata disagrees with capabilities: {rel}") - if entry["origin"] not in ORIGINS or origins.get(rel) != entry["origin"]: errors.append(f"artifact origin provenance mismatch for {rel}") + if entry["origin"] not in ALLOWED_ORIGINS or origins.get(rel) != entry["origin"]: errors.append(f"artifact origin provenance mismatch for {rel}") path = output / Path(*PurePosixPath(rel).parts) if not path.is_file(): errors.append(f"manifest artifact missing: {rel}"); continue size = path.stat().st_size @@ -297,6 +291,7 @@ def verify(args: argparse.Namespace) -> None: for path in base.rglob("*"): if not path.is_file(): continue rel, parts = path.relative_to(output).as_posix(), PurePosixPath(path.relative_to(output).as_posix()).parts + if rel in INTERNAL_ARTIFACT_FILES: continue if len(parts) != 3: errors.append(f"unexpected nested publishable file: {rel}"); continue expected = matrix.get((parts[0], parts[1])) if not expected or path.suffix != "." + expected["extension"]: errors.append(f"unexpected publishable file: {rel}"); continue diff --git a/scripts/tools/artifact_origins.py b/scripts/tools/artifact_origins.py new file mode 100644 index 000000000..0581f047c --- /dev/null +++ b/scripts/tools/artifact_origins.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import tempfile +from pathlib import Path, PurePosixPath + +from platform_capabilities import load_platform_capabilities + +ALLOWED_ORIGINS = {"generated-custom", "generated-upstream", "restored-published-branch"} + + +def origins_path(artifact_root: Path) -> Path: + return artifact_root / "artifact-origins.json" + + +def safe_origin_path(value: str) -> bool: + path = PurePosixPath(value) + return ( + value == path.as_posix() + and not path.is_absolute() + and len(path.parts) == 3 + and path.parts[0] in {"domain", "ip"} + and all(part not in {"", ".", ".."} for part in path.parts) + and "\\" not in value + ) + + +def load_origin_file(path: Path, *, require_nonempty: bool = False) -> dict[str, str]: + if not path.is_file() and not require_nonempty: + return {} + data = json.loads(path.read_text(encoding="utf-8")) + if ( + not isinstance(data, dict) + or (require_nonempty and not data) + or any( + not isinstance(key, str) + or not safe_origin_path(key) + or value not in ALLOWED_ORIGINS + for key, value in data.items() + ) + ): + raise SystemExit(f"invalid artifact origin map: {path}") + return data + + +def load_origins(artifact_root: Path) -> dict[str, str]: + return load_origin_file(origins_path(artifact_root)) + + +def write_origins(artifact_root: Path, origins: dict[str, str]) -> None: + artifact_root.mkdir(parents=True, exist_ok=True) + target = origins_path(artifact_root) + payload = json.dumps(origins, ensure_ascii=False, indent=2, sort_keys=True) + "\n" + temporary: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=artifact_root, prefix=".artifact-origins.", delete=False + ) as handle: + temporary = Path(handle.name) + handle.write(payload) + os.replace(temporary, target) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + + +def publishable_files(artifact_root: Path): + for section in ("domain", "ip"): + base = artifact_root / section + if not base.is_dir(): + continue + for path in sorted(base.glob("*/*")): + if path.is_file(): + yield path + + +def reset_origins(args: argparse.Namespace) -> None: + artifact_root = args.artifact_root.resolve() + origins = { + path.relative_to(artifact_root).as_posix(): args.origin + for path in publishable_files(artifact_root) + } + write_origins(artifact_root, origins) + + +def list_origins(args: argparse.Namespace) -> None: + for path, origin in sorted(load_origins(args.artifact_root.resolve()).items()): + if args.origin is None or origin == args.origin: + print(path) + + +def mark_custom(args: argparse.Namespace) -> None: + artifact_root = args.artifact_root.resolve() + origins = load_origins(artifact_root) + capabilities = load_platform_capabilities() + source_roots = { + "domain": args.domain_sources.resolve(), + "ip": args.ip_sources.resolve(), + } + + for section, source_root in source_roots.items(): + if not source_root.is_dir(): + continue + for source in sorted(source_root.glob("*.list")): + for platform, _details, capability_section, capability in capabilities.iter_capabilities(section): + if capability_section != section or (args.text_only and capability.format == "binary"): + continue + relative = f"{section}/{platform}/{source.stem}.{capability.extension}" + origins.pop(relative, None) + if (artifact_root / relative).is_file(): + origins[relative] = "generated-custom" + + write_origins(artifact_root, origins) + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + subparsers = result.add_subparsers(dest="command", required=True) + + reset = subparsers.add_parser("reset") + reset.add_argument("artifact_root", type=Path) + reset.add_argument("origin", choices=sorted(ALLOWED_ORIGINS)) + reset.set_defaults(func=reset_origins) + + listing = subparsers.add_parser("list") + listing.add_argument("artifact_root", type=Path) + listing.add_argument("--origin", choices=sorted(ALLOWED_ORIGINS)) + listing.set_defaults(func=list_origins) + + custom = subparsers.add_parser("mark-custom") + custom.add_argument("artifact_root", type=Path) + custom.add_argument("domain_sources", type=Path) + custom.add_argument("ip_sources", type=Path) + custom.add_argument("--text-only", action="store_true") + custom.set_defaults(func=mark_custom) + return result + + +def main() -> None: + args = parser().parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/tools/artifact_verifier.py b/scripts/tools/artifact_verifier.py index 80a375289..90737f100 100644 --- a/scripts/tools/artifact_verifier.py +++ b/scripts/tools/artifact_verifier.py @@ -48,13 +48,17 @@ def singbox_counts(data: dict[str, Any], kind: str) -> Counter[str]: if not isinstance(rule, dict): continue for field, canonical in mappings.items(): - values = rule.get(field, []) - if isinstance(values, list): - if kind == "ip" and field == "ip_cidr": - for value in values: - counts["IP-CIDR6" if ":" in value else "IP-CIDR"] += 1 - else: - counts[canonical] += len(values) + raw_values = rule.get(field) + if raw_values is None: + continue + values = raw_values if isinstance(raw_values, list) else [raw_values] + if not values or any(not isinstance(value, str) for value in values): + raise ValueError(f"sing-box decompile returned invalid {field} values") + if kind == "ip" and field == "ip_cidr": + for value in values: + counts["IP-CIDR6" if ":" in value else "IP-CIDR"] += 1 + else: + counts[canonical] += len(values) return counts diff --git a/scripts/tools/domain_rules.py b/scripts/tools/domain_rules.py index da1aabe7b..f829e07ac 100644 --- a/scripts/tools/domain_rules.py +++ b/scripts/tools/domain_rules.py @@ -39,7 +39,13 @@ def normalize_domain_value(kind: str, value: str) -> str: return value -def domain_value_errors(kind: str, value: str, *, require_canonical: bool) -> list[str]: +def domain_value_errors( + kind: str, + value: str, + *, + require_canonical: bool, + allow_single_label_suffix: bool = False, +) -> list[str]: errors: list[str] = [] if not value: return ["rule value must not be empty"] @@ -57,7 +63,7 @@ def domain_value_errors(kind: str, value: str, *, require_canonical: bool) -> li canonical = value.lower().rstrip(".") if len(canonical) > 253: errors.append(f"{kind} value is longer than 253 characters: {value}") - elif "." not in canonical: + elif "." not in canonical and not (allow_single_label_suffix and kind == "DOMAIN-SUFFIX"): errors.append(f"{kind} value is too broad; use a fully qualified domain: {value}") else: for label in canonical.split("."): @@ -82,7 +88,12 @@ def domain_value_errors(kind: str, value: str, *, require_canonical: bool) -> li return errors -def parse_classical_domain_file(path: Path, *, require_canonical: bool = True) -> tuple[list[ParsedDomainRule], list[str]]: +def parse_classical_domain_file( + path: Path, + *, + require_canonical: bool = True, + allow_single_label_suffix: bool = False, +) -> tuple[list[ParsedDomainRule], list[str]]: rules: list[ParsedDomainRule] = [] errors: list[str] = [] seen: dict[tuple[str, str], int] = {} @@ -105,7 +116,12 @@ def parse_classical_domain_file(path: Path, *, require_canonical: bool = True) - if require_canonical and value_raw != value_raw.strip(): errors.append(f"{path}:{line_no} rule value must not have surrounding whitespace: {value_raw!r}") continue - value_errors = domain_value_errors(kind, value_raw, require_canonical=require_canonical) + value_errors = domain_value_errors( + kind, + value_raw, + require_canonical=require_canonical, + allow_single_label_suffix=allow_single_label_suffix, + ) errors.extend(f"{path}:{line_no} {message}" for message in value_errors) if value_errors: continue diff --git a/scripts/tools/export-domain-rules.py b/scripts/tools/export-domain-rules.py index bfc22fa27..1ed0bf674 100644 --- a/scripts/tools/export-domain-rules.py +++ b/scripts/tools/export-domain-rules.py @@ -165,7 +165,12 @@ def parse_data_file(path: Path) -> tuple[list[Rule], list[Include], list[tuple[s raise ValueError(f"{path}:{line_no} {exc}") from exc value = normalize_rule_value(kind, value) canonical_kind = RULE_KIND_MAP[kind] - validation_errors = domain_value_errors(canonical_kind, value, require_canonical=False) + validation_errors = domain_value_errors( + canonical_kind, + value, + require_canonical=False, + allow_single_label_suffix=True, + ) if validation_errors: raise ValueError(f"{path}:{line_no} {validation_errors[0]}") rule = Rule(kind=canonical_kind, value=value, attrs=tuple(attrs)) @@ -296,7 +301,11 @@ def region_pairs_for_rule_set_names(names: set[str]) -> dict[str, list[str]]: def parse_classical_domain_rules(input_file: Path) -> list[Rule]: - parsed, errors = parse_classical_domain_file(input_file, require_canonical=True) + parsed, errors = parse_classical_domain_file( + input_file, + require_canonical=True, + allow_single_label_suffix=True, + ) if errors: raise ValueError("\n".join(errors)) return [Rule(kind=rule.kind, value=rule.value, attrs=tuple()) for rule in parsed]