diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a3e11fc2..ea056ba1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,6 +21,7 @@ on: - 'config/**' - 'scripts/**' - 'sources/custom/**' + - 'sources/builtin/**' - 'templates/**' - 'tests/**' schedule: diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index b4074ba8..ad4d7d5f 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -28,5 +28,5 @@ jobs: - name: Prepare scripts run: find scripts -type f \( -name '*.sh' -o -name '*.py' \) -exec chmod +x {} + - - name: Quick check - run: REQUIRE_SHELLCHECK=1 make lint + - name: Validate + run: REQUIRE_SHELLCHECK=1 make validate diff --git a/README.md b/README.md index f264bade..2e6e5687 100644 --- a/README.md +++ b/README.md @@ -34,12 +34,15 @@ https://raw.githubusercontent.com/KuGouGo/Rules/sing-box/domain/fakeip-filter.sr https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/ip/google.mrs ``` -常用规则包括 `cn`、`geolocation-cn`、`geolocation-!cn`、`google`、`telegram`、`emby-cn`、`emby` 和 `fakeip-filter`。其中 `cn` 与 `geolocation-cn` 在 `domain-list-community` 基础上合并 Loyalsoldier 发布的 `china-list.txt`，补充中国大陆可直连的长尾域名；不合并其宽泛的 `direct/proxy/reject` 列表。同时使用 `emby-cn` 与 `emby` 时，应先加载范围较小的 `emby-cn`。 +常用规则包括 `cn`、`geolocation-cn`、`geolocation-!cn`、`google`、`telegram`、`emby-cn`、`emby` 和 `fakeip-filter`。域名规则以 `domain-list-community` 源码树为主源，保留 include、affiliation 和属性派生；默认 `cn` / `geolocation-cn` 排除 `@ads`、`@!cn`，`geolocation-!cn` 排除 `@ads`、`@cn`，同时继续生成对应属性列表。该策略参考 `domain-list-custom`，但不重复克隆或反解析其 `geosite.dat`。 + +`china-list` 独立包含 `felixonmars/dnsmasq-china-list` 的 accelerated domains，仅用于有特殊需求的中国大陆 DNS 分流，不合并进 `cn` 或 `geolocation-cn`，也不建议作为路由规则使用。`ruleset.skk.moe` 用于补充 `cn` 中可在中国大陆直连的 `domestic` 域名，以及 AI、Apple 中国服务/CDN、Microsoft 中国 CDN、`apple-intelligence`、`icloudprivaterelay`、`category-game-platforms-download` 和 Apple 服务 IP；上游固定标记域名会在规范化阶段统一剔除。其中国 IP 列表因边际覆盖较低不默认引入，语义混合的 `direct`、宽泛的 `global/reject`、动态测速、通用 CDN 与下载大集合也不会默认引入。同时使用 `emby-cn` 与 `emby` 时，应先加载范围较小的 `emby-cn`。 ## 添加自定义规则 - 域名规则：`sources/custom/domain/*.list` - IP 规则：`sources/custom/ip/*.list` +- 内置稳定规则：`sources/builtin/`（例如无需远端下载的特殊用途网络） - 文件名使用小写字母、数字和连字符。 ```text diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 5edd35ff..2369b0da 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -5,8 +5,10 @@ | 内容 | 来源 | | --- | --- | | 域名规则 | [v2fly/domain-list-community](https://github.com/v2fly/domain-list-community)（MIT） | -| 中国大陆可直连域名补充 | [Loyalsoldier/v2ray-rules-dat](https://github.com/Loyalsoldier/v2ray-rules-dat) 发布的 `china-list.txt`（GPL-3.0；数据源为 [felixonmars/dnsmasq-china-list](https://github.com/felixonmars/dnsmasq-china-list)） | -| 中国 IP | 、、[Loyalsoldier/geoip](https://github.com/Loyalsoldier/geoip) | +| 域名属性过滤策略参考 | [Loyalsoldier/domain-list-custom](https://github.com/Loyalsoldier/domain-list-custom)（GPL-3.0；不直接分发其构建产物） | +| 中国大陆 DNS 分流域名 `china-list` | [Loyalsoldier/v2ray-rules-dat](https://github.com/Loyalsoldier/v2ray-rules-dat) 发布的 `china-list.txt`（GPL-3.0；数据源为 [felixonmars/dnsmasq-china-list](https://github.com/felixonmars/dnsmasq-china-list)） | +| 中国大陆直连域名、Apple Intelligence、iCloud Private Relay、游戏下载及 Apple 服务 IP 补充 | [SukkaW/Surge](https://github.com/SukkaW/Surge) 与其[发布仓库](https://github.com/SukkaLab/ruleset.skk.moe)（仓库代码 AGPL-3.0；发布规则仍按各文件头声明的许可证） | +| 中国 IP | 、 | | Google IP | | | Telegram IP | | | Cloudflare IP | 、 | diff --git a/config/upstreams.json b/config/upstreams.json index 3f950904..8552331e 100644 --- a/config/upstreams.json +++ b/config/upstreams.json @@ -13,25 +13,49 @@ "parser": "domain-suffix-text", "url": "https://raw.githubusercontent.com/Loyalsoldier/v2ray-rules-dat/release/china-list.txt", "health": { "requirement": "required", "min_raw_bytes": 1000000, "min_entries": 100000, "family": "any", "fallback_policy": "none" } + }, + "sukka-apple-intelligence": { + "kind": "text", "trust": "community", "parser": "sukka-classical-domain", "url": "https://ruleset.skk.moe/Clash/non_ip/apple_intelligence.txt", + "health": { "requirement": "required", "min_raw_bytes": 200, "min_entries": 5, "family": "any", "fallback_policy": "none" } + }, + "sukka-icloud-private-relay": { + "kind": "text", "trust": "community", "parser": "sukka-domain-set-text", "url": "https://ruleset.skk.moe/Clash/domainset/icloud_private_relay.txt", + "health": { "requirement": "required", "min_raw_bytes": 150, "min_entries": 6, "family": "any", "fallback_policy": "none" } + }, + "sukka-game-download": { + "kind": "text", "trust": "community", "parser": "sukka-domain-set-text", "url": "https://ruleset.skk.moe/Clash/domainset/game-download.txt", + "health": { "requirement": "required", "min_raw_bytes": 1000, "min_entries": 50, "family": "any", "fallback_policy": "none" } + }, + "sukka-domestic": { + "kind": "text", "trust": "community", "parser": "sukka-classical-domain", "url": "https://ruleset.skk.moe/Clash/non_ip/domestic.txt", + "health": { "requirement": "required", "min_raw_bytes": 20000, "min_entries": 750, "family": "any", "fallback_policy": "none" } + }, + "sukka-ai": { + "kind": "text", "trust": "community", "parser": "sukka-classical-domain", "url": "https://ruleset.skk.moe/Clash/non_ip/ai.txt", + "health": { "requirement": "required", "min_raw_bytes": 500, "min_entries": 40, "family": "any", "fallback_policy": "none" } + }, + "sukka-apple-cdn": { + "kind": "text", "trust": "community", "parser": "sukka-domain-set-text", "url": "https://ruleset.skk.moe/Clash/domainset/apple_cdn.txt", + "health": { "requirement": "required", "min_raw_bytes": 3000, "min_entries": 150, "family": "any", "fallback_policy": "none" } + }, + "sukka-apple-cn": { + "kind": "text", "trust": "community", "parser": "sukka-classical-domain", "url": "https://ruleset.skk.moe/Clash/non_ip/apple_cn.txt", + "health": { "requirement": "required", "min_raw_bytes": 500, "min_entries": 9, "family": "any", "fallback_policy": "none" } + }, + "sukka-microsoft-cdn": { + "kind": "text", "trust": "community", "parser": "sukka-classical-domain", "url": "https://ruleset.skk.moe/Clash/non_ip/microsoft_cdn.txt", + "health": { "requirement": "required", "min_raw_bytes": 1500, "min_entries": 50, "family": "any", "fallback_policy": "none" } } }, "ip": { "cn-ipv46": { - "kind": "text", "trust": "registry", "parser": "cidr-text", "url": "https://ispip.clang.cn/all_cn_ipv46.txt", + "kind": "text", "trust": "community", "parser": "cidr-text", "url": "https://ispip.clang.cn/all_cn_ipv46.txt", "health": { "requirement": "required", "min_raw_bytes": 1000, "min_entries": 100, "family": "dual", "fallback_policy": "none" } }, "cn-ipv46-apnic": { "kind": "text", "trust": "registry", "parser": "cidr-text", "url": "https://ispip.clang.cn/all_cn_ipv46_apnic.txt", "health": { "requirement": "required", "min_raw_bytes": 1000, "min_entries": 100, "family": "dual", "fallback_policy": "none" } }, - "loyalsoldier-geoip-cn": { - "kind": "text", "trust": "community", "parser": "cidr-text", "url": "https://raw.githubusercontent.com/Loyalsoldier/geoip/release/text/cn.txt", - "health": { "requirement": "required", "min_raw_bytes": 1000, "min_entries": 5000, "family": "any", "fallback_policy": "none" } - }, - "loyalsoldier-geoip-private": { - "kind": "text", "trust": "community", "parser": "cidr-text", "url": "https://raw.githubusercontent.com/Loyalsoldier/geoip/release/text/private.txt", - "health": { "requirement": "required", "min_raw_bytes": 100, "min_entries": 15, "family": "any", "fallback_policy": "none" } - }, "google": { "kind": "json", "trust": "official", "parser": "google-json", "url": "https://www.gstatic.com/ipranges/goog.json", "health": { "requirement": "required", "min_raw_bytes": 100, "min_entries": 80, "family": "dual", "fallback_policy": "none" } @@ -68,14 +92,18 @@ "kind": "html", "trust": "official", "parser": "html-cidr", "url": "https://support.apple.com/en-us/101555", "fallback_url": "https://support.apple.com/zh-cn/101555", "health": { "requirement": "required", "min_raw_bytes": 100, "min_entries": 3, "family": "any", "fallback_policy": "ordered" } }, + "sukka-apple-services": { + "kind": "text", "trust": "community", "parser": "sukka-classical-ip", "url": "https://ruleset.skk.moe/Clash/ip/apple_services.txt", + "health": { "requirement": "required", "min_raw_bytes": 300, "min_entries": 10, "family": "ipv4", "fallback_policy": "none" } + }, "ripe-stat": { "kind": "json-api", "trust": "registry", "parser": "ripe-stat-json", "base_url": "https://stat.ripe.net/data/announced-prefixes/data.json?resource=AS", "health": { "requirement": "required", "min_raw_bytes": 50, "min_entries": 1, "family": "any", "fallback_policy": "none" } } }, "asn_groups": { - "telegram": [44907, 59930, 62014, 62041, 211157], - "netflix": [2906, 40027], + "telegram": [62041], + "netflix": [2906], "spotify": [35228, 7441], "disney": [13379, 15260, 23344, 25932, 29736, 30311, 53578] } diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index b025e1bd..8c92f80f 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -17,6 +17,7 @@ make clean # 清理临时文件和产物 ```text sources/custom/domain/ 自定义域名规则 sources/custom/ip/ 自定义 IP 规则 +sources/builtin/ 不依赖远端下载的稳定内置规则 config/upstreams.json 上游地址和解析方式 config/tools-lock.json sing-box / mihomo 工具版本 scripts/ 同步、转换、校验和发布脚本 @@ -39,9 +40,13 @@ IP-CIDR,192.0.2.0/24 新增上游时修改 `config/upstreams.json`，并按实际情况更新根目录的 `THIRD_PARTY_NOTICES.md`。 +上游来源应优先选择官方或注册机构数据，并满足以下条件：能够声明稳定的健康阈值、对最终规则有可量化的边际覆盖、不会仅重复已有来源。共享同一载荷的不同规则集应只下载一次再分别解析；高重叠、低边际或容易引入地域误判的来源应移除。 + +ASN 补充仅保留官方列表之外仍有覆盖贡献的网络。所有 RIPE Stat ASN 在一次并行批次中下载，每个唯一 ASN 只规范化和检查一次，再由 Telegram、Netflix、Spotify、Disney 规则组复用。 + ## 自动构建 -- PR：只运行快速检查。 +- PR：运行完整 lint 和测试，阻止未经测试的构建脚本变更合并。 - `main`：检查、构建并发布五个平台分支。 - 定时任务：每天同步一次上游。 - 手动运行：Actions → Sync Rules，可选择 `auto`、`custom` 或 `full`。 diff --git a/scripts/commands/lint-custom-rules.sh b/scripts/commands/lint-custom-rules.sh index 8f209ab6..f5f5efd4 100755 --- a/scripts/commands/lint-custom-rules.sh +++ b/scripts/commands/lint-custom-rules.sh @@ -8,4 +8,8 @@ python3 "$ROOT/scripts/tools/lint-custom-rules.py" \ --domain-dir "$ROOT/sources/custom/domain" \ --ip-dir "$ROOT/sources/custom/ip" -echo "custom rule lint passed" +python3 "$ROOT/scripts/tools/lint-custom-rules.py" \ + --domain-dir "$ROOT/sources/builtin/domain" \ + --ip-dir "$ROOT/sources/builtin/ip" + +echo "custom and built-in rule lint passed" diff --git a/scripts/commands/select-build-scope.sh b/scripts/commands/select-build-scope.sh index 7eb78b8b..df454902 100755 --- a/scripts/commands/select-build-scope.sh +++ b/scripts/commands/select-build-scope.sh @@ -218,7 +218,7 @@ collect_deleted_custom_files() { } has_build_relevant_changes() { - grep -Eq '^(\.github/workflows/|Makefile$|config/|scripts/|sources/custom/|templates/|tests/)' + grep -Eq '^(\.github/workflows/|Makefile$|config/|scripts/|sources/(builtin|custom)/|templates/|tests/)' } has_only_non_build_changes() { diff --git a/scripts/commands/sync-upstream.sh b/scripts/commands/sync-upstream.sh index 924ede7a..0790485d 100755 --- a/scripts/commands/sync-upstream.sh +++ b/scripts/commands/sync-upstream.sh @@ -25,58 +25,84 @@ IP_CANONICAL_ARTIFACTS=("${IP_TEXT_ARTIFACTS[@]}" netflix spotify disney) UPSTREAMS_CONFIG_FILE="$ROOT_DIR/config/upstreams.json" UPSTREAM_SUMMARY_FILE="$WORK_TMP_DIR/upstream-summary.jsonl" FIRST_BATCH_BASELINES_FILE="$ROOT_DIR/config/upstream-first-batch-baselines.json" +BUILTIN_PRIVATE_SOURCE_FILE="$ROOT_DIR/sources/builtin/ip/private.list" +SUKKA_MARKER_DOMAIN="7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe" -upstream_value() { - local section="$1" - local name="$2" - local key="$3" - python3 - <<'PY' "$UPSTREAMS_CONFIG_FILE" "$section" "$name" "$key" -import json, sys -config, section, name, key = sys.argv[1:] -data = json.load(open(config, encoding="utf-8"))[section][name] -value = data.get(key, "") -if isinstance(value, list): - print(" ".join(str(item) for item in value)) -else: - print(value) -PY -} +mapfile -t UPSTREAM_SETTINGS < <(python3 - "$UPSTREAMS_CONFIG_FILE" <<'PY' +import json +import sys -upstream_asn_group() { - local name="$1" - python3 - <<'PY' "$UPSTREAMS_CONFIG_FILE" "$name" -import json, sys -config, name = sys.argv[1:] -print(" ".join(str(item) for item in json.load(open(config, encoding="utf-8"))["asn_groups"][name])) +with open(sys.argv[1], encoding="utf-8") as handle: + config = json.load(handle) + +values = ( + config["domain"]["dlc"]["url"], + config["domain"]["loyalsoldier-china-list"]["url"], + config["domain"]["sukka-apple-intelligence"]["url"], + config["domain"]["sukka-icloud-private-relay"]["url"], + config["domain"]["sukka-game-download"]["url"], + config["domain"]["sukka-domestic"]["url"], + config["domain"]["sukka-ai"]["url"], + config["domain"]["sukka-apple-cdn"]["url"], + config["domain"]["sukka-apple-cn"]["url"], + config["domain"]["sukka-microsoft-cdn"]["url"], + config["ip"]["cn-ipv46"]["url"], + config["ip"]["cn-ipv46-apnic"]["url"], + config["ip"]["google"]["url"], + config["ip"]["telegram"]["url"], + config["ip"]["cloudflare-ipv4"]["url"], + config["ip"]["cloudflare-ipv6"]["url"], + config["ip"]["aws"]["url"], + config["ip"]["cloudfront"]["url"], + config["ip"]["fastly"]["url"], + config["ip"]["github"]["url"], + config["ip"]["apple"]["url"], + config["ip"]["apple"]["fallback_url"], + config["ip"]["sukka-apple-services"]["url"], + config["ip"]["ripe-stat"]["base_url"], + *(" ".join(str(asn) for asn in config["asn_groups"][name]) for name in ("telegram", "netflix", "spotify", "disney")), +) +print("\n".join(values)) PY -} +) +if [ "${#UPSTREAM_SETTINGS[@]}" -ne 28 ]; then + echo "failed to load complete upstream settings" >&2 + exit 1 +fi -DOMAIN_SOURCE_REPO_URL="$(upstream_value domain dlc url)" -LOYALSOLDIER_CHINA_LIST_SOURCE_URL="$(upstream_value domain loyalsoldier-china-list url)" -CN_IPV46_SOURCE_URL="$(upstream_value ip cn-ipv46 url)" -CN_IPV46_APNIC_SOURCE_URL="$(upstream_value ip cn-ipv46-apnic url)" -LOYALSOLDIER_GEOIP_CN_SOURCE_URL="$(upstream_value ip loyalsoldier-geoip-cn url)" -LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL="$(upstream_value ip loyalsoldier-geoip-private url)" -GOOGLE_IP_SOURCE_URL="$(upstream_value ip google url)" -TELEGRAM_IP_SOURCE_URL="$(upstream_value ip telegram url)" -CLOUDFLARE_IPV4_SOURCE_URL="$(upstream_value ip cloudflare-ipv4 url)" -CLOUDFLARE_IPV6_SOURCE_URL="$(upstream_value ip cloudflare-ipv6 url)" -AWS_IP_SOURCE_URL="$(upstream_value ip aws url)" -CLOUDFRONT_IP_SOURCE_URL="$(upstream_value ip cloudfront url)" -FASTLY_IP_SOURCE_URL="$(upstream_value ip fastly url)" -GITHUB_IP_SOURCE_URL="$(upstream_value ip github url)" -APPLE_IP_SOURCE_URL="$(upstream_value ip apple url)" -APPLE_IP_SOURCE_FALLBACK_URL="$(upstream_value ip apple fallback_url)" -RIPE_STAT_BASE_URL="$(upstream_value ip ripe-stat base_url)" +DOMAIN_SOURCE_REPO_URL="${UPSTREAM_SETTINGS[0]}" +LOYALSOLDIER_CHINA_LIST_SOURCE_URL="${UPSTREAM_SETTINGS[1]}" +SUKKA_APPLE_INTELLIGENCE_SOURCE_URL="${UPSTREAM_SETTINGS[2]}" +SUKKA_ICLOUD_PRIVATE_RELAY_SOURCE_URL="${UPSTREAM_SETTINGS[3]}" +SUKKA_GAME_DOWNLOAD_SOURCE_URL="${UPSTREAM_SETTINGS[4]}" +SUKKA_DOMESTIC_SOURCE_URL="${UPSTREAM_SETTINGS[5]}" +SUKKA_AI_SOURCE_URL="${UPSTREAM_SETTINGS[6]}" +SUKKA_APPLE_CDN_SOURCE_URL="${UPSTREAM_SETTINGS[7]}" +SUKKA_APPLE_CN_SOURCE_URL="${UPSTREAM_SETTINGS[8]}" +SUKKA_MICROSOFT_CDN_SOURCE_URL="${UPSTREAM_SETTINGS[9]}" +CN_IPV46_SOURCE_URL="${UPSTREAM_SETTINGS[10]}" +CN_IPV46_APNIC_SOURCE_URL="${UPSTREAM_SETTINGS[11]}" +GOOGLE_IP_SOURCE_URL="${UPSTREAM_SETTINGS[12]}" +TELEGRAM_IP_SOURCE_URL="${UPSTREAM_SETTINGS[13]}" +CLOUDFLARE_IPV4_SOURCE_URL="${UPSTREAM_SETTINGS[14]}" +CLOUDFLARE_IPV6_SOURCE_URL="${UPSTREAM_SETTINGS[15]}" +AWS_IP_SOURCE_URL="${UPSTREAM_SETTINGS[16]}" +CLOUDFRONT_IP_SOURCE_URL="${UPSTREAM_SETTINGS[17]}" +FASTLY_IP_SOURCE_URL="${UPSTREAM_SETTINGS[18]}" +GITHUB_IP_SOURCE_URL="${UPSTREAM_SETTINGS[19]}" +APPLE_IP_SOURCE_URL="${UPSTREAM_SETTINGS[20]}" +APPLE_IP_SOURCE_FALLBACK_URL="${UPSTREAM_SETTINGS[21]}" +SUKKA_APPLE_SERVICES_SOURCE_URL="${UPSTREAM_SETTINGS[22]}" +RIPE_STAT_BASE_URL="${UPSTREAM_SETTINGS[23]}" DLC_MIN_ATTR_RULESETS="${DLC_MIN_ATTR_RULESETS:-300}" DLC_MIN_CN_ATTR_RULESETS="${DLC_MIN_CN_ATTR_RULESETS:-100}" DLC_MIN_NOT_CN_ATTR_RULESETS="${DLC_MIN_NOT_CN_ATTR_RULESETS:-30}" DLC_MIN_ADS_ATTR_RULESETS="${DLC_MIN_ADS_ATTR_RULESETS:-100}" DLC_MIN_REGIONAL_RULESETS="${DLC_MIN_REGIONAL_RULESETS:-40}" -read -r -a TELEGRAM_ASNS <<< "$(upstream_asn_group telegram)" -read -r -a NETFLIX_ASNS <<< "$(upstream_asn_group netflix)" -read -r -a SPOTIFY_ASNS <<< "$(upstream_asn_group spotify)" -read -r -a DISNEY_ASNS <<< "$(upstream_asn_group disney)" +read -r -a TELEGRAM_ASNS <<< "${UPSTREAM_SETTINGS[24]}" +read -r -a NETFLIX_ASNS <<< "${UPSTREAM_SETTINGS[25]}" +read -r -a SPOTIFY_ASNS <<< "${UPSTREAM_SETTINGS[26]}" +read -r -a DISNEY_ASNS <<< "${UPSTREAM_SETTINGS[27]}" # shellcheck source=scripts/lib/common.sh source "$ROOT_DIR/scripts/lib/common.sh" @@ -178,6 +204,7 @@ verify_and_record_upstream_health() { local fallback_used="${6:-0}" local context="${7:-}" local health_json status detail health_detail verifier_failed=0 + local -a health_fields=() if health_json="$(python3 "$ROOT_DIR/scripts/tools/verify-upstream-health.py" \ "$UPSTREAMS_CONFIG_FILE" "$category" "$name" "$raw_file" "$normalized_file")"; then @@ -185,9 +212,18 @@ verify_and_record_upstream_health() { else verifier_failed=1 fi - if ! status="$(printf '%s' "$health_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])' 2>/dev/null)"; then + mapfile -t health_fields < <( + printf '%s' "$health_json" \ + | python3 -c 'import json,sys; data=json.load(sys.stdin); print(data["status"]); print("; ".join(data.get("errors", [])))' \ + 2>/dev/null + ) + if [ "${#health_fields[@]}" -ne 2 ]; then status=semantic_regression + health_detail="health verifier failed" verifier_failed=1 + else + status="${health_fields[0]}" + health_detail="${health_fields[1]}" fi case "$status" in ok|semantic_regression) ;; @@ -198,7 +234,6 @@ verify_and_record_upstream_health() { else detail="" fi - health_detail="$(printf '%s' "$health_json" | python3 -c 'import json,sys; print("; ".join(json.load(sys.stdin).get("errors", [])))' 2>/dev/null || printf 'health verifier failed')" detail="${health_detail:-$detail}" if [ -n "$context" ]; then detail="${context}${detail:+; $detail}" @@ -273,39 +308,32 @@ render_ip_text_artifacts() { done } -sync_asn_ip_cidrs() { - local name="$1" - shift - local -a asns=("$@") - local -a cidr_files=() - local asn raw_json cidr_txt merge_mode health_json health_status health_detail +prepare_ripe_stat_asns() { + local -A seen=() + local -a unique_asns=() download_args=() + local asn raw_json cidr_txt health_json health_status health_detail - for asn in "${asns[@]}"; do - raw_json="$IP_BUILD_TMP_DIR/${name}_as${asn}.raw.json" - cidr_txt="$IP_BUILD_TMP_DIR/${name}_as${asn}.cidr.txt" - download_file "${RIPE_STAT_BASE_URL}${asn}" "$raw_json" - cidr_files+=("$cidr_txt") + for asn in "$@"; do + if [ -n "${seen[$asn]:-}" ]; then + continue + fi + seen[$asn]=1 + unique_asns+=("$asn") + raw_json="$IP_BUILD_TMP_DIR/ripe_as${asn}.raw.json" + download_args+=("ripe-stat-as${asn}" required "${RIPE_STAT_BASE_URL}${asn}" "$raw_json") done + download_files_parallel "${download_args[@]}" - # Normalize separately so malformed responses are attributed and reported - # before they block the transaction. - for asn in "${asns[@]}"; do - raw_json="$IP_BUILD_TMP_DIR/${name}_as${asn}.raw.json" - cidr_txt="$IP_BUILD_TMP_DIR/${name}_as${asn}.cidr.txt" + for asn in "${unique_asns[@]}"; do + raw_json="$IP_BUILD_TMP_DIR/ripe_as${asn}.raw.json" + cidr_txt="$IP_BUILD_TMP_DIR/ripe_as${asn}.cidr.txt" if ! python3 "$ROOT_DIR/scripts/tools/normalize-ip-rules.py" single ripe-stat-json "$raw_json" "$cidr_txt"; then : > "$cidr_txt" record_upstream_summary ip "ripe-stat-as${asn}" semantic_regression "${RIPE_STAT_BASE_URL}${asn}" "$raw_json" "$cidr_txt" 0 "invalid RIPE Stat response" echo "RIPE Stat response AS${asn} is invalid" >&2 return 1 fi - done - # Every RIPE Stat response is independently subject to the configured source - # policy. Record the failing response before returning so transaction - # diagnostics explain which ASN was undersized or invalid. - for asn in "${asns[@]}"; do - raw_json="$IP_BUILD_TMP_DIR/${name}_as${asn}.raw.json" - cidr_txt="$IP_BUILD_TMP_DIR/${name}_as${asn}.cidr.txt" if health_json="$(python3 "$ROOT_DIR/scripts/tools/verify-upstream-health.py" "$UPSTREAMS_CONFIG_FILE" ip ripe-stat "$raw_json" "$cidr_txt")"; then health_status=ok else @@ -318,6 +346,23 @@ sync_asn_ip_cidrs() { return 1 fi done +} + +sync_asn_ip_cidrs() { + local name="$1" + shift + local -a asns=("$@") cidr_files=() + local asn raw_json cidr_txt merge_mode health_json health_status health_detail + + for asn in "${asns[@]}"; do + raw_json="$IP_BUILD_TMP_DIR/ripe_as${asn}.raw.json" + cidr_txt="$IP_BUILD_TMP_DIR/ripe_as${asn}.cidr.txt" + if [ ! -s "$raw_json" ] || [ ! -s "$cidr_txt" ]; then + echo "RIPE Stat AS${asn} was not prepared" >&2 + return 1 + fi + cidr_files+=("$cidr_txt") + done merge_mode="${ASN_CIDR_MERGE_MODE:-collapse}" case "$merge_mode" in @@ -338,7 +383,7 @@ sync_asn_ip_cidrs() { local group_raw="$IP_BUILD_TMP_DIR/${name}.ripe-group.raw" : > "$group_raw" for asn in "${asns[@]}"; do - cat "$IP_BUILD_TMP_DIR/${name}_as${asn}.raw.json" >> "$group_raw" + cat "$IP_BUILD_TMP_DIR/ripe_as${asn}.raw.json" >> "$group_raw" done if health_json="$(python3 "$ROOT_DIR/scripts/tools/verify-upstream-health.py" "$UPSTREAMS_CONFIG_FILE" ip ripe-stat "$group_raw" "$IP_BUILD_TMP_DIR/${name}.cidr.txt")"; then health_status=ok @@ -427,10 +472,9 @@ generate_ip_normalize_manifest() { generate_normalize_manifest "$manifest_file" \ text "$tmp_dir/cn_ipv46.raw.txt" "$tmp_dir/cn_ipv46.cidr.txt" \ text "$tmp_dir/cn_ipv46_apnic.raw.txt" "$tmp_dir/cn_ipv46_apnic.cidr.txt" \ - text "$tmp_dir/loyalsoldier_geoip_cn.raw.txt" "$tmp_dir/loyalsoldier_geoip_cn.cidr.txt" \ - text "$tmp_dir/private.raw.txt" "$tmp_dir/private.cidr.txt" \ text "$tmp_dir/cloudflare_ipv4.raw.txt" "$tmp_dir/cloudflare_ipv4.cidr.txt" \ text "$tmp_dir/cloudflare_ipv6.raw.txt" "$tmp_dir/cloudflare_ipv6.cidr.txt" \ + sukka-classical-ip "$tmp_dir/sukka_apple_services.raw.txt" "$tmp_dir/sukka_apple_services.cidr.txt" \ aws-cloudfront-json "$tmp_dir/aws.raw.json" "$tmp_dir/cloudfront.cidr.txt" \ aws-json "$tmp_dir/aws.raw.json" "$tmp_dir/aws.cidr.txt" \ fastly-json "$tmp_dir/fastly.raw.json" "$tmp_dir/fastly.cidr.txt" \ @@ -576,6 +620,7 @@ first_batch_source_url() { classify_first_batch_source() { local source="$1" local raw_file result_json status reason + local -a result_fields=() raw_file="$(first_batch_raw_file "$source")" result_json="$( @@ -585,8 +630,16 @@ classify_first_batch_source() { "$raw_file" \ "$FIRST_BATCH_BASELINES_FILE" )" - status="$(printf '%s' "$result_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["status"])')" - reason="$(printf '%s' "$result_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["reason"])')" + mapfile -t result_fields < <( + printf '%s' "$result_json" \ + | python3 -c 'import json,sys; data=json.load(sys.stdin); print(data["status"]); print(data["reason"])' + ) + if [ "${#result_fields[@]}" -ne 2 ]; then + echo "failed to parse first-batch classification for $source" >&2 + return 1 + fi + status="${result_fields[0]}" + reason="${result_fields[1]}" set_first_batch_result "$source" "$status" "$reason" if [ "$status" != "ok" ]; then @@ -602,19 +655,6 @@ classify_first_batch_source() { fi } -download_and_classify_first_batch_source() { - local source="$1" - local url="$2" - local raw_file - - raw_file="$(first_batch_raw_file "$source")" - if ! download_file "$url" "$raw_file"; then - rm -f "$raw_file" - fi - - classify_first_batch_source "$source" -} - normalize_first_batch_source() { local source="$1" local raw_file output_file source_type @@ -806,15 +846,22 @@ verify_and_record_upstream_health \ 0 \ "commit=$(git -C "$WORK_TMP_DIR/domain-list-community" rev-parse HEAD)" -# Supplement the DLC China categories with the independently maintained -# dnsmasq-china-list export published by Loyalsoldier. Parse the 100k+ source -# once, verify its normalized form independently, and update both targets via -# a reverse-label trie (linear in domain label count). -download_file "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" +# Fetch independent domain supplements together. The large China list remains +# a DNS-only artifact; it must not change DLC's routing-oriented cn categories. +download_files_parallel \ + loyalsoldier-china-list required "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" \ + sukka-apple-intelligence required "$SUKKA_APPLE_INTELLIGENCE_SOURCE_URL" "$WORK_TMP_DIR/sukka-apple-intelligence.raw.conf" \ + sukka-icloud-private-relay required "$SUKKA_ICLOUD_PRIVATE_RELAY_SOURCE_URL" "$WORK_TMP_DIR/sukka-icloud-private-relay.raw.conf" \ + sukka-game-download required "$SUKKA_GAME_DOWNLOAD_SOURCE_URL" "$WORK_TMP_DIR/sukka-game-download.raw.conf" \ + sukka-domestic required "$SUKKA_DOMESTIC_SOURCE_URL" "$WORK_TMP_DIR/sukka-domestic.raw.conf" \ + sukka-ai required "$SUKKA_AI_SOURCE_URL" "$WORK_TMP_DIR/sukka-ai.raw.conf" \ + sukka-apple-cdn required "$SUKKA_APPLE_CDN_SOURCE_URL" "$WORK_TMP_DIR/sukka-apple-cdn.raw.conf" \ + sukka-apple-cn required "$SUKKA_APPLE_CN_SOURCE_URL" "$WORK_TMP_DIR/sukka-apple-cn.raw.conf" \ + sukka-microsoft-cdn required "$SUKKA_MICROSOFT_CDN_SOURCE_URL" "$WORK_TMP_DIR/sukka-microsoft-cdn.raw.conf" +: > "$DOMAIN_RULE_TMP_DIR/china-list.list" python3 "$ROOT_DIR/scripts/tools/merge-domain-suffixes.py" \ "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" \ - "$DOMAIN_RULE_TMP_DIR/cn.list" \ - "$DOMAIN_RULE_TMP_DIR/geolocation-cn.list" \ + "$DOMAIN_RULE_TMP_DIR/china-list.list" \ --normalized-output "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list" verify_and_record_upstream_health \ domain \ @@ -824,6 +871,52 @@ verify_and_record_upstream_health \ "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list" \ 0 +while IFS='|' read -r source_name source_url source_format target_name; do + raw_file="$WORK_TMP_DIR/${source_name}.raw.conf" + normalized_file="$WORK_TMP_DIR/${source_name}.normalized.list" + python3 "$ROOT_DIR/scripts/tools/merge-domain-rule-source.py" \ + "$raw_file" \ + "$DOMAIN_RULE_TMP_DIR/${target_name}.list" \ + "$normalized_file" \ + --format "$source_format" \ + --ignore-domain "$SUKKA_MARKER_DOMAIN" + verify_and_record_upstream_health \ + domain "$source_name" "$source_url" "$raw_file" "$normalized_file" 0 +done <&2 + return 2 + fi + + local log_dir="${RULES_DOWNLOAD_LOG_DIR:-${WORK_TMP_DIR:-}}" + if [ -z "$log_dir" ]; then + echo "parallel download log directory is not configured" >&2 + return 2 + fi + mkdir -p "$log_dir" + + local -a labels=() modes=() urls=() outputs=() pids=() logs=() + local label mode url output log index failed=0 + while [ "$#" -gt 0 ]; do + label="$1" + mode="$2" + url="$3" + output="$4" + shift 4 + case "$mode" in + required|classified) ;; + *) echo "unsupported download mode for $label: $mode" >&2; return 2 ;; + esac + labels+=("$label") + modes+=("$mode") + urls+=("$url") + outputs+=("$output") + done + + for index in "${!labels[@]}"; do + log="$log_dir/download-${index}.log" + logs+=("$log") + download_file "${urls[$index]}" "${outputs[$index]}" >"$log" 2>&1 & + pids+=("$!") + done + + for index in "${!labels[@]}"; do + if wait "${pids[$index]}"; then + cat "${logs[$index]}" + else + cat "${logs[$index]}" >&2 + rm -f "${outputs[$index]}" + if [ "${modes[$index]}" = "required" ]; then + echo "required download failed: ${labels[$index]}" >&2 + failed=1 + fi + fi + rm -f "${logs[$index]}" + done + + [ "$failed" -eq 0 ] +} + tool_lock_value() { local tool="$1" local field="$2" diff --git a/scripts/tests/test-build-scope.sh b/scripts/tests/test-build-scope.sh index 8059428f..ccd6a272 100755 --- a/scripts/tests/test-build-scope.sh +++ b/scripts/tests/test-build-scope.sh @@ -158,6 +158,10 @@ assert_scope full "pull_request includes changes outside custom sources" \ EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ CHANGED_FILES=$'README.md\nscripts/commands/build-custom.sh' +assert_scope full "pull_request includes changes outside custom sources" \ + EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ + CHANGED_FILES=$'sources/builtin/ip/private.list' + assert_scope full "pull_request includes unclassified changes" \ EVENT_NAME=pull_request CURRENT_SHA=HEAD BEFORE_SHA=base \ CHANGED_FILES=$'pyproject.toml' diff --git a/scripts/tests/test-domain-parsing.sh b/scripts/tests/test-domain-parsing.sh index aea96fdc..ecb07384 100755 --- a/scripts/tests/test-domain-parsing.sh +++ b/scripts/tests/test-domain-parsing.sh @@ -492,6 +492,76 @@ EOF } +test_regional_base_lists_apply_safe_attribute_policy() { + mkdir -p "$TMP_DIR/regional_policy/data" "$TMP_DIR/regional_policy/out" + cat > "$TMP_DIR/regional_policy/data/shared" <<'EOF' +domain:plain.example +domain:cn.example @cn +domain:not-cn.example @!cn +domain:ads.example @ads +domain:cn-ads.example @cn @ads +EOF + cat > "$TMP_DIR/regional_policy/data/cn" <<'EOF' +include:shared +EOF + cat > "$TMP_DIR/regional_policy/data/geolocation-cn" <<'EOF' +include:shared +EOF + cat > "$TMP_DIR/regional_policy/data/geolocation-!cn" <<'EOF' +include:shared +EOF + + python3 "$ROOT/scripts/tools/export-domain-rules.py" export \ + "$TMP_DIR/regional_policy/data" \ + "$TMP_DIR/regional_policy/out" + + cat > "$TMP_DIR/regional_policy/cn.expected" <<'EOF' +DOMAIN-SUFFIX,plain.example +DOMAIN-SUFFIX,cn.example +EOF + cat > "$TMP_DIR/regional_policy/not-cn.expected" <<'EOF' +DOMAIN-SUFFIX,plain.example +DOMAIN-SUFFIX,not-cn.example +EOF + cat > "$TMP_DIR/regional_policy/ads.expected" <<'EOF' +DOMAIN-SUFFIX,ads.example +DOMAIN-SUFFIX,cn-ads.example +EOF + cat > "$TMP_DIR/regional_policy/not-cn-attr.expected" <<'EOF' +DOMAIN-SUFFIX,not-cn.example +EOF + cat > "$TMP_DIR/regional_policy/cn-attr.expected" <<'EOF' +DOMAIN-SUFFIX,cn.example +DOMAIN-SUFFIX,cn-ads.example +EOF + + assert_file_equals \ + "$TMP_DIR/regional_policy/cn.expected" \ + "$TMP_DIR/regional_policy/out/cn.list" \ + "cn excludes ads and !cn rules from its default output" + assert_file_equals \ + "$TMP_DIR/regional_policy/cn.expected" \ + "$TMP_DIR/regional_policy/out/geolocation-cn.list" \ + "geolocation-cn excludes ads and !cn rules from its default output" + assert_file_equals \ + "$TMP_DIR/regional_policy/not-cn.expected" \ + "$TMP_DIR/regional_policy/out/geolocation-!cn.list" \ + "geolocation-!cn excludes ads and cn rules from its default output" + assert_file_equals \ + "$TMP_DIR/regional_policy/ads.expected" \ + "$TMP_DIR/regional_policy/out/cn@ads.list" \ + "regional base filtering preserves ads derivative output" + assert_file_equals \ + "$TMP_DIR/regional_policy/not-cn-attr.expected" \ + "$TMP_DIR/regional_policy/out/cn@!cn.list" \ + "regional base filtering preserves !cn derivative output" + assert_file_equals \ + "$TMP_DIR/regional_policy/cn-attr.expected" \ + "$TMP_DIR/regional_policy/out/geolocation-!cn@cn.list" \ + "regional base filtering preserves cn derivative output" +} + + test_export_materializes_attr_derivatives_with_sing_geosite_filter() { mkdir -p "$TMP_DIR/region_derivatives/data" "$TMP_DIR/region_derivatives/out" cat > "$TMP_DIR/region_derivatives/data/vendor" <<'EOF' @@ -771,6 +841,7 @@ test_batch_domain_dir_outputs test_include_filter_semantics test_export_preserves_upstream_order_and_cn_regex_policy test_attr_derivatives_merge_duplicate_rule_attrs +test_regional_base_lists_apply_safe_attribute_policy test_export_materializes_attr_derivatives_with_sing_geosite_filter test_domain_capability_summary test_mihomo_mrs_skip_summary diff --git a/scripts/tests/test-ip-normalization.sh b/scripts/tests/test-ip-normalization.sh index 851cc213..64456023 100755 --- a/scripts/tests/test-ip-normalization.sh +++ b/scripts/tests/test-ip-normalization.sh @@ -208,6 +208,26 @@ assert_file_content \ "$TMP_DIR/surge-source.plain" \ $'10.0.0.0/8\n2001:db8::/32' +python3 "$ROOT/scripts/tools/normalize-ip-rules.py" single classical-ip \ + "$TMP_DIR/surge-source.list" "$TMP_DIR/classical-ip.plain" +assert_file_content \ + "$TMP_DIR/classical-ip.plain" \ + $'10.0.0.0/8\n2001:db8::/32' +printf '%s\n' 'IP-CIDR,10.0.0.0/8,unsupported' > "$TMP_DIR/classical-ip-invalid.list" +if python3 "$ROOT/scripts/tools/normalize-ip-rules.py" single classical-ip \ + "$TMP_DIR/classical-ip-invalid.list" "$TMP_DIR/classical-ip-invalid.out" >/dev/null 2>&1; then + echo "test failed: classical IP parser accepted unsupported option" >&2 + exit 1 +fi +cat > "$TMP_DIR/sukka-classical-ip.list" <<'CIDRS' +DOMAIN,7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe +IP-CIDR,10.0.0.0/8,no-resolve +IP-CIDR6,2001:db8::/32,no-resolve +CIDRS +python3 "$ROOT/scripts/tools/normalize-ip-rules.py" single sukka-classical-ip \ + "$TMP_DIR/sukka-classical-ip.list" "$TMP_DIR/sukka-classical-ip.out" +assert_file_content "$TMP_DIR/sukka-classical-ip.out" $'10.0.0.0/8\n2001:db8::/32' + cat > "$TMP_DIR/singbox-source.txt" <<'CIDRS' 192.168.1.1/24 192.168.1.0/24 diff --git a/scripts/tests/test-pr-validation-workflow.sh b/scripts/tests/test-pr-validation-workflow.sh new file mode 100755 index 00000000..6fc672e4 --- /dev/null +++ b/scripts/tests/test-pr-validation-workflow.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +WORKFLOW="$ROOT/.github/workflows/pull-request.yml" + +if [ ! -f "$WORKFLOW" ]; then + echo "test failed: pull request workflow is missing" >&2 + exit 1 +fi + +python3 - "$WORKFLOW" <<'PY' +import re +import sys +from pathlib import Path + +workflow = Path(sys.argv[1]) +text = workflow.read_text(encoding="utf-8") +validate_command = re.compile( + r"^\s*run:\s*(?:REQUIRE_SHELLCHECK=1\s+)?make\s+validate\s*$", + re.MULTILINE, +) + +if not validate_command.search(text): + raise SystemExit( + f"{workflow}: pull request validation must run the full make validate target" + ) +PY + +echo "pull request validation workflow tests passed" diff --git a/scripts/tests/test-shell-utils.sh b/scripts/tests/test-shell-utils.sh index a92c1966..05a50409 100755 --- a/scripts/tests/test-shell-utils.sh +++ b/scripts/tests/test-shell-utils.sh @@ -148,6 +148,56 @@ test_sha256_file_uses_python_fallback_without_sha256sum() { "sha256_file Python fallback" } +test_parallel_downloads_preserve_failure_semantics() { + local download_root="$TMP_DIR/parallel-downloads" + local ready_dir="$download_root/ready" + mkdir -p "$ready_dir" + + download_file() { + local url="$1" + local output="$2" + local name="${url##*/}" + touch "$ready_dir/$name" + local attempts=0 + while [ "$(find "$ready_dir" -type f | wc -l | tr -d ' ')" -lt 3 ]; do + attempts=$((attempts + 1)) + if [ "$attempts" -gt 100 ]; then + echo "downloads did not start concurrently" >&2 + return 1 + fi + sleep 0.01 + done + if [ "$name" = "classified-fail" ] || [ "$name" = "required-fail" ]; then + return 1 + fi + printf '%s\n' "$name" > "$output" + } + + RULES_DOWNLOAD_LOG_DIR="$download_root/logs" download_files_parallel \ + one required mock://one "$download_root/one.out" \ + two required mock://two "$download_root/two.out" \ + classified-fail classified mock://classified-fail "$download_root/classified.out" + assert_file_content "$download_root/one.out" "one" + assert_file_content "$download_root/two.out" "two" + if [ -e "$download_root/classified.out" ]; then + echo "test failed: failed classified download output should be removed" >&2 + exit 1 + fi + + find "$ready_dir" -type f -delete + if RULES_DOWNLOAD_LOG_DIR="$download_root/logs" download_files_parallel \ + one required mock://one "$download_root/one.out" \ + two required mock://two "$download_root/two.out" \ + required-fail required mock://required-fail "$download_root/required.out"; then + echo "test failed: required parallel download failure should fail the batch" >&2 + exit 1 + fi + if [ -e "$download_root/required.out" ]; then + echo "test failed: failed required download output should be removed" >&2 + exit 1 + fi +} + test_list_rule_files_sorts_lists_only test_list_rule_files_missing_dir_is_empty test_write_if_changed_replaces_different_file @@ -157,5 +207,6 @@ test_write_if_nonempty_or_remove_deletes_empty_output_and_stale_target test_common_source_has_no_tool_cache_side_effects test_setup_tool_cache_creates_bin_and_updates_path_once test_sha256_file_uses_python_fallback_without_sha256sum +test_parallel_downloads_preserve_failure_semantics echo "shell utility tests passed" diff --git a/scripts/tests/test-sukka-domain-merge.sh b/scripts/tests/test-sukka-domain-merge.sh new file mode 100644 index 00000000..723b83f9 --- /dev/null +++ b/scripts/tests/test-sukka-domain-merge.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +cat > "$TMP_DIR/target.list" <<'EOF' +DOMAIN,api.example.com +DOMAIN,keep.example.net +EOF +cat > "$TMP_DIR/domain-set.conf" <<'EOF' +# source comment +.example.com +exact.example.org ++ .invalid.example +EOF + +if python3 "$ROOT/scripts/tools/merge-domain-rule-source.py" \ + "$TMP_DIR/domain-set.conf" "$TMP_DIR/target.list" "$TMP_DIR/normalized.list" \ + --format domain-set >/dev/null 2>&1; then + echo "test failed: malformed domain-set source was accepted" >&2 + exit 1 +fi + +cat > "$TMP_DIR/domain-set.conf" <<'EOF' +# source comment +.example.com +exact.example.org +EOF +output="$(python3 "$ROOT/scripts/tools/merge-domain-rule-source.py" \ + "$TMP_DIR/domain-set.conf" "$TMP_DIR/target.list" "$TMP_DIR/normalized.list" \ + --format domain-set)" +grep -q 'source=2 skipped=0 excluded=0 added=2 compacted=1' <<<"$output" +grep -qx 'DOMAIN-SUFFIX,example.com' "$TMP_DIR/target.list" +grep -qx 'DOMAIN,keep.example.net' "$TMP_DIR/target.list" +if grep -q 'api.example.com' "$TMP_DIR/target.list"; then + echo "test failed: suffix-covered exact rule was not compacted" >&2 + exit 1 +fi +grep -qx 'DOMAIN,exact.example.org' "$TMP_DIR/normalized.list" + +cat > "$TMP_DIR/classical.conf" <<'EOF' +DOMAIN,new.example.net +PROCESS-NAME,unsupported +EOF +if python3 "$ROOT/scripts/tools/merge-domain-rule-source.py" \ + "$TMP_DIR/classical.conf" "$TMP_DIR/target.list" "$TMP_DIR/classical.list" \ + --format classical >/dev/null 2>&1; then + echo "test failed: unsupported classical rule type was accepted" >&2 + exit 1 +fi + +cat > "$TMP_DIR/filtered.conf" <<'EOF' +DOMAIN,7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe +DOMAIN-SUFFIX,kept.example +DOMAIN-WILDCARD,stun.*.* +EOF +output="$(python3 "$ROOT/scripts/tools/merge-domain-rule-source.py" \ + "$TMP_DIR/filtered.conf" "$TMP_DIR/target.list" "$TMP_DIR/filtered.list" \ + --format classical \ + --ignore-domain 7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe \ + --allow-unsupported-rule 'DOMAIN-WILDCARD,stun.*.*')" +grep -q 'source=1 skipped=2 excluded=0 added=1' <<<"$output" +grep -qx 'DOMAIN-SUFFIX,kept.example' "$TMP_DIR/filtered.list" + +cat > "$TMP_DIR/unreviewed-unsupported.conf" <<'EOF' +DOMAIN-SUFFIX,kept.example +DOMAIN-WILDCARD,new.*.* +EOF +if python3 "$ROOT/scripts/tools/merge-domain-rule-source.py" \ + "$TMP_DIR/unreviewed-unsupported.conf" "$TMP_DIR/target.list" "$TMP_DIR/unreviewed.list" \ + --format classical \ + --allow-unsupported-rule 'DOMAIN-WILDCARD,stun.*.*' >/dev/null 2>&1; then + echo "test failed: unreviewed rule of an allowlisted unsupported kind was accepted" >&2 + exit 1 +fi + +cat > "$TMP_DIR/marked-domain-set.conf" <<'EOF' +7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe +.domain-set.example +EOF +output="$(python3 "$ROOT/scripts/tools/merge-domain-rule-source.py" \ + "$TMP_DIR/marked-domain-set.conf" "$TMP_DIR/target.list" "$TMP_DIR/marked-domain-set.list" \ + --format domain-set \ + --ignore-domain 7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe)" +grep -q 'source=1 skipped=1 excluded=0 added=1' <<<"$output" +if grep -q '7h1s_rul35et' "$TMP_DIR/marked-domain-set.list" "$TMP_DIR/target.list"; then + echo "test failed: Sukka marker leaked from domain-set source" >&2 + exit 1 +fi + +cat > "$TMP_DIR/exclusions.list" <<'EOF' +DOMAIN-SUFFIX,blocked.example +DOMAIN,ad.broad.example +EOF +cat > "$TMP_DIR/excluded-source.conf" <<'EOF' +DOMAIN,sub.blocked.example +DOMAIN-SUFFIX,broad.example +DOMAIN-SUFFIX,allowed.example +EOF +output="$(python3 "$ROOT/scripts/tools/merge-domain-rule-source.py" \ + "$TMP_DIR/excluded-source.conf" "$TMP_DIR/target.list" "$TMP_DIR/excluded.list" \ + --format classical --exclude-file "$TMP_DIR/exclusions.list")" +grep -q 'source=3 skipped=0 excluded=2 added=1' <<<"$output" +if grep -q 'blocked.example' "$TMP_DIR/excluded.list"; then + echo "test failed: excluded rule was normalized" >&2 + exit 1 +fi + +echo "Sukka domain merge tests passed" diff --git a/scripts/tests/test-sync-upstream-render-matrix.sh b/scripts/tests/test-sync-upstream-render-matrix.sh index e442f217..7843b197 100755 --- a/scripts/tests/test-sync-upstream-render-matrix.sh +++ b/scripts/tests/test-sync-upstream-render-matrix.sh @@ -50,29 +50,59 @@ for snippet in summary_required_snippets: if snippet not in script: raise SystemExit(f"test failed: sync-upstream summary lifecycle missing {snippet!r}") -loyalsoldier_required_snippets = [ - 'LOYALSOLDIER_GEOIP_CN_SOURCE_URL="$(upstream_value ip loyalsoldier-geoip-cn url)"', - 'LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL="$(upstream_value ip loyalsoldier-geoip-private url)"', - 'download_file "$LOYALSOLDIER_GEOIP_CN_SOURCE_URL" "$IP_BUILD_TMP_DIR/loyalsoldier_geoip_cn.raw.txt"', - 'download_file "$LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL" "$IP_BUILD_TMP_DIR/private.raw.txt"', - 'text "$tmp_dir/loyalsoldier_geoip_cn.raw.txt" "$tmp_dir/loyalsoldier_geoip_cn.cidr.txt"', - 'text "$tmp_dir/private.raw.txt" "$tmp_dir/private.cidr.txt"', +ip_source_required_snippets = [ + 'mapfile -t UPSTREAM_SETTINGS', + 'BUILTIN_PRIVATE_SOURCE_FILE="$ROOT_DIR/sources/builtin/ip/private.list"', + 'download_files_parallel', + 'SUKKA_MARKER_DOMAIN="7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe"', + '"$BUILTIN_PRIVATE_SOURCE_FILE"', + '"$IP_BUILD_TMP_DIR/private.cidr.txt"', 'verify_and_record_upstream_health', - 'loyalsoldier-geoip-cn|$LOYALSOLDIER_GEOIP_CN_SOURCE_URL|loyalsoldier_geoip_cn.raw.txt|loyalsoldier_geoip_cn.cidr.txt|0', - 'loyalsoldier-geoip-private|$LOYALSOLDIER_GEOIP_PRIVATE_SOURCE_URL|private.raw.txt|private.cidr.txt|0', - '"$IP_BUILD_TMP_DIR/loyalsoldier_geoip_cn.cidr.txt"', + 'prepare_ripe_stat_asns()', + 'raw_json="$IP_BUILD_TMP_DIR/ripe_as${asn}.raw.json"', + 'cidr_txt="$IP_BUILD_TMP_DIR/ripe_as${asn}.cidr.txt"', + 'download_args+=("ripe-stat-as${asn}" required "${RIPE_STAT_BASE_URL}${asn}" "$raw_json")', + 'download_files_parallel "${download_args[@]}"', + 'prepare_ripe_stat_asns \\', ] -for snippet in loyalsoldier_required_snippets: +for snippet in ip_source_required_snippets: if snippet not in script: - raise SystemExit(f"test failed: sync-upstream missing Loyalsoldier IP snippet {snippet!r}") + raise SystemExit(f"test failed: sync-upstream missing optimized IP source snippet {snippet!r}") + +for excluded in ( + "LOYALSOLDIER_GEOIP_CN_SOURCE_URL", + "loyalsoldier_geoip_cn", + "SUKKA_CHINA_IPV4_SOURCE_URL", + "SUKKA_CHINA_IPV6_SOURCE_URL", + "sukka_china_ipv4", + "sukka_china_ipv6", +): + if excluded in script: + raise SystemExit(f"test failed: low-marginal China GeoIP source was reintroduced: {excluded}") domain_required_snippets = [ 'clone_repository_shallow "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-community"', - 'LOYALSOLDIER_CHINA_LIST_SOURCE_URL="$(upstream_value domain loyalsoldier-china-list url)"', - 'download_file "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt"', - '"$DOMAIN_RULE_TMP_DIR/cn.list"', - '"$DOMAIN_RULE_TMP_DIR/geolocation-cn.list"', + 'LOYALSOLDIER_CHINA_LIST_SOURCE_URL="${UPSTREAM_SETTINGS[1]}"', + 'loyalsoldier-china-list required "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt"', + ': > "$DOMAIN_RULE_TMP_DIR/china-list.list"', + '"$DOMAIN_RULE_TMP_DIR/china-list.list"', '--normalized-output "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list"', + 'sukka-apple-intelligence|$SUKKA_APPLE_INTELLIGENCE_SOURCE_URL|classical|apple-intelligence', + 'sukka-icloud-private-relay|$SUKKA_ICLOUD_PRIVATE_RELAY_SOURCE_URL|domain-set|icloudprivaterelay', + 'sukka-game-download|$SUKKA_GAME_DOWNLOAD_SOURCE_URL|domain-set|category-game-platforms-download', + 'sukka-domestic required "$SUKKA_DOMESTIC_SOURCE_URL" "$WORK_TMP_DIR/sukka-domestic.raw.conf"', + 'sukka-ai required "$SUKKA_AI_SOURCE_URL" "$WORK_TMP_DIR/sukka-ai.raw.conf"', + 'sukka-apple-cdn|$SUKKA_APPLE_CDN_SOURCE_URL|domain-set|apple@cn', + 'sukka-apple-cn|$SUKKA_APPLE_CN_SOURCE_URL|classical|apple@cn', + 'sukka-microsoft-cdn|$SUKKA_MICROSOFT_CDN_SOURCE_URL|classical|microsoft@cn', + '--allow-unsupported-rule DOMAIN-KEYWORD,openai', + '--allow-unsupported-rule DOMAIN-KEYWORD,alkalimakersuite-pa.clients6.google.com', + '"$DOMAIN_RULE_TMP_DIR/cn.list"', + "--allow-unsupported-rule 'DOMAIN-WILDCARD,*.qhimgs?.com'", + '--exclude-file "$DOMAIN_RULE_TMP_DIR/category-ads-all.list"', + 'sukka-apple-services required "$SUKKA_APPLE_SERVICES_SOURCE_URL" "$IP_BUILD_TMP_DIR/sukka_apple_services.raw.txt"', + '--ignore-domain "$SUKKA_MARKER_DOMAIN"', + '"$IP_BUILD_TMP_DIR/sukka_apple_services.cidr.txt"', '"$WORK_TMP_DIR/domain-list-community/data"', 'assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE"', '"apple@cn"', @@ -86,6 +116,56 @@ for snippet in domain_required_snippets: if snippet not in script: raise SystemExit(f"test failed: sync-upstream missing domain derivative guard snippet {snippet!r}") +expected_sukka_merge_targets = { + "sukka-apple-intelligence": "apple-intelligence", + "sukka-icloud-private-relay": "icloudprivaterelay", + "sukka-game-download": "category-game-platforms-download", + "sukka-apple-cdn": "apple@cn", + "sukka-apple-cn": "apple@cn", + "sukka-microsoft-cdn": "microsoft@cn", +} +actual_sukka_merge_targets = dict( + re.findall(r"^(sukka-[^|]+)\|\$[^|]+\|[^|]+\|([^|\n]+)$", script, re.MULTILINE) +) +if actual_sukka_merge_targets != expected_sukka_merge_targets: + raise SystemExit( + "test failed: focused Sukka sources must merge into existing domain targets: " + f"{actual_sukka_merge_targets!r}" + ) + +for source_file, target_file in ( + ("sukka-ai.raw.conf", "category-ai-!cn.list"), + ("sukka-domestic.raw.conf", "cn.list"), +): + merge_pattern = re.compile( + r'merge-domain-rule-source\.py"(?P.*?)verify_and_record_upstream_health', + re.DOTALL, + ) + if not any( + source_file in match.group("body") and target_file in match.group("body") + for match in merge_pattern.finditer(script) + ): + raise SystemExit(f"test failed: {source_file} is not merged into existing target {target_file}") + +if re.search(r'\$DOMAIN_RULE_TMP_DIR/(?:\$\{)?sukka[-_][^"}]*\.list', script): + raise SystemExit("test failed: a focused Sukka source is published as a standalone domain artifact") + +apple_ip_merge_start = script.index('"$IP_BUILD_TMP_DIR/apple.merged.cidr.txt"') +apple_ip_merge_body = script[apple_ip_merge_start : apple_ip_merge_start + 300] +if "sukka_apple_services.cidr.txt" not in apple_ip_merge_body: + raise SystemExit("test failed: Sukka Apple IP source must merge into the existing apple artifact") + +china_merge = re.search( + r'merge-domain-suffixes\.py"(?P.*?)--normalized-output', + script, + re.DOTALL, +) +if not china_merge or 'china-list.list' not in china_merge.group("body"): + raise SystemExit("test failed: China List must produce its independent DNS artifact") +for routing_target in ('cn.list', 'geolocation-cn.list'): + if routing_target in china_merge.group("body"): + raise SystemExit(f"test failed: DNS-only China List leaked into routing target {routing_target}") + asn_function_match = re.search( r"sync_asn_ip_list\(\) \{(?P.*?)\n\}", script, diff --git a/scripts/tests/test-upstream-config.sh b/scripts/tests/test-upstream-config.sh index b0033b8d..f00d6408 100755 --- a/scripts/tests/test-upstream-config.sh +++ b/scripts/tests/test-upstream-config.sh @@ -49,6 +49,69 @@ if china.get("parser") != "domain-suffix-text": raise SystemExit("test failed: China List must use domain-suffix-text parser") if not china.get("url", "").endswith("/release/china-list.txt"): raise SystemExit("test failed: China List URL must use the narrow release text artifact") +domain_sources = json.loads(Path("config/upstreams.json").read_text(encoding="utf-8"))["domain"] +expected_sukka_domain_sources = { + "sukka-apple-intelligence": "sukka-classical-domain", + "sukka-icloud-private-relay": "sukka-domain-set-text", + "sukka-game-download": "sukka-domain-set-text", + "sukka-domestic": "sukka-classical-domain", + "sukka-ai": "sukka-classical-domain", + "sukka-apple-cdn": "sukka-domain-set-text", + "sukka-apple-cn": "sukka-classical-domain", + "sukka-microsoft-cdn": "sukka-classical-domain", +} +for name, parser in expected_sukka_domain_sources.items(): + source = domain_sources.get(name, {}) + if source.get("parser") != parser or not source.get("url", "").startswith("https://ruleset.skk.moe/"): + raise SystemExit(f"test failed: invalid focused Sukka domain source {name}") +ip_sources = json.loads(Path("config/upstreams.json").read_text(encoding="utf-8"))["ip"] +if ip_sources.get("cn-ipv46", {}).get("trust") != "community": + raise SystemExit("test failed: IPNetDB-derived China IP source must use community trust") +if ip_sources.get("cn-ipv46-apnic", {}).get("trust") != "registry": + raise SystemExit("test failed: APNIC-derived China IP source must retain registry trust") +if "loyalsoldier-geoip-cn" in ip_sources: + raise SystemExit("test failed: low-marginal China GeoIP source must remain excluded") +if "loyalsoldier-geoip-private" in ip_sources: + raise SystemExit("test failed: static private ranges must not depend on a remote source") +sukka_apple = ip_sources.get("sukka-apple-services", {}) +if sukka_apple.get("parser") != "sukka-classical-ip" or not sukka_apple.get("url", "").startswith("https://ruleset.skk.moe/"): + raise SystemExit("test failed: invalid focused Sukka Apple IP source") +for excluded in ("sukka-china-ipv4", "sukka-china-ipv6"): + if excluded in ip_sources: + raise SystemExit(f"test failed: low-marginal Sukka China IP source was reintroduced: {excluded}") +asn_groups = json.loads(Path("config/upstreams.json").read_text(encoding="utf-8"))["asn_groups"] +if asn_groups["telegram"] != [62041]: + raise SystemExit("test failed: Telegram must keep only the ASN with coverage beyond its official CIDR list") +if asn_groups["netflix"] != [2906]: + raise SystemExit("test failed: Netflix must exclude AS40027 because AS2906 fully covers it") +PY + +python3 - <<'PY' +from pathlib import Path + +expected = { + "IP-CIDR,0.0.0.0/8", + "IP-CIDR,10.0.0.0/8", + "IP-CIDR,100.64.0.0/10", + "IP-CIDR,127.0.0.0/8", + "IP-CIDR,169.254.0.0/16", + "IP-CIDR,172.16.0.0/12", + "IP-CIDR,192.0.0.0/24", + "IP-CIDR,192.0.2.0/24", + "IP-CIDR,192.88.99.0/24", + "IP-CIDR,192.168.0.0/16", + "IP-CIDR,198.18.0.0/15", + "IP-CIDR,198.51.100.0/24", + "IP-CIDR,203.0.113.0/24", + "IP-CIDR,224.0.0.0/3", + "IP-CIDR6,::/127", + "IP-CIDR6,fc00::/7", + "IP-CIDR6,fe80::/10", + "IP-CIDR6,ff00::/8", +} +actual = set(Path("sources/builtin/ip/private.list").read_text(encoding="utf-8").splitlines()) +if actual != expected: + raise SystemExit("test failed: built-in private ranges changed without updating the reviewed baseline") PY cp config/upstreams.json "$TMP_DIR/upstreams.invalid-url.json" @@ -115,6 +178,32 @@ assert_lint_fails_with \ "upstreams.ip.github.kind: must equal 'json' for source 'github', got 'text'" \ --upstreams "$TMP_DIR/upstreams.wrong-kind.json" +cp config/upstreams.json "$TMP_DIR/upstreams.unused.json" +python3 - <<'PY' "$TMP_DIR/upstreams.unused.json" +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +data = json.loads(path.read_text(encoding="utf-8")) +data["domain"]["unused-domain"] = data["domain"]["sukka-ai"].copy() +data["ip"]["unused-ip"] = data["ip"]["cloudflare-ipv4"].copy() +data["asn_groups"]["unused-group"] = [64512] +path.write_text(json.dumps(data), encoding="utf-8") +PY +assert_lint_fails_with \ + "unused-domain" \ + "upstreams.domain: unsupported sources: ['unused-domain']" \ + --upstreams "$TMP_DIR/upstreams.unused.json" +assert_lint_fails_with \ + "unused-ip" \ + "upstreams.ip: unsupported sources: ['unused-ip']" \ + --upstreams "$TMP_DIR/upstreams.unused.json" +assert_lint_fails_with \ + "unused-asn-group" \ + "upstreams.asn_groups: unsupported groups: ['unused-group']" \ + --upstreams "$TMP_DIR/upstreams.unused.json" + cp config/upstreams.json "$TMP_DIR/upstreams.cloudfront-url.json" python3 - <<'PY' "$TMP_DIR/upstreams.cloudfront-url.json" import json diff --git a/scripts/tools/export-domain-rules.py b/scripts/tools/export-domain-rules.py index 832dff6f..fa57a2fb 100644 --- a/scripts/tools/export-domain-rules.py +++ b/scripts/tools/export-domain-rules.py @@ -47,6 +47,11 @@ } REGIONAL_ATTRS = ("!cn", "cn") +BASE_RULE_EXCLUDED_ATTRS = { + "cn": frozenset({"!cn", "ads"}), + "geolocation-cn": frozenset({"!cn", "ads"}), + "geolocation-!cn": frozenset({"cn", "ads"}), +} CAPABILITY_SCHEMA: PlatformCapabilities = load_platform_capabilities(CAPABILITIES_FILE) @@ -243,6 +248,13 @@ def collect_rule_set_output(rules: list[Rule]) -> RuleSetOutput: return RuleSetOutput(unique_rules, rules_by_attr) +def filter_base_rule_set(name: str, rules: list[Rule]) -> list[Rule]: + excluded_attrs = BASE_RULE_EXCLUDED_ATTRS.get(name) + if not excluded_attrs: + return rules + return [rule for rule in rules if excluded_attrs.isdisjoint(rule.attrs)] + + def regional_suffix_for_rule_set_name(name: str) -> str | None: if name == "cn": return "cn" @@ -543,8 +555,10 @@ def write_rule_set(name: str, rules: list[Rule]) -> None: (output_dir / f"{name}.list").write_text("\n".join(rendered) + "\n", encoding="utf-8") for name in names: - rule_set_output = collect_rule_set_output(resolve(name)) - write_rule_set(name, rule_set_output.rules) + resolved_rules = resolve(name) + rule_set_output = collect_rule_set_output(resolved_rules) + base_output = collect_rule_set_output(filter_base_rule_set(name, resolved_rules)) + write_rule_set(name, base_output.rules) for attr, rules in sorted(rule_set_output.rules_by_attr.items()): if is_redundant_attr_rule_set_name(name, attr): diff --git a/scripts/tools/lint-config.py b/scripts/tools/lint-config.py index 6d5dc135..cf6cdc88 100644 --- a/scripts/tools/lint-config.py +++ b/scripts/tools/lint-config.py @@ -17,12 +17,18 @@ "domain": { "dlc": ("git", "git-tree"), "loyalsoldier-china-list": ("text", "domain-suffix-text"), + "sukka-apple-intelligence": ("text", "sukka-classical-domain"), + "sukka-icloud-private-relay": ("text", "sukka-domain-set-text"), + "sukka-game-download": ("text", "sukka-domain-set-text"), + "sukka-domestic": ("text", "sukka-classical-domain"), + "sukka-ai": ("text", "sukka-classical-domain"), + "sukka-apple-cdn": ("text", "sukka-domain-set-text"), + "sukka-apple-cn": ("text", "sukka-classical-domain"), + "sukka-microsoft-cdn": ("text", "sukka-classical-domain"), }, "ip": { "cn-ipv46": ("text", "cidr-text"), "cn-ipv46-apnic": ("text", "cidr-text"), - "loyalsoldier-geoip-cn": ("text", "cidr-text"), - "loyalsoldier-geoip-private": ("text", "cidr-text"), "google": ("json", "google-json"), "telegram": ("text", "telegram"), "cloudflare-ipv4": ("text", "cidr-text"), @@ -32,6 +38,7 @@ "fastly": ("json", "fastly-json"), "github": ("json", "github-json"), "apple": ("html", "html-cidr"), + "sukka-apple-services": ("text", "sukka-classical-ip"), "ripe-stat": ("json-api", "ripe-stat-json"), }, } @@ -40,7 +47,7 @@ REQUIRED_ASN_GROUPS = {"telegram", "netflix", "spotify", "disney"} REQUIRED_FIRST_BATCH_SOURCES = {"google-json", "github-json", "telegram"} SUPPORTED_PARSERS = { - "git-tree", "domain-suffix-text", "cidr-text", "google-json", "github-json", "telegram", + "git-tree", "domain-suffix-text", "classical-domain", "sukka-classical-domain", "domain-set-text", "sukka-domain-set-text", "cidr-text", "classical-ip", "sukka-classical-ip", "google-json", "github-json", "telegram", "aws-json", "aws-cloudfront-json", "fastly-json", "html-cidr", "ripe-stat-json", } ALLOWED_REQUIREMENTS = {"required", "optional"} @@ -209,12 +216,21 @@ def validate_upstreams(data: dict, reporter: Reporter) -> None: missing_domain = REQUIRED_DOMAIN_SOURCES - set(domain) missing_ip = REQUIRED_IP_SOURCES - set(ip) missing_asn_groups = REQUIRED_ASN_GROUPS - set(asn_groups) + unexpected_domain = set(domain) - REQUIRED_DOMAIN_SOURCES + unexpected_ip = set(ip) - REQUIRED_IP_SOURCES + unexpected_asn_groups = set(asn_groups) - REQUIRED_ASN_GROUPS if missing_domain: reporter.error("upstreams.domain", f"missing required sources: {sorted(missing_domain)}") if missing_ip: reporter.error("upstreams.ip", f"missing required sources: {sorted(missing_ip)}") if missing_asn_groups: reporter.error("upstreams.asn_groups", f"missing required groups: {sorted(missing_asn_groups)}") + if unexpected_domain: + reporter.error("upstreams.domain", f"unsupported sources: {sorted(unexpected_domain)}") + if unexpected_ip: + reporter.error("upstreams.ip", f"unsupported sources: {sorted(unexpected_ip)}") + if unexpected_asn_groups: + reporter.error("upstreams.asn_groups", f"unsupported groups: {sorted(unexpected_asn_groups)}") for name, item in sorted(domain.items()): validate_source("domain", name, item, reporter) diff --git a/scripts/tools/merge-domain-rule-source.py b/scripts/tools/merge-domain-rule-source.py new file mode 100644 index 00000000..ef879a75 --- /dev/null +++ b/scripts/tools/merge-domain-rule-source.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +from pathlib import Path + +from domain_rules import ( + ParsedDomainRule, + compact_domain_rules, + domain_value_errors, + normalize_domain_value, + normalize_rule_type, + parse_classical_domain_file, +) + + +SUPPORTED_FORMATS = {"classical", "domain-set"} +SUPPORTED_KINDS = {"DOMAIN", "DOMAIN-SUFFIX"} + + +def parse_source( + path: Path, + source_format: str, + allowed_unsupported_rules: set[tuple[str, str]], + ignored_domains: set[str], +) -> tuple[list[ParsedDomainRule], int]: + rules: list[ParsedDomainRule] = [] + seen: set[tuple[str, str]] = set() + skipped = 0 + + for line_no, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + + if source_format == "classical": + if "," not in line: + raise ValueError(f"{path}:{line_no} invalid classical domain rule: {line}") + kind_raw, value_raw = line.split(",", 1) + kind = normalize_rule_type(kind_raw) + value = value_raw.split(",", 1)[0].strip() + if kind not in SUPPORTED_KINDS: + unsupported_key = (kind, value.lower().rstrip(".")) + if unsupported_key in allowed_unsupported_rules: + skipped += 1 + continue + raise ValueError(f"{path}:{line_no} unsupported domain rule type: {kind}") + else: + if "," in line: + raise ValueError(f"{path}:{line_no} invalid domain-set rule: {line}") + if line.startswith("+."): + kind, value = "DOMAIN-SUFFIX", line[2:] + elif line.startswith("."): + kind, value = "DOMAIN-SUFFIX", line[1:] + else: + kind, value = "DOMAIN", line + + value = normalize_domain_value(kind, value) + if value in ignored_domains: + skipped += 1 + continue + errors = domain_value_errors( + kind, + value, + require_canonical=True, + allow_single_label_suffix=False, + ) + if errors: + raise ValueError(f"{path}:{line_no} {'; '.join(errors)}") + key = (kind, value) + if key not in seen: + seen.add(key) + rules.append(ParsedDomainRule(kind, value, line_no)) + + if not rules: + raise ValueError(f"{path}: source contains no supported domain rules") + return rules, skipped + + +def merge_rules(target: Path, additions: list[ParsedDomainRule]) -> tuple[int, int]: + current, errors = parse_classical_domain_file( + target, + require_canonical=True, + allow_single_label_suffix=True, + ) + if errors: + raise ValueError("\n".join(errors)) + + existing = {(rule.kind, rule.value) for rule in current} + appended = [rule for rule in additions if (rule.kind, rule.value) not in existing] + compacted, removed = compact_domain_rules(current + appended) + target.write_text("\n".join(rule.text for rule in compacted) + "\n", encoding="utf-8") + return len(appended), removed + + +def rule_conflicts_with_exclusion(rule: ParsedDomainRule, exclusions: list[ParsedDomainRule]) -> bool: + for excluded in exclusions: + if excluded.kind == "DOMAIN-SUFFIX" and ( + rule.value == excluded.value or rule.value.endswith(f".{excluded.value}") + ): + return True + if excluded.kind == "DOMAIN" and rule.kind == "DOMAIN" and rule.value == excluded.value: + return True + if rule.kind == "DOMAIN-SUFFIX" and ( + excluded.value == rule.value or excluded.value.endswith(f".{rule.value}") + ): + return True + return False + + +def parse_allowed_unsupported_rule(value: str) -> tuple[str, str]: + kind_raw, separator, rule_value = value.partition(",") + kind = normalize_rule_type(kind_raw) + normalized_value = rule_value.strip().lower().rstrip(".") + if not separator or not kind or not normalized_value or "," in rule_value: + raise ValueError(f"invalid unsupported-rule exception: {value!r}; expected KIND,VALUE") + if kind in SUPPORTED_KINDS: + raise ValueError(f"unsupported-rule exception uses supported kind: {kind}") + return kind, normalized_value + + +def main() -> int: + parser = argparse.ArgumentParser(description="Normalize and merge a narrow domain rule source.") + parser.add_argument("source") + parser.add_argument("target") + parser.add_argument("normalized_output") + parser.add_argument("--format", choices=sorted(SUPPORTED_FORMATS), required=True) + parser.add_argument("--allow-unsupported-rule", action="append", default=[]) + parser.add_argument("--ignore-domain", action="append", default=[]) + parser.add_argument("--exclude-file", action="append", default=[]) + args = parser.parse_args() + + allowed_unsupported_rules = { + parse_allowed_unsupported_rule(value) for value in args.allow_unsupported_rule + } + ignored_domains = {value.lower().rstrip(".") for value in args.ignore_domain} + rules, skipped = parse_source( + Path(args.source), + args.format, + allowed_unsupported_rules, + ignored_domains, + ) + exclusions: list[ParsedDomainRule] = [] + for exclude_name in args.exclude_file: + exclude_rules, errors = parse_classical_domain_file( + Path(exclude_name), + require_canonical=True, + allow_single_label_suffix=True, + ) + if errors: + raise ValueError("\n".join(errors)) + exclusions.extend(exclude_rules) + selected = [rule for rule in rules if not rule_conflicts_with_exclusion(rule, exclusions)] + excluded_count = len(rules) - len(selected) + + normalized = Path(args.normalized_output) + normalized.parent.mkdir(parents=True, exist_ok=True) + normalized.write_text("\n".join(rule.text for rule in selected) + "\n", encoding="utf-8") + added, removed = merge_rules(Path(args.target), selected) + print( + f"source={len(rules)} skipped={skipped} excluded={excluded_count} " + f"added={added} compacted={removed}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tools/normalize-ip-rules.py b/scripts/tools/normalize-ip-rules.py index 614c55d1..afa4a0c2 100644 --- a/scripts/tools/normalize-ip-rules.py +++ b/scripts/tools/normalize-ip-rules.py @@ -138,6 +138,45 @@ def extract_text_cidrs(input_file: Path, output_file: Path) -> None: write_deduplicated_cidrs(lines, output_file) +def extract_classical_ip_cidrs( + input_file: Path, + output_file: Path, + *, + allow_sukka_marker: bool = False, +) -> None: + values: list[str] = [] + for line_no, raw_line in enumerate(input_file.read_text(encoding="utf-8").splitlines(), start=1): + line = raw_line.split("#", 1)[0].strip() + if not line: + continue + fields = [field.strip() for field in line.split(",")] + if allow_sukka_marker and fields == [ + "DOMAIN", + "7h1s_rul35et_i5_mad3_by_5ukk4w-ruleset.skk.moe", + ]: + continue + if len(fields) not in {2, 3} or (len(fields) == 3 and fields[2] != "no-resolve"): + raise ValueError(f"{input_file}:{line_no} invalid classical IP rule: {line}") + kind, value = fields[:2] + if kind not in {"IP-CIDR", "IP-CIDR6"}: + raise ValueError(f"{input_file}:{line_no} invalid classical IP rule type: {kind}") + try: + network = ipaddress.ip_network(value, strict=False) + except ValueError as exc: + raise ValueError(f"{input_file}:{line_no} invalid CIDR value: {value}") from exc + expected_kind = "IP-CIDR6" if network.version == 6 else "IP-CIDR" + if kind != expected_kind: + raise ValueError( + f"{input_file}:{line_no} {kind} has wrong address family for {value}" + ) + values.append(value) + write_deduplicated_cidrs(values, output_file) + + +def extract_sukka_classical_ip_cidrs(input_file: Path, output_file: Path) -> None: + extract_classical_ip_cidrs(input_file, output_file, allow_sukka_marker=True) + + def require_object_list(data: object, field: str, source: Path) -> list[dict]: if not isinstance(data, dict) or field not in data or not isinstance(data[field], list): raise ValueError(f"{source} missing required JSON array: {field}") @@ -313,6 +352,8 @@ def build_singbox_json_from_plain(input_file: Path, output_file: Path) -> None: def run_single_task(source_type: str, input_file: Path, output_file: Path) -> None: source_to_handler = { "text": extract_text_cidrs, + "classical-ip": extract_classical_ip_cidrs, + "sukka-classical-ip": extract_sukka_classical_ip_cidrs, "google-json": extract_google_json_cidrs, "aws-cloudfront-json": extract_aws_cloudfront_json_cidrs, "aws-json": extract_aws_all_json_cidrs, @@ -369,6 +410,8 @@ def run_batch_tasks(manifest_file: Path) -> None: def main() -> int: source_types = { "text", + "classical-ip", + "sukka-classical-ip", "google-json", "aws-cloudfront-json", "aws-json", diff --git a/sources/builtin/ip/private.list b/sources/builtin/ip/private.list new file mode 100644 index 00000000..bea173d2 --- /dev/null +++ b/sources/builtin/ip/private.list @@ -0,0 +1,18 @@ +IP-CIDR,0.0.0.0/8 +IP-CIDR,10.0.0.0/8 +IP-CIDR,100.64.0.0/10 +IP-CIDR,127.0.0.0/8 +IP-CIDR,169.254.0.0/16 +IP-CIDR,172.16.0.0/12 +IP-CIDR,192.0.0.0/24 +IP-CIDR,192.0.2.0/24 +IP-CIDR,192.88.99.0/24 +IP-CIDR,192.168.0.0/16 +IP-CIDR,198.18.0.0/15 +IP-CIDR,198.51.100.0/24 +IP-CIDR,203.0.113.0/24 +IP-CIDR,224.0.0.0/3 +IP-CIDR6,::/127 +IP-CIDR6,fc00::/7 +IP-CIDR6,fe80::/10 +IP-CIDR6,ff00::/8