diff --git a/README.md b/README.md index 1cbab666..f264bade 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ https://raw.githubusercontent.com/KuGouGo/Rules/sing-box/domain/fakeip-filter.sr https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/ip/google.mrs ``` -常用规则包括 `cn`、`geolocation-cn`、`geolocation-!cn`、`google`、`telegram`、`emby-cn`、`emby` 和 `fakeip-filter`。同时使用 `emby-cn` 与 `emby` 时，应先加载范围较小的 `emby-cn`。 +常用规则包括 `cn`、`geolocation-cn`、`geolocation-!cn`、`google`、`telegram`、`emby-cn`、`emby` 和 `fakeip-filter`。其中 `cn` 与 `geolocation-cn` 在 `domain-list-community` 基础上合并 Loyalsoldier 发布的 `china-list.txt`，补充中国大陆可直连的长尾域名；不合并其宽泛的 `direct/proxy/reject` 列表。同时使用 `emby-cn` 与 `emby` 时，应先加载范围较小的 `emby-cn`。 ## 添加自定义规则 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 0c9f4f1d..5edd35ff 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -5,6 +5,7 @@ | 内容 | 来源 | | --- | --- | | 域名规则 | [v2fly/domain-list-community](https://github.com/v2fly/domain-list-community)（MIT） | +| 中国大陆可直连域名补充 | [Loyalsoldier/v2ray-rules-dat](https://github.com/Loyalsoldier/v2ray-rules-dat) 发布的 `china-list.txt`（GPL-3.0；数据源为 [felixonmars/dnsmasq-china-list](https://github.com/felixonmars/dnsmasq-china-list)） | | 中国 IP | 、、[Loyalsoldier/geoip](https://github.com/Loyalsoldier/geoip) | | Google IP | | | Telegram IP | | diff --git a/config/upstreams.json b/config/upstreams.json index 083fb579..3f950904 100644 --- a/config/upstreams.json +++ b/config/upstreams.json @@ -6,6 +6,13 @@ "parser": "git-tree", "url": "https://github.com/v2fly/domain-list-community.git", "health": { "requirement": "required", "min_raw_bytes": 1024, "min_entries": 1, "family": "any", "fallback_policy": "none" } + }, + "loyalsoldier-china-list": { + "kind": "text", + "trust": "community", + "parser": "domain-suffix-text", + "url": "https://raw.githubusercontent.com/Loyalsoldier/v2ray-rules-dat/release/china-list.txt", + "health": { "requirement": "required", "min_raw_bytes": 1000000, "min_entries": 100000, "family": "any", "fallback_policy": "none" } } }, "ip": { diff --git a/scripts/commands/sync-upstream.sh b/scripts/commands/sync-upstream.sh index fadf7ddf..924ede7a 100755 --- a/scripts/commands/sync-upstream.sh +++ b/scripts/commands/sync-upstream.sh @@ -52,6 +52,7 @@ PY } DOMAIN_SOURCE_REPO_URL="$(upstream_value domain dlc url)" +LOYALSOLDIER_CHINA_LIST_SOURCE_URL="$(upstream_value domain loyalsoldier-china-list url)" CN_IPV46_SOURCE_URL="$(upstream_value ip cn-ipv46 url)" CN_IPV46_APNIC_SOURCE_URL="$(upstream_value ip cn-ipv46-apnic url)" LOYALSOLDIER_GEOIP_CN_SOURCE_URL="$(upstream_value ip loyalsoldier-geoip-cn url)" @@ -795,13 +796,7 @@ clone_repository_shallow "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-co python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" export \ "$WORK_TMP_DIR/domain-list-community/data" \ "$DOMAIN_RULE_TMP_DIR" -python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" domain-rule-manifest \ - "$DOMAIN_RULE_TMP_DIR" \ - "$DOMAIN_RULE_MANIFEST_FILE" -stage_domain_canonical_rules \ - "$DOMAIN_RULE_TMP_DIR" \ - "$CANONICAL_ARTIFACTS_DIR/domain" -assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE" +# Verify the DLC export before any supplemental source mutates the rule tree. verify_and_record_upstream_health \ domain \ dlc \ @@ -810,6 +805,33 @@ verify_and_record_upstream_health \ "$DOMAIN_RULE_TMP_DIR" \ 0 \ "commit=$(git -C "$WORK_TMP_DIR/domain-list-community" rev-parse HEAD)" + +# Supplement the DLC China categories with the independently maintained +# dnsmasq-china-list export published by Loyalsoldier. Parse the 100k+ source +# once, verify its normalized form independently, and update both targets via +# a reverse-label trie (linear in domain label count). +download_file "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" +python3 "$ROOT_DIR/scripts/tools/merge-domain-suffixes.py" \ + "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" \ + "$DOMAIN_RULE_TMP_DIR/cn.list" \ + "$DOMAIN_RULE_TMP_DIR/geolocation-cn.list" \ + --normalized-output "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list" +verify_and_record_upstream_health \ + domain \ + loyalsoldier-china-list \ + "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" \ + "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" \ + "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list" \ + 0 + +# Build the manifest and canonical tree once from the final merged rule set. +python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" domain-rule-manifest \ + "$DOMAIN_RULE_TMP_DIR" \ + "$DOMAIN_RULE_MANIFEST_FILE" +stage_domain_canonical_rules \ + "$DOMAIN_RULE_TMP_DIR" \ + "$CANONICAL_ARTIFACTS_DIR/domain" +assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE" assert_files_present "$DOMAIN_RULE_TMP_DIR" "$DOMAIN_RULE_TMP_DIR/*.list" render_domain_rule_dir_to_text_platform_dirs \ "$DOMAIN_RULE_TMP_DIR" \ diff --git a/scripts/tests/test-domain-suffix-merge.sh b/scripts/tests/test-domain-suffix-merge.sh new file mode 100644 index 00000000..7fbe7b47 --- /dev/null +++ b/scripts/tests/test-domain-suffix-merge.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT +for target in one two; do + cat > "$TMP/$target.list" <<'EOF' +DOMAIN-SUFFIX,example.cn +DOMAIN,exact.cn +EOF +done +cat > "$TMP/source.txt" <<'EOF' +example.cn +www.example.cn +new.cn +sub.new.cn +INVALID DOMAIN +EOF +out=$(PYTHONPATH="$ROOT_DIR/scripts/tools" python3 "$ROOT_DIR/scripts/tools/merge-domain-suffixes.py" \ + "$TMP/source.txt" "$TMP/one.list" "$TMP/two.list" --normalized-output "$TMP/normalized.list") +grep -q 'candidate=4 invalid=1' <<<"$out" +[ "$(grep -c 'added=1' <<<"$out")" -eq 2 ] +for target in one two; do + grep -qx 'DOMAIN-SUFFIX,new.cn' "$TMP/$target.list" + if grep -q 'www.example.cn\|sub.new.cn\|INVALID' "$TMP/$target.list"; then + echo 'covered or invalid domain unexpectedly present' >&2; exit 1 + fi +done +grep -qx 'DOMAIN-SUFFIX,sub.new.cn' "$TMP/normalized.list" +python3 - "$TMP/one.list" <<'PY' +import sys +from pathlib import Path +sys.path.insert(0,'scripts/tools') +from domain_rules import parse_classical_domain_file +_,e=parse_classical_domain_file(Path(sys.argv[1]),require_canonical=True,allow_single_label_suffix=True) +assert not e,e +PY +echo PASS diff --git a/scripts/tests/test-sync-upstream-render-matrix.sh b/scripts/tests/test-sync-upstream-render-matrix.sh index 14bd774d..e442f217 100755 --- a/scripts/tests/test-sync-upstream-render-matrix.sh +++ b/scripts/tests/test-sync-upstream-render-matrix.sh @@ -68,6 +68,11 @@ for snippet in loyalsoldier_required_snippets: domain_required_snippets = [ 'clone_repository_shallow "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-community"', + 'LOYALSOLDIER_CHINA_LIST_SOURCE_URL="$(upstream_value domain loyalsoldier-china-list url)"', + 'download_file "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt"', + '"$DOMAIN_RULE_TMP_DIR/cn.list"', + '"$DOMAIN_RULE_TMP_DIR/geolocation-cn.list"', + '--normalized-output "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list"', '"$WORK_TMP_DIR/domain-list-community/data"', 'assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE"', '"apple@cn"', diff --git a/scripts/tests/test-upstream-config.sh b/scripts/tests/test-upstream-config.sh index bed8bfda..b0033b8d 100755 --- a/scripts/tests/test-upstream-config.sh +++ b/scripts/tests/test-upstream-config.sh @@ -44,6 +44,11 @@ if dlc.get("kind") != "git": raise SystemExit("test failed: domain.dlc must use the git source tree to preserve @attribute filters") if dlc.get("url") != "https://github.com/v2fly/domain-list-community.git": raise SystemExit("test failed: domain.dlc URL must point at domain-list-community.git") +china = json.loads(Path("config/upstreams.json").read_text(encoding="utf-8"))["domain"]["loyalsoldier-china-list"] +if china.get("parser") != "domain-suffix-text": + raise SystemExit("test failed: China List must use domain-suffix-text parser") +if not china.get("url", "").endswith("/release/china-list.txt"): + raise SystemExit("test failed: China List URL must use the narrow release text artifact") PY cp config/upstreams.json "$TMP_DIR/upstreams.invalid-url.json" diff --git a/scripts/tools/lint-config.py b/scripts/tools/lint-config.py index ca144bd7..6d5dc135 100644 --- a/scripts/tools/lint-config.py +++ b/scripts/tools/lint-config.py @@ -16,6 +16,7 @@ SOURCE_IMPLEMENTATIONS = { "domain": { "dlc": ("git", "git-tree"), + "loyalsoldier-china-list": ("text", "domain-suffix-text"), }, "ip": { "cn-ipv46": ("text", "cidr-text"), @@ -39,7 +40,7 @@ REQUIRED_ASN_GROUPS = {"telegram", "netflix", "spotify", "disney"} REQUIRED_FIRST_BATCH_SOURCES = {"google-json", "github-json", "telegram"} SUPPORTED_PARSERS = { - "git-tree", "cidr-text", "google-json", "github-json", "telegram", + "git-tree", "domain-suffix-text", "cidr-text", "google-json", "github-json", "telegram", "aws-json", "aws-cloudfront-json", "fastly-json", "html-cidr", "ripe-stat-json", } ALLOWED_REQUIREMENTS = {"required", "optional"} diff --git a/scripts/tools/merge-domain-suffixes.py b/scripts/tools/merge-domain-suffixes.py new file mode 100644 index 00000000..5a4b8d4a --- /dev/null +++ b/scripts/tools/merge-domain-suffixes.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +from __future__ import annotations +import argparse +from pathlib import Path +from domain_rules import domain_value_errors, parse_classical_domain_file + +_TERMINAL = "" + +class SuffixTrie: + def __init__(self, values=()): + self.root: dict = {} + for value in values: self.add(value) + + def covers(self, value: str) -> bool: + node = self.root + for label in reversed(value.split('.')): + if _TERMINAL in node: return True + node = node.get(label) + if node is None: return False + return _TERMINAL in node + + def add(self, value: str) -> None: + node = self.root + for label in reversed(value.split('.')): + if _TERMINAL in node: return + node = node.setdefault(label, {}) + node.clear(); node[_TERMINAL] = True + +def load_plain(path: Path) -> tuple[list[str], int]: + values=set(); invalid=0 + for raw in path.read_text(encoding='utf-8').splitlines(): + value=raw.split('#',1)[0].strip().lower().rstrip('.') + if not value: continue + if domain_value_errors('DOMAIN-SUFFIX',value,require_canonical=True): invalid+=1; continue + values.add(value) + return sorted(values,key=lambda x:(x.count('.'),x)),invalid + +def merge_target(target: Path, candidates: list[str]) -> tuple[int,int]: + rules,errors=parse_classical_domain_file(target,require_canonical=True,allow_single_label_suffix=True) + if errors: raise ValueError('\n'.join(errors)) + trie=SuffixTrie(r.value for r in rules if r.kind=='DOMAIN-SUFFIX') + additions=[] + for value in candidates: + if trie.covers(value): continue + trie.add(value); additions.append(value) + target.write_text('\n'.join([r.text for r in rules]+[f'DOMAIN-SUFFIX,{x}' for x in additions])+'\n',encoding='utf-8') + return len(additions),len(candidates)-len(additions) + +def main() -> int: + p=argparse.ArgumentParser(description='Merge a plain domain suffix source into classical rule files in one pass.') + p.add_argument('plain_source');p.add_argument('targets',nargs='+');p.add_argument('--normalized-output') + a=p.parse_args();candidates,invalid=load_plain(Path(a.plain_source)) + if a.normalized_output: + Path(a.normalized_output).write_text('\n'.join(f'DOMAIN-SUFFIX,{x}' for x in candidates)+'\n',encoding='utf-8') + print(f'candidate={len(candidates)} invalid={invalid}') + for name in a.targets: + added,covered=merge_target(Path(name),candidates) + print(f'target={name} added={added} covered_or_duplicate={covered}') + return 0 +if __name__=='__main__':raise SystemExit(main())