feat: enrich China domains with optimized merge

README.md

@@ -34,7 +34,7 @@ https://raw.githubusercontent.com/KuGouGo/Rules/sing-box/domain/fakeip-filter.sr
 https://raw.githubusercontent.com/KuGouGo/Rules/mihomo/ip/google.mrs
 ```
 
-常用规则包括 `cn`、`geolocation-cn`、`geolocation-!cn`、`google`、`telegram`、`emby-cn`、`emby` 和 `fakeip-filter`。同时使用 `emby-cn` 与 `emby` 时，应先加载范围较小的 `emby-cn`。
+常用规则包括 `cn`、`geolocation-cn`、`geolocation-!cn`、`google`、`telegram`、`emby-cn`、`emby` 和 `fakeip-filter`。其中 `cn` 与 `geolocation-cn` 在 `domain-list-community` 基础上合并 Loyalsoldier 发布的 `china-list.txt`，补充中国大陆可直连的长尾域名；不合并其宽泛的 `direct/proxy/reject` 列表。同时使用 `emby-cn` 与 `emby` 时，应先加载范围较小的 `emby-cn`。
 
 ## 添加自定义规则
 

THIRD_PARTY_NOTICES.md

@@ -5,6 +5,7 @@
 | 内容 | 来源 |
 | --- | --- |
 | 域名规则 | [v2fly/domain-list-community](https://github.com/v2fly/domain-list-community)（MIT） |
+| 中国大陆可直连域名补充 | [Loyalsoldier/v2ray-rules-dat](https://github.com/Loyalsoldier/v2ray-rules-dat) 发布的 `china-list.txt`（GPL-3.0；数据源为 [felixonmars/dnsmasq-china-list](https://github.com/felixonmars/dnsmasq-china-list)） |
 | 中国 IP | <https://ispip.clang.cn/all_cn_ipv46.txt>、<https://ispip.clang.cn/all_cn_ipv46_apnic.txt>、[Loyalsoldier/geoip](https://github.com/Loyalsoldier/geoip) |
 | Google IP | <https://www.gstatic.com/ipranges/goog.json> |
 | Telegram IP | <https://core.telegram.org/resources/cidr.txt> |

config/upstreams.json

@@ -6,6 +6,13 @@
       "parser": "git-tree",
       "url": "https://github.com/v2fly/domain-list-community.git",
       "health": { "requirement": "required", "min_raw_bytes": 1024, "min_entries": 1, "family": "any", "fallback_policy": "none" }
+    },
+    "loyalsoldier-china-list": {
+      "kind": "text",
+      "trust": "community",
+      "parser": "domain-suffix-text",
+      "url": "https://raw.githubusercontent.com/Loyalsoldier/v2ray-rules-dat/release/china-list.txt",
+      "health": { "requirement": "required", "min_raw_bytes": 1000000, "min_entries": 100000, "family": "any", "fallback_policy": "none" }
     }
   },
   "ip": {

scripts/commands/sync-upstream.sh

@@ -52,6 +52,7 @@ PY
 }
 
 DOMAIN_SOURCE_REPO_URL="$(upstream_value domain dlc url)"
+LOYALSOLDIER_CHINA_LIST_SOURCE_URL="$(upstream_value domain loyalsoldier-china-list url)"
 CN_IPV46_SOURCE_URL="$(upstream_value ip cn-ipv46 url)"
 CN_IPV46_APNIC_SOURCE_URL="$(upstream_value ip cn-ipv46-apnic url)"
 LOYALSOLDIER_GEOIP_CN_SOURCE_URL="$(upstream_value ip loyalsoldier-geoip-cn url)"
@@ -795,13 +796,7 @@ clone_repository_shallow "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-co
 python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" export \
   "$WORK_TMP_DIR/domain-list-community/data" \
   "$DOMAIN_RULE_TMP_DIR"
-python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" domain-rule-manifest \
-  "$DOMAIN_RULE_TMP_DIR" \
-  "$DOMAIN_RULE_MANIFEST_FILE"
-stage_domain_canonical_rules \
-  "$DOMAIN_RULE_TMP_DIR" \
-  "$CANONICAL_ARTIFACTS_DIR/domain"
-assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE"
+# Verify the DLC export before any supplemental source mutates the rule tree.
 verify_and_record_upstream_health \
   domain \
   dlc \
@@ -810,6 +805,33 @@ verify_and_record_upstream_health \
   "$DOMAIN_RULE_TMP_DIR" \
   0 \
   "commit=$(git -C "$WORK_TMP_DIR/domain-list-community" rev-parse HEAD)"
+
+# Supplement the DLC China categories with the independently maintained
+# dnsmasq-china-list export published by Loyalsoldier. Parse the 100k+ source
+# once, verify its normalized form independently, and update both targets via
+# a reverse-label trie (linear in domain label count).
+download_file "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt"
+python3 "$ROOT_DIR/scripts/tools/merge-domain-suffixes.py" \
+  "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" \
+  "$DOMAIN_RULE_TMP_DIR/cn.list" \
+  "$DOMAIN_RULE_TMP_DIR/geolocation-cn.list" \
+  --normalized-output "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list"
+verify_and_record_upstream_health \
+  domain \
+  loyalsoldier-china-list \
+  "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" \
+  "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt" \
+  "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list" \
+  0
+
+# Build the manifest and canonical tree once from the final merged rule set.
+python3 "$ROOT_DIR/scripts/tools/export-domain-rules.py" domain-rule-manifest \
+  "$DOMAIN_RULE_TMP_DIR" \
+  "$DOMAIN_RULE_MANIFEST_FILE"
+stage_domain_canonical_rules \
+  "$DOMAIN_RULE_TMP_DIR" \
+  "$CANONICAL_ARTIFACTS_DIR/domain"
+assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE"
 assert_files_present "$DOMAIN_RULE_TMP_DIR" "$DOMAIN_RULE_TMP_DIR/*.list"
 render_domain_rule_dir_to_text_platform_dirs \
   "$DOMAIN_RULE_TMP_DIR" \

scripts/tests/test-domain-suffix-merge.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -euo pipefail
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+TMP=$(mktemp -d); trap 'rm -rf "$TMP"' EXIT
+for target in one two; do
+  cat > "$TMP/$target.list" <<'EOF'
+DOMAIN-SUFFIX,example.cn
+DOMAIN,exact.cn
+EOF
+done
+cat > "$TMP/source.txt" <<'EOF'
+example.cn
+www.example.cn
+new.cn
+sub.new.cn
+INVALID DOMAIN
+EOF
+out=$(PYTHONPATH="$ROOT_DIR/scripts/tools" python3 "$ROOT_DIR/scripts/tools/merge-domain-suffixes.py" \
+  "$TMP/source.txt" "$TMP/one.list" "$TMP/two.list" --normalized-output "$TMP/normalized.list")
+grep -q 'candidate=4 invalid=1' <<<"$out"
+[ "$(grep -c 'added=1' <<<"$out")" -eq 2 ]
+for target in one two; do
+  grep -qx 'DOMAIN-SUFFIX,new.cn' "$TMP/$target.list"
+  if grep -q 'www.example.cn\|sub.new.cn\|INVALID' "$TMP/$target.list"; then
+    echo 'covered or invalid domain unexpectedly present' >&2; exit 1
+  fi
+done
+grep -qx 'DOMAIN-SUFFIX,sub.new.cn' "$TMP/normalized.list"
+python3 - "$TMP/one.list" <<'PY'
+import sys
+from pathlib import Path
+sys.path.insert(0,'scripts/tools')
+from domain_rules import parse_classical_domain_file
+_,e=parse_classical_domain_file(Path(sys.argv[1]),require_canonical=True,allow_single_label_suffix=True)
+assert not e,e
+PY
+echo PASS

scripts/tests/test-sync-upstream-render-matrix.sh

@@ -68,6 +68,11 @@ for snippet in loyalsoldier_required_snippets:
 
 domain_required_snippets = [
     'clone_repository_shallow "$DOMAIN_SOURCE_REPO_URL" "$WORK_TMP_DIR/domain-list-community"',
+    'LOYALSOLDIER_CHINA_LIST_SOURCE_URL="$(upstream_value domain loyalsoldier-china-list url)"',
+    'download_file "$LOYALSOLDIER_CHINA_LIST_SOURCE_URL" "$WORK_TMP_DIR/loyalsoldier-china-list.raw.txt"',
+    '"$DOMAIN_RULE_TMP_DIR/cn.list"',
+    '"$DOMAIN_RULE_TMP_DIR/geolocation-cn.list"',
+    '--normalized-output "$WORK_TMP_DIR/loyalsoldier-china-list.normalized.list"',
     '"$WORK_TMP_DIR/domain-list-community/data"',
     'assert_domain_attr_derivatives "$DOMAIN_RULE_MANIFEST_FILE"',
     '"apple@cn"',

scripts/tests/test-upstream-config.sh

@@ -44,6 +44,11 @@ if dlc.get("kind") != "git":
     raise SystemExit("test failed: domain.dlc must use the git source tree to preserve @attribute filters")
 if dlc.get("url") != "https://github.com/v2fly/domain-list-community.git":
     raise SystemExit("test failed: domain.dlc URL must point at domain-list-community.git")
+china = json.loads(Path("config/upstreams.json").read_text(encoding="utf-8"))["domain"]["loyalsoldier-china-list"]
+if china.get("parser") != "domain-suffix-text":
+    raise SystemExit("test failed: China List must use domain-suffix-text parser")
+if not china.get("url", "").endswith("/release/china-list.txt"):
+    raise SystemExit("test failed: China List URL must use the narrow release text artifact")
 PY
 
 cp config/upstreams.json "$TMP_DIR/upstreams.invalid-url.json"

scripts/tools/lint-config.py

@@ -16,6 +16,7 @@
 SOURCE_IMPLEMENTATIONS = {
     "domain": {
         "dlc": ("git", "git-tree"),
+        "loyalsoldier-china-list": ("text", "domain-suffix-text"),
     },
     "ip": {
         "cn-ipv46": ("text", "cidr-text"),
@@ -39,7 +40,7 @@
 REQUIRED_ASN_GROUPS = {"telegram", "netflix", "spotify", "disney"}
 REQUIRED_FIRST_BATCH_SOURCES = {"google-json", "github-json", "telegram"}
 SUPPORTED_PARSERS = {
-    "git-tree", "cidr-text", "google-json", "github-json", "telegram",
+    "git-tree", "domain-suffix-text", "cidr-text", "google-json", "github-json", "telegram",
     "aws-json", "aws-cloudfront-json", "fastly-json", "html-cidr", "ripe-stat-json",
 }
 ALLOWED_REQUIREMENTS = {"required", "optional"}

scripts/tools/merge-domain-suffixes.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+import argparse
+from pathlib import Path
+from domain_rules import domain_value_errors, parse_classical_domain_file
+
+_TERMINAL = ""
+
+class SuffixTrie:
+    def __init__(self, values=()):
+        self.root: dict = {}
+        for value in values: self.add(value)
+
+    def covers(self, value: str) -> bool:
+        node = self.root
+        for label in reversed(value.split('.')):
+            if _TERMINAL in node: return True
+            node = node.get(label)
+            if node is None: return False
+        return _TERMINAL in node
+
+    def add(self, value: str) -> None:
+        node = self.root
+        for label in reversed(value.split('.')):
+            if _TERMINAL in node: return
+            node = node.setdefault(label, {})
+        node.clear(); node[_TERMINAL] = True
+
+def load_plain(path: Path) -> tuple[list[str], int]:
+    values=set(); invalid=0
+    for raw in path.read_text(encoding='utf-8').splitlines():
+        value=raw.split('#',1)[0].strip().lower().rstrip('.')
+        if not value: continue
+        if domain_value_errors('DOMAIN-SUFFIX',value,require_canonical=True): invalid+=1; continue
+        values.add(value)
+    return sorted(values,key=lambda x:(x.count('.'),x)),invalid
+
+def merge_target(target: Path, candidates: list[str]) -> tuple[int,int]:
+    rules,errors=parse_classical_domain_file(target,require_canonical=True,allow_single_label_suffix=True)
+    if errors: raise ValueError('\n'.join(errors))
+    trie=SuffixTrie(r.value for r in rules if r.kind=='DOMAIN-SUFFIX')
+    additions=[]
+    for value in candidates:
+        if trie.covers(value): continue
+        trie.add(value); additions.append(value)
+    target.write_text('\n'.join([r.text for r in rules]+[f'DOMAIN-SUFFIX,{x}' for x in additions])+'\n',encoding='utf-8')
+    return len(additions),len(candidates)-len(additions)
+
+def main() -> int:
+    p=argparse.ArgumentParser(description='Merge a plain domain suffix source into classical rule files in one pass.')
+    p.add_argument('plain_source');p.add_argument('targets',nargs='+');p.add_argument('--normalized-output')
+    a=p.parse_args();candidates,invalid=load_plain(Path(a.plain_source))
+    if a.normalized_output:
+        Path(a.normalized_output).write_text('\n'.join(f'DOMAIN-SUFFIX,{x}' for x in candidates)+'\n',encoding='utf-8')
+    print(f'candidate={len(candidates)} invalid={invalid}')
+    for name in a.targets:
+        added,covered=merge_target(Path(name),candidates)
+        print(f'target={name} added={added} covered_or_duplicate={covered}')
+    return 0
+if __name__=='__main__':raise SystemExit(main())