ref-check

#!/usr/bin/env python3
# @ref https://github.com/ccheever/llp/blob/main/llp/0000-linked-literate-programming.explainer.md#6-validation-ref-check — implements the checker specified there
"""ref-check — deterministic validator for LLP corpora and @ref annotations.

Usage:
    ref-check [--root DIR] [--verbose]

Checks (broken = exit 1):
  * @ref targets resolve: LLP numbers, anchors, repo paths. URLs are
    shape-validated only (never fetched); shorthands are listed as
    unchecked unless the project defines a mapping.
  * LLP metadata headers parse; filename type matches **Type:**;
    LLP numbers are unique across the tree (tombstones included).
  * No [inferred] claim survives in an Accepted/Active document.

Judgment calls (stale glosses, drift, orphaned annotations) are
deliberately not gated — that is interactive work for llp-maintain.
Directories containing their own llp/ are separate corpus roots and are
skipped; check them with --root. llp/tombstones/ and llp/reviews/ are
exempt from ref validation.
"""

import argparse
import os
import re
import sys

SKIP_DIRS = {".git", ".hg", "node_modules", "__pycache__", ".venv", "venv",
             "dist", "build", "target", "coverage", ".cache", ".next",
             ".pytest_cache", ".turbo", ".idea", ".vscode"}
# `ref-check` is a source/document validator, not a generic text crawler.
# Keeping the scan surface explicit prevents machine-local build products and
# source maps from changing counts or inventing refs while retaining every
# source/config format used by this repository.
SCAN_EXTENSIONS = {
    ".c", ".cc", ".contract", ".cpp", ".css", ".go", ".h", ".hpp", ".html",
    ".java", ".js", ".json", ".jsonc", ".jsonl", ".kt", ".md", ".mjs", ".mts",
    ".py", ".rb", ".rs", ".sh", ".sql", ".swift", ".toml", ".ts", ".tsx",
    ".txt", ".webmanifest", ".yaml", ".yml",
}
SCAN_BASENAMES = {
    ".gitattributes", ".gitignore", ".gitmodules", "Dockerfile", "Makefile",
}
SCAN_EXTENSIONLESS_DIRS = {"scripts", ".githooks"}
REQUIRED_FIELDS = ("Type", "Status", "Systems", "Author", "Date")
GATED_STATUSES = {"accepted", "active"}
MAX_BYTES = 2_000_000

# Built by concatenation so this file doesn't match its own scan patterns.
REF_MARK = "@" + "ref"
CODE_REF_RE = re.compile(re.escape(REF_MARK) + r"\s+(.+)")
MD_REF_RE = re.compile(r"<!--\s*" + re.escape(REF_MARK) + r"\s+(.+?)-->", re.DOTALL)
LLP_TARGET_RE = re.compile(r"^LLP\s+(\d{1,4})(?:#(\S+))?$")
SHORTHAND_RE = re.compile(r"^[A-Z][A-Z0-9_]*(?:#\S+)?$")
# Shared with `ecr ref-check`, which validates the SAME @ref grammar inside a repo's
# .expo-code-review/ setup: `glob:<pattern>` targets a set of files, and a `<…>` target
# documents the syntax rather than citing a file. Both checkers must agree on these or
# the same annotation passes one and fails the other.
GLOB_PREFIX = "glob:"
PLACEHOLDER_RE = re.compile(r"^[<`]")
URL_RE = re.compile(r"^https?://[^\s/]+\.[^\s/]+\S*$")
HEADING_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$")
LLP_FILE_RE = re.compile(r"^(\d{4})-[a-z0-9-]+\.([a-z0-9]+)\.md$")
FIELD_RE = re.compile(r"^\*\*(\w+):\*\*\s*(.+)$")
INFERRED = "[" + "inferred]"


def slugify(text):
    """GitHub-style heading slug: lowercase; spaces -> hyphens; keep
    alphanumerics, hyphens, underscores; drop everything else."""
    text = text.replace("`", "")
    out = []
    for ch in text.strip().lower():
        if ch.isalnum() or ch in "-_":
            out.append(ch)
        elif ch in " \t":
            out.append("-")
    return "".join(out)


def read_text(path):
    try:
        if os.path.getsize(path) > MAX_BYTES:
            return None
        with open(path, encoding="utf-8") as f:
            return f.read()
    except (UnicodeDecodeError, OSError):
        return None


def is_scan_candidate(relpath):
    """Whether a repository-relative path is source/config/document text."""
    normalized = relpath.replace(os.sep, "/")
    name = os.path.basename(normalized)
    if name in SCAN_BASENAMES:
        return True
    _, extension = os.path.splitext(name)
    if extension.lower() in SCAN_EXTENSIONS:
        return True
    return not extension and normalized.split("/", 1)[0] in SCAN_EXTENSIONLESS_DIRS


def md_headings(text):
    """Heading texts outside fenced code blocks."""
    headings, fenced = [], False
    for line in text.splitlines():
        if line.lstrip().startswith("```"):
            fenced = not fenced
            continue
        if fenced:
            continue
        m = HEADING_RE.match(line)
        if m:
            headings.append(m.group(2))
    return headings


def anchor_resolves(anchor, headings):
    anchor = anchor.lstrip("#")
    slugs = {slugify(h) for h in headings}
    if anchor.lower() in slugs:
        return True
    if re.fullmatch(r"\d+(\.\d+)*", anchor):  # numbered-section form: #3, #3.2
        return any(re.match(rf"^{re.escape(anchor)}([.)\s]|$)", h) for h in headings)
    return False


class Report:
    def __init__(self):
        self.errors, self.warnings, self.infos = [], [], []

    def error(self, msg):
        self.errors.append(msg)

    def info(self, msg):
        self.infos.append(msg)


def parse_target(rest):
    """Split an annotation body into (kind, target, anchor)."""
    rest = rest.strip()
    rest = re.split(r"\s+(?:[—–]|--|-)(?:\s+|$)", rest)[0].strip()  # drop gloss
    rest = re.sub(r"\s*\[[a-z-]+\]\s*$", "", rest)        # drop [relation]
    m = LLP_TARGET_RE.match(rest)
    if m:
        return "llp", m.group(1), m.group(2)
    token = rest.split()[0] if rest.split() else ""
    if token.startswith(("http://", "https://")):
        return "url", token, None
    if PLACEHOLDER_RE.match(token):
        return "placeholder", token, None
    if token.startswith(GLOB_PREFIX):
        return "glob", token[len(GLOB_PREFIX):], None
    if SHORTHAND_RE.match(token):
        return "shorthand", token, None
    path, _, anchor = token.partition("#")
    return "path", path, anchor or None


def glob_to_regex(pattern):
    """The mini-glob dialect shared with the CLI: `**` crosses `/`, `*` does not."""
    out, i = "", 0
    while i < len(pattern):
        ch = pattern[i]
        if ch == "*":
            if pattern[i + 1:i + 2] == "*":
                out += ".*"
                i += 1
            else:
                out += "[^/]*"
        elif ch in ".+^${}()|[]\\?":
            out += "\\" + ch
        else:
            out += ch
        i += 1
    return re.compile(f"^{out}$")


def repo_files(root):
    """Every file in the tree (SKIP_DIRS pruned), root-relative with / separators."""
    found = []
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
        for name in filenames:
            rel = os.path.relpath(os.path.join(dirpath, name), root)
            found.append(rel.replace(os.sep, "/"))
    return found


def fenced_lines(text):
    """1-based line numbers inside a fenced code block (the fence lines included)."""
    fenced, open_fence = set(), False
    for i, line in enumerate(text.splitlines(), 1):
        if line.lstrip().startswith("```"):
            fenced.add(i)
            open_fence = not open_fence
        elif open_fence:
            fenced.add(i)
    return fenced


def collect_refs(path, text):
    refs = []  # (lineno, rest)
    if path.endswith(".md"):
        # An annotation inside a fence documents the grammar; resolving it would make
        # every doc that explains refs fail. Same rule in ecr's config-refs.ts.
        fenced = fenced_lines(text)
        for m in MD_REF_RE.finditer(text):
            lineno = text.count("\n", 0, m.start()) + 1
            if lineno in fenced:
                continue
            refs.append((lineno, m.group(1)))
    else:
        for i, line in enumerate(text.splitlines(), 1):
            m = CODE_REF_RE.search(line)
            if m:
                refs.append((i, m.group(1)))
    return refs


def check(root):
    rep = Report()
    llp_dir = os.path.join(root, "llp")

    # --- corpus: collect LLP docs, check metadata ---
    docs = {}  # number -> relpath
    headings = {}  # number -> [heading, ...]
    if os.path.isdir(llp_dir):
        for dirpath, dirnames, filenames in os.walk(llp_dir):
            dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS and d != "reviews"]
            for name in sorted(filenames):
                m = LLP_FILE_RE.match(name)
                if not m:
                    continue
                num, ftype = m.group(1), m.group(2)
                rel = os.path.relpath(os.path.join(dirpath, name), root)
                text = read_text(os.path.join(dirpath, name))
                if text is None:
                    continue
                if num in docs:
                    rep.error(f"{rel}: duplicate LLP number {num} (also {docs[num]})")
                else:
                    docs[num] = rel
                    headings[num] = md_headings(text)
                fields = {}
                for line in text.splitlines()[1:40]:
                    fm = FIELD_RE.match(line.strip())
                    if fm:
                        fields.setdefault(fm.group(1), fm.group(2).strip())
                for req in REQUIRED_FIELDS:
                    if req not in fields:
                        rep.error(f"{rel}: missing required metadata field **{req}:**")
                doc_type = fields.get("Type", "").split()[0].lower() if fields.get("Type") else ""
                if doc_type and doc_type != ftype:
                    rep.error(f"{rel}: filename type '.{ftype}.md' != **Type:** {fields['Type']}")
                status = fields.get("Status", "").split()[0].lower()
                if status in GATED_STATUSES:
                    bare = text.count(INFERRED) - text.count("`" + INFERRED)
                    if bare > 0:
                        rep.error(f"{rel}: {INFERRED} claim in {fields.get('Status')} document "
                                  f"(ratify to [confirmed] or remove before promotion)")

    # --- refs: scan the tree ---
    n_files = n_refs = 0
    all_files = repo_files(root)
    exempt = (os.path.join("llp", "tombstones"), os.path.join("llp", "reviews"))
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames if d not in SKIP_DIRS]
        if dirpath != root and "llp" in dirnames:
            # a nested llp/ marks a separate corpus root (fixture, vendored
            # project): its refs resolve against its own tree, not this one
            dirnames[:] = []
            continue
        for name in sorted(filenames):
            full = os.path.join(dirpath, name)
            rel = os.path.relpath(full, root)
            if rel.startswith(exempt) or os.path.basename(rel) == "ref-check":
                continue
            if not is_scan_candidate(rel):
                continue
            text = read_text(full)
            if text is None:
                continue
            n_files += 1
            for lineno, rest in collect_refs(rel, text):
                n_refs += 1
                kind, target, anchor = parse_target(rest)
                where = f"{rel}:{lineno}"
                if kind == "llp":
                    num = target.zfill(4)
                    if num not in docs:
                        rep.error(f"{where}: broken ref: LLP {num} not found under llp/")
                    elif anchor and not anchor_resolves(anchor, headings[num]):
                        rep.error(f"{where}: broken ref: no anchor #{anchor} in {docs[num]}")
                elif kind == "url":
                    if not URL_RE.match(target):
                        rep.error(f"{where}: malformed URL target: {target}")
                elif kind == "placeholder":
                    rep.info(f"{where}: documentation placeholder, not a citation: {target}")
                elif kind == "glob":
                    if not target:
                        rep.error(f"{where}: empty glob target")
                    elif not any(glob_to_regex(target).match(f) for f in all_files):
                        rep.error(f"{where}: broken ref: glob matches no file: {target}")
                elif kind == "shorthand":
                    rep.info(f"{where}: unchecked shorthand: {target} (no mapping defined)")
                else:  # path
                    tpath = os.path.join(root, target)
                    if not os.path.exists(tpath):
                        rep.error(f"{where}: broken ref: path not found: {target}")
                    elif anchor:
                        if target.endswith(".md"):
                            ttext = read_text(tpath) or ""
                            if not anchor_resolves(anchor, md_headings(ttext)):
                                rep.error(f"{where}: broken ref: no anchor #{anchor} in {target}")
                        else:
                            rep.info(f"{where}: anchor #{anchor} unchecked (non-markdown target)")
    return rep, n_files, n_refs, len(docs)


def main():
    ap = argparse.ArgumentParser(prog="ref-check", description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--root", default=".", help="project root to check (default: .)")
    ap.add_argument("--verbose", action="store_true", help="also print informational notes")
    args = ap.parse_args()

    rep, n_files, n_refs, n_docs = check(os.path.abspath(args.root))
    for e in rep.errors:
        print(f"ERROR {e}")
    if args.verbose:
        for i in rep.infos:
            print(f"INFO  {i}")
    status = "FAIL" if rep.errors else "ok"
    print(f"ref-check: {status} — {n_docs} LLP docs, {n_refs} refs in {n_files} files, "
          f"{len(rep.errors)} errors, {len(rep.infos)} unchecked")
    sys.exit(1 if rep.errors else 0)


if __name__ == "__main__":
    main()