feat: bounded backend-to-schema import lineage with evidence

.github/workflows/github-source-smoke.yml

@@ -3,6 +3,9 @@ on:
   pull_request:
     paths:
       - 'ontology-extraction/scripts/github_discovery.py'
+      - 'ontology-extraction/scripts/code_imports.py'
+      - 'ontology-extraction/scripts/snapshot_store.py'
+      - 'tests/test_code_surface_discovery.py'
       - 'ontology-extraction/scripts/discover_repo.py'
       - 'tests/test_github_discovery.py'
       - '.github/workflows/github-source-smoke.yml'
@@ -47,6 +50,12 @@ jobs:
           assert x['coverage']['schema_source_files_read']==1
           assert x['coverage']['import_route_files_inspected']>=20
           assert x['coverage']['import_relationships_found']>=10
+          assert x['coverage']['second_hop_modules_read']>=10
+          assert x['coverage']['second_hop_import_edges']>=10
+          assert x['coverage']['second_hop_schema_table_declarations']>=1
+          assert any(e['relation']=='DECLARES_TABLE_CANDIDATE' and
+                     e['state']=='SOURCE_SYNTAX_CANDIDATE'
+                     for e in x['edges'])
           assert any(e['relation']=='STATIC_IMPORTS' and
                      e['state']=='SOURCE_SYNTAX_CANDIDATE'
                      for e in x['edges'])
@@ -63,6 +72,10 @@ jobs:
               c['scope']=='CI_ONLY_NOT_PRODUCTION_HEALTH'
               for c in x['ci_evidence'])
           print('CI evidence for pinned commit:',len(x['ci_evidence']),'workflow runs')
+          print('second hop:',x['coverage']['second_hop_modules_read'],
+                'modules,',x['coverage']['second_hop_import_edges'],
+                'imports,',x['coverage']['second_hop_schema_table_declarations'],
+                'schema declaration links')
           print('repo.ing source discovery validated:',
                 len(x['nodes']), 'nodes,',len(x['edges']), 'edges,',
                 x['coverage']['candidate_manifests'],'manifests; production health UNKNOWN')

docs/second-hop-source-lineage.md

@@ -0,0 +1,71 @@
+# Second-hop source lineage — module imports and schema declarations
+
+## What changed
+
+OntologyEX now explores **one additional bounded import hop** after its
+source-pinned API route scan. The displayed claim is precise:
+
+API route (file) --STATIC_IMPORTS--> backend module (file)
+  --STATIC_IMPORTS--> schema module (file)
+  --DECLARES_TABLE_CANDIDATE--> database table declaration (source syntax)
+
+The first two kinds of arrows mean a static relative import appears in
+source code whose bytes matched the pinned Git object. The final arrow
+means a simple Drizzle pgTable declaration was recognized in
+`src/db/schema.mjs`. It does **not** mean that backend code read, wrote,
+or queried any of the tables; the table may not exist in the live DB.
+
+## Bounded discovery contract
+
+- Select at most 36 API route files from the pinned tree.
+- Traverse **one more hop** from at most 48 modules directly imported
+  by those routes. 32 selections favor frequently imported modules; up
+  to 16 additional selections cover `src/` business modules.
+- Read up to 96 KiB per module and 2 MiB total for second-hop discovery.
+- Add up to 384 second-hop static import edges and a total of 256 unique
+  source-module nodes across both phases.
+- Resolve only same-repository relative paths which actually exist in the
+  pinned Git tree. Do not execute source code, npm scripts, build steps,
+  network imports, dynamic code, SQL or application functions.
+- Preserve per-edge source-file hash and source line, including unresolved
+  references and skipped/unsupported input counts.
+- Imported target files remain *tree metadata* nodes unless their source
+  bytes were actually read for a source-backed edge.
+- On a failed/oversized read, emit a named coverage gap and retain UNKNOWN
+  rather than pretending to have completed the import graph.
+
+## What is not proven
+
+Static imports are necessary evidence for many dependency paths, but
+do not establish execution paths, HTTP verbs, SQL queries, production
+connectivity or operational health. The limited import recognizer is not
+a full JavaScript parser and may miss dynamic/multiline imports or match
+unusual comment layouts. Every parsed import remains an unreviewed
+`SOURCE_SYNTAX_CANDIDATE`; downstream automated repair is prohibited.
+
+Source files may include sensitive secrets. The scanner processes
+bounded bytes on the backend, emitting only paths, SHA hashes, line
+numbers and candidate claims; no raw code bodies are sent to the UI.
+Private repository deployments still require tenant-authorized
+GitHub App credentials and isolated workers before hosted release.
+
+## Verification
+
+Offline fixtures test a real two-hop chain, byte-corruption refusal,
+missing imports, bounded module reads, stable source identity,
+persistence and that no runtime usage claims are created.
+CI additionally scans the public repo.ing source via GitHub's read API
+with blob verification. The CI evidence concerns static source only;
+actual production database health remains UNKNOWN.
+
+## Next engineering direction
+
+To establish specific table usage, use an AST parser to identify
+Drizzle query-building operations and tie them to declared imports.
+Treat those as source-code candidates until confirmed with
+authenticated OpenTelemetry traces and database telemetry.
+
+## Stacked history
+
+Built on drafts #11 → #14 → #17 → #18 → #19 → #20.
+No branch is merged or deployed automatically.

ontology-extraction/scripts/code_imports.py

@@ -16,6 +16,10 @@
 MAX_LINKS = 512
 MAX_MODULES = 256
 MAX_UNRESOLVED = 256
+MAX_SECOND_HOP_MODULES = 48
+MAX_SECOND_HOP_BYTES = 2 * 1024 * 1024
+MAX_SECOND_HOP_LINKS = 384
+MAX_MODULE_BYTES = 96 * 1024
 EXTENSIONS = (".js", ".mjs", ".ts", ".jsx", ".tsx")
 IMPORT = re.compile(
     r"""(?m)^[ \t]*import[ \t]+(?:[^\r\n;'"]{1,500}?[ \t]+from[ \t]+)?(?P<q>['"])(?P<path>\.[^'"\r\n]{1,180})(?P=q)"""
@@ -49,6 +53,143 @@ def resolve_import(source: str, specifier: str, known: dict):
     return None
 
 
+def scan_module_imports(mapped, known, unique_modules, inbound,
+                        client, api, commit, repo_id, read_blob, link_node, require,
+                        evidence_ids):
+    """One additional import hop from *existing* source modules, bounded.
+
+    STATIC_IMPORTS means source-text import declaration; it does not claim a
+    runtime invocation, an observed database query or execution authority.
+    """
+    # Prioritize commonly imported files, plus a spread of business modules.
+    ranked = sorted(unique_modules, key=lambda path: (-inbound.get(path, 0), path))
+    chosen = ranked[:32]
+    chosen += [path for path in sorted(unique_modules)
+               if path.startswith("src/") and path not in chosen][:16]
+    chosen = chosen[:MAX_SECOND_HOP_MODULES]
+    modules_read = 0
+    module_bytes = 0
+    relationships = 0
+    unresolved = 0
+    limits = 0
+    target_modules = set(unique_modules)
+    schema_module_id = "discovery:source_module:gh" + str(repo_id) + "/src/db/schema.mjs"
+    for source in chosen:
+        entry = known[source]
+        size = entry.get("size")
+        # Only explicit tracked application modules, not deps, vendored code or arbitrary files.
+        if (not (source.startswith("app/") or source.startswith("src/"))
+                or type(size) is not int or size < 0 or size > MAX_MODULE_BYTES):
+            mapped["gaps"].append({
+                "source": source, "reason": "SECOND_HOP_MODULE_UNSUPPORTED"})
+            continue
+        if module_bytes + size > MAX_SECOND_HOP_BYTES:
+            limits += 1
+            continue
+        try:
+            raw = read_blob(client, api, entry["sha"], size)
+            content = raw.decode("utf-8")
+        except (ValueError, TypeError, UnicodeError):
+            mapped["gaps"].append({
+                "source": source, "reason": "SECOND_HOP_SOURCE_UNAVAILABLE"})
+            continue
+        modules_read += 1
+        module_bytes += len(raw)
+        evidence_id = hashlib.sha256(
+            (str(repo_id) + "\0" + commit + "\0" + source +
+             "\0" + hashlib.sha256(raw).hexdigest() +
+             "\0second-hop-static-imports").encode("utf-8")
+        ).hexdigest()
+        require(evidence_id not in evidence_ids, "duplicate module evidence")
+        evidence_ids.add(evidence_id)
+        mapped["source_evidence"].append({
+            "id": evidence_id, "path": source,
+            "sha256": hashlib.sha256(raw).hexdigest(),
+            "source_kind": "STATIC_IMPORT_SYNTAX",
+            "authenticity": "GIT_BLOB_VERIFIED_SYNTAX_UNREVIEWED",
+            "origin": "GITHUB_COMMIT_PINNED_BLOB",
+            "repository_id": repo_id, "commit_sha": commit,
+            "git_blob_sha": entry["sha"],
+        })
+        module_id = "discovery:source_module:gh" + str(repo_id) + "/" + quote(source, safe="/-._~")
+        for match in IMPORT.finditer(content):
+            specifier = match.group("path")
+            target = resolve_import(source, specifier, known)
+            if target is None:
+                unresolved += 1
+                if len(mapped["unresolved"]) < MAX_UNRESOLVED:
+                    mapped["unresolved"].append({
+                        "source": module_id, "target_name": specifier,
+                        "relation": "STATIC_IMPORT_UNRESOLVED",
+                        "evidence_id": evidence_id,
+                    })
+                else:
+                    limits += 1
+                continue
+            if target not in target_modules:
+                if len(target_modules) >= MAX_MODULES:
+                    limits += 1
+                    continue
+                target_entry = known[target]
+                sha = target_entry.get("sha")
+                if (type(sha) is not str or
+                        re.fullmatch(r"[0-9a-f]{40}", sha) is None):
+                    limits += 1
+                    continue
+                digest = hashlib.sha256(
+                    (commit + "\0" + target + "\0" + sha).encode("utf-8")
+                ).hexdigest()
+                link_node(target, "source_module", target, "GITHUB_TREE_ENTRY_METADATA",
+                          digest, sha, "TREE_PATH_ONLY_SOURCE_BYTES_NOT_READ",
+                          "GITHUB_REPOSITORY_TREE_ENTRY")
+                target_modules.add(target)
+            if relationships >= MAX_SECOND_HOP_LINKS:
+                limits += 1
+                continue
+            mapped["edges"].append({
+                "source": module_id,
+                "target": "discovery:source_module:gh" + str(repo_id) +
+                          "/" + quote(target, safe="/-._~"),
+                "relation": "STATIC_IMPORTS",
+                "evidence_id": evidence_id,
+                "state": "SOURCE_SYNTAX_CANDIDATE",
+                "source_line": content.count("\n", 0, match.start()) + 1,
+            })
+            relationships += 1
+
+    # The schema module owns these *source-syntax candidates*. This edge never
+    # means a table is deployed or queried, and only appears if that exact
+    # module was actually found in the static import graph.
+    declared_tables = 0
+    if "src/db/schema.mjs" in target_modules:
+        for table in [node for node in mapped["nodes"]
+                      if node["kind"] == "declared_db_table" and
+                      node["state"] == "SOURCE_SYNTAX_CANDIDATE"]:
+            mapped["edges"].append({
+                "source": schema_module_id, "target": table["id"],
+                "relation": "DECLARES_TABLE_CANDIDATE",
+                "evidence_id": table["evidence_id"],
+                "state": "SOURCE_SYNTAX_CANDIDATE",
+                "source_line": table["source_line"],
+            })
+            declared_tables += 1
+    if limits:
+        mapped["gaps"].append({
+            "source": "github_second_hop_imports",
+            "reason": "SECOND_HOP_RELATIONSHIPS_BOUNDED",
+        })
+    mapped["coverage"].update({
+        "second_hop_modules_selected": len(chosen),
+        "second_hop_modules_read": modules_read,
+        "second_hop_modules_not_read": len(unique_modules) - modules_read,
+        "second_hop_source_bytes_read": module_bytes,
+        "second_hop_import_edges": relationships,
+        "second_hop_unresolved_imports": unresolved,
+        "second_hop_limited_links": limits,
+        "second_hop_schema_table_declarations": declared_tables,
+    })
+
+
 def attach(mapped, selected_routes, entries, client, api, commit, repo_id,
            read_blob, link_node, require):
     known = {
@@ -61,6 +202,7 @@ def attach(mapped, selected_routes, entries, client, api, commit, repo_id,
     inspected = 0
     links = 0
     unique_modules = set()
+    inbound = {}
     unrecognized = 0
     skipped = 0
     gaps = 0
@@ -151,7 +293,11 @@ def gap(path, reason):
                 "state": "SOURCE_SYNTAX_CANDIDATE", "source_line": line,
             })
             links += 1
+            inbound[target] = inbound.get(target, 0) + 1
 
+    scan_module_imports(mapped, known, unique_modules, inbound,
+                        client, api, commit, repo_id, read_blob, link_node,
+                        require, evidence_ids)
     if skipped:
         gap("github_static_imports", "IMPORT_RELATIONSHIPS_BOUNDED")
     mapped["unresolved"].sort(key=lambda x: (

ontology-extraction/scripts/snapshot_store.py

@@ -78,6 +78,10 @@ def encode(graph: dict) -> tuple[bytes, int, str, str]:
                 "import_route_files_not_inspected", "import_source_bytes_read",
                 "import_relationships_found", "import_modules_found",
                 "import_unresolved_specifiers", "import_skipped_relations",
+                "second_hop_modules_selected", "second_hop_modules_read",
+                "second_hop_modules_not_read", "second_hop_source_bytes_read",
+                "second_hop_import_edges", "second_hop_unresolved_imports",
+                "second_hop_limited_links", "second_hop_schema_table_declarations",
             } and type(graph["gaps"]) is list and len(graph["gaps"]) <= 256 and
             type(graph["unresolved"]) is list and len(graph["unresolved"]) <= 256 and
             type(graph["ci_evidence"]) is list and len(graph["ci_evidence"]) <= 100,