diff --git a/.github/workflows/github-source-smoke.yml b/.github/workflows/github-source-smoke.yml index 5ff99df..fec76ec 100644 --- a/.github/workflows/github-source-smoke.yml +++ b/.github/workflows/github-source-smoke.yml @@ -3,6 +3,9 @@ on: pull_request: paths: - 'ontology-extraction/scripts/github_discovery.py' + - 'ontology-extraction/scripts/code_imports.py' + - 'ontology-extraction/scripts/snapshot_store.py' + - 'tests/test_code_surface_discovery.py' - 'ontology-extraction/scripts/discover_repo.py' - 'tests/test_github_discovery.py' - '.github/workflows/github-source-smoke.yml' @@ -47,6 +50,12 @@ jobs: assert x['coverage']['schema_source_files_read']==1 assert x['coverage']['import_route_files_inspected']>=20 assert x['coverage']['import_relationships_found']>=10 + assert x['coverage']['second_hop_modules_read']>=10 + assert x['coverage']['second_hop_import_edges']>=10 + assert x['coverage']['second_hop_schema_table_declarations']>=1 + assert any(e['relation']=='DECLARES_TABLE_CANDIDATE' and + e['state']=='SOURCE_SYNTAX_CANDIDATE' + for e in x['edges']) assert any(e['relation']=='STATIC_IMPORTS' and e['state']=='SOURCE_SYNTAX_CANDIDATE' for e in x['edges']) @@ -63,6 +72,10 @@ jobs: c['scope']=='CI_ONLY_NOT_PRODUCTION_HEALTH' for c in x['ci_evidence']) print('CI evidence for pinned commit:',len(x['ci_evidence']),'workflow runs') + print('second hop:',x['coverage']['second_hop_modules_read'], + 'modules,',x['coverage']['second_hop_import_edges'], + 'imports,',x['coverage']['second_hop_schema_table_declarations'], + 'schema declaration links') print('repo.ing source discovery validated:', len(x['nodes']), 'nodes,',len(x['edges']), 'edges,', x['coverage']['candidate_manifests'],'manifests; production health UNKNOWN') diff --git a/docs/second-hop-source-lineage.md b/docs/second-hop-source-lineage.md new file mode 100644 index 0000000..b635455 --- /dev/null +++ b/docs/second-hop-source-lineage.md @@ -0,0 +1,71 @@ +# Second-hop source lineage — module imports and schema declarations + +## What changed + +OntologyEX now explores **one additional bounded import hop** after its +source-pinned API route scan. The displayed claim is precise: + +API route (file) --STATIC_IMPORTS--> backend module (file) + --STATIC_IMPORTS--> schema module (file) + --DECLARES_TABLE_CANDIDATE--> database table declaration (source syntax) + +The first two kinds of arrows mean a static relative import appears in +source code whose bytes matched the pinned Git object. The final arrow +means a simple Drizzle pgTable declaration was recognized in +`src/db/schema.mjs`. It does **not** mean that backend code read, wrote, +or queried any of the tables; the table may not exist in the live DB. + +## Bounded discovery contract + +- Select at most 36 API route files from the pinned tree. +- Traverse **one more hop** from at most 48 modules directly imported + by those routes. 32 selections favor frequently imported modules; up + to 16 additional selections cover `src/` business modules. +- Read up to 96 KiB per module and 2 MiB total for second-hop discovery. +- Add up to 384 second-hop static import edges and a total of 256 unique + source-module nodes across both phases. +- Resolve only same-repository relative paths which actually exist in the + pinned Git tree. Do not execute source code, npm scripts, build steps, + network imports, dynamic code, SQL or application functions. +- Preserve per-edge source-file hash and source line, including unresolved + references and skipped/unsupported input counts. +- Imported target files remain *tree metadata* nodes unless their source + bytes were actually read for a source-backed edge. +- On a failed/oversized read, emit a named coverage gap and retain UNKNOWN + rather than pretending to have completed the import graph. + +## What is not proven + +Static imports are necessary evidence for many dependency paths, but +do not establish execution paths, HTTP verbs, SQL queries, production +connectivity or operational health. The limited import recognizer is not +a full JavaScript parser and may miss dynamic/multiline imports or match +unusual comment layouts. Every parsed import remains an unreviewed +`SOURCE_SYNTAX_CANDIDATE`; downstream automated repair is prohibited. + +Source files may include sensitive secrets. The scanner processes +bounded bytes on the backend, emitting only paths, SHA hashes, line +numbers and candidate claims; no raw code bodies are sent to the UI. +Private repository deployments still require tenant-authorized +GitHub App credentials and isolated workers before hosted release. + +## Verification + +Offline fixtures test a real two-hop chain, byte-corruption refusal, +missing imports, bounded module reads, stable source identity, +persistence and that no runtime usage claims are created. +CI additionally scans the public repo.ing source via GitHub's read API +with blob verification. The CI evidence concerns static source only; +actual production database health remains UNKNOWN. + +## Next engineering direction + +To establish specific table usage, use an AST parser to identify +Drizzle query-building operations and tie them to declared imports. +Treat those as source-code candidates until confirmed with +authenticated OpenTelemetry traces and database telemetry. + +## Stacked history + +Built on drafts #11 → #14 → #17 → #18 → #19 → #20. +No branch is merged or deployed automatically. diff --git a/ontology-extraction/scripts/code_imports.py b/ontology-extraction/scripts/code_imports.py index 5e15985..310df70 100644 --- a/ontology-extraction/scripts/code_imports.py +++ b/ontology-extraction/scripts/code_imports.py @@ -16,6 +16,10 @@ MAX_LINKS = 512 MAX_MODULES = 256 MAX_UNRESOLVED = 256 +MAX_SECOND_HOP_MODULES = 48 +MAX_SECOND_HOP_BYTES = 2 * 1024 * 1024 +MAX_SECOND_HOP_LINKS = 384 +MAX_MODULE_BYTES = 96 * 1024 EXTENSIONS = (".js", ".mjs", ".ts", ".jsx", ".tsx") IMPORT = re.compile( r"""(?m)^[ \t]*import[ \t]+(?:[^\r\n;'"]{1,500}?[ \t]+from[ \t]+)?(?P['"])(?P\.[^'"\r\n]{1,180})(?P=q)""" @@ -49,6 +53,143 @@ def resolve_import(source: str, specifier: str, known: dict): return None +def scan_module_imports(mapped, known, unique_modules, inbound, + client, api, commit, repo_id, read_blob, link_node, require, + evidence_ids): + """One additional import hop from *existing* source modules, bounded. + + STATIC_IMPORTS means source-text import declaration; it does not claim a + runtime invocation, an observed database query or execution authority. + """ + # Prioritize commonly imported files, plus a spread of business modules. + ranked = sorted(unique_modules, key=lambda path: (-inbound.get(path, 0), path)) + chosen = ranked[:32] + chosen += [path for path in sorted(unique_modules) + if path.startswith("src/") and path not in chosen][:16] + chosen = chosen[:MAX_SECOND_HOP_MODULES] + modules_read = 0 + module_bytes = 0 + relationships = 0 + unresolved = 0 + limits = 0 + target_modules = set(unique_modules) + schema_module_id = "discovery:source_module:gh" + str(repo_id) + "/src/db/schema.mjs" + for source in chosen: + entry = known[source] + size = entry.get("size") + # Only explicit tracked application modules, not deps, vendored code or arbitrary files. + if (not (source.startswith("app/") or source.startswith("src/")) + or type(size) is not int or size < 0 or size > MAX_MODULE_BYTES): + mapped["gaps"].append({ + "source": source, "reason": "SECOND_HOP_MODULE_UNSUPPORTED"}) + continue + if module_bytes + size > MAX_SECOND_HOP_BYTES: + limits += 1 + continue + try: + raw = read_blob(client, api, entry["sha"], size) + content = raw.decode("utf-8") + except (ValueError, TypeError, UnicodeError): + mapped["gaps"].append({ + "source": source, "reason": "SECOND_HOP_SOURCE_UNAVAILABLE"}) + continue + modules_read += 1 + module_bytes += len(raw) + evidence_id = hashlib.sha256( + (str(repo_id) + "\0" + commit + "\0" + source + + "\0" + hashlib.sha256(raw).hexdigest() + + "\0second-hop-static-imports").encode("utf-8") + ).hexdigest() + require(evidence_id not in evidence_ids, "duplicate module evidence") + evidence_ids.add(evidence_id) + mapped["source_evidence"].append({ + "id": evidence_id, "path": source, + "sha256": hashlib.sha256(raw).hexdigest(), + "source_kind": "STATIC_IMPORT_SYNTAX", + "authenticity": "GIT_BLOB_VERIFIED_SYNTAX_UNREVIEWED", + "origin": "GITHUB_COMMIT_PINNED_BLOB", + "repository_id": repo_id, "commit_sha": commit, + "git_blob_sha": entry["sha"], + }) + module_id = "discovery:source_module:gh" + str(repo_id) + "/" + quote(source, safe="/-._~") + for match in IMPORT.finditer(content): + specifier = match.group("path") + target = resolve_import(source, specifier, known) + if target is None: + unresolved += 1 + if len(mapped["unresolved"]) < MAX_UNRESOLVED: + mapped["unresolved"].append({ + "source": module_id, "target_name": specifier, + "relation": "STATIC_IMPORT_UNRESOLVED", + "evidence_id": evidence_id, + }) + else: + limits += 1 + continue + if target not in target_modules: + if len(target_modules) >= MAX_MODULES: + limits += 1 + continue + target_entry = known[target] + sha = target_entry.get("sha") + if (type(sha) is not str or + re.fullmatch(r"[0-9a-f]{40}", sha) is None): + limits += 1 + continue + digest = hashlib.sha256( + (commit + "\0" + target + "\0" + sha).encode("utf-8") + ).hexdigest() + link_node(target, "source_module", target, "GITHUB_TREE_ENTRY_METADATA", + digest, sha, "TREE_PATH_ONLY_SOURCE_BYTES_NOT_READ", + "GITHUB_REPOSITORY_TREE_ENTRY") + target_modules.add(target) + if relationships >= MAX_SECOND_HOP_LINKS: + limits += 1 + continue + mapped["edges"].append({ + "source": module_id, + "target": "discovery:source_module:gh" + str(repo_id) + + "/" + quote(target, safe="/-._~"), + "relation": "STATIC_IMPORTS", + "evidence_id": evidence_id, + "state": "SOURCE_SYNTAX_CANDIDATE", + "source_line": content.count("\n", 0, match.start()) + 1, + }) + relationships += 1 + + # The schema module owns these *source-syntax candidates*. This edge never + # means a table is deployed or queried, and only appears if that exact + # module was actually found in the static import graph. + declared_tables = 0 + if "src/db/schema.mjs" in target_modules: + for table in [node for node in mapped["nodes"] + if node["kind"] == "declared_db_table" and + node["state"] == "SOURCE_SYNTAX_CANDIDATE"]: + mapped["edges"].append({ + "source": schema_module_id, "target": table["id"], + "relation": "DECLARES_TABLE_CANDIDATE", + "evidence_id": table["evidence_id"], + "state": "SOURCE_SYNTAX_CANDIDATE", + "source_line": table["source_line"], + }) + declared_tables += 1 + if limits: + mapped["gaps"].append({ + "source": "github_second_hop_imports", + "reason": "SECOND_HOP_RELATIONSHIPS_BOUNDED", + }) + mapped["coverage"].update({ + "second_hop_modules_selected": len(chosen), + "second_hop_modules_read": modules_read, + "second_hop_modules_not_read": len(unique_modules) - modules_read, + "second_hop_source_bytes_read": module_bytes, + "second_hop_import_edges": relationships, + "second_hop_unresolved_imports": unresolved, + "second_hop_limited_links": limits, + "second_hop_schema_table_declarations": declared_tables, + }) + + def attach(mapped, selected_routes, entries, client, api, commit, repo_id, read_blob, link_node, require): known = { @@ -61,6 +202,7 @@ def attach(mapped, selected_routes, entries, client, api, commit, repo_id, inspected = 0 links = 0 unique_modules = set() + inbound = {} unrecognized = 0 skipped = 0 gaps = 0 @@ -151,7 +293,11 @@ def gap(path, reason): "state": "SOURCE_SYNTAX_CANDIDATE", "source_line": line, }) links += 1 + inbound[target] = inbound.get(target, 0) + 1 + scan_module_imports(mapped, known, unique_modules, inbound, + client, api, commit, repo_id, read_blob, link_node, + require, evidence_ids) if skipped: gap("github_static_imports", "IMPORT_RELATIONSHIPS_BOUNDED") mapped["unresolved"].sort(key=lambda x: ( diff --git a/ontology-extraction/scripts/snapshot_store.py b/ontology-extraction/scripts/snapshot_store.py index 7b0c2c9..9ece483 100644 --- a/ontology-extraction/scripts/snapshot_store.py +++ b/ontology-extraction/scripts/snapshot_store.py @@ -78,6 +78,10 @@ def encode(graph: dict) -> tuple[bytes, int, str, str]: "import_route_files_not_inspected", "import_source_bytes_read", "import_relationships_found", "import_modules_found", "import_unresolved_specifiers", "import_skipped_relations", + "second_hop_modules_selected", "second_hop_modules_read", + "second_hop_modules_not_read", "second_hop_source_bytes_read", + "second_hop_import_edges", "second_hop_unresolved_imports", + "second_hop_limited_links", "second_hop_schema_table_declarations", } and type(graph["gaps"]) is list and len(graph["gaps"]) <= 256 and type(graph["unresolved"]) is list and len(graph["unresolved"]) <= 256 and type(graph["ci_evidence"]) is list and len(graph["ci_evidence"]) <= 100, diff --git a/tests/test_code_surface_discovery.py b/tests/test_code_surface_discovery.py index c1c2ecf..52434c4 100644 --- a/tests/test_code_surface_discovery.py +++ b/tests/test_code_surface_discovery.py @@ -40,12 +40,22 @@ def fixture(self): "size": len(raw), "sha": sha, }) mock.maps[base + "/git/blobs/" + sha] = response - for path in ("app/lib/server.mjs", "src/claim.mjs"): - sha, _ = fixtures.blob(path.encode("utf-8")) + module_contents = { + "app/lib/server.mjs": ( + b"import { createDatabasePool } from '../../src/database-pool.mjs'\n" + ), + "src/claim.mjs": ( + b"import { users, orders } from './db/schema.mjs'\n" + ), + "src/database-pool.mjs": b"export function createDatabasePool() {}\n", + } + for path, raw in module_contents.items(): + sha, response = fixtures.blob(raw) mock.tree.append({ "path": path, "type": "blob", "mode": "100644", - "size": len(path), "sha": sha, + "size": len(raw), "sha": sha, }) + mock.maps[base + "/git/blobs/" + sha] = response schema = (b"import { pgTable } from 'drizzle-orm/pg-core'\n" b"export const users = pgTable('users', {})\n" b"export const orders = pgTable(\"orders\", {})\n" @@ -72,11 +82,24 @@ def test_routes_from_tree_and_schema_from_verified_blob(self): self.assertEqual(graph["coverage"]["import_route_files_inspected"], 2) self.assertEqual(graph["coverage"]["import_relationships_found"], 2) self.assertEqual(graph["coverage"]["import_modules_found"], 2) + self.assertEqual(graph["coverage"]["second_hop_modules_read"], 2) + self.assertEqual(graph["coverage"]["second_hop_import_edges"], 2) + self.assertEqual(graph["coverage"]["second_hop_schema_table_declarations"], 2) + self.assertEqual(len([e for e in graph["edges"] + if e["relation"] == "DECLARES_TABLE_CANDIDATE"]), 2) + self.assertTrue(any( + e["relation"] == "STATIC_IMPORTS" and + "src/claim.mjs" in e["source"] and + "src/db/schema.mjs" in e["target"] + for e in graph["edges"])) + self.assertTrue(any(n["label"] == "src/db/schema.mjs" + for n in graph["nodes"] if n["kind"] == "source_module")) imported = [e for e in graph["edges"] if e["relation"] == "STATIC_IMPORTS"] self.assertEqual({e["source_line"] for e in imported}, {1}) self.assertEqual({n["label"] for n in graph["nodes"] if n["kind"] == "source_module"}, - {"src/claim.mjs", "app/lib/server.mjs"}) + {"src/claim.mjs", "app/lib/server.mjs", + "src/db/schema.mjs", "src/database-pool.mjs"}) self.assertTrue(all(e["state"] == "SOURCE_SYNTAX_CANDIDATE" for e in imported)) self.assertEqual(graph["runtime_health"], "UNKNOWN") @@ -129,6 +152,56 @@ def test_all_evidence_unique_and_graph_consistent(self): self.assertFalse(any(e["relation"] == "DEPENDS_ON" for e in table_edges)) + def test_corrupted_module_does_not_fabricate_schema_dependency(self): + mock = self.fixture() + entry = next(x for x in mock.tree if x["path"] == "src/claim.mjs") + key = f"/repos/{mock.repo}/git/blobs/{entry['sha']}" + mock.maps[key]["content"] = "not-base64!" + graph = gh.discover_github(mock.repo, mock) + self.assertIn("SECOND_HOP_SOURCE_UNAVAILABLE", + [g["reason"] for g in graph["gaps"]]) + self.assertFalse(any(e["relation"] == "DECLARES_TABLE_CANDIDATE" + for e in graph["edges"])) + self.assertEqual(graph["coverage"]["second_hop_modules_read"], 1) + self.assertEqual(graph["runtime_health"], "UNKNOWN") + + def test_unresolved_module_import_does_not_create_a_target(self): + mock = self.fixture() + entry = next(x for x in mock.tree if x["path"] == "src/claim.mjs") + raw = b"import { missing } from './db/not-present.mjs'\n" + sha, response = fixtures.blob(raw) + entry["sha"], entry["size"] = sha, len(raw) + mock.maps[f"/repos/{mock.repo}/git/blobs/{sha}"] = response + graph = gh.discover_github(mock.repo, mock) + self.assertIn("STATIC_IMPORT_UNRESOLVED", + [u["relation"] for u in graph["unresolved"]]) + self.assertFalse(any("not-present.mjs" in n["id"] for n in graph["nodes"])) + self.assertEqual(graph["coverage"]["second_hop_unresolved_imports"], 1) + + def test_no_implicit_table_usage_from_schema_declaration(self): + graph = gh.discover_github(self.fixture().repo, self.fixture()) + self.assertFalse(any(e["relation"] in ("USES_TABLE", "QUERIES_TABLE", + "RUNTIME_CALL", "DATABASE_CALL") + for e in graph["edges"])) + for e in graph["edges"]: + if e["relation"] == "DECLARES_TABLE_CANDIDATE": + self.assertIn("src/db/schema.mjs", e["source"]) + self.assertEqual(e["state"], "SOURCE_SYNTAX_CANDIDATE") + self.assertIsInstance(e["source_line"], int) + + def test_bounded_module_reads_show_partial_coverage(self): + from unittest.mock import patch + import code_imports + mock = self.fixture() + with patch.object(code_imports, "MAX_MODULE_BYTES", 0): + graph = gh.discover_github(mock.repo, mock) + self.assertEqual(graph["coverage"]["second_hop_modules_read"], 0) + self.assertEqual(graph["coverage"]["second_hop_modules_not_read"], 2) + self.assertFalse(any(e["relation"] == "DECLARES_TABLE_CANDIDATE" + for e in graph["edges"])) + self.assertIn("SECOND_HOP_MODULE_UNSUPPORTED", + [g["reason"] for g in graph["gaps"]]) + def test_bad_schema_not_executed_and_not_health(self): mock = self.fixture() graph = gh.discover_github(mock.repo, mock)