diff --git a/flex/modules/claude_code/__init__.py b/flex/modules/claude_code/__init__.py index 78c0667..b8c47ab 100644 --- a/flex/modules/claude_code/__init__.py +++ b/flex/modules/claude_code/__init__.py @@ -9,8 +9,20 @@ (formerly install._run_enrichment_quiet). """ -# Single source of truth for coding-agent enrichment stub tables. -ENRICHMENT_STUBS: list[str] = [ +# Base enrichment stubs -- generic tables any flex module may need. +BASE_ENRICHMENT_STUBS: list[str] = [ + """CREATE TABLE IF NOT EXISTS _ops ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp INTEGER DEFAULT (strftime('%s','now')), + operation TEXT, target TEXT, sql TEXT, params TEXT, + rows_affected INTEGER, source TEXT)""", + """CREATE TABLE IF NOT EXISTS _views ( + name TEXT PRIMARY KEY, sql TEXT NOT NULL, + description TEXT, created_at INTEGER)""", +] + +# CC-specific enrichment stubs -- coding-agent graph intelligence tables. +_CC_ENRICHMENT_STUBS: list[str] = [ """CREATE TABLE IF NOT EXISTS _enrich_source_graph ( source_id TEXT PRIMARY KEY, centrality REAL, is_hub INTEGER DEFAULT 0, is_bridge INTEGER DEFAULT 0, community_id INTEGER, community_label TEXT)""", @@ -27,16 +39,11 @@ source_id TEXT PRIMARY KEY, agents_spawned INTEGER, is_orchestrator INTEGER DEFAULT 0, delegation_depth INTEGER, parent_session TEXT)""", - """CREATE TABLE IF NOT EXISTS _ops ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - timestamp INTEGER DEFAULT (strftime('%s','now')), - operation TEXT, target TEXT, sql TEXT, params TEXT, - rows_affected INTEGER, source TEXT)""", - """CREATE TABLE IF NOT EXISTS _views ( - name TEXT PRIMARY KEY, sql TEXT NOT NULL, - description TEXT, created_at INTEGER)""", ] +# Full coding-agent enrichment stubs (base + CC). Backward-compatible. +ENRICHMENT_STUBS: list[str] = BASE_ENRICHMENT_STUBS + _CC_ENRICHMENT_STUBS + def __getattr__(name): """Lazy import of run_enrichment — avoids heavy module load at package init.""" diff --git a/flex/modules/claude_code/coding_agent_install.py b/flex/modules/claude_code/coding_agent_install.py index 8fe7da0..479a8d6 100644 --- a/flex/modules/claude_code/coding_agent_install.py +++ b/flex/modules/claude_code/coding_agent_install.py @@ -70,16 +70,17 @@ def run_from_spec(args, console, spec: dict[str, Any]) -> None: from rich.progress import BarColumn, Progress, SpinnerColumn, TextColumn from rich.text import Text - from flex.modules.claude_code import ENRICHMENT_STUBS, run_enrichment + from flex.modules.claude_code import BASE_ENRICHMENT_STUBS, ENRICHMENT_STUBS, run_enrichment from flex.modules.claude_code.compile.worker import ( _batch_embed_chunks, - bootstrap_claude_code_cell, + bootstrap_cell, ) - from flex.modules.claude_code.contract import validate_coding_agent_cell + from flex.modules.claude_code.contract import validate_base_cell, validate_coding_agent_cell from flex.registry import register_cell from flex.cli import _install_claude_assets cell_type = spec["cell_type"] + substrate = spec.get("substrate", "claude_code") name = getattr(args, "name", None) or spec.get("default_cell_name") or cell_type description = spec.get("description") or f"{cell_type} coding-agent session provenance." source_attr = spec["source_arg"].lstrip("-").replace("-", "_") @@ -94,13 +95,14 @@ def run_from_spec(args, console, spec: dict[str, Any]) -> None: console.print(f" [yellow]not found[/yellow] — {spec.get('missing_hint', 'run the source agent at least once.')}") return - db_path = bootstrap_claude_code_cell(name=name, cell_type=cell_type) + db_path = bootstrap_cell(name=name, cell_type=cell_type, substrate=substrate) conn = sqlite3.connect(str(db_path), timeout=30.0) conn.row_factory = sqlite3.Row conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA busy_timeout=30000") - for ddl in ENRICHMENT_STUBS: + stubs = ENRICHMENT_STUBS if substrate == "claude_code" else BASE_ENRICHMENT_STUBS + for ddl in stubs: conn.execute(ddl) conn.execute( "INSERT OR REPLACE INTO _meta (key, value) VALUES ('description', ?)", @@ -161,7 +163,10 @@ def _e_cb(done, total): def _g_cb(step): progress.update(t_graph, info=step) - n_comm, failed = run_enrichment(conn, cell_type=cell_type, progress_cb=_g_cb) + if substrate == "claude_code": + n_comm, failed = run_enrichment(conn, cell_type=cell_type, progress_cb=_g_cb) + else: + n_comm, failed = 0, [] progress.update( t_graph, visible=True, @@ -181,7 +186,10 @@ def _g_cb(step): except OSError: pass - report = validate_coding_agent_cell(conn, cell_type=cell_type) + if substrate == "claude_code": + report = validate_coding_agent_cell(conn, cell_type=cell_type) + else: + report = validate_base_cell(conn, cell_type=cell_type) if not report.ok or report.warnings: console.print() console.print(f" [yellow]{report.summary()}[/yellow]") diff --git a/flex/modules/claude_code/compile/worker.py b/flex/modules/claude_code/compile/worker.py index ed69d6d..d8d7102 100644 --- a/flex/modules/claude_code/compile/worker.py +++ b/flex/modules/claude_code/compile/worker.py @@ -289,8 +289,12 @@ def update_source_stats(conn: sqlite3.Connection, session_id: str, chunk: dict): """, (clean[:250], session_id)) -def _ensure_core_tables(conn: sqlite3.Connection): - """Create all chunk-atom tables for a fresh cell. Idempotent.""" +def _ensure_base_tables(conn: sqlite3.Connection): + """Create generic flex chunk-atom tables. Idempotent. + + These tables are the shared storage contract for all flex modules -- + not specific to any particular source type (Claude Code, Hermes, Matrix, etc.). + """ conn.executescript(""" CREATE TABLE IF NOT EXISTS _raw_chunks ( id TEXT PRIMARY KEY, @@ -327,6 +331,50 @@ def _ensure_core_tables(conn: sqlite3.Connection): CREATE INDEX IF NOT EXISTS idx_es_chunk ON _edges_source(chunk_id); CREATE INDEX IF NOT EXISTS idx_es_source ON _edges_source(source_id); + CREATE TABLE IF NOT EXISTS _meta ( + key TEXT PRIMARY KEY, + value TEXT + ); + + CREATE TABLE IF NOT EXISTS _presets ( + name TEXT PRIMARY KEY, + description TEXT, + params TEXT DEFAULT '', + sql TEXT + ); + + CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5( + content, + content='_raw_chunks', + content_rowid='rowid' + ); + """) + # FTS triggers -- can't use IF NOT EXISTS, so check first + has_trigger = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='trigger' AND name='raw_chunks_ai'" + ).fetchone() + if not has_trigger: + conn.executescript(""" + CREATE TRIGGER raw_chunks_ai AFTER INSERT ON _raw_chunks BEGIN + INSERT INTO chunks_fts(rowid, content) VALUES (new.rowid, new.content); + END; + CREATE TRIGGER raw_chunks_ad AFTER DELETE ON _raw_chunks BEGIN + INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', old.rowid, old.content); + END; + CREATE TRIGGER raw_chunks_au AFTER UPDATE ON _raw_chunks BEGIN + INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', old.rowid, old.content); + INSERT INTO chunks_fts(rowid, content) VALUES (new.rowid, new.content); + END; + """) + + +def _ensure_cc_tables(conn: sqlite3.Connection): + """Create Claude Code specific extension tables. Idempotent. + + These tables capture coding-agent concepts: tool operations, message + threading, agent delegation, soft file-op detection, and file bodies. + """ + conn.executescript(""" CREATE TABLE IF NOT EXISTS _edges_tool_ops ( chunk_id TEXT PRIMARY KEY, tool_name TEXT, @@ -382,43 +430,16 @@ def _ensure_core_tables(conn: sqlite3.Connection): position INTEGER ); CREATE INDEX IF NOT EXISTS idx_tfb_file ON _types_file_body(target_file); + """) - CREATE TABLE IF NOT EXISTS _meta ( - key TEXT PRIMARY KEY, - value TEXT - ); - - CREATE TABLE IF NOT EXISTS _presets ( - name TEXT PRIMARY KEY, - description TEXT, - params TEXT DEFAULT '', - sql TEXT - ); - CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5( - content, - content='_raw_chunks', - content_rowid='rowid' - ); - """) - # FTS triggers — can't use IF NOT EXISTS, so check first - has_trigger = conn.execute( - "SELECT 1 FROM sqlite_master WHERE type='trigger' AND name='raw_chunks_ai'" - ).fetchone() - if not has_trigger: - conn.executescript(""" - CREATE TRIGGER raw_chunks_ai AFTER INSERT ON _raw_chunks BEGIN - INSERT INTO chunks_fts(rowid, content) VALUES (new.rowid, new.content); - END; - CREATE TRIGGER raw_chunks_ad AFTER DELETE ON _raw_chunks BEGIN - INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', old.rowid, old.content); - END; - CREATE TRIGGER raw_chunks_au AFTER UPDATE ON _raw_chunks BEGIN - INSERT INTO chunks_fts(chunks_fts, rowid, content) VALUES('delete', old.rowid, old.content); - INSERT INTO chunks_fts(rowid, content) VALUES (new.rowid, new.content); - END; - """) +def _ensure_core_tables(conn: sqlite3.Connection): + """Create all chunk-atom tables for a fresh cell. Idempotent. + Backward-compatible wrapper -- calls base + CC tables. + """ + _ensure_base_tables(conn) + _ensure_cc_tables(conn) def _ensure_content_tables(conn: sqlite3.Connection): """Create content store tables if they don't exist.""" @@ -582,22 +603,37 @@ def _normalize_tool_result(content) -> str | None: return None -def insert_chunk_atom(conn: sqlite3.Connection, chunk: dict): - """Insert a chunk into all chunk-atom tables.""" +def insert_base_chunk(conn: sqlite3.Connection, chunk: dict, + source_type: str = 'claude-code'): + """Insert a chunk into base tables only. Module-agnostic. + + Writes to _raw_chunks and _edges_source. Any flex module can call + this without requiring CC-specific extension tables. + """ cur = conn.cursor() chunk_id = chunk['id'] - # _raw_chunks cur.execute(""" INSERT OR IGNORE INTO _raw_chunks (id, content, embedding, timestamp) VALUES (?, ?, ?, ?) """, (chunk_id, chunk['content'], chunk.get('embedding'), chunk['timestamp'])) - # _edges_source cur.execute(""" INSERT OR IGNORE INTO _edges_source (chunk_id, source_id, source_type, position) - VALUES (?, ?, 'claude-code', ?) - """, (chunk_id, chunk['doc_id'], chunk['chunk_number'])) + VALUES (?, ?, ?, ?) + """, (chunk_id, chunk['doc_id'], source_type, chunk['chunk_number'])) + + +def insert_chunk_atom(conn: sqlite3.Connection, chunk: dict): + """Insert a chunk into all CC chunk-atom tables. + + Calls insert_base_chunk() for generic storage, then writes + CC-specific extension tables (message types, tool ops, etc.). + """ + insert_base_chunk(conn, chunk, source_type='claude-code') + + cur = conn.cursor() + chunk_id = chunk['id'] # _types_message cur.execute(""" @@ -1289,17 +1325,19 @@ def process_queue(conn: sqlite3.Connection) -> dict: ) -def bootstrap_claude_code_cell( +def bootstrap_cell( name: str = 'claude_code', cell_type: str = 'claude-code', description: str | None = None, + substrate: str = 'claude_code', ) -> Path: - """Create a coding-agent cell with the CC canonical schema. Idempotent. + """Create a flex cell with the appropriate schema. Idempotent. + + substrate='claude_code' -- full coding-agent schema (base + CC + content + SOMA) + substrate='base' -- generic chunk schema only (base + content) Defaults preserve the original behavior — existing CC callers pass nothing and get a cell named 'claude_code' / cell_type='claude-code'. - Compatible coding-agent modules pass their own name/cell_type to reuse - the same substrate. """ desc = description or _DEFAULT_CC_DESCRIPTION existing = resolve_cell(name) @@ -1314,10 +1352,14 @@ def bootstrap_claude_code_cell( conn = sqlite3.connect(str(db_path), timeout=30.0) conn.execute("PRAGMA journal_mode=WAL") conn.execute("PRAGMA busy_timeout=30000") - _ensure_core_tables(conn) + + _ensure_base_tables(conn) _ensure_content_tables(conn) - if soma_ensure_tables: - soma_ensure_tables(conn) + + if substrate == 'claude_code': + _ensure_cc_tables(conn) + if soma_ensure_tables: + soma_ensure_tables(conn) conn.execute("INSERT OR IGNORE INTO _meta VALUES ('description', ?)", (desc,)) conn.execute("INSERT OR IGNORE INTO _meta VALUES ('cell_type', ?)", (cell_type,)) @@ -1328,6 +1370,10 @@ def bootstrap_claude_code_cell( return db_path +# Backward-compatible alias +bootstrap_claude_code_cell = bootstrap_cell + + def _batch_embed_chunks(conn, batch_size: int = 500, quiet: bool = False, progress_cb=None, embedder=None) -> int: """Phase 2 of decoupled backfill: batch embed all NULL-embedding chunks. diff --git a/flex/modules/claude_code/contract.py b/flex/modules/claude_code/contract.py index c86a09b..83ce04b 100644 --- a/flex/modules/claude_code/contract.py +++ b/flex/modules/claude_code/contract.py @@ -18,6 +18,16 @@ from dataclasses import dataclass, field +# Tables that MUST exist after ingest for any flex cell (base contract). +REQUIRED_BASE_TABLES: tuple[str, ...] = ( + "_raw_sources", + "_raw_chunks", + "_raw_content", + "_edges_source", + "_edges_raw_content", +) + + # Tables that MUST exist after ingest for any coding-agent cell. # A missing table = schema-level violation (probably a transpiler bug). REQUIRED_TABLES: tuple[str, ...] = ( @@ -157,3 +167,35 @@ def validate_coding_agent_cell( )) return report + + +def validate_base_cell( + conn: sqlite3.Connection, + cell_type: str = "unknown", +) -> ContractReport: + """ + Validate any flex cell against the base (non-coding-agent) contract. + + Checks only the generic tables that all flex modules produce. + Use this for modules with substrate='base'. + """ + n_sources = conn.execute("SELECT COUNT(*) FROM _raw_sources").fetchone()[0] + report = ContractReport(cell_type=cell_type, n_sources=n_sources) + + existing = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + for tbl in REQUIRED_BASE_TABLES: + if tbl not in existing: + report.violations.append(ContractViolation( + severity="error", + table=tbl, + message="required base table missing", + )) + + return report +