From b48ef9d829cd735e563dc2cfb0e57daa2b9e7717 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 26 Oct 2026 01:50:26 +0000 Subject: [PATCH] experiment(index): embed only a chunk's path and file name, not the repeated metadata Every chunk was embedded with nine lines of identical metadata (ids, absolute root, collection, pipeline version). That boilerplate can blur short chunks. Only rel_path and file_name are embedded now; the rest stays on the node. extractor_version 3 makes existing indexes rebuild. Pending a measured comparison on the code Q&A evaluation (baseline: recall@5 1.0, MRR 0.7, citation precision 0.6). Co-Authored-By: Claude Sonnet 5.5 Claude-Session: https://claude.ai/code/session_01SZ4EYrCPbq3hhNsUVRT46K Signed-off-by: Claude --- tests/test_index_helpers.py | 23 +++++++++++++++++++++++ trinaxai/indexing/indexer.py | 21 ++++++++++++++++++--- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/tests/test_index_helpers.py b/tests/test_index_helpers.py index 7809487..1bb2590 100644 --- a/tests/test_index_helpers.py +++ b/tests/test_index_helpers.py @@ -288,3 +288,26 @@ class FakeIndex: manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) assert manifest["default:active.txt"] == new_state["default:active.txt"] assert manifest["other:foreign.txt"] == {"unverified": True} + + +def test_only_the_file_name_and_path_are_embedded_with_a_chunk() -> None: + from llama_index.core.schema import MetadataMode, TextNode + + from trinaxai.indexing.indexer import _embed_excluded_keys + + metadata = { + "rel_path": "pkg/git_tool.py", + "file_name": "git_tool.py", + "source_id": "code-1234", + "source_root": "/home/u/proj", + "pipeline_version": "abc", + } + assert _embed_excluded_keys(metadata) == ["pipeline_version", "source_id", "source_root"] + node = TextNode( + text="def run(): ...", metadata=metadata, excluded_embed_metadata_keys=_embed_excluded_keys(metadata) + ) + embedded = node.get_content(metadata_mode=MetadataMode.EMBED) + assert "rel_path: pkg/git_tool.py" in embedded and "file_name: git_tool.py" in embedded + assert "source_id" not in embedded and "/home/u/proj" not in embedded and "pipeline_version" not in embedded + # The metadata is still stored on the node for filtering and citations. + assert node.metadata["source_id"] == "code-1234" diff --git a/trinaxai/indexing/indexer.py b/trinaxai/indexing/indexer.py index b530032..94be7b7 100644 --- a/trinaxai/indexing/indexer.py +++ b/trinaxai/indexing/indexer.py @@ -210,6 +210,15 @@ def _load_file_documents_result(path: str) -> tuple[str, list[Document], Excepti return path, [], exc +# Only the file's name and path say something about its content. The rest (ids, absolute root, collection, +# pipeline version) is the same boilerplate on every chunk, so embedding it only blurs short chunks. +_EMBED_METADATA_KEYS = frozenset({"rel_path", "file_name"}) + + +def _embed_excluded_keys(metadata: dict[str, Any]) -> list[str]: + return sorted(key for key in metadata if key not in _EMBED_METADATA_KEYS) + + def _pipeline_version() -> str: """Stable version of every setting that changes generated chunks/vectors.""" inputs = { @@ -221,7 +230,7 @@ def _pipeline_version() -> str: "code_chunk_lines": config.CODE_CHUNK_LINES, "code_chunk_overlap": config.CODE_CHUNK_LINES_OVERLAP, "code_max_chars": config.CODE_MAX_CHARS, - "extractor_version": 2, + "extractor_version": 3, } payload = json.dumps(inputs, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(payload).hexdigest()[:20] @@ -457,7 +466,7 @@ def load_docs_with_status( } ) d.metadata = metadata - d.excluded_embed_metadata_keys = [] + d.excluded_embed_metadata_keys = _embed_excluded_keys(metadata) d.excluded_llm_metadata_keys = [] result.documents.append(d) result.loaded_paths.add(fp) @@ -503,7 +512,13 @@ def build_nodes(documents: list[Document]) -> list[BaseNode]: doc_nodes = _sentence_splitter().get_nodes_from_documents([doc]) or [] prose_count += 1 if not doc_nodes and str(doc.text or "").strip(): - doc_nodes = [TextNode(text=str(doc.text).strip(), metadata=dict(doc.metadata or {}))] + doc_nodes = [ + TextNode( + text=str(doc.text).strip(), + metadata=dict(doc.metadata or {}), + excluded_embed_metadata_keys=list(getattr(doc, "excluded_embed_metadata_keys", None) or []), + ) + ] fallback += 1 nodes.extend(doc_nodes)