diff --git a/tests/test_index_helpers.py b/tests/test_index_helpers.py index 7809487..1bb2590 100644 --- a/tests/test_index_helpers.py +++ b/tests/test_index_helpers.py @@ -288,3 +288,26 @@ class FakeIndex: manifest = json.loads((tmp_path / "manifest.json").read_text(encoding="utf-8")) assert manifest["default:active.txt"] == new_state["default:active.txt"] assert manifest["other:foreign.txt"] == {"unverified": True} + + +def test_only_the_file_name_and_path_are_embedded_with_a_chunk() -> None: + from llama_index.core.schema import MetadataMode, TextNode + + from trinaxai.indexing.indexer import _embed_excluded_keys + + metadata = { + "rel_path": "pkg/git_tool.py", + "file_name": "git_tool.py", + "source_id": "code-1234", + "source_root": "/home/u/proj", + "pipeline_version": "abc", + } + assert _embed_excluded_keys(metadata) == ["pipeline_version", "source_id", "source_root"] + node = TextNode( + text="def run(): ...", metadata=metadata, excluded_embed_metadata_keys=_embed_excluded_keys(metadata) + ) + embedded = node.get_content(metadata_mode=MetadataMode.EMBED) + assert "rel_path: pkg/git_tool.py" in embedded and "file_name: git_tool.py" in embedded + assert "source_id" not in embedded and "/home/u/proj" not in embedded and "pipeline_version" not in embedded + # The metadata is still stored on the node for filtering and citations. + assert node.metadata["source_id"] == "code-1234" diff --git a/trinaxai/indexing/indexer.py b/trinaxai/indexing/indexer.py index b530032..94be7b7 100644 --- a/trinaxai/indexing/indexer.py +++ b/trinaxai/indexing/indexer.py @@ -210,6 +210,15 @@ def _load_file_documents_result(path: str) -> tuple[str, list[Document], Excepti return path, [], exc +# Only the file's name and path say something about its content. The rest (ids, absolute root, collection, +# pipeline version) is the same boilerplate on every chunk, so embedding it only blurs short chunks. +_EMBED_METADATA_KEYS = frozenset({"rel_path", "file_name"}) + + +def _embed_excluded_keys(metadata: dict[str, Any]) -> list[str]: + return sorted(key for key in metadata if key not in _EMBED_METADATA_KEYS) + + def _pipeline_version() -> str: """Stable version of every setting that changes generated chunks/vectors.""" inputs = { @@ -221,7 +230,7 @@ def _pipeline_version() -> str: "code_chunk_lines": config.CODE_CHUNK_LINES, "code_chunk_overlap": config.CODE_CHUNK_LINES_OVERLAP, "code_max_chars": config.CODE_MAX_CHARS, - "extractor_version": 2, + "extractor_version": 3, } payload = json.dumps(inputs, sort_keys=True, separators=(",", ":")).encode("utf-8") return hashlib.sha256(payload).hexdigest()[:20] @@ -457,7 +466,7 @@ def load_docs_with_status( } ) d.metadata = metadata - d.excluded_embed_metadata_keys = [] + d.excluded_embed_metadata_keys = _embed_excluded_keys(metadata) d.excluded_llm_metadata_keys = [] result.documents.append(d) result.loaded_paths.add(fp) @@ -503,7 +512,13 @@ def build_nodes(documents: list[Document]) -> list[BaseNode]: doc_nodes = _sentence_splitter().get_nodes_from_documents([doc]) or [] prose_count += 1 if not doc_nodes and str(doc.text or "").strip(): - doc_nodes = [TextNode(text=str(doc.text).strip(), metadata=dict(doc.metadata or {}))] + doc_nodes = [ + TextNode( + text=str(doc.text).strip(), + metadata=dict(doc.metadata or {}), + excluded_embed_metadata_keys=list(getattr(doc, "excluded_embed_metadata_keys", None) or []), + ) + ] fallback += 1 nodes.extend(doc_nodes)