diff --git a/ai-ml/knowledge_graph/.gitignore b/ai-ml/knowledge_graph/.gitignore new file mode 100644 index 0000000..75c6182 --- /dev/null +++ b/ai-ml/knowledge_graph/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/ai-ml/knowledge_graph/README.md b/ai-ml/knowledge_graph/README.md new file mode 100644 index 0000000..0939041 --- /dev/null +++ b/ai-ml/knowledge_graph/README.md @@ -0,0 +1,85 @@ +# Knowledge Graph — Team Lambda + +Connects related content a user has ingested, by reusing the vectors already +produced by the embedding pipeline. Two layers: + +- **Document graph** — connects whole documents that are topically related. +- **Topic graph** — finer connections between individual chunks, within and + across documents. + +## Where this fits + +``` +ai-ml/ +├── ingestion/ +├── embedding/ +├── quiz_generator/ +└── knowledge_graph/ ← this module +``` + +## Install + +From `ai-ml/`: + +```bash +pip install -r knowledge_graph/requirements.txt +``` + +(Most of these are likely already installed via `embedding/requirements.txt` +— this file just makes the module's own dependencies explicit.) + +## Run the API + +```bash +cd ai-ml +uvicorn knowledge_graph.app.api.graph_routes:app --reload --port 8005 +``` + +Docs: http://127.0.0.1:8005/docs + +All endpoints require `Authorization: Bearer `, same as quiz_generator. + +| Endpoint | Method | Description | +|---|---|---| +| `/health` | GET | No auth — basic liveness check | +| `/graph` | GET | Returns the authenticated user's current graph (nodes + edges) | +| `/graph/rebuild` | POST | Rebuilds the graph from the user's current embedded content | +| `/graph` | DELETE | Deletes all graph edges for the authenticated user | + +## Run the tests + +```bash +cd ai-ml +pytest knowledge_graph/tests/ -v +``` + +14 tests, covering: utility math (vectorizer, similarity, labeling), graph +building against a real Chroma collection, related vs. unrelated document +detection, empty-user handling, and delete behavior. + +## How it works, briefly + +1. `document_graph_builder.py` groups a user's chunks by document, averages + each document's chunk vectors, and compares every pair using cosine + similarity (`utils/similarity.py`). +2. Pairs above `SIMILARITY_THRESHOLD_DOCUMENT` (default 0.5, see `config.py`) + become edges, each labeled with shared keywords (`utils/topic_labeler.py` + — no LLM, no paid API). +3. `topic_graph_builder.py` does the same at the individual-chunk level. +4. `graph_service.py` orchestrates both builders and persists edges via + `storage/graph_store.py` — a JSON file living alongside the existing + shared ChromaDB data directory (no new database introduced). +5. `api/graph_routes.py` exposes it all over HTTP, authenticated with the + same JWT pattern as quiz_generator (`quiz_generator.app.auth`). + +## Not included in this version + +- Frontend visualization (this module returns data, not a rendered graph) +- LLM-based relationship explanations (labeling uses keyword overlap only, + to avoid requiring a paid API key) + +## Known limitation + +`storage/graph_store.py` uses a simple JSON file, not a proper database — +fine for how this is used today (rebuilt on-demand per user), but not +safe under heavy concurrent writes. Worth revisiting if usage grows. diff --git a/ai-ml/knowledge_graph/__init__.py b/ai-ml/knowledge_graph/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/__init__.py b/ai-ml/knowledge_graph/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/api/__init__.py b/ai-ml/knowledge_graph/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/api/graph_routes.py b/ai-ml/knowledge_graph/app/api/graph_routes.py new file mode 100644 index 0000000..61d4f93 --- /dev/null +++ b/ai-ml/knowledge_graph/app/api/graph_routes.py @@ -0,0 +1,61 @@ +""" +Knowledge Graph API — FastAPI service. + +Run from ai-ml/ (so the knowledge_graph.* and embedding.* imports resolve): + uvicorn knowledge_graph.app.api.graph_routes:app --reload --port 8003 + +Interactive docs: http://127.0.0.1:8003/docs + +[Built in from the start, per NV-2] Every endpoint below requires a +valid JWT, reusing quiz_generator.app.auth's existing, working +implementation — the same fix already applied to ingestion. user_id +is never accepted from the client; it comes only from the verified +token's `sub` claim. +""" +from fastapi import FastAPI, Depends +from fastapi.middleware.cors import CORSMiddleware + +from quiz_generator.app.auth import get_current_user_id +from knowledge_graph.app.services.graph_service import GraphService + +app = FastAPI(title="StudyMind Knowledge Graph API — Team Lambda") + +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:5173"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +_service: GraphService | None = None + + +def get_service() -> GraphService: + global _service + if _service is None: + _service = GraphService() + return _service + + +@app.get("/health") +def health_check(): + return {"status": "ok"} + + +@app.get("/graph") +def get_graph_endpoint(user_id: str = Depends(get_current_user_id)) -> dict: + """Returns the authenticated user's current knowledge graph.""" + return get_service().get_graph(user_id) + + +@app.post("/graph/rebuild") +def rebuild_graph_endpoint(user_id: str = Depends(get_current_user_id)) -> dict: + """Rebuilds the authenticated user's knowledge graph from their current content.""" + return get_service().build_graph(user_id) + + +@app.delete("/graph") +def delete_graph_endpoint(user_id: str = Depends(get_current_user_id)) -> dict: + """Deletes all graph edges for the authenticated user.""" + return get_service().delete_graph(user_id) diff --git a/ai-ml/knowledge_graph/app/builders/__init__.py b/ai-ml/knowledge_graph/app/builders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/builders/document_graph_builder.py b/ai-ml/knowledge_graph/app/builders/document_graph_builder.py new file mode 100644 index 0000000..95109cd --- /dev/null +++ b/ai-ml/knowledge_graph/app/builders/document_graph_builder.py @@ -0,0 +1,81 @@ +""" +Builds document-to-document edges: groups a user's chunks by +document, averages each document's chunk vectors, compares every +pair of documents, and creates an edge wherever similarity is above +the configured threshold. +""" +from itertools import combinations + +from embedding.chroma_store import get_collection +from knowledge_graph.app.config import SIMILARITY_THRESHOLD_DOCUMENT, CONTENT_COLLECTION_NAME +from knowledge_graph.app.utils.vectorizer import get_document_vector +from knowledge_graph.app.utils.similarity import cosine_similarity +from knowledge_graph.app.utils.topic_labeler import generate_label +from knowledge_graph.app.validators.graph_validators import validate_documents_have_vectors +from knowledge_graph.app.models.graph_edge import GraphEdge + + +def get_user_documents(user_id: str) -> dict: + """ + Fetches this user's chunks from the shared ChromaDB collection and + groups them by document_id. + + Returns: { document_id: {"title": str, "vectors": [...], "texts": [...]} } + """ + collection = get_collection(name=CONTENT_COLLECTION_NAME) + result = collection.get( + where={"user_id": user_id}, + include=["embeddings", "documents", "metadatas"], + ) + + docs: dict = {} + ids = result.get("ids", []) + embeddings = result.get("embeddings", []) + documents = result.get("documents", []) + metadatas = result.get("metadatas", []) + + for i in range(len(ids)): + meta = metadatas[i] or {} + doc_id = meta.get("document_id") + if not doc_id: + continue + docs.setdefault(doc_id, {"title": meta.get("document", ""), "vectors": [], "texts": []}) + docs[doc_id]["vectors"].append(embeddings[i]) + docs[doc_id]["texts"].append(documents[i]) + + return docs + + +def build_document_graph(user_id: str) -> list: + """ + Returns a list of GraphEdge.to_dict() for this user's + document-to-document relationships. + """ + docs = validate_documents_have_vectors(get_user_documents(user_id)) + + doc_vectors = { + doc_id: get_document_vector(data["vectors"]) + for doc_id, data in docs.items() + } + + edges = [] + for (id_a, vec_a), (id_b, vec_b) in combinations(doc_vectors.items(), 2): + score = cosine_similarity(vec_a, vec_b) + if score >= SIMILARITY_THRESHOLD_DOCUMENT: + # label from a sample of each document's text, not the full text + sample_a = " ".join(docs[id_a]["texts"][:2]) + sample_b = " ".join(docs[id_b]["texts"][:2]) + + edge = GraphEdge( + user_id=user_id, + source_id=id_a, + target_id=id_b, + node_type="document", + similarity=round(score, 4), + source_title=docs[id_a]["title"], + target_title=docs[id_b]["title"], + label=generate_label(sample_a, sample_b), + ) + edges.append(edge.to_dict()) + + return edges diff --git a/ai-ml/knowledge_graph/app/builders/topic_graph_builder.py b/ai-ml/knowledge_graph/app/builders/topic_graph_builder.py new file mode 100644 index 0000000..07367df --- /dev/null +++ b/ai-ml/knowledge_graph/app/builders/topic_graph_builder.py @@ -0,0 +1,55 @@ +""" +Builds finer-grained edges between individual chunks ("topics"), +within and across a user's documents — the detail layer the +document-level graph misses. +""" +from itertools import combinations + +from knowledge_graph.app.config import SIMILARITY_THRESHOLD_TOPIC +from knowledge_graph.app.utils.similarity import cosine_similarity +from knowledge_graph.app.utils.topic_labeler import generate_label +from knowledge_graph.app.models.graph_edge import GraphEdge +from knowledge_graph.app.builders.document_graph_builder import get_user_documents + + +def build_topic_graph(user_id: str, max_chunks: int = 200) -> list: + """ + Returns a list of GraphEdge.to_dict() for chunk-level relationships. + + max_chunks caps how many of the user's chunks are compared, since + comparing every pair is O(n^2) — protects against runaway cost for + users with very large amounts of content. Chunks beyond the cap + are simply not included in this pass. + """ + docs = get_user_documents(user_id) + + # flatten into a single list of (chunk_id, vector, text, title) + chunks = [] + for doc_id, data in docs.items(): + for i, (vector, text) in enumerate(zip(data["vectors"], data["texts"])): + chunks.append({ + "chunk_id": f"{doc_id}_{i}", + "vector": vector, + "text": text, + "title": data["title"], + }) + + chunks = chunks[:max_chunks] + + edges = [] + for a, b in combinations(chunks, 2): + score = cosine_similarity(a["vector"], b["vector"]) + if score >= SIMILARITY_THRESHOLD_TOPIC: + edge = GraphEdge( + user_id=user_id, + source_id=a["chunk_id"], + target_id=b["chunk_id"], + node_type="topic", + similarity=round(score, 4), + source_title=a["title"], + target_title=b["title"], + label=generate_label(a["text"], b["text"]), + ) + edges.append(edge.to_dict()) + + return edges diff --git a/ai-ml/knowledge_graph/app/config.py b/ai-ml/knowledge_graph/app/config.py new file mode 100644 index 0000000..64d40e4 --- /dev/null +++ b/ai-ml/knowledge_graph/app/config.py @@ -0,0 +1,36 @@ +""" +Configuration for the knowledge graph module. + +Reuses the shared .env at the ai-ml/ root, same pattern as +embedding/config.py and quiz_generator/app/auth.py. +""" +import os +from pathlib import Path +from dotenv import load_dotenv + +# ai-ml/.env (app -> knowledge-graph -> ai-ml) +load_dotenv(dotenv_path=Path(__file__).resolve().parent.parent.parent / ".env") + +# --- Similarity thresholds --- +# Document-level pairs below this score are not considered related. +SIMILARITY_THRESHOLD_DOCUMENT = float(os.getenv("KG_DOC_SIMILARITY_THRESHOLD", "0.5")) + +# Topic/chunk-level pairs below this score are not considered related. +# Slightly higher than the document threshold, since chunk-level +# comparisons are noisier (less text per vector). +SIMILARITY_THRESHOLD_TOPIC = float(os.getenv("KG_TOPIC_SIMILARITY_THRESHOLD", "0.6")) + +# --- Storage --- +# Reuses the same local ChromaDB path as embedding/chroma_store.py, +# so this module reads the same underlying data directory rather than +# introducing a second database location. Edge data is written to its +# own JSON file inside that same directory (see storage/graph_store.py) — +# not a new database technology, just a new file alongside Chroma's data. +CHROMA_DB_PATH = os.getenv( + "CHROMA_DB_PATH", + r"C:\Dev\QuantumLearningWorkspace\shared_chroma_data", +) +CONTENT_COLLECTION_NAME = "study_chunks" # matches chroma_store.py's DEFAULT_COLLECTION_NAME + +# --- API --- +API_PORT = int(os.getenv("KNOWLEDGE_GRAPH_PORT", "8005")) diff --git a/ai-ml/knowledge_graph/app/models/__init__.py b/ai-ml/knowledge_graph/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/models/graph_edge.py b/ai-ml/knowledge_graph/app/models/graph_edge.py new file mode 100644 index 0000000..4cc8705 --- /dev/null +++ b/ai-ml/knowledge_graph/app/models/graph_edge.py @@ -0,0 +1,20 @@ +""" +Represents a relationship (edge) between two nodes in the graph. +""" +from dataclasses import dataclass, asdict +from typing import Optional + + +@dataclass +class GraphEdge: + user_id: str + source_id: str + target_id: str + node_type: str # "document" or "topic" — matches the nodes it connects + similarity: float + source_title: str + target_title: str + label: Optional[str] = None # short human-readable reason for the connection + + def to_dict(self) -> dict: + return asdict(self) diff --git a/ai-ml/knowledge_graph/app/models/graph_node.py b/ai-ml/knowledge_graph/app/models/graph_node.py new file mode 100644 index 0000000..a6f3589 --- /dev/null +++ b/ai-ml/knowledge_graph/app/models/graph_node.py @@ -0,0 +1,16 @@ +""" +Represents a single node in the knowledge graph — either a whole +document, or (for the topic-level graph) an individual chunk. +""" +from dataclasses import dataclass, asdict + + +@dataclass +class GraphNode: + node_id: str # document_id, or "documentid_chunkindex" for topic nodes + node_type: str # "document" or "topic" + title: str + user_id: str + + def to_dict(self) -> dict: + return asdict(self) diff --git a/ai-ml/knowledge_graph/app/services/__init__.py b/ai-ml/knowledge_graph/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/services/graph_service.py b/ai-ml/knowledge_graph/app/services/graph_service.py new file mode 100644 index 0000000..9478b52 --- /dev/null +++ b/ai-ml/knowledge_graph/app/services/graph_service.py @@ -0,0 +1,64 @@ +""" +Main integration interface for the knowledge graph module — what +api/graph_routes.py and any other module calls. + +user_id is accepted as an explicit parameter on every method here +(not sourced from a request/token internally), so this class stays +easy to unit test with any user_id value. Only the API layer +(api/graph_routes.py) is responsible for sourcing a real, verified +user_id from a JWT before calling into this service. +""" +from knowledge_graph.app.validators.graph_validators import validate_user_id +from knowledge_graph.app.builders.document_graph_builder import build_document_graph, get_user_documents +from knowledge_graph.app.builders.topic_graph_builder import build_topic_graph +from knowledge_graph.app.storage import graph_store + + +class GraphService: + def build_graph(self, user_id: str, include_topics: bool = True) -> dict: + """ + Builds (or rebuilds) this user's full graph and persists it. + Returns a summary dict, not the full graph — call get_graph() + to read it back. + """ + validate_user_id(user_id) + + document_edges = build_document_graph(user_id) + graph_store.save_edges(user_id, document_edges, node_type="document") + + topic_edge_count = 0 + if include_topics: + topic_edges = build_topic_graph(user_id) + graph_store.save_edges(user_id, topic_edges, node_type="topic") + topic_edge_count = len(topic_edges) + + return { + "user_id": user_id, + "document_edges_created": len(document_edges), + "topic_edges_created": topic_edge_count, + } + + def get_graph(self, user_id: str) -> dict: + """ + Returns { "nodes": [...], "edges": [...] } for this user, + combining both document- and topic-level edges. Nodes are + derived from the user's currently embedded documents, so the + node list always reflects current content even if the graph + hasn't been rebuilt since the last edit. + """ + validate_user_id(user_id) + + docs = get_user_documents(user_id) + nodes = [ + {"id": doc_id, "title": data["title"], "node_type": "document"} + for doc_id, data in docs.items() + ] + + edges = graph_store.get_edges(user_id) + + return {"nodes": nodes, "edges": edges} + + def delete_graph(self, user_id: str) -> dict: + validate_user_id(user_id) + graph_store.delete_edges(user_id) + return {"user_id": user_id, "deleted": True} diff --git a/ai-ml/knowledge_graph/app/storage/__init__.py b/ai-ml/knowledge_graph/app/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/storage/graph_store.py b/ai-ml/knowledge_graph/app/storage/graph_store.py new file mode 100644 index 0000000..5c40d2f --- /dev/null +++ b/ai-ml/knowledge_graph/app/storage/graph_store.py @@ -0,0 +1,76 @@ +""" +Stores graph edges as a JSON file inside the same shared data +directory ChromaDB already uses (CHROMA_DB_PATH). This is +deliberately not a new database technology — edges are small, +relationship-only records, so a flat JSON file keeps this module +free of new infrastructure dependencies while still living +alongside the "one shared data location" the rest of the project +already uses. + +Not safe for many concurrent writers, but fine for how this module +is used today: rebuilt on-demand per user, not written to under +sustained concurrent load. +""" +import json +import os +from pathlib import Path +from threading import Lock + +from knowledge_graph.app.config import CHROMA_DB_PATH + +_EDGES_FILENAME = "knowledge_graph_edges.json" +_lock = Lock() + + +def _edges_path() -> Path: + return Path(CHROMA_DB_PATH) / _EDGES_FILENAME + + +def _read_all() -> list: + path = _edges_path() + if not path.exists(): + return [] + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _write_all(edges: list) -> None: + path = _edges_path() + os.makedirs(path.parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(edges, f, indent=2) + + +def save_edges(user_id: str, edges: list, node_type: str) -> int: + """ + Replaces all existing edges of the given node_type for this user + with the new set (a full rebuild, not an incremental append — + keeps the graph consistent with the latest embedded content). + edges: list of GraphEdge.to_dict() results. + Returns the number of edges written. + """ + with _lock: + all_edges = _read_all() + # drop this user's existing edges of this type, keep everyone else's + all_edges = [ + e for e in all_edges + if not (e["user_id"] == user_id and e["node_type"] == node_type) + ] + all_edges.extend(edges) + _write_all(all_edges) + return len(edges) + + +def get_edges(user_id: str, node_type: str = None) -> list: + all_edges = _read_all() + result = [e for e in all_edges if e["user_id"] == user_id] + if node_type: + result = [e for e in result if e["node_type"] == node_type] + return result + + +def delete_edges(user_id: str) -> None: + with _lock: + all_edges = _read_all() + all_edges = [e for e in all_edges if e["user_id"] != user_id] + _write_all(all_edges) diff --git a/ai-ml/knowledge_graph/app/utils/__init__.py b/ai-ml/knowledge_graph/app/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/utils/similarity.py b/ai-ml/knowledge_graph/app/utils/similarity.py new file mode 100644 index 0000000..0f7f8f0 --- /dev/null +++ b/ai-ml/knowledge_graph/app/utils/similarity.py @@ -0,0 +1,23 @@ +""" +Cosine similarity between two vectors. +""" +import numpy as np + + +def cosine_similarity(vec_a: list, vec_b: list) -> float: + """ + Returns a score from -1 to 1. Closer to 1 means more related. + + Raises ValueError if either vector has zero magnitude (all zeros), + since cosine similarity is undefined in that case. + """ + a = np.array(vec_a) + b = np.array(vec_b) + + norm_a = np.linalg.norm(a) + norm_b = np.linalg.norm(b) + + if norm_a == 0 or norm_b == 0: + raise ValueError("cosine_similarity() received a zero-magnitude vector") + + return float(np.dot(a, b) / (norm_a * norm_b)) diff --git a/ai-ml/knowledge_graph/app/utils/topic_labeler.py b/ai-ml/knowledge_graph/app/utils/topic_labeler.py new file mode 100644 index 0000000..76623a4 --- /dev/null +++ b/ai-ml/knowledge_graph/app/utils/topic_labeler.py @@ -0,0 +1,40 @@ +""" +Generates a short, human-readable label explaining why two chunks of +text are related — using shared keyword overlap, not an LLM. Keeps +this free of any paid API dependency. +""" +import re +from collections import Counter + +# Small built-in stopword list — enough to filter common noise words +# without pulling in a heavier NLP dependency for this lightweight task. +_STOPWORDS = { + "the", "a", "an", "and", "or", "but", "is", "are", "was", "were", + "in", "on", "at", "to", "for", "of", "with", "by", "as", "that", + "this", "it", "be", "from", "which", "these", "those", "their", + "its", "has", "have", "had", "not", "can", "will", "would", "also", +} + + +def _extract_keywords(text: str, top_n: int = 15) -> set: + words = re.findall(r"[a-zA-Z]{3,}", text.lower()) + words = [w for w in words if w not in _STOPWORDS] + most_common = [w for w, _ in Counter(words).most_common(top_n)] + return set(most_common) + + +def generate_label(text_a: str, text_b: str, max_terms: int = 3) -> str: + """ + Returns a short label like "shared terms: neural networks, training" + based on keyword overlap between two texts. Returns a generic + fallback label if no meaningful overlap is found. + """ + keywords_a = _extract_keywords(text_a) + keywords_b = _extract_keywords(text_b) + shared = keywords_a & keywords_b + + if not shared: + return "related topics" + + top_shared = sorted(shared)[:max_terms] + return f"shared terms: {', '.join(top_shared)}" diff --git a/ai-ml/knowledge_graph/app/utils/vectorizer.py b/ai-ml/knowledge_graph/app/utils/vectorizer.py new file mode 100644 index 0000000..fd82676 --- /dev/null +++ b/ai-ml/knowledge_graph/app/utils/vectorizer.py @@ -0,0 +1,21 @@ +""" +Turns a document's many chunk vectors into a single vector +representing the whole document. +""" +import numpy as np + + +def get_document_vector(chunk_vectors: list) -> list: + """ + Averages a list of chunk vectors into one document-level vector. + + chunk_vectors: list of vectors (list[float]), all the same length, + belonging to one document. + + Raises ValueError if no vectors are given, so callers can decide + how to handle documents with no embedded chunks (e.g. skip them). + """ + if not chunk_vectors: + raise ValueError("get_document_vector() requires at least one chunk vector") + + return np.mean(np.array(chunk_vectors), axis=0).tolist() diff --git a/ai-ml/knowledge_graph/app/validators/__init__.py b/ai-ml/knowledge_graph/app/validators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/validators/graph_validators.py b/ai-ml/knowledge_graph/app/validators/graph_validators.py new file mode 100644 index 0000000..5045717 --- /dev/null +++ b/ai-ml/knowledge_graph/app/validators/graph_validators.py @@ -0,0 +1,21 @@ +""" +Validation checks used before building or reading a graph. +""" + + +def validate_user_id(user_id: str) -> None: + if not user_id or not isinstance(user_id, str): + raise ValueError("A valid user_id is required") + + +def validate_documents_have_vectors(docs: dict) -> dict: + """ + Filters out any document with no chunk vectors, so a document + that failed to embed properly doesn't crash graph building. + Returns the filtered dict; does not raise. + """ + return { + doc_id: data + for doc_id, data in docs.items() + if data.get("vectors") + } diff --git a/ai-ml/knowledge_graph/requirements.txt b/ai-ml/knowledge_graph/requirements.txt new file mode 100644 index 0000000..8370127 --- /dev/null +++ b/ai-ml/knowledge_graph/requirements.txt @@ -0,0 +1,12 @@ +# Core dependencies — numpy and chromadb are likely already installed +# via embedding/requirements.txt, but listed here so this module's +# requirements are self-contained. +numpy +chromadb +fastapi +uvicorn +python-dotenv +pyjwt + +# For running tests +pytest diff --git a/ai-ml/knowledge_graph/tests/__init__.py b/ai-ml/knowledge_graph/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/tests/test_graph_service.py b/ai-ml/knowledge_graph/tests/test_graph_service.py new file mode 100644 index 0000000..ea8e143 --- /dev/null +++ b/ai-ml/knowledge_graph/tests/test_graph_service.py @@ -0,0 +1,93 @@ +""" +Integration tests for GraphService, against a real (ephemeral, +in-memory) ChromaDB collection — not mocked, so these catch real +wiring issues between the builders, storage, and Chroma. + +Run: pytest knowledge_graph/tests/test_graph_service.py +""" +import numpy as np +import pytest + +from embedding.chroma_store import get_collection +from knowledge_graph.app.services.graph_service import GraphService + + +def _make_vec(base: float, dim: int = 16, seed: int = 0) -> list: + rng = np.random.default_rng(seed) + return rng.normal(loc=base, scale=0.03, size=dim).tolist() + + +@pytest.fixture +def seeded_user(tmp_path): + """Inserts two related documents and one unrelated document for a test user.""" + user_id = "pytest_user" + collection = get_collection() + + rows = [ + ("docA", 0, "ML Basics", "Neural networks learn from data.", 0.8, 1), + ("docA", 1, "ML Basics", "Training uses gradient descent.", 0.8, 2), + ("docB", 0, "Deep Learning", "Gradient descent trains neural networks.", 0.82, 3), + ("docC", 0, "Cooking", "Bread requires yeast and flour.", -0.8, 4), + ] + ids, embeddings, documents, metadatas = [], [], [], [] + for doc_id, idx, title, text, base, seed in rows: + ids.append(f"{doc_id}_{idx}") + embeddings.append(_make_vec(base, seed=seed)) + documents.append(text) + metadatas.append({"user_id": user_id, "document_id": doc_id, "document": title, "chunk_index": idx}) + + collection.upsert(ids=ids, embeddings=embeddings, documents=documents, metadatas=metadatas) + return user_id + + +def test_build_graph_links_related_documents(seeded_user): + service = GraphService() + result = service.build_graph(seeded_user) + assert result["document_edges_created"] >= 1 + + graph = service.get_graph(seeded_user) + doc_edges = [e for e in graph["edges"] if e["node_type"] == "document"] + titles = {(e["source_title"], e["target_title"]) for e in doc_edges} + + assert ("ML Basics", "Deep Learning") in titles or ("Deep Learning", "ML Basics") in titles + + +def test_build_graph_does_not_link_unrelated_documents(seeded_user): + service = GraphService() + service.build_graph(seeded_user) + graph = service.get_graph(seeded_user) + + doc_edges = [e for e in graph["edges"] if e["node_type"] == "document"] + cooking_involved = any( + "Cooking" in (e["source_title"], e["target_title"]) for e in doc_edges + ) + assert not cooking_involved + + +def test_get_graph_nodes_match_documents(seeded_user): + service = GraphService() + graph = service.get_graph(seeded_user) + titles = {n["title"] for n in graph["nodes"]} + assert titles == {"ML Basics", "Deep Learning", "Cooking"} + + +def test_delete_graph_clears_edges(seeded_user): + service = GraphService() + service.build_graph(seeded_user) + service.delete_graph(seeded_user) + + graph = service.get_graph(seeded_user) + assert graph["edges"] == [] + + +def test_user_with_no_documents_does_not_crash(): + service = GraphService() + result = service.build_graph("nobody_has_this_id") + assert result["document_edges_created"] == 0 + assert result["topic_edges_created"] == 0 + + +def test_empty_user_id_raises(): + service = GraphService() + with pytest.raises(ValueError): + service.get_graph("") diff --git a/ai-ml/knowledge_graph/tests/test_utils.py b/ai-ml/knowledge_graph/tests/test_utils.py new file mode 100644 index 0000000..dc5c7d1 --- /dev/null +++ b/ai-ml/knowledge_graph/tests/test_utils.py @@ -0,0 +1,53 @@ +""" +Tests for the standalone utility functions — no database needed. +Run: pytest knowledge_graph/tests/test_utils.py +""" +import pytest + +from knowledge_graph.app.utils.vectorizer import get_document_vector +from knowledge_graph.app.utils.similarity import cosine_similarity +from knowledge_graph.app.utils.topic_labeler import generate_label + + +def test_get_document_vector_averages_correctly(): + chunks = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]] + result = get_document_vector(chunks) + assert abs(result[0] - 1 / 3) < 0.001 + assert abs(result[1] - 1 / 3) < 0.001 + assert abs(result[2] - 1 / 3) < 0.001 + + +def test_get_document_vector_empty_raises(): + with pytest.raises(ValueError): + get_document_vector([]) + + +def test_cosine_similarity_identical_vectors(): + assert abs(cosine_similarity([1.0, 0.0], [1.0, 0.0]) - 1.0) < 0.001 + + +def test_cosine_similarity_orthogonal_vectors(): + assert abs(cosine_similarity([1.0, 0.0], [0.0, 1.0]) - 0.0) < 0.001 + + +def test_cosine_similarity_opposite_vectors(): + assert abs(cosine_similarity([1.0, 0.0], [-1.0, 0.0]) - (-1.0)) < 0.001 + + +def test_cosine_similarity_zero_vector_raises(): + with pytest.raises(ValueError): + cosine_similarity([0.0, 0.0], [1.0, 0.0]) + + +def test_generate_label_finds_shared_terms(): + label = generate_label( + "Backpropagation computes gradients in a neural network", + "Gradient descent updates the neural network weights", + ) + assert "shared terms" in label + assert "neural" in label or "gradient" in label + + +def test_generate_label_no_overlap_falls_back(): + label = generate_label("Roman Empire history ancient", "Quantum physics particles") + assert label == "related topics"