Knowledge Graph enhancements: relationship types, auto-update, definitions

.env.example

@@ -1,7 +1,12 @@
+ feature/knowledge-graph
+GROQ_API_KEY=your-groq-api-key
+JWT_SECRET_KEY=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0X3VzZXIifQ.kr2UKjrAiugP8-X_4wCC_rlFIKXnk_Vh4DCjNFxQHsQ
+
 GROQ_API_KEY=groq_api_key_here
 
 # Email OTP Verification (Optional for local dev)
 SMTP_HOST=smtp.gmail.com
 SMTP_PORT=587
 SMTP_USER=your_email@gmail.com
 SMTP_PASSWORD=your_16_character_app_password
+

ai-ml/.env.example

@@ -1,4 +1,4 @@
-GROQ_API_KEY=groq_api_key_here
+GROQ_API_KEY=your-groq-api-key
 
 # Pinecone (RAG vector database)
 PINECONE_API_KEY=your-pinecone-api-key

ai-ml/embedding/embedder.py

@@ -43,6 +43,12 @@ def embed_document(self, document: dict, user_id: str = "unknown") -> dict:
         this method. This method exists for standalone/manual use
         (see the CLI at the bottom of this file) and now writes to the
         exact same store, so both paths stay consistent.
+
+        [Task 5, pending] Auto-updating the knowledge graph on upload
+        needs to be hooked in here AND in ingestion/main.py's
+        _chunk_and_store() (the actual path real uploads use) —
+        intentionally not added yet, since the ingestion-side change
+        is being held off for review before touching shared code.
         """
         chunks = chunk_document(document)
         if not chunks:
@@ -55,6 +61,7 @@ def embed_document(self, document: dict, user_id: str = "unknown") -> dict:
             document_id=document_id,
             title=document.get("title", ""),
         )
+
         return {"document_id": document_id, "chunks_stored": stored_count}
 
     def search(
@@ -99,9 +106,7 @@ def close(self):
 # NOTE: this was previously "http://127.0.0.1:8000", which is Mu's
 # confirmed port, not Lambda ingestion's. Per the confirmed port
 # scheme (Mu=8000, Pluto=5000, Lambda ingestion=8001, Lambda
-# quiz=8002), 8001 is correct here. Flagging in case this was
-# intentional for some other reason — worth a quick sanity check
-# against P1-6's verification pass.
+# quiz=8002), 8001 is correct here.
 
 
 def _fetch_from_ingestion(pdf=None, youtube=None, article=None) -> dict:

ai-ml/ingestion/main.py

@@ -17,6 +17,13 @@
 caller can never claim to be another user. This closes the gap
 identified in NV-2: previously these endpoints had no authentication
 at all, unlike quiz_generator's endpoints, which were already correct.
+
+[Task 5] After storing a new document's chunks, the knowledge graph
+is incrementally updated (new document compared only against the
+user's existing documents, not a full rebuild) — so /graph reflects
+new uploads immediately without a manual /graph/rebuild call. Wrapped
+defensively so a graph-update failure never breaks the actual
+ingestion request the user is waiting on.
 """
 from __future__ import annotations
 import shutil
@@ -48,6 +55,20 @@ class URLRequest(BaseModel):
     # the verified JWT, never from client-supplied request data.
 
 
+def _update_knowledge_graph(user_id: str, document_id: str) -> None:
+    """
+    [Task 5] Best-effort incremental graph update. Any failure here
+    (LLM call down, graph module misconfigured, etc.) is swallowed so
+    it never blocks or breaks the ingestion response — the user's
+    upload should succeed even if the graph update doesn't.
+    """
+    try:
+        from knowledge_graph.app.services.graph_service import GraphService
+        GraphService().add_document(user_id, document_id)
+    except Exception:
+        pass
+
+
 def _chunk_and_store(result: dict, user_id: str) -> dict:
     """Shared helper: chunk an ingested document and store it in ChromaDB."""
     document_id = str(uuid.uuid4())
@@ -58,6 +79,9 @@ def _chunk_and_store(result: dict, user_id: str) -> dict:
         document_id=document_id,
         title=result.get("title", ""),
     )
+
+    _update_knowledge_graph(user_id, document_id)
+
     return {
         "document_id": document_id,
         "title": result.get("title", ""),
@@ -72,7 +96,6 @@ def _chunk_and_store(result: dict, user_id: str) -> dict:
 async def ingest_pdf_endpoint(
     file: UploadFile = File(...),
     user_id: str = Depends(get_current_user_id),
-    # user_id="test_user",
 ):
     if not file.filename or not file.filename.lower().endswith(".pdf"):
         raise HTTPException(status_code=400, detail="File must be a PDF")
@@ -100,8 +123,7 @@ async def ingest_pdf_endpoint(
 @app.post("/ingest/youtube")
 async def ingest_youtube_endpoint(
     payload: URLRequest,
-   user_id: str = Depends(get_current_user_id),
-    # user_id = "test_user",
+    user_id: str = Depends(get_current_user_id),
 ):
     try:
         result = ingest_youtube(payload.url)
@@ -111,7 +133,7 @@ async def ingest_youtube_endpoint(
         error_msg = str(e)
         status_code = 400 if "YouTube" in error_msg else 500
         raise HTTPException(status_code=status_code, detail=error_msg)
-        
+
     return {**result, **storage_info}
 
 

ai-ml/knowledge_graph/app/builders/document_graph_builder.py

@@ -13,6 +13,7 @@
 from knowledge_graph.app.utils.topic_labeler import generate_label
 from knowledge_graph.app.validators.graph_validators import validate_documents_have_vectors
 from knowledge_graph.app.models.graph_edge import GraphEdge
+from knowledge_graph.app.utils.relationship_classifier import classify_relationship
 
 
 def get_user_documents(user_id: str) -> dict:
@@ -47,10 +48,6 @@ def get_user_documents(user_id: str) -> dict:
 
 
 def build_document_graph(user_id: str) -> list:
-    """
-    Returns a list of GraphEdge.to_dict() for this user's
-    document-to-document relationships.
-    """
     docs = validate_documents_have_vectors(get_user_documents(user_id))
 
     doc_vectors = {
@@ -59,14 +56,24 @@ def build_document_graph(user_id: str) -> list:
     }
 
     edges = []
+    seen_pairs = set()  # NEW — prevents duplicate edges, either direction
+
     for (id_a, vec_a), (id_b, vec_b) in combinations(doc_vectors.items(), 2):
+        if id_a == id_b:  # NEW — explicit self-link guard, defensive
+            continue
+
+        pair_key = frozenset((id_a, id_b))  # NEW — order-independent identity
+        if pair_key in seen_pairs:  # NEW
+            continue
+        seen_pairs.add(pair_key)  # NEW
+
         score = cosine_similarity(vec_a, vec_b)
         if score >= SIMILARITY_THRESHOLD_DOCUMENT:
-            # label from a sample of each document's text, not the full text
             sample_a = " ".join(docs[id_a]["texts"][:2])
             sample_b = " ".join(docs[id_b]["texts"][:2])
 
             edge = GraphEdge(
+
                 user_id=user_id,
                 source_id=id_a,
                 target_id=id_b,
@@ -75,7 +82,8 @@ def build_document_graph(user_id: str) -> list:
                 source_title=docs[id_a]["title"],
                 target_title=docs[id_b]["title"],
                 label=generate_label(sample_a, sample_b),
+                relationship_type=classify_relationship(sample_a, sample_b),  # NEW
             )
             edges.append(edge.to_dict())
 
-    return edges
+    return edges

ai-ml/knowledge_graph/app/builders/topic_graph_builder.py

@@ -10,20 +10,18 @@
 from knowledge_graph.app.utils.topic_labeler import generate_label
 from knowledge_graph.app.models.graph_edge import GraphEdge
 from knowledge_graph.app.builders.document_graph_builder import get_user_documents
+from knowledge_graph.app.utils.relationship_classifier import classify_relationship
 
 
 def build_topic_graph(user_id: str, max_chunks: int = 200) -> list:
     """
     Returns a list of GraphEdge.to_dict() for chunk-level relationships.
 
     max_chunks caps how many of the user's chunks are compared, since
-    comparing every pair is O(n^2) — protects against runaway cost for
-    users with very large amounts of content. Chunks beyond the cap
-    are simply not included in this pass.
+    comparing every pair is O(n^2).
     """
     docs = get_user_documents(user_id)
 
-    # flatten into a single list of (chunk_id, vector, text, title)
     chunks = []
     for doc_id, data in docs.items():
         for i, (vector, text) in enumerate(zip(data["vectors"], data["texts"])):
@@ -37,10 +35,21 @@ def build_topic_graph(user_id: str, max_chunks: int = 200) -> list:
     chunks = chunks[:max_chunks]
 
     edges = []
+    seen_pairs = set()  # NEW — prevents duplicate edges, either direction
+
     for a, b in combinations(chunks, 2):
+        if a["chunk_id"] == b["chunk_id"]:  # NEW — explicit self-link guard
+            continue
+
+        pair_key = frozenset((a["chunk_id"], b["chunk_id"]))  # NEW
+        if pair_key in seen_pairs:  # NEW
+            continue
+        seen_pairs.add(pair_key)  # NEW
+
         score = cosine_similarity(a["vector"], b["vector"])
         if score >= SIMILARITY_THRESHOLD_TOPIC:
             edge = GraphEdge(
+
                 user_id=user_id,
                 source_id=a["chunk_id"],
                 target_id=b["chunk_id"],
@@ -49,7 +58,8 @@ def build_topic_graph(user_id: str, max_chunks: int = 200) -> list:
                 source_title=a["title"],
                 target_title=b["title"],
                 label=generate_label(a["text"], b["text"]),
+                relationship_type=classify_relationship(a["text"], b["text"]),  # NEW
             )
             edges.append(edge.to_dict())
 
-    return edges
+    return edges

ai-ml/knowledge_graph/app/models/graph_edge.py

@@ -10,11 +10,12 @@ class GraphEdge:
     user_id: str
     source_id: str
     target_id: str
-    node_type: str          # "document" or "topic" — matches the nodes it connects
+    node_type: str          # "document" or "topic"
     similarity: float
     source_title: str
     target_title: str
-    label: Optional[str] = None   # short human-readable reason for the connection
+    label: Optional[str] = None            # keyword-based explanation (existing)
+    relationship_type: str = "related_to"  # NEW — placeholder until LLM classification (task 3) fills this in with real types like "prerequisite_of", "example_of", etc.
 
     def to_dict(self) -> dict:
-        return asdict(self)
+        return asdict(self)

ai-ml/knowledge_graph/app/services/graph_service.py

@@ -1,6 +1,6 @@
 """
 Main integration interface for the knowledge graph module — what
-api/graph_routes.py and any other module calls.
+api/graph_routes.py, ingestion, and any other module calls.
 
 user_id is accepted as an explicit parameter on every method here
 (not sourced from a request/token internally), so this class stays
@@ -11,15 +11,24 @@
 from knowledge_graph.app.validators.graph_validators import validate_user_id
 from knowledge_graph.app.builders.document_graph_builder import build_document_graph, get_user_documents
 from knowledge_graph.app.builders.topic_graph_builder import build_topic_graph
-from knowledge_graph.app.storage import graph_store
+from knowledge_graph.app.utils.vectorizer import get_document_vector
+from knowledge_graph.app.utils.similarity import cosine_similarity
+from knowledge_graph.app.utils.topic_labeler import generate_label
+from knowledge_graph.app.utils.relationship_classifier import classify_relationship
+from knowledge_graph.app.utils.definition_generator import generate_definition
+from knowledge_graph.app.config import SIMILARITY_THRESHOLD_DOCUMENT
+from knowledge_graph.app.models.graph_edge import GraphEdge
+from knowledge_graph.app.storage import graph_store, node_store
 
 
 class GraphService:
     def build_graph(self, user_id: str, include_topics: bool = True) -> dict:
         """
-        Builds (or rebuilds) this user's full graph and persists it.
-        Returns a summary dict, not the full graph — call get_graph()
-        to read it back.
+        Full rebuild: recomputes ALL of this user's document (and
+        optionally topic) edges from scratch, replacing whatever was
+        stored before. Use add_document() instead for incrementally
+        updating the graph after a single new upload — much cheaper,
+        since it avoids re-classifying every existing pair.
         """
         validate_user_id(user_id)
 
@@ -38,6 +47,73 @@ def build_graph(self, user_id: str, include_topics: bool = True) -> dict:
             "topic_edges_created": topic_edge_count,
         }
 
+    def add_document(self, user_id: str, document_id: str) -> dict:
+        """
+        [Task 5] Incrementally updates the graph after a single new
+        document is embedded — compares only the new document against
+        existing ones, rather than re-comparing every pair in the
+        user's whole history. Avoids re-running LLM classification on
+        unchanged pairs, so cost scales with new content, not the
+        user's full document count.
+
+        Safe to call even if document_id has no chunks yet, or if the
+        user has no other documents to compare against — returns
+        edges_created: 0 in either case rather than raising.
+        """
+        validate_user_id(user_id)
+
+        docs = get_user_documents(user_id)
+        if document_id not in docs or not docs[document_id]["vectors"]:
+            return {"user_id": user_id, "document_id": document_id, "edges_created": 0}
+
+        new_vector = get_document_vector(docs[document_id]["vectors"])
+        new_title = docs[document_id]["title"]
+        new_sample = " ".join(docs[document_id]["texts"][:2])
+
+        # [Task 4] Generate this document's definition once, when it's
+        # first added — not on every /graph read, to avoid an LLM call
+        # per node per request.
+        full_text = " ".join(docs[document_id]["texts"])
+        definition = generate_definition(full_text)
+        node_store.save_node_metadata(user_id, document_id, definition)
+
+        existing_edges = graph_store.get_edges(user_id, node_type="document")
+        new_edges = []
+
+        for other_id, other_data in docs.items():
+            if other_id == document_id:
+                continue
+
+            already_linked = any(
+                {e["source_id"], e["target_id"]} == {document_id, other_id}
+                for e in existing_edges
+            )
+            if already_linked:
+                continue
+
+            other_vector = get_document_vector(other_data["vectors"])
+            score = cosine_similarity(new_vector, other_vector)
+
+            if score >= SIMILARITY_THRESHOLD_DOCUMENT:
+                other_sample = " ".join(other_data["texts"][:2])
+                edge = GraphEdge(
+                    user_id=user_id,
+                    source_id=document_id,
+                    target_id=other_id,
+                    node_type="document",
+                    similarity=round(score, 4),
+                    source_title=new_title,
+                    target_title=other_data["title"],
+                    label=generate_label(new_sample, other_sample),
+                    relationship_type=classify_relationship(new_sample, other_sample),
+                )
+                new_edges.append(edge.to_dict())
+
+        if new_edges:
+            graph_store.append_edges(user_id, new_edges, node_type="document")
+
+        return {"user_id": user_id, "document_id": document_id, "edges_created": len(new_edges)}
+
     def get_graph(self, user_id: str) -> dict:
         """
         Returns { "nodes": [...], "edges": [...] } for this user,
@@ -49,10 +125,16 @@ def get_graph(self, user_id: str) -> dict:
         validate_user_id(user_id)
 
         docs = get_user_documents(user_id)
-        nodes = [
-            {"id": doc_id, "title": data["title"], "node_type": "document"}
-            for doc_id, data in docs.items()
-        ]
+        nodes = []
+        for doc_id, data in docs.items():
+            metadata = node_store.get_node_metadata(user_id, doc_id)
+            nodes.append({
+                "id": doc_id,
+                "title": data["title"],
+                "node_type": "document",
+                "definition": metadata["definition"],  # [Task 4] "" if not yet generated
+                "source_document": data["title"],       # [Task 4] same as title for document-level nodes
+            })
 
         edges = graph_store.get_edges(user_id)
 
@@ -61,4 +143,5 @@ def get_graph(self, user_id: str) -> dict:
     def delete_graph(self, user_id: str) -> dict:
         validate_user_id(user_id)
         graph_store.delete_edges(user_id)
-        return {"user_id": user_id, "deleted": True}
+        node_store.delete_node_metadata(user_id)
+        return {"user_id": user_id, "deleted": True}