From 5c864affca12c479b75d559a9d0c9e35090bbc6e Mon Sep 17 00:00:00 2001 From: Matee ur Rehman Date: Sat, 19 Sep 2026 17:12:00 +0500 Subject: [PATCH] Add multi-mode roadmap generation (topic/document/quiz_performance); fix .env path bug in chroma_store.py and relative path bug in weak_topic_service.py --- ai-ml/embedding/chroma_store.py | 35 ++++- ai-ml/roadmap_generator/app/main.py | 50 +++++--- .../app/models/api_models.py | 51 ++++++-- .../app/services/roadmap_service.py | 118 ++++++++++++++--- .../app/utils/topic_extractor.py | 38 ++++++ .../app/services/weak_topic_service.py | 20 ++- docs/api-contracts.md | 121 +++++++++++++++++- 7 files changed, 379 insertions(+), 54 deletions(-) create mode 100644 ai-ml/roadmap_generator/app/utils/topic_extractor.py diff --git a/ai-ml/embedding/chroma_store.py b/ai-ml/embedding/chroma_store.py index b4467cb..c9ea8f4 100644 --- a/ai-ml/embedding/chroma_store.py +++ b/ai-ml/embedding/chroma_store.py @@ -27,7 +27,13 @@ from embedding.model import get_embedding_model -load_dotenv(dotenv_path=Path(__file__).parent.parent.parent / ".env") +# [Fix] This file lives at ai-ml/embedding/chroma_store.py — two +# .parent calls reach ai-ml/, which is where .env actually lives. +# The previous three .parent calls pointed one level too high (the +# repo root), where no .env exists at all, so CHROMA_API_KEY / +# CHROMA_TENANT / CHROMA_DATABASE (and anything else loaded here) +# were silently never found regardless of what was in ai-ml/.env. +load_dotenv(dotenv_path=Path(__file__).parent.parent / ".env") DEFAULT_COLLECTION_NAME = "study_chunks" @@ -136,4 +142,29 @@ def delete_chunks(document_id: str) -> None: what P0-5's purge endpoint will need. """ collection = get_collection() - collection.delete(where={"document_id": document_id}) \ No newline at end of file + collection.delete(where={"document_id": document_id}) + + +def get_document_chunks(document_id: str, user_id: str = None) -> list: + """ + [Roadmap multi-mode] Returns all chunk texts for a document — a + direct metadata lookup, not a similarity search (unlike + query_chunks). Used for mode="document" roadmap generation, where + the whole document's content is needed, not just the top-k + results for some query. + + If user_id is given, only chunks whose stored user_id matches are + returned — this prevents a caller from pulling another user's + document content by guessing/supplying a document_id that isn't + theirs. Returns an empty list if the document doesn't exist, or + exists but belongs to a different user (the two cases are + intentionally indistinguishable to the caller). + """ + collection = get_collection() + + where = {"document_id": document_id} + if user_id is not None: + where = {"$and": [{"document_id": document_id}, {"user_id": user_id}]} + + results = collection.get(where=where) + return results.get("documents", []) or [] \ No newline at end of file diff --git a/ai-ml/roadmap_generator/app/main.py b/ai-ml/roadmap_generator/app/main.py index 151225c..7e13c67 100644 --- a/ai-ml/roadmap_generator/app/main.py +++ b/ai-ml/roadmap_generator/app/main.py @@ -6,13 +6,15 @@ uvicorn roadmap_generator.app.main:app --reload --port Interactive docs (once running): http://127.0.0.1:/docs + +Supports three generation modes via the "mode" field on +POST /generate-roadmap: "topic", "document", "quiz_performance". +See docs/api-contracts.md for the full contract. """ from fastapi import FastAPI, HTTPException, Depends from fastapi.middleware.cors import CORSMiddleware -# Reusing the JWT auth dependency already built for quiz_generator -# (Contract v1, Section 10) — same as weak_topic_detection's endpoint. from quiz_generator.app.auth import get_current_user_id from roadmap_generator.app.models.api_models import ( @@ -52,26 +54,35 @@ def generate_roadmap_endpoint( user_id: str = Depends(get_current_user_id), ) -> GenerateRoadmapResponse: """ - Requires "Authorization: Bearer ". - - user_id is derived from the verified token and required for - consistency with Contract v1's auth pattern (every Lambda - endpoint requires auth), but this module is general-purpose and - doesn't read per-user content — it only uses the topic_names the - caller supplies. If a future version personalizes roadmaps using - a user's own ingested content or weak-topic results, user_id is - already available here to wire that in without another contract - change. + Requires "Authorization: Bearer ". Dispatches to the correct + generation path based on body.mode: + - "topic": uses body.topic_names / body.priorities + - "document": uses body.document_id, scoped to this user + - "quiz_performance": uses this user's weak-topic results """ service = get_service() try: - roadmap = service.generate_roadmap( - topic_names=body.topic_names, - subject=body.subject, - step_count=body.step_count, - priorities=body.priorities, - ) + if body.mode == "topic": + roadmap = service.generate_roadmap( + topic_names=body.topic_names, + subject=body.subject, + step_count=body.step_count, + priorities=body.priorities, + ) + elif body.mode == "document": + roadmap = service.generate_roadmap_from_document( + user_id=user_id, + document_id=body.document_id, + subject=body.subject, + step_count=body.step_count, + ) + else: # "quiz_performance" + roadmap = service.generate_roadmap_from_quiz_performance( + user_id=user_id, + subject=body.subject, + step_count=body.step_count, + ) except ValueError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc except RuntimeError as exc: @@ -80,7 +91,8 @@ def generate_roadmap_endpoint( return GenerateRoadmapResponse( success=True, message=f"Generated {roadmap.total_steps} steps.", + mode=body.mode, subject=roadmap.subject, steps=[step.model_dump() for step in roadmap.steps], total_steps=roadmap.total_steps, - ) \ No newline at end of file + ) diff --git a/ai-ml/roadmap_generator/app/models/api_models.py b/ai-ml/roadmap_generator/app/models/api_models.py index f52087c..25b6bb6 100644 --- a/ai-ml/roadmap_generator/app/models/api_models.py +++ b/ai-ml/roadmap_generator/app/models/api_models.py @@ -1,31 +1,58 @@ -from typing import Optional +from typing import Literal, Optional -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator class GenerateRoadmapRequest(BaseModel): - """Request body for POST /generate-roadmap.""" + """ + Request body for POST /generate-roadmap. - topic_names: list[str] = Field( - ..., min_length=1, description="List of topic/subject names to build a roadmap for." - ) - subject: str = Field( - default="", description="Optional overall subject/title for the roadmap." + mode determines which fields are required: + - "topic": topic_names is required + - "document": document_id is required + - "quiz_performance": no extra fields required — uses the + authenticated user's weak-topic results + """ + + mode: Literal["document", "topic", "quiz_performance"] = Field( + ..., description="Which generation mode to use." ) - step_count: int = Field( - default=6, ge=3, le=15, description="Number of roadmap steps to generate." + + # mode="topic" + topic_names: Optional[list[str]] = Field( + default=None, description="Required when mode='topic'." ) priorities: Optional[dict[str, str]] = Field( default=None, - description="Optional {topic_name: 'high'|'normal'|'low'} hints to influence ordering.", + description="Optional {topic_name: 'high'|'normal'|'low'} hints. Only used with mode='topic'.", + ) + + # mode="document" + document_id: Optional[str] = Field( + default=None, description="Required when mode='document'." + ) + + # shared across all modes + subject: str = Field(default="", description="Optional overall subject/title for the roadmap.") + step_count: int = Field( + default=6, ge=3, le=15, description="Number of roadmap steps to generate." ) + @model_validator(mode="after") + def _check_mode_specific_fields(self): + if self.mode == "topic" and not self.topic_names: + raise ValueError("topic_names is required when mode='topic'.") + if self.mode == "document" and not self.document_id: + raise ValueError("document_id is required when mode='document'.") + return self + class GenerateRoadmapResponse(BaseModel): """Response body for POST /generate-roadmap.""" success: bool message: str + mode: str subject: str = "" steps: list[dict] = Field(default_factory=list) - total_steps: int = 0 \ No newline at end of file + total_steps: int = 0 diff --git a/ai-ml/roadmap_generator/app/services/roadmap_service.py b/ai-ml/roadmap_generator/app/services/roadmap_service.py index cb11f7b..d34e96a 100644 --- a/ai-ml/roadmap_generator/app/services/roadmap_service.py +++ b/ai-ml/roadmap_generator/app/services/roadmap_service.py @@ -5,19 +5,40 @@ validate_topic_names, validate_step_count, ) +from roadmap_generator.app.utils.topic_extractor import extract_topics from roadmap_generator.app.config import DEFAULT_STEP_COUNT +from embedding.chroma_store import get_document_chunks + +# Cross-module import, same convention already used elsewhere +# (e.g. weak_topic_detection/app/main.py importing quiz_generator's +# auth) — not an HTTP call, a direct Python import within the same +# codebase. +from weak_topic_detection.app.services.weak_topic_service import WeakTopicService + class RoadmapService: """ - Coordinates the roadmap-generation workflow. This is the - integration point other code (a future API layer, or another - module) should call, rather than using RoadmapGenerator directly. + Coordinates roadmap generation across all three supported modes: + - "topic" — caller supplies topic names directly + - "document" — topics are extracted from an ingested document's content + - "quiz_performance" — topics come from Weak Topic Detection's output """ def __init__(self): self.generator = RoadmapGenerator() + self._weak_topic_service: WeakTopicService | None = None + def _get_weak_topic_service(self) -> WeakTopicService: + # Lazy — avoids paying WeakTopicService's init cost for + # callers who only ever use mode="topic" or mode="document". + if self._weak_topic_service is None: + self._weak_topic_service = WeakTopicService() + return self._weak_topic_service + + # ------------------------------------------------------------------ + # mode="topic" — unchanged from the original single-mode version + # ------------------------------------------------------------------ def generate_roadmap( self, topic_names: list[str], @@ -25,22 +46,87 @@ def generate_roadmap( step_count: int = DEFAULT_STEP_COUNT, priorities: dict[str, str] | None = None, ) -> Roadmap: - """ - Generates a study roadmap from a plain list of topic names. - - priorities: optional {topic_name: "high"|"normal"|"low"} map. - This is how a caller (e.g. a future weak-topic integration) - could nudge topic ordering WITHOUT this module importing or - depending on that other module — the caller does the mapping, - this service just accepts plain data. - """ validate_topic_names(topic_names) validate_step_count(step_count) priorities = priorities or {} - topics = [ - Topic(name=name, priority=priorities.get(name)) - for name in topic_names - ] + topics = [Topic(name=name, priority=priorities.get(name)) for name in topic_names] return self.generator.generate(topics, subject=subject, step_count=step_count) + + # ------------------------------------------------------------------ + # mode="document" + # ------------------------------------------------------------------ + def generate_roadmap_from_document( + self, + user_id: str, + document_id: str, + subject: str = "", + step_count: int = DEFAULT_STEP_COUNT, + ) -> Roadmap: + """ + Builds a roadmap from an already-ingested document's own + content. Pulls every chunk belonging to (document_id, user_id) + — ownership enforced, so a caller can't generate a roadmap + from a document that isn't theirs — extracts topic keywords + with YAKE, then generates the same way mode="topic" does. + """ + validate_step_count(step_count) + + chunks = get_document_chunks(document_id=document_id, user_id=user_id) + if not chunks: + raise ValueError( + f"No content found for document_id '{document_id}'. It may not " + "exist, may still be processing, or may belong to another user." + ) + + full_text = "\n\n".join(chunks) + topic_names = extract_topics(full_text) + if not topic_names: + raise ValueError("Could not extract any topics from this document's content.") + + topics = [Topic(name=name) for name in topic_names] + return self.generator.generate( + topics, + subject=subject or f"Document {document_id}", + step_count=step_count, + ) + + # ------------------------------------------------------------------ + # mode="quiz_performance" + # ------------------------------------------------------------------ + def generate_roadmap_from_quiz_performance( + self, + user_id: str, + subject: str = "", + step_count: int = DEFAULT_STEP_COUNT, + ) -> Roadmap: + """ + Builds a roadmap focused on topics the user is weak in, using + Weak Topic Detection's output. Weak topics are marked "high" + priority so the generator sequences them early. + + NOTE (known limitation — same one documented on Weak Topic + Detection's own endpoint): WeakTopicService currently reads a + static demo dataset, not this user's live quiz history, so + results aren't truly personalized per-user yet. user_id is + still threaded through so this works correctly once live + per-user quiz ingestion exists — no further contract change + needed here when that lands. + """ + validate_step_count(step_count) + + weak_topics = self._get_weak_topic_service().get_weak_topics() + if not weak_topics: + raise ValueError( + "No weak topics found — take a quiz first to get a personalized roadmap." + ) + + topic_names = [wt.get("topic", "Unknown Topic") for wt in weak_topics] + topics = [Topic(name=name, priority="high") for name in topic_names] + + return self.generator.generate( + topics, + subject=subject or "Focus Areas From Your Quiz Performance", + step_count=step_count, + ) diff --git a/ai-ml/roadmap_generator/app/utils/topic_extractor.py b/ai-ml/roadmap_generator/app/utils/topic_extractor.py new file mode 100644 index 0000000..57b7948 --- /dev/null +++ b/ai-ml/roadmap_generator/app/utils/topic_extractor.py @@ -0,0 +1,38 @@ +""" +Topic extraction for mode="document" roadmap generation. + +Uses YAKE (unsupervised keyword extraction — no API key, no cost, +runs locally) to pull candidate topic phrases out of a document's +raw text. Reuses the exact YAKE settings quiz_generator's config.py +already defines, rather than duplicating magic numbers. +""" + +import yake + +from quiz_generator.app.config import ( + YAKE_MAX_KEYWORDS, + YAKE_NGRAM_SIZE, + YAKE_DEDUP_THRESHOLD, +) + + +def extract_topics(text: str, max_keywords: int = None) -> list[str]: + """ + Extracts candidate topic/keyword phrases from raw text. + + Returns a list of phrase strings, most relevant first (YAKE + scores lower = more relevant, so we sort ascending by score). + Returns an empty list for empty/whitespace-only input. + """ + if not text or not text.strip(): + return [] + + extractor = yake.KeywordExtractor( + n=YAKE_NGRAM_SIZE, + dedupLim=YAKE_DEDUP_THRESHOLD, + top=max_keywords or YAKE_MAX_KEYWORDS, + ) + keywords = extractor.extract_keywords(text) + keywords.sort(key=lambda pair: pair[1]) + + return [phrase for phrase, _score in keywords] diff --git a/ai-ml/weak_topic_detection/app/services/weak_topic_service.py b/ai-ml/weak_topic_detection/app/services/weak_topic_service.py index 6146f23..648a4e9 100644 --- a/ai-ml/weak_topic_detection/app/services/weak_topic_service.py +++ b/ai-ml/weak_topic_detection/app/services/weak_topic_service.py @@ -1,8 +1,20 @@ +from pathlib import Path + from weak_topic_detection.app.detectors.weak_topic_detector import WeakTopicDetector from weak_topic_detection.app.models.quiz_result import QuizResult from weak_topic_detection.app.utils.data_loader import load_json_data from weak_topic_detection.app.config import WEAK_TOPIC_THRESHOLD, MIN_TOPIC_ATTEMPTS +# [Fix] weak_topic_detection/app/services/ -> weak_topic_detection/data/ +# Computed from this file's own location so it resolves correctly +# regardless of the process's working directory (previously a bare +# relative "data/quiz_results.json" string, which only worked if the +# process happened to be started from inside weak_topic_detection/ +# itself — it broke for any other caller, e.g. roadmap_generator's +# quiz_performance mode, or this module's own HTTP endpoint, when run +# from ai-ml/ as the docs instruct). +DEFAULT_DATA_FILE = Path(__file__).resolve().parent.parent.parent / "data" / "quiz_results.json" + class WeakTopicService: """ @@ -12,11 +24,11 @@ class WeakTopicService: def __init__( self, - data_file: str = "data/quiz_results.json", + data_file: str = None, weak_threshold: float = WEAK_TOPIC_THRESHOLD, -min_attempts: int = MIN_TOPIC_ATTEMPTS, + min_attempts: int = MIN_TOPIC_ATTEMPTS, ): - self.data_file = data_file + self.data_file = data_file or DEFAULT_DATA_FILE self.detector = WeakTopicDetector( weak_threshold=weak_threshold, min_attempts=min_attempts, @@ -34,4 +46,4 @@ def get_weak_topics(self) -> list[dict]: results = self.load_results() - return self.detector.detect(results) + return self.detector.detect(results) \ No newline at end of file diff --git a/docs/api-contracts.md b/docs/api-contracts.md index 6b01e21..af16a24 100644 --- a/docs/api-contracts.md +++ b/docs/api-contracts.md @@ -333,4 +333,123 @@ Requires `Authorization: Bearer ` (same scheme as `/ask`). `user_id` comes | Status | When | |--------|------| | `400` | Invalid `quiz_type`, or no relevant content found for `topic` (returned as `success: false` in body) | -| `502` | Upstream generation error (e.g. LLM returned malformed data) | \ No newline at end of file +| `502` | Upstream generation error (e.g. LLM returned malformed data) | + +## Roadmap Generation (Team Lambda) + +**Service root:** `ai-ml/roadmap_generator/` + +**Default local base URL:** `http://127.0.0.1:8004` *(pending final port confirmation)* + +**Run:** + +```bash +cd ai-ml +uvicorn roadmap_generator.app.main:app --reload --port 8004 +``` + +Interactive docs: [http://127.0.0.1:8004/docs](http://127.0.0.1:8004/docs) + +**Auth:** Requires `Authorization: Bearer ` on every endpoint (same scheme as Quiz Generation and Weak Topic Detection — HS256, `JWT_SECRET_KEY`, identity from the token's `sub` claim). + +--- + +### `GET /health` + +**Response `200`:** +```json +{"status": "ok"} +``` + +--- + +### `POST /generate-roadmap` + +Supports three generation modes via the `mode` field. Exactly one mode's required field(s) must be supplied. + +#### Request body — common fields + +| Field | Type | Required | Default | Notes | +|-------|------|----------|---------|-------| +| `mode` | string | yes | — | One of: `topic`, `document`, `quiz_performance` | +| `subject` | string | no | `""` | Optional overall title for the roadmap | +| `step_count` | integer | no | `6` | Clamped **3–15** | + +#### Mode: `topic` + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `topic_names` | array[string] | **yes** (for this mode) | List of topic/subject names | +| `priorities` | object | no | `{topic_name: "high"\|"normal"\|"low"}` | + +**Example request:** +```json +{ + "mode": "topic", + "topic_names": ["Recursion", "Dynamic programming"], + "subject": "Algorithms", + "step_count": 4, + "priorities": {"Recursion": "high"} +} +``` + +#### Mode: `document` + +| Field | Type | Required | Notes | +|-------|------|----------|-------| +| `document_id` | string | **yes** (for this mode) | Must belong to the authenticated user | + +Topics are extracted automatically from the document's ingested content (YAKE keyword extraction). Returns **400** if the document has no content, doesn't exist, or belongs to a different user (these cases are indistinguishable in the response, by design). + +**Example request:** +```json +{ + "mode": "document", + "document_id": "3fa85f64-5717-4562-b3fc-2c963f66afa6", + "step_count": 6 +} +``` + +#### Mode: `quiz_performance` + +No mode-specific fields — uses the authenticated user's Weak Topic Detection results. Weak topics are prioritized ("high") in the generated sequence. Returns **400** if no weak topics are found (e.g. user hasn't taken a quiz yet). + +**Example request:** +```json +{ + "mode": "quiz_performance", + "step_count": 5 +} +``` + +**Known limitation:** Weak Topic Detection currently reads from a static demo dataset, not live per-user quiz history — `quiz_performance` mode inherits this limitation until per-user quiz ingestion is wired in. + +#### Response `200` (all modes — same shape) + +```json +{ + "success": true, + "message": "Generated 4 steps.", + "mode": "topic", + "subject": "Algorithms", + "steps": [ + { + "step_number": 1, + "topic": "Recursion Fundamentals", + "description": "Learn base cases, call stack mechanics...", + "estimated_duration": "2-3 days" + } + ], + "total_steps": 4 +} +``` + +#### Errors + +| Status | When | +|--------|------| +| `400` | Missing mode-specific required field; invalid `mode`; no content/topics found for the given input | +| `401` | Token present but invalid/expired | +| `403` | Missing `Authorization` header | +| `422` | Request body fails schema validation | +| `502` | Upstream generation error (e.g. LLM returned malformed data) |