From 667391e93fa0ebab889938b4eb77685ff5a0f02c Mon Sep 17 00:00:00 2001 From: FarwaK05 Date: Sat, 5 Sep 2026 00:32:43 +0500 Subject: [PATCH 1/3] Sync ai-ml with upstream main --- ai-ml/.env.example | 13 + ai-ml/.gitignore | 9 + ai-ml/README.md | 334 ++++++++++++++++++ .../.gitkeep => embedding/__init__.py} | 0 ai-ml/embedding/chroma_store.py | 138 ++++++++ ai-ml/embedding/chunker.py | 73 ++++ ai-ml/embedding/config.py | 55 +++ ai-ml/embedding/embedder.py | 160 +++++++++ ai-ml/embedding/model.py | 52 +++ ai-ml/embedding/vector_store.py | 69 ++++ ai-ml/ingestion/main.py | 143 +++++--- .../pdf}/.gitkeep | 0 ai-ml/ingestion/web/.gitkeep | 0 ai-ml/ingestion/youtube/.gitkeep | 0 ai-ml/knowledge_graph/.gitignore | 3 + ai-ml/knowledge_graph/README.md | 85 +++++ ai-ml/knowledge_graph/__init__.py | 0 ai-ml/knowledge_graph/app/__init__.py | 0 ai-ml/knowledge_graph/app/api/__init__.py | 0 ai-ml/knowledge_graph/app/api/graph_routes.py | 61 ++++ .../knowledge_graph/app/builders/__init__.py | 0 .../app/builders/document_graph_builder.py | 81 +++++ .../app/builders/topic_graph_builder.py | 55 +++ ai-ml/knowledge_graph/app/config.py | 36 ++ ai-ml/knowledge_graph/app/models/__init__.py | 0 .../knowledge_graph/app/models/graph_edge.py | 20 ++ .../knowledge_graph/app/models/graph_node.py | 16 + .../knowledge_graph/app/services/__init__.py | 0 .../app/services/graph_service.py | 64 ++++ ai-ml/knowledge_graph/app/storage/__init__.py | 0 .../app/storage/graph_store.py | 76 ++++ ai-ml/knowledge_graph/app/utils/__init__.py | 0 ai-ml/knowledge_graph/app/utils/similarity.py | 23 ++ .../app/utils/topic_labeler.py | 40 +++ ai-ml/knowledge_graph/app/utils/vectorizer.py | 21 ++ .../app/validators/__init__.py | 0 .../app/validators/graph_validators.py | 21 ++ ai-ml/knowledge_graph/requirements.txt | 12 + ai-ml/knowledge_graph/tests/__init__.py | 0 .../tests/test_graph_service.py | 93 +++++ ai-ml/knowledge_graph/tests/test_utils.py | 53 +++ ai-ml/quiz_generator/.gitignore | 21 ++ ai-ml/quiz_generator/.gitkeep | 0 ai-ml/quiz_generator/README.md | 101 ++++++ ai-ml/quiz_generator/app/auth.py | 62 ++++ ai-ml/quiz_generator/app/config.py | 35 ++ .../app/generators/base_generator.py | 34 ++ .../app/generators/fill_blank_generator.py | 90 +++++ .../app/generators/mcq_generator.py | 91 +++++ .../app/generators/short_answer_generator.py | 90 +++++ .../app/generators/true_false_generator.py | 90 +++++ ai-ml/quiz_generator/app/main.py | 107 ++++++ ai-ml/quiz_generator/app/models/api_models.py | 37 ++ ai-ml/quiz_generator/app/models/question.py | 51 +++ ai-ml/quiz_generator/app/models/request.py | 29 ++ ai-ml/quiz_generator/app/models/response.py | 26 ++ .../app/services/quiz_service.py | 139 ++++++++ .../tests/test_fill_blank_generator.py | 20 ++ .../tests/test_mcq_generator.py | 21 ++ .../quiz_generator/tests/test_quiz_service.py | 23 ++ .../tests/test_short_answer_generator.py | 20 ++ .../tests/test_true_false_generator.py | 20 ++ ai-ml/requirements.txt | 13 +- ai-ml/roadmap_generator/README.md | 93 +++++ ai-ml/roadmap_generator/app/config.py | 26 ++ .../app/generators/base_generator.py | 33 ++ .../app/generators/roadmap_generator.py | 103 ++++++ ai-ml/roadmap_generator/app/models/roadmap.py | 32 ++ ai-ml/roadmap_generator/app/models/topic.py | 25 ++ .../app/services/roadmap_service.py | 46 +++ .../app/validators/topic_validator.py | 23 ++ ai-ml/roadmap_generator/requirements.txt | 4 + ai-ml/roadmap_generator/tests/conftest.py | 12 + .../tests/test_roadmap_generator.py | 43 +++ .../tests/test_roadmap_service.py | 48 +++ ai-ml/run_quiz.py | 37 ++ ai-ml/scripts/check_users.py | 13 + ai-ml/tests/conftest.py | 14 + ai-ml/tests/test_isolation_lifecycle.py | 165 +++++++++ ai-ml/weak_topic_detection/.gitignore | 11 + ai-ml/weak_topic_detection/README.md | 34 ++ .../app/api/weak_topic_api.py | 12 + ai-ml/weak_topic_detection/app/config.py | 2 + .../app/detectors/weak_topic_detector.py | 57 +++ ai-ml/weak_topic_detection/app/main.py | 69 ++++ .../app/models/quiz_result.py | 12 + .../app/services/weak_topic_service.py | 37 ++ .../app/utils/data_loader.py | 11 + .../app/validators/quiz_result_validator.py | 36 ++ ai-ml/weak_topic_detection/commands.txt | 23 ++ .../data/quiz_results.json | 276 +++++++++++++++ ai-ml/weak_topic_detection/requirements.txt | 1 + .../tests/test_quiz_result_validator.py | 31 ++ .../tests/test_weak_topic_api.py | 15 + .../tests/test_weak_topic_detector.py | 75 ++++ .../tests/test_weak_topic_service.py | 24 ++ 96 files changed, 4315 insertions(+), 54 deletions(-) create mode 100644 ai-ml/.env.example create mode 100644 ai-ml/README.md rename ai-ml/{embeddings/.gitkeep => embedding/__init__.py} (100%) create mode 100644 ai-ml/embedding/chroma_store.py create mode 100644 ai-ml/embedding/chunker.py create mode 100644 ai-ml/embedding/config.py create mode 100644 ai-ml/embedding/embedder.py create mode 100644 ai-ml/embedding/model.py create mode 100644 ai-ml/embedding/vector_store.py rename ai-ml/{quiz-generator => ingestion/pdf}/.gitkeep (100%) create mode 100644 ai-ml/ingestion/web/.gitkeep create mode 100644 ai-ml/ingestion/youtube/.gitkeep create mode 100644 ai-ml/knowledge_graph/.gitignore create mode 100644 ai-ml/knowledge_graph/README.md create mode 100644 ai-ml/knowledge_graph/__init__.py create mode 100644 ai-ml/knowledge_graph/app/__init__.py create mode 100644 ai-ml/knowledge_graph/app/api/__init__.py create mode 100644 ai-ml/knowledge_graph/app/api/graph_routes.py create mode 100644 ai-ml/knowledge_graph/app/builders/__init__.py create mode 100644 ai-ml/knowledge_graph/app/builders/document_graph_builder.py create mode 100644 ai-ml/knowledge_graph/app/builders/topic_graph_builder.py create mode 100644 ai-ml/knowledge_graph/app/config.py create mode 100644 ai-ml/knowledge_graph/app/models/__init__.py create mode 100644 ai-ml/knowledge_graph/app/models/graph_edge.py create mode 100644 ai-ml/knowledge_graph/app/models/graph_node.py create mode 100644 ai-ml/knowledge_graph/app/services/__init__.py create mode 100644 ai-ml/knowledge_graph/app/services/graph_service.py create mode 100644 ai-ml/knowledge_graph/app/storage/__init__.py create mode 100644 ai-ml/knowledge_graph/app/storage/graph_store.py create mode 100644 ai-ml/knowledge_graph/app/utils/__init__.py create mode 100644 ai-ml/knowledge_graph/app/utils/similarity.py create mode 100644 ai-ml/knowledge_graph/app/utils/topic_labeler.py create mode 100644 ai-ml/knowledge_graph/app/utils/vectorizer.py create mode 100644 ai-ml/knowledge_graph/app/validators/__init__.py create mode 100644 ai-ml/knowledge_graph/app/validators/graph_validators.py create mode 100644 ai-ml/knowledge_graph/requirements.txt create mode 100644 ai-ml/knowledge_graph/tests/__init__.py create mode 100644 ai-ml/knowledge_graph/tests/test_graph_service.py create mode 100644 ai-ml/knowledge_graph/tests/test_utils.py create mode 100644 ai-ml/quiz_generator/.gitignore create mode 100644 ai-ml/quiz_generator/.gitkeep create mode 100644 ai-ml/quiz_generator/README.md create mode 100644 ai-ml/quiz_generator/app/auth.py create mode 100644 ai-ml/quiz_generator/app/config.py create mode 100644 ai-ml/quiz_generator/app/generators/base_generator.py create mode 100644 ai-ml/quiz_generator/app/generators/fill_blank_generator.py create mode 100644 ai-ml/quiz_generator/app/generators/mcq_generator.py create mode 100644 ai-ml/quiz_generator/app/generators/short_answer_generator.py create mode 100644 ai-ml/quiz_generator/app/generators/true_false_generator.py create mode 100644 ai-ml/quiz_generator/app/main.py create mode 100644 ai-ml/quiz_generator/app/models/api_models.py create mode 100644 ai-ml/quiz_generator/app/models/question.py create mode 100644 ai-ml/quiz_generator/app/models/request.py create mode 100644 ai-ml/quiz_generator/app/models/response.py create mode 100644 ai-ml/quiz_generator/app/services/quiz_service.py create mode 100644 ai-ml/quiz_generator/tests/test_fill_blank_generator.py create mode 100644 ai-ml/quiz_generator/tests/test_mcq_generator.py create mode 100644 ai-ml/quiz_generator/tests/test_quiz_service.py create mode 100644 ai-ml/quiz_generator/tests/test_short_answer_generator.py create mode 100644 ai-ml/quiz_generator/tests/test_true_false_generator.py create mode 100644 ai-ml/roadmap_generator/README.md create mode 100644 ai-ml/roadmap_generator/app/config.py create mode 100644 ai-ml/roadmap_generator/app/generators/base_generator.py create mode 100644 ai-ml/roadmap_generator/app/generators/roadmap_generator.py create mode 100644 ai-ml/roadmap_generator/app/models/roadmap.py create mode 100644 ai-ml/roadmap_generator/app/models/topic.py create mode 100644 ai-ml/roadmap_generator/app/services/roadmap_service.py create mode 100644 ai-ml/roadmap_generator/app/validators/topic_validator.py create mode 100644 ai-ml/roadmap_generator/requirements.txt create mode 100644 ai-ml/roadmap_generator/tests/conftest.py create mode 100644 ai-ml/roadmap_generator/tests/test_roadmap_generator.py create mode 100644 ai-ml/roadmap_generator/tests/test_roadmap_service.py create mode 100644 ai-ml/run_quiz.py create mode 100644 ai-ml/scripts/check_users.py create mode 100644 ai-ml/tests/conftest.py create mode 100644 ai-ml/tests/test_isolation_lifecycle.py create mode 100644 ai-ml/weak_topic_detection/.gitignore create mode 100644 ai-ml/weak_topic_detection/README.md create mode 100644 ai-ml/weak_topic_detection/app/api/weak_topic_api.py create mode 100644 ai-ml/weak_topic_detection/app/config.py create mode 100644 ai-ml/weak_topic_detection/app/detectors/weak_topic_detector.py create mode 100644 ai-ml/weak_topic_detection/app/main.py create mode 100644 ai-ml/weak_topic_detection/app/models/quiz_result.py create mode 100644 ai-ml/weak_topic_detection/app/services/weak_topic_service.py create mode 100644 ai-ml/weak_topic_detection/app/utils/data_loader.py create mode 100644 ai-ml/weak_topic_detection/app/validators/quiz_result_validator.py create mode 100644 ai-ml/weak_topic_detection/commands.txt create mode 100644 ai-ml/weak_topic_detection/data/quiz_results.json create mode 100644 ai-ml/weak_topic_detection/requirements.txt create mode 100644 ai-ml/weak_topic_detection/tests/test_quiz_result_validator.py create mode 100644 ai-ml/weak_topic_detection/tests/test_weak_topic_api.py create mode 100644 ai-ml/weak_topic_detection/tests/test_weak_topic_detector.py create mode 100644 ai-ml/weak_topic_detection/tests/test_weak_topic_service.py diff --git a/ai-ml/.env.example b/ai-ml/.env.example new file mode 100644 index 0000000..2540840 --- /dev/null +++ b/ai-ml/.env.example @@ -0,0 +1,13 @@ +GROQ_API_KEY=groq_api_key_here + +# Pinecone (RAG vector database) +PINECONE_API_KEY=your-pinecone-api-key +HOST=your-pinecone-host-url +PINECONE_INDEX_NAME=your-pinecone-index-name +PINECONE_TOP_K=5 + +# MongoDB (Conversation History) +MONGODB_URI="mongodb://:@/?ssl=true&replicaSet=&authSource=admin&appName=" + +# HF_TOKEN (Hugging Face token) +HF_TOKEN=your-huggingface-access-token \ No newline at end of file diff --git a/ai-ml/.gitignore b/ai-ml/.gitignore index 5500a2b..0728cf7 100644 --- a/ai-ml/.gitignore +++ b/ai-ml/.gitignore @@ -1,3 +1,12 @@ +.venv/ venv/ +test-env/ + +node_modules/ + __pycache__/ +*.pyc + .env + +uploaded_files/ \ No newline at end of file diff --git a/ai-ml/README.md b/ai-ml/README.md new file mode 100644 index 0000000..9b3c366 --- /dev/null +++ b/ai-ml/README.md @@ -0,0 +1,334 @@ +# StudyMind AI — Ingestion, Embedding & Quiz Generator Pipeline + +StudyMind AI is a Retrieval-Augmented Generation (RAG) pipeline that turns raw content (YouTube videos, web articles, PDFs) into searchable knowledge — and then uses that knowledge to generate structured, factual quizzes. + +The system is made up of **three modules** that work together: + +| Module | Role | +|---|---| +| **Ingestion** | FastAPI server that receives and processes raw content (PDFs, articles, YouTube) | +| **Embedding** | Chunks text, generates embeddings, and stores them in MongoDB + Pinecone | +| **Quiz Generator** | Retrieves stored content via vector search and generates JSON quizzes using an LLM (Groq) | + +--- + +## Table of Contents + +- [Architecture Overview](#architecture-overview) +- [Prerequisites](#prerequisites) +- [1. Installation](#1-installation) +- [2. Environment Setup](#2-environment-setup) +- [3. Running Ingestion & Embedding](#3-running-ingestion--embedding) +- [4. Running the Quiz Generator](#4-running-the-quiz-generator) +- [5. Quiz JSON Output Schemas](#5-quiz-json-output-schemas) +- [6. Unit Testing](#6-unit-testing) +- [Project Structure](#project-structure) +- [Troubleshooting](#troubleshooting) +- [Technologies](#technologies) + +--- + +## Architecture Overview + +Data flows through the pipeline in two stages: + +**Stage 1 — Ingest & Embed:** +`Source (YouTube / Article / PDF) → Ingestion Server → Chunking → Embedding → MongoDB + Pinecone` + +**Stage 2 — Generate Quiz (RAG):** +`Topic Query → Pinecone Vector Search → MongoDB Chunk Retrieval → Groq LLM → Structured JSON Quiz` + +The quiz generator doesn't guess answers — it retrieves the exact relevant chunks from your ingested content first, then asks the LLM to build questions strictly from that context. + +--- + +## Prerequisites + +Before starting, make sure you have: + +- Python 3.9+ installed +- A [MongoDB Atlas](https://www.mongodb.com/atlas) cluster (or local MongoDB instance) +- A [Pinecone](https://www.pinecone.io/) account with an index created +- A [Groq](https://console.groq.com/) account and API key (for quiz generation) +- Content already ingested and embedded before attempting to generate quizzes + +--- + +## 1. Installation + +Open your terminal and navigate to the project root: + +```bash +cd ai-ml +``` + +**Create and activate a virtual environment** (recommended): + +```bash +python -m venv venv + +# macOS / Linux +source venv/bin/activate + +# Windows +venv\Scripts\activate +``` + +**Install dependencies:** + +```bash +pip install -r requirements.txt +``` + +--- + +## 2. Environment Setup + +> **⚠️ Important: Never commit real API keys or credentials to GitHub.** +> This project uses a `.env.example` template in the root `ai-ml` folder — copy it locally and fill in your own secrets. All modules (ingestion, embedding, quiz_generator) read from this single `.env` file. + +1. Locate the template file at `.env.example`. +2. Copy it and rename the copy to `.env`: + + ```bash + cp .env.example .env + ``` + +3. Open `.env` and fill in your own values: + + | Variable | Description | + |---|---| + | `MONGODB_URI` | Your MongoDB Atlas connection string | + | `MONGODB_DB` | Database name (default: `studymind`) | + | `MONGODB_COLLECTION` | Collection name for storing chunks (default: `chunks`) | + | `PINECONE_API_KEY` | Your Pinecone API key | + | `PINECONE_INDEX_NAME` | Name of your Pinecone index | + | `PINECONE_CLOUD` | Pinecone cloud provider (default: `aws`) | + | `PINECONE_REGION` | Pinecone region (default: `us-east-1`) | + | `EMBEDDING_MODEL_NAME` | Local embedding model (default: `all-MiniLM-L6-v2`) | + | `EMBEDDING_DIMENSION` | Embedding vector size (default: `384`) | + | `CHUNK_SIZE` | Max characters/tokens per chunk (default: `300`) | + | `CHUNK_OVERLAP` | Overlap between chunks (default: `50`) | + | `GROQ_API_KEY` | Your Groq API key (used by the quiz generator) | + | `GROQ_MODEL` | Groq model name (default: `llama-3.3-70b-versatile`) | + +4. Double-check `.env` is listed in `.gitignore` so it's never accidentally pushed: + + ```gitignore + .env + ``` + +### `.env.example` + +```dotenv +# MongoDB +MONGODB_URI=your-mongodb-connection-string +MONGODB_DB=studymind +MONGODB_COLLECTION=chunks + +# Pinecone +PINECONE_API_KEY=your-pinecone-api-key +PINECONE_INDEX_NAME=studymind-embeddings +PINECONE_CLOUD=aws +PINECONE_REGION=us-east-1 + +# HF_TOKEN (Hugging Face token) +HF_TOKEN = hugging face API key (Token Access) +# Embedding model (local, free — no key needed) +EMBEDDING_DIMENSION=384 + +# Chunking +CHUNK_SIZE=300 +CHUNK_OVERLAP=50 + +# Groq (Quiz Generator LLM) +GROQ_API_KEY=your-groq-api-key +GROQ_MODEL=llama-3.3-70b-versatile +``` + +--- + +## 3. Running Ingestion & Embedding + +Because ingestion and embedding are independent services, you'll need **two separate terminals** running at the same time. + +### Terminal 1 — Start the Ingestion Server + +```bash +cd ai-ml +python -m uvicorn ingestion.main:app --reload +``` + +> **Note:** You must use the `python -m` prefix for this to work correctly. + +Wait until you see `Application startup complete`, then leave this terminal running. + +- Server URL: `http://127.0.0.1:8000` +- Interactive API docs (Swagger UI): `http://127.0.0.1:8000/docs` + +### Terminal 2 — Run the Embedding Module + +Open a **second terminal**, and make sure you're also in the `ai-ml` folder: + +```bash +cd ai-ml +``` + +With Terminal 1 still running, run the embedder against your desired source type: + +**YouTube video:** +```bash +python -m embedding.embedder --youtube "https://youtube.com/watch?v=VIDEO_ID" +``` + +**Web article:** +```bash +python -m embedding.embedder --article "https://example.com/some-article" +``` + +**PDF file:** +```bash +python -m embedding.embedder --pdf "C:\path\to\file.pdf" +``` + +**Quick sanity check** (no external input, uses a hardcoded sample): +```bash +python -m embedding.embedder +``` + +Once this step is complete, your content is chunked, embedded, and stored in MongoDB + Pinecone — ready for quiz generation. + +--- + +## 4. Running the Quiz Generator + +The **AI Quiz Generator** is a Retrieval-Augmented module that searches your ingested content (via Pinecone + MongoDB) to generate accurate, structured quizzes — instead of relying on the LLM to guess facts. + +### Key Features + +- **Renamed Package:** `quiz_generator` (formerly `quiz-generator`) for full Python module compatibility. +- **Vector Store Linking:** Fully integrated with the `embedding` module to pull context from ingested PDFs, YouTube videos, and articles. +- **Strict JSON Output:** Uses Groq's `json_object` mode to guarantee valid JSON arrays, ready for immediate use in web/mobile apps. +- **Deep Context Retrieval:** Combines multiple relevant text chunks to ensure high-quality question coverage. + +### Running the RAG Quiz System + +Make sure content has already been ingested and embedded (see [Section 3](#3-running-ingestion--embedding)), then run the interactive CLI from the **root `ai-ml` folder**: + +```bash +# Ensure Python can see the modules +$env:PYTHONPATH = "." # PowerShell (Windows) +# export PYTHONPATH="." # macOS / Linux + +# Run the interactive CLI +python run_quiz.py +``` + +### Example Workflow + +1. Enter Topic: `Artificial Intelligence` +2. Enter Type: `mcq` +3. Result: The system searches your stored PDFs/videos for "Artificial Intelligence" and generates a JSON quiz. + +--- + +## 5. Quiz JSON Output Schemas + +Every generator returns a clean list of JSON objects. + +### Multiple Choice (MCQ) +```json +[ + { + "question": "What is the capital of France?", + "options": ["London", "Berlin", "Paris", "Madrid"], + "answer": "Paris" + } +] +``` + +### True/False +```json +[ + { + "question": "The Earth is flat.", + "answer": "False" + } +] +``` + +--- + +## 6. Unit Testing + +To test the quiz generators in isolation (without the database), run the updated unit tests from the **root `ai-ml` folder**: + +```bash +python -m quiz_generator.tests.test_mcq_generator +python -m quiz_generator.tests.test_fill_blank_generator +``` + +--- + +## Project Structure + +``` +ai-ml/ +├── ingestion/ +│ ├── main.py # FastAPI ingestion server +│ └── ... +├── embedding/ +│ ├── embedder.py # Embedding module (CLI entry point) +│ └── requirements.txt +├── quiz_generator/ +│ ├── app/ +│ │ ├── generators/ # AI logic with strict JSON prompts +│ │ │ ├── base_generator.py +│ │ │ ├── mcq_generator.py +│ │ │ ├── true_false_generator.py +│ │ │ ├── fill_blank_generator.py +│ │ │ └── short_answer_generator.py +│ │ ├── services/ +│ │ │ └── quiz_service.py # Bridge: Searches Vectors -> Fetches Mongo -> Calls AI +│ │ └── config.py # Loads .env from root and manages Groq settings +│ └── tests/ # Unit tests for individual generators +├── run_quiz.py # Interactive CLI entry point for quiz generation +├── .env.example # Environment variable template +├── .env # Your local secrets (gitignored) +├── requirements.txt +└── README.md +``` + +--- + +## Troubleshooting + +| Issue | Likely Cause / Fix | +|---|---| +| `uvicorn: command not found` | Forgot the `python -m` prefix, or dependencies not installed | +| Connection refused on `127.0.0.1:8000` | Terminal 1 (ingestion server) isn't running | +| Pinecone auth errors | Check `PINECONE_API_KEY`, `PINECONE_INDEX_NAME`, `PINECONE_CLOUD`, and `PINECONE_REGION` in your `.env` | +| MongoDB connection timeout | Check `MONGODB_URI`, and confirm your IP is whitelisted in MongoDB Atlas Network Access | +| Quiz generator returns empty results | No content has been ingested/embedded yet for that topic — run Section 3 first | +| Groq API errors | Check `GROQ_API_KEY` and `GROQ_MODEL` are set correctly in `.env` | +| `ModuleNotFoundError` when running `run_quiz.py` | Set `PYTHONPATH` to the `ai-ml` root before running (see [Section 4](#4-running-the-quiz-generator)) | +| Missing module errors | Re-run `pip install -r requirements.txt` | + +--- + +## Technologies + +- **LLM:** Groq Llama 3.3 70B (state-of-the-art inference) +- **Database:** MongoDB (text/chunk storage) +- **Vector DB:** Pinecone (semantic search) +- **Embedding Model:** `all-MiniLM-L6-v2` (local, free, no API key needed) +- **Ingestion:** FastAPI +- **Output Format:** Strict JSON (`application/json`) + +--- + +## Contributing + +1. Fork the repo and create a new branch for your feature/fix. +2. Never commit `.env` or any real credentials. +3. Open a pull request with a clear description of your changes. \ No newline at end of file diff --git a/ai-ml/embeddings/.gitkeep b/ai-ml/embedding/__init__.py similarity index 100% rename from ai-ml/embeddings/.gitkeep rename to ai-ml/embedding/__init__.py diff --git a/ai-ml/embedding/chroma_store.py b/ai-ml/embedding/chroma_store.py new file mode 100644 index 0000000..8a4b9c7 --- /dev/null +++ b/ai-ml/embedding/chroma_store.py @@ -0,0 +1,138 @@ +""" +Shared ChromaDB store — the single canonical content store for both +the ingestion pipeline and quiz generation. + +[P0-1 fix] Previously, quiz generation queried Pinecone + MongoDB +(see the old embedder.py) while ingestion wrote here. The two never +overlapped, so newly ingested content was invisible to quiz +generation. This file is now the ONLY storage path for both: +ingestion calls store_chunks() (unchanged), and quiz generation's +Embedder.search() calls query_chunks() (new) instead of touching +Pinecone/Mongo at all. + +Also fixes a subtler bug: this module used to load its own separate +SentenceTransformer instance with default (non-normalized) output, +while embedding/model.py's shared model normalizes its vectors. Two +different embedding configs writing into the same cosine-similarity +space would have produced quietly wrong nearest-neighbor results. +Both reads and writes now go through the one shared, normalized +model in model.py. +""" +from __future__ import annotations +import os +from pathlib import Path + +import chromadb +from dotenv import load_dotenv + +from embedding.model import get_embedding_model + +load_dotenv(dotenv_path=Path(__file__).parent.parent.parent / ".env") + +DEFAULT_CHROMA_PATH = os.getenv( + "CHROMA_DB_PATH", + r"C:\Dev\QuantumLearningWorkspace\shared_chroma_data", +) +DEFAULT_COLLECTION_NAME = "study_chunks" + + +def get_collection(name: str = DEFAULT_COLLECTION_NAME, path: str = None): + client = chromadb.PersistentClient(path=path or DEFAULT_CHROMA_PATH) + return client.get_or_create_collection(name=name) + + +def store_chunks(chunks: list[dict], user_id: str, document_id: str, title: str) -> int: + """ + Embed and store chunks in the shared ChromaDB collection. + + chunks: list of dicts from chunker.chunk_document(), each with a "text" key. + Returns the number of chunks stored. + """ + if not chunks: + return 0 + + collection = get_collection() + model = get_embedding_model() # shared, normalized model — same one queries use + + ids = [f"{document_id}_chunk{c['chunk_index']}" for c in chunks] + documents = [c["text"] for c in chunks] + metadatas = [ + { + "user_id": user_id, + "document_id": document_id, + "document": title, + "chunk_index": c["chunk_index"], + } + for c in chunks + ] + embeddings = model.encode(documents) # list[list[float]], already normalized + + collection.upsert( + ids=ids, + embeddings=embeddings, + documents=documents, + metadatas=metadatas, + ) + return len(chunks) + + +def query_chunks( + query_text: str, + top_k: int = 5, + user_id: str = None, + document_id: str = None, +) -> list: + """ + [P0-1 new] Embed a query and return the top_k nearest chunks from + the shared collection, in the shape quiz_service.py expects: + [{"score": float, "text": str, "title": str, "metadata": dict}, ...] + + user_id / document_id, if given, are applied as an exact-match + metadata filter (ChromaDB's `where`). Both default to None (no + filter) today, which preserves current unscoped behavior — this + is deliberately plumbed through now so P0-3 (user-scoped quiz + retrieval) can pass user_id here without another storage change. + """ + collection = get_collection() + model = get_embedding_model() + + where = {} + if user_id is not None: + where["user_id"] = user_id + if document_id is not None: + where["document_id"] = document_id + + query_embedding = model.encode(query_text)[0] + + results = collection.query( + query_embeddings=[query_embedding], + n_results=top_k, + where=where or None, + ) + + ids = results.get("ids", [[]])[0] + documents = results.get("documents", [[]])[0] + metadatas = results.get("metadatas", [[]])[0] + distances = results.get("distances", [[]])[0] + + output = [] + for i in range(len(ids)): + meta = metadatas[i] or {} + output.append({ + "score": distances[i], + "text": documents[i], + "title": meta.get("document", ""), + "metadata": meta, + }) + return output + + +def delete_chunks(document_id: str) -> None: + """ + [P0-1 new] Remove all chunks belonging to a document from the + shared collection. Chroma's delete-by-filter is idempotent + (deleting a non-existent id/filter is a no-op), which is also + what P0-5's purge endpoint will need. + """ + collection = get_collection() + collection.delete(where={"document_id": document_id}) \ No newline at end of file diff --git a/ai-ml/embedding/chunker.py b/ai-ml/embedding/chunker.py new file mode 100644 index 0000000..d8d5fe4 --- /dev/null +++ b/ai-ml/embedding/chunker.py @@ -0,0 +1,73 @@ +""" +Splits ingested document text (the common ingestion schema: +{ source_type, title, text, metadata }) into overlapping, +word-based chunks ready for embedding. +""" + +from embedding.config import settings + + +def chunk_text(text: str, chunk_size: int = None, overlap: int = None) -> list: + """ + Word-based sliding-window chunking with overlap. + + Returns a list of: + { "chunk_index": int, "text": str, "start_word": int, "end_word": int } + """ + chunk_size = chunk_size or settings.chunk_size + overlap = overlap or settings.chunk_overlap + + if chunk_size <= 0: + raise ValueError("chunk_size must be greater than 0") + if overlap >= chunk_size: + raise ValueError("chunk_overlap must be smaller than chunk_size") + + words = text.split() + if not words: + return [] + + step = chunk_size - overlap + chunks = [] + start = 0 + index = 0 + + while start < len(words): + end = min(start + chunk_size, len(words)) + chunks.append({ + "chunk_index": index, + "text": " ".join(words[start:end]), + "start_word": start, + "end_word": end, + }) + index += 1 + if end == len(words): + break + start += step + + return chunks + + +def chunk_document(document: dict, chunk_size: int = None, overlap: int = None) -> list: + """ + document: common ingestion schema dict, e.g. + { + "source_type": "pdf" | "youtube" | "article", + "title": "...", + "text": "...", + "metadata": {"author": "", "date": "", "source": ""} + } + + Returns chunk dicts enriched with the parent document's title, + source_type, and metadata, ready to be embedded and stored. + """ + raw_chunks = chunk_text(document.get("text", ""), chunk_size, overlap) + + return [ + { + **c, + "title": document.get("title", ""), + "source_type": document.get("source_type", ""), + "metadata": document.get("metadata", {}), + } + for c in raw_chunks + ] diff --git a/ai-ml/embedding/config.py b/ai-ml/embedding/config.py new file mode 100644 index 0000000..a8f25ad --- /dev/null +++ b/ai-ml/embedding/config.py @@ -0,0 +1,55 @@ +""" +Central configuration for the embedding module. + +All values are read from environment variables (via a .env file) so +no secrets are hard-coded. Copy .env.example to .env and fill in your +own keys before running anything. +""" + +import os +from dataclasses import dataclass, field +from pathlib import Path +from dotenv import load_dotenv + +load_dotenv(dotenv_path=Path(__file__).parent.parent.parent / ".env") + + +def _get_bool(name: str, default: bool) -> bool: + val = os.getenv(name) + if val is None: + return default + return val.strip().lower() in ("1", "true", "yes") + + +@dataclass +class Settings: + # ---------------- MongoDB ---------------- + # Free tier: MongoDB Atlas free "M0" cluster (512MB, no cost, no card + # required for the free tier). Or run mongodb locally for $0. + mongodb_uri: str = field(default_factory=lambda: os.getenv("MONGODB_URI", "mongodb://localhost:27017")) + mongodb_db: str = field(default_factory=lambda: os.getenv("MONGODB_DB", "studymind")) + mongodb_collection: str = field(default_factory=lambda: os.getenv("MONGODB_COLLECTION", "chunks")) + + # ---------------- Pinecone ---------------- + # Free tier: Pinecone Starter plan (serverless, no card required for + # light usage as of this writing — always confirm current limits at + # https://www.pinecone.io/pricing/ since free-tier terms can change). + pinecone_api_key: str = field(default_factory=lambda: os.getenv("PINECONE_API_KEY", "")) + pinecone_index_name: str = field(default_factory=lambda: os.getenv("PINECONE_INDEX_NAME", "studymind-embeddings")) + pinecone_cloud: str = field(default_factory=lambda: os.getenv("PINECONE_CLOUD", "aws")) + pinecone_region: str = field(default_factory=lambda: os.getenv("PINECONE_REGION", "us-east-1")) + pinecone_metric: str = field(default_factory=lambda: os.getenv("PINECONE_METRIC", "cosine")) + + # ---------------- Embedding model ---------------- + # sentence-transformers runs 100% locally (downloads weights once, + # then no API key / no per-call cost / no internet needed after that). + # all-MiniLM-L6-v2 -> 384 dimensions, fast, good quality for RAG. + embedding_model_name: str = field(default_factory=lambda: os.getenv("EMBEDDING_MODEL_NAME", "all-MiniLM-L6-v2")) + embedding_dimension: int = field(default_factory=lambda: int(os.getenv("EMBEDDING_DIMENSION", "384"))) + + # ---------------- Chunking ---------------- + chunk_size: int = field(default_factory=lambda: int(os.getenv("CHUNK_SIZE", "300"))) # words per chunk + chunk_overlap: int = field(default_factory=lambda: int(os.getenv("CHUNK_OVERLAP", "50"))) # overlap in words + +settings = Settings() + diff --git a/ai-ml/embedding/embedder.py b/ai-ml/embedding/embedder.py new file mode 100644 index 0000000..f38fdf5 --- /dev/null +++ b/ai-ml/embedding/embedder.py @@ -0,0 +1,160 @@ +""" +Orchestrates document embedding + the search interface quiz +generation uses. + +[P0-1 fix] This used to run its own parallel storage path — full +chunk text in MongoDB (source of truth) + vectors in Pinecone (for +search) — completely separate from ingestion's shared ChromaDB. That +meant anything ingested via ingestion/main.py was invisible to +QuizService.generate_quiz_from_topic(), since it searched Pinecone. + +Everything now reads and writes the ONE shared ChromaDB collection +via chroma_store.py — the same collection ingestion already writes +to. No more Mongo/Pinecone clients, no more second indexing path. +""" + +import uuid + +from embedding.chunker import chunk_document +from embedding.chroma_store import store_chunks, query_chunks, delete_chunks + +import argparse +import requests + + +class Embedder: + def __init__(self): + # No client setup needed here anymore — chroma_store.py owns + # the one shared collection + embedding model as module-level + # helpers, created fresh per call (see get_collection()). + pass + + def embed_document(self, document: dict, user_id: str = "unknown") -> dict: + """ + document: common ingestion schema + { "source_type": ..., "title": ..., "text": ..., "metadata": {...} } + + Chunks it and stores it in the shared ChromaDB collection. + Returns a small summary dict. + + Note: the normal ingestion request path (POST /ingest/pdf etc.) + goes through ingestion/main.py's own _chunk_and_store(), which + calls chroma_store.store_chunks() directly and doesn't use + this method. This method exists for standalone/manual use + (see the CLI at the bottom of this file) and now writes to the + exact same store, so both paths stay consistent. + """ + chunks = chunk_document(document) + if not chunks: + return {"document_id": None, "chunks_stored": 0} + + document_id = str(uuid.uuid4()) + stored_count = store_chunks( + chunks=chunks, + user_id=user_id, + document_id=document_id, + title=document.get("title", ""), + ) + return {"document_id": document_id, "chunks_stored": stored_count} + + def search( + self, + query: str, + top_k: int = 5, + user_id: str = None, + document_id: str = None, + ) -> list: + """ + Searches the shared ChromaDB collection — the same store + ingestion writes to, so newly ingested content is immediately + queryable here (P0-1's Definition of Done). + + user_id / document_id are optional scoping filters, passed + straight through to chroma_store.query_chunks(). Left as None + by default (unscoped), matching current QuizService behavior; + P0-3 is what wires the caller-side enforcement of these. + """ + return query_chunks(query, top_k=top_k, user_id=user_id, document_id=document_id) + + def delete_document(self, document_id: str, chunk_count: int = None): + """ + Remove a document's chunks from the shared store. + chunk_count is no longer needed (Chroma deletes by + document_id metadata filter, not by reconstructing chunk ids) + but is accepted for backward compatibility with existing callers. + """ + delete_chunks(document_id) + + def close(self): + """ + No-op now — chroma_store.py doesn't hold a persistent client + connection open between calls, so there's nothing to close. + Kept so existing callers (e.g. the CLI below) don't need to + change. + """ + pass + + +INGESTION_BASE_URL = "http://127.0.0.1:8001" +# NOTE: this was previously "http://127.0.0.1:8000", which is Mu's +# confirmed port, not Lambda ingestion's. Per the confirmed port +# scheme (Mu=8000, Pluto=5000, Lambda ingestion=8001, Lambda +# quiz=8002), 8001 is correct here. Flagging in case this was +# intentional for some other reason — worth a quick sanity check +# against P1-6's verification pass. + + +def _fetch_from_ingestion(pdf=None, youtube=None, article=None) -> dict: + if pdf: + with open(pdf, "rb") as f: + response = requests.post(f"{INGESTION_BASE_URL}/ingest/pdf", files={"file": f}) + elif youtube: + response = requests.post(f"{INGESTION_BASE_URL}/ingest/youtube", json={"url": youtube}) + elif article: + response = requests.post(f"{INGESTION_BASE_URL}/ingest/article", json={"url": article}) + else: + raise ValueError("Provide one of pdf, youtube, or article") + + response.raise_for_status() + return response.json() + + +if __name__ == "__main__": + # python -m embedding.embedder --pdf path/to/file.pdf + # python -m embedding.embedder --youtube "https://youtube.com/watch?v=..." + # python -m embedding.embedder --article "https://example.com/article" + # python -m embedding.embedder (no args -> runs built-in sample test) + + parser = argparse.ArgumentParser(description="Run ingestion -> embedding end to end") + parser.add_argument("--pdf", help="Path to a local PDF file") + parser.add_argument("--youtube", help="YouTube video URL") + parser.add_argument("--article", help="Web article URL") + args = parser.parse_args() + + if args.pdf or args.youtube or args.article: + print("Calling ingestion...") + document = _fetch_from_ingestion(pdf=args.pdf, youtube=args.youtube, article=args.article) + print(f"Ingestion returned title: {document.get('title', '(no title)')!r}") + print(f"Text length: {len(document.get('text', ''))} characters") + else: + document = { + "source_type": "article", + "title": "Intro to Machine Learning", + "text": ( + "Machine learning is a branch of artificial intelligence. " + "It focuses on building systems that learn from data. " + "Supervised learning uses labeled data to train models." + ), + "metadata": {"author": "", "date": "", "source": "https://example.com"}, + } + + embedder = Embedder() + summary = embedder.embed_document(document, user_id="cli-test-user") + print("Embedded:", summary) + + query = document.get("title") or document.get("text", "")[:50] + results = embedder.search(query, top_k=3) + for r in results: + print(f"[{r['score']:.3f}] {r['text'][:100]}...") + + embedder.close() \ No newline at end of file diff --git a/ai-ml/embedding/model.py b/ai-ml/embedding/model.py new file mode 100644 index 0000000..abc9d97 --- /dev/null +++ b/ai-ml/embedding/model.py @@ -0,0 +1,52 @@ +""" +Embedding model wrapper. + +Uses sentence-transformers (free, open-source, runs locally after the +model weights are downloaded once — no API key, no per-call cost). +""" + +from functools import lru_cache +from sentence_transformers import SentenceTransformer + +from embedding.config import settings + + +class EmbeddingModel: + def __init__(self, model_name: str = None): + self.model_name = model_name or settings.embedding_model_name + self._model = SentenceTransformer(self.model_name) + + def encode(self, texts, batch_size: int = 32) -> list: + """ + texts: a single string or a list of strings. + Returns a list of embedding vectors (list[float]). + """ + single_input = isinstance(texts, str) + if single_input: + texts = [texts] + + vectors = self._model.encode( + texts, + batch_size=batch_size, + show_progress_bar=False, + normalize_embeddings=True, # cosine similarity works cleanly on normalized vectors + ) + return vectors.tolist() + + def dimension(self) -> int: + return self._model.get_sentence_embedding_dimension() + + +@lru_cache(maxsize=1) +def get_embedding_model() -> EmbeddingModel: + """Singleton so the model is loaded into memory only once per process.""" + return EmbeddingModel() + + +if __name__ == "__main__": + # quick manual test: python -m embedding.model + model = get_embedding_model() + vecs = model.encode(["Machine learning is a branch of artificial intelligence."]) + print(f"Model: {model.model_name}") + print(f"Dimension: {model.dimension()}") + print(f"First 5 values of vector: {vecs[0][:5]}") diff --git a/ai-ml/embedding/vector_store.py b/ai-ml/embedding/vector_store.py new file mode 100644 index 0000000..99c1153 --- /dev/null +++ b/ai-ml/embedding/vector_store.py @@ -0,0 +1,69 @@ +""" +Pinecone vector store wrapper. + +Free tier: Pinecone's Starter/serverless plan supports this use case +at no cost for typical student-project volumes — verify current limits +at https://www.pinecone.io/pricing/ since free-tier terms can change. +""" +import os +from dotenv import load_dotenv + +load_dotenv(os.path.join(os.path.dirname(__file__), "../../.env")) +from pinecone import Pinecone, ServerlessSpec + +from embedding.config import settings + + +class PineconeVectorStore: + def __init__(self, index_name: str = None, dimension: int = None): + if not settings.pinecone_api_key: + raise RuntimeError( + "PINECONE_API_KEY is not set. Add it to your .env file " + "(get a free key at https://www.pinecone.io/)." + ) + + self.index_name = index_name or settings.pinecone_index_name + self.dimension = dimension or settings.embedding_dimension + + self._pc = Pinecone(api_key=settings.pinecone_api_key) + self._ensure_index() + self.index = self._pc.Index(self.index_name) + + def _ensure_index(self): + existing = [i["name"] for i in self._pc.list_indexes()] + if self.index_name not in existing: + self._pc.create_index( + name=self.index_name, + dimension=self.dimension, + metric=settings.pinecone_metric, + spec=ServerlessSpec(cloud=settings.pinecone_cloud, region=settings.pinecone_region), + ) + + def upsert(self, vectors: list): + """ + vectors: list of dicts, each: + {"id": str, "values": [float, ...], "metadata": {...}} + """ + if not vectors: + return + self.index.upsert(vectors=vectors) + + def query(self, vector: list, top_k: int = 5, filter: dict = None) -> list: + """Returns Pinecone's list of matches: [{id, score, metadata}, ...]""" + result = self.index.query( + vector=vector, + top_k=top_k, + include_metadata=True, + filter=filter, + ) + return result.get("matches", []) + + def delete(self, ids: list): + if ids: + self.index.delete(ids=ids) + + def delete_by_filter(self, filter: dict): + self.index.delete(filter=filter) + + def stats(self) -> dict: + return self.index.describe_index_stats() diff --git a/ai-ml/ingestion/main.py b/ai-ml/ingestion/main.py index 17a3a5e..0ffb87e 100644 --- a/ai-ml/ingestion/main.py +++ b/ai-ml/ingestion/main.py @@ -2,112 +2,126 @@ 8. API Design -------------- Free stack: FastAPI + Uvicorn — pip install fastapi uvicorn python-multipart - Run locally (from inside the ai-ml/ folder): - uvicorn ingestion.main:app --reload - + uvicorn ingestion.main:app --reload --port 8001 Endpoints: POST /ingest/pdf (multipart file upload) POST /ingest/youtube ({"url": "..."}) POST /ingest/article ({"url": "..."}) -""" +[NV-2 fix] All three endpoints now require a valid JWT +("Authorization: Bearer "), matching Contract v1 and the same +pattern quiz_generator/app/auth.py already implements. user_id is no +longer accepted from the client (form field / request body) — it is +derived exclusively from the verified token's `sub` claim, so a +caller can never claim to be another user. This closes the gap +identified in NV-2: previously these endpoints had no authentication +at all, unlike quiz_generator's endpoints, which were already correct. +""" +from __future__ import annotations import shutil import tempfile import os +import uuid -from fastapi import FastAPI, UploadFile, File, HTTPException +from fastapi import FastAPI, UploadFile, File, HTTPException, Depends from pydantic import BaseModel - +from embedding.chroma_store import delete_chunks from ingestion.pdf.extractor import ingest_pdf from ingestion.youtube.transcript import ingest_youtube from ingestion.web.scraper import ingest_article +from embedding.chunker import chunk_document +from embedding.chroma_store import store_chunks +# Reusing the existing, working JWT verification already implemented +# for quiz_generator — same secret, same failure modes, same Contract +# v1 behavior. Not duplicated; imported directly so both services +# stay in sync if the auth scheme ever changes. +from quiz_generator.app.auth import get_current_user_id app = FastAPI(title="StudyMind AI - Content Ingestion Pipeline") class URLRequest(BaseModel): url: str + # user_id intentionally removed — identity now comes only from + # the verified JWT, never from client-supplied request data. + + +def _chunk_and_store(result: dict, user_id: str) -> dict: + """Shared helper: chunk an ingested document and store it in ChromaDB.""" + document_id = str(uuid.uuid4()) + chunks = chunk_document(result) + stored_count = store_chunks( + chunks=chunks, + user_id=user_id, + document_id=document_id, + title=result.get("title", ""), + ) + return { + "document_id": document_id, + "title": result.get("title", ""), + "chunks_stored": stored_count, + } # ----------------------------- # PDF INGESTION # ----------------------------- @app.post("/ingest/pdf") -async def ingest_pdf_endpoint(file: UploadFile = File(...)): - - # Validate file type +async def ingest_pdf_endpoint( + file: UploadFile = File(...), + user_id: str = Depends(get_current_user_id), +): if not file.filename.lower().endswith(".pdf"): - raise HTTPException( - status_code=400, - detail="File must be a PDF" - ) - - # Save uploaded PDF temporarily - with tempfile.NamedTemporaryFile( - delete=False, - suffix=".pdf" - ) as tmp: + raise HTTPException(status_code=400, detail="File must be a PDF") + with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp: shutil.copyfileobj(file.file, tmp) tmp_path = tmp.name try: - # Run PDF ingestion pipeline - result = ingest_pdf( - file_path=tmp_path, - original_filename=file.filename - ) - + result = ingest_pdf(file_path=tmp_path, original_filename=file.filename) + storage_info = _chunk_and_store(result, user_id) except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e) - ) - + raise HTTPException(status_code=500, detail=str(e)) finally: - # Remove temporary file after processing if os.path.exists(tmp_path): os.remove(tmp_path) - return result + return {**result, **storage_info} # ----------------------------- # YOUTUBE INGESTION # ----------------------------- @app.post("/ingest/youtube") -async def ingest_youtube_endpoint(payload: URLRequest): - +async def ingest_youtube_endpoint( + payload: URLRequest, + user_id: str = Depends(get_current_user_id), +): try: result = ingest_youtube(payload.url) - + storage_info = _chunk_and_store(result, user_id) except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e) - ) - - return result + raise HTTPException(status_code=500, detail=str(e)) + return {**result, **storage_info} # ----------------------------- # ARTICLE INGESTION # ----------------------------- @app.post("/ingest/article") -async def ingest_article_endpoint(payload: URLRequest): - +async def ingest_article_endpoint( + payload: URLRequest, + user_id: str = Depends(get_current_user_id), +): try: result = ingest_article(payload.url) - + storage_info = _chunk_and_store(result, user_id) except Exception as e: - raise HTTPException( - status_code=500, - detail=str(e) - ) - - return result + raise HTTPException(status_code=500, detail=str(e)) + return {**result, **storage_info} # ----------------------------- @@ -115,8 +129,31 @@ async def ingest_article_endpoint(payload: URLRequest): # ----------------------------- @app.get("/") async def root(): - return { "status": "ok", - "service": "StudyMind AI Content Ingestion Pipeline" - } \ No newline at end of file + "service": "StudyMind AI Content Ingestion Pipeline", + } + + +# ----------------------------- +# DOCUMENT PURGE (P0-5) +# ----------------------------- +@app.delete("/documents/{document_id}") +async def delete_document_endpoint( + document_id: str, + user_id: str = Depends(get_current_user_id), +): + """ + [P0-5] Purge a document's chunks from the shared ChromaDB store. + Matches the DELETE /documents/{document_id} contract Pluto's + delete_upload() already calls, and reuses the same + chroma_store.delete_chunks() Lambda already verified working for + quiz_generator's own purge endpoint. Idempotent. + + Note: does not verify document_id actually belongs to user_id + before deleting -- same ownership-check gap already flagged on + quiz_generator's purge endpoint. Caller (Pluto) is expected to + have already confirmed ownership before calling this. + """ + delete_chunks(document_id) + return {"success": True, "message": f"Document {document_id} purged."} \ No newline at end of file diff --git a/ai-ml/quiz-generator/.gitkeep b/ai-ml/ingestion/pdf/.gitkeep similarity index 100% rename from ai-ml/quiz-generator/.gitkeep rename to ai-ml/ingestion/pdf/.gitkeep diff --git a/ai-ml/ingestion/web/.gitkeep b/ai-ml/ingestion/web/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/ingestion/youtube/.gitkeep b/ai-ml/ingestion/youtube/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/.gitignore b/ai-ml/knowledge_graph/.gitignore new file mode 100644 index 0000000..75c6182 --- /dev/null +++ b/ai-ml/knowledge_graph/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/ai-ml/knowledge_graph/README.md b/ai-ml/knowledge_graph/README.md new file mode 100644 index 0000000..0939041 --- /dev/null +++ b/ai-ml/knowledge_graph/README.md @@ -0,0 +1,85 @@ +# Knowledge Graph — Team Lambda + +Connects related content a user has ingested, by reusing the vectors already +produced by the embedding pipeline. Two layers: + +- **Document graph** — connects whole documents that are topically related. +- **Topic graph** — finer connections between individual chunks, within and + across documents. + +## Where this fits + +``` +ai-ml/ +├── ingestion/ +├── embedding/ +├── quiz_generator/ +└── knowledge_graph/ ← this module +``` + +## Install + +From `ai-ml/`: + +```bash +pip install -r knowledge_graph/requirements.txt +``` + +(Most of these are likely already installed via `embedding/requirements.txt` +— this file just makes the module's own dependencies explicit.) + +## Run the API + +```bash +cd ai-ml +uvicorn knowledge_graph.app.api.graph_routes:app --reload --port 8005 +``` + +Docs: http://127.0.0.1:8005/docs + +All endpoints require `Authorization: Bearer `, same as quiz_generator. + +| Endpoint | Method | Description | +|---|---|---| +| `/health` | GET | No auth — basic liveness check | +| `/graph` | GET | Returns the authenticated user's current graph (nodes + edges) | +| `/graph/rebuild` | POST | Rebuilds the graph from the user's current embedded content | +| `/graph` | DELETE | Deletes all graph edges for the authenticated user | + +## Run the tests + +```bash +cd ai-ml +pytest knowledge_graph/tests/ -v +``` + +14 tests, covering: utility math (vectorizer, similarity, labeling), graph +building against a real Chroma collection, related vs. unrelated document +detection, empty-user handling, and delete behavior. + +## How it works, briefly + +1. `document_graph_builder.py` groups a user's chunks by document, averages + each document's chunk vectors, and compares every pair using cosine + similarity (`utils/similarity.py`). +2. Pairs above `SIMILARITY_THRESHOLD_DOCUMENT` (default 0.5, see `config.py`) + become edges, each labeled with shared keywords (`utils/topic_labeler.py` + — no LLM, no paid API). +3. `topic_graph_builder.py` does the same at the individual-chunk level. +4. `graph_service.py` orchestrates both builders and persists edges via + `storage/graph_store.py` — a JSON file living alongside the existing + shared ChromaDB data directory (no new database introduced). +5. `api/graph_routes.py` exposes it all over HTTP, authenticated with the + same JWT pattern as quiz_generator (`quiz_generator.app.auth`). + +## Not included in this version + +- Frontend visualization (this module returns data, not a rendered graph) +- LLM-based relationship explanations (labeling uses keyword overlap only, + to avoid requiring a paid API key) + +## Known limitation + +`storage/graph_store.py` uses a simple JSON file, not a proper database — +fine for how this is used today (rebuilt on-demand per user), but not +safe under heavy concurrent writes. Worth revisiting if usage grows. diff --git a/ai-ml/knowledge_graph/__init__.py b/ai-ml/knowledge_graph/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/__init__.py b/ai-ml/knowledge_graph/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/api/__init__.py b/ai-ml/knowledge_graph/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/api/graph_routes.py b/ai-ml/knowledge_graph/app/api/graph_routes.py new file mode 100644 index 0000000..61d4f93 --- /dev/null +++ b/ai-ml/knowledge_graph/app/api/graph_routes.py @@ -0,0 +1,61 @@ +""" +Knowledge Graph API — FastAPI service. + +Run from ai-ml/ (so the knowledge_graph.* and embedding.* imports resolve): + uvicorn knowledge_graph.app.api.graph_routes:app --reload --port 8003 + +Interactive docs: http://127.0.0.1:8003/docs + +[Built in from the start, per NV-2] Every endpoint below requires a +valid JWT, reusing quiz_generator.app.auth's existing, working +implementation — the same fix already applied to ingestion. user_id +is never accepted from the client; it comes only from the verified +token's `sub` claim. +""" +from fastapi import FastAPI, Depends +from fastapi.middleware.cors import CORSMiddleware + +from quiz_generator.app.auth import get_current_user_id +from knowledge_graph.app.services.graph_service import GraphService + +app = FastAPI(title="StudyMind Knowledge Graph API — Team Lambda") + +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:5173"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +_service: GraphService | None = None + + +def get_service() -> GraphService: + global _service + if _service is None: + _service = GraphService() + return _service + + +@app.get("/health") +def health_check(): + return {"status": "ok"} + + +@app.get("/graph") +def get_graph_endpoint(user_id: str = Depends(get_current_user_id)) -> dict: + """Returns the authenticated user's current knowledge graph.""" + return get_service().get_graph(user_id) + + +@app.post("/graph/rebuild") +def rebuild_graph_endpoint(user_id: str = Depends(get_current_user_id)) -> dict: + """Rebuilds the authenticated user's knowledge graph from their current content.""" + return get_service().build_graph(user_id) + + +@app.delete("/graph") +def delete_graph_endpoint(user_id: str = Depends(get_current_user_id)) -> dict: + """Deletes all graph edges for the authenticated user.""" + return get_service().delete_graph(user_id) diff --git a/ai-ml/knowledge_graph/app/builders/__init__.py b/ai-ml/knowledge_graph/app/builders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/builders/document_graph_builder.py b/ai-ml/knowledge_graph/app/builders/document_graph_builder.py new file mode 100644 index 0000000..95109cd --- /dev/null +++ b/ai-ml/knowledge_graph/app/builders/document_graph_builder.py @@ -0,0 +1,81 @@ +""" +Builds document-to-document edges: groups a user's chunks by +document, averages each document's chunk vectors, compares every +pair of documents, and creates an edge wherever similarity is above +the configured threshold. +""" +from itertools import combinations + +from embedding.chroma_store import get_collection +from knowledge_graph.app.config import SIMILARITY_THRESHOLD_DOCUMENT, CONTENT_COLLECTION_NAME +from knowledge_graph.app.utils.vectorizer import get_document_vector +from knowledge_graph.app.utils.similarity import cosine_similarity +from knowledge_graph.app.utils.topic_labeler import generate_label +from knowledge_graph.app.validators.graph_validators import validate_documents_have_vectors +from knowledge_graph.app.models.graph_edge import GraphEdge + + +def get_user_documents(user_id: str) -> dict: + """ + Fetches this user's chunks from the shared ChromaDB collection and + groups them by document_id. + + Returns: { document_id: {"title": str, "vectors": [...], "texts": [...]} } + """ + collection = get_collection(name=CONTENT_COLLECTION_NAME) + result = collection.get( + where={"user_id": user_id}, + include=["embeddings", "documents", "metadatas"], + ) + + docs: dict = {} + ids = result.get("ids", []) + embeddings = result.get("embeddings", []) + documents = result.get("documents", []) + metadatas = result.get("metadatas", []) + + for i in range(len(ids)): + meta = metadatas[i] or {} + doc_id = meta.get("document_id") + if not doc_id: + continue + docs.setdefault(doc_id, {"title": meta.get("document", ""), "vectors": [], "texts": []}) + docs[doc_id]["vectors"].append(embeddings[i]) + docs[doc_id]["texts"].append(documents[i]) + + return docs + + +def build_document_graph(user_id: str) -> list: + """ + Returns a list of GraphEdge.to_dict() for this user's + document-to-document relationships. + """ + docs = validate_documents_have_vectors(get_user_documents(user_id)) + + doc_vectors = { + doc_id: get_document_vector(data["vectors"]) + for doc_id, data in docs.items() + } + + edges = [] + for (id_a, vec_a), (id_b, vec_b) in combinations(doc_vectors.items(), 2): + score = cosine_similarity(vec_a, vec_b) + if score >= SIMILARITY_THRESHOLD_DOCUMENT: + # label from a sample of each document's text, not the full text + sample_a = " ".join(docs[id_a]["texts"][:2]) + sample_b = " ".join(docs[id_b]["texts"][:2]) + + edge = GraphEdge( + user_id=user_id, + source_id=id_a, + target_id=id_b, + node_type="document", + similarity=round(score, 4), + source_title=docs[id_a]["title"], + target_title=docs[id_b]["title"], + label=generate_label(sample_a, sample_b), + ) + edges.append(edge.to_dict()) + + return edges diff --git a/ai-ml/knowledge_graph/app/builders/topic_graph_builder.py b/ai-ml/knowledge_graph/app/builders/topic_graph_builder.py new file mode 100644 index 0000000..07367df --- /dev/null +++ b/ai-ml/knowledge_graph/app/builders/topic_graph_builder.py @@ -0,0 +1,55 @@ +""" +Builds finer-grained edges between individual chunks ("topics"), +within and across a user's documents — the detail layer the +document-level graph misses. +""" +from itertools import combinations + +from knowledge_graph.app.config import SIMILARITY_THRESHOLD_TOPIC +from knowledge_graph.app.utils.similarity import cosine_similarity +from knowledge_graph.app.utils.topic_labeler import generate_label +from knowledge_graph.app.models.graph_edge import GraphEdge +from knowledge_graph.app.builders.document_graph_builder import get_user_documents + + +def build_topic_graph(user_id: str, max_chunks: int = 200) -> list: + """ + Returns a list of GraphEdge.to_dict() for chunk-level relationships. + + max_chunks caps how many of the user's chunks are compared, since + comparing every pair is O(n^2) — protects against runaway cost for + users with very large amounts of content. Chunks beyond the cap + are simply not included in this pass. + """ + docs = get_user_documents(user_id) + + # flatten into a single list of (chunk_id, vector, text, title) + chunks = [] + for doc_id, data in docs.items(): + for i, (vector, text) in enumerate(zip(data["vectors"], data["texts"])): + chunks.append({ + "chunk_id": f"{doc_id}_{i}", + "vector": vector, + "text": text, + "title": data["title"], + }) + + chunks = chunks[:max_chunks] + + edges = [] + for a, b in combinations(chunks, 2): + score = cosine_similarity(a["vector"], b["vector"]) + if score >= SIMILARITY_THRESHOLD_TOPIC: + edge = GraphEdge( + user_id=user_id, + source_id=a["chunk_id"], + target_id=b["chunk_id"], + node_type="topic", + similarity=round(score, 4), + source_title=a["title"], + target_title=b["title"], + label=generate_label(a["text"], b["text"]), + ) + edges.append(edge.to_dict()) + + return edges diff --git a/ai-ml/knowledge_graph/app/config.py b/ai-ml/knowledge_graph/app/config.py new file mode 100644 index 0000000..64d40e4 --- /dev/null +++ b/ai-ml/knowledge_graph/app/config.py @@ -0,0 +1,36 @@ +""" +Configuration for the knowledge graph module. + +Reuses the shared .env at the ai-ml/ root, same pattern as +embedding/config.py and quiz_generator/app/auth.py. +""" +import os +from pathlib import Path +from dotenv import load_dotenv + +# ai-ml/.env (app -> knowledge-graph -> ai-ml) +load_dotenv(dotenv_path=Path(__file__).resolve().parent.parent.parent / ".env") + +# --- Similarity thresholds --- +# Document-level pairs below this score are not considered related. +SIMILARITY_THRESHOLD_DOCUMENT = float(os.getenv("KG_DOC_SIMILARITY_THRESHOLD", "0.5")) + +# Topic/chunk-level pairs below this score are not considered related. +# Slightly higher than the document threshold, since chunk-level +# comparisons are noisier (less text per vector). +SIMILARITY_THRESHOLD_TOPIC = float(os.getenv("KG_TOPIC_SIMILARITY_THRESHOLD", "0.6")) + +# --- Storage --- +# Reuses the same local ChromaDB path as embedding/chroma_store.py, +# so this module reads the same underlying data directory rather than +# introducing a second database location. Edge data is written to its +# own JSON file inside that same directory (see storage/graph_store.py) — +# not a new database technology, just a new file alongside Chroma's data. +CHROMA_DB_PATH = os.getenv( + "CHROMA_DB_PATH", + r"C:\Dev\QuantumLearningWorkspace\shared_chroma_data", +) +CONTENT_COLLECTION_NAME = "study_chunks" # matches chroma_store.py's DEFAULT_COLLECTION_NAME + +# --- API --- +API_PORT = int(os.getenv("KNOWLEDGE_GRAPH_PORT", "8005")) diff --git a/ai-ml/knowledge_graph/app/models/__init__.py b/ai-ml/knowledge_graph/app/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/models/graph_edge.py b/ai-ml/knowledge_graph/app/models/graph_edge.py new file mode 100644 index 0000000..4cc8705 --- /dev/null +++ b/ai-ml/knowledge_graph/app/models/graph_edge.py @@ -0,0 +1,20 @@ +""" +Represents a relationship (edge) between two nodes in the graph. +""" +from dataclasses import dataclass, asdict +from typing import Optional + + +@dataclass +class GraphEdge: + user_id: str + source_id: str + target_id: str + node_type: str # "document" or "topic" — matches the nodes it connects + similarity: float + source_title: str + target_title: str + label: Optional[str] = None # short human-readable reason for the connection + + def to_dict(self) -> dict: + return asdict(self) diff --git a/ai-ml/knowledge_graph/app/models/graph_node.py b/ai-ml/knowledge_graph/app/models/graph_node.py new file mode 100644 index 0000000..a6f3589 --- /dev/null +++ b/ai-ml/knowledge_graph/app/models/graph_node.py @@ -0,0 +1,16 @@ +""" +Represents a single node in the knowledge graph — either a whole +document, or (for the topic-level graph) an individual chunk. +""" +from dataclasses import dataclass, asdict + + +@dataclass +class GraphNode: + node_id: str # document_id, or "documentid_chunkindex" for topic nodes + node_type: str # "document" or "topic" + title: str + user_id: str + + def to_dict(self) -> dict: + return asdict(self) diff --git a/ai-ml/knowledge_graph/app/services/__init__.py b/ai-ml/knowledge_graph/app/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/services/graph_service.py b/ai-ml/knowledge_graph/app/services/graph_service.py new file mode 100644 index 0000000..9478b52 --- /dev/null +++ b/ai-ml/knowledge_graph/app/services/graph_service.py @@ -0,0 +1,64 @@ +""" +Main integration interface for the knowledge graph module — what +api/graph_routes.py and any other module calls. + +user_id is accepted as an explicit parameter on every method here +(not sourced from a request/token internally), so this class stays +easy to unit test with any user_id value. Only the API layer +(api/graph_routes.py) is responsible for sourcing a real, verified +user_id from a JWT before calling into this service. +""" +from knowledge_graph.app.validators.graph_validators import validate_user_id +from knowledge_graph.app.builders.document_graph_builder import build_document_graph, get_user_documents +from knowledge_graph.app.builders.topic_graph_builder import build_topic_graph +from knowledge_graph.app.storage import graph_store + + +class GraphService: + def build_graph(self, user_id: str, include_topics: bool = True) -> dict: + """ + Builds (or rebuilds) this user's full graph and persists it. + Returns a summary dict, not the full graph — call get_graph() + to read it back. + """ + validate_user_id(user_id) + + document_edges = build_document_graph(user_id) + graph_store.save_edges(user_id, document_edges, node_type="document") + + topic_edge_count = 0 + if include_topics: + topic_edges = build_topic_graph(user_id) + graph_store.save_edges(user_id, topic_edges, node_type="topic") + topic_edge_count = len(topic_edges) + + return { + "user_id": user_id, + "document_edges_created": len(document_edges), + "topic_edges_created": topic_edge_count, + } + + def get_graph(self, user_id: str) -> dict: + """ + Returns { "nodes": [...], "edges": [...] } for this user, + combining both document- and topic-level edges. Nodes are + derived from the user's currently embedded documents, so the + node list always reflects current content even if the graph + hasn't been rebuilt since the last edit. + """ + validate_user_id(user_id) + + docs = get_user_documents(user_id) + nodes = [ + {"id": doc_id, "title": data["title"], "node_type": "document"} + for doc_id, data in docs.items() + ] + + edges = graph_store.get_edges(user_id) + + return {"nodes": nodes, "edges": edges} + + def delete_graph(self, user_id: str) -> dict: + validate_user_id(user_id) + graph_store.delete_edges(user_id) + return {"user_id": user_id, "deleted": True} diff --git a/ai-ml/knowledge_graph/app/storage/__init__.py b/ai-ml/knowledge_graph/app/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/storage/graph_store.py b/ai-ml/knowledge_graph/app/storage/graph_store.py new file mode 100644 index 0000000..5c40d2f --- /dev/null +++ b/ai-ml/knowledge_graph/app/storage/graph_store.py @@ -0,0 +1,76 @@ +""" +Stores graph edges as a JSON file inside the same shared data +directory ChromaDB already uses (CHROMA_DB_PATH). This is +deliberately not a new database technology — edges are small, +relationship-only records, so a flat JSON file keeps this module +free of new infrastructure dependencies while still living +alongside the "one shared data location" the rest of the project +already uses. + +Not safe for many concurrent writers, but fine for how this module +is used today: rebuilt on-demand per user, not written to under +sustained concurrent load. +""" +import json +import os +from pathlib import Path +from threading import Lock + +from knowledge_graph.app.config import CHROMA_DB_PATH + +_EDGES_FILENAME = "knowledge_graph_edges.json" +_lock = Lock() + + +def _edges_path() -> Path: + return Path(CHROMA_DB_PATH) / _EDGES_FILENAME + + +def _read_all() -> list: + path = _edges_path() + if not path.exists(): + return [] + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _write_all(edges: list) -> None: + path = _edges_path() + os.makedirs(path.parent, exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + json.dump(edges, f, indent=2) + + +def save_edges(user_id: str, edges: list, node_type: str) -> int: + """ + Replaces all existing edges of the given node_type for this user + with the new set (a full rebuild, not an incremental append — + keeps the graph consistent with the latest embedded content). + edges: list of GraphEdge.to_dict() results. + Returns the number of edges written. + """ + with _lock: + all_edges = _read_all() + # drop this user's existing edges of this type, keep everyone else's + all_edges = [ + e for e in all_edges + if not (e["user_id"] == user_id and e["node_type"] == node_type) + ] + all_edges.extend(edges) + _write_all(all_edges) + return len(edges) + + +def get_edges(user_id: str, node_type: str = None) -> list: + all_edges = _read_all() + result = [e for e in all_edges if e["user_id"] == user_id] + if node_type: + result = [e for e in result if e["node_type"] == node_type] + return result + + +def delete_edges(user_id: str) -> None: + with _lock: + all_edges = _read_all() + all_edges = [e for e in all_edges if e["user_id"] != user_id] + _write_all(all_edges) diff --git a/ai-ml/knowledge_graph/app/utils/__init__.py b/ai-ml/knowledge_graph/app/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/utils/similarity.py b/ai-ml/knowledge_graph/app/utils/similarity.py new file mode 100644 index 0000000..0f7f8f0 --- /dev/null +++ b/ai-ml/knowledge_graph/app/utils/similarity.py @@ -0,0 +1,23 @@ +""" +Cosine similarity between two vectors. +""" +import numpy as np + + +def cosine_similarity(vec_a: list, vec_b: list) -> float: + """ + Returns a score from -1 to 1. Closer to 1 means more related. + + Raises ValueError if either vector has zero magnitude (all zeros), + since cosine similarity is undefined in that case. + """ + a = np.array(vec_a) + b = np.array(vec_b) + + norm_a = np.linalg.norm(a) + norm_b = np.linalg.norm(b) + + if norm_a == 0 or norm_b == 0: + raise ValueError("cosine_similarity() received a zero-magnitude vector") + + return float(np.dot(a, b) / (norm_a * norm_b)) diff --git a/ai-ml/knowledge_graph/app/utils/topic_labeler.py b/ai-ml/knowledge_graph/app/utils/topic_labeler.py new file mode 100644 index 0000000..76623a4 --- /dev/null +++ b/ai-ml/knowledge_graph/app/utils/topic_labeler.py @@ -0,0 +1,40 @@ +""" +Generates a short, human-readable label explaining why two chunks of +text are related — using shared keyword overlap, not an LLM. Keeps +this free of any paid API dependency. +""" +import re +from collections import Counter + +# Small built-in stopword list — enough to filter common noise words +# without pulling in a heavier NLP dependency for this lightweight task. +_STOPWORDS = { + "the", "a", "an", "and", "or", "but", "is", "are", "was", "were", + "in", "on", "at", "to", "for", "of", "with", "by", "as", "that", + "this", "it", "be", "from", "which", "these", "those", "their", + "its", "has", "have", "had", "not", "can", "will", "would", "also", +} + + +def _extract_keywords(text: str, top_n: int = 15) -> set: + words = re.findall(r"[a-zA-Z]{3,}", text.lower()) + words = [w for w in words if w not in _STOPWORDS] + most_common = [w for w, _ in Counter(words).most_common(top_n)] + return set(most_common) + + +def generate_label(text_a: str, text_b: str, max_terms: int = 3) -> str: + """ + Returns a short label like "shared terms: neural networks, training" + based on keyword overlap between two texts. Returns a generic + fallback label if no meaningful overlap is found. + """ + keywords_a = _extract_keywords(text_a) + keywords_b = _extract_keywords(text_b) + shared = keywords_a & keywords_b + + if not shared: + return "related topics" + + top_shared = sorted(shared)[:max_terms] + return f"shared terms: {', '.join(top_shared)}" diff --git a/ai-ml/knowledge_graph/app/utils/vectorizer.py b/ai-ml/knowledge_graph/app/utils/vectorizer.py new file mode 100644 index 0000000..fd82676 --- /dev/null +++ b/ai-ml/knowledge_graph/app/utils/vectorizer.py @@ -0,0 +1,21 @@ +""" +Turns a document's many chunk vectors into a single vector +representing the whole document. +""" +import numpy as np + + +def get_document_vector(chunk_vectors: list) -> list: + """ + Averages a list of chunk vectors into one document-level vector. + + chunk_vectors: list of vectors (list[float]), all the same length, + belonging to one document. + + Raises ValueError if no vectors are given, so callers can decide + how to handle documents with no embedded chunks (e.g. skip them). + """ + if not chunk_vectors: + raise ValueError("get_document_vector() requires at least one chunk vector") + + return np.mean(np.array(chunk_vectors), axis=0).tolist() diff --git a/ai-ml/knowledge_graph/app/validators/__init__.py b/ai-ml/knowledge_graph/app/validators/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/app/validators/graph_validators.py b/ai-ml/knowledge_graph/app/validators/graph_validators.py new file mode 100644 index 0000000..5045717 --- /dev/null +++ b/ai-ml/knowledge_graph/app/validators/graph_validators.py @@ -0,0 +1,21 @@ +""" +Validation checks used before building or reading a graph. +""" + + +def validate_user_id(user_id: str) -> None: + if not user_id or not isinstance(user_id, str): + raise ValueError("A valid user_id is required") + + +def validate_documents_have_vectors(docs: dict) -> dict: + """ + Filters out any document with no chunk vectors, so a document + that failed to embed properly doesn't crash graph building. + Returns the filtered dict; does not raise. + """ + return { + doc_id: data + for doc_id, data in docs.items() + if data.get("vectors") + } diff --git a/ai-ml/knowledge_graph/requirements.txt b/ai-ml/knowledge_graph/requirements.txt new file mode 100644 index 0000000..8370127 --- /dev/null +++ b/ai-ml/knowledge_graph/requirements.txt @@ -0,0 +1,12 @@ +# Core dependencies — numpy and chromadb are likely already installed +# via embedding/requirements.txt, but listed here so this module's +# requirements are self-contained. +numpy +chromadb +fastapi +uvicorn +python-dotenv +pyjwt + +# For running tests +pytest diff --git a/ai-ml/knowledge_graph/tests/__init__.py b/ai-ml/knowledge_graph/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/knowledge_graph/tests/test_graph_service.py b/ai-ml/knowledge_graph/tests/test_graph_service.py new file mode 100644 index 0000000..ea8e143 --- /dev/null +++ b/ai-ml/knowledge_graph/tests/test_graph_service.py @@ -0,0 +1,93 @@ +""" +Integration tests for GraphService, against a real (ephemeral, +in-memory) ChromaDB collection — not mocked, so these catch real +wiring issues between the builders, storage, and Chroma. + +Run: pytest knowledge_graph/tests/test_graph_service.py +""" +import numpy as np +import pytest + +from embedding.chroma_store import get_collection +from knowledge_graph.app.services.graph_service import GraphService + + +def _make_vec(base: float, dim: int = 16, seed: int = 0) -> list: + rng = np.random.default_rng(seed) + return rng.normal(loc=base, scale=0.03, size=dim).tolist() + + +@pytest.fixture +def seeded_user(tmp_path): + """Inserts two related documents and one unrelated document for a test user.""" + user_id = "pytest_user" + collection = get_collection() + + rows = [ + ("docA", 0, "ML Basics", "Neural networks learn from data.", 0.8, 1), + ("docA", 1, "ML Basics", "Training uses gradient descent.", 0.8, 2), + ("docB", 0, "Deep Learning", "Gradient descent trains neural networks.", 0.82, 3), + ("docC", 0, "Cooking", "Bread requires yeast and flour.", -0.8, 4), + ] + ids, embeddings, documents, metadatas = [], [], [], [] + for doc_id, idx, title, text, base, seed in rows: + ids.append(f"{doc_id}_{idx}") + embeddings.append(_make_vec(base, seed=seed)) + documents.append(text) + metadatas.append({"user_id": user_id, "document_id": doc_id, "document": title, "chunk_index": idx}) + + collection.upsert(ids=ids, embeddings=embeddings, documents=documents, metadatas=metadatas) + return user_id + + +def test_build_graph_links_related_documents(seeded_user): + service = GraphService() + result = service.build_graph(seeded_user) + assert result["document_edges_created"] >= 1 + + graph = service.get_graph(seeded_user) + doc_edges = [e for e in graph["edges"] if e["node_type"] == "document"] + titles = {(e["source_title"], e["target_title"]) for e in doc_edges} + + assert ("ML Basics", "Deep Learning") in titles or ("Deep Learning", "ML Basics") in titles + + +def test_build_graph_does_not_link_unrelated_documents(seeded_user): + service = GraphService() + service.build_graph(seeded_user) + graph = service.get_graph(seeded_user) + + doc_edges = [e for e in graph["edges"] if e["node_type"] == "document"] + cooking_involved = any( + "Cooking" in (e["source_title"], e["target_title"]) for e in doc_edges + ) + assert not cooking_involved + + +def test_get_graph_nodes_match_documents(seeded_user): + service = GraphService() + graph = service.get_graph(seeded_user) + titles = {n["title"] for n in graph["nodes"]} + assert titles == {"ML Basics", "Deep Learning", "Cooking"} + + +def test_delete_graph_clears_edges(seeded_user): + service = GraphService() + service.build_graph(seeded_user) + service.delete_graph(seeded_user) + + graph = service.get_graph(seeded_user) + assert graph["edges"] == [] + + +def test_user_with_no_documents_does_not_crash(): + service = GraphService() + result = service.build_graph("nobody_has_this_id") + assert result["document_edges_created"] == 0 + assert result["topic_edges_created"] == 0 + + +def test_empty_user_id_raises(): + service = GraphService() + with pytest.raises(ValueError): + service.get_graph("") diff --git a/ai-ml/knowledge_graph/tests/test_utils.py b/ai-ml/knowledge_graph/tests/test_utils.py new file mode 100644 index 0000000..dc5c7d1 --- /dev/null +++ b/ai-ml/knowledge_graph/tests/test_utils.py @@ -0,0 +1,53 @@ +""" +Tests for the standalone utility functions — no database needed. +Run: pytest knowledge_graph/tests/test_utils.py +""" +import pytest + +from knowledge_graph.app.utils.vectorizer import get_document_vector +from knowledge_graph.app.utils.similarity import cosine_similarity +from knowledge_graph.app.utils.topic_labeler import generate_label + + +def test_get_document_vector_averages_correctly(): + chunks = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]] + result = get_document_vector(chunks) + assert abs(result[0] - 1 / 3) < 0.001 + assert abs(result[1] - 1 / 3) < 0.001 + assert abs(result[2] - 1 / 3) < 0.001 + + +def test_get_document_vector_empty_raises(): + with pytest.raises(ValueError): + get_document_vector([]) + + +def test_cosine_similarity_identical_vectors(): + assert abs(cosine_similarity([1.0, 0.0], [1.0, 0.0]) - 1.0) < 0.001 + + +def test_cosine_similarity_orthogonal_vectors(): + assert abs(cosine_similarity([1.0, 0.0], [0.0, 1.0]) - 0.0) < 0.001 + + +def test_cosine_similarity_opposite_vectors(): + assert abs(cosine_similarity([1.0, 0.0], [-1.0, 0.0]) - (-1.0)) < 0.001 + + +def test_cosine_similarity_zero_vector_raises(): + with pytest.raises(ValueError): + cosine_similarity([0.0, 0.0], [1.0, 0.0]) + + +def test_generate_label_finds_shared_terms(): + label = generate_label( + "Backpropagation computes gradients in a neural network", + "Gradient descent updates the neural network weights", + ) + assert "shared terms" in label + assert "neural" in label or "gradient" in label + + +def test_generate_label_no_overlap_falls_back(): + label = generate_label("Roman Empire history ancient", "Quantum physics particles") + assert label == "related topics" diff --git a/ai-ml/quiz_generator/.gitignore b/ai-ml/quiz_generator/.gitignore new file mode 100644 index 0000000..7f7cfdb --- /dev/null +++ b/ai-ml/quiz_generator/.gitignore @@ -0,0 +1,21 @@ +# Python +__pycache__/ +*.py[cod] + +# Environment +.env + +# VS Code +.vscode/ + +# Pytest cache +.pytest_cache/ + +# Virtual environments +venv/ +.venv/ + +# OS files +.DS_Store +Thumbs.dbd +demo_commands.txt \ No newline at end of file diff --git a/ai-ml/quiz_generator/.gitkeep b/ai-ml/quiz_generator/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ai-ml/quiz_generator/README.md b/ai-ml/quiz_generator/README.md new file mode 100644 index 0000000..85abca7 --- /dev/null +++ b/ai-ml/quiz_generator/README.md @@ -0,0 +1,101 @@ +# AI Quiz Generator + +## Overview + +The AI Quiz Generator is a module of the QuantumLearningWorkspace project. It generates different types of quiz questions from input text using the Groq API. + +## Features + +- Multiple Choice Questions (MCQs) +- True/False Questions +- Fill in the Blank Questions +- Short Answer Questions +- Centralized QuizService for selecting question types + +## Project Structure + +``` +quiz-generator/ +│ +├── app/ +│ ├── api/ +│ ├── generators/ +│ ├── models/ +│ ├── services/ +│ ├── utils/ +│ ├── validators/ +│ └── config.py +│ +├── tests/ +├── .env +├── requirements.txt +└── README.md +``` + +## Installation + +Clone the repository and install the required packages. + +```bash +pip install -r requirements.txt +``` + +## Environment Variables + +Create a `.env` file in the project root. + +```env +GROQ_API_KEY=your_groq_api_key_here +``` + +## Running the Tests + +### MCQ Generator + +```bash +python -m tests.test_mcq_generator +``` + +### True/False Generator + +```bash +python -m tests.test_true_false_generator +``` + +### Fill in the Blank Generator + +```bash +python -m tests.test_fill_blank_generator +``` + +### Short Answer Generator + +```bash +python -m tests.test_short_answer_generator +``` + +### Quiz Service + +```bash +python -m tests.test_quiz_service +``` + +## Supported Question Types + +- mcq +- true_false +- fill_blank +- short_answer + +## Technologies Used + +- Python 3.11 +- Groq API +- FastAPI +- Pydantic +- python-dotenv +- YAKE + +## Author + +Developed as part of the QuantumLearningWorkspace Internship Project. \ No newline at end of file diff --git a/ai-ml/quiz_generator/app/auth.py b/ai-ml/quiz_generator/app/auth.py new file mode 100644 index 0000000..2bffda6 --- /dev/null +++ b/ai-ml/quiz_generator/app/auth.py @@ -0,0 +1,62 @@ +""" +[Contract v1, Section 10 — Quiz Security] JWT authentication for +quiz endpoints. + +Matches the scheme documented in docs/api-contracts.md for Mu's +/ask: HS256, JWT_SECRET_KEY env var, "Authorization: Bearer ", +identity read from the token's `sub` claim (the user's login email, +per Contract v1 Section 2). No user_id is ever trusted from a +request body or header directly — only from a verified token. + +Requires PyJWT (`pip install pyjwt`) — add it to requirements.txt +if it isn't already there (Mu's service already depends on it for +the same purpose, so it's likely already in the root/shared +requirements somewhere; worth checking before assuming it needs +adding here too). +""" + +import os +from pathlib import Path + +import jwt +from dotenv import load_dotenv +from fastapi import Header, HTTPException + +load_dotenv(dotenv_path=Path(__file__).resolve().parent.parent.parent / ".env") + +JWT_SECRET_KEY = os.getenv("JWT_SECRET_KEY", "") +JWT_ALGORITHM = "HS256" + + +def get_current_user_id(authorization: str = Header(default=None)) -> str: + """ + FastAPI dependency. Verifies the Authorization header and returns + the authenticated user's identity from the JWT's `sub` claim. + + Failure modes match Contract v1 / Mu's documented /ask behavior + exactly, so error handling is consistent across services: + - missing header -> 403 "Not authenticated" + - invalid/expired token -> 401 "Could not validate credentials." + - server missing the secret -> 500 "Server is not configured with a JWT secret." + """ + if not JWT_SECRET_KEY: + raise HTTPException( + status_code=500, + detail="Server is not configured with a JWT secret.", + ) + + if not authorization or not authorization.startswith("Bearer "): + raise HTTPException(status_code=403, detail="Not authenticated") + + token = authorization.split(" ", 1)[1] + + try: + payload = jwt.decode(token, JWT_SECRET_KEY, algorithms=[JWT_ALGORITHM]) + except jwt.PyJWTError: + raise HTTPException(status_code=401, detail="Could not validate credentials.") + + user_id = payload.get("sub") + if not user_id: + raise HTTPException(status_code=401, detail="Could not validate credentials.") + + return user_id \ No newline at end of file diff --git a/ai-ml/quiz_generator/app/config.py b/ai-ml/quiz_generator/app/config.py new file mode 100644 index 0000000..15840dd --- /dev/null +++ b/ai-ml/quiz_generator/app/config.py @@ -0,0 +1,35 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + +# ========================================== +# Groq Configuration +# ========================================== + +GROQ_API_KEY = os.getenv("GROQ_API_KEY", "") + +GROQ_MODEL = os.getenv("GROQ_MODEL", "llama-3.1-8b-instant") + + +# ========================================== +# YAKE Configuration +# ========================================== + +YAKE_MAX_KEYWORDS = 15 +YAKE_NGRAM_SIZE = 2 +YAKE_DEDUP_THRESHOLD = 0.9 + + +# ========================================== +# Quiz Configuration +# ========================================== + +DEFAULT_QUESTION_COUNT = 10 + +SUPPORTED_QUESTION_TYPES = [ + "mcq", + "true_false", + "fill_blank", + "short_answer" +] \ No newline at end of file diff --git a/ai-ml/quiz_generator/app/generators/base_generator.py b/ai-ml/quiz_generator/app/generators/base_generator.py new file mode 100644 index 0000000..e22189f --- /dev/null +++ b/ai-ml/quiz_generator/app/generators/base_generator.py @@ -0,0 +1,34 @@ +from __future__ import annotations +from abc import ABC, abstractmethod + +from quiz_generator.app.models.question import Question + + +class BaseGenerator(ABC): + """ + Base class for all quiz generators. + Every quiz generator (MCQ, True/False, Fill in the Blank, + Short Answer) will inherit from this class. + """ + + @abstractmethod + def generate( + self, + text: str, + number_of_questions: int = 5, + difficulty: str = "medium", + topic: str = "", + ) -> list[Question]: + """ + Generate quiz questions from the given text. + + Parameters: + text (str): Input text from which questions are generated. + number_of_questions (int): How many questions to generate. + difficulty (str): Difficulty level (easy, medium, hard). + topic (str): Topic label to attach to each generated question. + + Returns: + list[Question]: A list of generated, structured questions. + """ + pass diff --git a/ai-ml/quiz_generator/app/generators/fill_blank_generator.py b/ai-ml/quiz_generator/app/generators/fill_blank_generator.py new file mode 100644 index 0000000..2ae019c --- /dev/null +++ b/ai-ml/quiz_generator/app/generators/fill_blank_generator.py @@ -0,0 +1,90 @@ +from __future__ import annotations +import json +import uuid + +from groq import Groq + +from quiz_generator.app.generators.base_generator import BaseGenerator +from quiz_generator.app.config import GROQ_API_KEY, GROQ_MODEL +from quiz_generator.app.models.question import Question + + +class FillBlankGenerator(BaseGenerator): + """ + Generator responsible for creating Fill in the Blank questions + from the provided text using the Groq API. + """ + + def __init__(self): + self.client = Groq(api_key=GROQ_API_KEY) + + def generate( + self, + text: str, + number_of_questions: int = 5, + difficulty: str = "medium", + topic: str = "", + ) -> list[Question]: + """ + Generate Fill in the Blank questions using the Groq API, returning + structured Question objects (not raw text). + """ + prompt = f""" + Generate exactly {number_of_questions} Fill in the Blank questions + from the following text. + + Rules: + - Replace only one important word or phrase in each sentence with a blank (_____). + - Clearly mark the correct answer (the word/phrase that fills the blank). + - Questions should cover different concepts from the text. + - Keep the difficulty at {difficulty} level. + + Respond with ONLY a JSON array, no other text, in this exact shape: + [ + {{ + "question": "... _____ ...", + "answer": "...", + "explanation": "..." + }} + ] + + Text: + {text} + """ + + response = self.client.chat.completions.create( + model=GROQ_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.5, + ) + + raw = response.choices[0].message.content or "" + raw = raw.strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.startswith("json"): + raw = raw[4:] + raw = raw.strip() + + try: + items = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"FillBlankGenerator: could not parse LLM response as JSON: {exc}" + ) from exc + + questions: list[Question] = [] + for item in items: + questions.append( + Question( + question=item["question"], + question_id=str(uuid.uuid4()), + topic=topic, + question_type="fill_blank", + options=None, + answer=item["answer"], + difficulty=difficulty, + explanation=item.get("explanation"), + ) + ) + return questions diff --git a/ai-ml/quiz_generator/app/generators/mcq_generator.py b/ai-ml/quiz_generator/app/generators/mcq_generator.py new file mode 100644 index 0000000..a43123b --- /dev/null +++ b/ai-ml/quiz_generator/app/generators/mcq_generator.py @@ -0,0 +1,91 @@ +from __future__ import annotations +import json +import uuid + +from groq import Groq + +from quiz_generator.app.generators.base_generator import BaseGenerator +from quiz_generator.app.config import GROQ_API_KEY, GROQ_MODEL +from quiz_generator.app.models.question import Question + + +class MCQGenerator(BaseGenerator): + """ + Generator responsible for creating Multiple Choice Questions (MCQs) + from the provided text using the Groq API. + """ + + def __init__(self): + self.client = Groq(api_key=GROQ_API_KEY) + + def generate( + self, + text: str, + number_of_questions: int = 5, + difficulty: str = "medium", + topic: str = "", + ) -> list[Question]: + """ + Generate Multiple Choice Questions using the Groq API, returning + structured Question objects (not raw text). + """ + prompt = f""" + Generate exactly {number_of_questions} multiple-choice questions + from the following text. + + Rules: + - Each question must have exactly 4 options. + - Clearly mark the correct answer (it must match one option exactly). + - Questions should cover different concepts from the text. + - Keep the difficulty at {difficulty} level. + + Respond with ONLY a JSON array, no other text, in this exact shape: + [ + {{ + "question": "...", + "options": ["...", "...", "...", "..."], + "answer": "...", + "explanation": "..." + }} + ] + + Text: + {text} + """ + + response = self.client.chat.completions.create( + model=GROQ_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.5, + ) + + raw = response.choices[0].message.content or "" + raw = raw.strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.startswith("json"): + raw = raw[4:] + raw = raw.strip() + + try: + items = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"MCQGenerator: could not parse LLM response as JSON: {exc}" + ) from exc + + questions: list[Question] = [] + for item in items: + questions.append( + Question( + question=item["question"], + question_id=str(uuid.uuid4()), + topic=topic, + question_type="mcq", + options=item.get("options"), + answer=item["answer"], + difficulty=difficulty, + explanation=item.get("explanation"), + ) + ) + return questions diff --git a/ai-ml/quiz_generator/app/generators/short_answer_generator.py b/ai-ml/quiz_generator/app/generators/short_answer_generator.py new file mode 100644 index 0000000..eba5546 --- /dev/null +++ b/ai-ml/quiz_generator/app/generators/short_answer_generator.py @@ -0,0 +1,90 @@ +from __future__ import annotations +import json +import uuid + +from groq import Groq + +from quiz_generator.app.generators.base_generator import BaseGenerator +from quiz_generator.app.config import GROQ_API_KEY, GROQ_MODEL +from quiz_generator.app.models.question import Question + + +class ShortAnswerGenerator(BaseGenerator): + """ + Generator responsible for creating Short Answer questions + from the provided text using the Groq API. + """ + + def __init__(self): + self.client = Groq(api_key=GROQ_API_KEY) + + def generate( + self, + text: str, + number_of_questions: int = 5, + difficulty: str = "medium", + topic: str = "", + ) -> list[Question]: + """ + Generate Short Answer questions using the Groq API, returning + structured Question objects (not raw text). + """ + prompt = f""" + Generate exactly {number_of_questions} short answer questions + from the following text. + + Rules: + - Each question should require a short, factual answer (a few words). + - Clearly mark the correct answer. + - Questions should cover different concepts from the text. + - Keep the difficulty at {difficulty} level. + + Respond with ONLY a JSON array, no other text, in this exact shape: + [ + {{ + "question": "...", + "answer": "...", + "explanation": "..." + }} + ] + + Text: + {text} + """ + + response = self.client.chat.completions.create( + model=GROQ_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.5, + ) + + raw = response.choices[0].message.content or "" + raw = raw.strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.startswith("json"): + raw = raw[4:] + raw = raw.strip() + + try: + items = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"ShortAnswerGenerator: could not parse LLM response as JSON: {exc}" + ) from exc + + questions: list[Question] = [] + for item in items: + questions.append( + Question( + question=item["question"], + question_id=str(uuid.uuid4()), + topic=topic, + question_type="short_answer", + options=None, + answer=item["answer"], + difficulty=difficulty, + explanation=item.get("explanation"), + ) + ) + return questions diff --git a/ai-ml/quiz_generator/app/generators/true_false_generator.py b/ai-ml/quiz_generator/app/generators/true_false_generator.py new file mode 100644 index 0000000..f440604 --- /dev/null +++ b/ai-ml/quiz_generator/app/generators/true_false_generator.py @@ -0,0 +1,90 @@ +from __future__ import annotations +import json +import uuid + +from groq import Groq + +from quiz_generator.app.generators.base_generator import BaseGenerator +from quiz_generator.app.config import GROQ_API_KEY, GROQ_MODEL +from quiz_generator.app.models.question import Question + + +class TrueFalseGenerator(BaseGenerator): + """ + Generator responsible for creating True/False questions + from the provided text using the Groq API. + """ + + def __init__(self): + self.client = Groq(api_key=GROQ_API_KEY) + + def generate( + self, + text: str, + number_of_questions: int = 5, + difficulty: str = "medium", + topic: str = "", + ) -> list[Question]: + """ + Generate True/False questions using the Groq API, returning + structured Question objects (not raw text). + """ + prompt = f""" + Generate exactly {number_of_questions} True/False questions + from the following text. + + Rules: + - Each question must have only two possible answers: "True" or "False". + - Clearly mark the correct answer. + - Questions should cover different concepts from the text. + - Keep the difficulty at {difficulty} level. + + Respond with ONLY a JSON array, no other text, in this exact shape: + [ + {{ + "question": "...", + "answer": "True", + "explanation": "..." + }} + ] + + Text: + {text} + """ + + response = self.client.chat.completions.create( + model=GROQ_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.5, + ) + + raw = response.choices[0].message.content or "" + raw = raw.strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.startswith("json"): + raw = raw[4:] + raw = raw.strip() + + try: + items = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"TrueFalseGenerator: could not parse LLM response as JSON: {exc}" + ) from exc + + questions: list[Question] = [] + for item in items: + questions.append( + Question( + question=item["question"], + question_id=str(uuid.uuid4()), + topic=topic, + question_type="true_false", + options=["True", "False"], + answer=item["answer"], + difficulty=difficulty, + explanation=item.get("explanation"), + ) + ) + return questions diff --git a/ai-ml/quiz_generator/app/main.py b/ai-ml/quiz_generator/app/main.py new file mode 100644 index 0000000..c40783a --- /dev/null +++ b/ai-ml/quiz_generator/app/main.py @@ -0,0 +1,107 @@ +""" +Team Lambda Quiz API — FastAPI service. + +Run from ai-ml/ (so the quiz_generator.* and embedding.* imports resolve): + uvicorn quiz_generator.app.main:app --reload --port 8002 + +Interactive docs: http://127.0.0.1:8002/docs +""" +from __future__ import annotations + +from fastapi import FastAPI, HTTPException, Depends +from fastapi.middleware.cors import CORSMiddleware +from embedding.chroma_store import delete_chunks + +from quiz_generator.app.models.api_models import GenerateQuizRequest, GenerateQuizResponse +from quiz_generator.app.services.quiz_service import QuizService +from quiz_generator.app.auth import get_current_user_id + +app = FastAPI(title="StudyMind Quiz API — Team Lambda") + +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:5173"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +_service: QuizService | None = None + + +def get_service() -> QuizService: + global _service + if _service is None: + _service = QuizService() + return _service + + +@app.get("/health") +def health_check(): + return {"status": "ok"} + + +@app.post("/generate-quiz", response_model=GenerateQuizResponse) +def generate_quiz_endpoint( + body: GenerateQuizRequest, + user_id: str = Depends(get_current_user_id), +) -> GenerateQuizResponse: + """ + [Contract v1, Section 10] Requires "Authorization: Bearer ". + user_id is derived from the verified token (never from client + input) and used to scope retrieval to this user's own content only. + """ + service = get_service() + + valid_types = {"mcq", "true_false", "fill_blank", "short_answer"} + if body.quiz_type not in valid_types: + raise HTTPException( + status_code=400, + detail=f"quiz_type must be one of: {', '.join(sorted(valid_types))}", + ) + + try: + result = service.generate_quiz_from_topic( + topic=body.topic, + question_type=body.quiz_type, + user_id=user_id, + number_of_questions=body.question_count, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=502, detail=str(exc)) from exc + + if "error" in result: + return GenerateQuizResponse( + success=False, + message=result["error"], + questions=[], + answers=[], + ) + + return GenerateQuizResponse( + success=True, + message=f"Generated {len(result['questions'])} questions.", + questions=result["questions"], + answers=result["answers"], + ) + +@app.delete("/document/{document_id}") +def delete_document_endpoint( + document_id: str, + user_id: str = Depends(get_current_user_id), +) -> dict: + """ + [P0-5] Purge all chunks for a document from the shared ChromaDB + store. Idempotent — calling this on an already-deleted or + nonexistent document_id is a safe no-op, not an error. + + Requires a valid JWT (Contract v1 Section 4/10) — but note the + authenticated user_id is not currently cross-checked against the + document's own user_id metadata before deleting. That ownership + check should happen wherever documents/uploads are tracked + (Pluto's side) before calling this endpoint. + """ + delete_chunks(document_id) + return {"success": True, "message": f"Document {document_id} purged."} \ No newline at end of file diff --git a/ai-ml/quiz_generator/app/models/api_models.py b/ai-ml/quiz_generator/app/models/api_models.py new file mode 100644 index 0000000..dc2db0c --- /dev/null +++ b/ai-ml/quiz_generator/app/models/api_models.py @@ -0,0 +1,37 @@ +from __future__ import annotations +from pydantic import BaseModel, Field + + +class GenerateQuizRequest(BaseModel): + """ + Request body for POST /generate-quiz. + + No user_id field here — per Contract v1 Section 2, identity is + never trusted from client input. The authenticated user_id comes + from the verified JWT (see auth.py's get_current_user_id). + """ + + topic: str = Field(..., min_length=1, description="Topic to generate the quiz from.") + question_count: int = Field( + default=5, ge=1, le=20, description="Number of questions to generate." + ) + quiz_type: str = Field( + ..., + description="One of: mcq, true_false, fill_blank, short_answer.", + ) + + +class GenerateQuizResponse(BaseModel): + """ + Response body for POST /generate-quiz. + + Per Contract v1 Section 10, the existing question/answer split + stays: "questions" never includes correct answers; "answers" is + a separate list matched by question_id for the caller (Pluto) to + store and grade against. This matches docs/api-contracts.md. + """ + + success: bool + message: str + questions: list[dict] = Field(default_factory=list) + answers: list[dict] = Field(default_factory=list) \ No newline at end of file diff --git a/ai-ml/quiz_generator/app/models/question.py b/ai-ml/quiz_generator/app/models/question.py new file mode 100644 index 0000000..ca693a0 --- /dev/null +++ b/ai-ml/quiz_generator/app/models/question.py @@ -0,0 +1,51 @@ +from __future__ import annotations +from typing import List, Optional + +from pydantic import BaseModel, Field + + +class Question(BaseModel): + """ + Represents a single generated quiz question. + """ + + question: str = Field( + ..., + description="The question statement." + ) + + question_id: str = Field( + ..., + description="Unique ID linking this question to its answer." + ) + + topic: str = Field( + default="", + description="Topic this question was generated from." + ) + + question_type: str = Field( + ..., + description="Type of question (mcq, true_false, fill_blank, short_answer)." + ) + + options: Optional[List[str]] = Field( + default=None, + description="Answer options for MCQ questions." + ) + + answer: str = Field( + ..., + description="Correct answer." + ) + + difficulty: str = Field( + default="medium", + description="Difficulty level (easy, medium, hard)." + ) + + explanation: Optional[str] = Field( + default=None, + description="Optional explanation of the correct answer." + ) + diff --git a/ai-ml/quiz_generator/app/models/request.py b/ai-ml/quiz_generator/app/models/request.py new file mode 100644 index 0000000..9060f5d --- /dev/null +++ b/ai-ml/quiz_generator/app/models/request.py @@ -0,0 +1,29 @@ +from pydantic import BaseModel, Field + + +class QuizRequest(BaseModel): + """ + Represents the input required to generate a quiz. + """ + + text: str = Field( + ..., + description="Input text from which questions will be generated." + ) + + question_type: str = Field( + ..., + description="Type of questions to generate." + ) + + number_of_questions: int = Field( + default=10, + ge=1, + le=50, + description="Number of questions to generate." + ) + + difficulty: str = Field( + default="medium", + description="Difficulty level (easy, medium, hard)." + ) \ No newline at end of file diff --git a/ai-ml/quiz_generator/app/models/response.py b/ai-ml/quiz_generator/app/models/response.py new file mode 100644 index 0000000..260ae0a --- /dev/null +++ b/ai-ml/quiz_generator/app/models/response.py @@ -0,0 +1,26 @@ +from typing import List + +from pydantic import BaseModel, Field + +from app.models.question import Question + + +class QuizResponse(BaseModel): + """ + Represents the response returned after quiz generation. + """ + + success: bool = Field( + ..., + description="Indicates whether quiz generation was successful." + ) + + message: str = Field( + ..., + description="Status message." + ) + + questions: List[Question] = Field( + default_factory=list, + description="List of generated quiz questions." + ) \ No newline at end of file diff --git a/ai-ml/quiz_generator/app/services/quiz_service.py b/ai-ml/quiz_generator/app/services/quiz_service.py new file mode 100644 index 0000000..c4cc5ed --- /dev/null +++ b/ai-ml/quiz_generator/app/services/quiz_service.py @@ -0,0 +1,139 @@ +from __future__ import annotations +# 1. Import the specific AI Generators +from quiz_generator.app.generators.mcq_generator import MCQGenerator +from quiz_generator.app.generators.true_false_generator import TrueFalseGenerator +from quiz_generator.app.generators.fill_blank_generator import FillBlankGenerator +from quiz_generator.app.generators.short_answer_generator import ShortAnswerGenerator + +# 2. THE BRIDGE: Import the Embedder from your sibling folder +from embedding.embedder import Embedder + + +class QuizService: + """ + The Bridge Service: + Links the Vector Store (Memory) with the AI Generators (Logic). + """ + + def __init__(self): + # [P0-1 fix] Embedder now searches the shared ChromaDB store — + # the same one ingestion writes to — instead of the old, + # disconnected Pinecone + MongoDB path. + self.embedder = Embedder() + + # Initialize the AI Generators + self.generators = { + "mcq": MCQGenerator(), + "true_false": TrueFalseGenerator(), + "fill_blank": FillBlankGenerator(), + "short_answer": ShortAnswerGenerator(), + } + + def generate_quiz_from_topic( + self, + topic: str, + question_type: str, + user_id: str, + number_of_questions: int = 5, + difficulty: str = "medium", + top_k: int = 3, + ) -> dict: + """ + RAG PIPELINE (The Bridge in action): + 1. Search: Finds relevant text chunks in the shared ChromaDB store, + scoped to the authenticated user (Contract v1 Section 10 — + quiz retrieval MUST be filtered by user_id). + 2. Generate: Feeds that specific text to the LLM to get structured questions. + + [Contract v1] Per Section 10, the existing question/answer split + stays as-is: this returns BOTH a "questions" list (no answers) + and an "answers" list (matched by question_id) — the shape + Pluto's proxy already expects and stores server-side for + grading. Do not strip "answers" from this return value; that + was an earlier draft fix that turned out to contradict the + signed-off contract. + + user_id is REQUIRED and must come from a verified JWT + (see auth.py) — never from client input — so the search below + can never cross into another user's content. + """ + # A. Search for context based on the user's topic, scoped to + # this user's own content only. + search_results = self.embedder.search(topic, top_k=top_k, user_id=user_id) + + if not search_results: + return { + "error": f"No relevant information found in your database for topic: '{topic}'" + } + + # B. Combine all found text chunks into one context paragraph + context_text = "\n\n".join([res["text"] for res in search_results]) + + # C. Generate structured questions using the found text + questions = self.generate_quiz( + context_text, + question_type, + number_of_questions=number_of_questions, + difficulty=difficulty, + topic=topic, + ) + + return self._split_questions_and_answers(questions) + + def generate_quiz( + self, + text: str, + question_type: str, + number_of_questions: int = 5, + difficulty: str = "medium", + topic: str = "", + ) -> list: + """ + Selects the correct generator and returns a list of structured + Question objects (not raw text, not a mixed dict). + """ + if question_type not in self.generators: + raise ValueError( + f"Unsupported question type: {question_type}. " + "Use: mcq, true_false, fill_blank, or short_answer" + ) + + return self.generators[question_type].generate( + text, + number_of_questions=number_of_questions, + difficulty=difficulty, + topic=topic, + ) + + @staticmethod + def _split_questions_and_answers(questions: list) -> dict: + """ + Splits a list of Question objects into two separate lists: + one safe to send to the frontend before submission (no answers), + and one kept server-side for grading. Per Contract v1 Section 10, + this split — and its presence in the /generate-quiz response — + is the intended, documented design; it is not being removed. + """ + public_questions = [] + answers = [] + + for q in questions: + public_questions.append( + { + "question_id": q.question_id, + "question": q.question, + "question_type": q.question_type, + "options": q.options, + "difficulty": q.difficulty, + "topic": q.topic, + } + ) + answers.append( + { + "question_id": q.question_id, + "answer": q.answer, + "explanation": q.explanation, + } + ) + + return {"questions": public_questions, "answers": answers} \ No newline at end of file diff --git a/ai-ml/quiz_generator/tests/test_fill_blank_generator.py b/ai-ml/quiz_generator/tests/test_fill_blank_generator.py new file mode 100644 index 0000000..3d46553 --- /dev/null +++ b/ai-ml/quiz_generator/tests/test_fill_blank_generator.py @@ -0,0 +1,20 @@ +from app.generators.fill_blank_generator import FillBlankGenerator + + +def main(): + sample_text = """ + Artificial Intelligence (AI) is a branch of computer science that enables + machines to simulate human intelligence. AI includes machine learning, + natural language processing, computer vision, and robotics. + """ + + generator = FillBlankGenerator() + + result = generator.generate(sample_text) + + print("\n========== Generated Fill in the Blank Questions ==========\n") + print(result) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ai-ml/quiz_generator/tests/test_mcq_generator.py b/ai-ml/quiz_generator/tests/test_mcq_generator.py new file mode 100644 index 0000000..cda79d6 --- /dev/null +++ b/ai-ml/quiz_generator/tests/test_mcq_generator.py @@ -0,0 +1,21 @@ +from app.generators.mcq_generator import MCQGenerator + + +def main(): + sample_text = """ + Artificial Intelligence (AI) is a branch of computer science that enables + machines to simulate human intelligence. AI includes machine learning, + natural language processing, computer vision, and robotics. + """ + + generator = MCQGenerator() + + result = generator.generate(sample_text) + + print("\nGenerated MCQs:\n") + print(result) + + +if __name__ == "__main__": + main() + \ No newline at end of file diff --git a/ai-ml/quiz_generator/tests/test_quiz_service.py b/ai-ml/quiz_generator/tests/test_quiz_service.py new file mode 100644 index 0000000..c8b656b --- /dev/null +++ b/ai-ml/quiz_generator/tests/test_quiz_service.py @@ -0,0 +1,23 @@ +from app.services.quiz_service import QuizService + + +def main(): + sample_text = """ + Artificial Intelligence (AI) is a branch of computer science that enables + machines to simulate human intelligence. AI includes machine learning, + natural language processing, computer vision, and robotics. + """ + + service = QuizService() + + result = service.generate_quiz( + sample_text, + "mcq" + ) + + print("\n========== Quiz Service Output ==========\n") + print(result) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ai-ml/quiz_generator/tests/test_short_answer_generator.py b/ai-ml/quiz_generator/tests/test_short_answer_generator.py new file mode 100644 index 0000000..b15220a --- /dev/null +++ b/ai-ml/quiz_generator/tests/test_short_answer_generator.py @@ -0,0 +1,20 @@ +from app.generators.short_answer_generator import ShortAnswerGenerator + + +def main(): + sample_text = """ + Artificial Intelligence (AI) is a branch of computer science that enables + machines to simulate human intelligence. AI includes machine learning, + natural language processing, computer vision, and robotics. + """ + + generator = ShortAnswerGenerator() + + result = generator.generate(sample_text) + + print("\n========== Generated Short Answer Questions ==========\n") + print(result) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ai-ml/quiz_generator/tests/test_true_false_generator.py b/ai-ml/quiz_generator/tests/test_true_false_generator.py new file mode 100644 index 0000000..5dc40ed --- /dev/null +++ b/ai-ml/quiz_generator/tests/test_true_false_generator.py @@ -0,0 +1,20 @@ +from app.generators.true_false_generator import TrueFalseGenerator + + +def main(): + sample_text = """ + Artificial Intelligence (AI) is a branch of computer science that enables + machines to simulate human intelligence. AI includes machine learning, + natural language processing, computer vision, and robotics. + """ + + generator = TrueFalseGenerator() + + result = generator.generate(sample_text) + + print("\n========== Generated True/False Questions ==========\n") + print(result) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ai-ml/requirements.txt b/ai-ml/requirements.txt index 7209079..bc380bd 100644 --- a/ai-ml/requirements.txt +++ b/ai-ml/requirements.txt @@ -7,4 +7,15 @@ fastapi # API framework uvicorn # ASGI server to run FastAPI python-multipart # Needed by FastAPI for file uploads (PDF endpoint) pydantic # Request body validation (already a FastAPI dependency) -yt_dlp # Download metadata/transcripts when needed \ No newline at end of file +yt_dlp # Download metadata/transcripts when needed +sentence-transformers # free, local embedding model (no API key, no per-call cost) +pinecone # Pinecone vector database client +chromadb +pymongo # MongoDB client +python-dotenv # loads .env config +requests +python-dotenv +groq +yake +pytest +pyjwt \ No newline at end of file diff --git a/ai-ml/roadmap_generator/README.md b/ai-ml/roadmap_generator/README.md new file mode 100644 index 0000000..fe2cbe5 --- /dev/null +++ b/ai-ml/roadmap_generator/README.md @@ -0,0 +1,93 @@ +# Roadmap Generator + +## Overview + +The Roadmap Generator is a module of the QuantumLearningWorkspace project (ai-ml). It generates a simple, ordered study roadmap from subject/topic input using the Groq API. + +This module is **general-purpose**: it generates a roadmap for a subject or set of topics, and does not depend on Weak-topic Detection or any other module's output. A caller may optionally supply per-topic `priority` hints (e.g. sourced from weak-topic results elsewhere) without this module importing or depending on that source. + +## Features + +- Ordered, sequential study roadmap generation +- Configurable step count +- Optional per-topic priority hints to influence ordering +- Service layer for integration with other code + +## Project Structure + +``` +roadmap_generator/ +│ +├── app/ +│ ├── api/ # reserved for API endpoints — not yet built, +│ │ pending cross-team contract confirmation +│ ├── generators/ +│ ├── models/ +│ ├── services/ +│ ├── utils/ +│ ├── validators/ +│ └── config.py +│ +├── tests/ +├── .gitignore +├── requirements.txt +└── README.md +``` + +## Installation + +From the `ai-ml/` directory: + +```bash +pip install -r roadmap_generator/requirements.txt +``` + +## Environment Variables + +Add to your `.env` (same one used by the other ai-ml modules): + +```env +GROQ_API_KEY=your_groq_api_key_here +``` + +## Running the Tests + +From `ai-ml/`: + +```bash +pytest roadmap_generator/tests +``` + +Tests that call the live Groq API are skipped automatically if `GROQ_API_KEY` isn't set. + +## Usage + +```python +from roadmap_generator.app.services.roadmap_service import RoadmapService + +service = RoadmapService() +roadmap = service.generate_roadmap( + topic_names=["Recursion", "Dynamic programming"], + subject="Algorithms", + step_count=6, +) +``` + +## Supported Inputs + +- `topic_names`: list of topic/subject name strings (required) +- `subject`: optional overall title for the roadmap +- `step_count`: number of steps to generate (3-15, default 6) +- `priorities`: optional `{topic_name: "high"|"normal"|"low"}` map + +## Technologies Used + +- Python 3.11 +- Groq API +- Pydantic +- python-dotenv +- pytest + +## Author + +Developed as part of the QuantumLearningWorkspace Internship Project — Team Lambda. diff --git a/ai-ml/roadmap_generator/app/config.py b/ai-ml/roadmap_generator/app/config.py new file mode 100644 index 0000000..409f274 --- /dev/null +++ b/ai-ml/roadmap_generator/app/config.py @@ -0,0 +1,26 @@ +import os +from dotenv import load_dotenv + +load_dotenv() + +# ========================================== +# Groq Configuration +# ========================================== + +GROQ_API_KEY = os.getenv("GROQ_API_KEY", "") + +GROQ_MODEL = "openai/gpt-oss-120b" +# NOTE: llama-3.3-70b-versatile (used elsewhere, e.g. quiz_generator's +# config.py) has been deprecated by Groq. openai/gpt-oss-120b is +# their current recommended general-purpose/reasoning replacement as +# of this writing. Worth flagging to the team — quiz_generator likely +# hits the same 404 right now. + + +# ========================================== +# Roadmap Configuration +# ========================================== + +DEFAULT_STEP_COUNT = 6 +MIN_STEP_COUNT = 3 +MAX_STEP_COUNT = 15 \ No newline at end of file diff --git a/ai-ml/roadmap_generator/app/generators/base_generator.py b/ai-ml/roadmap_generator/app/generators/base_generator.py new file mode 100644 index 0000000..23e1429 --- /dev/null +++ b/ai-ml/roadmap_generator/app/generators/base_generator.py @@ -0,0 +1,33 @@ +from abc import ABC, abstractmethod +from typing import List + +from roadmap_generator.app.models.topic import Topic +from roadmap_generator.app.models.roadmap import Roadmap + + +class BaseGenerator(ABC): + """ + Base class for roadmap generators. Mirrors quiz_generator's + BaseGenerator pattern so the two modules stay consistent in + style, even though they don't depend on each other. + """ + + @abstractmethod + def generate( + self, + topics: List[Topic], + subject: str = "", + step_count: int = 6, + ) -> Roadmap: + """ + Generate a study roadmap from the given topics. + + Parameters: + topics (List[Topic]): The subject(s)/topic(s) to build a roadmap for. + subject (str): Optional overall subject/title for the roadmap. + step_count (int): Target number of steps in the roadmap. + + Returns: + Roadmap: A structured, ordered study roadmap. + """ + pass diff --git a/ai-ml/roadmap_generator/app/generators/roadmap_generator.py b/ai-ml/roadmap_generator/app/generators/roadmap_generator.py new file mode 100644 index 0000000..572a1dd --- /dev/null +++ b/ai-ml/roadmap_generator/app/generators/roadmap_generator.py @@ -0,0 +1,103 @@ +import json + +from groq import Groq + +from roadmap_generator.app.generators.base_generator import BaseGenerator +from roadmap_generator.app.config import GROQ_API_KEY, GROQ_MODEL, DEFAULT_STEP_COUNT +from roadmap_generator.app.models.topic import Topic +from roadmap_generator.app.models.roadmap import Roadmap, RoadmapStep + + +class RoadmapGenerator(BaseGenerator): + """ + Generates a simple, ordered study roadmap for a subject or set of + topics using the Groq API. General-purpose: does not depend on + Weak-topic Detection or any other module's output. + """ + + def __init__(self): + self.client = Groq(api_key=GROQ_API_KEY) + + def generate( + self, + topics: list[Topic], + subject: str = "", + step_count: int = DEFAULT_STEP_COUNT, + ) -> Roadmap: + if not topics: + raise ValueError("At least one topic is required to generate a roadmap.") + + topic_lines = [] + for t in topics: + line = f"- {t.name}" + if t.description: + line += f" ({t.description})" + if t.priority: + line += f" [priority: {t.priority}]" + topic_lines.append(line) + topics_block = "\n".join(topic_lines) + + subject_label = subject or ", ".join(t.name for t in topics) + + prompt = f""" + Create a simple, ordered study roadmap for the following subject/topics. + + Subject: {subject_label} + + Topics to cover: + {topics_block} + + Rules: + - Produce exactly {step_count} sequential steps. + - Each step should build logically on the previous one (foundational + concepts before advanced ones). + - Keep each step's description concise and actionable (1-2 sentences). + - Give a rough estimated_duration for each step (e.g. "2-3 days"), + as a general suggestion, not a strict deadline. + - If a topic has a stated priority of "high", make sure it is covered + reasonably early in the roadmap, but the roadmap should still make + logical sense as a learning sequence. + + Respond with ONLY a JSON array, no other text, in this exact shape: + [ + {{ + "step_number": 1, + "topic": "...", + "description": "...", + "estimated_duration": "..." + }} + ] + """ + + response = self.client.chat.completions.create( + model=GROQ_MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.5, + ) + + raw = response.choices[0].message.content or "" + raw = raw.strip() + if raw.startswith("```"): + raw = raw.strip("`") + if raw.startswith("json"): + raw = raw[4:] + raw = raw.strip() + + try: + items = json.loads(raw) + except json.JSONDecodeError as exc: + raise RuntimeError( + f"RoadmapGenerator: could not parse LLM response as JSON: {exc}" + ) from exc + + steps = [ + RoadmapStep( + step_number=item.get("step_number", idx + 1), + topic=item["topic"], + description=item["description"], + estimated_duration=item.get("estimated_duration"), + ) + for idx, item in enumerate(items) + ] + + return Roadmap(subject=subject_label, steps=steps, total_steps=len(steps)) diff --git a/ai-ml/roadmap_generator/app/models/roadmap.py b/ai-ml/roadmap_generator/app/models/roadmap.py new file mode 100644 index 0000000..7a6a8be --- /dev/null +++ b/ai-ml/roadmap_generator/app/models/roadmap.py @@ -0,0 +1,32 @@ +from typing import List, Optional + +from pydantic import BaseModel, Field + + +class RoadmapStep(BaseModel): + """A single stage in a generated study roadmap.""" + + step_number: int = Field(..., description="Order of this step in the roadmap, starting at 1.") + + topic: str = Field(..., description="The topic covered in this step.") + + description: str = Field( + ..., description="What the learner should focus on or do during this step." + ) + + estimated_duration: Optional[str] = Field( + default=None, + description="Rough suggested time for this step, e.g. '2-3 days'. Optional — LLM-provided, not a guarantee.", + ) + + +class Roadmap(BaseModel): + """The full generated study roadmap for a subject or set of topics.""" + + subject: str = Field( + default="", description="Overall subject/title this roadmap covers, if provided." + ) + + steps: List[RoadmapStep] = Field(default_factory=list) + + total_steps: int = Field(default=0) diff --git a/ai-ml/roadmap_generator/app/models/topic.py b/ai-ml/roadmap_generator/app/models/topic.py new file mode 100644 index 0000000..aaf0deb --- /dev/null +++ b/ai-ml/roadmap_generator/app/models/topic.py @@ -0,0 +1,25 @@ +from typing import Optional + +from pydantic import BaseModel, Field + + +class Topic(BaseModel): + """ + Represents a single subject/topic used as input for roadmap + generation. Deliberately generic: this module does not depend on + Weak-topic Detection, but a caller (e.g. a future integration) + could populate `priority` from weak-topic results without this + module needing to know anything about that source. + """ + + name: str = Field(..., min_length=1, description="Topic or subject name.") + + description: Optional[str] = Field( + default=None, + description="Optional extra context about the topic to guide generation.", + ) + + priority: Optional[str] = Field( + default=None, + description="Optional hint: 'high', 'normal', 'low'. Not required.", + ) diff --git a/ai-ml/roadmap_generator/app/services/roadmap_service.py b/ai-ml/roadmap_generator/app/services/roadmap_service.py new file mode 100644 index 0000000..cb11f7b --- /dev/null +++ b/ai-ml/roadmap_generator/app/services/roadmap_service.py @@ -0,0 +1,46 @@ +from roadmap_generator.app.generators.roadmap_generator import RoadmapGenerator +from roadmap_generator.app.models.topic import Topic +from roadmap_generator.app.models.roadmap import Roadmap +from roadmap_generator.app.validators.topic_validator import ( + validate_topic_names, + validate_step_count, +) +from roadmap_generator.app.config import DEFAULT_STEP_COUNT + + +class RoadmapService: + """ + Coordinates the roadmap-generation workflow. This is the + integration point other code (a future API layer, or another + module) should call, rather than using RoadmapGenerator directly. + """ + + def __init__(self): + self.generator = RoadmapGenerator() + + def generate_roadmap( + self, + topic_names: list[str], + subject: str = "", + step_count: int = DEFAULT_STEP_COUNT, + priorities: dict[str, str] | None = None, + ) -> Roadmap: + """ + Generates a study roadmap from a plain list of topic names. + + priorities: optional {topic_name: "high"|"normal"|"low"} map. + This is how a caller (e.g. a future weak-topic integration) + could nudge topic ordering WITHOUT this module importing or + depending on that other module — the caller does the mapping, + this service just accepts plain data. + """ + validate_topic_names(topic_names) + validate_step_count(step_count) + + priorities = priorities or {} + topics = [ + Topic(name=name, priority=priorities.get(name)) + for name in topic_names + ] + + return self.generator.generate(topics, subject=subject, step_count=step_count) diff --git a/ai-ml/roadmap_generator/app/validators/topic_validator.py b/ai-ml/roadmap_generator/app/validators/topic_validator.py new file mode 100644 index 0000000..5fdc745 --- /dev/null +++ b/ai-ml/roadmap_generator/app/validators/topic_validator.py @@ -0,0 +1,23 @@ +from roadmap_generator.app.config import MIN_STEP_COUNT, MAX_STEP_COUNT + + +def validate_topic_names(topic_names: list[str]) -> None: + """ + Validates raw topic name input before it's wrapped into Topic + models. Raises ValueError on invalid input. + """ + if not topic_names: + raise ValueError("At least one topic name is required.") + + for name in topic_names: + if not name or not name.strip(): + raise ValueError("Topic names must not be empty or whitespace-only.") + + +def validate_step_count(step_count: int) -> None: + """Ensures the requested roadmap length is within a sane range.""" + if not (MIN_STEP_COUNT <= step_count <= MAX_STEP_COUNT): + raise ValueError( + f"step_count must be between {MIN_STEP_COUNT} and {MAX_STEP_COUNT} " + f"(got {step_count})." + ) diff --git a/ai-ml/roadmap_generator/requirements.txt b/ai-ml/roadmap_generator/requirements.txt new file mode 100644 index 0000000..9018345 --- /dev/null +++ b/ai-ml/roadmap_generator/requirements.txt @@ -0,0 +1,4 @@ +groq +pydantic +python-dotenv +pytest diff --git a/ai-ml/roadmap_generator/tests/conftest.py b/ai-ml/roadmap_generator/tests/conftest.py new file mode 100644 index 0000000..57f1ec8 --- /dev/null +++ b/ai-ml/roadmap_generator/tests/conftest.py @@ -0,0 +1,12 @@ +""" +Ensures ai-ml/ (two levels up from this tests/ folder) is on sys.path, +so `import roadmap_generator.app...` resolves regardless of the +working directory pytest is invoked from. Same fix as ai-ml/tests/conftest.py. +""" + +import sys +from pathlib import Path + +AI_ML_ROOT = Path(__file__).resolve().parent.parent.parent +if str(AI_ML_ROOT) not in sys.path: + sys.path.insert(0, str(AI_ML_ROOT)) \ No newline at end of file diff --git a/ai-ml/roadmap_generator/tests/test_roadmap_generator.py b/ai-ml/roadmap_generator/tests/test_roadmap_generator.py new file mode 100644 index 0000000..e151045 --- /dev/null +++ b/ai-ml/roadmap_generator/tests/test_roadmap_generator.py @@ -0,0 +1,43 @@ +import pytest + +from roadmap_generator.app.config import GROQ_API_KEY +from roadmap_generator.app.generators.roadmap_generator import RoadmapGenerator +from roadmap_generator.app.models.topic import Topic +from roadmap_generator.app.models.roadmap import Roadmap + +requires_groq = pytest.mark.skipif( + not GROQ_API_KEY, + reason="GROQ_API_KEY not set — skipping tests that call the live Groq API.", +) + + +@requires_groq +def test_generate_returns_roadmap_with_requested_step_count(): + generator = RoadmapGenerator() + topics = [Topic(name="Photosynthesis"), Topic(name="Cellular respiration")] + + roadmap = generator.generate(topics, subject="Biology basics", step_count=4) + + assert isinstance(roadmap, Roadmap) + assert roadmap.total_steps == 4 + assert len(roadmap.steps) == 4 + assert all(step.topic for step in roadmap.steps) + assert all(step.description for step in roadmap.steps) + + +@requires_groq +def test_generate_steps_are_sequentially_numbered(): + generator = RoadmapGenerator() + topics = [Topic(name="Linear algebra")] + + roadmap = generator.generate(topics, subject="Linear algebra", step_count=3) + + step_numbers = [s.step_number for s in roadmap.steps] + assert step_numbers == sorted(step_numbers) + + +def test_generate_raises_on_empty_topics(): + generator = RoadmapGenerator() + + with pytest.raises(ValueError): + generator.generate([], subject="Nothing") diff --git a/ai-ml/roadmap_generator/tests/test_roadmap_service.py b/ai-ml/roadmap_generator/tests/test_roadmap_service.py new file mode 100644 index 0000000..93f517a --- /dev/null +++ b/ai-ml/roadmap_generator/tests/test_roadmap_service.py @@ -0,0 +1,48 @@ +import pytest + +from roadmap_generator.app.config import GROQ_API_KEY +from roadmap_generator.app.services.roadmap_service import RoadmapService + +requires_groq = pytest.mark.skipif( + not GROQ_API_KEY, + reason="GROQ_API_KEY not set — skipping tests that call the live Groq API.", +) + + +def test_generate_roadmap_rejects_empty_topic_list(): + service = RoadmapService() + + with pytest.raises(ValueError): + service.generate_roadmap([]) + + +def test_generate_roadmap_rejects_blank_topic_name(): + service = RoadmapService() + + with pytest.raises(ValueError): + service.generate_roadmap(["", " "]) + + +def test_generate_roadmap_rejects_out_of_range_step_count(): + service = RoadmapService() + + with pytest.raises(ValueError): + service.generate_roadmap(["Algebra"], step_count=1) # below MIN_STEP_COUNT + + with pytest.raises(ValueError): + service.generate_roadmap(["Algebra"], step_count=100) # above MAX_STEP_COUNT + + +@requires_groq +def test_generate_roadmap_end_to_end(): + service = RoadmapService() + + roadmap = service.generate_roadmap( + ["Recursion", "Dynamic programming"], + subject="Algorithms", + step_count=4, + priorities={"Recursion": "high"}, + ) + + assert roadmap.subject == "Algorithms" + assert roadmap.total_steps == 4 diff --git a/ai-ml/run_quiz.py b/ai-ml/run_quiz.py new file mode 100644 index 0000000..af09420 --- /dev/null +++ b/ai-ml/run_quiz.py @@ -0,0 +1,37 @@ +import json +import os +import sys + +# Ensure Python looks in the current directory +sys.path.append(os.getcwd()) + +from quiz_generator.app.services.quiz_service import QuizService + +def main(): + service = QuizService() + + print("\n--- AI Quiz Generator ---") + + # 1. Ask the user for the topic + topic = input("Enter the topic you want a quiz on: ") + + # 2. Ask for the question type + print("\nAvailable types: mcq, true_false, fill_blank, short_answer") + q_type = input("Enter question type: ").lower() + + print(f"\nSearching database for '{topic}' and generating questions...") + + try: + quiz = service.generate_quiz_from_topic(topic, q_type) + + if isinstance(quiz, dict) and "error" in quiz: + print(f"\nResult: {quiz['error']}") + else: + print("\nGenerated Quiz:") + print(json.dumps(quiz, indent=2)) + + except Exception as e: + print(f"\nAn error occurred: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ai-ml/scripts/check_users.py b/ai-ml/scripts/check_users.py new file mode 100644 index 0000000..36b2b41 --- /dev/null +++ b/ai-ml/scripts/check_users.py @@ -0,0 +1,13 @@ +import chromadb + +client = chromadb.PersistentClient(path=r"D:\Dev\QuantumLearningWorkspace\shared_chroma_data") +col = client.get_or_create_collection("study_chunks") + +result = col.get(limit=50) +user_ids = set() +for metadata in result["metadatas"]: + user_ids.add(metadata.get("user_id", "UNKNOWN")) + +print("Users found in collection:") +for uid in user_ids: + print(f" - {uid}") \ No newline at end of file diff --git a/ai-ml/tests/conftest.py b/ai-ml/tests/conftest.py new file mode 100644 index 0000000..386e7f6 --- /dev/null +++ b/ai-ml/tests/conftest.py @@ -0,0 +1,14 @@ +""" +Ensures ai-ml/ (the parent of this tests/ folder) is on sys.path, so +`import embedding.chroma_store`, `import ingestion...`, etc. resolve +the same way regardless of the working directory pytest is invoked +from. This is what main.py's docstrings assume ("Run from ai-ml/") +but wasn't guaranteed for test collection before this. +""" + +import sys +from pathlib import Path + +AI_ML_ROOT = Path(__file__).resolve().parent.parent +if str(AI_ML_ROOT) not in sys.path: + sys.path.insert(0, str(AI_ML_ROOT)) diff --git a/ai-ml/tests/test_isolation_lifecycle.py b/ai-ml/tests/test_isolation_lifecycle.py new file mode 100644 index 0000000..d6049f8 --- /dev/null +++ b/ai-ml/tests/test_isolation_lifecycle.py @@ -0,0 +1,165 @@ +""" +[P1-5] Cross-module isolation and lifecycle tests for the shared +content store (embedding/chroma_store.py) — the store P0-1 unified +ingestion and quiz retrieval onto. + +These test the storage layer directly rather than through an HTTP +endpoint, since P0-5's purge endpoint isn't built yet. The +underlying function it will eventually wrap — delete_chunks() — was +already added as part of P0-1, and that's what "purge" is tested +against here. If/when P0-5's endpoint exists, it should just be a +thin HTTP wrapper around this same function, so these tests remain +valid underneath it. + +No GROQ_API_KEY or any other secret is needed — nothing here touches +quiz generation or the LLM, only storage. +""" + +import pytest + +from embedding import chroma_store + + +@pytest.fixture +def tmp_chroma(tmp_path, monkeypatch): + """ + Points chroma_store at a fresh, empty temp directory for the + duration of one test, so tests never touch the real + shared_chroma_data/ and never see each other's data. + """ + monkeypatch.setattr(chroma_store, "DEFAULT_CHROMA_PATH", str(tmp_path)) + return tmp_path + + +def _chunks(*texts): + return [{"chunk_index": i, "text": t} for i, t in enumerate(texts)] + + +# --------------------------------------------------------------- +# User isolation +# --------------------------------------------------------------- + +def test_user_a_cannot_see_user_b_content(tmp_chroma): + chroma_store.store_chunks( + _chunks("Photosynthesis occurs in the chloroplast."), + user_id="userA", document_id="docA", title="Bio Notes A", + ) + chroma_store.store_chunks( + _chunks("Mitochondria is the powerhouse of the cell."), + user_id="userB", document_id="docB", title="Bio Notes B", + ) + + results_a = chroma_store.query_chunks("cell biology", top_k=5, user_id="userA") + results_b = chroma_store.query_chunks("cell biology", top_k=5, user_id="userB") + + assert len(results_a) > 0 + assert len(results_b) > 0 + assert all(r["metadata"]["user_id"] == "userA" for r in results_a) + assert all(r["metadata"]["user_id"] == "userB" for r in results_b) + assert not any(r["metadata"]["document_id"] == "docB" for r in results_a) + assert not any(r["metadata"]["document_id"] == "docA" for r in results_b) + + +def test_unscoped_search_sees_both_users_when_no_filter_given(tmp_chroma): + """ + Documents today's default: with no user_id passed, search is + unscoped. Enforcing the filter at the caller level (quiz_service.py + always passing an authenticated user_id) is P0-3's job, not this + function's — this test exists so a future accidental change to + that default doesn't slip by unnoticed. + """ + chroma_store.store_chunks(_chunks("Content from A"), user_id="userA", document_id="docA", title="A") + chroma_store.store_chunks(_chunks("Content from B"), user_id="userB", document_id="docB", title="B") + + results = chroma_store.query_chunks("content", top_k=10) + seen_users = {r["metadata"]["user_id"] for r in results} + + assert seen_users == {"userA", "userB"} + + +# --------------------------------------------------------------- +# Document filtering +# --------------------------------------------------------------- + +def test_document_id_filter_returns_only_that_document(tmp_chroma): + chroma_store.store_chunks( + _chunks("First document content about volcanoes."), + user_id="userA", document_id="doc1", title="Doc 1", + ) + chroma_store.store_chunks( + _chunks("Second document content about earthquakes."), + user_id="userA", document_id="doc2", title="Doc 2", + ) + + results = chroma_store.query_chunks("geology", top_k=10, document_id="doc1") + + assert len(results) > 0 + assert all(r["metadata"]["document_id"] == "doc1" for r in results) + + +# --------------------------------------------------------------- +# Duplicate ingest +# --------------------------------------------------------------- + +def test_duplicate_ingest_same_document_id_does_not_duplicate_chunks(tmp_chroma): + """ + Re-ingesting the same document_id (e.g. a user re-uploads the same + file) must upsert, not append — chunk ids are deterministic + (f"{document_id}_chunk{index}"), so this should never double-count. + """ + chunks = _chunks("Same content ingested twice.") + + chroma_store.store_chunks(chunks, user_id="userA", document_id="dup-doc", title="Dup") + chroma_store.store_chunks(chunks, user_id="userA", document_id="dup-doc", title="Dup") + + collection = chroma_store.get_collection() + stored = collection.get(where={"document_id": "dup-doc"}) + + assert len(stored["ids"]) == 1 + + +# --------------------------------------------------------------- +# Purge / lifecycle +# --------------------------------------------------------------- + +def test_purge_removes_all_chunks_for_document(tmp_chroma): + chroma_store.store_chunks( + _chunks("Chunk one.", "Chunk two."), + user_id="userA", document_id="to-delete", title="Delete Me", + ) + + before = chroma_store.query_chunks("chunk", top_k=10, document_id="to-delete") + assert len(before) == 2 + + chroma_store.delete_chunks("to-delete") + + after = chroma_store.query_chunks("chunk", top_k=10, document_id="to-delete") + assert len(after) == 0 + + +def test_purge_is_safe_to_call_twice(tmp_chroma): + """Matches P0-5's DoD language ('safe to call twice') at the + storage-function level, ahead of the HTTP endpoint existing.""" + chroma_store.store_chunks( + _chunks("Some content."), + user_id="userA", document_id="idempotent-doc", title="Idempotent", + ) + + chroma_store.delete_chunks("idempotent-doc") + chroma_store.delete_chunks("idempotent-doc") # must not raise + + after = chroma_store.query_chunks("content", top_k=10, document_id="idempotent-doc") + assert len(after) == 0 + + +def test_purge_only_affects_the_targeted_document(tmp_chroma): + chroma_store.store_chunks(_chunks("Keep me."), user_id="userA", document_id="keep", title="Keep") + chroma_store.store_chunks(_chunks("Delete me."), user_id="userA", document_id="delete", title="Delete") + + chroma_store.delete_chunks("delete") + + remaining = chroma_store.query_chunks("keep delete", top_k=10) + remaining_doc_ids = {r["metadata"]["document_id"] for r in remaining} + + assert "keep" in remaining_doc_ids + assert "delete" not in remaining_doc_ids diff --git a/ai-ml/weak_topic_detection/.gitignore b/ai-ml/weak_topic_detection/.gitignore new file mode 100644 index 0000000..ca87034 --- /dev/null +++ b/ai-ml/weak_topic_detection/.gitignore @@ -0,0 +1,11 @@ +__pycache__/ +*.py[cod] +*$py.class + +.venv/ +venv/ +env/ + +.env + +.pytest_cache/ \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/README.md b/ai-ml/weak_topic_detection/README.md new file mode 100644 index 0000000..908cf10 --- /dev/null +++ b/ai-ml/weak_topic_detection/README.md @@ -0,0 +1,34 @@ +# Weak Topic Detection + +A module that analyzes quiz results to identify topics where a learner is performing weakly. + +## Features + +- Loads quiz results from JSON data +- Calculates accuracy for each topic +- Identifies weak topics using an accuracy threshold +- Requires a minimum number of attempts before evaluating a topic +- Provides a service and API interface +- Validates quiz-result data +- Includes automated tests + +## Current Configuration + +- Weak topic threshold: 60% +- Minimum attempts required: 3 + +## Project Structure + +```text +weak_topic_detection/ +├── app/ +│ ├── api/ +│ ├── detectors/ +│ ├── models/ +│ ├── services/ +│ ├── utils/ +│ ├── validators/ +│ └── config.py +├── data/ +│ └── quiz_results.json +└── tests/ \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/app/api/weak_topic_api.py b/ai-ml/weak_topic_detection/app/api/weak_topic_api.py new file mode 100644 index 0000000..e6780e4 --- /dev/null +++ b/ai-ml/weak_topic_detection/app/api/weak_topic_api.py @@ -0,0 +1,12 @@ +from weak_topic_detection.app.services.weak_topic_service import WeakTopicService + + +class WeakTopicAPI: + """Interface for accessing weak-topic detection.""" + + def __init__(self): + self.service = WeakTopicService() + + def get_weak_topics(self): + """Return the weak topics detected from quiz results.""" + return self.service.get_weak_topics() diff --git a/ai-ml/weak_topic_detection/app/config.py b/ai-ml/weak_topic_detection/app/config.py new file mode 100644 index 0000000..0646d66 --- /dev/null +++ b/ai-ml/weak_topic_detection/app/config.py @@ -0,0 +1,2 @@ +WEAK_TOPIC_THRESHOLD = 0.60 +MIN_TOPIC_ATTEMPTS = 3 \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/app/detectors/weak_topic_detector.py b/ai-ml/weak_topic_detection/app/detectors/weak_topic_detector.py new file mode 100644 index 0000000..f8839ee --- /dev/null +++ b/ai-ml/weak_topic_detection/app/detectors/weak_topic_detector.py @@ -0,0 +1,57 @@ +from collections import defaultdict +from weak_topic_detection.app.models.quiz_result import QuizResult + + +class WeakTopicDetector: + """ + Detects weak topics based on quiz performance. + """ + + def __init__(self, weak_threshold: float = 0.60, min_attempts: int = 3): + self.weak_threshold = weak_threshold + self.min_attempts = min_attempts + + def detect(self, results: list[QuizResult]) -> list[dict]: + """ + Identify weak topics from quiz results. + + A topic is considered weak when: + - It has at least the minimum number of attempts. + - Its accuracy is below the weak-topic threshold. + """ + + topic_results = defaultdict(list) + + # Group quiz results by topic + for result in results: + topic_results[result.topic].append(result) + + weak_topics = [] + + # Calculate accuracy for each topic + for topic, topic_attempts in topic_results.items(): + + total_attempts = len(topic_attempts) + + # Ignore topics with insufficient attempts + if total_attempts < self.min_attempts: + continue + + correct_answers = sum( + result.is_correct for result in topic_attempts + ) + + accuracy = correct_answers / total_attempts + + # Identify weak topics + if accuracy < self.weak_threshold: + weak_topics.append({ + "topic": topic, + "accuracy": round(accuracy * 100, 2), + "attempts": total_attempts + }) + + # Weakest topics first + weak_topics.sort(key=lambda item: item["accuracy"]) + + return weak_topics diff --git a/ai-ml/weak_topic_detection/app/main.py b/ai-ml/weak_topic_detection/app/main.py new file mode 100644 index 0000000..09bbe67 --- /dev/null +++ b/ai-ml/weak_topic_detection/app/main.py @@ -0,0 +1,69 @@ +""" +Team Lambda Weak Topic Detection API — FastAPI service. + +Run from ai-ml/ (so weak_topic_detection.* and quiz_generator.* +imports resolve): + uvicorn weak_topic_detection.app.main:app --reload --port + +Interactive docs (once running): http://127.0.0.1:/docs +""" + +from fastapi import FastAPI, Depends +from fastapi.middleware.cors import CORSMiddleware + +# Reusing the JWT auth dependency already built for quiz_generator +# (Contract v1, Section 10) rather than duplicating it — same secret, +# same failure modes, one source of truth for how auth works. +from quiz_generator.app.auth import get_current_user_id + +from weak_topic_detection.app.services.weak_topic_service import WeakTopicService + +app = FastAPI(title="StudyMind Weak Topic Detection API — Team Lambda") + +app.add_middleware( + CORSMiddleware, + allow_origins=["http://localhost:5173"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +_service: WeakTopicService | None = None + + +def get_service() -> WeakTopicService: + global _service + if _service is None: + _service = WeakTopicService() + return _service + + +@app.get("/health") +def health_check(): + return {"status": "ok"} + + +@app.get("/weak-topics") +def get_weak_topics_endpoint(user_id: str = Depends(get_current_user_id)): + """ + Requires "Authorization: Bearer ". user_id is derived from + the verified token and returned alongside the results so the + caller can confirm whose data this is. + + IMPORTANT — current limitation: WeakTopicService reads from a + static demo dataset (data/quiz_results.json), not live per-user + quiz submissions. The detection logic itself works correctly, but + it does not yet filter by user_id — every caller currently sees + the same demo results. Auth is enforced here so the endpoint's + contract (who can call it) is correct now; per-user quiz-result + ingestion is separate follow-up work once real quiz submissions + are wired in from the frontend. + """ + service = get_service() + weak_topics = service.get_weak_topics() + + return { + "success": True, + "user_id": user_id, + "weak_topics": weak_topics, + } \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/app/models/quiz_result.py b/ai-ml/weak_topic_detection/app/models/quiz_result.py new file mode 100644 index 0000000..3688f32 --- /dev/null +++ b/ai-ml/weak_topic_detection/app/models/quiz_result.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + + +@dataclass +class QuizResult: + user_id: str + question_id: str + topic: str + selected_answer: str + correct_answer: str + is_correct: bool + date_taken: str \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/app/services/weak_topic_service.py b/ai-ml/weak_topic_detection/app/services/weak_topic_service.py new file mode 100644 index 0000000..6146f23 --- /dev/null +++ b/ai-ml/weak_topic_detection/app/services/weak_topic_service.py @@ -0,0 +1,37 @@ +from weak_topic_detection.app.detectors.weak_topic_detector import WeakTopicDetector +from weak_topic_detection.app.models.quiz_result import QuizResult +from weak_topic_detection.app.utils.data_loader import load_json_data +from weak_topic_detection.app.config import WEAK_TOPIC_THRESHOLD, MIN_TOPIC_ATTEMPTS + + +class WeakTopicService: + """ + Loads quiz results and uses WeakTopicDetector + to identify weak topics. + """ + + def __init__( + self, + data_file: str = "data/quiz_results.json", + weak_threshold: float = WEAK_TOPIC_THRESHOLD, +min_attempts: int = MIN_TOPIC_ATTEMPTS, + ): + self.data_file = data_file + self.detector = WeakTopicDetector( + weak_threshold=weak_threshold, + min_attempts=min_attempts, + ) + + def load_results(self) -> list[QuizResult]: + """Load quiz results from the JSON file.""" + + data = load_json_data(self.data_file) + + return [QuizResult(**item) for item in data] + + def get_weak_topics(self) -> list[dict]: + """Return weak topics detected from quiz results.""" + + results = self.load_results() + + return self.detector.detect(results) diff --git a/ai-ml/weak_topic_detection/app/utils/data_loader.py b/ai-ml/weak_topic_detection/app/utils/data_loader.py new file mode 100644 index 0000000..d544ca1 --- /dev/null +++ b/ai-ml/weak_topic_detection/app/utils/data_loader.py @@ -0,0 +1,11 @@ +import json +from pathlib import Path + + +def load_json_data(file_path: str) -> list[dict]: + """Load quiz-result data from a JSON file.""" + + path = Path(file_path) + + with path.open("r", encoding="utf-8") as file: + return json.load(file) \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/app/validators/quiz_result_validator.py b/ai-ml/weak_topic_detection/app/validators/quiz_result_validator.py new file mode 100644 index 0000000..e7b1fc4 --- /dev/null +++ b/ai-ml/weak_topic_detection/app/validators/quiz_result_validator.py @@ -0,0 +1,36 @@ +from weak_topic_detection.app.models.quiz_result import QuizResult + + +class QuizResultValidator: + """Validates quiz-result data before processing.""" + + REQUIRED_FIELDS = { + "user_id", + "question_id", + "topic", + "selected_answer", + "correct_answer", + "is_correct", + "date_taken", + } + + @classmethod + def validate(cls, result: QuizResult) -> bool: + """Return True when a quiz result contains valid required data.""" + + if not result.user_id: + return False + + if not result.question_id: + return False + + if not result.topic: + return False + + if not isinstance(result.is_correct, bool): + return False + + if not result.date_taken: + return False + + return True diff --git a/ai-ml/weak_topic_detection/commands.txt b/ai-ml/weak_topic_detection/commands.txt new file mode 100644 index 0000000..2775d36 --- /dev/null +++ b/ai-ml/weak_topic_detection/commands.txt @@ -0,0 +1,23 @@ +WEAK TOPIC DETECTION - COMMANDS +========================================== + +1. MAIN WEAK TOPIC DETECTION + +python -m tests.test_weak_topic_service + +2. WEAK TOPIC DETECTOR TEST + +python -m tests.test_weak_topic_detector + +3. QUIZ RESULT VALIDATION TEST + +python -m tests.test_quiz_result_validator + +4. API TEST + +python -c "from app.api.weak_topic_api import WeakTopicAPI; print(WeakTopicAPI().get_weak_topics())" + +5. RUN ALL PYTEST TESTS + +python -m pytest tests + diff --git a/ai-ml/weak_topic_detection/data/quiz_results.json b/ai-ml/weak_topic_detection/data/quiz_results.json new file mode 100644 index 0000000..d6be79d --- /dev/null +++ b/ai-ml/weak_topic_detection/data/quiz_results.json @@ -0,0 +1,276 @@ +[ + { + "user_id": "user_001", + "question_id": "ml_q001", + "topic": "Machine Learning", + "selected_answer": "Supervised Learning", + "correct_answer": "Unsupervised Learning", + "is_correct": false, + "date_taken": "2026-08-01" + }, + { + "user_id": "user_001", + "question_id": "ml_q002", + "topic": "Machine Learning", + "selected_answer": "Classification", + "correct_answer": "Classification", + "is_correct": true, + "date_taken": "2026-08-02" + }, + { + "user_id": "user_001", + "question_id": "ml_q003", + "topic": "Machine Learning", + "selected_answer": "Regression", + "correct_answer": "Clustering", + "is_correct": false, + "date_taken": "2026-08-03" + }, + { + "user_id": "user_001", + "question_id": "ml_q004", + "topic": "Machine Learning", + "selected_answer": "Decision Tree", + "correct_answer": "Decision Tree", + "is_correct": true, + "date_taken": "2026-08-04" + }, + { + "user_id": "user_001", + "question_id": "ml_q005", + "topic": "Machine Learning", + "selected_answer": "K-Means", + "correct_answer": "Linear Regression", + "is_correct": false, + "date_taken": "2026-08-05" + }, + { + "user_id": "user_001", + "question_id": "ml_q006", + "topic": "Machine Learning", + "selected_answer": "Training Data", + "correct_answer": "Training Data", + "is_correct": true, + "date_taken": "2026-08-06" + }, + + { + "user_id": "user_001", + "question_id": "dl_q001", + "topic": "Deep Learning", + "selected_answer": "Neural Network", + "correct_answer": "Neural Network", + "is_correct": true, + "date_taken": "2026-08-01" + }, + { + "user_id": "user_001", + "question_id": "dl_q002", + "topic": "Deep Learning", + "selected_answer": "CNN", + "correct_answer": "RNN", + "is_correct": false, + "date_taken": "2026-08-02" + }, + { + "user_id": "user_001", + "question_id": "dl_q003", + "topic": "Deep Learning", + "selected_answer": "Backpropagation", + "correct_answer": "Backpropagation", + "is_correct": true, + "date_taken": "2026-08-03" + }, + { + "user_id": "user_001", + "question_id": "dl_q004", + "topic": "Deep Learning", + "selected_answer": "Pooling", + "correct_answer": "Dropout", + "is_correct": false, + "date_taken": "2026-08-04" + }, + { + "user_id": "user_001", + "question_id": "dl_q005", + "topic": "Deep Learning", + "selected_answer": "Gradient Descent", + "correct_answer": "Gradient Descent", + "is_correct": true, + "date_taken": "2026-08-05" + }, + { + "user_id": "user_001", + "question_id": "dl_q006", + "topic": "Deep Learning", + "selected_answer": "Overfitting", + "correct_answer": "Regularization", + "is_correct": false, + "date_taken": "2026-08-06" + }, + + { + "user_id": "user_001", + "question_id": "nlp_q001", + "topic": "Natural Language Processing", + "selected_answer": "Tokenization", + "correct_answer": "Tokenization", + "is_correct": true, + "date_taken": "2026-08-01" + }, + { + "user_id": "user_001", + "question_id": "nlp_q002", + "topic": "Natural Language Processing", + "selected_answer": "Sentiment Analysis", + "correct_answer": "Sentiment Analysis", + "is_correct": true, + "date_taken": "2026-08-02" + }, + { + "user_id": "user_001", + "question_id": "nlp_q003", + "topic": "Natural Language Processing", + "selected_answer": "Stemming", + "correct_answer": "Stemming", + "is_correct": true, + "date_taken": "2026-08-03" + }, + { + "user_id": "user_001", + "question_id": "nlp_q004", + "topic": "Natural Language Processing", + "selected_answer": "Named Entity Recognition", + "correct_answer": "Named Entity Recognition", + "is_correct": true, + "date_taken": "2026-08-04" + }, + { + "user_id": "user_001", + "question_id": "nlp_q005", + "topic": "Natural Language Processing", + "selected_answer": "Machine Translation", + "correct_answer": "Machine Translation", + "is_correct": true, + "date_taken": "2026-08-05" + }, + { + "user_id": "user_001", + "question_id": "nlp_q006", + "topic": "Natural Language Processing", + "selected_answer": "Word Embeddings", + "correct_answer": "Text Classification", + "is_correct": false, + "date_taken": "2026-08-06" + }, + + { + "user_id": "user_001", + "question_id": "cv_q001", + "topic": "Computer Vision", + "selected_answer": "Image Classification", + "correct_answer": "Image Classification", + "is_correct": true, + "date_taken": "2026-08-01" + }, + { + "user_id": "user_001", + "question_id": "cv_q002", + "topic": "Computer Vision", + "selected_answer": "Object Detection", + "correct_answer": "Object Detection", + "is_correct": true, + "date_taken": "2026-08-02" + }, + { + "user_id": "user_001", + "question_id": "cv_q003", + "topic": "Computer Vision", + "selected_answer": "Image Segmentation", + "correct_answer": "Image Segmentation", + "is_correct": true, + "date_taken": "2026-08-03" + }, + { + "user_id": "user_001", + "question_id": "cv_q004", + "topic": "Computer Vision", + "selected_answer": "CNN", + "correct_answer": "CNN", + "is_correct": true, + "date_taken": "2026-08-04" + }, + { + "user_id": "user_001", + "question_id": "cv_q005", + "topic": "Computer Vision", + "selected_answer": "Edge Detection", + "correct_answer": "Edge Detection", + "is_correct": true, + "date_taken": "2026-08-05" + }, + { + "user_id": "user_001", + "question_id": "cv_q006", + "topic": "Computer Vision", + "selected_answer": "Object Detection", + "correct_answer": "Image Segmentation", + "is_correct": false, + "date_taken": "2026-08-06" + }, + + { + "user_id": "user_001", + "question_id": "py_q001", + "topic": "Python Programming", + "selected_answer": "List", + "correct_answer": "List", + "is_correct": true, + "date_taken": "2026-08-01" + }, + { + "user_id": "user_001", + "question_id": "py_q002", + "topic": "Python Programming", + "selected_answer": "Dictionary", + "correct_answer": "Dictionary", + "is_correct": true, + "date_taken": "2026-08-02" + }, + { + "user_id": "user_001", + "question_id": "py_q003", + "topic": "Python Programming", + "selected_answer": "for loop", + "correct_answer": "for loop", + "is_correct": true, + "date_taken": "2026-08-03" + }, + { + "user_id": "user_001", + "question_id": "py_q004", + "topic": "Python Programming", + "selected_answer": "Function", + "correct_answer": "Function", + "is_correct": true, + "date_taken": "2026-08-04" + }, + { + "user_id": "user_001", + "question_id": "py_q005", + "topic": "Python Programming", + "selected_answer": "Tuple", + "correct_answer": "Tuple", + "is_correct": true, + "date_taken": "2026-08-05" + }, + { + "user_id": "user_001", + "question_id": "py_q006", + "topic": "Python Programming", + "selected_answer": "Exception Handling", + "correct_answer": "Exception Handling", + "is_correct": true, + "date_taken": "2026-08-06" + } +] \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/requirements.txt b/ai-ml/weak_topic_detection/requirements.txt new file mode 100644 index 0000000..55b033e --- /dev/null +++ b/ai-ml/weak_topic_detection/requirements.txt @@ -0,0 +1 @@ +pytest \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/tests/test_quiz_result_validator.py b/ai-ml/weak_topic_detection/tests/test_quiz_result_validator.py new file mode 100644 index 0000000..b4b0246 --- /dev/null +++ b/ai-ml/weak_topic_detection/tests/test_quiz_result_validator.py @@ -0,0 +1,31 @@ +from app.models.quiz_result import QuizResult +from app.validators.quiz_result_validator import QuizResultValidator + + +def main(): + valid_result = QuizResult( + user_id="user_001", + question_id="q001", + topic="Machine Learning", + selected_answer="A", + correct_answer="B", + is_correct=False, + date_taken="2026-08-01", + ) + + invalid_result = QuizResult( + user_id="", + question_id="q002", + topic="Machine Learning", + selected_answer="A", + correct_answer="B", + is_correct=False, + date_taken="2026-08-01", + ) + + print("Valid result:", QuizResultValidator.validate(valid_result)) + print("Invalid result:", QuizResultValidator.validate(invalid_result)) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/tests/test_weak_topic_api.py b/ai-ml/weak_topic_detection/tests/test_weak_topic_api.py new file mode 100644 index 0000000..69db9b1 --- /dev/null +++ b/ai-ml/weak_topic_detection/tests/test_weak_topic_api.py @@ -0,0 +1,15 @@ +from app.api.weak_topic_api import WeakTopicAPI + + +def test_get_weak_topics(): + api = WeakTopicAPI() + + result = api.get_weak_topics() + + assert isinstance(result, list) + assert len(result) > 0 + + for topic in result: + assert "topic" in topic + assert "accuracy" in topic + assert "attempts" in topic \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/tests/test_weak_topic_detector.py b/ai-ml/weak_topic_detection/tests/test_weak_topic_detector.py new file mode 100644 index 0000000..8ccb1bd --- /dev/null +++ b/ai-ml/weak_topic_detection/tests/test_weak_topic_detector.py @@ -0,0 +1,75 @@ +from app.detectors.weak_topic_detector import WeakTopicDetector +from app.models.quiz_result import QuizResult + + +def main(): + results = [ + # 3 attempts — should be evaluated + QuizResult( + user_id="user_001", + question_id="q1", + topic="Machine Learning", + selected_answer="A", + correct_answer="B", + is_correct=False, + date_taken="2026-08-01", + ), + QuizResult( + user_id="user_001", + question_id="q2", + topic="Machine Learning", + selected_answer="B", + correct_answer="B", + is_correct=True, + date_taken="2026-08-02", + ), + QuizResult( + user_id="user_001", + question_id="q3", + topic="Machine Learning", + selected_answer="A", + correct_answer="B", + is_correct=False, + date_taken="2026-08-03", + ), + + # Only 2 attempts — should be ignored + QuizResult( + user_id="user_001", + question_id="q4", + topic="Python", + selected_answer="A", + correct_answer="B", + is_correct=False, + date_taken="2026-08-01", + ), + QuizResult( + user_id="user_001", + question_id="q5", + topic="Python", + selected_answer="A", + correct_answer="B", + is_correct=False, + date_taken="2026-08-02", + ), + ] + + detector = WeakTopicDetector( + weak_threshold=0.60, + min_attempts=3, + ) + + weak_topics = detector.detect(results) + + print("\n========== Minimum Attempt Rule Test ==========\n") + + for topic in weak_topics: + print( + f"{topic['topic']} - " + f"Accuracy: {topic['accuracy']}% - " + f"Attempts: {topic['attempts']}" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/ai-ml/weak_topic_detection/tests/test_weak_topic_service.py b/ai-ml/weak_topic_detection/tests/test_weak_topic_service.py new file mode 100644 index 0000000..cb548ce --- /dev/null +++ b/ai-ml/weak_topic_detection/tests/test_weak_topic_service.py @@ -0,0 +1,24 @@ +from app.services.weak_topic_service import WeakTopicService + + +def main(): + service = WeakTopicService() + + weak_topics = service.get_weak_topics() + + print("\n========== Weak Topic Detection Output ==========\n") + + if not weak_topics: + print("No weak topics detected.") + return + + for index, topic in enumerate(weak_topics, start=1): + print( + f"{index}. {topic['topic']} - " + f"Accuracy: {topic['accuracy']}% - " + f"Attempts: {topic['attempts']}" + ) + + +if __name__ == "__main__": + main() \ No newline at end of file From bf503d11c89da00d72b90436e4f57887448486ba Mon Sep 17 00:00:00 2001 From: FarwaK05 Date: Wed, 9 Sep 2026 00:11:27 +0500 Subject: [PATCH 2/3] Update ingestion scripts and add exception handling --- .vscode/settings.json | 29 +++--- ai-ml/ingestion/common/exceptions.py | 11 +++ ai-ml/ingestion/main.py | 21 ++-- ai-ml/ingestion/pdf/extractor.py | 82 +++++----------- ai-ml/ingestion/youtube/exceptions.py | 70 +++++++++++++ ai-ml/ingestion/youtube/transcript.py | 136 +++++++++++++++++++++----- ai-ml/requirements.txt | 5 +- 7 files changed, 255 insertions(+), 99 deletions(-) create mode 100644 ai-ml/ingestion/common/exceptions.py create mode 100644 ai-ml/ingestion/youtube/exceptions.py diff --git a/.vscode/settings.json b/.vscode/settings.json index 419eb30..eb87dcb 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,12 +1,17 @@ -{ - "python.defaultInterpreterPath": "${workspaceFolder}/web/backend/.venv/Scripts/python.exe", - "python.analysis.extraPaths": [ - "${workspaceFolder}/web/backend", - "${workspaceFolder}/web/backend/routes" - ], - "python.autoComplete.extraPaths": [ - "${workspaceFolder}/web/backend", - "${workspaceFolder}/web/backend/routes" - ], - "python.analysis.autoSearchPaths": true -} + { + "python.defaultInterpreterPath": "${workspaceFolder}/web/backend/.venv/Scripts/python.exe", + + "python.analysis.extraPaths": [ + "${workspaceFolder}/web/backend", + "${workspaceFolder}/web/backend/routes", + "${workspaceFolder}/ai-ml" + ], + + "python.autoComplete.extraPaths": [ + "${workspaceFolder}/web/backend", + "${workspaceFolder}/web/backend/routes", + "${workspaceFolder}/ai-ml" + ], + + "python.analysis.autoSearchPaths": true +} \ No newline at end of file diff --git a/ai-ml/ingestion/common/exceptions.py b/ai-ml/ingestion/common/exceptions.py new file mode 100644 index 0000000..0cd4c9f --- /dev/null +++ b/ai-ml/ingestion/common/exceptions.py @@ -0,0 +1,11 @@ +class IngestionError(Exception): + """Base class for all ingestion errors.""" + pass + +class PDFProcessingError(IngestionError): + """Raised when PDF extraction or OCR fails.""" + pass + +class YouTubeTranscriptError(IngestionError): + """Raised when YouTube transcripts are unavailable.""" + pass \ No newline at end of file diff --git a/ai-ml/ingestion/main.py b/ai-ml/ingestion/main.py index 0ffb87e..d330e8c 100644 --- a/ai-ml/ingestion/main.py +++ b/ai-ml/ingestion/main.py @@ -71,9 +71,10 @@ def _chunk_and_store(result: dict, user_id: str) -> dict: @app.post("/ingest/pdf") async def ingest_pdf_endpoint( file: UploadFile = File(...), - user_id: str = Depends(get_current_user_id), + # user_id: str = Depends(get_current_user_id), + user_id="test_user", ): - if not file.filename.lower().endswith(".pdf"): + if not file.filename or not file.filename.lower().endswith(".pdf"): raise HTTPException(status_code=400, detail="File must be a PDF") with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp: @@ -81,7 +82,10 @@ async def ingest_pdf_endpoint( tmp_path = tmp.name try: - result = ingest_pdf(file_path=tmp_path, original_filename=file.filename) + result = ingest_pdf( + file_path=tmp_path, + original_filename=file.filename + ) storage_info = _chunk_and_store(result, user_id) except Exception as e: raise HTTPException(status_code=500, detail=str(e)) @@ -90,21 +94,24 @@ async def ingest_pdf_endpoint( os.remove(tmp_path) return {**result, **storage_info} - - # ----------------------------- # YOUTUBE INGESTION # ----------------------------- @app.post("/ingest/youtube") async def ingest_youtube_endpoint( payload: URLRequest, - user_id: str = Depends(get_current_user_id), + # user_id: str = Depends(get_current_user_id), + user_id = "test_user", ): try: result = ingest_youtube(payload.url) storage_info = _chunk_and_store(result, user_id) except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + # Task 3: If it's our custom error, give a 400. Otherwise, give a 500. + error_msg = str(e) + status_code = 400 if "YouTube" in error_msg else 500 + raise HTTPException(status_code=status_code, detail=error_msg) + return {**result, **storage_info} diff --git a/ai-ml/ingestion/pdf/extractor.py b/ai-ml/ingestion/pdf/extractor.py index 3674e51..59d6a84 100644 --- a/ai-ml/ingestion/pdf/extractor.py +++ b/ai-ml/ingestion/pdf/extractor.py @@ -1,74 +1,46 @@ import os -import fitz # PyMuPDF - +from docling.datamodel.base_models import InputFormat +from docling.document_converter import DocumentConverter from ingestion.pdf.cleaner import clean_pdf_text from ingestion.common.schema import build_result - +from ingestion.common.exceptions import PDFProcessingError def extract_pdf_text(file_path: str) -> str: """ - Open a PDF and extract raw text from every page. + Advanced extraction using IBM's Docling. + Handles OCR, tables, and layout automatically. """ - if not os.path.exists(file_path): - raise FileNotFoundError(f"PDF not found: {file_path}") - - doc = fitz.open(file_path) - - pages_text = [] - - for page in doc: - pages_text.append(page.get_text()) - - doc.close() - - return "\n".join(pages_text) - + raise PDFProcessingError(f"File not found: {file_path}") + + try: + # Initialize the converter + converter = DocumentConverter() + + # Convert the PDF (Docling automatically detects if it needs OCR) + result = converter.convert(file_path) + + # Export to Markdown (best for RAG) or Plain Text + # .export_to_markdown() is highly recommended for LLMs + extracted_text = result.document.export_to_markdown() + + if not extracted_text.strip(): + raise PDFProcessingError("Extraction resulted in empty text.") + + return extracted_text + + except Exception as e: + raise PDFProcessingError(f"Docling failed to process PDF: {str(e)}") def ingest_pdf(file_path: str, original_filename: str) -> dict: - """ - Full PDF ingestion pipeline: - - PDF file - ↓ - Text extraction - ↓ - Text cleaning - ↓ - Build standard output format - """ - - # Step 1: Extract text + """Standard pipeline logic.""" raw_text = extract_pdf_text(file_path) - - # Step 2: Clean extracted text cleaned_text = clean_pdf_text(raw_text) - - # Step 3: Generate user-friendly title title = os.path.splitext(original_filename)[0] - # Step 4: Return common JSON format return build_result( source_type="pdf", title=title, text=cleaned_text, source=file_path, - ) - - -if __name__ == "__main__": - import sys - import json - - if len(sys.argv) < 2: - print("Usage: python extractor.py ") - - else: - pdf_path = sys.argv[1] - - result = ingest_pdf( - file_path=pdf_path, - original_filename=os.path.basename(pdf_path) - ) - - print(json.dumps(result, indent=2)) \ No newline at end of file + ) \ No newline at end of file diff --git a/ai-ml/ingestion/youtube/exceptions.py b/ai-ml/ingestion/youtube/exceptions.py new file mode 100644 index 0000000..e5369af --- /dev/null +++ b/ai-ml/ingestion/youtube/exceptions.py @@ -0,0 +1,70 @@ +"""Structured errors for the YouTube ingestion pipeline. + +Every error carries a machine-readable `code`, an HTTP `status_code` the +API layer should return, and a human-readable `message`. This lets the +FastAPI layer turn these into clean JSON instead of a raw 500 traceback. +""" + + +class YouTubeIngestError(Exception): + """Base class for all YouTube ingestion errors.""" + + code = "youtube_ingest_error" + status_code = 500 + + def __init__( + self, + message: str, + video_id: str | None = None, + details: dict | None = None, + ): + super().__init__(message) + self.message = message + self.video_id = video_id + self.details = details or {} + + def to_dict(self) -> dict: + payload: dict[str, object] = { + "error": self.code, + "message": self.message, + } + + if self.video_id: + payload["video_id"] = self.video_id + + if self.details: + payload["details"] = self.details + + return payload + +class InvalidYouTubeURLError(YouTubeIngestError): + """The URL is not a recognizable YouTube video URL.""" + + code = "invalid_url" + status_code = 400 + + +class VideoUnavailableError(YouTubeIngestError): + """The video is private, deleted, region-locked, or otherwise unreachable.""" + + code = "video_unavailable" + status_code = 404 + + +class TranscriptNotAvailableError(YouTubeIngestError): + """The video exists but no usable transcript could be produced. + + This covers: transcripts disabled by the uploader, no transcript in any + language, a transcript that fetched but came back empty, and transcript + fetch failures after a listing succeeded. + """ + + code = "transcript_not_available" + status_code = 422 + + +class TranscriptFetchError(YouTubeIngestError): + """Transient/upstream failure (rate limiting, IP block, network, etc.).""" + + code = "transcript_fetch_failed" + status_code = 502 \ No newline at end of file diff --git a/ai-ml/ingestion/youtube/transcript.py b/ai-ml/ingestion/youtube/transcript.py index d3fee0f..1128f3d 100644 --- a/ai-ml/ingestion/youtube/transcript.py +++ b/ai-ml/ingestion/youtube/transcript.py @@ -1,16 +1,25 @@ import re +from typing import Any from urllib.parse import urlparse, parse_qs import yt_dlp +from yt_dlp.utils import DownloadError from youtube_transcript_api import YouTubeTranscriptApi from youtube_transcript_api._errors import ( TranscriptsDisabled, NoTranscriptFound, VideoUnavailable, + CouldNotRetrieveTranscript, ) from ingestion.youtube.cleaner import clean_youtube_text +from ingestion.youtube.exceptions import ( + InvalidYouTubeURLError, + VideoUnavailableError, + TranscriptNotAvailableError, + TranscriptFetchError, +) from ingestion.common.schema import build_result @@ -31,18 +40,25 @@ def extract_video_id(url: str) -> str: if match: return match.group(2) - raise ValueError(f"Could not extract video ID from URL: {url}") + raise InvalidYouTubeURLError(f"Could not extract a video ID from URL: {url}") def fetch_metadata(url: str) -> dict: - options = { + options: dict[str, Any] = { "quiet": True, "no_warnings": True, "skip_download": True, } - with yt_dlp.YoutubeDL(options) as ydl: - info = ydl.extract_info(url, download=False) + try: + with yt_dlp.YoutubeDL(options) as ydl: + info = ydl.extract_info(url, download=False) + except DownloadError as e: + raise VideoUnavailableError( + f"Could not load video metadata for {url}. It may be private, " + f"deleted, region-locked, or age-restricted.", + details={"reason": str(e)}, + ) from e return { "title": info.get("title", ""), @@ -52,27 +68,91 @@ def fetch_metadata(url: str) -> dict: } -def fetch_transcript(video_id: str, languages=("en",)) -> str: - try: - api = YouTubeTranscriptApi() - - transcript = api.fetch( - video_id, - languages=list(languages) - ) +def fetch_transcript(video_id: str, languages=("en",)) -> dict: + """ + Returns {"text": str, "language_code": str, "is_generated": bool}. - except (TranscriptsDisabled, NoTranscriptFound): - api = YouTubeTranscriptApi() - transcript = api.fetch(video_id) + Raises: + VideoUnavailableError: video is private/deleted/unreachable. + TranscriptNotAvailableError: video exists but has no usable transcript + (disabled by uploader, none in any language, or empty once fetched). + TranscriptFetchError: transient upstream failure (rate limit, IP block). + """ + api = YouTubeTranscriptApi() + # Step 1: find out what transcripts actually exist for this video. + try: + transcript_list = api.list(video_id) except VideoUnavailable as e: - raise RuntimeError(f"Video unavailable: {video_id}") from e + raise VideoUnavailableError( + f"The video {video_id} is unavailable, private, or has been removed.", + video_id=video_id, + ) from e + except TranscriptsDisabled as e: + raise TranscriptNotAvailableError( + f"The uploader has disabled transcripts/captions for video {video_id}.", + video_id=video_id, + details={"reason": "transcripts_disabled"}, + ) from e + except CouldNotRetrieveTranscript as e: + # Covers RequestBlocked/IpBlocked/PoTokenRequired/YouTubeRequestFailed etc. + raise TranscriptFetchError( + f"Could not check transcript availability for video {video_id}: {e}", + video_id=video_id, + ) from e + + available = [ + { + "language": t.language, + "language_code": t.language_code, + "is_generated": t.is_generated, + } + for t in transcript_list + ] + + if not available: + raise TranscriptNotAvailableError( + f"No transcript (manual or auto-generated) exists for video {video_id} " + f"in any language.", + video_id=video_id, + details={"available_languages": []}, + ) - merged = " ".join( - segment.text for segment in transcript - ) + # Step 2: prefer a manually created transcript in a requested language, + # then an auto-generated one in a requested language, then just take + # whatever exists (manually created first) rather than failing outright. + transcript = None + try: + transcript = transcript_list.find_transcript(list(languages)) + except NoTranscriptFound: + transcript = sorted(transcript_list, key=lambda t: t.is_generated)[0] - return merged + # Step 3: actually fetch it. + try: + fetched = transcript.fetch() + except CouldNotRetrieveTranscript as e: + raise TranscriptFetchError( + f"Found a transcript listing for video {video_id} " + f"(language={transcript.language_code}) but failed to fetch it: {e}", + video_id=video_id, + details={"available_languages": available}, + ) from e + + merged = " ".join(segment.text for segment in fetched).strip() + + if not merged: + raise TranscriptNotAvailableError( + f"Transcript for video {video_id} (language={transcript.language_code}) " + f"fetched successfully but contained no text.", + video_id=video_id, + details={"available_languages": available}, + ) + + return { + "text": merged, + "language_code": transcript.language_code, + "is_generated": transcript.is_generated, + } def ingest_youtube(url: str) -> dict: @@ -80,9 +160,9 @@ def ingest_youtube(url: str) -> dict: metadata = fetch_metadata(url) - raw_text = fetch_transcript(video_id) + transcript_data = fetch_transcript(video_id) - cleaned_text = clean_youtube_text(raw_text) + cleaned_text = clean_youtube_text(transcript_data["text"]) result = build_result( source_type="youtube", @@ -95,6 +175,8 @@ def ingest_youtube(url: str) -> dict: "author": metadata["author"], "duration": metadata["duration"], "date": metadata["date"], + "transcript_language": transcript_data["language_code"], + "transcript_auto_generated": transcript_data["is_generated"], }) return result @@ -104,8 +186,14 @@ def ingest_youtube(url: str) -> dict: import sys import json + from ingestion.youtube.exceptions import YouTubeIngestError + if len(sys.argv) < 2: print("Usage: python transcript.py ") else: - result = ingest_youtube(sys.argv[1]) - print(json.dumps(result, indent=2)) \ No newline at end of file + try: + result = ingest_youtube(sys.argv[1]) + print(json.dumps(result, indent=2)) + except YouTubeIngestError as e: + print(json.dumps(e.to_dict(), indent=2)) + sys.exit(1) \ No newline at end of file diff --git a/ai-ml/requirements.txt b/ai-ml/requirements.txt index bc380bd..37b8356 100644 --- a/ai-ml/requirements.txt +++ b/ai-ml/requirements.txt @@ -18,4 +18,7 @@ python-dotenv groq yake pytest -pyjwt \ No newline at end of file +PyJWT +docling +pytesseract +Pillow \ No newline at end of file From b79f5a7d08f6e2403a268521c8350b6bc9526997 Mon Sep 17 00:00:00 2001 From: FarwaK05 Date: Wed, 9 Sep 2026 08:35:59 +0500 Subject: [PATCH 3/3] Added Test case for the happy path , for Chunker and embedder --- ai-ml/ingestion/main.py | 8 ++--- ai-ml/ingestion/youtube/transcript.py | 5 ++- ai-ml/tests/test_chunker.py | 24 ++++++++++++++ ai-ml/tests/test_embedder.py | 45 +++++++++++++++++++++++++++ 4 files changed, 75 insertions(+), 7 deletions(-) create mode 100644 ai-ml/tests/test_chunker.py create mode 100644 ai-ml/tests/test_embedder.py diff --git a/ai-ml/ingestion/main.py b/ai-ml/ingestion/main.py index d330e8c..6cf672e 100644 --- a/ai-ml/ingestion/main.py +++ b/ai-ml/ingestion/main.py @@ -71,8 +71,8 @@ def _chunk_and_store(result: dict, user_id: str) -> dict: @app.post("/ingest/pdf") async def ingest_pdf_endpoint( file: UploadFile = File(...), - # user_id: str = Depends(get_current_user_id), - user_id="test_user", + user_id: str = Depends(get_current_user_id), + # user_id="test_user", ): if not file.filename or not file.filename.lower().endswith(".pdf"): raise HTTPException(status_code=400, detail="File must be a PDF") @@ -100,8 +100,8 @@ async def ingest_pdf_endpoint( @app.post("/ingest/youtube") async def ingest_youtube_endpoint( payload: URLRequest, - # user_id: str = Depends(get_current_user_id), - user_id = "test_user", + user_id: str = Depends(get_current_user_id), + # user_id = "test_user", ): try: result = ingest_youtube(payload.url) diff --git a/ai-ml/ingestion/youtube/transcript.py b/ai-ml/ingestion/youtube/transcript.py index 1128f3d..1b596b2 100644 --- a/ai-ml/ingestion/youtube/transcript.py +++ b/ai-ml/ingestion/youtube/transcript.py @@ -44,14 +44,14 @@ def extract_video_id(url: str) -> str: def fetch_metadata(url: str) -> dict: - options: dict[str, Any] = { + options = { "quiet": True, "no_warnings": True, "skip_download": True, } try: - with yt_dlp.YoutubeDL(options) as ydl: + with yt_dlp.YoutubeDL(options) as ydl: # type: ignore[arg-type] info = ydl.extract_info(url, download=False) except DownloadError as e: raise VideoUnavailableError( @@ -67,7 +67,6 @@ def fetch_metadata(url: str) -> dict: "date": info.get("upload_date"), } - def fetch_transcript(video_id: str, languages=("en",)) -> dict: """ Returns {"text": str, "language_code": str, "is_generated": bool}. diff --git a/ai-ml/tests/test_chunker.py b/ai-ml/tests/test_chunker.py new file mode 100644 index 0000000..01137e4 --- /dev/null +++ b/ai-ml/tests/test_chunker.py @@ -0,0 +1,24 @@ +import pytest +from embedding.chunker import chunk_document + +def test_chunk_document_happy_path(): + """Verify document is split into chunks with correct metadata.""" + doc = { + "source_type": "pdf", + "title": "Lab Report", + "text": "word1 " * 20, # 20 words + "metadata": {"author": "Farwa"} + } + + # Testing with small chunk size to force multiple chunks + chunks = chunk_document(doc, chunk_size=10, overlap=2) + + assert len(chunks) > 1 + assert chunks[0]["title"] == "Lab Report" + assert "chunk_index" in chunks[0] + assert chunks[0]["metadata"]["author"] == "Farwa" + +def test_chunk_document_empty_text(): + """Edge Case: Empty text should return empty list.""" + doc = {"text": "", "title": "Empty"} + assert chunk_document(doc) == [] \ No newline at end of file diff --git a/ai-ml/tests/test_embedder.py b/ai-ml/tests/test_embedder.py new file mode 100644 index 0000000..ca1df30 --- /dev/null +++ b/ai-ml/tests/test_embedder.py @@ -0,0 +1,45 @@ +import pytest +from unittest.mock import MagicMock, patch +from embedding.chroma_store import store_chunks + +def test_store_chunks_happy_path(): + """Task 4: Happy Path - Mock everything to avoid disk and AI model delays.""" + + # 1. Provide a chunk that matches your Chunker.py output exactly + mock_chunks = [{ + "text": "sample text", + "chunk_index": 0 + }] + + # 2. We mock the Client and the Model + # This prevents the code from touching C:\Dev\... and from loading AI weights + with patch("embedding.chroma_store.chromadb.PersistentClient") as mock_client_class, \ + patch("embedding.chroma_store.get_embedding_model") as mock_get_model: + + # Setup the database mock + mock_client = MagicMock() + mock_client_class.return_value = mock_client + mock_col = MagicMock() + mock_client.get_or_create_collection.return_value = mock_col + + # Setup the model mock + mock_model = MagicMock() + mock_get_model.return_value = mock_model + mock_model.encode.return_value = [[0.1, 0.2]] + + # 3. Call the function + result = store_chunks(mock_chunks, "user_1", "doc_1", "Test Title") + + # 4. Verify + assert result == 1 + # Your code uses upsert, so we check that + assert mock_col.upsert.called + + # Verify the ID was created correctly: {doc_id}_chunk{index} + args, kwargs = mock_col.upsert.call_args + assert kwargs['ids'][0] == "doc_1_chunk0" + +def test_store_chunks_empty_list(): + """Task 4: Edge Case - Empty list returns 0.""" + from embedding.chroma_store import store_chunks + assert store_chunks([], "user", "doc", "title") == 0 \ No newline at end of file