Lambda/ai ml

.vscode/settings.json

@@ -1,12 +1,17 @@
-{
-  "python.defaultInterpreterPath": "${workspaceFolder}/web/backend/.venv/Scripts/python.exe",
-  "python.analysis.extraPaths": [
-    "${workspaceFolder}/web/backend",
-    "${workspaceFolder}/web/backend/routes"
-  ],
-  "python.autoComplete.extraPaths": [
-    "${workspaceFolder}/web/backend",
-    "${workspaceFolder}/web/backend/routes"
-  ],
-  "python.analysis.autoSearchPaths": true
-}
+ {
+    "python.defaultInterpreterPath": "${workspaceFolder}/web/backend/.venv/Scripts/python.exe",
+
+    "python.analysis.extraPaths": [
+        "${workspaceFolder}/web/backend",
+        "${workspaceFolder}/web/backend/routes",
+        "${workspaceFolder}/ai-ml"
+    ],
+
+    "python.autoComplete.extraPaths": [
+        "${workspaceFolder}/web/backend",
+        "${workspaceFolder}/web/backend/routes",
+        "${workspaceFolder}/ai-ml"
+    ],
+
+    "python.analysis.autoSearchPaths": true
+}

ai-ml/ingestion/common/exceptions.py

@@ -0,0 +1,11 @@
+class IngestionError(Exception):
+    """Base class for all ingestion errors."""
+    pass
+
+class PDFProcessingError(IngestionError):
+    """Raised when PDF extraction or OCR fails."""
+    pass
+
+class YouTubeTranscriptError(IngestionError):
+    """Raised when YouTube transcripts are unavailable."""
+    pass

ai-ml/ingestion/main.py

@@ -72,16 +72,20 @@ def _chunk_and_store(result: dict, user_id: str) -> dict:
 async def ingest_pdf_endpoint(
     file: UploadFile = File(...),
     user_id: str = Depends(get_current_user_id),
+    # user_id="test_user",
 ):
-    if not file.filename.lower().endswith(".pdf"):
+    if not file.filename or not file.filename.lower().endswith(".pdf"):
         raise HTTPException(status_code=400, detail="File must be a PDF")
 
     with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as tmp:
         shutil.copyfileobj(file.file, tmp)
         tmp_path = tmp.name
 
     try:
-        result = ingest_pdf(file_path=tmp_path, original_filename=file.filename)
+        result = ingest_pdf(
+            file_path=tmp_path,
+            original_filename=file.filename
+        )
         storage_info = _chunk_and_store(result, user_id)
     except Exception as e:
         raise HTTPException(status_code=500, detail=str(e))
@@ -90,21 +94,24 @@ async def ingest_pdf_endpoint(
             os.remove(tmp_path)
 
     return {**result, **storage_info}
-
-
 # -----------------------------
 # YOUTUBE INGESTION
 # -----------------------------
 @app.post("/ingest/youtube")
 async def ingest_youtube_endpoint(
     payload: URLRequest,
-    user_id: str = Depends(get_current_user_id),
+   user_id: str = Depends(get_current_user_id),
+    # user_id = "test_user",
 ):
     try:
         result = ingest_youtube(payload.url)
         storage_info = _chunk_and_store(result, user_id)
     except Exception as e:
-        raise HTTPException(status_code=500, detail=str(e))
+        # Task 3: If it's our custom error, give a 400. Otherwise, give a 500.
+        error_msg = str(e)
+        status_code = 400 if "YouTube" in error_msg else 500
+        raise HTTPException(status_code=status_code, detail=error_msg)
+
     return {**result, **storage_info}
 
 

ai-ml/ingestion/pdf/extractor.py

@@ -1,74 +1,46 @@
 import os
-import fitz  # PyMuPDF
-
+from docling.datamodel.base_models import InputFormat
+from docling.document_converter import DocumentConverter
 from ingestion.pdf.cleaner import clean_pdf_text
 from ingestion.common.schema import build_result
-
+from ingestion.common.exceptions import PDFProcessingError
 
 def extract_pdf_text(file_path: str) -> str:
     """
-    Open a PDF and extract raw text from every page.
+    Advanced extraction using IBM's Docling. 
+    Handles OCR, tables, and layout automatically.
     """
-
     if not os.path.exists(file_path):
-        raise FileNotFoundError(f"PDF not found: {file_path}")
-
-    doc = fitz.open(file_path)
-
-    pages_text = []
-
-    for page in doc:
-        pages_text.append(page.get_text())
-
-    doc.close()
-
-    return "\n".join(pages_text)
-
+        raise PDFProcessingError(f"File not found: {file_path}")
+
+    try:
+        # Initialize the converter
+        converter = DocumentConverter()
+
+        # Convert the PDF (Docling automatically detects if it needs OCR)
+        result = converter.convert(file_path)
+
+        # Export to Markdown (best for RAG) or Plain Text
+        # .export_to_markdown() is highly recommended for LLMs
+        extracted_text = result.document.export_to_markdown()
+
+        if not extracted_text.strip():
+            raise PDFProcessingError("Extraction resulted in empty text.")
+
+        return extracted_text
+
+    except Exception as e:
+        raise PDFProcessingError(f"Docling failed to process PDF: {str(e)}")
 
 def ingest_pdf(file_path: str, original_filename: str) -> dict:
-    """
-    Full PDF ingestion pipeline:
-
-    PDF file
-        ↓
-    Text extraction
-        ↓
-    Text cleaning
-        ↓
-    Build standard output format
-    """
-
-    # Step 1: Extract text
+    """Standard pipeline logic."""
     raw_text = extract_pdf_text(file_path)
-
-    # Step 2: Clean extracted text
     cleaned_text = clean_pdf_text(raw_text)
-
-    # Step 3: Generate user-friendly title
     title = os.path.splitext(original_filename)[0]
 
-    # Step 4: Return common JSON format
     return build_result(
         source_type="pdf",
         title=title,
         text=cleaned_text,
         source=file_path,
-    )
-
-
-if __name__ == "__main__":
-    import sys
-    import json
-
-    if len(sys.argv) < 2:
-        print("Usage: python extractor.py <path_to_pdf>")
-
-    else:
-        pdf_path = sys.argv[1]
-
-        result = ingest_pdf(
-            file_path=pdf_path,
-            original_filename=os.path.basename(pdf_path)
-        )
-
-        print(json.dumps(result, indent=2))
+    )

ai-ml/ingestion/youtube/exceptions.py

@@ -0,0 +1,70 @@
+"""Structured errors for the YouTube ingestion pipeline.
+
+Every error carries a machine-readable `code`, an HTTP `status_code` the
+API layer should return, and a human-readable `message`. This lets the
+FastAPI layer turn these into clean JSON instead of a raw 500 traceback.
+"""
+
+
+class YouTubeIngestError(Exception):
+    """Base class for all YouTube ingestion errors."""
+
+    code = "youtube_ingest_error"
+    status_code = 500
+
+    def __init__(
+        self,
+        message: str,
+        video_id: str | None = None,
+        details: dict | None = None,
+    ):
+        super().__init__(message)
+        self.message = message
+        self.video_id = video_id
+        self.details = details or {}
+
+    def to_dict(self) -> dict:
+        payload: dict[str, object] = {
+            "error": self.code,
+            "message": self.message,
+        }
+
+        if self.video_id:
+            payload["video_id"] = self.video_id
+
+        if self.details:
+            payload["details"] = self.details
+
+        return payload
+
+class InvalidYouTubeURLError(YouTubeIngestError):
+    """The URL is not a recognizable YouTube video URL."""
+
+    code = "invalid_url"
+    status_code = 400
+
+
+class VideoUnavailableError(YouTubeIngestError):
+    """The video is private, deleted, region-locked, or otherwise unreachable."""
+
+    code = "video_unavailable"
+    status_code = 404
+
+
+class TranscriptNotAvailableError(YouTubeIngestError):
+    """The video exists but no usable transcript could be produced.
+
+    This covers: transcripts disabled by the uploader, no transcript in any
+    language, a transcript that fetched but came back empty, and transcript
+    fetch failures after a listing succeeded.
+    """
+
+    code = "transcript_not_available"
+    status_code = 422
+
+
+class TranscriptFetchError(YouTubeIngestError):
+    """Transient/upstream failure (rate limiting, IP block, network, etc.)."""
+
+    code = "transcript_fetch_failed"
+    status_code = 502