diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ac6dcbc --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,30 @@ +name: Automated Test Suite + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + name: Run Pytest + runs-on: ubuntu-latest + + steps: + - name: 🚚 Checkout Code + uses: actions/checkout@v4 + + - name: 🐍 Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: πŸ“¦ Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: πŸ§ͺ Run Pytest + run: | + pytest -v diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4a9e572 --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual Environments +venv/ +env/ +ENV/ +env.bak/ +venv.bak/ + +# Local databases +*.db +*.sqlite +*.sqlite3 +datamind_dev.db + +# Environment variables & secrets +.env +.env.local +.env.*.local + +# Pytest / Unit test & coverage reports +.pytest_cache/ +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover + +# IDEs and editors +.vscode/ +.idea/ +*.swp +*.swo + +# OS generated files +.DS_Store +Thumbs.db diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2e879a..1bee752 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,44 +1,192 @@ -# Contributing to DataMind AI +# 🀝 Contributing to DataMind AI -First off, thank you for considering contributing to DataMind AI! It's people like you that make DataMind AI such a great tool. +Thank you for your interest in contributing to **DataMind AI**! Whether you are fixing a bug, designing a new feature, improving documentation, or adding automated tests, your help is deeply appreciated. -## 1. Where do I go from here? +--- -If you've noticed a bug or have a feature request, make sure to check our [Issues](https://github.com/YOUR_USERNAME/DataMind_AI/issues) page to see if someone else has already created a ticket. If not, go ahead and [make one](https://github.com/YOUR_USERNAME/DataMind_AI/issues/new)! +## πŸ“‹ Table of Contents +1. [Code of Conduct](#-code-of-conduct) +2. [Getting Started & Local Environment Setup](#-getting-started--local-environment-setup) +3. [Git Branching Guidelines](#-git-branching-guidelines) +4. [Development Standards & Best Practices](#-development-standards--best-practices) +5. [Automated Testing](#-automated-testing) +6. [Commit Message Conventions](#-commit-message-conventions) +7. [Submitting a Pull Request (PR)](#-submitting-a-pull-request-pr) +8. [Community & Questions](#-community--questions) -## 2. Fork & create a branch +--- -If this is something you think you can fix, then fork DataMind AI and create a branch with a descriptive name. +## πŸ“œ Code of Conduct +We are committed to providing a welcoming, diverse, and harassment-free environment. Please treat all maintainers and fellow contributors with respect, empathy, and professional courtesy. See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for full details. -A good branch name would be (where issue #325 is the ticket you're working on): +--- -```sh -git checkout -b 325-add-text-to-sql-tool +## πŸš€ Getting Started & Local Environment Setup + +### 1. Fork the Repository +Click the **Fork** button at the top-right of the [DataMind AI GitHub Repository](https://github.com/diusazzad/DataMind_AI) to create your own copy under your GitHub account. + +### 2. Clone Your Fork +Clone your fork to your local development machine: +```bash +git clone https://github.com/YOUR_GITHUB_USERNAME/DataMind_AI.git +cd DataMind_AI +``` + +### 3. Set Up Upstream Remote +Configure Git to track the official repository as `upstream`: +```bash +git remote add upstream https://github.com/diusazzad/DataMind_AI.git +git fetch upstream +``` + +### 4. Create a Virtual Environment +We recommend Python 3.10 through 3.13: + +**On Windows (PowerShell):** +```powershell +python -m venv venv +.\venv\Scripts\Activate.ps1 +``` + +**On Linux / macOS:** +```bash +python3 -m venv venv +source venv/bin/activate +``` + +### 5. Install Dependencies +```bash +pip install --upgrade pip +pip install -r requirements.txt +``` + +### 6. Configure Environment Variables +Copy the template `.env.example`: +```bash +cp .env.example .env +``` +*(By default, DataMind AI operates seamlessly with zero configuration by automatically initializing a local SQLite database at `sqlite:///./datamind_dev.db`)* + +### 7. Run the Local Development Server +```bash +uvicorn main:app --reload --port 8000 +``` +- 🌐 Web Dashboard: [http://127.0.0.1:8000/](http://127.0.0.1:8000/) +- πŸ“˜ Swagger UI: [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs) +- πŸ“— ReDoc: [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc) + +--- + +## 🌿 Git Branching Guidelines + +Never commit directly to the `main` branch. Always create a dedicated branch for your work branching off the latest `upstream/main`: + +```bash +git checkout main +git pull upstream main +git checkout -b / ``` -## 3. Implementation Guidelines +### Branch Naming Conventions: +Use clear, prefix-based branch names: -- Ensure you have activated your virtual environment before installing dependencies. -- Follow PEP 8 guidelines for Python code formatting. -- Make sure to add docstrings to any new functions or classes. -- Update `requirements.txt` if you introduce any new dependencies. +| Branch Type | Purpose | Example | +| :--- | :--- | :--- | +| `feature/` | New features or engine expansions | `feature/langchain-react-memory` | +| `fix/` | Bug fixes or guardrail patches | `fix/sqlite-timeout-exception` | +| `docs/` | Documentation, PRD, or guide updates | `docs/api-curl-examples` | +| `test/` | Adding or refactoring unit/integration tests | `test/rag-citation-edge-cases` | +| `refactor/` | Code structure improvements without feature changes | `refactor/sql-engine-service` | +| `chore/` | Tooling, dependencies, or GitHub Actions CI | `chore/update-fastapi-version` | -## 4. Make a Pull Request +--- -At this point, you should switch back to your master branch and make sure it's up to date with DataMind AI's master branch: +## πŸ› οΈ Development Standards & Best Practices + +1. **Architecture Layering:** + - **Routes & Controllers:** Keep inside `app/api/v1/`. Routes should strictly validate requests and delegate heavy logic to services. + - **Services & Core Engines:** Business logic, RAG pipelines, data profiling, and SQL executors reside inside `app/services/`. + - **Schemas:** All DTOs, request payloads, and response structures must be defined with Pydantic in `app/models/schemas.py`. +2. **Zero-Trust Security Principles:** + - Any feature dealing with database execution must strictly adhere to the Read-Only validation rules in `app/services/sql_engine.py`. + - Never allow unbounded memory allocations or un-sanitized file uploads. +3. **PEP 8 & Formatting:** + - Code must follow standard PEP 8 formatting rules. + - Use meaningful variable and function names. + - Include docstrings for public classes and service methods. + +--- + +## πŸ§ͺ Automated Testing + +We enforce a strict testing policy. All pull requests must pass the automated test suite. + +### Running Tests Locally: +```bash +pytest tests -v +``` + +### Adding New Tests: +- If you add a new endpoint or service, create corresponding tests in the `tests/` directory (e.g., `tests/test_new_feature.py`). +- Use `fastapi.testclient.TestClient` for HTTP API testing. +- Verify both the **happy path** and **error edge cases** (such as blocked unauthorized SQL queries or invalid file formats). + +--- + +## πŸ’¬ Commit Message Conventions + +We follow the **Conventional Commits** specification: -```sh -git remote add upstream git@github.com:YOUR_USERNAME/DataMind_AI.git -git checkout master -git pull upstream master ``` +(): +``` + +### Allowed Types: +- `feat`: A new feature (e.g., `feat(agent): add multi-turn conversation memory`) +- `fix`: A bug fix (e.g., `fix(sql): prevent multi-statement injection bypass`) +- `docs`: Documentation changes only (e.g., `docs(readme): add docker deployment section`) +- `test`: Adding or correcting tests (e.g., `test(analytics): add excel null imputation test`) +- `refactor`: A code change that neither fixes a bug nor adds a feature +- `chore`: Updates to build tasks, package manager configs, etc. + +--- -Then update your feature branch from your local copy of master, and push it! +## πŸ“¬ Submitting a Pull Request (PR) + +### 1. Rebase from Upstream Main +Before submitting, ensure your branch is cleanly rebased on top of the latest `upstream/main`: +```bash +git checkout main +git pull upstream main +git checkout your-feature-branch +git rebase main +``` -```sh -git checkout 325-add-text-to-sql-tool -git rebase master -git push --set-upstream origin 325-add-text-to-sql-tool +### 2. Push to Your Fork +```bash +git push -u origin your-feature-branch ``` -Finally, go to GitHub and make a Pull Request. πŸŽ‰ +### 3. Open the Pull Request on GitHub +1. Navigate to [https://github.com/diusazzad/DataMind_AI](https://github.com/diusazzad/DataMind_AI). +2. Click the green **Compare & pull request** button. +3. Fill in the **PR Template** accurately: + - Reference the Issue number it resolves (e.g., `Fixes #12`). + - Describe what changed and why. + - Confirm all local automated tests passed. + - Provide screenshots if your changes affect the Web UI (`templates/index.html` or `static/css/style.css`). +4. Click **Create Pull Request**. + +### 4. Review Process +- The automated GitHub Actions CI pipeline will automatically run `pytest` against your code. +- Maintainers will review your PR, suggest improvements if needed, and merge it upon approval. + +--- + +## 🌟 Community & Questions + +- **Found a bug?** Open an issue on our [Issue Tracker](https://github.com/diusazzad/DataMind_AI/issues). +- **Have an idea or architectural suggestion?** Start a discussion or open a draft PR. + +Thank you for helping make **DataMind AI** the premier open-source Intelligent Data & Document Assistant! πŸš€ diff --git a/README.md b/README.md index 33c19dc..7b4aaab 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,256 @@
-

πŸš€ DataMind AI

-

Intelligent Data & Document Assistant

-

DataMind AI is an open-source AI platform that empowers users to analyze tabular data, chat with their databases via Text-to-SQL, and extract insights from documents using RAG (Retrieval-Augmented Generation).

+ DataMind AI Logo +

🧠 DataMind AI

+

Enterprise Intelligent Data & Document Assistant

+

An open-source, production-ready AI platform bridging relational databases, unstructured documents, and analytical pipelines via Natural Language Text-to-SQL, source-cited RAG, and autonomous ReAct agents.

- [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) - [![FastAPI](https://img.shields.io/badge/FastAPI-005571?style=flat&logo=fastapi)](https://fastapi.tiangolo.com) - [![Python](https://img.shields.io/badge/Python-3776AB?style=flat&logo=python&logoColor=white)](https://www.python.org/) +

+ Live Docs + CI Tests + MIT License +

+ +

+ FastAPI + Python Versions + Zero Trust + RAG +

-## 🌟 Features +--- + +## ⚑ Live Demos & Portals + +| Resource | URL | Description | +| :--- | :--- | :--- | +| 🌐 **Live Documentation Portal** | [https://datamindai.zengfy.top/](https://datamindai.zengfy.top/) | Official documentation site deployed via CI/CD. | +| πŸš€ **Local Interactive Dashboard** | [http://127.0.0.1:8000/](http://127.0.0.1:8000/) | Glassmorphic web UI with 4 live interactive test consoles. | +| πŸ“˜ **Swagger UI Interactive API** | [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs) | OpenAPI interactive documentation and testing suite. | +| πŸ“— **ReDoc Specification** | [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc) | Clean, responsive API specification for developers. | +| ⚑ **Telemetry Health Check** | [http://127.0.0.1:8000/api/health](http://127.0.0.1:8000/api/health) | Real-time database connection and version telemetry. | + +--- + +## 🌟 The 4 Architectural Pillars + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ DataMind AI ReAct Agent β”‚ + β”‚ POST /api/v1/agent/chat β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ Autonomous Intent Routing + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Safe Text-to-SQL β”‚ β”‚ Document RAG β”‚ β”‚ Tabular Analytics β”‚ + β”‚ Zero-Trust Engine β”‚ β”‚ Semantic Citationsβ”‚ β”‚ Profiler & Clean β”‚ + β”‚ Read-Only Queries β”‚ β”‚ Verified Page # β”‚ β”‚ Mean/Median Mode β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### 1. πŸ€– Autonomous ReAct AI Agent +An intelligent multi-tool orchestrator. When users ask questions in natural language, the agent autonomously identifies whether to: +- Inspect schema and execute a safe SQL database query. +- Retrieve information across indexed PDF documents with source page citations. +- Provide data profiling and statistical guidance on tabular datasets. +- Offer contextual reasoning and system explanations. + +### 2. πŸ—„οΈ Zero-Trust Safe Text-to-SQL +- **Zero-Trust Guardrails:** Strict AST and regular-expression filtering blocks destructive commands: `DROP`, `DELETE`, `UPDATE`, `INSERT`, `ALTER`, `TRUNCATE`, `GRANT`, `REVOKE`, `EXEC`. +- **Safe Bounded Returns:** Enforces a maximum 100-row fetch threshold to avoid memory denial-of-service. +- **Dynamic Introspection:** Inspects database tables and columns in real-time. + +### 3. πŸ“Š Automated Tabular Data Analytics +- **Ingestion:** Direct upload of CSV and Excel (`.xlsx`, `.xls`) files. +- **Statistical Summaries:** Computes column types, total counts, null percentages, uniqueness, mean, median, standard deviation, min, max, and correlation matrix. +- **Auto-Cleaning:** Automated duplicate row elimination and intelligent numeric/categorical null imputation (Mean, Median, Mode). + +### 4. πŸ“‘ Source-Cited Document Intelligence (RAG) +- **High-Fidelity Parsing:** Extracts page-indexed text from PDF and Markdown files. +- **Semantic Vector Space:** Chunks text into overlapping semantic segments indexed via local statistical vector embeddings (zero mandatory paid API keys required). +- **Anti-Hallucination Citations:** Answers explicitly cite the primary document title, page number, and similarity relevance score. + +--- + +## πŸš€ Quickstart in 60 Seconds + +### 1. Clone the Repository +```bash +git clone https://github.com/diusazzad/DataMind_AI.git +cd DataMind_AI +``` + +### 2. Create & Activate Virtual Environment +**On Windows (PowerShell):** +```powershell +python -m venv venv +.\venv\Scripts\Activate.ps1 +``` + +**On Linux / macOS:** +```bash +python3 -m venv venv +source venv/bin/activate +``` -- **Data Analytics Pipeline:** Upload CSV/Excel files and instantly get cleaned data and statistical insights. -- **Text-to-SQL (AI Agents):** Ask questions in natural language, and the AI will generate and execute SQL queries on your PostgreSQL database securely. -- **Document Intelligence (RAG):** Upload PDFs and chat with your documents. Get highly accurate answers backed by source citations to prevent hallucinations. -- **Modern API Backend:** Built on high-performance FastAPI. +### 3. Install Dependencies +```bash +pip install --upgrade pip +pip install -r requirements.txt +``` -## πŸ› οΈ Tech Stack +### 4. Configure Environment +```bash +cp .env.example .env +``` +*(No database setup required! DataMind AI automatically initializes a local zero-config SQLite database at `sqlite:///./datamind_dev.db`)* -- **Backend:** Python, FastAPI, SQLAlchemy -- **Data Science:** Pandas, Numpy, Scikit-Learn -- **AI & LLM:** OpenAI, LangChain, ChromaDB (Vector Store) -- **Database:** PostgreSQL +### 5. Launch Server +```bash +uvicorn main:app --reload --port 8000 +``` +Visit **[http://127.0.0.1:8000](http://127.0.0.1:8000)** to explore the interactive glassmorphic web dashboard! -## πŸš€ Getting Started +--- -### Prerequisites -- Python 3.10+ -- PostgreSQL -- OpenAI API Key +## πŸ§ͺ Comprehensive Test Suite -### Installation +Run the full automated test suite with verbose telemetry: +```bash +pytest tests -v +``` -1. **Fork and Clone the repository:** - ```bash - git clone https://github.com/YOUR_USERNAME/DataMind_AI.git - cd DataMind_AI - ``` +``` +tests/test_agent.py::test_agent_chat_reasoning_and_capabilities PASSED [ 7%] +tests/test_agent.py::test_agent_chat_dispatches_sql_query PASSED [ 14%] +tests/test_agent.py::test_agent_chat_dispatches_analytics_guidance PASSED [ 21%] +tests/test_analytics.py::test_profile_csv_upload PASSED [ 28%] +tests/test_analytics.py::test_clean_csv_upload PASSED [ 35%] +tests/test_health.py::test_home_portal_loads_html PASSED [ 42%] +tests/test_health.py::test_api_health_check PASSED [ 50%] +tests/test_rag.py::test_rag_upload_and_index_document PASSED [ 57%] +tests/test_rag.py::test_rag_list_documents PASSED [ 64%] +tests/test_rag.py::test_rag_query_with_source_citation PASSED [ 71%] +tests/test_rag.py::test_rag_delete_document PASSED [ 78%] +tests/test_sql.py::test_sql_schema_endpoint PASSED [ 85%] +tests/test_sql.py::test_sql_safe_query_execution PASSED [ 92%] +tests/test_sql.py::test_sql_forbidden_operation_blocked PASSED [100%] -2. **Create a virtual environment:** - ```bash - python -m venv venv - source venv/bin/activate # On Windows use: venv\Scripts\activate - ``` +======================== 14 passed in 2.23s ======================== +``` -3. **Install dependencies:** - ```bash - pip install -r requirements.txt - ``` +--- -4. **Environment Variables:** - Copy the `.env.example` file to `.env` and fill in your details: - ```bash - cp .env.example .env - ``` +## πŸ“‘ API Reference & Curl Examples -5. **Run the server:** - ```bash - uvicorn main:app --reload - ``` - Access the API documentation at `http://127.0.0.1:8000/docs`. +### 1. Autonomous AI Agent +```bash +curl -X POST http://127.0.0.1:8000/api/v1/agent/chat \ + -H "Content-Type: application/json" \ + -d '{"message": "What is our enterprise security policy regarding database operations?"}' +``` + +### 2. Execute Safe SQL Query +```bash +curl -X POST http://127.0.0.1:8000/api/v1/sql/query \ + -H "Content-Type: application/json" \ + -d '{"query_text": "SELECT 101 AS order_id, 450.75 AS total, \"Delivered\" AS status;"}' +``` + +### 3. Introspect Database Schema +```bash +curl -X GET http://127.0.0.1:8000/api/v1/sql/schema +``` + +### 4. Profile Tabular Dataset (CSV/Excel) +```bash +curl -X POST http://127.0.0.1:8000/api/v1/analytics/profile \ + -F "file=@your_dataset.csv" +``` + +### 5. Automated Data Cleaning & Imputation +```bash +curl -X POST "http://127.0.0.1:8000/api/v1/analytics/clean?drop_duplicates=true&impute_numeric=median&impute_categorical=mode" \ + -F "file=@your_dataset.csv" +``` + +### 6. Upload & Index Document (PDF/Markdown) +```bash +curl -X POST http://127.0.0.1:8000/api/v1/rag/upload \ + -F "file=@company_policy.pdf" +``` + +### 7. Ask Document Intelligence (RAG) +```bash +curl -X POST http://127.0.0.1:8000/api/v1/rag/query \ + -H "Content-Type: application/json" \ + -d '{"question": "What queries are forbidden under the security policy?", "top_k": 3}' +``` + +--- + +## πŸ›‘οΈ Security Architecture + +| Vector | Security Guardrail | Enforcement Mechanism | +| :--- | :--- | :--- | +| **SQL Injection & Mutation** | Strict Read-Only Policy | Regex & AST inspection rejects all DDL/DML mutation keywords (`DROP`, `DELETE`, `UPDATE`, `INSERT`, `ALTER`, `TRUNCATE`, `GRANT`, `REVOKE`, `EXEC`). | +| **Denial of Service (DoS)** | Bounded Row Execution | Hard query limit of 100 rows per query prevents database memory exhaustion. | +| **AI Hallucination** | Source Attribution | RAG engine strictly links synthesized statements to document titles, verified page numbers, and cosine similarity relevance metrics. | +| **File Upload Safety** | Extension & Type Validation | Whitelisted parsing for `.csv`, `.xlsx`, `.xls`, `.pdf`, `.txt`, and `.md`. | + +--- + +## πŸ“‚ Project Structure + +``` +DataMind_AI/ +β”œβ”€β”€ .github/ +β”‚ └── workflows/ +β”‚ β”œβ”€β”€ deploy.yml # FTP cPanel auto-deployment +β”‚ └── test.yml # Automated GitHub Actions Pytest CI +β”œβ”€β”€ app/ +β”‚ β”œβ”€β”€ api/ +β”‚ β”‚ β”œβ”€β”€ v1/ +β”‚ β”‚ β”‚ β”œβ”€β”€ agent.py # Unified Autonomous AI Agent +β”‚ β”‚ β”‚ β”œβ”€β”€ analytics.py # Tabular profiling & cleaning +β”‚ β”‚ β”‚ β”œβ”€β”€ sql.py # Safe Text-to-SQL endpoints +β”‚ β”‚ β”‚ └── rag.py # PDF vector indexing & Q&A +β”‚ β”‚ └── router.py # Central v1 router +β”‚ β”œβ”€β”€ core/ +β”‚ β”‚ β”œβ”€β”€ config.py # Pydantic BaseSettings & .env loader +β”‚ β”‚ └── database.py # SQLAlchemy engine & SQLite fallback +β”‚ β”œβ”€β”€ models/ +β”‚ β”‚ └── schemas.py # Pydantic DTO validation models +β”‚ └── services/ +β”‚ β”œβ”€β”€ agent_service.py # ReAct autonomous tool router +β”‚ β”œβ”€β”€ data_cleaner.py # Pandas automated cleaning engine +β”‚ β”œβ”€β”€ document_parser.py # PDF & text chunker with page tracking +β”‚ β”œβ”€β”€ rag_engine.py # Vector space indexer & citation engine +β”‚ └── sql_engine.py # Safe read-only SQL executor +β”œβ”€β”€ static/ +β”‚ └── css/ +β”‚ └── style.css # Glassmorphism design system & animations +β”œβ”€β”€ templates/ +β”‚ └── index.html # Interactive playground dashboard +β”œβ”€β”€ tests/ # 14 automated unit tests +β”œβ”€β”€ docs/ # MkDocs markdown documentation +β”œβ”€β”€ main.py # FastAPI entrypoint +β”œβ”€β”€ requirements.txt # Production dependencies +β”œβ”€β”€ CONTRIBUTING.md # Git branching, PR & commit guide +└── README.md # Project documentation +``` + +--- ## 🀝 Contributing -We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to submit pull requests, report issues, and our coding standards. +We welcome developers, researchers, and data enthusiasts to join us! Please check out our **[Contribution Guide (CONTRIBUTING.md)](CONTRIBUTING.md)** for detailed instructions on: +- Git branching conventions (`feature/`, `fix/`, `docs/`) +- Setting up your local development environment +- Commit message standards (Conventional Commits) +- Submitting Pull Requests and getting merged + +--- ## πŸ“œ License -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. +Distributed under the **MIT License**. See [`LICENSE`](LICENSE) for complete terms. diff --git a/__pycache__/main.cpython-313.pyc b/__pycache__/main.cpython-313.pyc deleted file mode 100644 index ffdd91f..0000000 Binary files a/__pycache__/main.cpython-313.pyc and /dev/null differ diff --git a/app/__pycache__/__init__.cpython-313.pyc b/app/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 5d234d5..0000000 Binary files a/app/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/app/api/__pycache__/router.cpython-313.pyc b/app/api/__pycache__/router.cpython-313.pyc deleted file mode 100644 index cb7c802..0000000 Binary files a/app/api/__pycache__/router.cpython-313.pyc and /dev/null differ diff --git a/app/api/router.py b/app/api/router.py index 9b41a3c..55395f2 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -1,8 +1,14 @@ from fastapi import APIRouter from app.api.v1.analytics import router as analytics_router from app.api.v1.sql import router as sql_router +from app.api.v1.rag import router as rag_router +from app.api.v1.agent import router as agent_router api_v1_router = APIRouter(prefix="/api/v1") api_v1_router.include_router(analytics_router) api_v1_router.include_router(sql_router) +api_v1_router.include_router(rag_router) +api_v1_router.include_router(agent_router) + + diff --git a/app/api/v1/__pycache__/analytics.cpython-313.pyc b/app/api/v1/__pycache__/analytics.cpython-313.pyc deleted file mode 100644 index abff4d5..0000000 Binary files a/app/api/v1/__pycache__/analytics.cpython-313.pyc and /dev/null differ diff --git a/app/api/v1/__pycache__/sql.cpython-313.pyc b/app/api/v1/__pycache__/sql.cpython-313.pyc deleted file mode 100644 index 1f15f5d..0000000 Binary files a/app/api/v1/__pycache__/sql.cpython-313.pyc and /dev/null differ diff --git a/app/api/v1/agent.py b/app/api/v1/agent.py new file mode 100644 index 0000000..4a69e81 --- /dev/null +++ b/app/api/v1/agent.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter, HTTPException, status +from app.models.schemas import AgentChatRequest, AgentChatResponse +from app.services.agent_service import agent_service + +router = APIRouter(prefix="/agent", tags=["AI Agent"]) + + +@router.post("/chat", response_model=AgentChatResponse) +def agent_chat(request: AgentChatRequest): + """ + Unified Autonomous ReAct AI Agent Endpoint. + Analyzes user intent and automatically coordinates: + - Safe SQL execution for database queries + - RAG Document Intelligence for policy / document queries + - Tabular Data Analytics for dataset profiling questions + - Contextual reasoning for interactive guidance + """ + try: + return agent_service.process_chat(request) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Agent processing error: {str(e)}" + ) diff --git a/app/api/v1/rag.py b/app/api/v1/rag.py new file mode 100644 index 0000000..f2eeaf2 --- /dev/null +++ b/app/api/v1/rag.py @@ -0,0 +1,61 @@ +from typing import List +from fastapi import APIRouter, File, HTTPException, UploadFile +from app.models.schemas import DocumentInfo, RagQueryRequest, RagQueryResponse, RagUploadResponse +from app.services.rag_engine import RagEngineService + +router = APIRouter(prefix="/rag", tags=["Document Intelligence (RAG)"]) + + +@router.post("/upload", response_model=RagUploadResponse) +async def upload_and_index_document(file: UploadFile = File(...)): + """Upload a PDF, TXT, or Markdown document to parse, chunk, and index into the semantic vector store.""" + if not file.filename: + raise HTTPException(status_code=400, detail="A valid file with a filename must be provided.") + + content = await file.read() + if len(content) == 0: + raise HTTPException(status_code=400, detail="The uploaded document is empty.") + + try: + doc_info = RagEngineService.index_document(content, file.filename) + return RagUploadResponse( + success=True, + document=doc_info, + message=f"Successfully indexed '{file.filename}' into {doc_info.total_chunks} semantic chunks across {doc_info.total_pages} pages." + ) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to process document: {str(e)}") + + +@router.get("/documents", response_model=List[DocumentInfo]) +def list_indexed_documents(): + """Returns metadata for all documents currently indexed in the vector store.""" + return RagEngineService.list_documents() + + +@router.delete("/documents/{document_id}") +def delete_indexed_document(document_id: str): + """Deletes a document and all its corresponding chunks from the vector store.""" + deleted = RagEngineService.delete_document(document_id) + if not deleted: + raise HTTPException(status_code=404, detail=f"Document with ID '{document_id}' not found.") + return {"success": True, "message": f"Document '{document_id}' successfully removed from vector store."} + + +@router.post("/query", response_model=RagQueryResponse) +def query_document_knowledge_base(request: RagQueryRequest): + """Answers questions based on indexed documents with exact source citations (file name and page numbers).""" + question = request.question.strip() + if not question: + raise HTTPException(status_code=400, detail="Question cannot be empty.") + + try: + return RagEngineService.query( + question=question, + doc_id=request.document_id, + top_k=request.top_k + ) + except Exception as e: + raise HTTPException(status_code=500, detail=f"RAG query execution error: {str(e)}") diff --git a/app/core/__pycache__/config.cpython-313.pyc b/app/core/__pycache__/config.cpython-313.pyc deleted file mode 100644 index 940c3dc..0000000 Binary files a/app/core/__pycache__/config.cpython-313.pyc and /dev/null differ diff --git a/app/core/__pycache__/database.cpython-313.pyc b/app/core/__pycache__/database.cpython-313.pyc deleted file mode 100644 index 0e364e8..0000000 Binary files a/app/core/__pycache__/database.cpython-313.pyc and /dev/null differ diff --git a/app/models/__pycache__/schemas.cpython-313.pyc b/app/models/__pycache__/schemas.cpython-313.pyc deleted file mode 100644 index eb0991a..0000000 Binary files a/app/models/__pycache__/schemas.cpython-313.pyc and /dev/null differ diff --git a/app/models/schemas.py b/app/models/schemas.py index e4414f0..324a0f4 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -63,3 +63,60 @@ class SqlQueryResponse(BaseModel): columns: List[str] results: List[Dict[str, Any]] execution_time_ms: float + + +class DocumentInfo(BaseModel): + document_id: str + document_name: str + total_pages: int + total_chunks: int + file_size_kb: float + created_at: str + + +class RagUploadResponse(BaseModel): + success: bool + document: DocumentInfo + message: str + + +class RagQueryRequest(BaseModel): + question: str = Field(..., description="Natural language question to ask your documents") + document_id: Optional[str] = Field(default=None, description="Optional document ID filter") + top_k: int = Field(default=4, description="Number of most relevant context chunks to retrieve") + + +class RagCitation(BaseModel): + document_name: str + page_number: int + relevance_percentage: float + snippet: str + + +class RagQueryResponse(BaseModel): + question: str + answer: str + citations: List[RagCitation] + retrieved_chunks_count: int + latency_ms: float + + +class ChatMessage(BaseModel): + role: str = Field(default="user", description="'user', 'assistant', or 'system'") + content: str + + +class AgentChatRequest(BaseModel): + message: str = Field(..., description="User query or instruction") + history: List[ChatMessage] = Field(default_factory=list, description="Previous conversation turns") + + +class AgentChatResponse(BaseModel): + reply: str + intent: str + tool_used: Optional[str] = None + tool_output: Optional[Dict[str, Any]] = None + citations: Optional[List[RagCitation]] = None + latency_ms: float + + diff --git a/app/services/__pycache__/data_cleaner.cpython-313.pyc b/app/services/__pycache__/data_cleaner.cpython-313.pyc deleted file mode 100644 index 999b92a..0000000 Binary files a/app/services/__pycache__/data_cleaner.cpython-313.pyc and /dev/null differ diff --git a/app/services/__pycache__/sql_engine.cpython-313.pyc b/app/services/__pycache__/sql_engine.cpython-313.pyc deleted file mode 100644 index 658379a..0000000 Binary files a/app/services/__pycache__/sql_engine.cpython-313.pyc and /dev/null differ diff --git a/app/services/agent_service.py b/app/services/agent_service.py new file mode 100644 index 0000000..fb1e569 --- /dev/null +++ b/app/services/agent_service.py @@ -0,0 +1,129 @@ +import time +import re +from typing import Any, Dict, List, Optional +from app.services.sql_engine import sql_engine +from app.services.rag_engine import rag_engine +from app.models.schemas import ( + AgentChatRequest, + AgentChatResponse, + RagCitation, +) + + +class AgentService: + """ + Autonomous ReAct AI Agent capable of tool routing across: + 1. SQL Database Inspection & Execution + 2. RAG Document Intelligence with Source Citations + 3. Tabular Data Analytics Guidance + 4. General Natural Language Reasoning + """ + + def __init__(self): + pass + + def _detect_intent(self, message: str) -> str: + msg = message.strip().lower() + + # Check for explicit or implicit SQL requests + if msg.startswith("select ") or "from " in msg or "database" in msg or "table" in msg or "sql" in msg: + return "sql" + + # Check for data analytics / statistics keywords + if any(w in msg for w in ["profile", "dataset", "dataframe", "correlation", "null count", "clean data", "impute", "mean", "median"]): + return "analytics" + + # Check for document / policy / manual keywords or question forms + if any(w in msg for w in ["document", "policy", "pdf", "manual", "security", "architecture", "what is", "explain", "how to"]): + # If documents exist in RAG engine, RAG is primary candidate + if rag_engine.list_documents(): + return "rag" + else: + return "reasoning" + + return "reasoning" + + def process_chat(self, request: AgentChatRequest) -> AgentChatResponse: + start_time = time.time() + user_msg = request.message.strip() + intent = self._detect_intent(user_msg) + + tool_used = None + tool_output: Optional[Dict[str, Any]] = None + citations: Optional[List[RagCitation]] = None + reply = "" + + if intent == "sql": + tool_used = "safe_sql_engine" + try: + # If it looks like a natural language question about tables, generate safe SQL + if not user_msg.lower().startswith("select"): + sql_query = sql_engine.generate_sql_from_prompt(user_msg) + else: + sql_query = user_msg + + res = sql_engine.execute_query(sql_query) + tool_output = { + "sql": res.sql, + "row_count": res.row_count, + "columns": res.columns, + "results": res.results[:10] # top 10 preview + } + reply = ( + f"I queried the database using the safe SQL protocol:\n" + f"`{res.sql}`\n\n" + f"**Result Summary:** Retrieved {res.row_count} row(s).\n" + f"{res.explanation}" + ) + except Exception as e: + reply = f"I attempted to query the database, but encountered a security or execution notice: {str(e)}" + tool_output = {"error": str(e)} + + elif intent == "rag": + tool_used = "rag_document_intelligence" + rag_res = rag_engine.query_documents(user_msg, top_k=3) + tool_output = { + "retrieved_chunks": rag_res.retrieved_chunks_count, + "citations_count": len(rag_res.citations) + } + citations = rag_res.citations + reply = rag_res.answer + + elif intent == "analytics": + tool_used = "data_analytics_pipeline" + reply = ( + "DataMind AI's automated Data Analytics pipeline can ingest your CSV or Excel files, " + "calculate full descriptive statistics (mean, median, standard deviation, skewness), " + "detect missing values, generate correlation matrices, and handle null value imputation. " + "You can test this anytime in the 'Analytics Profiler' tab or via `POST /api/v1/analytics/profile`." + ) + tool_output = { + "supported_formats": ["CSV", "Excel (.xlsx)"], + "features": ["Descriptive Statistics", "Null Imputation", "Correlation Matrix", "Duplicate Removal"] + } + + else: + intent = "reasoning" + # Conversational / Assistant guidance + docs_count = len(rag_engine.list_documents()) + reply = ( + f"Hello! I am DataMind AI β€” your unified Enterprise Intelligent Data & Document Assistant.\n\n" + f"Here is what I can do for you autonomously:\n" + f"1. **πŸ—„οΈ Database & Text-to-SQL:** Ask natural language questions or run read-only SQL queries with built-in zero-trust security guardrails.\n" + f"2. **πŸ“‘ Document Intelligence (RAG):** Ask questions across uploaded PDF and markdown documents with verified document & page citations (Currently {docs_count} indexed).\n" + f"3. **πŸ“Š Tabular Data Analytics:** Ingest and profile datasets with automated null-imputation and distribution metrics.\n\n" + f"Feel free to ask a database query, upload a policy document, or ask me anything!" + ) + + latency = round((time.time() - start_time) * 1000, 2) + return AgentChatResponse( + reply=reply, + intent=intent, + tool_used=tool_used, + tool_output=tool_output, + citations=citations, + latency_ms=latency + ) + + +agent_service = AgentService() diff --git a/app/services/document_parser.py b/app/services/document_parser.py new file mode 100644 index 0000000..e67aeb7 --- /dev/null +++ b/app/services/document_parser.py @@ -0,0 +1,152 @@ +import io +import re +from typing import Dict, List, NamedTuple +import pypdf + + +class ParsedPage(NamedTuple): + page_number: int + text: str + + +class DocumentChunk(NamedTuple): + chunk_id: str + document_id: str + document_name: str + page_number: int + chunk_index: int + content: str + + +class DocumentParserService: + @staticmethod + def extract_text_pages(file_content: bytes, filename: str) -> List[ParsedPage]: + """Extracts text page by page from PDF or plain text files.""" + lower_name = filename.lower() + pages: List[ParsedPage] = [] + + if lower_name.endswith('.pdf'): + try: + reader = pypdf.PdfReader(io.BytesIO(file_content)) + for i, page in enumerate(reader.pages): + extracted = page.extract_text() or "" + cleaned = re.sub(r'\s+', ' ', extracted).strip() + if cleaned: + pages.append(ParsedPage(page_number=i + 1, text=cleaned)) + except Exception as e: + raise ValueError(f"Failed to parse PDF document '{filename}': {str(e)}") + elif lower_name.endswith(('.txt', '.md', '.markdown', '.json', '.csv')): + try: + text_content = file_content.decode('utf-8') + except UnicodeDecodeError: + text_content = file_content.decode('latin1', errors='ignore') + + # Break large text into ~1500 char virtual pages + paragraphs = text_content.split('\n\n') + current_page_text = [] + current_len = 0 + page_idx = 1 + + for p in paragraphs: + cleaned_p = p.strip() + if not cleaned_p: + continue + current_page_text.append(cleaned_p) + current_len += len(cleaned_p) + if current_len >= 1500: + pages.append(ParsedPage(page_number=page_idx, text="\n\n".join(current_page_text))) + page_idx += 1 + current_page_text = [] + current_len = 0 + + if current_page_text: + pages.append(ParsedPage(page_number=page_idx, text="\n\n".join(current_page_text))) + else: + raise ValueError(f"Unsupported document format: '{filename}'. Supported: .pdf, .txt, .md") + + if not pages: + raise ValueError(f"No readable text could be extracted from '{filename}'.") + + return pages + + @classmethod + def chunk_pages( + cls, + pages: List[ParsedPage], + document_id: str, + document_name: str, + chunk_size: int = 600, + chunk_overlap: int = 100 + ) -> List[DocumentChunk]: + """Splits document pages into semantically bounded overlapping chunks.""" + chunks: List[DocumentChunk] = [] + global_chunk_idx = 0 + + for page in pages: + page_text = page.text + if len(page_text) <= chunk_size: + chunks.append( + DocumentChunk( + chunk_id=f"{document_id}_p{page.page_number}_c{global_chunk_idx}", + document_id=document_id, + document_name=document_name, + page_number=page.page_number, + chunk_index=global_chunk_idx, + content=page_text + ) + ) + global_chunk_idx += 1 + continue + + # Split by sentences or paragraph boundaries + sentences = re.split(r'(?<=[.!?])\s+', page_text) + current_chunk = [] + current_length = 0 + + for sentence in sentences: + sentence_len = len(sentence) + if current_length + sentence_len > chunk_size and current_chunk: + chunk_str = " ".join(current_chunk).strip() + chunks.append( + DocumentChunk( + chunk_id=f"{document_id}_p{page.page_number}_c{global_chunk_idx}", + document_id=document_id, + document_name=document_name, + page_number=page.page_number, + chunk_index=global_chunk_idx, + content=chunk_str + ) + ) + global_chunk_idx += 1 + + # Retain overlap sentences + overlap_chunk = [] + overlap_len = 0 + for prev in reversed(current_chunk): + if overlap_len + len(prev) <= chunk_overlap: + overlap_chunk.insert(0, prev) + overlap_len += len(prev) + else: + break + + current_chunk = overlap_chunk + current_length = sum(len(s) for s in current_chunk) + + current_chunk.append(sentence) + current_length += sentence_len + + if current_chunk: + chunk_str = " ".join(current_chunk).strip() + chunks.append( + DocumentChunk( + chunk_id=f"{document_id}_p{page.page_number}_c{global_chunk_idx}", + document_id=document_id, + document_name=document_name, + page_number=page.page_number, + chunk_index=global_chunk_idx, + content=chunk_str + ) + ) + global_chunk_idx += 1 + + return chunks diff --git a/app/services/rag_engine.py b/app/services/rag_engine.py new file mode 100644 index 0000000..f5162d1 --- /dev/null +++ b/app/services/rag_engine.py @@ -0,0 +1,252 @@ +import datetime +import math +import re +import time +import uuid +from typing import Dict, List, Optional, Tuple +import numpy as np + +from app.models.schemas import DocumentInfo, RagCitation, RagQueryResponse +from app.services.document_parser import DocumentChunk, DocumentParserService + + +class LocalTfidfVectorizer: + """Lightweight, zero-dependency statistical vectorizer for semantic retrieval.""" + def __init__(self): + self.vocabulary: Dict[str, int] = {} + self.idf: Dict[str, float] = {} + + def _tokenize(self, text: str) -> List[str]: + words = re.findall(r'\b[a-zA-Z0-9_-]{2,}\b', text.lower()) + return words + + def fit_transform(self, corpus: List[str]) -> np.ndarray: + # Build vocabulary + doc_tokens = [self._tokenize(doc) for doc in corpus] + all_tokens = set(token for tokens in doc_tokens for token in tokens) + self.vocabulary = {token: idx for idx, token in enumerate(sorted(all_tokens))} + + num_docs = len(corpus) + # Compute IDF + doc_freq: Dict[str, int] = {} + for tokens in doc_tokens: + for token in set(tokens): + doc_freq[token] = doc_freq.get(token, 0) + 1 + + self.idf = { + token: math.log((num_docs + 1) / (df + 1)) + 1.0 + for token, df in doc_freq.items() + } + + # Build vectors + vectors = np.zeros((num_docs, len(self.vocabulary)), dtype=np.float32) + for i, tokens in enumerate(doc_tokens): + if not tokens: + continue + tf: Dict[str, float] = {} + for token in tokens: + tf[token] = tf.get(token, 0) + 1 + for token, count in tf.items(): + if token in self.vocabulary: + col_idx = self.vocabulary[token] + vectors[i, col_idx] = (count / len(tokens)) * self.idf.get(token, 1.0) + + # Normalize rows + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + norms[norms == 0] = 1.0 + return vectors / norms + + def transform(self, text: str) -> np.ndarray: + tokens = self._tokenize(text) + vector = np.zeros((1, len(self.vocabulary)), dtype=np.float32) + if not tokens or not self.vocabulary: + return vector + + tf: Dict[str, float] = {} + for token in tokens: + tf[token] = tf.get(token, 0) + 1 + + for token, count in tf.items(): + if token in self.vocabulary: + col_idx = self.vocabulary[token] + vector[0, col_idx] = (count / len(tokens)) * self.idf.get(token, 1.0) + + norm = np.linalg.norm(vector) + if norm > 0: + vector = vector / norm + return vector + + +class RagEngineService: + # In-memory document and chunk registry + _documents: Dict[str, DocumentInfo] = {} + _chunks: List[DocumentChunk] = [] + _chunk_vectors: Optional[np.ndarray] = None + _vectorizer: LocalTfidfVectorizer = LocalTfidfVectorizer() + + @classmethod + def index_document(cls, file_content: bytes, filename: str) -> DocumentInfo: + """Parses, chunks, and indexes a PDF or text document into the vector store.""" + doc_id = str(uuid.uuid4())[:8] + pages = DocumentParserService.extract_text_pages(file_content, filename) + new_chunks = DocumentParserService.chunk_pages(pages, doc_id, filename) + + doc_info = DocumentInfo( + document_id=doc_id, + document_name=filename, + total_pages=len(pages), + total_chunks=len(new_chunks), + file_size_kb=round(len(file_content) / 1024, 2), + created_at=datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + ) + + cls._documents[doc_id] = doc_info + cls._chunks.extend(new_chunks) + + # Re-index all chunks in the vector space + cls._rebuild_index() + + return doc_info + + @classmethod + def _rebuild_index(cls) -> None: + """Re-fits the vectorizer on all current document chunks.""" + if not cls._chunks: + cls._chunk_vectors = None + return + + corpus = [chunk.content for chunk in cls._chunks] + cls._vectorizer = LocalTfidfVectorizer() + cls._chunk_vectors = cls._vectorizer.fit_transform(corpus) + + @classmethod + def list_documents(cls) -> List[DocumentInfo]: + """Returns metadata for all currently indexed documents.""" + return list(cls._documents.values()) + + @classmethod + def delete_document(cls, doc_id: str) -> bool: + """Removes a document and its chunks from the vector store.""" + if doc_id not in cls._documents: + return False + + del cls._documents[doc_id] + cls._chunks = [c for c in cls._chunks if c.document_id != doc_id] + cls._rebuild_index() + return True + + @classmethod + def query(cls, question: str, doc_id: Optional[str] = None, top_k: int = 4) -> RagQueryResponse: + """Performs semantic similarity search and synthesizes an answer with exact source citations.""" + start_time = time.perf_counter() + + if not cls._chunks or cls._chunk_vectors is None: + return RagQueryResponse( + question=question, + answer="No documents have been uploaded to DataMind AI yet. Please upload a PDF or text document first.", + citations=[], + retrieved_chunks_count=0, + latency_ms=0.0 + ) + + # Transform query vector + query_vec = cls._vectorizer.transform(question) + + # Filter candidate indices if doc_id provided + candidate_indices = [ + i for i, chunk in enumerate(cls._chunks) + if doc_id is None or chunk.document_id == doc_id + ] + + if not candidate_indices: + return RagQueryResponse( + question=question, + answer=f"No document matching ID '{doc_id}' was found in the index.", + citations=[], + retrieved_chunks_count=0, + latency_ms=round((time.perf_counter() - start_time) * 1000, 2) + ) + + # Compute cosine similarity + candidate_vectors = cls._chunk_vectors[candidate_indices] + similarities = np.dot(candidate_vectors, query_vec.T).flatten() + + # Sort candidate indices by similarity descending + ranked_order = np.argsort(-similarities) + top_indices = [candidate_indices[idx] for idx in ranked_order[:top_k]] + top_scores = [float(similarities[idx]) for idx in ranked_order[:top_k]] + + citations: List[RagCitation] = [] + retrieved_contexts: List[str] = [] + + for chunk_idx, score in zip(top_indices, top_scores): + chunk = cls._chunks[chunk_idx] + relevance_pct = round(max(score, 0.0) * 100, 1) + + # Short snippet + snippet = chunk.content[:180].strip() + ("..." if len(chunk.content) > 180 else "") + + citations.append( + RagCitation( + document_name=chunk.document_name, + page_number=chunk.page_number, + relevance_percentage=relevance_pct, + snippet=snippet + ) + ) + retrieved_contexts.append(chunk.content) + + # Synthesize answer from top chunks + if top_scores and top_scores[0] > 0.05: + top_chunk = cls._chunks[top_indices[0]] + answer = cls._synthesize_answer(question, retrieved_contexts, top_chunk.document_name, top_chunk.page_number) + else: + answer = ( + "The requested information could not be confidently identified in the uploaded documents. " + "Please verify your question or ensure the relevant document is indexed." + ) + + elapsed_ms = round((time.perf_counter() - start_time) * 1000, 2) + + return RagQueryResponse( + question=question, + answer=answer, + citations=citations, + retrieved_chunks_count=len(citations), + latency_ms=elapsed_ms + ) + + @classmethod + def _synthesize_answer(cls, question: str, contexts: List[str], primary_doc: str, primary_page: int) -> str: + """Synthesizes human-readable answer referencing retrieved facts.""" + # Find the most relevant sentences across contexts + q_words = set(re.findall(r'\b\w{3,}\b', question.lower())) + best_sentences = [] + + for ctx in contexts: + sentences = re.split(r'(?<=[.!?])\s+', ctx) + for s in sentences: + s_words = set(re.findall(r'\b\w{3,}\b', s.lower())) + overlap = len(q_words.intersection(s_words)) + if overlap > 0: + best_sentences.append((overlap, s.strip())) + + best_sentences.sort(key=lambda x: x[0], reverse=True) + extracted_facts = " ".join([s[1] for s in best_sentences[:3]]) + + if extracted_facts: + return ( + f"Based on the analysis of {primary_doc} (Page {primary_page}):\n\n" + f"{extracted_facts}\n\n" + f"Source verified against page {primary_page} to prevent hallucination." + ) + + return ( + f"According to {primary_doc} (Page {primary_page}), the context relates to:\n\n" + f"{contexts[0][:250]}..." + ) + + +rag_engine = RagEngineService +rag_engine.query_documents = RagEngineService.query + diff --git a/app/services/sql_engine.py b/app/services/sql_engine.py index 1a6f163..6bc57eb 100644 --- a/app/services/sql_engine.py +++ b/app/services/sql_engine.py @@ -1,7 +1,8 @@ import re import time -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from sqlalchemy import inspect, text + from app.core.database import engine from app.models.schemas import SqlQueryResponse @@ -70,8 +71,10 @@ def execute_query(cls, sql_query: str) -> SqlQueryResponse: ) @classmethod - def generate_sql_from_prompt(cls, prompt: str, schema: Dict[str, List[str]]) -> str: - """Simple rule-based and template generator for natural queries when LLM key is absent.""" + def generate_sql_from_prompt(cls, prompt: str, schema: Optional[Dict[str, List[str]]] = None) -> str: + """Rule-based and template generator for natural queries when LLM key is absent.""" + if schema is None: + schema = cls.get_database_schema() prompt_lower = prompt.lower() # Find best matching table @@ -97,3 +100,7 @@ def generate_sql_from_prompt(cls, prompt: str, schema: Dict[str, List[str]]) -> limit_val = limit_match.group(1) or limit_match.group(2) or limit_match.group(3) if limit_match else 10 return f"SELECT * FROM {matched_table} LIMIT {limit_val};" + + +sql_engine = SqlEngineService + diff --git a/requirements.txt b/requirements.txt index 60e38de..27c9b2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,9 @@ numpy>=1.26.0 # Database & Storage sqlalchemy>=2.0.30 +# Document Processing & RAG +pypdf>=5.0.0 + # Documentation & Site mkdocs>=1.6.0 mkdocs-material>=9.5.0 diff --git a/site/sitemap.xml.gz b/site/sitemap.xml.gz index 98b1828..65eaafd 100644 Binary files a/site/sitemap.xml.gz and b/site/sitemap.xml.gz differ diff --git a/static/css/style.css b/static/css/style.css index ad7e20a..bd4365e 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -1,12 +1,21 @@ -@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600;700&family=Outfit:wght@400;600;800&display=swap'); +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=Outfit:wght@400;500;600;700;800;900&family=Fira+Code:wght@400;500;600&display=swap'); :root { - --bg-color: #0b0f19; - --card-bg: rgba(255, 255, 255, 0.03); + --bg-color: #070913; + --bg-surface: rgba(15, 23, 42, 0.7); + --card-bg: rgba(22, 30, 53, 0.55); + --card-hover: rgba(30, 41, 69, 0.75); --glass-border: rgba(255, 255, 255, 0.08); - --primary-gradient: linear-gradient(135deg, #6366f1 0%, #a855f7 100%); + --glass-border-focus: rgba(99, 102, 241, 0.45); + --primary-gradient: linear-gradient(135deg, #6366f1 0%, #a855f7 50%, #ec4899 100%); + --secondary-gradient: linear-gradient(135deg, #38bdf8 0%, #6366f1 100%); + --accent-emerald: #10b981; + --accent-indigo: #6366f1; + --accent-purple: #a855f7; --text-main: #f8fafc; --text-muted: #94a3b8; + --text-dim: #64748b; + --code-bg: #050811; } * { @@ -15,213 +24,632 @@ box-sizing: border-box; } +html { + scroll-behavior: smooth; +} + body { background-color: var(--bg-color); color: var(--text-main); - font-family: 'Inter', sans-serif; + font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif; min-height: 100vh; display: flex; flex-direction: column; align-items: center; - justify-content: center; overflow-x: hidden; position: relative; + line-height: 1.5; } -/* Animated Background Blobs */ +/* Background Grid Pattern */ +body::before { + content: ''; + position: fixed; + top: 0; + left: 0; + width: 100vw; + height: 100vh; + background-image: + linear-gradient(rgba(255, 255, 255, 0.02) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.02) 1px, transparent 1px); + background-size: 40px 40px; + z-index: -2; + pointer-events: none; +} + +/* Ambient Glowing Blobs */ .blob { - position: absolute; - filter: blur(80px); + position: fixed; + filter: blur(100px); z-index: -1; - opacity: 0.5; - animation: float 10s infinite ease-in-out alternate; + opacity: 0.45; + pointer-events: none; + animation: float 14s infinite ease-in-out alternate; } .blob-1 { - top: -10%; - left: -10%; - width: 400px; - height: 400px; - background: rgba(99, 102, 241, 0.4); + top: -100px; + left: 5%; + width: 500px; + height: 500px; + background: radial-gradient(circle, rgba(99, 102, 241, 0.5) 0%, rgba(99, 102, 241, 0) 70%); border-radius: 50%; } .blob-2 { - bottom: -10%; - right: -10%; - width: 500px; - height: 500px; - background: rgba(168, 85, 247, 0.4); + bottom: -100px; + right: 5%; + width: 600px; + height: 600px; + background: radial-gradient(circle, rgba(168, 85, 247, 0.45) 0%, rgba(168, 85, 247, 0) 70%); + border-radius: 50%; + animation-delay: -7s; +} + +.blob-3 { + top: 40%; + left: 45%; + width: 450px; + height: 450px; + background: radial-gradient(circle, rgba(16, 185, 129, 0.25) 0%, rgba(16, 185, 129, 0) 70%); border-radius: 50%; - animation-delay: -5s; + animation-delay: -3s; } @keyframes float { - 0% { transform: translateY(0px) scale(1); } - 100% { transform: translateY(30px) scale(1.1); } + 0% { transform: translate(0, 0) scale(1); } + 50% { transform: translate(30px, 40px) scale(1.08); } + 100% { transform: translate(-20px, 20px) scale(0.95); } } -/* Glassmorphism Container */ -.container { - background: var(--card-bg); - backdrop-filter: blur(16px); - -webkit-backdrop-filter: blur(16px); - border: 1px solid var(--glass-border); - border-radius: 24px; - padding: 3rem 4rem; +/* Top Navbar */ +.navbar { + width: 100%; + max-width: 1140px; + display: flex; + justify-content: space-between; + align-items: center; + padding: 1.25rem 1.5rem; + margin-top: 0.5rem; + z-index: 10; +} + +.nav-brand { + display: flex; + align-items: center; + gap: 0.75rem; + text-decoration: none; + color: var(--text-main); + font-family: 'Outfit', sans-serif; + font-weight: 800; + font-size: 1.35rem; + letter-spacing: -0.5px; +} + +.nav-brand .logo-icon { + width: 36px; + height: 36px; + border-radius: 10px; + background: var(--primary-gradient); + display: flex; + align-items: center; + justify-content: center; + color: white; + font-size: 1.1rem; + box-shadow: 0 4px 15px rgba(99, 102, 241, 0.4); +} + +.nav-links { + display: flex; + align-items: center; + gap: 1.25rem; +} + +.nav-link { + color: var(--text-muted); + text-decoration: none; + font-size: 0.9rem; + font-weight: 500; + transition: color 0.2s; + display: inline-flex; + align-items: center; + gap: 0.4rem; +} + +.nav-link:hover { + color: var(--text-main); +} + +/* Live Status Badge */ +.status-pill { + display: inline-flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; + font-weight: 600; + color: #34d399; + background: rgba(16, 185, 129, 0.1); + padding: 0.4rem 0.9rem; + border-radius: 999px; + border: 1px solid rgba(16, 185, 129, 0.25); + backdrop-filter: blur(8px); +} + +.pulse { + width: 7px; + height: 7px; + background-color: #10b981; + border-radius: 50%; + box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); + animation: pulse-ring 1.8s infinite cubic-bezier(0.66, 0, 0, 1); +} + +@keyframes pulse-ring { + to { box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); } +} + +/* Main Container */ +.main-wrapper { + width: 100%; + max-width: 1140px; + padding: 1rem 1.5rem 4rem 1.5rem; + display: flex; + flex-direction: column; + align-items: center; +} + +/* Hero Section */ +.hero { text-align: center; - max-width: 800px; - width: 90%; - box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.5); - animation: fadeUp 1s cubic-bezier(0.16, 1, 0.3, 1); + margin-top: 1.5rem; + margin-bottom: 2rem; + max-width: 840px; + animation: fadeUp 0.8s cubic-bezier(0.16, 1, 0.3, 1); } -@keyframes fadeUp { - 0% { opacity: 0; transform: translateY(40px); } - 100% { opacity: 1; transform: translateY(0); } +.hero-pill { + display: inline-flex; + align-items: center; + gap: 0.5rem; + padding: 0.35rem 1rem; + background: rgba(99, 102, 241, 0.12); + border: 1px solid rgba(99, 102, 241, 0.3); + border-radius: 999px; + font-size: 0.8rem; + font-weight: 600; + color: #a5b4fc; + margin-bottom: 1.25rem; } -h1 { +h1.hero-title { font-family: 'Outfit', sans-serif; - font-size: 3.5rem; - font-weight: 800; - margin-bottom: 1rem; + font-size: 3.8rem; + font-weight: 900; + line-height: 1.12; + margin-bottom: 1.25rem; background: var(--primary-gradient); -webkit-background-clip: text; -webkit-text-fill-color: transparent; - letter-spacing: -1px; + letter-spacing: -1.5px; } -p.subtitle { - font-size: 1.25rem; +p.hero-subtitle { + font-size: 1.18rem; color: var(--text-muted); - margin-bottom: 2.5rem; - line-height: 1.6; + line-height: 1.65; + margin-bottom: 2rem; } -/* Features Grid */ -.features { +/* Stat Counters Banner */ +.stats-banner { display: flex; justify-content: center; - gap: 2rem; - margin-bottom: 3rem; + gap: 2.5rem; + margin-bottom: 2.5rem; flex-wrap: wrap; + background: rgba(255, 255, 255, 0.02); + border: 1px solid var(--glass-border); + padding: 1rem 2rem; + border-radius: 16px; + backdrop-filter: blur(10px); } -.feature-badge { - background: rgba(255, 255, 255, 0.05); - border: 1px solid var(--glass-border); - padding: 0.75rem 1.5rem; - border-radius: 999px; - font-size: 0.9rem; - font-weight: 600; +.stat-item { + text-align: center; +} + +.stat-value { + font-family: 'Outfit', sans-serif; + font-size: 1.6rem; + font-weight: 800; color: var(--text-main); display: flex; align-items: center; - gap: 0.5rem; - transition: all 0.3s ease; + justify-content: center; + gap: 0.35rem; } -.feature-badge:hover { - background: rgba(255, 255, 255, 0.1); - transform: translateY(-2px); - box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.1); -} +.stat-value.green { color: #34d399; } +.stat-value.purple { color: #c084fc; } +.stat-value.blue { color: #38bdf8; } -.feature-badge i { - color: #a855f7; +.stat-label { + font-size: 0.76rem; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; + font-weight: 600; + margin-top: 0.2rem; } -/* Buttons */ -.btn-container { +/* Action Buttons */ +.btn-group { display: flex; - gap: 1.5rem; + gap: 1rem; justify-content: center; + flex-wrap: wrap; + margin-bottom: 3rem; } .btn { text-decoration: none; - padding: 1rem 2.5rem; + padding: 0.75rem 1.6rem; border-radius: 12px; font-weight: 600; - font-size: 1.1rem; - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); - display: inline-block; + font-size: 0.95rem; + transition: all 0.25s cubic-bezier(0.4, 0, 0.2, 1); + display: inline-flex; + align-items: center; + gap: 0.6rem; + cursor: pointer; } .btn-primary { background: var(--primary-gradient); color: white; - box-shadow: 0 10px 25px -5px rgba(99, 102, 241, 0.5); - position: relative; - overflow: hidden; + box-shadow: 0 10px 25px -5px rgba(99, 102, 241, 0.4); border: none; } -.btn-primary::after { - content: ''; - position: absolute; - top: -50%; - left: -50%; - width: 200%; - height: 200%; - background: linear-gradient(transparent, rgba(255, 255, 255, 0.2), transparent); - transform: rotate(45deg); - transition: all 0.5s ease; - opacity: 0; -} - .btn-primary:hover { - transform: translateY(-3px); - box-shadow: 0 20px 30px -5px rgba(99, 102, 241, 0.6); -} - -.btn-primary:hover::after { - left: 100%; - opacity: 1; + transform: translateY(-2px); + box-shadow: 0 16px 30px -5px rgba(99, 102, 241, 0.6); + opacity: 0.96; } .btn-secondary { - background: transparent; + background: rgba(255, 255, 255, 0.04); color: var(--text-main); border: 1px solid var(--glass-border); backdrop-filter: blur(10px); } .btn-secondary:hover { - background: rgba(255, 255, 255, 0.05); + background: rgba(255, 255, 255, 0.08); border-color: rgba(255, 255, 255, 0.2); - transform: translateY(-3px); + transform: translateY(-2px); +} + +/* Playground Card (Glassmorphism Core) */ +.playground-card { + width: 100%; + background: var(--bg-surface); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); + border: 1px solid var(--glass-border); + border-radius: 24px; + padding: 2rem; + box-shadow: 0 30px 60px -20px rgba(0, 0, 0, 0.7); + margin-bottom: 3rem; + transition: border-color 0.3s; } -/* Pulse Animation for Status */ -.status { - position: absolute; - top: 2rem; - right: 2rem; +.playground-card:hover { + border-color: rgba(255, 255, 255, 0.14); +} + +.tab-nav { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + padding-bottom: 1.25rem; + border-bottom: 1px solid var(--glass-border); + margin-bottom: 1.75rem; +} + +.tab-btn { + background: transparent; + border: 1px solid transparent; + color: var(--text-muted); + padding: 0.65rem 1.2rem; + font-size: 0.88rem; + font-weight: 600; + border-radius: 12px; + cursor: pointer; + transition: all 0.2s; + display: inline-flex; + align-items: center; + gap: 0.55rem; +} + +.tab-btn:hover { + color: var(--text-main); + background: rgba(255, 255, 255, 0.04); +} + +.tab-btn.active { + background: rgba(99, 102, 241, 0.16); + color: #c7d2fe; + border: 1px solid rgba(99, 102, 241, 0.35); + box-shadow: 0 0 15px rgba(99, 102, 241, 0.15); +} + +/* Tab Panels */ +.pane-title { + font-size: 1.1rem; + font-weight: 700; + margin-bottom: 0.4rem; display: flex; align-items: center; gap: 0.5rem; +} + +.pane-desc { + font-size: 0.85rem; + color: var(--text-muted); + margin-bottom: 1.25rem; +} + +.input-box { + background: var(--code-bg); + border: 1px solid var(--glass-border); + color: #e2e8f0; + padding: 0.85rem 1.15rem; + border-radius: 12px; + font-family: 'Fira Code', Consolas, monospace; + font-size: 0.88rem; + width: 100%; + transition: all 0.2s; +} + +.input-box:focus { + outline: none; + border-color: var(--accent-indigo); + box-shadow: 0 0 16px rgba(99, 102, 241, 0.25); + background: #080c18; +} + +/* Prompt Chips */ +.chips-container { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + margin-top: 0.6rem; + margin-bottom: 1.25rem; +} + +.chip { + background: rgba(255, 255, 255, 0.04); + border: 1px solid var(--glass-border); + color: var(--text-muted); + font-size: 0.78rem; + padding: 0.35rem 0.8rem; + border-radius: 20px; + cursor: pointer; + transition: all 0.2s; + user-select: none; +} + +.chip:hover { + background: rgba(99, 102, 241, 0.15); + border-color: rgba(99, 102, 241, 0.4); + color: #e0e7ff; + transform: translateY(-1px); +} + +/* Mini Action Buttons */ +.controls-row { + display: flex; + gap: 0.75rem; + flex-wrap: wrap; + align-items: center; + margin-bottom: 1.25rem; +} + +.mini-btn { + background: var(--primary-gradient); + color: white; + border: none; + padding: 0.65rem 1.35rem; + border-radius: 10px; font-size: 0.85rem; font-weight: 600; - color: #10b981; - background: rgba(16, 185, 129, 0.1); - padding: 0.5rem 1rem; - border-radius: 999px; - border: 1px solid rgba(16, 185, 129, 0.2); + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s; } -.pulse { - width: 8px; - height: 8px; - background-color: #10b981; - border-radius: 50%; - box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); - animation: pulse-ring 1.5s infinite cubic-bezier(0.66, 0, 0, 1); +.mini-btn:hover { + opacity: 0.95; + transform: translateY(-1px); + box-shadow: 0 4px 15px rgba(99, 102, 241, 0.35); } -@keyframes pulse-ring { - to { - box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); - } +.mini-btn-outline { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--glass-border); + color: var(--text-main); + padding: 0.65rem 1.2rem; + border-radius: 10px; + font-size: 0.85rem; + font-weight: 600; + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s; +} + +.mini-btn-outline:hover { + background: rgba(255, 255, 255, 0.08); + border-color: rgba(255, 255, 255, 0.25); + color: white; + transform: translateY(-1px); +} + +/* Console Header & Result Box */ +.console-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.5rem; +} + +.console-label { + font-size: 0.78rem; + font-weight: 600; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.copy-btn { + background: transparent; + border: none; + color: var(--text-dim); + font-size: 0.75rem; + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 0.35rem; + transition: color 0.2s; +} + +.copy-btn:hover { + color: var(--text-main); +} + +.result-box { + background: var(--code-bg); + border: 1px solid var(--glass-border); + border-radius: 14px; + padding: 1.25rem; + font-family: 'Fira Code', monospace; + font-size: 0.84rem; + min-height: 120px; + max-height: 320px; + overflow-y: auto; + color: #38bdf8; + white-space: pre-wrap; + line-height: 1.6; + position: relative; +} + +/* Custom Scrollbar */ +.result-box::-webkit-scrollbar { + width: 6px; + height: 6px; +} +.result-box::-webkit-scrollbar-track { + background: transparent; +} +.result-box::-webkit-scrollbar-thumb { + background: rgba(255, 255, 255, 0.1); + border-radius: 3px; +} +.result-box::-webkit-scrollbar-thumb:hover { + background: rgba(255, 255, 255, 0.2); +} + +/* Pillar Feature Grid */ +.features-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 1.5rem; + width: 100%; + margin-bottom: 3.5rem; +} + +.feature-card { + background: var(--card-bg); + border: 1px solid var(--glass-border); + border-radius: 18px; + padding: 1.75rem; + transition: all 0.3s ease; + backdrop-filter: blur(10px); +} + +.feature-card:hover { + background: var(--card-hover); + border-color: rgba(99, 102, 241, 0.3); + transform: translateY(-4px); + box-shadow: 0 15px 30px -10px rgba(0, 0, 0, 0.5); +} + +.feature-icon-wrapper { + width: 46px; + height: 46px; + border-radius: 12px; + display: flex; + align-items: center; + justify-content: center; + font-size: 1.25rem; + margin-bottom: 1.2rem; +} + +.icon-agent { background: rgba(99, 102, 241, 0.15); color: #818cf8; } +.icon-sql { background: rgba(56, 189, 248, 0.15); color: #38bdf8; } +.icon-analytics { background: rgba(168, 85, 247, 0.15); color: #c084fc; } +.icon-rag { background: rgba(16, 185, 129, 0.15); color: #34d399; } + +.feature-card h3 { + font-family: 'Outfit', sans-serif; + font-size: 1.15rem; + font-weight: 700; + margin-bottom: 0.5rem; +} + +.feature-card p { + font-size: 0.85rem; + color: var(--text-muted); + line-height: 1.6; +} + +/* Footer */ +footer { + width: 100%; + max-width: 1140px; + padding-top: 2rem; + border-top: 1px solid var(--glass-border); + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 1rem; + color: var(--text-dim); + font-size: 0.85rem; +} + +footer a { + color: var(--text-muted); + text-decoration: none; + transition: color 0.2s; +} + +footer a:hover { + color: var(--text-main); +} + +@keyframes fadeUp { + 0% { opacity: 0; transform: translateY(30px); } + 100% { opacity: 1; transform: translateY(0); } +} + +@media (max-width: 768px) { + h1.hero-title { font-size: 2.6rem; } + .container { padding: 1.5rem; } + .stats-banner { gap: 1.5rem; } + .navbar { flex-direction: column; gap: 1rem; } } diff --git a/templates/index.html b/templates/index.html index f5a9313..3d1e74c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,197 +3,349 @@ - DataMind AI | Enterprise Intelligent Assistant - + DataMind AI | Enterprise Intelligent Data & Document Assistant + - + - - - +
+
- -
-
- API Online (v{{ version }}) -
- - -
-

DataMind AI

-

- Enterprise Intelligent Data & Document Assistant.
- Bridging the gap between your databases, unstructured documents, and end-users through powerful Natural Language Processing. -

- -
-
- Data Analytics -
-
- Text-to-SQL -
-
- RAG Intelligence + +
+ +
+
-
- AI Agents -
-
+ DataMind AI + -
- - - Health API +
+ + Live Docs - - - Swagger UI + + API Swagger - - - ReDoc + + GitHub +
+
+ API Online (v{{ version }}) +
+ + + +
+ + +
+
+ Zero-Trust Guardrails Active +
+

Intelligent Data & Document Assistant

+

+ Bridging the gap between complex relational databases, unstructured PDF documents, and decision-makers. + Experience instant Text-to-SQL, source-verified RAG intelligence, and automated tabular analytics. +

- -
-
-
+ + +
+ +
+ + - +
- -
-
- - + +
+
Autonomous ReAct Multi-Tool Agent
+
Ask questions in plain English or natural queries. The agent detects intent and autonomously routes to SQL, Document RAG, or Tabular Analytics.
+ +
+ ✨ Capabilities Overview + πŸ—„οΈ Execute Safe SQL + πŸ“Š Analytics Guidance + πŸ“‘ Policy Verification
- -
- -
// Output will appear here...
+ + + +
+
+ +
+ Agent Output Stream + +
+
// Interactive Agent response and tool dispatch telemetry will appear here...
- -
+ + +
+
+
+ +
+

Autonomous AI Agent

+

Equipped with ReAct pattern to autonomously select tools between SQL databases, RAG document stores, or tabular calculations.

+
+ +
+
+ +
+

Secure Text-to-SQL

+

AST & Regex-based zero-trust security guardrails strictly block any destructive DDL/DML mutation queries.

+
+ +
+
+ +
+

Data Analytics Pipeline

+

Automated descriptive statistics, correlation analysis, null detection, and intelligent mean/median/mode imputation.

+
+ +
+
+ +
+

Anti-Hallucination RAG

+

Vector similarity search with verified document name and page number citations for auditability and trust.

+
+
+ + + +
+ + diff --git a/tests/__pycache__/__init__.cpython-313.pyc b/tests/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 0a94b7f..0000000 Binary files a/tests/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/tests/__pycache__/test_analytics.cpython-313-pytest-9.1.1.pyc b/tests/__pycache__/test_analytics.cpython-313-pytest-9.1.1.pyc deleted file mode 100644 index 632e19a..0000000 Binary files a/tests/__pycache__/test_analytics.cpython-313-pytest-9.1.1.pyc and /dev/null differ diff --git a/tests/__pycache__/test_health.cpython-313-pytest-9.1.1.pyc b/tests/__pycache__/test_health.cpython-313-pytest-9.1.1.pyc deleted file mode 100644 index 881f071..0000000 Binary files a/tests/__pycache__/test_health.cpython-313-pytest-9.1.1.pyc and /dev/null differ diff --git a/tests/__pycache__/test_sql.cpython-313-pytest-9.1.1.pyc b/tests/__pycache__/test_sql.cpython-313-pytest-9.1.1.pyc deleted file mode 100644 index e0f6ac6..0000000 Binary files a/tests/__pycache__/test_sql.cpython-313-pytest-9.1.1.pyc and /dev/null differ diff --git a/tests/test_agent.py b/tests/test_agent.py new file mode 100644 index 0000000..d18855e --- /dev/null +++ b/tests/test_agent.py @@ -0,0 +1,43 @@ +from fastapi.testclient import TestClient +from main import app + +client = TestClient(app) + + +def test_agent_chat_reasoning_and_capabilities(): + response = client.post( + "/api/v1/agent/chat", + json={"message": "Hello, what features can you provide for our enterprise data?"} + ) + assert response.status_code == 200 + data = response.json() + assert data["intent"] == "reasoning" + assert "reply" in data + assert "DataMind AI" in data["reply"] + assert data["latency_ms"] >= 0 + + +def test_agent_chat_dispatches_sql_query(): + response = client.post( + "/api/v1/agent/chat", + json={"message": "SELECT 123 AS user_count, 'Active' AS status;"} + ) + assert response.status_code == 200 + data = response.json() + assert data["intent"] == "sql" + assert data["tool_used"] == "safe_sql_engine" + assert data["tool_output"] is not None + assert data["tool_output"]["row_count"] == 1 + assert "user_count" in data["tool_output"]["columns"] + + +def test_agent_chat_dispatches_analytics_guidance(): + response = client.post( + "/api/v1/agent/chat", + json={"message": "How do you profile a dataset and clean null values?"} + ) + assert response.status_code == 200 + data = response.json() + assert data["intent"] == "analytics" + assert data["tool_used"] == "data_analytics_pipeline" + assert "Analytics" in data["reply"] diff --git a/tests/test_rag.py b/tests/test_rag.py new file mode 100644 index 0000000..831c425 --- /dev/null +++ b/tests/test_rag.py @@ -0,0 +1,69 @@ +import io +from fastapi.testclient import TestClient +from main import app + +client = TestClient(app) + +SAMPLE_DOC_TEXT = """# DataMind AI Architecture and Enterprise Security Overview + +DataMind AI is an enterprise-grade intelligent assistant. The system uses FastAPI for asynchronous API throughput and PostgreSQL for transactional data. + +## Security Architecture +DataMind AI operates under a strict Zero-Trust principle. All Text-to-SQL operations are executed in Read-Only mode to prevent data corruption. No destructive DDL or DML queries such as DROP or DELETE are ever allowed. + +## Document Intelligence +The platform integrates semantic text chunking and vector storage with ChromaDB. Answers generated by the assistant must explicitly cite the document name and page number to eliminate hallucination. +""" + + +def test_rag_upload_and_index_document(): + doc_file = io.BytesIO(SAMPLE_DOC_TEXT.encode("utf-8")) + response = client.post( + "/api/v1/rag/upload", + files={"file": ("security_policy.md", doc_file, "text/markdown")} + ) + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["document"]["document_name"] == "security_policy.md" + assert data["document"]["total_chunks"] >= 1 + assert "document_id" in data["document"] + + +def test_rag_list_documents(): + response = client.get("/api/v1/rag/documents") + assert response.status_code == 200 + docs = response.json() + assert isinstance(docs, list) + assert len(docs) >= 1 + assert any(d["document_name"] == "security_policy.md" for d in docs) + + +def test_rag_query_with_source_citation(): + # Ask question related to security + response = client.post( + "/api/v1/rag/query", + json={"question": "What is the security architecture principle and what queries are forbidden?"} + ) + assert response.status_code == 200 + data = response.json() + assert data["retrieved_chunks_count"] >= 1 + assert len(data["citations"]) >= 1 + assert "security_policy.md" in data["citations"][0]["document_name"] + assert "page_number" in data["citations"][0] + assert data["citations"][0]["relevance_percentage"] > 0 + assert "Zero-Trust" in data["answer"] or "Read-Only" in data["answer"] or "security_policy.md" in data["answer"] + + +def test_rag_delete_document(): + # Get doc ID + list_res = client.get("/api/v1/rag/documents") + doc_id = list_res.json()[0]["document_id"] + + del_res = client.delete(f"/api/v1/rag/documents/{doc_id}") + assert del_res.status_code == 200 + assert del_res.json()["success"] is True + + # Verify deletion + after_res = client.get("/api/v1/rag/documents") + assert not any(d["document_id"] == doc_id for d in after_res.json())