From 33986d022f1f7e1c3ba9bd2241ddbfffc1e2df9e Mon Sep 17 00:00:00 2001 From: diusazzad Date: Fri, 11 Sep 2026 07:02:08 +0600 Subject: [PATCH] Save work --- .github/workflows/test.yml | 30 + .gitignore | 68 ++ CONTRIBUTING.md | 200 +++++- README.md | 285 ++++++-- __pycache__/main.cpython-313.pyc | Bin 2923 -> 0 bytes app/__pycache__/__init__.cpython-313.pyc | Bin 235 -> 0 bytes app/api/__pycache__/router.cpython-313.pyc | Bin 462 -> 0 bytes app/api/router.py | 6 + .../v1/__pycache__/analytics.cpython-313.pyc | Bin 3085 -> 0 bytes app/api/v1/__pycache__/sql.cpython-313.pyc | Bin 2883 -> 0 bytes app/api/v1/agent.py | 24 + app/api/v1/rag.py | 61 ++ app/core/__pycache__/config.cpython-313.pyc | Bin 2117 -> 0 bytes app/core/__pycache__/database.cpython-313.pyc | Bin 1742 -> 0 bytes .../__pycache__/schemas.cpython-313.pyc | Bin 3612 -> 0 bytes app/models/schemas.py | 57 ++ .../__pycache__/data_cleaner.cpython-313.pyc | Bin 8587 -> 0 bytes .../__pycache__/sql_engine.cpython-313.pyc | Bin 5624 -> 0 bytes app/services/agent_service.py | 129 ++++ app/services/document_parser.py | 152 ++++ app/services/rag_engine.py | 252 +++++++ app/services/sql_engine.py | 13 +- requirements.txt | 3 + site/sitemap.xml.gz | Bin 127 -> 127 bytes static/css/style.css | 672 ++++++++++++++---- templates/index.html | 545 ++++++++++---- tests/__pycache__/__init__.cpython-313.pyc | Bin 136 -> 0 bytes ...est_analytics.cpython-313-pytest-9.1.1.pyc | Bin 7609 -> 0 bytes .../test_health.cpython-313-pytest-9.1.1.pyc | Bin 5292 -> 0 bytes .../test_sql.cpython-313-pytest-9.1.1.pyc | Bin 6909 -> 0 bytes tests/test_agent.py | 43 ++ tests/test_rag.py | 69 ++ 32 files changed, 2256 insertions(+), 353 deletions(-) create mode 100644 .github/workflows/test.yml create mode 100644 .gitignore delete mode 100644 __pycache__/main.cpython-313.pyc delete mode 100644 app/__pycache__/__init__.cpython-313.pyc delete mode 100644 app/api/__pycache__/router.cpython-313.pyc delete mode 100644 app/api/v1/__pycache__/analytics.cpython-313.pyc delete mode 100644 app/api/v1/__pycache__/sql.cpython-313.pyc create mode 100644 app/api/v1/agent.py create mode 100644 app/api/v1/rag.py delete mode 100644 app/core/__pycache__/config.cpython-313.pyc delete mode 100644 app/core/__pycache__/database.cpython-313.pyc delete mode 100644 app/models/__pycache__/schemas.cpython-313.pyc delete mode 100644 app/services/__pycache__/data_cleaner.cpython-313.pyc delete mode 100644 app/services/__pycache__/sql_engine.cpython-313.pyc create mode 100644 app/services/agent_service.py create mode 100644 app/services/document_parser.py create mode 100644 app/services/rag_engine.py delete mode 100644 tests/__pycache__/__init__.cpython-313.pyc delete mode 100644 tests/__pycache__/test_analytics.cpython-313-pytest-9.1.1.pyc delete mode 100644 tests/__pycache__/test_health.cpython-313-pytest-9.1.1.pyc delete mode 100644 tests/__pycache__/test_sql.cpython-313-pytest-9.1.1.pyc create mode 100644 tests/test_agent.py create mode 100644 tests/test_rag.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..ac6dcbc --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,30 @@ +name: Automated Test Suite + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test: + name: Run Pytest + runs-on: ubuntu-latest + + steps: + - name: 🚚 Checkout Code + uses: actions/checkout@v4 + + - name: 🐍 Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: πŸ“¦ Install Dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: πŸ§ͺ Run Pytest + run: | + pytest -v diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4a9e572 --- /dev/null +++ b/.gitignore @@ -0,0 +1,68 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual Environments +venv/ +env/ +ENV/ +env.bak/ +venv.bak/ + +# Local databases +*.db +*.sqlite +*.sqlite3 +datamind_dev.db + +# Environment variables & secrets +.env +.env.local +.env.*.local + +# Pytest / Unit test & coverage reports +.pytest_cache/ +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover + +# IDEs and editors +.vscode/ +.idea/ +*.swp +*.swo + +# OS generated files +.DS_Store +Thumbs.db diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c2e879a..1bee752 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,44 +1,192 @@ -# Contributing to DataMind AI +# 🀝 Contributing to DataMind AI -First off, thank you for considering contributing to DataMind AI! It's people like you that make DataMind AI such a great tool. +Thank you for your interest in contributing to **DataMind AI**! Whether you are fixing a bug, designing a new feature, improving documentation, or adding automated tests, your help is deeply appreciated. -## 1. Where do I go from here? +--- -If you've noticed a bug or have a feature request, make sure to check our [Issues](https://github.com/YOUR_USERNAME/DataMind_AI/issues) page to see if someone else has already created a ticket. If not, go ahead and [make one](https://github.com/YOUR_USERNAME/DataMind_AI/issues/new)! +## πŸ“‹ Table of Contents +1. [Code of Conduct](#-code-of-conduct) +2. [Getting Started & Local Environment Setup](#-getting-started--local-environment-setup) +3. [Git Branching Guidelines](#-git-branching-guidelines) +4. [Development Standards & Best Practices](#-development-standards--best-practices) +5. [Automated Testing](#-automated-testing) +6. [Commit Message Conventions](#-commit-message-conventions) +7. [Submitting a Pull Request (PR)](#-submitting-a-pull-request-pr) +8. [Community & Questions](#-community--questions) -## 2. Fork & create a branch +--- -If this is something you think you can fix, then fork DataMind AI and create a branch with a descriptive name. +## πŸ“œ Code of Conduct +We are committed to providing a welcoming, diverse, and harassment-free environment. Please treat all maintainers and fellow contributors with respect, empathy, and professional courtesy. See [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) for full details. -A good branch name would be (where issue #325 is the ticket you're working on): +--- -```sh -git checkout -b 325-add-text-to-sql-tool +## πŸš€ Getting Started & Local Environment Setup + +### 1. Fork the Repository +Click the **Fork** button at the top-right of the [DataMind AI GitHub Repository](https://github.com/diusazzad/DataMind_AI) to create your own copy under your GitHub account. + +### 2. Clone Your Fork +Clone your fork to your local development machine: +```bash +git clone https://github.com/YOUR_GITHUB_USERNAME/DataMind_AI.git +cd DataMind_AI +``` + +### 3. Set Up Upstream Remote +Configure Git to track the official repository as `upstream`: +```bash +git remote add upstream https://github.com/diusazzad/DataMind_AI.git +git fetch upstream +``` + +### 4. Create a Virtual Environment +We recommend Python 3.10 through 3.13: + +**On Windows (PowerShell):** +```powershell +python -m venv venv +.\venv\Scripts\Activate.ps1 +``` + +**On Linux / macOS:** +```bash +python3 -m venv venv +source venv/bin/activate +``` + +### 5. Install Dependencies +```bash +pip install --upgrade pip +pip install -r requirements.txt +``` + +### 6. Configure Environment Variables +Copy the template `.env.example`: +```bash +cp .env.example .env +``` +*(By default, DataMind AI operates seamlessly with zero configuration by automatically initializing a local SQLite database at `sqlite:///./datamind_dev.db`)* + +### 7. Run the Local Development Server +```bash +uvicorn main:app --reload --port 8000 +``` +- 🌐 Web Dashboard: [http://127.0.0.1:8000/](http://127.0.0.1:8000/) +- πŸ“˜ Swagger UI: [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs) +- πŸ“— ReDoc: [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc) + +--- + +## 🌿 Git Branching Guidelines + +Never commit directly to the `main` branch. Always create a dedicated branch for your work branching off the latest `upstream/main`: + +```bash +git checkout main +git pull upstream main +git checkout -b / ``` -## 3. Implementation Guidelines +### Branch Naming Conventions: +Use clear, prefix-based branch names: -- Ensure you have activated your virtual environment before installing dependencies. -- Follow PEP 8 guidelines for Python code formatting. -- Make sure to add docstrings to any new functions or classes. -- Update `requirements.txt` if you introduce any new dependencies. +| Branch Type | Purpose | Example | +| :--- | :--- | :--- | +| `feature/` | New features or engine expansions | `feature/langchain-react-memory` | +| `fix/` | Bug fixes or guardrail patches | `fix/sqlite-timeout-exception` | +| `docs/` | Documentation, PRD, or guide updates | `docs/api-curl-examples` | +| `test/` | Adding or refactoring unit/integration tests | `test/rag-citation-edge-cases` | +| `refactor/` | Code structure improvements without feature changes | `refactor/sql-engine-service` | +| `chore/` | Tooling, dependencies, or GitHub Actions CI | `chore/update-fastapi-version` | -## 4. Make a Pull Request +--- -At this point, you should switch back to your master branch and make sure it's up to date with DataMind AI's master branch: +## πŸ› οΈ Development Standards & Best Practices + +1. **Architecture Layering:** + - **Routes & Controllers:** Keep inside `app/api/v1/`. Routes should strictly validate requests and delegate heavy logic to services. + - **Services & Core Engines:** Business logic, RAG pipelines, data profiling, and SQL executors reside inside `app/services/`. + - **Schemas:** All DTOs, request payloads, and response structures must be defined with Pydantic in `app/models/schemas.py`. +2. **Zero-Trust Security Principles:** + - Any feature dealing with database execution must strictly adhere to the Read-Only validation rules in `app/services/sql_engine.py`. + - Never allow unbounded memory allocations or un-sanitized file uploads. +3. **PEP 8 & Formatting:** + - Code must follow standard PEP 8 formatting rules. + - Use meaningful variable and function names. + - Include docstrings for public classes and service methods. + +--- + +## πŸ§ͺ Automated Testing + +We enforce a strict testing policy. All pull requests must pass the automated test suite. + +### Running Tests Locally: +```bash +pytest tests -v +``` + +### Adding New Tests: +- If you add a new endpoint or service, create corresponding tests in the `tests/` directory (e.g., `tests/test_new_feature.py`). +- Use `fastapi.testclient.TestClient` for HTTP API testing. +- Verify both the **happy path** and **error edge cases** (such as blocked unauthorized SQL queries or invalid file formats). + +--- + +## πŸ’¬ Commit Message Conventions + +We follow the **Conventional Commits** specification: -```sh -git remote add upstream git@github.com:YOUR_USERNAME/DataMind_AI.git -git checkout master -git pull upstream master ``` +(): +``` + +### Allowed Types: +- `feat`: A new feature (e.g., `feat(agent): add multi-turn conversation memory`) +- `fix`: A bug fix (e.g., `fix(sql): prevent multi-statement injection bypass`) +- `docs`: Documentation changes only (e.g., `docs(readme): add docker deployment section`) +- `test`: Adding or correcting tests (e.g., `test(analytics): add excel null imputation test`) +- `refactor`: A code change that neither fixes a bug nor adds a feature +- `chore`: Updates to build tasks, package manager configs, etc. + +--- -Then update your feature branch from your local copy of master, and push it! +## πŸ“¬ Submitting a Pull Request (PR) + +### 1. Rebase from Upstream Main +Before submitting, ensure your branch is cleanly rebased on top of the latest `upstream/main`: +```bash +git checkout main +git pull upstream main +git checkout your-feature-branch +git rebase main +``` -```sh -git checkout 325-add-text-to-sql-tool -git rebase master -git push --set-upstream origin 325-add-text-to-sql-tool +### 2. Push to Your Fork +```bash +git push -u origin your-feature-branch ``` -Finally, go to GitHub and make a Pull Request. πŸŽ‰ +### 3. Open the Pull Request on GitHub +1. Navigate to [https://github.com/diusazzad/DataMind_AI](https://github.com/diusazzad/DataMind_AI). +2. Click the green **Compare & pull request** button. +3. Fill in the **PR Template** accurately: + - Reference the Issue number it resolves (e.g., `Fixes #12`). + - Describe what changed and why. + - Confirm all local automated tests passed. + - Provide screenshots if your changes affect the Web UI (`templates/index.html` or `static/css/style.css`). +4. Click **Create Pull Request**. + +### 4. Review Process +- The automated GitHub Actions CI pipeline will automatically run `pytest` against your code. +- Maintainers will review your PR, suggest improvements if needed, and merge it upon approval. + +--- + +## 🌟 Community & Questions + +- **Found a bug?** Open an issue on our [Issue Tracker](https://github.com/diusazzad/DataMind_AI/issues). +- **Have an idea or architectural suggestion?** Start a discussion or open a draft PR. + +Thank you for helping make **DataMind AI** the premier open-source Intelligent Data & Document Assistant! πŸš€ diff --git a/README.md b/README.md index 33c19dc..7b4aaab 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,256 @@
-

πŸš€ DataMind AI

-

Intelligent Data & Document Assistant

-

DataMind AI is an open-source AI platform that empowers users to analyze tabular data, chat with their databases via Text-to-SQL, and extract insights from documents using RAG (Retrieval-Augmented Generation).

+ DataMind AI Logo +

🧠 DataMind AI

+

Enterprise Intelligent Data & Document Assistant

+

An open-source, production-ready AI platform bridging relational databases, unstructured documents, and analytical pipelines via Natural Language Text-to-SQL, source-cited RAG, and autonomous ReAct agents.

- [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) - [![FastAPI](https://img.shields.io/badge/FastAPI-005571?style=flat&logo=fastapi)](https://fastapi.tiangolo.com) - [![Python](https://img.shields.io/badge/Python-3776AB?style=flat&logo=python&logoColor=white)](https://www.python.org/) +

+ Live Docs + CI Tests + MIT License +

+ +

+ FastAPI + Python Versions + Zero Trust + RAG +

-## 🌟 Features +--- + +## ⚑ Live Demos & Portals + +| Resource | URL | Description | +| :--- | :--- | :--- | +| 🌐 **Live Documentation Portal** | [https://datamindai.zengfy.top/](https://datamindai.zengfy.top/) | Official documentation site deployed via CI/CD. | +| πŸš€ **Local Interactive Dashboard** | [http://127.0.0.1:8000/](http://127.0.0.1:8000/) | Glassmorphic web UI with 4 live interactive test consoles. | +| πŸ“˜ **Swagger UI Interactive API** | [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs) | OpenAPI interactive documentation and testing suite. | +| πŸ“— **ReDoc Specification** | [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc) | Clean, responsive API specification for developers. | +| ⚑ **Telemetry Health Check** | [http://127.0.0.1:8000/api/health](http://127.0.0.1:8000/api/health) | Real-time database connection and version telemetry. | + +--- + +## 🌟 The 4 Architectural Pillars + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ DataMind AI ReAct Agent β”‚ + β”‚ POST /api/v1/agent/chat β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ Autonomous Intent Routing + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β–Ό β–Ό β–Ό + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ Safe Text-to-SQL β”‚ β”‚ Document RAG β”‚ β”‚ Tabular Analytics β”‚ + β”‚ Zero-Trust Engine β”‚ β”‚ Semantic Citationsβ”‚ β”‚ Profiler & Clean β”‚ + β”‚ Read-Only Queries β”‚ β”‚ Verified Page # β”‚ β”‚ Mean/Median Mode β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### 1. πŸ€– Autonomous ReAct AI Agent +An intelligent multi-tool orchestrator. When users ask questions in natural language, the agent autonomously identifies whether to: +- Inspect schema and execute a safe SQL database query. +- Retrieve information across indexed PDF documents with source page citations. +- Provide data profiling and statistical guidance on tabular datasets. +- Offer contextual reasoning and system explanations. + +### 2. πŸ—„οΈ Zero-Trust Safe Text-to-SQL +- **Zero-Trust Guardrails:** Strict AST and regular-expression filtering blocks destructive commands: `DROP`, `DELETE`, `UPDATE`, `INSERT`, `ALTER`, `TRUNCATE`, `GRANT`, `REVOKE`, `EXEC`. +- **Safe Bounded Returns:** Enforces a maximum 100-row fetch threshold to avoid memory denial-of-service. +- **Dynamic Introspection:** Inspects database tables and columns in real-time. + +### 3. πŸ“Š Automated Tabular Data Analytics +- **Ingestion:** Direct upload of CSV and Excel (`.xlsx`, `.xls`) files. +- **Statistical Summaries:** Computes column types, total counts, null percentages, uniqueness, mean, median, standard deviation, min, max, and correlation matrix. +- **Auto-Cleaning:** Automated duplicate row elimination and intelligent numeric/categorical null imputation (Mean, Median, Mode). + +### 4. πŸ“‘ Source-Cited Document Intelligence (RAG) +- **High-Fidelity Parsing:** Extracts page-indexed text from PDF and Markdown files. +- **Semantic Vector Space:** Chunks text into overlapping semantic segments indexed via local statistical vector embeddings (zero mandatory paid API keys required). +- **Anti-Hallucination Citations:** Answers explicitly cite the primary document title, page number, and similarity relevance score. + +--- + +## πŸš€ Quickstart in 60 Seconds + +### 1. Clone the Repository +```bash +git clone https://github.com/diusazzad/DataMind_AI.git +cd DataMind_AI +``` + +### 2. Create & Activate Virtual Environment +**On Windows (PowerShell):** +```powershell +python -m venv venv +.\venv\Scripts\Activate.ps1 +``` + +**On Linux / macOS:** +```bash +python3 -m venv venv +source venv/bin/activate +``` -- **Data Analytics Pipeline:** Upload CSV/Excel files and instantly get cleaned data and statistical insights. -- **Text-to-SQL (AI Agents):** Ask questions in natural language, and the AI will generate and execute SQL queries on your PostgreSQL database securely. -- **Document Intelligence (RAG):** Upload PDFs and chat with your documents. Get highly accurate answers backed by source citations to prevent hallucinations. -- **Modern API Backend:** Built on high-performance FastAPI. +### 3. Install Dependencies +```bash +pip install --upgrade pip +pip install -r requirements.txt +``` -## πŸ› οΈ Tech Stack +### 4. Configure Environment +```bash +cp .env.example .env +``` +*(No database setup required! DataMind AI automatically initializes a local zero-config SQLite database at `sqlite:///./datamind_dev.db`)* -- **Backend:** Python, FastAPI, SQLAlchemy -- **Data Science:** Pandas, Numpy, Scikit-Learn -- **AI & LLM:** OpenAI, LangChain, ChromaDB (Vector Store) -- **Database:** PostgreSQL +### 5. Launch Server +```bash +uvicorn main:app --reload --port 8000 +``` +Visit **[http://127.0.0.1:8000](http://127.0.0.1:8000)** to explore the interactive glassmorphic web dashboard! -## πŸš€ Getting Started +--- -### Prerequisites -- Python 3.10+ -- PostgreSQL -- OpenAI API Key +## πŸ§ͺ Comprehensive Test Suite -### Installation +Run the full automated test suite with verbose telemetry: +```bash +pytest tests -v +``` -1. **Fork and Clone the repository:** - ```bash - git clone https://github.com/YOUR_USERNAME/DataMind_AI.git - cd DataMind_AI - ``` +``` +tests/test_agent.py::test_agent_chat_reasoning_and_capabilities PASSED [ 7%] +tests/test_agent.py::test_agent_chat_dispatches_sql_query PASSED [ 14%] +tests/test_agent.py::test_agent_chat_dispatches_analytics_guidance PASSED [ 21%] +tests/test_analytics.py::test_profile_csv_upload PASSED [ 28%] +tests/test_analytics.py::test_clean_csv_upload PASSED [ 35%] +tests/test_health.py::test_home_portal_loads_html PASSED [ 42%] +tests/test_health.py::test_api_health_check PASSED [ 50%] +tests/test_rag.py::test_rag_upload_and_index_document PASSED [ 57%] +tests/test_rag.py::test_rag_list_documents PASSED [ 64%] +tests/test_rag.py::test_rag_query_with_source_citation PASSED [ 71%] +tests/test_rag.py::test_rag_delete_document PASSED [ 78%] +tests/test_sql.py::test_sql_schema_endpoint PASSED [ 85%] +tests/test_sql.py::test_sql_safe_query_execution PASSED [ 92%] +tests/test_sql.py::test_sql_forbidden_operation_blocked PASSED [100%] -2. **Create a virtual environment:** - ```bash - python -m venv venv - source venv/bin/activate # On Windows use: venv\Scripts\activate - ``` +======================== 14 passed in 2.23s ======================== +``` -3. **Install dependencies:** - ```bash - pip install -r requirements.txt - ``` +--- -4. **Environment Variables:** - Copy the `.env.example` file to `.env` and fill in your details: - ```bash - cp .env.example .env - ``` +## πŸ“‘ API Reference & Curl Examples -5. **Run the server:** - ```bash - uvicorn main:app --reload - ``` - Access the API documentation at `http://127.0.0.1:8000/docs`. +### 1. Autonomous AI Agent +```bash +curl -X POST http://127.0.0.1:8000/api/v1/agent/chat \ + -H "Content-Type: application/json" \ + -d '{"message": "What is our enterprise security policy regarding database operations?"}' +``` + +### 2. Execute Safe SQL Query +```bash +curl -X POST http://127.0.0.1:8000/api/v1/sql/query \ + -H "Content-Type: application/json" \ + -d '{"query_text": "SELECT 101 AS order_id, 450.75 AS total, \"Delivered\" AS status;"}' +``` + +### 3. Introspect Database Schema +```bash +curl -X GET http://127.0.0.1:8000/api/v1/sql/schema +``` + +### 4. Profile Tabular Dataset (CSV/Excel) +```bash +curl -X POST http://127.0.0.1:8000/api/v1/analytics/profile \ + -F "file=@your_dataset.csv" +``` + +### 5. Automated Data Cleaning & Imputation +```bash +curl -X POST "http://127.0.0.1:8000/api/v1/analytics/clean?drop_duplicates=true&impute_numeric=median&impute_categorical=mode" \ + -F "file=@your_dataset.csv" +``` + +### 6. Upload & Index Document (PDF/Markdown) +```bash +curl -X POST http://127.0.0.1:8000/api/v1/rag/upload \ + -F "file=@company_policy.pdf" +``` + +### 7. Ask Document Intelligence (RAG) +```bash +curl -X POST http://127.0.0.1:8000/api/v1/rag/query \ + -H "Content-Type: application/json" \ + -d '{"question": "What queries are forbidden under the security policy?", "top_k": 3}' +``` + +--- + +## πŸ›‘οΈ Security Architecture + +| Vector | Security Guardrail | Enforcement Mechanism | +| :--- | :--- | :--- | +| **SQL Injection & Mutation** | Strict Read-Only Policy | Regex & AST inspection rejects all DDL/DML mutation keywords (`DROP`, `DELETE`, `UPDATE`, `INSERT`, `ALTER`, `TRUNCATE`, `GRANT`, `REVOKE`, `EXEC`). | +| **Denial of Service (DoS)** | Bounded Row Execution | Hard query limit of 100 rows per query prevents database memory exhaustion. | +| **AI Hallucination** | Source Attribution | RAG engine strictly links synthesized statements to document titles, verified page numbers, and cosine similarity relevance metrics. | +| **File Upload Safety** | Extension & Type Validation | Whitelisted parsing for `.csv`, `.xlsx`, `.xls`, `.pdf`, `.txt`, and `.md`. | + +--- + +## πŸ“‚ Project Structure + +``` +DataMind_AI/ +β”œβ”€β”€ .github/ +β”‚ └── workflows/ +β”‚ β”œβ”€β”€ deploy.yml # FTP cPanel auto-deployment +β”‚ └── test.yml # Automated GitHub Actions Pytest CI +β”œβ”€β”€ app/ +β”‚ β”œβ”€β”€ api/ +β”‚ β”‚ β”œβ”€β”€ v1/ +β”‚ β”‚ β”‚ β”œβ”€β”€ agent.py # Unified Autonomous AI Agent +β”‚ β”‚ β”‚ β”œβ”€β”€ analytics.py # Tabular profiling & cleaning +β”‚ β”‚ β”‚ β”œβ”€β”€ sql.py # Safe Text-to-SQL endpoints +β”‚ β”‚ β”‚ └── rag.py # PDF vector indexing & Q&A +β”‚ β”‚ └── router.py # Central v1 router +β”‚ β”œβ”€β”€ core/ +β”‚ β”‚ β”œβ”€β”€ config.py # Pydantic BaseSettings & .env loader +β”‚ β”‚ └── database.py # SQLAlchemy engine & SQLite fallback +β”‚ β”œβ”€β”€ models/ +β”‚ β”‚ └── schemas.py # Pydantic DTO validation models +β”‚ └── services/ +β”‚ β”œβ”€β”€ agent_service.py # ReAct autonomous tool router +β”‚ β”œβ”€β”€ data_cleaner.py # Pandas automated cleaning engine +β”‚ β”œβ”€β”€ document_parser.py # PDF & text chunker with page tracking +β”‚ β”œβ”€β”€ rag_engine.py # Vector space indexer & citation engine +β”‚ └── sql_engine.py # Safe read-only SQL executor +β”œβ”€β”€ static/ +β”‚ └── css/ +β”‚ └── style.css # Glassmorphism design system & animations +β”œβ”€β”€ templates/ +β”‚ └── index.html # Interactive playground dashboard +β”œβ”€β”€ tests/ # 14 automated unit tests +β”œβ”€β”€ docs/ # MkDocs markdown documentation +β”œβ”€β”€ main.py # FastAPI entrypoint +β”œβ”€β”€ requirements.txt # Production dependencies +β”œβ”€β”€ CONTRIBUTING.md # Git branching, PR & commit guide +└── README.md # Project documentation +``` + +--- ## 🀝 Contributing -We welcome contributions! Please see our [CONTRIBUTING.md](CONTRIBUTING.md) for details on how to submit pull requests, report issues, and our coding standards. +We welcome developers, researchers, and data enthusiasts to join us! Please check out our **[Contribution Guide (CONTRIBUTING.md)](CONTRIBUTING.md)** for detailed instructions on: +- Git branching conventions (`feature/`, `fix/`, `docs/`) +- Setting up your local development environment +- Commit message standards (Conventional Commits) +- Submitting Pull Requests and getting merged + +--- ## πŸ“œ License -This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. +Distributed under the **MIT License**. See [`LICENSE`](LICENSE) for complete terms. diff --git a/__pycache__/main.cpython-313.pyc b/__pycache__/main.cpython-313.pyc deleted file mode 100644 index ffdd91f0a74cec5ca3ae71826877a1a16d3d2ccd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2923 zcmb_e&2JmW6`$pj%a0`~ilR(Pl#`VuJECNfh~ofN(Atr0sUaE30#`|ZiUF}A$K=|} zU3PXUODs^xKmohx#W|$NF-0#idhyZ69&_x%2uKX17D3WOZxqx&pl_C3QZd?F7vRiq z-n@D9zK7?LNC?6A<>sFs{w{#f4{YIn!67(5^Z*Bvk<1lQfg_y6u&3xL@PsdTiPu^4 zMPET6f`h$9e<45u4)zs;g%AljSSX5xFbO-@UyKx@BDo?-94KymPx+;$6_lK~Q9n)MyQe_^_M@u(c%6OJVpx+#?T_lI)h#Xz> zk#sp-T63x6xh%*NIZlo_BSVvGKF~7^_gscwN{+9J2)P=|Q)?V~3nbkhOK_2#D2GO; zrr$gzPs^9qc#xculWdPjWvO)6Jv*LV`C(nn zvWw(&IUvu0rI*)u`K_||rbm8z7CrR3HP?u^dvs#FnM%b+$cKZQ$yoN^QSV zHFOX=)womUK_C6zQ6D zPWY`t!ps%a?He661Bx{r~U&>Oa=msxzZ*Y z4LbY|5lTHqnRbYclRYfyhjwZGmMs2igV5J7qY0$dhk6=wD zV7Wn#I{q9e3*Mu`UTOs}ts^!cF+Eq=4&`#4&}VpG z+A)ZwXb{6F8Ahg3)fCgDUQ4N))K3Nhtq$wl$qDF6h=2O3GU z4L>BIg%7QaKvHZSbJNyMM8cG%f%To8%J%x!1{E3lbfYBa?{8CaWBb#5>HhZC#`bQ8 zbHtj=!4kc^^FjHSs?})kmsx@`11jtJa#K_ujNOiA5T#!h8Ul5cmDD|T|_B)!m0=j?Ru%0S2FN z4pLA@%t86&Ap4-GXmA&67!s2l{Ukw}Mh$C30`@99<9?&G_#;ptEk-Hhwig}kjS}P& zv@|@Yz*8o3_c_^fA=LlpG=U`K*L+TrY?QFoCb}sRuyNH=n^-b2IZ&%u8d!Z@8kY5S zNrCjKK^XQGXdZ6cIZ&;mEcFkVkD&H(oj|SDph-Bj*fz-mnC2u9Sp-Dmt_-(z0kc5h zId*MZ^7V%KO(&j&mV7` zh%4)2hCzRz)(7=8Iik8Qb=*kfjP9hRnKi*ZO~rIvOQUQ9 zOvnyi0}^4f1Cp;BS(%`~(FOobM_QMyF~ch=>N%DsGP91qHa3=mYSRmMhB-n7V| z^nFdE%}%nmT7d>l1$+oEiAni7R1PapCsZ`5>Q${>!^4N^1#-MTh-~2JpnO}yx5yUo zv%)cd1!JG*IPMgs{(%-x(T!8Ie2P|1QTCr~`R*xNJw;33ixbZSf4J2Z=WTKRwYbm~ zmu+$RIPkp?>4oCmP}&Zqd$G-4IMEGDc3A2~620gp2)*BP%=hDn^LA0(M)96E-kZAg z{A%yg>|kb#zKFP~GbC`Hvp9;({i*uR(#t)2`NP-O*6r|}<6sZ*PlHc_&oW=^y?F1N z!&ggo?#}C#yY}?GZ+~_iw9#%KdQ#%~16!Os4)l2O>Fp=CyL`gt6Fsm0Y5qz6S^bOO zypUg}{_^=>6Lxa*+s}{lws)@|Kpx-ISWe0*7IQE_H|UVQvYhR+~7Z>a`Y#kgmdWRxbwz^sXP^o&U?D2R!V&&Sf)r~q-r{r& z@C?c?ElDi`a@dN%tSWZ>#DYxyGDDD1K~ZX2W`&<7=PmZM#Nra5L=iJk>n)MQf&x7t zPp`~SFEKAMr?Mn7xwr`AyITTqemum$TYN~0iwkpZaRCXK952w2_%cHnmoGCfIj1xw z70O@9@EPRNTj~K;G47cq8Kp@vE{P?HzL|L`@s6G`Ky!gOGX`RhUP0w84x8Nkl+v73 zyCMOgK2WF>n*xas%#4hTcNt{wGFabb(0?c(xj^!wfOZ4#0|wsf3^ErPWG=|+USZI? Q0preyy2_wY#0S&~04*7HQ~&?~ diff --git a/app/api/router.py b/app/api/router.py index 9b41a3c..55395f2 100644 --- a/app/api/router.py +++ b/app/api/router.py @@ -1,8 +1,14 @@ from fastapi import APIRouter from app.api.v1.analytics import router as analytics_router from app.api.v1.sql import router as sql_router +from app.api.v1.rag import router as rag_router +from app.api.v1.agent import router as agent_router api_v1_router = APIRouter(prefix="/api/v1") api_v1_router.include_router(analytics_router) api_v1_router.include_router(sql_router) +api_v1_router.include_router(rag_router) +api_v1_router.include_router(agent_router) + + diff --git a/app/api/v1/__pycache__/analytics.cpython-313.pyc b/app/api/v1/__pycache__/analytics.cpython-313.pyc deleted file mode 100644 index abff4d562bba4729b7e0e3a28c166a7c74ca5809..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3085 zcmchZO>7&-6@X`dNG>Unq9{tSLP_4phGU{J1;bJlIg$`cjcw7WB6CGq774H`awx4q z?y|E>)gpRulS6e&iXdo?0SXk!t%(c71qu{afTBmz3Sf4iA_;nOPZXp>f%MQfORhvI zKgTYxvomkr&+NQ6-|nH`??Z5n-}wFgl!VZ~*uc5$85kV?8=4Q0f)wsHN^pcr@PtnY zL`aB4l$o5r?Mk?bJK-UoghV7;CfxQWe8eZC3^$4tQE`oNihGJDU74=rpcCTcsdzGy zB2Dob9<;p^Tx({WQ+ye5oL9PDMEAw}+(g@e_n%oO#Qc(mzUmrki%t5Gj!@pm5YX& zXH!vu27jtr>U5Lr!*Y_JV()L@cULi!MBuj zzQew|!?4{2d-+H0g_Lj+NoeusFQIh6vG*YL974L-xhLsfXEr4szni^Fil~cmvB0H6 zjt1CAz~0qi4+Y%e!R2@eplj{D1~0C-tH2e|jBp+mxJjYFMZML{RyE41Jg=BVe=I^&+5$DF0(!}${B>=T}Lv6vMz zn}cek?mV_st++^7da9A-O)OXJ(}dd|HB-i=vb7z1e#F8>Zh$t-SqAfIzZ6@miLL8$ z)JG*JGGS#%MS|5l6<{%hl?wH;ahB=O;0D$)Q7xQp`=P!SwOGM#5@HZ4m=>Yl<0Bq* z6P9Y~$r-wZb&K-(0`)kC{1&DhHr=2h+ZVmEbUkwuwxY70VJG%ynx4yA?$yTE>Vb4kN`KoFn9hIAH{ey? z9-ANHKIP^@;%CFqdlHOub78dG%fnzd?3*7Hc1QZ=t_r(X{j5JbFdr7a2)m&F#h?H> zRrgqq@pYNHOE|BoIu#k&XiuIPric9e+L2*q9y1T>c_oHsb!D0o@ zU->~wveA|}gEodHZwH#(il+lj$6x7FX!6FrXV4^7)A5RBlz<|z@xUKDiGY`r6W>0{ z^7^(+a0!s1Z^$S4Y~hQgl*vn)X|gBKr-IEG3Bm?cM^4e`JM*!{VBMfW{^ToY+oMEY}06nv^6kGyw~0%tX(x%Wb49#NWss?ut^X@C#aD58Xg?UiwYs*O8sI+U3_DMrz#jKdwza79LFp-;rb4`!k VmC)eOk*&vkPaW_~N?wI^{||N>;LQL4 diff --git a/app/api/v1/__pycache__/sql.cpython-313.pyc b/app/api/v1/__pycache__/sql.cpython-313.pyc deleted file mode 100644 index 1f15f5dd5470afcc27d4b4520533aca9e2131c41..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2883 zcmbtWTWlLy89p<|Gxj*X#px!+0XiF2-MGqy&1FNfPSjl7HkZxC6Zf*Vu%`A&I*T)Q z&y2H)A0P`41RkI-6@6U9ODZ9;2qY@O1GGHw=p>aQ4Mi33)VHi0p+aBy&rHT~DxTP zGTY;M&zwMoIgyI?-ZSr=^HJZNL?wGK%=_nBXp4-pLKjj*#oNy*zEPeAvVm0R=`Bq! zMauHoP!?xH0-O%4>S46<)G37syLKvuGOgIV7YuY_XgV^2e-b)Acc zUog(aIB$7ow7h9~7Bs_zoiw>Tn<`aILapGn)z#(XR-Tkit)v4jtlTZGR0!Qpk-HUQ zm{#QMz$lk=gTy)eEUB++I#CGS)bgb2={N2ctI<`mW%in-US(xLb{IgVSf*r6+p;{S zx^7gx{YL%{*-$}Eh!RI-ZlhEn#eDN8`e1|Msec=$`{=7E)1apK#KYIy4%^ft

O)@H8hSdz&kEf7gEHmP~DK# zVo^5L+eHFPy&#)+h(=}h86;Pfh(&q z#dd?N>5gzom)BISND5=}L*8mJOx3IyxjZ<=5(~rx(qdeo%y6r1otQaSUCwc*B{iZ* zMaF~;lY);qVz3XCt#_BlvR5_pPUUuX1}ynd(+j!D*{oVF!=q(4-^)S-`^wui3heg8 z%Eh<>(+)Zov5?`Og)bb12M@x7hvDIxZ}{c2mUDH)F+!+w57*vbe!8++OJ6@+xlvob zSsS{wANr`~`{>m%W<38gBB1KK=Q_|&{0aQi_7VIWA%Ux6KcpGmw!GBIxBO{1fES@g z5?q3ie^b1V(kNvUUY}AdQQjDCA#8{lR#Wy!i$k2kL7Djap|DrD40Dt{QtYNTMlbB! z8foRGiO!|D?o&=RhFD^rX|Vz3bv0dRRdi)#SnGJHr%8f2`;Bpy%DShC^V}P3uw}Z_ zp5fDS(D7@wwErjmGSEn3lfRl9Ppf5T5!LOcUJKkPJAL zhLpM153c0?8(t|W&4r+&4B}P3iMN4Vjyu>qe45^y=)NGM58p$m1@2nsMlfI#U*)s6 z7vtFCDFB5*RGPnI`Dd>#E~S#wlS-0N`o3=Q-}=OosQO9Hc7(w+%JL3nNn%Nfb)u84 zGM%vAA&%Q}fCUa~U!qvbt3_j?ufg>t5Hf~fdijDu1Hv)!Z_)Jb)CopX%a5fWO7~I^ z_#rVVVBj0GvH)=)G|1{VKes5%q1H(imxsaS*wLCXwSFc&#++$<+r~mXF}((1Qy-0g;jmqLCWZr zrPS2y%uI4IH=n$9V<|PGSm8~zr~!JCoT07}bKCM?SBn*rWEySgq^#6z1hmdG&rq8e z9Kt|7=j4PHbUkHXPuoBw?xh{@vcj5?19uj+yk;^-TKp!l1e*^cWTQTPi(PR6CWAF5 z*&dgjaJ0Ib8!?~|a>y%S7Qrr&Un8(s2_bl+`qd?C=pl9#a)4&MR8T)|%N}^py zt-}Yc!-uUSyW+Emd=wcyh>Sjp|Ni=6WMNl0_6Tk-=sgJc?vcarP|Y{=@}vz#V$ctm z=IDjJ>EGc;Q@{1@M=#d`mst;h^H(S5P@tXljB&eV{CfHk|EsI}p+wD>c=bPalLYwV z4}tcnZuB_%)^rd(?)Fdn@#BHEX#qbG1Xw=_`e&l}Nk`lCCH&-4kgZ4BXM*^TK>^nB zC^Cu)o}iV zfDV~L?4Khi2Sw`qYGLTwO{wK6mmphNWFMkB%*ctpF9Fw7ie!R*0A~eXov6J#zAsJeh!8>E_x;$XvEA9v3VVZ}dp=iwDeX<{ zUmV+s)wmD7-~}$j)iL4|$M*6#!uUHs?cBXtyZHW2XN?;<=3~I~;5&81-i-GfU)V=< H42<|U(5#ac diff --git a/app/api/v1/agent.py b/app/api/v1/agent.py new file mode 100644 index 0000000..4a69e81 --- /dev/null +++ b/app/api/v1/agent.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter, HTTPException, status +from app.models.schemas import AgentChatRequest, AgentChatResponse +from app.services.agent_service import agent_service + +router = APIRouter(prefix="/agent", tags=["AI Agent"]) + + +@router.post("/chat", response_model=AgentChatResponse) +def agent_chat(request: AgentChatRequest): + """ + Unified Autonomous ReAct AI Agent Endpoint. + Analyzes user intent and automatically coordinates: + - Safe SQL execution for database queries + - RAG Document Intelligence for policy / document queries + - Tabular Data Analytics for dataset profiling questions + - Contextual reasoning for interactive guidance + """ + try: + return agent_service.process_chat(request) + except Exception as e: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Agent processing error: {str(e)}" + ) diff --git a/app/api/v1/rag.py b/app/api/v1/rag.py new file mode 100644 index 0000000..f2eeaf2 --- /dev/null +++ b/app/api/v1/rag.py @@ -0,0 +1,61 @@ +from typing import List +from fastapi import APIRouter, File, HTTPException, UploadFile +from app.models.schemas import DocumentInfo, RagQueryRequest, RagQueryResponse, RagUploadResponse +from app.services.rag_engine import RagEngineService + +router = APIRouter(prefix="/rag", tags=["Document Intelligence (RAG)"]) + + +@router.post("/upload", response_model=RagUploadResponse) +async def upload_and_index_document(file: UploadFile = File(...)): + """Upload a PDF, TXT, or Markdown document to parse, chunk, and index into the semantic vector store.""" + if not file.filename: + raise HTTPException(status_code=400, detail="A valid file with a filename must be provided.") + + content = await file.read() + if len(content) == 0: + raise HTTPException(status_code=400, detail="The uploaded document is empty.") + + try: + doc_info = RagEngineService.index_document(content, file.filename) + return RagUploadResponse( + success=True, + document=doc_info, + message=f"Successfully indexed '{file.filename}' into {doc_info.total_chunks} semantic chunks across {doc_info.total_pages} pages." + ) + except ValueError as ve: + raise HTTPException(status_code=400, detail=str(ve)) + except Exception as e: + raise HTTPException(status_code=500, detail=f"Failed to process document: {str(e)}") + + +@router.get("/documents", response_model=List[DocumentInfo]) +def list_indexed_documents(): + """Returns metadata for all documents currently indexed in the vector store.""" + return RagEngineService.list_documents() + + +@router.delete("/documents/{document_id}") +def delete_indexed_document(document_id: str): + """Deletes a document and all its corresponding chunks from the vector store.""" + deleted = RagEngineService.delete_document(document_id) + if not deleted: + raise HTTPException(status_code=404, detail=f"Document with ID '{document_id}' not found.") + return {"success": True, "message": f"Document '{document_id}' successfully removed from vector store."} + + +@router.post("/query", response_model=RagQueryResponse) +def query_document_knowledge_base(request: RagQueryRequest): + """Answers questions based on indexed documents with exact source citations (file name and page numbers).""" + question = request.question.strip() + if not question: + raise HTTPException(status_code=400, detail="Question cannot be empty.") + + try: + return RagEngineService.query( + question=question, + doc_id=request.document_id, + top_k=request.top_k + ) + except Exception as e: + raise HTTPException(status_code=500, detail=f"RAG query execution error: {str(e)}") diff --git a/app/core/__pycache__/config.cpython-313.pyc b/app/core/__pycache__/config.cpython-313.pyc deleted file mode 100644 index 940c3dc9b3046739d000697bfff4c61c9be5015f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2117 zcmZuy%}*Og6rZ)%YlDsXu(A2ZfT36|CMiTpX+&dUH#p!Ac}<#B+l&@_aW+}jW_Cfv zN-OtNq+XIkC3%Z%G)+pN;G zt`^KfdkHoPZO|O!XiySBoCuIj|&@Jxg`%lZiYOlhgtS8Selqh6Yd{Lq3N5sGp%hh5~3n@<|SwMY3|R zE%U!28)C9yG|0S07#c+ZhQ=5gM?tpt1Vz#iN%Eu^5~sxJkL+kz8WCqC$FfZfkAMgK zd7?+fSu|D(MCP!olgpPhh^jj-n{rCki=kK&yJFcaOvh4*s^fC>QuGpbQ^D;-PE2Of z*j-c(6s=s@Rdf@(6Y1N@Tqd1Lr1R?aNdPdP&E)cV?1(4Uwr=7stz3|``(?w#&iEQ! z&xm>T>L4+rh+1M-g(yo%F_>seOyn@YZmNxm;%Av$oNP#9+aqYD$@B`@aP>l2NG zXk2d?x7oH2Ya(xJiAEqk3+0=Zp6Xv2wC4sk%)*-;9ndZ0iW&o=2#a35)Svh$vz?=}dNeGHS3 z8hGLEJ#~+f&+-LNec=;dxb6wR==GiUPLl7Z<D zj`gRpr^VCg>Pd99zVOL!d(ZrTjQu|L%==ufXY;37=_D&X|GNIgy*ezM!iOjDVf~&~ z@7=AtcAI2h@C%5HV;6+FysJQnyCK{y7xy*Fdm!A~m$kO07m_kYhN-E#qL(2=UBnyP z4HIFal6Ado%CtipL`5&RmE;~~r0OQ}QjXs)m$f&N&hCI+2~B_NhG^9z`pnP>C8Ly# zQ$kx3_K@%t4YpEG+5{?B6l5Nv2vrV3XwWDMP&Q4ZZtt544d(;u{XQkM?2)rW=z_+d zgnSDu|5R=LsLkG#*5AjQcBxMOmlWGTQ%AMEd~CGfNV7oh#l>A$DDB wRmm#)n0V0FV$2Y7X6HEWFEIKySbZ(baXNSO=ry48s~iVB{_j@4aj|&*1G4}KivR!s diff --git a/app/core/__pycache__/database.cpython-313.pyc b/app/core/__pycache__/database.cpython-313.pyc deleted file mode 100644 index 0e364e8267cdc6d52c28ef25d53d41ba08ed88ce..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1742 zcmah}%WoS+7@yg%@S5$!D(wi$6fq(b=q)?0;@^C#1sA2uD^CZ?A)DC7#nTyJ2&$px4bNbRsa?R- z3ZW_`m#W0|GN&03UF?<|(r~9ZK}x4w=fDFBOTG&FAP4%#1^Kp8@dz)zB}2!S@f;=9?TZ#v9Std&-YW@VatM z`5^fFyIe>4%ViaMH8hJq6jJEp{#kr$V21yipT&Csmm!G(*iztlHvrfbFgoqOfaQ?Q zU}g3ASX=3l3@s~rT^56mb@iX0$P~p59$G%=J zJh(85o2{mQEZyqfVVkE56j>nlE?LeVJS#W)$-F`b)3;z@tY6#&SAypSrjsB5FAo@@if?Q2{7)G(-jq0X&`((Xuu@=53 t@4F)>YeG|!aQvgd&8Ih}zesHFpQy(&j}W+;12RtBJo77pfu`Xoe*>ZVj`#oo diff --git a/app/models/__pycache__/schemas.cpython-313.pyc b/app/models/__pycache__/schemas.cpython-313.pyc deleted file mode 100644 index eb0991ab32b6018d58eb7be47d00008fa034a1a7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3612 zcma)8&u`mC7N#VMlqgZM<=A!YxK@668?{Lav~jyl3m1ulY@*n58>VRs5Flu2qOhPy zWk|YqP6iBYo11bd;A4Sa+<&0wp7tC=K%mSehdueWfYp27o6#@3YOwW$149SSylgcp~E5~WPoS=!aOywft@q5X# zLY1;g)uJ>jotKQnoMgzRvYZfe&=g0LfGQ(snxiV9sS#A;Xc|y$1kG?X188;x&2m%+ zG&h3k9L)ndF@ommlcmBDE~q54#0_tU$+w(p$mBIA2w7^Q89KgayQMfw-?Rg=?$?OR zlD8e=)~ez&MC^T&bbJTyf0Rs##tezZjTlWVNv3QjjriNp!&`|#OCGe6yrsZ61zQH} zT0K=tv)mnGyW!*eBxw3xK#p+#nG%F{7zHe4H=CAcHwaU=2n}GREJM64hx%TFcp)p) z?9kqXWm{F>^GG!$wO^!4DaBG2f6TI&W?2ot7P$o4S<8A7*)C29cX`Wtj|cqPU;)DSQYV|=yVuE0?yYz9iM_SX zXcyrjKA*#NLx=&dd+S>3m#{PJjUNOkefgsNsE1{YQ>Vj&vB$WX^fA7Vb0& z%Y?v&3k*lVxaLJ}zn$mpCZSc(f?X#p8+p!?2*m(HD@<+>+e21sEYWbF)v&jj9@vd0 zsA0=?BNA`|OOvz!Z>3Wh&S03vFoWR@2rS{io0E@R-wx>#qHt(P^dA`TQz&|aUcykq zfZIXe#DMY;+UXA5n4;ndyYx!v{Fwf)5Nk6+5?#42zB zCnSU)g1_T`L--pBp)`c6hZ0iFRFC++kka6O?OGhsUQgfa88Zu_kojL1BTkG|U zuCA6eHiZuA31VTyijn-d6V&$GkpzPTmtW6YaG*8umc0GI=Bx3n9djWDjiPm)J$8R>FATK zD~Ea!S%v1sRoW|T_;H+nh`ku_ zy90axI0y4?v>pJ(ujUpHQZH7%Je+&DTWPJf<%h4{x^Qs&x7^`d8!zOU{X|>d=oByR zmRqU)sptAj`J&(#Cp^I~>JWd&E`-zkjwIr7mPJgM6sfjzi--pL`!15HbAwXUoC`F(!)QAOr z{uZgx4`6mqTBVd7%i55zpT+58-?1)y(|<>WeE64`;pt zm4n{D8mD)B=)_JA^iHAY&pQb^7KP_Hy<=@7^bVyq-_^eAQR&Jlt@GN)%)m2!)n~ct zCx$!UD^jn+Qs^oKs4kwkP?K&qVf%S?E6!gZh z_{7O7P^~FYl7VGKBIHKHZRJ8>spd}2_Mk?lDq>tU8n$@;fE5W5Zk%^se-R(+n>7G?$ wpY4~r5?s0kIkwc==}K_v=G55v{d8A?OZQA1aHlK5rK_nh&(80FFYq<|0aKefkN^Mx diff --git a/app/models/schemas.py b/app/models/schemas.py index e4414f0..324a0f4 100644 --- a/app/models/schemas.py +++ b/app/models/schemas.py @@ -63,3 +63,60 @@ class SqlQueryResponse(BaseModel): columns: List[str] results: List[Dict[str, Any]] execution_time_ms: float + + +class DocumentInfo(BaseModel): + document_id: str + document_name: str + total_pages: int + total_chunks: int + file_size_kb: float + created_at: str + + +class RagUploadResponse(BaseModel): + success: bool + document: DocumentInfo + message: str + + +class RagQueryRequest(BaseModel): + question: str = Field(..., description="Natural language question to ask your documents") + document_id: Optional[str] = Field(default=None, description="Optional document ID filter") + top_k: int = Field(default=4, description="Number of most relevant context chunks to retrieve") + + +class RagCitation(BaseModel): + document_name: str + page_number: int + relevance_percentage: float + snippet: str + + +class RagQueryResponse(BaseModel): + question: str + answer: str + citations: List[RagCitation] + retrieved_chunks_count: int + latency_ms: float + + +class ChatMessage(BaseModel): + role: str = Field(default="user", description="'user', 'assistant', or 'system'") + content: str + + +class AgentChatRequest(BaseModel): + message: str = Field(..., description="User query or instruction") + history: List[ChatMessage] = Field(default_factory=list, description="Previous conversation turns") + + +class AgentChatResponse(BaseModel): + reply: str + intent: str + tool_used: Optional[str] = None + tool_output: Optional[Dict[str, Any]] = None + citations: Optional[List[RagCitation]] = None + latency_ms: float + + diff --git a/app/services/__pycache__/data_cleaner.cpython-313.pyc b/app/services/__pycache__/data_cleaner.cpython-313.pyc deleted file mode 100644 index 999b92ab6c2d45d2314fe3e3d19cc18e1ff2fc5d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8587 zcmb_iZ)_V!b|3zy_(zmTNfarXA|=WWtv{43DUM_-K3jIINOt6Ky|x@FW@vI{Ga;AE zEGhpB9l%6b0h;3e@+dg%@==@PeGIR zQ{U`zNm)^xG}i$&^LFOVn>TM}-u~XKHtlu`f)sk`qc`_8BlMr-K~4HHvH1oNA0P@* znhAtSsh!Y_Yq54*hjrt6tRFXELqKUaVw22g#x+ptCM@GtY#p~@+qfOu$7^wI05zjF zMCrYVG7M^!5wPQ;CTx669$pFSCEa26s-!=fO5`Q|i4>QY45tgZGy^U6(QLYqVNc=g zY%0x2bw}g*_-LAmvy-`eD$8t^(ncE&*IW{i}6#uTmF+D)McjFiEP-qa74n{dsHg);8UTb5&mKBlMo*q+hH zJY$EkYj>77;~3ByU}TV3^OVFo%7W`JTEjNUS+NL%DF)A_63k;ebdhYtgt0`H%`-X7UBhCdz9g&&vq84*oWOt05+$HBl(HC)EV3mexiI_qIf72FtykE4Rcu z5%QyHb)TUq8E0E+L(^upDk|?)^+_q)5RFv1Y8gV`p@ClIsA#YkZ5_Y%Y1!oI$BjQ- zuS%i}w5AqO#(teTHlqomX`9+^Lh&wy0$a8C6RkjLDcz8k)3%}znnfeJRy3;#o36cZ zDvmjZ3ye}{x&o(3mc@Z%R}xG*K-QiMq}Y5m5D%P+v&lFYAlv383>%^+ti7i1NpN$J z=da`?V>+Hsv3+5UWMBxT`&*(}b)^TUB_ zyLtkr;J|WB0FHDvo+Rp26zK`qNQQLwGJ}ciBzHNLza(jMNy&_vcrpe$NZM3ZG99^^ zXSkOqB`3|MKm+C|L-J!7XR&0FSs5}f$#N#1E-3Y39VUy59gs;Dd5jp@?i|^ufj~go zcWQX%IE+`gFhiz(GQ}oihhLtF=W;Wga%{MnBs9bl#NRNuCwEn78$k#> zSHnkw=pTapqi)^rn>3J1+T?6v>-i>JB8Y|a5Z&rq%8V{_ zTs!vf2(iYgCgRdq(=;jMM4O@}m9Cbmzlo|2yxN4+HucQ0=4q?iOzSAk@ZTU52boiU z-VODXwj0Kp)KEG~ufCJmny1DmE^}L=43u%G;?r2mG}&8n0)2OQRaA7bQYO|0e|Gq* zg+Ir%Q|%dTP^oGeZK~2L74@i8wWLkz2o>-4YlJ|TmMV`*srF&(=(?y+t*w@-gyj3+ zA1judZdEy;S5T#^r7Bg`gECjFX`60W`RKYzdyTvf(DEH)bf|r*<#&%k){nJ6rQ@Dz zKb6|vpRIkWM_A>nmfz~(nEc{LtXOANqDt+sVyC}%#p>z$XRg>jux529)<=Js4Yv09 zVKzATJv+-uJD;iN3s33k{2_W$Cd#5muwqqhozWklZMmNVH11YyUo9!Cx*wyFYOPAC ztM5W*T%0z7IIRnfO@n`OZP^38DtEO6T;fvYs;4OqevdLu4o~;1e6(wNK&7M4t5mg| z_N(-ORy|7cyL?A^*}lUqGerS_EeB zb+nnTp)ItHw$rt=gKnT~Llx@Kcv5ny8M^t&$a?5;Qtk$YjL_^t!G;9;y z^vu2UA)Jz?sM={uN!5^-2Br8F@Lj-lcuRAcRM#|G@oZHd@L3w=iFMQNo!?Yrufh5^ z_aeHHZYIYI_>NR1s1)oIsAWCSdQ`2emb8a%R=_*!{obAARdrI=KKb<8SK5gWBw7(G z&>o`ONf?`~`pFrkgjLz;4AiP4M2}Z>RH-d%Hm259OUl$+c?(7-s`V;GIjA~V)0XLz zY7MQ0ousD?bPL@$qN&UYP6tGhCaQkO7=m*afIrqXi^9%pLc z#{{@MAW<%#N^o6)Op43aCu{ie%^F8HjLWa|Ew)pJ*gg zicO>oN#-$$cq9`BfQyr)!I;G1YS%~ ziKL6?Qj&ofgp=x0T#PMb7%+28wh<<;Z^^)9a`~&0VHV(i9urR~=^)g^eMAT`FDBkl zazk&x*GxW@VV)m|#o4Qh5J`Utu)k!?XVc^=Ko82rB})nx8&7h@K< zVahQG!Eow|q)qg}-#$qPD&RxF$`J$zLC|+5OkP7$-9-ih9~iwd-}W7%p6yJ#$N-n& zM9INV$DTW5x5s#M5Q3jb-@M}yL@&c149}028r|2=&cC$o@hzTNetqfnTQPB0FTbl- z*tJja^v{p4yM6a94_^I!?c$rffAF5}bN_+j>9fUiuL}g#)2oNX&O?0XA))iI&~fDcVWIVrFR(*L`CBAYVVYHY?VT@bx35!0tQt+xCwf3-*#bB)a#)Z`~hU z=@#4i`L=$cZBXz(Ph@_p>YGRY&~j!eb1S=GExFr7_jB+ge4Te9w<7}d^$)C3YZr?C zA*KIL^&4j74J@}TwXD2)%P)F*cu&tmPw%Ed*HZf*p5BGSU+IviW!dyg(~4%Tcue+jpBEo{twVB!u(H&+C|f_4IBSkiB^W zfv61&vin3^8*gh{uiHg-1a+&qJBp3Nu=yMw(a{ROm2ojNz=sCb zUWGNe7ZgGVO5V_NWGNzeJ4J6F@9kR)3f_VFiBeNgYzjYY3NL6&?HvoY#VB7FD0w?X zZ|6gA=fV*nNw?Lu)eEb;q1(df`pBWr&i?J$`_yMMpU&JJTyxzWe!wl9SaI?Vp_Qo6 z@Z6>WwRAl(Aot!48-jzd=lY(-wi}(Ivx9ebe2nkR-JV;`udsr1P;`#)&XN1)iq4Uu zb8_DFrK7FX9DfPwb{T%LX#lSpNHu$24F~j<~`-(y&Ij-y>RYODd9xLrkz`7KVy!6X4z*Fke+RZQ7|{H;g5ZS-G{V z4oqnYXpXM{-ZDfhaY?m(x^fc`byjOt3h=IONV8Ma59l!uqnf9th_Rn(aknb1TBXO;{CTJx<*}(si{5C zc5<$aOtsNUv{UUu8D!m-f3$t!j^;b;%l-e`w>)bC7Ph5|{VcR4+P19`9xE#adk8o} z-{zn+WG4W^H?jX0>lPI2ZaKe(XYQsy_550QIzg-rP6t`+ZK)b(83G$K?WrMiG29bxAHqv&8G~xbj>imEXgn81A#n9#jb=!deB<8Z%5X6=yM7Wu-=+fh%c}3}32cB0jTz?=;V(`;p6X3u?BEfK+WJ|HBdDcaE0w4Ay2H@Q>snL zk(LTO@fp&ABH3AzDPaPjb9t5+HW!kN5Q(G#l{?E_i3%4XPDuc~gtLt>2?_zZ1JcHm zErszDBZlA(JeZv-mAC&*UtAPp-JYzy}3}sEyws z73AwxT87OAVs;My6l(E1BqR8id_PdW26DthZ-coCxV6Z(CLu^%+O-3$tv07Y1Xl+R z^=IfSBWiA0+;`*hylowbBg-!@y?kq8d2(q|@O6v6KHk?S`1*zBfq6S1k(+0)pS^Kz z-tuUdzi9TBtWMF|20sFDUc1#gKVI^*i=KYo(=T`i;VIa8H?g+&*I7O|QndNu>fh!P ztwG)zEOzV{toz~RFS;YVJMwqV&pe-c?pcfONYQ;%bf0+WKCz%HxprMYB)Y=9D=fG= zMOTD(MMPJh;OZ;6{i3^@cXzKf3GV%+_U_w<#P3;M=!w ztTZt6yNTaTEEblpEnQm;{_>~y&n=AYs7DRa<0~5i^#7*lkMRD8;O`UtgAgGI{{5o= z0PjB__(ufqi}Q|kw|8;)#-9Nuboy>au18kd1ZRin?B<=_01oFqnOn=R0@NB2oyU0R zu?GOPjuo9}0cyEIrG{pJSL6q1%hy?I?E}E|#9?l*%$qkFP@T8vXfK-EH(_I-Pv8@_ zQ!>ZouS>C*WQoOKOBNuwk4d&z?CnAvj!K3+IbyNd6z1~j6w9z#(kx$SB;cmvQZ^|W zE|9NW_&CT^V(l8TE9FmenB2pY2r$lZ3b!1p;qI|CKUlP{FSWJ-}i|3Uq z5%3W){MUr*uSiB-yqF*ZIkN-7eFWKGppv0}Lu)f!(5x6X5GmJON_o%uneS8Ird+qF zLq?~AKyufe3xCT`7x4=uNqTbSd*~%xGo0%MgMSbYkhm1_H9! zKPs0G3=b~+OZpG#;!LLaN`@a}H%Y79eNzjRqCZhRT!-O57{DFDbp&_|OaLp^lZ?DK zWs^t-h>3GoWq&3abMlu?#kfwmH|Qb0BhB@2iA&&GIr$DjF(>g*iiu?#J#9D1d{9~8 z7x9O{N6<4zURUrDO{4h&)%+U@3n=^ra(;naUuz8-?Iwck>$($KP3Vt^-S6&(%VkIK^d5izr6D&$e(-BLAjaVbB8Km}`^ps~>&`ZpL ziUMo{Idg>NY2LD*;n^*HcB3!ifIjP%J}30qV4VHYIbdFPzlC>fk$3SSdLMZm6OZ@KN@}; zzlG!~ny&UuS6(4P2P$ncwGYXrFcl)Z_|Ri)W<5IDjgU-(6#I|}RKv8Ps~(+U6w_X$ zm|z4*rUE=g)Sfbp!&W#Dhh($NoC}5NE@&O^(c8kNP8h+#PiAJ3&y+hdEN5^^b0CGw z2s4tXaDp7?Vks#*FLSam4-K<8HP2<{L{5lhk{3lTE`X$t^bf6$-1bkFyr*vY zCzq7P{-3;3^iSqb6uhU_p4fBcP|4YT&x~68*Da{ln|FQll^HdjrWEqduGb9KSyUUb z`~nHriCEQjpqIFT=3u`DHv$KPs{Vu?l&Z7=eo#EP*aK1qUKzt6=jnsY^&(c?QeUT#W9X5rw88VwWAZI1msu~YM*gRg=`iSh#Z~NUP>-8f4YF zB03s8e|9ny-~kUr0mtT4`x!yXib0G4BkW2>z!~KdM840A+u?0v0&1!)CW(R^g}71K z?~4nUQaG-%X#p(2vKCkxYT!;H>%J&RNdN*dstB`UWqo8m}FH z`*@*s-^$LFT}hVvM~eL;H~Y)|p<;iiFdZ*Xr;5|5LVBS*ohwe~ z?$osvwueh~X9~_U>okf`uTTY$bN6n*I?xtaApxnuiUc8d6+uu}0*6Q%Pna^?nnRoC zS?Dp4vJ-6r#593y?Fktq!)SnfQ_@7vIXVP^^zbw7GFmn*Ff0LBse(h8c^n8BmQ+WW zL9X7=xS8|S)ybBy1x{4}sQ-|Nq_Ju&Ec|m#2H-=-?SclkDKl)5K-9 z=WG>@0E#USw404Rfp#|RfYjQ@%FgLlV@0c$^`I*Bf=A5)+t>kc&7p~lN9Y}Bmhw4r=ODPXVARQm4uu3nIR5mQ1}O_1 z2~%mI2N07`uB4nG^du_@Sfd{rNvT38H!l)^Rs473u#So3Kftl?!Ye_!XZUOk09cvL zN>$7ish%culLDoQ7wc*qyU_M!3Sl~)XWwEGarHrC(s~R57BZ%jKaaJEGVx3k28`MEI?6dNHH9YNTmTGae$QUq9mjhF%G8VW6+?G zc-7F>NOaV7Rjk!{joSVQ=5FaHexvk&)4Ya&CE1*1YirTkx;T8>y8CYp&E z0S?zUm+L!<^&Lw?rTRVjz&$H!yC9Za(TW&Apf-k)wJKQL*CZ+0_C>eTW!6iw!JIe#kRw%OtCev zDwG;d=Ivm;`vi4Ex%u=TF0ab#hze8QJKpBSu6Lv>FD?x%=a!}G9qTmZdKN(1=DBZ2 zwcMJgX|1+-t*N6TxaMgk|LuG3xvjPKb!4^Kzu9O&+m3v5pEeBn#I^lD%0Ae&-w!$h zyXcQuPoUlYacfPWsqSN+m6V@ip#4UZ1%pj6j9FjC$I*-M=BigDQ=Jl-U(xs9sNh`KGR=a6V9+thyT@>`eR`9ak4i7TUJ&#t0p*OE~3>?wQBEsvIajud;2To+0` z&p~~4v>XT*1L4A%v!%efe;4LcK)AQoI&7R`>yUxN41R zUm~JW)gFz)Wd|G`%FZa9NrFVaQ9#WdjcRTp!G|+B6^-IP7_B;DlAtKSE#^{j%#aou z?k|{t4&Fr)a%O91D?Up~2}sn2Xq32mG8To42u{vqGok_tlSF{nq#cZya9%ATC$sW9 zNPdLAGSMxT0^?n`Bg?jR6KnP^1(xfV<12k1B$wNZozH9_XxpHT-t{`I|vl2>G)w0=|*X)zB`%OFGY>-$&f)v22uADv&V zDGp5PjfJVR8(Ob6bi+gh^)->@0Ef}S;W(64OJ*SroDGw((zZ_9HMq%LkOYZ*@mbKn z8EGtL8+^d=1J@G$1`z0^r;fCf>#3yp^-nj7zT4XFZQW7TdRTwAM{ySBB!R6EP=!;L kqCP{m&yeE_w6lnIet|kaC;xlDwmeT!L)2GD%d~a;7qL7xxc~qF diff --git a/app/services/agent_service.py b/app/services/agent_service.py new file mode 100644 index 0000000..fb1e569 --- /dev/null +++ b/app/services/agent_service.py @@ -0,0 +1,129 @@ +import time +import re +from typing import Any, Dict, List, Optional +from app.services.sql_engine import sql_engine +from app.services.rag_engine import rag_engine +from app.models.schemas import ( + AgentChatRequest, + AgentChatResponse, + RagCitation, +) + + +class AgentService: + """ + Autonomous ReAct AI Agent capable of tool routing across: + 1. SQL Database Inspection & Execution + 2. RAG Document Intelligence with Source Citations + 3. Tabular Data Analytics Guidance + 4. General Natural Language Reasoning + """ + + def __init__(self): + pass + + def _detect_intent(self, message: str) -> str: + msg = message.strip().lower() + + # Check for explicit or implicit SQL requests + if msg.startswith("select ") or "from " in msg or "database" in msg or "table" in msg or "sql" in msg: + return "sql" + + # Check for data analytics / statistics keywords + if any(w in msg for w in ["profile", "dataset", "dataframe", "correlation", "null count", "clean data", "impute", "mean", "median"]): + return "analytics" + + # Check for document / policy / manual keywords or question forms + if any(w in msg for w in ["document", "policy", "pdf", "manual", "security", "architecture", "what is", "explain", "how to"]): + # If documents exist in RAG engine, RAG is primary candidate + if rag_engine.list_documents(): + return "rag" + else: + return "reasoning" + + return "reasoning" + + def process_chat(self, request: AgentChatRequest) -> AgentChatResponse: + start_time = time.time() + user_msg = request.message.strip() + intent = self._detect_intent(user_msg) + + tool_used = None + tool_output: Optional[Dict[str, Any]] = None + citations: Optional[List[RagCitation]] = None + reply = "" + + if intent == "sql": + tool_used = "safe_sql_engine" + try: + # If it looks like a natural language question about tables, generate safe SQL + if not user_msg.lower().startswith("select"): + sql_query = sql_engine.generate_sql_from_prompt(user_msg) + else: + sql_query = user_msg + + res = sql_engine.execute_query(sql_query) + tool_output = { + "sql": res.sql, + "row_count": res.row_count, + "columns": res.columns, + "results": res.results[:10] # top 10 preview + } + reply = ( + f"I queried the database using the safe SQL protocol:\n" + f"`{res.sql}`\n\n" + f"**Result Summary:** Retrieved {res.row_count} row(s).\n" + f"{res.explanation}" + ) + except Exception as e: + reply = f"I attempted to query the database, but encountered a security or execution notice: {str(e)}" + tool_output = {"error": str(e)} + + elif intent == "rag": + tool_used = "rag_document_intelligence" + rag_res = rag_engine.query_documents(user_msg, top_k=3) + tool_output = { + "retrieved_chunks": rag_res.retrieved_chunks_count, + "citations_count": len(rag_res.citations) + } + citations = rag_res.citations + reply = rag_res.answer + + elif intent == "analytics": + tool_used = "data_analytics_pipeline" + reply = ( + "DataMind AI's automated Data Analytics pipeline can ingest your CSV or Excel files, " + "calculate full descriptive statistics (mean, median, standard deviation, skewness), " + "detect missing values, generate correlation matrices, and handle null value imputation. " + "You can test this anytime in the 'Analytics Profiler' tab or via `POST /api/v1/analytics/profile`." + ) + tool_output = { + "supported_formats": ["CSV", "Excel (.xlsx)"], + "features": ["Descriptive Statistics", "Null Imputation", "Correlation Matrix", "Duplicate Removal"] + } + + else: + intent = "reasoning" + # Conversational / Assistant guidance + docs_count = len(rag_engine.list_documents()) + reply = ( + f"Hello! I am DataMind AI β€” your unified Enterprise Intelligent Data & Document Assistant.\n\n" + f"Here is what I can do for you autonomously:\n" + f"1. **πŸ—„οΈ Database & Text-to-SQL:** Ask natural language questions or run read-only SQL queries with built-in zero-trust security guardrails.\n" + f"2. **πŸ“‘ Document Intelligence (RAG):** Ask questions across uploaded PDF and markdown documents with verified document & page citations (Currently {docs_count} indexed).\n" + f"3. **πŸ“Š Tabular Data Analytics:** Ingest and profile datasets with automated null-imputation and distribution metrics.\n\n" + f"Feel free to ask a database query, upload a policy document, or ask me anything!" + ) + + latency = round((time.time() - start_time) * 1000, 2) + return AgentChatResponse( + reply=reply, + intent=intent, + tool_used=tool_used, + tool_output=tool_output, + citations=citations, + latency_ms=latency + ) + + +agent_service = AgentService() diff --git a/app/services/document_parser.py b/app/services/document_parser.py new file mode 100644 index 0000000..e67aeb7 --- /dev/null +++ b/app/services/document_parser.py @@ -0,0 +1,152 @@ +import io +import re +from typing import Dict, List, NamedTuple +import pypdf + + +class ParsedPage(NamedTuple): + page_number: int + text: str + + +class DocumentChunk(NamedTuple): + chunk_id: str + document_id: str + document_name: str + page_number: int + chunk_index: int + content: str + + +class DocumentParserService: + @staticmethod + def extract_text_pages(file_content: bytes, filename: str) -> List[ParsedPage]: + """Extracts text page by page from PDF or plain text files.""" + lower_name = filename.lower() + pages: List[ParsedPage] = [] + + if lower_name.endswith('.pdf'): + try: + reader = pypdf.PdfReader(io.BytesIO(file_content)) + for i, page in enumerate(reader.pages): + extracted = page.extract_text() or "" + cleaned = re.sub(r'\s+', ' ', extracted).strip() + if cleaned: + pages.append(ParsedPage(page_number=i + 1, text=cleaned)) + except Exception as e: + raise ValueError(f"Failed to parse PDF document '{filename}': {str(e)}") + elif lower_name.endswith(('.txt', '.md', '.markdown', '.json', '.csv')): + try: + text_content = file_content.decode('utf-8') + except UnicodeDecodeError: + text_content = file_content.decode('latin1', errors='ignore') + + # Break large text into ~1500 char virtual pages + paragraphs = text_content.split('\n\n') + current_page_text = [] + current_len = 0 + page_idx = 1 + + for p in paragraphs: + cleaned_p = p.strip() + if not cleaned_p: + continue + current_page_text.append(cleaned_p) + current_len += len(cleaned_p) + if current_len >= 1500: + pages.append(ParsedPage(page_number=page_idx, text="\n\n".join(current_page_text))) + page_idx += 1 + current_page_text = [] + current_len = 0 + + if current_page_text: + pages.append(ParsedPage(page_number=page_idx, text="\n\n".join(current_page_text))) + else: + raise ValueError(f"Unsupported document format: '{filename}'. Supported: .pdf, .txt, .md") + + if not pages: + raise ValueError(f"No readable text could be extracted from '{filename}'.") + + return pages + + @classmethod + def chunk_pages( + cls, + pages: List[ParsedPage], + document_id: str, + document_name: str, + chunk_size: int = 600, + chunk_overlap: int = 100 + ) -> List[DocumentChunk]: + """Splits document pages into semantically bounded overlapping chunks.""" + chunks: List[DocumentChunk] = [] + global_chunk_idx = 0 + + for page in pages: + page_text = page.text + if len(page_text) <= chunk_size: + chunks.append( + DocumentChunk( + chunk_id=f"{document_id}_p{page.page_number}_c{global_chunk_idx}", + document_id=document_id, + document_name=document_name, + page_number=page.page_number, + chunk_index=global_chunk_idx, + content=page_text + ) + ) + global_chunk_idx += 1 + continue + + # Split by sentences or paragraph boundaries + sentences = re.split(r'(?<=[.!?])\s+', page_text) + current_chunk = [] + current_length = 0 + + for sentence in sentences: + sentence_len = len(sentence) + if current_length + sentence_len > chunk_size and current_chunk: + chunk_str = " ".join(current_chunk).strip() + chunks.append( + DocumentChunk( + chunk_id=f"{document_id}_p{page.page_number}_c{global_chunk_idx}", + document_id=document_id, + document_name=document_name, + page_number=page.page_number, + chunk_index=global_chunk_idx, + content=chunk_str + ) + ) + global_chunk_idx += 1 + + # Retain overlap sentences + overlap_chunk = [] + overlap_len = 0 + for prev in reversed(current_chunk): + if overlap_len + len(prev) <= chunk_overlap: + overlap_chunk.insert(0, prev) + overlap_len += len(prev) + else: + break + + current_chunk = overlap_chunk + current_length = sum(len(s) for s in current_chunk) + + current_chunk.append(sentence) + current_length += sentence_len + + if current_chunk: + chunk_str = " ".join(current_chunk).strip() + chunks.append( + DocumentChunk( + chunk_id=f"{document_id}_p{page.page_number}_c{global_chunk_idx}", + document_id=document_id, + document_name=document_name, + page_number=page.page_number, + chunk_index=global_chunk_idx, + content=chunk_str + ) + ) + global_chunk_idx += 1 + + return chunks diff --git a/app/services/rag_engine.py b/app/services/rag_engine.py new file mode 100644 index 0000000..f5162d1 --- /dev/null +++ b/app/services/rag_engine.py @@ -0,0 +1,252 @@ +import datetime +import math +import re +import time +import uuid +from typing import Dict, List, Optional, Tuple +import numpy as np + +from app.models.schemas import DocumentInfo, RagCitation, RagQueryResponse +from app.services.document_parser import DocumentChunk, DocumentParserService + + +class LocalTfidfVectorizer: + """Lightweight, zero-dependency statistical vectorizer for semantic retrieval.""" + def __init__(self): + self.vocabulary: Dict[str, int] = {} + self.idf: Dict[str, float] = {} + + def _tokenize(self, text: str) -> List[str]: + words = re.findall(r'\b[a-zA-Z0-9_-]{2,}\b', text.lower()) + return words + + def fit_transform(self, corpus: List[str]) -> np.ndarray: + # Build vocabulary + doc_tokens = [self._tokenize(doc) for doc in corpus] + all_tokens = set(token for tokens in doc_tokens for token in tokens) + self.vocabulary = {token: idx for idx, token in enumerate(sorted(all_tokens))} + + num_docs = len(corpus) + # Compute IDF + doc_freq: Dict[str, int] = {} + for tokens in doc_tokens: + for token in set(tokens): + doc_freq[token] = doc_freq.get(token, 0) + 1 + + self.idf = { + token: math.log((num_docs + 1) / (df + 1)) + 1.0 + for token, df in doc_freq.items() + } + + # Build vectors + vectors = np.zeros((num_docs, len(self.vocabulary)), dtype=np.float32) + for i, tokens in enumerate(doc_tokens): + if not tokens: + continue + tf: Dict[str, float] = {} + for token in tokens: + tf[token] = tf.get(token, 0) + 1 + for token, count in tf.items(): + if token in self.vocabulary: + col_idx = self.vocabulary[token] + vectors[i, col_idx] = (count / len(tokens)) * self.idf.get(token, 1.0) + + # Normalize rows + norms = np.linalg.norm(vectors, axis=1, keepdims=True) + norms[norms == 0] = 1.0 + return vectors / norms + + def transform(self, text: str) -> np.ndarray: + tokens = self._tokenize(text) + vector = np.zeros((1, len(self.vocabulary)), dtype=np.float32) + if not tokens or not self.vocabulary: + return vector + + tf: Dict[str, float] = {} + for token in tokens: + tf[token] = tf.get(token, 0) + 1 + + for token, count in tf.items(): + if token in self.vocabulary: + col_idx = self.vocabulary[token] + vector[0, col_idx] = (count / len(tokens)) * self.idf.get(token, 1.0) + + norm = np.linalg.norm(vector) + if norm > 0: + vector = vector / norm + return vector + + +class RagEngineService: + # In-memory document and chunk registry + _documents: Dict[str, DocumentInfo] = {} + _chunks: List[DocumentChunk] = [] + _chunk_vectors: Optional[np.ndarray] = None + _vectorizer: LocalTfidfVectorizer = LocalTfidfVectorizer() + + @classmethod + def index_document(cls, file_content: bytes, filename: str) -> DocumentInfo: + """Parses, chunks, and indexes a PDF or text document into the vector store.""" + doc_id = str(uuid.uuid4())[:8] + pages = DocumentParserService.extract_text_pages(file_content, filename) + new_chunks = DocumentParserService.chunk_pages(pages, doc_id, filename) + + doc_info = DocumentInfo( + document_id=doc_id, + document_name=filename, + total_pages=len(pages), + total_chunks=len(new_chunks), + file_size_kb=round(len(file_content) / 1024, 2), + created_at=datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + ) + + cls._documents[doc_id] = doc_info + cls._chunks.extend(new_chunks) + + # Re-index all chunks in the vector space + cls._rebuild_index() + + return doc_info + + @classmethod + def _rebuild_index(cls) -> None: + """Re-fits the vectorizer on all current document chunks.""" + if not cls._chunks: + cls._chunk_vectors = None + return + + corpus = [chunk.content for chunk in cls._chunks] + cls._vectorizer = LocalTfidfVectorizer() + cls._chunk_vectors = cls._vectorizer.fit_transform(corpus) + + @classmethod + def list_documents(cls) -> List[DocumentInfo]: + """Returns metadata for all currently indexed documents.""" + return list(cls._documents.values()) + + @classmethod + def delete_document(cls, doc_id: str) -> bool: + """Removes a document and its chunks from the vector store.""" + if doc_id not in cls._documents: + return False + + del cls._documents[doc_id] + cls._chunks = [c for c in cls._chunks if c.document_id != doc_id] + cls._rebuild_index() + return True + + @classmethod + def query(cls, question: str, doc_id: Optional[str] = None, top_k: int = 4) -> RagQueryResponse: + """Performs semantic similarity search and synthesizes an answer with exact source citations.""" + start_time = time.perf_counter() + + if not cls._chunks or cls._chunk_vectors is None: + return RagQueryResponse( + question=question, + answer="No documents have been uploaded to DataMind AI yet. Please upload a PDF or text document first.", + citations=[], + retrieved_chunks_count=0, + latency_ms=0.0 + ) + + # Transform query vector + query_vec = cls._vectorizer.transform(question) + + # Filter candidate indices if doc_id provided + candidate_indices = [ + i for i, chunk in enumerate(cls._chunks) + if doc_id is None or chunk.document_id == doc_id + ] + + if not candidate_indices: + return RagQueryResponse( + question=question, + answer=f"No document matching ID '{doc_id}' was found in the index.", + citations=[], + retrieved_chunks_count=0, + latency_ms=round((time.perf_counter() - start_time) * 1000, 2) + ) + + # Compute cosine similarity + candidate_vectors = cls._chunk_vectors[candidate_indices] + similarities = np.dot(candidate_vectors, query_vec.T).flatten() + + # Sort candidate indices by similarity descending + ranked_order = np.argsort(-similarities) + top_indices = [candidate_indices[idx] for idx in ranked_order[:top_k]] + top_scores = [float(similarities[idx]) for idx in ranked_order[:top_k]] + + citations: List[RagCitation] = [] + retrieved_contexts: List[str] = [] + + for chunk_idx, score in zip(top_indices, top_scores): + chunk = cls._chunks[chunk_idx] + relevance_pct = round(max(score, 0.0) * 100, 1) + + # Short snippet + snippet = chunk.content[:180].strip() + ("..." if len(chunk.content) > 180 else "") + + citations.append( + RagCitation( + document_name=chunk.document_name, + page_number=chunk.page_number, + relevance_percentage=relevance_pct, + snippet=snippet + ) + ) + retrieved_contexts.append(chunk.content) + + # Synthesize answer from top chunks + if top_scores and top_scores[0] > 0.05: + top_chunk = cls._chunks[top_indices[0]] + answer = cls._synthesize_answer(question, retrieved_contexts, top_chunk.document_name, top_chunk.page_number) + else: + answer = ( + "The requested information could not be confidently identified in the uploaded documents. " + "Please verify your question or ensure the relevant document is indexed." + ) + + elapsed_ms = round((time.perf_counter() - start_time) * 1000, 2) + + return RagQueryResponse( + question=question, + answer=answer, + citations=citations, + retrieved_chunks_count=len(citations), + latency_ms=elapsed_ms + ) + + @classmethod + def _synthesize_answer(cls, question: str, contexts: List[str], primary_doc: str, primary_page: int) -> str: + """Synthesizes human-readable answer referencing retrieved facts.""" + # Find the most relevant sentences across contexts + q_words = set(re.findall(r'\b\w{3,}\b', question.lower())) + best_sentences = [] + + for ctx in contexts: + sentences = re.split(r'(?<=[.!?])\s+', ctx) + for s in sentences: + s_words = set(re.findall(r'\b\w{3,}\b', s.lower())) + overlap = len(q_words.intersection(s_words)) + if overlap > 0: + best_sentences.append((overlap, s.strip())) + + best_sentences.sort(key=lambda x: x[0], reverse=True) + extracted_facts = " ".join([s[1] for s in best_sentences[:3]]) + + if extracted_facts: + return ( + f"Based on the analysis of {primary_doc} (Page {primary_page}):\n\n" + f"{extracted_facts}\n\n" + f"Source verified against page {primary_page} to prevent hallucination." + ) + + return ( + f"According to {primary_doc} (Page {primary_page}), the context relates to:\n\n" + f"{contexts[0][:250]}..." + ) + + +rag_engine = RagEngineService +rag_engine.query_documents = RagEngineService.query + diff --git a/app/services/sql_engine.py b/app/services/sql_engine.py index 1a6f163..6bc57eb 100644 --- a/app/services/sql_engine.py +++ b/app/services/sql_engine.py @@ -1,7 +1,8 @@ import re import time -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from sqlalchemy import inspect, text + from app.core.database import engine from app.models.schemas import SqlQueryResponse @@ -70,8 +71,10 @@ def execute_query(cls, sql_query: str) -> SqlQueryResponse: ) @classmethod - def generate_sql_from_prompt(cls, prompt: str, schema: Dict[str, List[str]]) -> str: - """Simple rule-based and template generator for natural queries when LLM key is absent.""" + def generate_sql_from_prompt(cls, prompt: str, schema: Optional[Dict[str, List[str]]] = None) -> str: + """Rule-based and template generator for natural queries when LLM key is absent.""" + if schema is None: + schema = cls.get_database_schema() prompt_lower = prompt.lower() # Find best matching table @@ -97,3 +100,7 @@ def generate_sql_from_prompt(cls, prompt: str, schema: Dict[str, List[str]]) -> limit_val = limit_match.group(1) or limit_match.group(2) or limit_match.group(3) if limit_match else 10 return f"SELECT * FROM {matched_table} LIMIT {limit_val};" + + +sql_engine = SqlEngineService + diff --git a/requirements.txt b/requirements.txt index 60e38de..27c9b2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,6 +14,9 @@ numpy>=1.26.0 # Database & Storage sqlalchemy>=2.0.30 +# Document Processing & RAG +pypdf>=5.0.0 + # Documentation & Site mkdocs>=1.6.0 mkdocs-material>=9.5.0 diff --git a/site/sitemap.xml.gz b/site/sitemap.xml.gz index 98b1828d5a2b79985488ebfc9f569e310f6d3dcb..65eaafde7498f518d1a764503c663c7881ed13a2 100644 GIT binary patch delta 13 Ucmb=gXP58h;An7JJdwQu032%sHvj+t delta 13 Ucmb=gXP58h;Aqe;oXB1Q02?L)>;M1& diff --git a/static/css/style.css b/static/css/style.css index ad7e20a..bd4365e 100644 --- a/static/css/style.css +++ b/static/css/style.css @@ -1,12 +1,21 @@ -@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600;700&family=Outfit:wght@400;600;800&display=swap'); +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800&family=Outfit:wght@400;500;600;700;800;900&family=Fira+Code:wght@400;500;600&display=swap'); :root { - --bg-color: #0b0f19; - --card-bg: rgba(255, 255, 255, 0.03); + --bg-color: #070913; + --bg-surface: rgba(15, 23, 42, 0.7); + --card-bg: rgba(22, 30, 53, 0.55); + --card-hover: rgba(30, 41, 69, 0.75); --glass-border: rgba(255, 255, 255, 0.08); - --primary-gradient: linear-gradient(135deg, #6366f1 0%, #a855f7 100%); + --glass-border-focus: rgba(99, 102, 241, 0.45); + --primary-gradient: linear-gradient(135deg, #6366f1 0%, #a855f7 50%, #ec4899 100%); + --secondary-gradient: linear-gradient(135deg, #38bdf8 0%, #6366f1 100%); + --accent-emerald: #10b981; + --accent-indigo: #6366f1; + --accent-purple: #a855f7; --text-main: #f8fafc; --text-muted: #94a3b8; + --text-dim: #64748b; + --code-bg: #050811; } * { @@ -15,213 +24,632 @@ box-sizing: border-box; } +html { + scroll-behavior: smooth; +} + body { background-color: var(--bg-color); color: var(--text-main); - font-family: 'Inter', sans-serif; + font-family: 'Inter', -apple-system, BlinkMacSystemFont, sans-serif; min-height: 100vh; display: flex; flex-direction: column; align-items: center; - justify-content: center; overflow-x: hidden; position: relative; + line-height: 1.5; } -/* Animated Background Blobs */ +/* Background Grid Pattern */ +body::before { + content: ''; + position: fixed; + top: 0; + left: 0; + width: 100vw; + height: 100vh; + background-image: + linear-gradient(rgba(255, 255, 255, 0.02) 1px, transparent 1px), + linear-gradient(90deg, rgba(255, 255, 255, 0.02) 1px, transparent 1px); + background-size: 40px 40px; + z-index: -2; + pointer-events: none; +} + +/* Ambient Glowing Blobs */ .blob { - position: absolute; - filter: blur(80px); + position: fixed; + filter: blur(100px); z-index: -1; - opacity: 0.5; - animation: float 10s infinite ease-in-out alternate; + opacity: 0.45; + pointer-events: none; + animation: float 14s infinite ease-in-out alternate; } .blob-1 { - top: -10%; - left: -10%; - width: 400px; - height: 400px; - background: rgba(99, 102, 241, 0.4); + top: -100px; + left: 5%; + width: 500px; + height: 500px; + background: radial-gradient(circle, rgba(99, 102, 241, 0.5) 0%, rgba(99, 102, 241, 0) 70%); border-radius: 50%; } .blob-2 { - bottom: -10%; - right: -10%; - width: 500px; - height: 500px; - background: rgba(168, 85, 247, 0.4); + bottom: -100px; + right: 5%; + width: 600px; + height: 600px; + background: radial-gradient(circle, rgba(168, 85, 247, 0.45) 0%, rgba(168, 85, 247, 0) 70%); + border-radius: 50%; + animation-delay: -7s; +} + +.blob-3 { + top: 40%; + left: 45%; + width: 450px; + height: 450px; + background: radial-gradient(circle, rgba(16, 185, 129, 0.25) 0%, rgba(16, 185, 129, 0) 70%); border-radius: 50%; - animation-delay: -5s; + animation-delay: -3s; } @keyframes float { - 0% { transform: translateY(0px) scale(1); } - 100% { transform: translateY(30px) scale(1.1); } + 0% { transform: translate(0, 0) scale(1); } + 50% { transform: translate(30px, 40px) scale(1.08); } + 100% { transform: translate(-20px, 20px) scale(0.95); } } -/* Glassmorphism Container */ -.container { - background: var(--card-bg); - backdrop-filter: blur(16px); - -webkit-backdrop-filter: blur(16px); - border: 1px solid var(--glass-border); - border-radius: 24px; - padding: 3rem 4rem; +/* Top Navbar */ +.navbar { + width: 100%; + max-width: 1140px; + display: flex; + justify-content: space-between; + align-items: center; + padding: 1.25rem 1.5rem; + margin-top: 0.5rem; + z-index: 10; +} + +.nav-brand { + display: flex; + align-items: center; + gap: 0.75rem; + text-decoration: none; + color: var(--text-main); + font-family: 'Outfit', sans-serif; + font-weight: 800; + font-size: 1.35rem; + letter-spacing: -0.5px; +} + +.nav-brand .logo-icon { + width: 36px; + height: 36px; + border-radius: 10px; + background: var(--primary-gradient); + display: flex; + align-items: center; + justify-content: center; + color: white; + font-size: 1.1rem; + box-shadow: 0 4px 15px rgba(99, 102, 241, 0.4); +} + +.nav-links { + display: flex; + align-items: center; + gap: 1.25rem; +} + +.nav-link { + color: var(--text-muted); + text-decoration: none; + font-size: 0.9rem; + font-weight: 500; + transition: color 0.2s; + display: inline-flex; + align-items: center; + gap: 0.4rem; +} + +.nav-link:hover { + color: var(--text-main); +} + +/* Live Status Badge */ +.status-pill { + display: inline-flex; + align-items: center; + gap: 0.5rem; + font-size: 0.8rem; + font-weight: 600; + color: #34d399; + background: rgba(16, 185, 129, 0.1); + padding: 0.4rem 0.9rem; + border-radius: 999px; + border: 1px solid rgba(16, 185, 129, 0.25); + backdrop-filter: blur(8px); +} + +.pulse { + width: 7px; + height: 7px; + background-color: #10b981; + border-radius: 50%; + box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); + animation: pulse-ring 1.8s infinite cubic-bezier(0.66, 0, 0, 1); +} + +@keyframes pulse-ring { + to { box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); } +} + +/* Main Container */ +.main-wrapper { + width: 100%; + max-width: 1140px; + padding: 1rem 1.5rem 4rem 1.5rem; + display: flex; + flex-direction: column; + align-items: center; +} + +/* Hero Section */ +.hero { text-align: center; - max-width: 800px; - width: 90%; - box-shadow: 0 25px 50px -12px rgba(0, 0, 0, 0.5); - animation: fadeUp 1s cubic-bezier(0.16, 1, 0.3, 1); + margin-top: 1.5rem; + margin-bottom: 2rem; + max-width: 840px; + animation: fadeUp 0.8s cubic-bezier(0.16, 1, 0.3, 1); } -@keyframes fadeUp { - 0% { opacity: 0; transform: translateY(40px); } - 100% { opacity: 1; transform: translateY(0); } +.hero-pill { + display: inline-flex; + align-items: center; + gap: 0.5rem; + padding: 0.35rem 1rem; + background: rgba(99, 102, 241, 0.12); + border: 1px solid rgba(99, 102, 241, 0.3); + border-radius: 999px; + font-size: 0.8rem; + font-weight: 600; + color: #a5b4fc; + margin-bottom: 1.25rem; } -h1 { +h1.hero-title { font-family: 'Outfit', sans-serif; - font-size: 3.5rem; - font-weight: 800; - margin-bottom: 1rem; + font-size: 3.8rem; + font-weight: 900; + line-height: 1.12; + margin-bottom: 1.25rem; background: var(--primary-gradient); -webkit-background-clip: text; -webkit-text-fill-color: transparent; - letter-spacing: -1px; + letter-spacing: -1.5px; } -p.subtitle { - font-size: 1.25rem; +p.hero-subtitle { + font-size: 1.18rem; color: var(--text-muted); - margin-bottom: 2.5rem; - line-height: 1.6; + line-height: 1.65; + margin-bottom: 2rem; } -/* Features Grid */ -.features { +/* Stat Counters Banner */ +.stats-banner { display: flex; justify-content: center; - gap: 2rem; - margin-bottom: 3rem; + gap: 2.5rem; + margin-bottom: 2.5rem; flex-wrap: wrap; + background: rgba(255, 255, 255, 0.02); + border: 1px solid var(--glass-border); + padding: 1rem 2rem; + border-radius: 16px; + backdrop-filter: blur(10px); } -.feature-badge { - background: rgba(255, 255, 255, 0.05); - border: 1px solid var(--glass-border); - padding: 0.75rem 1.5rem; - border-radius: 999px; - font-size: 0.9rem; - font-weight: 600; +.stat-item { + text-align: center; +} + +.stat-value { + font-family: 'Outfit', sans-serif; + font-size: 1.6rem; + font-weight: 800; color: var(--text-main); display: flex; align-items: center; - gap: 0.5rem; - transition: all 0.3s ease; + justify-content: center; + gap: 0.35rem; } -.feature-badge:hover { - background: rgba(255, 255, 255, 0.1); - transform: translateY(-2px); - box-shadow: 0 10px 15px -3px rgba(0, 0, 0, 0.1); -} +.stat-value.green { color: #34d399; } +.stat-value.purple { color: #c084fc; } +.stat-value.blue { color: #38bdf8; } -.feature-badge i { - color: #a855f7; +.stat-label { + font-size: 0.76rem; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; + font-weight: 600; + margin-top: 0.2rem; } -/* Buttons */ -.btn-container { +/* Action Buttons */ +.btn-group { display: flex; - gap: 1.5rem; + gap: 1rem; justify-content: center; + flex-wrap: wrap; + margin-bottom: 3rem; } .btn { text-decoration: none; - padding: 1rem 2.5rem; + padding: 0.75rem 1.6rem; border-radius: 12px; font-weight: 600; - font-size: 1.1rem; - transition: all 0.3s cubic-bezier(0.4, 0, 0.2, 1); - display: inline-block; + font-size: 0.95rem; + transition: all 0.25s cubic-bezier(0.4, 0, 0.2, 1); + display: inline-flex; + align-items: center; + gap: 0.6rem; + cursor: pointer; } .btn-primary { background: var(--primary-gradient); color: white; - box-shadow: 0 10px 25px -5px rgba(99, 102, 241, 0.5); - position: relative; - overflow: hidden; + box-shadow: 0 10px 25px -5px rgba(99, 102, 241, 0.4); border: none; } -.btn-primary::after { - content: ''; - position: absolute; - top: -50%; - left: -50%; - width: 200%; - height: 200%; - background: linear-gradient(transparent, rgba(255, 255, 255, 0.2), transparent); - transform: rotate(45deg); - transition: all 0.5s ease; - opacity: 0; -} - .btn-primary:hover { - transform: translateY(-3px); - box-shadow: 0 20px 30px -5px rgba(99, 102, 241, 0.6); -} - -.btn-primary:hover::after { - left: 100%; - opacity: 1; + transform: translateY(-2px); + box-shadow: 0 16px 30px -5px rgba(99, 102, 241, 0.6); + opacity: 0.96; } .btn-secondary { - background: transparent; + background: rgba(255, 255, 255, 0.04); color: var(--text-main); border: 1px solid var(--glass-border); backdrop-filter: blur(10px); } .btn-secondary:hover { - background: rgba(255, 255, 255, 0.05); + background: rgba(255, 255, 255, 0.08); border-color: rgba(255, 255, 255, 0.2); - transform: translateY(-3px); + transform: translateY(-2px); +} + +/* Playground Card (Glassmorphism Core) */ +.playground-card { + width: 100%; + background: var(--bg-surface); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); + border: 1px solid var(--glass-border); + border-radius: 24px; + padding: 2rem; + box-shadow: 0 30px 60px -20px rgba(0, 0, 0, 0.7); + margin-bottom: 3rem; + transition: border-color 0.3s; } -/* Pulse Animation for Status */ -.status { - position: absolute; - top: 2rem; - right: 2rem; +.playground-card:hover { + border-color: rgba(255, 255, 255, 0.14); +} + +.tab-nav { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + padding-bottom: 1.25rem; + border-bottom: 1px solid var(--glass-border); + margin-bottom: 1.75rem; +} + +.tab-btn { + background: transparent; + border: 1px solid transparent; + color: var(--text-muted); + padding: 0.65rem 1.2rem; + font-size: 0.88rem; + font-weight: 600; + border-radius: 12px; + cursor: pointer; + transition: all 0.2s; + display: inline-flex; + align-items: center; + gap: 0.55rem; +} + +.tab-btn:hover { + color: var(--text-main); + background: rgba(255, 255, 255, 0.04); +} + +.tab-btn.active { + background: rgba(99, 102, 241, 0.16); + color: #c7d2fe; + border: 1px solid rgba(99, 102, 241, 0.35); + box-shadow: 0 0 15px rgba(99, 102, 241, 0.15); +} + +/* Tab Panels */ +.pane-title { + font-size: 1.1rem; + font-weight: 700; + margin-bottom: 0.4rem; display: flex; align-items: center; gap: 0.5rem; +} + +.pane-desc { + font-size: 0.85rem; + color: var(--text-muted); + margin-bottom: 1.25rem; +} + +.input-box { + background: var(--code-bg); + border: 1px solid var(--glass-border); + color: #e2e8f0; + padding: 0.85rem 1.15rem; + border-radius: 12px; + font-family: 'Fira Code', Consolas, monospace; + font-size: 0.88rem; + width: 100%; + transition: all 0.2s; +} + +.input-box:focus { + outline: none; + border-color: var(--accent-indigo); + box-shadow: 0 0 16px rgba(99, 102, 241, 0.25); + background: #080c18; +} + +/* Prompt Chips */ +.chips-container { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + margin-top: 0.6rem; + margin-bottom: 1.25rem; +} + +.chip { + background: rgba(255, 255, 255, 0.04); + border: 1px solid var(--glass-border); + color: var(--text-muted); + font-size: 0.78rem; + padding: 0.35rem 0.8rem; + border-radius: 20px; + cursor: pointer; + transition: all 0.2s; + user-select: none; +} + +.chip:hover { + background: rgba(99, 102, 241, 0.15); + border-color: rgba(99, 102, 241, 0.4); + color: #e0e7ff; + transform: translateY(-1px); +} + +/* Mini Action Buttons */ +.controls-row { + display: flex; + gap: 0.75rem; + flex-wrap: wrap; + align-items: center; + margin-bottom: 1.25rem; +} + +.mini-btn { + background: var(--primary-gradient); + color: white; + border: none; + padding: 0.65rem 1.35rem; + border-radius: 10px; font-size: 0.85rem; font-weight: 600; - color: #10b981; - background: rgba(16, 185, 129, 0.1); - padding: 0.5rem 1rem; - border-radius: 999px; - border: 1px solid rgba(16, 185, 129, 0.2); + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s; } -.pulse { - width: 8px; - height: 8px; - background-color: #10b981; - border-radius: 50%; - box-shadow: 0 0 0 0 rgba(16, 185, 129, 0.7); - animation: pulse-ring 1.5s infinite cubic-bezier(0.66, 0, 0, 1); +.mini-btn:hover { + opacity: 0.95; + transform: translateY(-1px); + box-shadow: 0 4px 15px rgba(99, 102, 241, 0.35); } -@keyframes pulse-ring { - to { - box-shadow: 0 0 0 10px rgba(16, 185, 129, 0); - } +.mini-btn-outline { + background: rgba(255, 255, 255, 0.03); + border: 1px solid var(--glass-border); + color: var(--text-main); + padding: 0.65rem 1.2rem; + border-radius: 10px; + font-size: 0.85rem; + font-weight: 600; + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 0.5rem; + transition: all 0.2s; +} + +.mini-btn-outline:hover { + background: rgba(255, 255, 255, 0.08); + border-color: rgba(255, 255, 255, 0.25); + color: white; + transform: translateY(-1px); +} + +/* Console Header & Result Box */ +.console-header { + display: flex; + justify-content: space-between; + align-items: center; + margin-bottom: 0.5rem; +} + +.console-label { + font-size: 0.78rem; + font-weight: 600; + color: var(--text-dim); + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.copy-btn { + background: transparent; + border: none; + color: var(--text-dim); + font-size: 0.75rem; + cursor: pointer; + display: inline-flex; + align-items: center; + gap: 0.35rem; + transition: color 0.2s; +} + +.copy-btn:hover { + color: var(--text-main); +} + +.result-box { + background: var(--code-bg); + border: 1px solid var(--glass-border); + border-radius: 14px; + padding: 1.25rem; + font-family: 'Fira Code', monospace; + font-size: 0.84rem; + min-height: 120px; + max-height: 320px; + overflow-y: auto; + color: #38bdf8; + white-space: pre-wrap; + line-height: 1.6; + position: relative; +} + +/* Custom Scrollbar */ +.result-box::-webkit-scrollbar { + width: 6px; + height: 6px; +} +.result-box::-webkit-scrollbar-track { + background: transparent; +} +.result-box::-webkit-scrollbar-thumb { + background: rgba(255, 255, 255, 0.1); + border-radius: 3px; +} +.result-box::-webkit-scrollbar-thumb:hover { + background: rgba(255, 255, 255, 0.2); +} + +/* Pillar Feature Grid */ +.features-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); + gap: 1.5rem; + width: 100%; + margin-bottom: 3.5rem; +} + +.feature-card { + background: var(--card-bg); + border: 1px solid var(--glass-border); + border-radius: 18px; + padding: 1.75rem; + transition: all 0.3s ease; + backdrop-filter: blur(10px); +} + +.feature-card:hover { + background: var(--card-hover); + border-color: rgba(99, 102, 241, 0.3); + transform: translateY(-4px); + box-shadow: 0 15px 30px -10px rgba(0, 0, 0, 0.5); +} + +.feature-icon-wrapper { + width: 46px; + height: 46px; + border-radius: 12px; + display: flex; + align-items: center; + justify-content: center; + font-size: 1.25rem; + margin-bottom: 1.2rem; +} + +.icon-agent { background: rgba(99, 102, 241, 0.15); color: #818cf8; } +.icon-sql { background: rgba(56, 189, 248, 0.15); color: #38bdf8; } +.icon-analytics { background: rgba(168, 85, 247, 0.15); color: #c084fc; } +.icon-rag { background: rgba(16, 185, 129, 0.15); color: #34d399; } + +.feature-card h3 { + font-family: 'Outfit', sans-serif; + font-size: 1.15rem; + font-weight: 700; + margin-bottom: 0.5rem; +} + +.feature-card p { + font-size: 0.85rem; + color: var(--text-muted); + line-height: 1.6; +} + +/* Footer */ +footer { + width: 100%; + max-width: 1140px; + padding-top: 2rem; + border-top: 1px solid var(--glass-border); + display: flex; + justify-content: space-between; + align-items: center; + flex-wrap: wrap; + gap: 1rem; + color: var(--text-dim); + font-size: 0.85rem; +} + +footer a { + color: var(--text-muted); + text-decoration: none; + transition: color 0.2s; +} + +footer a:hover { + color: var(--text-main); +} + +@keyframes fadeUp { + 0% { opacity: 0; transform: translateY(30px); } + 100% { opacity: 1; transform: translateY(0); } +} + +@media (max-width: 768px) { + h1.hero-title { font-size: 2.6rem; } + .container { padding: 1.5rem; } + .stats-banner { gap: 1.5rem; } + .navbar { flex-direction: column; gap: 1rem; } } diff --git a/templates/index.html b/templates/index.html index f5a9313..3d1e74c 100644 --- a/templates/index.html +++ b/templates/index.html @@ -3,197 +3,349 @@ - DataMind AI | Enterprise Intelligent Assistant - + DataMind AI | Enterprise Intelligent Data & Document Assistant + - + - - - +

+
- -
-
- API Online (v{{ version }}) -
- - -
-

DataMind AI

-

- Enterprise Intelligent Data & Document Assistant.
- Bridging the gap between your databases, unstructured documents, and end-users through powerful Natural Language Processing. -

- -
-
- Data Analytics -
-
- Text-to-SQL -
-
- RAG Intelligence + +
+ +
+
-
- AI Agents -
-
+ DataMind AI + -
- - - Health API +
+ + Live Docs - - - Swagger UI + + API Swagger - - - ReDoc + + GitHub +
+
+ API Online (v{{ version }}) +
+ + + +
+ + +
+
+ Zero-Trust Guardrails Active +
+

Intelligent Data & Document Assistant

+

+ Bridging the gap between complex relational databases, unstructured PDF documents, and decision-makers. + Experience instant Text-to-SQL, source-verified RAG intelligence, and automated tabular analytics. +

- -
-
-
+ + +
+ +
+ + - +
- -
-
- - + +
+
Autonomous ReAct Multi-Tool Agent
+
Ask questions in plain English or natural queries. The agent detects intent and autonomously routes to SQL, Document RAG, or Tabular Analytics.
+ +
+ ✨ Capabilities Overview + πŸ—„οΈ Execute Safe SQL + πŸ“Š Analytics Guidance + πŸ“‘ Policy Verification
- -
- -
// Output will appear here...
+ + + +
+
+ +
+ Agent Output Stream + +
+
// Interactive Agent response and tool dispatch telemetry will appear here...
- -
+ + +
+
+
+ +
+

Autonomous AI Agent

+

Equipped with ReAct pattern to autonomously select tools between SQL databases, RAG document stores, or tabular calculations.

+
+ +
+
+ +
+

Secure Text-to-SQL

+

AST & Regex-based zero-trust security guardrails strictly block any destructive DDL/DML mutation queries.

+
+ +
+
+ +
+

Data Analytics Pipeline

+

Automated descriptive statistics, correlation analysis, null detection, and intelligent mean/median/mode imputation.

+
+ +
+
+ +
+

Anti-Hallucination RAG

+

Vector similarity search with verified document name and page number citations for auditability and trust.

+
+
+ + + +
+ + diff --git a/tests/__pycache__/__init__.cpython-313.pyc b/tests/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 0a94b7f61052361482f1e38512398831c22b7c66..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 136 zcmey&%ge<81VL_#vm}A^V-N=h7@>^MEI`IohI9r^M!%H|MNB~6XOPq_^#H3F_so)v z(xe!d#F9kc%)FF%N6(m&)Z&ujnE3e2yv&mLc)fzkTO2mI`6;D2sdh!IKz$%{i$RQ! M%#4hTMa)1J0AD{IHUIzs diff --git a/tests/__pycache__/test_analytics.cpython-313-pytest-9.1.1.pyc b/tests/__pycache__/test_analytics.cpython-313-pytest-9.1.1.pyc deleted file mode 100644 index 632e19a8efd3548575984560fa068cd7e8b2779d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7609 zcmeHM-ESMm5#J+M6v?AVeL1$iDq5ChQ=#=qO0sKNktN5HZP|t{Oby5=aVAePQ|gZ0 zJ35il0z#VyyFp>Kh*8z(gPNj%_2xeH<1Z-CK9Hgc35Nu2(I=rVM%0)1rJcDiQmNB8 ztql~h52x9g+28HQ?Cff0xmQ!;C-8``|KhD*R}u0z6zs=sF?-(x<{lASuwq7J-;y*`4@M_$iVS**4#uM= zp>F?IvFKz*l*Q=4NHi*lSxNQBqvvi((bzz=A6vqvWEhRn|9_}TnNv1tH56#my@x)u z=bTeF>vC9!=^%5|7H09w&pZNW+tV&(GUv5Hv%JlfOTi_$1)3m2Rm|PT37(TKnM-i9 zfilA8WS3B_3?$sArU+3ck6Y^W?+BR-+L+l|n=6+cMDQkPj1zpKGXYy5(LuHMGX zHriad>>zPV_&Sv7mVLdvQpVX_2T3?{x5p*|u3~J5Y;3u#{OSq^V=B+&$!M2t*Dpl6 zZKS~WxKDTpk*kD&P?M1Jw|+iYztFY!%IjA#<|kXqYmfB{k<>;C zwLLDwHh!T_s89GUOYvXAQoP-kU0Rm%f~7tHOEg%PXnf%~D=)ky4x%L*$eRJx z)k{=Y1DS%8Rr#ZE2=!N8GjJ~RBuwNQIUqDCw-YsZA}Qq^^vkE%LCp!=K`qyQ_}}ax zq4k-cl{NPMu+M>~IxC&8;H+#F+N|9g6uu_3CxX^)t@;wXwaOlaU7EW!cSi`nv@HBOGofa6Vdl!T#@IuZtW?goF<3#5nR|pBM{pMug!{4$PBC=jF#$d2Rm0 z$Z}J!n9KCu>hFaRFuSB>QfhCGDf5}Eq*J7iYox_oR#}oHwI`+CdhFGtceGyMHaNWs z0qXAWVI4NO$F9-Q4Q_ex=<(drKt%PwYcb)`FminnwMW%NEw3h1N?P)NH#}-q#3QQi zWs;gxWL4U5>u$gz89)+$928YmVw%y$FhD^$toyLzBu3HDV?8$CA5rZFV-a=3g}8ns zv3)HJnJotULl#W;{Iz<=!bEN8s$fM-p(t zPu0P>T_+G6lfa$225}|C*fTW*V#izL9^@=D4dz+D;2L(%a}hT?0#McoU^}2M6+^{cvO)^=llPUix`#u*YWj^-?4S5a z|Bw3bE1#-fsQ+G|h5NCu1#%@hpUr}EL`yg{*P|oWxjsUs4t4C2TP)Pjo=_3z+dE99e4-QDFM zEdQ)~yZO~(^Q#ZemzraxP<&Lr~(XQJgj;>gipu$0yussLl zFrePy8}5$Y8GV0jWpalPt@1^_wLp8(GJRCK#lE%ihQ_0kFJj!q{tuLrq;Vjj~4i_ zDQ;EpB}6+YhXM6pI*&hS`tVArHum9_wQsKuebDstD=X85+Sm?XTcB;L(=hgJYQ3AZ z?RMOhw#uwi4Z787M=I`<9CeAdfw>Dbyt)L_*rwLINyE5;C~cKlry6vtOO90BCpqd8 z4gbSKLJh_G$+h(Q$@@yFezFvtTsgnP*RP&_Rz?H6e)>rbsf(-lkK+#{C^XO~UyXbR z057jbz(~gP@@k}PiDz~-62hSNQe|5AG#4YaMp{?*+%HDzor{qMq4Bv~j69p`fP-HF zOBiWCT$VNcPif9q!4l0FD1#+hUTjM&kVxxghX13M&;EOom&!__JeFokE}4cPK9dq5 zyc^Y6Uh2v$=JFb(SNTQw)gd*yD5c@&1BTc33>tO2uMG@P2>|AsI$8g5)F;Ot=`{ zkFnE8MvsB2)%w@gj6|~=|R^s zQ)uSK3W~dkQDDt&V(c2s?+kZj-N?cH12H|}nLqBR9u!ejW@70)AD9i%J%Pa5mFuDx!aJ8rc0FYKyEj{pDw diff --git a/tests/__pycache__/test_health.cpython-313-pytest-9.1.1.pyc b/tests/__pycache__/test_health.cpython-313-pytest-9.1.1.pyc deleted file mode 100644 index 881f0710c412575d215d8fe1afc673a0d2abb371..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5292 zcmeHLTW=Fr5Z<*6kbRS*yGQpr4lR8?yJLVrOY?Of#Oh^ngk#9P(8@zj~U*^VMXs$N9PYBgu(oH?^| z=A7C6W_Rl9A{<pZ3id$lm0j_k*UVGZ`iRc)l6Fks;AVu zTIIF_ymnXM`lcMNnsuP8?Q)f~uJhnic_n*mdr^9+rqA_83dH=U!LCN6pcG1ye%_L5 zrkh+ZDr_Bc`^qi&SmWAqTNRE|g4g&2pBt3I`lpky(%qNdmGoik^gC`mUTLo)$sp{N zFg07=s|x@2j!MAw>owig@?O>cZF?1fy`oaxo4u+&r~A&pe{8SjII;d3Le=P1pm&qQOE`HFCOUsB3=a z2VEqCNQPh3D~6#`)3PxNs?a(5Lb#-u%0xy>N5`(L@ywUeM1Hv!A2T2t(TrwkgJIhV zU>qlgzzI{mYew#g)`JGn+z@Sm1uRF#Zd{38*RmN+6>C`_Wqmf7e@-=-P~kX>yq8-t z;kzO$lpT~)N;WG~HAhqWLQbJ7tA$~Cb}^GRGn&BySsgSC)*>78`eHV1smgO%eOAfJ znzEp>uxuzdRkXnB$E>AhbnSggb;_FLn>t-kOj*5~%PN{;!U!YizZjwd_{;=+m>r;j zN+Dh;4B3KhYot;*Q>oTpDGVC;p^7>eJ9Xpi%!e6sesOljjl4W|Wd_4wSX7=@m8>}* z%Pp}sD>(D|f-2{9YQj3Rx{@~Jd2=C4-vO)m*)?Y1>%GD~YiNGBe1AEA^1&A?m$!wc zlF(ih+6$!X>5xT=LRX16_d4k+2<@I|Gh3u<^^Dh7EK4xpX>XuCPswATds}EJ5b?>x z;|XZ;6GcHR5$9ef;%dT{Hp{Fx41M{_o>VMLTWH7|@R$uEZVN{Wq~}TEaRQorqA2u~ zh;y%#p4D@tMqm;EgXK8V8+dG-4zy?7qK9 z_^n7Z{Rbj($caSr|2q=WVf{`ruq!9Nd*z%sGIjgU4nm)Cn<)!}CLE7g16EO-O9 zNJpLg6QowDEh#wpNA{6_#LYXmwev4`7x3e;Ys*5PlFRhj@PQ&Up!mX}So>NShhptZ z@nua90$~9gi?N_RWr>B->Ks+lYMNrb2sp5D7Y>YzhUlZsI2q7he7O=)j@XEDgoWVe zfCb2C<1Q->Oa&_}cw40gfUxD7fmna6pEafdhR!O63XoRQ)Rd{F_jo>lPDW7E@q%J4 zTIB@*R11TOwG`jov<1mgB&|rGw&&Umq*EVCPy zo~Pt7(2q7FMWL-gdh_=zQWSbi#JSf=Z$W6Y#mx$S1GPOZkAZHq7ruX?Ksxht7AXpy zCF0!cq;vIxEp3+B4b=5?JO;YA5lVMIg%x&}h;y%#ZVMNqxLLt_3efo!qT#3)t0b=* zlxQ#PvRKDLZmnaTP*^^?3UvhR!YM>BX#io?e7;{3~c+2K?sddH!dP`-Q>oc|()v;7Nvg@Qe6EGkF*AscNE+y}n z-D54uFd(3Pu#p0BP!#oJiUg?j*602O1^PmYawHsD6m6dj^udVU{M4Df4<0SSii|dB znjDU^GqZEEyR&zi{Y>S;KbsynRCWw<$b;?+y% zrGU|s30%6)amMg@r#tsYj++m;wsR4etF^tH6wCxicuBa^HWC>8iTz*&e&u|w@ncU_$K-JSUkZ&c@ayJIRaA72Fajr-oG zVYg!6(lZotck!TAXKH6i{EzQP?G(Jpe(ZS~_Cq{yJz;m{Qe!g2-h#*>-hznxwA(sw zK?vT0{<~bF>DF4?pi_I$jcZ?>e6(K4u8ah=sf>sc0kz7PF`m$M}R3#?!F%G7n z9A8`VDJR^qfT@jj9>)aN_B4QTx`tg9Fpg>nX7(X9^w@**iP(byHquyv0&m4Ms_8Im zJ*!$mPRp8!5W&AvoW&55VS?`>a$`s)63td9%kqU4ysMg)=v%TC`(bg_goGk1#A=nZ zNPj)M=!ywFmVk>StuSBG1|$(Bm>(<=hqdE5OTj!6F8NY@aq+)pJEd(@in}4#V)H@4)-D&i$q3*e7d`*2Yd|fep~poP!Y;=s00cUREB)74A1`b+S%?6+{GXE=o2Q3clUY;0yr82G~U zbl};Q;JNaid4O^k!pPM@8w@OiHnYGT3~i)<6ifjCc{}hLaMSRH;3mYtO}Gx+L=N?L z{mp=z$bo;)Zvt+j9=M4~;(rC)oWR|$zg%djhcY3lu>m^uu+(IXXM#65CTxNji=jFqBG-ZhKliLKEoHMRrMIDs5zuIYUu#2v4A3{O zQHFR&5_BU#@QWFN>_{E%1~g2LVNaA8_W-_^bn`iW)A-#uXdc{H;jED_6%fPPleYwh zSHl4EOL>zL{3w2&Lw^x>Q5%f}vydee4QdknmL~WCL?Cfb>a(bL3&}YoeMtI&0C)$G=?AANf=SGeP)Gg3IK=?EXcb!W12jKf1ET=FWqa4u z6*XHzgiU6^rU`y@8kT4y()*9@??g`RG~A||&5=iU^OWwGBDw|smR4LvK50d<|@ zrHXj2EDS&GWTYa(BhI}k441`owz%cvt7z?6c?_743vinD$K?S)=ilRUC~fXr&Z?Cv zp=|Q4#N|?>@zc52Aakh+bE+|x31a!SR$%;^m^{oddG-*P{OcFh_G0n~!{iYMlSdDM z$-P?rE0`QhQWWxO`znAB7Ypv6B?|%@#edXPoGw~8sjp$`E1bXc;B&*ja$h9^J z{=|u9uEu9pvTBhPOhP%;RJ1%xnYusMX}X&-$p{WUh6EENnF5lCy*?R{chM5l54nnD z8VJQwOfB8m+2$=T+d$oAQOo62T{em;VdWfo5eh^e? zI#7j~{C)a&>CdPBFi|->2~zp&WJR3Z7Myz%G+2*8am&Y7nWbmuF<`^~9HHPAacgD_ zC|DzzqdO(}@5q z!zZlf#L_$q`v%A6VDDCBvS8#&dG#Ig6EH`3L~)N_1bLqS8^`^Ud&VXH${qcCwEgkb ZkGt1HUxkAFg|8Y#eq^^R#HX33e*=2xeqR6p diff --git a/tests/test_agent.py b/tests/test_agent.py new file mode 100644 index 0000000..d18855e --- /dev/null +++ b/tests/test_agent.py @@ -0,0 +1,43 @@ +from fastapi.testclient import TestClient +from main import app + +client = TestClient(app) + + +def test_agent_chat_reasoning_and_capabilities(): + response = client.post( + "/api/v1/agent/chat", + json={"message": "Hello, what features can you provide for our enterprise data?"} + ) + assert response.status_code == 200 + data = response.json() + assert data["intent"] == "reasoning" + assert "reply" in data + assert "DataMind AI" in data["reply"] + assert data["latency_ms"] >= 0 + + +def test_agent_chat_dispatches_sql_query(): + response = client.post( + "/api/v1/agent/chat", + json={"message": "SELECT 123 AS user_count, 'Active' AS status;"} + ) + assert response.status_code == 200 + data = response.json() + assert data["intent"] == "sql" + assert data["tool_used"] == "safe_sql_engine" + assert data["tool_output"] is not None + assert data["tool_output"]["row_count"] == 1 + assert "user_count" in data["tool_output"]["columns"] + + +def test_agent_chat_dispatches_analytics_guidance(): + response = client.post( + "/api/v1/agent/chat", + json={"message": "How do you profile a dataset and clean null values?"} + ) + assert response.status_code == 200 + data = response.json() + assert data["intent"] == "analytics" + assert data["tool_used"] == "data_analytics_pipeline" + assert "Analytics" in data["reply"] diff --git a/tests/test_rag.py b/tests/test_rag.py new file mode 100644 index 0000000..831c425 --- /dev/null +++ b/tests/test_rag.py @@ -0,0 +1,69 @@ +import io +from fastapi.testclient import TestClient +from main import app + +client = TestClient(app) + +SAMPLE_DOC_TEXT = """# DataMind AI Architecture and Enterprise Security Overview + +DataMind AI is an enterprise-grade intelligent assistant. The system uses FastAPI for asynchronous API throughput and PostgreSQL for transactional data. + +## Security Architecture +DataMind AI operates under a strict Zero-Trust principle. All Text-to-SQL operations are executed in Read-Only mode to prevent data corruption. No destructive DDL or DML queries such as DROP or DELETE are ever allowed. + +## Document Intelligence +The platform integrates semantic text chunking and vector storage with ChromaDB. Answers generated by the assistant must explicitly cite the document name and page number to eliminate hallucination. +""" + + +def test_rag_upload_and_index_document(): + doc_file = io.BytesIO(SAMPLE_DOC_TEXT.encode("utf-8")) + response = client.post( + "/api/v1/rag/upload", + files={"file": ("security_policy.md", doc_file, "text/markdown")} + ) + assert response.status_code == 200 + data = response.json() + assert data["success"] is True + assert data["document"]["document_name"] == "security_policy.md" + assert data["document"]["total_chunks"] >= 1 + assert "document_id" in data["document"] + + +def test_rag_list_documents(): + response = client.get("/api/v1/rag/documents") + assert response.status_code == 200 + docs = response.json() + assert isinstance(docs, list) + assert len(docs) >= 1 + assert any(d["document_name"] == "security_policy.md" for d in docs) + + +def test_rag_query_with_source_citation(): + # Ask question related to security + response = client.post( + "/api/v1/rag/query", + json={"question": "What is the security architecture principle and what queries are forbidden?"} + ) + assert response.status_code == 200 + data = response.json() + assert data["retrieved_chunks_count"] >= 1 + assert len(data["citations"]) >= 1 + assert "security_policy.md" in data["citations"][0]["document_name"] + assert "page_number" in data["citations"][0] + assert data["citations"][0]["relevance_percentage"] > 0 + assert "Zero-Trust" in data["answer"] or "Read-Only" in data["answer"] or "security_policy.md" in data["answer"] + + +def test_rag_delete_document(): + # Get doc ID + list_res = client.get("/api/v1/rag/documents") + doc_id = list_res.json()[0]["document_id"] + + del_res = client.delete(f"/api/v1/rag/documents/{doc_id}") + assert del_res.status_code == 200 + assert del_res.json()["success"] is True + + # Verify deletion + after_res = client.get("/api/v1/rag/documents") + assert not any(d["document_id"] == doc_id for d in after_res.json())