diff --git a/backend/alembic/versions/005_durable_artifacts.py b/backend/alembic/versions/005_durable_artifacts.py new file mode 100644 index 0000000..6674f62 --- /dev/null +++ b/backend/alembic/versions/005_durable_artifacts.py @@ -0,0 +1,132 @@ +"""Durable analytics artifacts (Phase 2 — Milestone 1) + +Revision ID: 005 +Revises: 004 +Create Date: 2026-06-08 + +Adds the first durable-artifact tables: saved_queries (named, owned, +re-runnable question + pinned SQL + typed params), charts (a persisted +visualization config per saved query), and result_snapshots (persisted +results that double as the result cache, keyed by sql_hash + taken_at). + +All tables are connection-scoped (the workspace cascade root) and carry +organization_id for SaaS-readiness, matching the Phase-1 metadata pattern. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects.postgresql import JSONB, UUID + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "005" +down_revision: str = "004" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.create_table( + "saved_queries", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "organization_id", + UUID(as_uuid=True), + sa.ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "connection_id", + UUID(as_uuid=True), + sa.ForeignKey("database_connections.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "owner_id", + UUID(as_uuid=True), + sa.ForeignKey("users.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("description", sa.Text, nullable=True), + sa.Column("nl_question", sa.Text, nullable=True), + sa.Column("pinned_sql", sa.Text, nullable=False), + sa.Column("params", JSONB, server_default=sa.text("'[]'::jsonb")), + sa.Column("version", sa.Integer, nullable=False, server_default=sa.text("1")), + sa.Column("status", sa.String(20), nullable=False, server_default=sa.text("'draft'")), + sa.Column("is_public", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + ) + op.create_index("ix_saved_queries_connection_id", "saved_queries", ["connection_id"]) + + op.create_table( + "charts", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "organization_id", + UUID(as_uuid=True), + sa.ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "saved_query_id", + UUID(as_uuid=True), + sa.ForeignKey("saved_queries.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("chart_type", sa.String(20), nullable=False), + sa.Column("config", JSONB, server_default=sa.text("'{}'::jsonb")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + ) + op.create_index("ix_charts_saved_query_id", "charts", ["saved_query_id"]) + + op.create_table( + "result_snapshots", + sa.Column("id", UUID(as_uuid=True), primary_key=True), + sa.Column( + "organization_id", + UUID(as_uuid=True), + sa.ForeignKey("organizations.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "connection_id", + UUID(as_uuid=True), + sa.ForeignKey("database_connections.id", ondelete="CASCADE"), + nullable=False, + ), + sa.Column( + "saved_query_id", + UUID(as_uuid=True), + sa.ForeignKey("saved_queries.id", ondelete="SET NULL"), + nullable=True, + ), + sa.Column("sql_hash", sa.String(64), nullable=False), + sa.Column("columns", JSONB, server_default=sa.text("'[]'::jsonb")), + sa.Column("column_types", JSONB, server_default=sa.text("'[]'::jsonb")), + sa.Column("rows", JSONB, server_default=sa.text("'[]'::jsonb")), + sa.Column("row_count", sa.Integer, nullable=True), + sa.Column("params_used", JSONB, server_default=sa.text("'{}'::jsonb")), + sa.Column("execution_time_ms", sa.Float, nullable=True), + sa.Column("truncated", sa.Boolean, server_default=sa.text("false")), + sa.Column("taken_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + ) + op.create_index( + "ix_result_snapshots_sql_hash_taken_at", + "result_snapshots", + ["sql_hash", sa.text("taken_at DESC")], + ) + + +def downgrade() -> None: + op.drop_index("ix_result_snapshots_sql_hash_taken_at", table_name="result_snapshots") + op.drop_table("result_snapshots") + op.drop_index("ix_charts_saved_query_id", table_name="charts") + op.drop_table("charts") + op.drop_index("ix_saved_queries_connection_id", table_name="saved_queries") + op.drop_table("saved_queries") diff --git a/backend/app/api/v1/endpoints/saved_queries.py b/backend/app/api/v1/endpoints/saved_queries.py new file mode 100644 index 0000000..d04b955 --- /dev/null +++ b/backend/app/api/v1/endpoints/saved_queries.py @@ -0,0 +1,325 @@ +import csv +import io +import json +import uuid + +from fastapi import APIRouter, Depends, Query +from fastapi.responses import StreamingResponse +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.api.v1.deps import require_connection_read, require_connection_write +from app.api.v1.schemas.chart import ChartCreate, ChartResponse, ChartUpdate +from app.api.v1.schemas.saved_query import ( + SavedQueryCreate, + SavedQueryResponse, + SavedQueryRunRequest, + SavedQueryRunResponse, + SavedQueryUpdate, +) +from app.core.auth import AuthContext +from app.core.exceptions import AppError, NotFoundError +from app.db.models.chart import Chart +from app.db.models.saved_query import SavedQuery +from app.db.session import get_db +from app.services import saved_query_service + +router = APIRouter(tags=["saved-queries"]) + + +async def _get_saved_query( + db: AsyncSession, connection_id: uuid.UUID, saved_query_id: uuid.UUID +) -> SavedQuery: + saved = await db.get(SavedQuery, saved_query_id) + if not saved or saved.connection_id != connection_id: + raise NotFoundError("SavedQuery", str(saved_query_id)) + return saved + + +# --------------------------------------------------------------------------- # +# Saved-query CRUD +# --------------------------------------------------------------------------- # +@router.get( + "/connections/{connection_id}/saved-queries", + response_model=list[SavedQueryResponse], +) +async def list_saved_queries( + connection_id: uuid.UUID, + _ctx: AuthContext = Depends(require_connection_read), + db: AsyncSession = Depends(get_db), +): + result = await db.execute( + select(SavedQuery) + .where(SavedQuery.connection_id == connection_id) + .order_by(SavedQuery.updated_at.desc()) + ) + return list(result.scalars().all()) + + +@router.post( + "/connections/{connection_id}/saved-queries", + response_model=SavedQueryResponse, + status_code=201, +) +async def create_saved_query( + connection_id: uuid.UUID, + body: SavedQueryCreate, + ctx: AuthContext = Depends(require_connection_write), + db: AsyncSession = Depends(get_db), +): + data = body.model_dump() + data["params"] = [p.model_dump() for p in body.params] + saved = SavedQuery( + connection_id=connection_id, + organization_id=ctx.organization_id, + owner_id=ctx.user_id, + **data, + ) + db.add(saved) + await db.flush() + return saved + + +@router.get( + "/connections/{connection_id}/saved-queries/{saved_query_id}", + response_model=SavedQueryResponse, +) +async def get_saved_query( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + _ctx: AuthContext = Depends(require_connection_read), + db: AsyncSession = Depends(get_db), +): + return await _get_saved_query(db, connection_id, saved_query_id) + + +@router.put( + "/connections/{connection_id}/saved-queries/{saved_query_id}", + response_model=SavedQueryResponse, +) +async def update_saved_query( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + body: SavedQueryUpdate, + _ctx: AuthContext = Depends(require_connection_write), + db: AsyncSession = Depends(get_db), +): + saved = await _get_saved_query(db, connection_id, saved_query_id) + updates = body.model_dump(exclude_unset=True) + if "params" in updates and body.params is not None: + updates["params"] = [p.model_dump() for p in body.params] + # Bump version when the executable SQL changes. + if "pinned_sql" in updates and updates["pinned_sql"] != saved.pinned_sql: + saved.version += 1 + for key, value in updates.items(): + setattr(saved, key, value) + await db.flush() + return saved + + +@router.delete( + "/connections/{connection_id}/saved-queries/{saved_query_id}", + status_code=204, +) +async def delete_saved_query( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + _ctx: AuthContext = Depends(require_connection_write), + db: AsyncSession = Depends(get_db), +): + saved = await _get_saved_query(db, connection_id, saved_query_id) + await db.delete(saved) + await db.flush() + + +@router.post( + "/connections/{connection_id}/saved-queries/{saved_query_id}/clone", + response_model=SavedQueryResponse, + status_code=201, +) +async def clone_saved_query( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + ctx: AuthContext = Depends(require_connection_write), + db: AsyncSession = Depends(get_db), +): + src = await _get_saved_query(db, connection_id, saved_query_id) + clone = SavedQuery( + connection_id=connection_id, + organization_id=ctx.organization_id, + owner_id=ctx.user_id, + name=f"{src.name} (copy)", + description=src.description, + nl_question=src.nl_question, + pinned_sql=src.pinned_sql, + params=src.params, + status="draft", + is_public=False, + ) + db.add(clone) + await db.flush() + return clone + + +# --------------------------------------------------------------------------- # +# Run + export +# --------------------------------------------------------------------------- # +@router.post( + "/connections/{connection_id}/saved-queries/{saved_query_id}/run", + response_model=SavedQueryRunResponse, +) +async def run_saved_query( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + body: SavedQueryRunRequest, + ctx: AuthContext = Depends(require_connection_read), + db: AsyncSession = Depends(get_db), +): + saved = await _get_saved_query(db, connection_id, saved_query_id) + return await saved_query_service.run_saved_query( + db, saved, ctx, body.params, refresh=body.refresh + ) + + +@router.get("/connections/{connection_id}/saved-queries/{saved_query_id}/export") +async def export_saved_query( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + format: str = Query("csv", pattern="^(csv|json|xlsx)$"), + ctx: AuthContext = Depends(require_connection_read), + db: AsyncSession = Depends(get_db), +): + saved = await _get_saved_query(db, connection_id, saved_query_id) + result = await saved_query_service.run_saved_query(db, saved, ctx, {}) + columns: list[str] = result["columns"] + rows: list[list] = result["rows"] + base = saved.name.replace(" ", "_") or "result" + + if format == "json": + payload = [dict(zip(columns, row, strict=False)) for row in rows] + return StreamingResponse( + iter([json.dumps(payload, default=str)]), + media_type="application/json", + headers={"Content-Disposition": f'attachment; filename="{base}.json"'}, + ) + + if format == "csv": + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerow(columns) + writer.writerows(rows) + return StreamingResponse( + iter([buf.getvalue()]), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{base}.csv"'}, + ) + + # xlsx — optional dependency + try: + from openpyxl import Workbook # type: ignore[import-untyped] + except ImportError as exc: + raise AppError( + "XLSX export requires the 'openpyxl' package (install the backend '[export]' extra).", + status_code=422, + ) from exc + + wb = Workbook() + ws = wb.active + ws.append(columns) + for row in rows: + ws.append(list(row)) + buf_bytes = io.BytesIO() + wb.save(buf_bytes) + buf_bytes.seek(0) + return StreamingResponse( + buf_bytes, + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + headers={"Content-Disposition": f'attachment; filename="{base}.xlsx"'}, + ) + + +# --------------------------------------------------------------------------- # +# Charts (sub-resource of a saved query) +# --------------------------------------------------------------------------- # +async def _get_chart(db: AsyncSession, saved_query_id: uuid.UUID, chart_id: uuid.UUID) -> Chart: + chart = await db.get(Chart, chart_id) + if not chart or chart.saved_query_id != saved_query_id: + raise NotFoundError("Chart", str(chart_id)) + return chart + + +@router.get( + "/connections/{connection_id}/saved-queries/{saved_query_id}/charts", + response_model=list[ChartResponse], +) +async def list_charts( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + _ctx: AuthContext = Depends(require_connection_read), + db: AsyncSession = Depends(get_db), +): + await _get_saved_query(db, connection_id, saved_query_id) + result = await db.execute( + select(Chart).where(Chart.saved_query_id == saved_query_id).order_by(Chart.created_at) + ) + return list(result.scalars().all()) + + +@router.post( + "/connections/{connection_id}/saved-queries/{saved_query_id}/charts", + response_model=ChartResponse, + status_code=201, +) +async def create_chart( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + body: ChartCreate, + ctx: AuthContext = Depends(require_connection_write), + db: AsyncSession = Depends(get_db), +): + await _get_saved_query(db, connection_id, saved_query_id) + chart = Chart( + organization_id=ctx.organization_id, + saved_query_id=saved_query_id, + **body.model_dump(), + ) + db.add(chart) + await db.flush() + return chart + + +@router.put( + "/connections/{connection_id}/saved-queries/{saved_query_id}/charts/{chart_id}", + response_model=ChartResponse, +) +async def update_chart( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + chart_id: uuid.UUID, + body: ChartUpdate, + _ctx: AuthContext = Depends(require_connection_write), + db: AsyncSession = Depends(get_db), +): + await _get_saved_query(db, connection_id, saved_query_id) + chart = await _get_chart(db, saved_query_id, chart_id) + for key, value in body.model_dump(exclude_unset=True).items(): + setattr(chart, key, value) + await db.flush() + return chart + + +@router.delete( + "/connections/{connection_id}/saved-queries/{saved_query_id}/charts/{chart_id}", + status_code=204, +) +async def delete_chart( + connection_id: uuid.UUID, + saved_query_id: uuid.UUID, + chart_id: uuid.UUID, + _ctx: AuthContext = Depends(require_connection_write), + db: AsyncSession = Depends(get_db), +): + await _get_saved_query(db, connection_id, saved_query_id) + chart = await _get_chart(db, saved_query_id, chart_id) + await db.delete(chart) + await db.flush() diff --git a/backend/app/api/v1/router.py b/backend/app/api/v1/router.py index 0a1e572..698f901 100644 --- a/backend/app/api/v1/router.py +++ b/backend/app/api/v1/router.py @@ -13,6 +13,7 @@ query, query_history, sample_queries, + saved_queries, schemas, teams, ) @@ -31,5 +32,6 @@ api_router.include_router(metrics.router) api_router.include_router(dictionary.router) api_router.include_router(sample_queries.router) +api_router.include_router(saved_queries.router) api_router.include_router(query_history.router) api_router.include_router(knowledge.router) diff --git a/backend/app/api/v1/schemas/chart.py b/backend/app/api/v1/schemas/chart.py new file mode 100644 index 0000000..4fefde2 --- /dev/null +++ b/backend/app/api/v1/schemas/chart.py @@ -0,0 +1,31 @@ +from datetime import datetime +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field + +ChartType = Literal["table", "line", "bar", "pie", "area", "scatter"] + + +class ChartCreate(BaseModel): + name: str = Field(min_length=1, max_length=255) + chart_type: ChartType + config: dict = Field(default_factory=dict) + + +class ChartUpdate(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=255) + chart_type: ChartType | None = None + config: dict | None = None + + +class ChartResponse(BaseModel): + id: UUID + saved_query_id: UUID + name: str + chart_type: str + config: dict | None + created_at: datetime + updated_at: datetime + + model_config = {"from_attributes": True} diff --git a/backend/app/api/v1/schemas/saved_query.py b/backend/app/api/v1/schemas/saved_query.py new file mode 100644 index 0000000..99030fd --- /dev/null +++ b/backend/app/api/v1/schemas/saved_query.py @@ -0,0 +1,68 @@ +from datetime import datetime +from typing import Any, Literal +from uuid import UUID + +from pydantic import BaseModel, Field + +ParamType = Literal["string", "number", "date", "boolean"] + + +class ParamDef(BaseModel): + name: str = Field(min_length=1, max_length=64, pattern=r"^[A-Za-z_][A-Za-z0-9_]*$") + type: ParamType = "string" + label: str | None = None + default: Any | None = None + + +class SavedQueryCreate(BaseModel): + name: str = Field(min_length=1, max_length=255) + description: str | None = None + nl_question: str | None = None + pinned_sql: str = Field(min_length=1) + params: list[ParamDef] = Field(default_factory=list) + status: str = "draft" + is_public: bool = False + + +class SavedQueryUpdate(BaseModel): + name: str | None = Field(default=None, min_length=1, max_length=255) + description: str | None = None + nl_question: str | None = None + pinned_sql: str | None = Field(default=None, min_length=1) + params: list[ParamDef] | None = None + status: str | None = None + is_public: bool | None = None + + +class SavedQueryResponse(BaseModel): + id: UUID + connection_id: UUID + owner_id: UUID | None + name: str + description: str | None + nl_question: str | None + pinned_sql: str + params: list[ParamDef] | None + version: int + status: str + is_public: bool + created_at: datetime + updated_at: datetime + + model_config = {"from_attributes": True} + + +class SavedQueryRunRequest(BaseModel): + params: dict[str, Any] = Field(default_factory=dict) + refresh: bool = False + + +class SavedQueryRunResponse(BaseModel): + columns: list[str] + column_types: list[str] + rows: list[list[Any]] + row_count: int + truncated: bool + execution_time_ms: float | None + cached: bool + taken_at: datetime diff --git a/backend/app/config.py b/backend/app/config.py index 162be4f..0fbf59a 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -39,6 +39,8 @@ class Settings(BaseSettings): default_query_timeout_seconds: int = 30 default_max_rows: int = 1000 max_retry_attempts: int = 3 + # Result-cache freshness window for saved-query runs / dashboards. + result_cache_ttl_seconds: int = 300 # LLM defaults default_llm_provider: str = "anthropic" diff --git a/backend/app/db/models/__init__.py b/backend/app/db/models/__init__.py index 6da6387..fe4e81f 100644 --- a/backend/app/db/models/__init__.py +++ b/backend/app/db/models/__init__.py @@ -1,4 +1,5 @@ from app.db.models.api_key import ApiKey +from app.db.models.chart import Chart from app.db.models.connection import DatabaseConnection from app.db.models.dictionary import DictionaryEntry from app.db.models.glossary import GlossaryTerm @@ -7,7 +8,9 @@ from app.db.models.metric import MetricDefinition from app.db.models.organization import Organization from app.db.models.query_history import QueryExecution +from app.db.models.result_snapshot import ResultSnapshot from app.db.models.sample_query import SampleQuery +from app.db.models.saved_query import SavedQuery from app.db.models.schema_cache import CachedColumn, CachedRelationship, CachedTable from app.db.models.team import Team from app.db.models.user import User @@ -29,4 +32,7 @@ "QueryExecution", "KnowledgeDocument", "KnowledgeChunk", + "SavedQuery", + "Chart", + "ResultSnapshot", ] diff --git a/backend/app/db/models/chart.py b/backend/app/db/models/chart.py new file mode 100644 index 0000000..b77c138 --- /dev/null +++ b/backend/app/db/models/chart.py @@ -0,0 +1,34 @@ +import uuid +from datetime import datetime + +from sqlalchemy import DateTime, ForeignKey, String, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base + + +class Chart(Base): + """A persisted visualization config attached to a saved query.""" + + __tablename__ = "charts" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False + ) + saved_query_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("saved_queries.id", ondelete="CASCADE"), nullable=False + ) + name: Mapped[str] = mapped_column(String(255), nullable=False) + # table | line | bar | pie | area | scatter + chart_type: Mapped[str] = mapped_column(String(20), nullable=False) + # {x_axis, y_axis: [...], series, options} + config: Mapped[dict | None] = mapped_column(JSONB, default=dict) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + # Relationships + saved_query: Mapped["SavedQuery"] = relationship(back_populates="charts") # noqa: F821 diff --git a/backend/app/db/models/result_snapshot.py b/backend/app/db/models/result_snapshot.py new file mode 100644 index 0000000..a565405 --- /dev/null +++ b/backend/app/db/models/result_snapshot.py @@ -0,0 +1,45 @@ +import uuid +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, Float, ForeignKey, Index, Integer, String, func, text +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column + +from app.db.base import Base + + +class ResultSnapshot(Base): + """A persisted query result — doubles as the result cache. + + A cache hit is the newest snapshot for a given ``sql_hash`` within the + freshness window (``RESULT_CACHE_TTL_SECONDS``). ``sql_hash`` is + ``sha256(final_sql + json(params_used) + connection_id)``. + """ + + __tablename__ = "result_snapshots" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False + ) + connection_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("database_connections.id", ondelete="CASCADE"), + nullable=False, + ) + saved_query_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("saved_queries.id", ondelete="SET NULL") + ) + sql_hash: Mapped[str] = mapped_column(String(64), nullable=False) + columns: Mapped[list | None] = mapped_column(JSONB, default=list) + column_types: Mapped[list | None] = mapped_column(JSONB, default=list) + rows: Mapped[list | None] = mapped_column(JSONB, default=list) + row_count: Mapped[int | None] = mapped_column(Integer) + params_used: Mapped[dict | None] = mapped_column(JSONB, default=dict) + execution_time_ms: Mapped[float | None] = mapped_column(Float) + truncated: Mapped[bool] = mapped_column(Boolean, default=False) + taken_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + __table_args__ = ( + Index("ix_result_snapshots_sql_hash_taken_at", "sql_hash", text("taken_at DESC")), + ) diff --git a/backend/app/db/models/saved_query.py b/backend/app/db/models/saved_query.py new file mode 100644 index 0000000..0237040 --- /dev/null +++ b/backend/app/db/models/saved_query.py @@ -0,0 +1,52 @@ +import uuid +from datetime import datetime + +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from app.db.base import Base + + +class SavedQuery(Base): + """A named, owned, re-runnable question + pinned SQL with typed parameters. + + Scoped by ``connection_id`` (the workspace cascade root) like the rest of + the semantic layer; workspace isolation is enforced through the connection. + """ + + __tablename__ = "saved_queries" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("organizations.id", ondelete="CASCADE"), nullable=False + ) + connection_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("database_connections.id", ondelete="CASCADE"), + nullable=False, + ) + owner_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="SET NULL") + ) + name: Mapped[str] = mapped_column(String(255), nullable=False) + description: Mapped[str | None] = mapped_column(Text) + nl_question: Mapped[str | None] = mapped_column(Text) + pinned_sql: Mapped[str] = mapped_column(Text, nullable=False) + # List of param defs: {name, type: string|number|date|boolean, label, default} + params: Mapped[list | None] = mapped_column(JSONB, default=list) + version: Mapped[int] = mapped_column(Integer, default=1, nullable=False) + # Forward-compat for Phase 3 certification (draft|certified|deprecated). + status: Mapped[str] = mapped_column(String(20), default="draft", nullable=False) + # Visible to the whole workspace vs. owner-only. + is_public: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + # Relationships + connection: Mapped["DatabaseConnection"] = relationship() # noqa: F821 + charts: Mapped[list["Chart"]] = relationship( # noqa: F821 + back_populates="saved_query", cascade="all, delete-orphan" + ) diff --git a/backend/app/services/saved_query_service.py b/backend/app/services/saved_query_service.py new file mode 100644 index 0000000..c34857f --- /dev/null +++ b/backend/app/services/saved_query_service.py @@ -0,0 +1,197 @@ +"""Saved-query service — parameter rendering, result caching, re-runs. + +Re-running a saved query renders its typed parameters into the pinned SQL and +goes through :func:`query_service.execute_raw_sql`, which already enforces the +read-only SQL safety blocklist, executes via the connector, and records history. +Results are persisted as :class:`ResultSnapshot` rows which double as the cache. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import uuid +from datetime import UTC, date, datetime, timedelta +from typing import Any + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from app.config import settings +from app.core.auth import AuthContext +from app.core.exceptions import AppError +from app.db.models.result_snapshot import ResultSnapshot +from app.db.models.saved_query import SavedQuery +from app.services import query_service + +_PLACEHOLDER_RE = re.compile(r"\{\{\s*(\w+)\s*\}\}") + + +class ParamError(AppError): + """Raised when supplied parameters are missing, unknown, or ill-typed.""" + + def __init__(self, message: str) -> None: + super().__init__(message, status_code=422) + + +def _render_value(name: str, ptype: str, value: Any) -> str: + """Render a single parameter as a safe SQL literal. + + Numbers/booleans are validated and inlined bare; strings/dates are validated + and emitted as single-quoted literals with embedded quotes doubled. The + rendered SQL is still run through ``check_sql_safety`` downstream, so this is + defense-in-depth against breaking out of a literal — not the only guard. + """ + if ptype == "number": + if isinstance(value, bool): + raise ParamError(f"Parameter '{name}' must be a number.") + try: + num = float(value) + except (TypeError, ValueError) as exc: + raise ParamError(f"Parameter '{name}' must be a number.") from exc + if num != num or num in (float("inf"), float("-inf")): + raise ParamError(f"Parameter '{name}' must be a finite number.") + # Preserve integers without a trailing .0 + rendered = int(num) if num.is_integer() else num + return str(rendered) + + if ptype == "boolean": + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, str) and value.strip().lower() in {"true", "false"}: + return "TRUE" if value.strip().lower() == "true" else "FALSE" + raise ParamError(f"Parameter '{name}' must be a boolean.") + + if ptype == "date": + if isinstance(value, (date, datetime)): + iso = value.date().isoformat() if isinstance(value, datetime) else value.isoformat() + else: + try: + iso = date.fromisoformat(str(value)).isoformat() + except ValueError as exc: + raise ParamError(f"Parameter '{name}' must be a date (YYYY-MM-DD).") from exc + return f"'{iso}'" + + # string (default) + text = str(value).replace("\x00", "") + escaped = text.replace("'", "''") + return f"'{escaped}'" + + +def render_sql( + pinned_sql: str, + param_defs: list[dict] | None, + supplied: dict[str, Any] | None, +) -> str: + """Substitute ``{{name}}`` placeholders with type-safe SQL literals.""" + defs = {d["name"]: d for d in (param_defs or [])} + supplied = supplied or {} + + placeholders = set(_PLACEHOLDER_RE.findall(pinned_sql)) + unknown = placeholders - set(defs) + if unknown: + raise ParamError(f"SQL references undefined parameter(s): {', '.join(sorted(unknown))}.") + + rendered: dict[str, str] = {} + for name, d in defs.items(): + if name not in placeholders: + continue + if name in supplied and supplied[name] is not None: + value = supplied[name] + elif d.get("default") is not None: + value = d["default"] + else: + raise ParamError(f"Missing required parameter '{name}'.") + rendered[name] = _render_value(name, d.get("type", "string"), value) + + return _PLACEHOLDER_RE.sub(lambda m: rendered[m.group(1)], pinned_sql) + + +def compute_sql_hash(final_sql: str, params_used: dict[str, Any], connection_id: uuid.UUID) -> str: + payload = "|".join( + [ + final_sql, + json.dumps(params_used, sort_keys=True, default=str), + str(connection_id), + ] + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +async def _latest_fresh_snapshot(db: AsyncSession, sql_hash: str) -> ResultSnapshot | None: + cutoff = datetime.now(UTC) - timedelta(seconds=settings.result_cache_ttl_seconds) + result = await db.execute( + select(ResultSnapshot) + .where(ResultSnapshot.sql_hash == sql_hash, ResultSnapshot.taken_at >= cutoff) + .order_by(ResultSnapshot.taken_at.desc()) + .limit(1) + ) + return result.scalar_one_or_none() + + +async def run_saved_query( + db: AsyncSession, + saved: SavedQuery, + ctx: AuthContext, + supplied_params: dict[str, Any] | None = None, + *, + refresh: bool = False, +) -> dict: + """Run a saved query (cache-first) and return a run-result dict. + + Returns the same fields as the query pipeline plus ``cached`` and ``taken_at``. + """ + supplied_params = supplied_params or {} + final_sql = render_sql(saved.pinned_sql, saved.params, supplied_params) + sql_hash = compute_sql_hash(final_sql, supplied_params, saved.connection_id) + + if not refresh: + snap = await _latest_fresh_snapshot(db, sql_hash) + if snap is not None: + return { + "columns": snap.columns or [], + "column_types": snap.column_types or [], + "rows": snap.rows or [], + "row_count": snap.row_count or 0, + "truncated": snap.truncated, + "execution_time_ms": snap.execution_time_ms, + "cached": True, + "taken_at": snap.taken_at, + } + + # Cache miss (or forced refresh): execute and persist a fresh snapshot. + result = await query_service.execute_raw_sql( + db, + saved.connection_id, + final_sql, + ctx, + original_question=saved.nl_question or saved.name, + ) + + snapshot = ResultSnapshot( + organization_id=ctx.organization_id, + connection_id=saved.connection_id, + saved_query_id=saved.id, + sql_hash=sql_hash, + columns=result["columns"], + column_types=result["column_types"], + rows=result["rows"], + row_count=result["row_count"], + params_used=supplied_params, + execution_time_ms=result["execution_time_ms"], + truncated=result["truncated"], + ) + db.add(snapshot) + await db.flush() + + return { + "columns": result["columns"], + "column_types": result["column_types"], + "rows": result["rows"], + "row_count": result["row_count"], + "truncated": result["truncated"], + "execution_time_ms": result["execution_time_ms"], + "cached": False, + "taken_at": snapshot.taken_at, + } diff --git a/backend/pyproject.toml b/backend/pyproject.toml index c3c3b5c..d91af8e 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -31,6 +31,9 @@ bigquery = [ databricks = [ "databricks-sql-connector[pyarrow]>=3.0", ] +export = [ + "openpyxl>=3.1", +] observability = [ "structlog>=24.0", "prometheus-client>=0.20", diff --git a/backend/tests/test_saved_query_service.py b/backend/tests/test_saved_query_service.py new file mode 100644 index 0000000..566f2db --- /dev/null +++ b/backend/tests/test_saved_query_service.py @@ -0,0 +1,104 @@ +"""Unit tests for saved_query_service param rendering + cache hashing (no DB).""" + +import uuid + +import pytest + +from app.core.exceptions import AppError +from app.services import saved_query_service as svc + + +def _defs(*items): + return [dict(item) for item in items] + + +# --------------------------------------------------------------------------- # +# render_sql — type coercion + escaping +# --------------------------------------------------------------------------- # +def test_render_string_escapes_single_quotes(): + sql = svc.render_sql( + "select * from t where name = {{n}}", + _defs({"name": "n", "type": "string"}), + {"n": "O'Brien"}, + ) + assert sql == "select * from t where name = 'O''Brien'" + + +def test_render_number_inlined_bare_and_integers_have_no_decimal(): + sql = svc.render_sql( + "select {{a}}, {{b}}", + _defs({"name": "a", "type": "number"}, {"name": "b", "type": "number"}), + {"a": 5, "b": 2.5}, + ) + assert sql == "select 5, 2.5" + + +def test_render_boolean_and_date(): + sql = svc.render_sql( + "select {{b}} where d > {{d}}", + _defs({"name": "b", "type": "boolean"}, {"name": "d", "type": "date"}), + {"b": True, "d": "2024-01-31"}, + ) + assert sql == "select TRUE where d > '2024-01-31'" + + +def test_render_uses_default_when_value_missing(): + sql = svc.render_sql( + "select {{r}}", + _defs({"name": "r", "type": "string", "default": "EU"}), + {}, + ) + assert sql == "select 'EU'" + + +def test_render_no_params_passthrough(): + assert svc.render_sql("select 1", None, None) == "select 1" + + +def test_render_rejects_unknown_placeholder(): + with pytest.raises(AppError): + svc.render_sql("select {{missing}}", [], {}) + + +def test_render_rejects_missing_required_param(): + with pytest.raises(AppError): + svc.render_sql("select {{x}}", _defs({"name": "x", "type": "number"}), {}) + + +@pytest.mark.parametrize( + "ptype,value", + [("number", "notnum"), ("number", True), ("boolean", "maybe"), ("date", "31-01-2024")], +) +def test_render_rejects_bad_types(ptype, value): + with pytest.raises(AppError): + svc.render_sql("select {{x}}", _defs({"name": "x", "type": ptype}), {"x": value}) + + +def test_render_number_rejects_infinity(): + with pytest.raises(AppError): + svc.render_sql("select {{x}}", _defs({"name": "x", "type": "number"}), {"x": float("inf")}) + + +# --------------------------------------------------------------------------- # +# compute_sql_hash — determinism + sensitivity +# --------------------------------------------------------------------------- # +def test_sql_hash_is_deterministic(): + cid = uuid.uuid4() + h1 = svc.compute_sql_hash("select 1", {"a": 1}, cid) + h2 = svc.compute_sql_hash("select 1", {"a": 1}, cid) + assert h1 == h2 and len(h1) == 64 + + +def test_sql_hash_param_order_independent(): + cid = uuid.uuid4() + h1 = svc.compute_sql_hash("select 1", {"a": 1, "b": 2}, cid) + h2 = svc.compute_sql_hash("select 1", {"b": 2, "a": 1}, cid) + assert h1 == h2 + + +def test_sql_hash_changes_with_sql_params_or_connection(): + cid = uuid.uuid4() + base = svc.compute_sql_hash("select 1", {"a": 1}, cid) + assert base != svc.compute_sql_hash("select 2", {"a": 1}, cid) + assert base != svc.compute_sql_hash("select 1", {"a": 2}, cid) + assert base != svc.compute_sql_hash("select 1", {"a": 1}, uuid.uuid4()) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d0c37db..828af56 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,11 +1,11 @@ { - "name": "frontend", + "name": "querywise-frontend", "version": "0.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "frontend", + "name": "querywise-frontend", "version": "0.0.0", "dependencies": { "@mantine/code-highlight": "^8.3.15", @@ -19,7 +19,8 @@ "axios": "^1.13.5", "react": "^19.2.0", "react-dom": "^19.2.0", - "react-router-dom": "^7.13.0" + "react-router-dom": "^7.13.0", + "recharts": "^3.8.1" }, "devDependencies": { "@eslint/js": "^9.39.1", @@ -1185,6 +1186,42 @@ "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/@reduxjs/toolkit": { + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/@reduxjs/toolkit/-/toolkit-2.12.0.tgz", + "integrity": "sha512-KiT+RzZbp6mQET+Mg+h2c97+9j1sNflUxQkIHI7Yuzf6Peu+OYpmkn6nbHWmLLWj+1ZODUJFwGZ7gx3L9R9EOw==", + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@standard-schema/utils": "^0.3.0", + "immer": "^11.0.0", + "redux": "^5.0.1", + "redux-thunk": "^3.1.0", + "reselect": "^5.1.0" + }, + "peerDependencies": { + "react": "^16.9.0 || ^17.0.0 || ^18 || ^19", + "react-redux": "^7.2.1 || ^8.1.3 || ^9.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-redux": { + "optional": true + } + } + }, + "node_modules/@reduxjs/toolkit/node_modules/immer": { + "version": "11.1.8", + "resolved": "https://registry.npmjs.org/immer/-/immer-11.1.8.tgz", + "integrity": "sha512-/tbkHMW7y10Lx6i1crLjD4/OhNkRG+Fo7byZHtah0547nIeXYcpIXaUh0IAQY6gO5459qpGGYapcEOHtFXkIuA==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/immer" + } + }, "node_modules/@rolldown/pluginutils": { "version": "1.0.0-rc.3", "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.3.tgz", @@ -1542,6 +1579,18 @@ "win32" ] }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@standard-schema/utils": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@standard-schema/utils/-/utils-0.3.0.tgz", + "integrity": "sha512-e7Mew686owMaPJVNNLs55PUvgz371nKgwsc4vxE49zsODpJEnxgxRo2y/OKrqueavXgZNMDVj3DdHFlaSAeU8g==", + "license": "MIT" + }, "node_modules/@tabler/icons": { "version": "3.37.1", "resolved": "https://registry.npmjs.org/@tabler/icons/-/icons-3.37.1.tgz", @@ -1639,6 +1688,69 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/d3-array": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.2.2.tgz", + "integrity": "sha512-hOLWVbm7uRza0BYXpIIW5pxfrKe0W+D5lrFiAEYR+pb6w3N2SwSMaJbXdUfSEv+dT4MfHBLtn5js0LAWaO6otw==", + "license": "MIT" + }, + "node_modules/@types/d3-color": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.3.tgz", + "integrity": "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A==", + "license": "MIT" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.2.tgz", + "integrity": "sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA==", + "license": "MIT" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz", + "integrity": "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA==", + "license": "MIT", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.1.1.tgz", + "integrity": "sha512-VMZBYyQvbGmWyWVea0EHs/BwLgxc+MKi1zLDCONksozI4YJMcTt8ZEuIR4Sb1MMTE8MMW49v0IwI5+b7RmfWlg==", + "license": "MIT" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.9", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.9.tgz", + "integrity": "sha512-dLmtwB8zkAeO/juAMfnV+sItKjlsw2lKdZVVy6LRr0cBmegxSABiLEpGVmSJJ8O08i4+sGR6qQtb6WtuwJdvVw==", + "license": "MIT", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-shape": { + "version": "3.1.8", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.8.tgz", + "integrity": "sha512-lae0iWfcDeR7qt7rA88BNiqdvPS5pFVPpo5OfjElwNaT2yyekbM0C9vK+yqBqEmHr6lDkRnYNoTBYlAgJa7a4w==", + "license": "MIT", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.4.tgz", + "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", + "license": "MIT" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.2.tgz", + "integrity": "sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -1691,6 +1803,12 @@ "optional": true, "peer": true }, + "node_modules/@types/use-sync-external-store": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.6.tgz", + "integrity": "sha512-zFDAD+tlpf2r4asuHEj0XH6pY6i0g5NeAHPn+15wk3BV6JA69eERFXC1gyGThDkVa1zCyKr5jox1+2LbV/AMLg==", + "license": "MIT" + }, "node_modules/@typescript-eslint/eslint-plugin": { "version": "8.56.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-8.56.0.tgz", @@ -2289,6 +2407,127 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, + "node_modules/d3-array": { + "version": "3.2.4", + "resolved": "https://registry.npmjs.org/d3-array/-/d3-array-3.2.4.tgz", + "integrity": "sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg==", + "license": "ISC", + "dependencies": { + "internmap": "1 - 2" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-format": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/d3-format/-/d3-format-3.1.2.tgz", + "integrity": "sha512-AJDdYOdnyRDV5b6ArilzCPPwc1ejkHcoyFarqlPqT7zRYjhavcT3uSrqcMvsgh2CgoPbK3RCwyHaVyxYcP2Arg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "license": "ISC", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-path": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-path/-/d3-path-3.1.0.tgz", + "integrity": "sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-scale": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/d3-scale/-/d3-scale-4.0.2.tgz", + "integrity": "sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ==", + "license": "ISC", + "dependencies": { + "d3-array": "2.10.0 - 3", + "d3-format": "1 - 3", + "d3-interpolate": "1.2.0 - 3", + "d3-time": "2.1.1 - 3", + "d3-time-format": "2 - 4" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-shape": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/d3-shape/-/d3-shape-3.2.0.tgz", + "integrity": "sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA==", + "license": "ISC", + "dependencies": { + "d3-path": "^3.1.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-time/-/d3-time-3.1.0.tgz", + "integrity": "sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q==", + "license": "ISC", + "dependencies": { + "d3-array": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-time-format": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/d3-time-format/-/d3-time-format-4.1.0.tgz", + "integrity": "sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg==", + "license": "ISC", + "dependencies": { + "d3-time": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/debug": { "version": "4.4.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", @@ -2307,6 +2546,12 @@ } } }, + "node_modules/decimal.js-light": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/decimal.js-light/-/decimal.js-light-2.5.1.tgz", + "integrity": "sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg==", + "license": "MIT" + }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -2415,6 +2660,16 @@ "node": ">= 0.4" } }, + "node_modules/es-toolkit": { + "version": "1.47.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.47.0.tgz", + "integrity": "sha512-n1GuoD0WEQZMBk5tttoZSqwgyLx01oqa5XsBmCHwPyNe1S9jPBEmtR2pSgp2kJuWE3ciFZ6yRHmY4pM4C3OOkw==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, "node_modules/esbuild": { "version": "0.27.3", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.3.tgz", @@ -2664,6 +2919,12 @@ "node": ">=0.10.0" } }, + "node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -2983,6 +3244,16 @@ "node": ">= 4" } }, + "node_modules/immer": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/immer/-/immer-10.2.0.tgz", + "integrity": "sha512-d/+XTN3zfODyjr89gM3mPq1WNX2B8pYsu7eORitdwyA2sBubnTl3laYlBk4sXY5FUa5qTZGBDPJICVbvqzjlbw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/immer" + } + }, "node_modules/import-fresh": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", @@ -3010,6 +3281,15 @@ "node": ">=0.8.19" } }, + "node_modules/internmap": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", + "integrity": "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg==", + "license": "ISC", + "engines": { + "node": ">=12" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -3506,6 +3786,29 @@ "react-dom": "^0.14 || ^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, + "node_modules/react-redux": { + "version": "9.3.0", + "resolved": "https://registry.npmjs.org/react-redux/-/react-redux-9.3.0.tgz", + "integrity": "sha512-KQopgqFo/p/fgmAs5qz6p5RWaNAzq40WAu7fJIXnQpYxFPbJYtsJPWvGeF2rOBaY/kEuV77AVsX8TsQzKm+A/g==", + "license": "MIT", + "dependencies": { + "@types/use-sync-external-store": "^0.0.6", + "use-sync-external-store": "^1.4.0" + }, + "peerDependencies": { + "@types/react": "^18.2.25 || ^19", + "react": "^18.0 || ^19", + "redux": "^5.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "redux": { + "optional": true + } + } + }, "node_modules/react-refresh": { "version": "0.18.0", "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.18.0.tgz", @@ -3656,6 +3959,57 @@ "react-dom": ">=16.6.0" } }, + "node_modules/recharts": { + "version": "3.8.1", + "resolved": "https://registry.npmjs.org/recharts/-/recharts-3.8.1.tgz", + "integrity": "sha512-mwzmO1s9sFL0TduUpwndxCUNoXsBw3u3E/0+A+cLcrSfQitSG62L32N69GhqUrrT5qKcAE3pCGVINC6pqkBBQg==", + "license": "MIT", + "workspaces": [ + "www" + ], + "dependencies": { + "@reduxjs/toolkit": "^1.9.0 || 2.x.x", + "clsx": "^2.1.1", + "decimal.js-light": "^2.5.1", + "es-toolkit": "^1.39.3", + "eventemitter3": "^5.0.1", + "immer": "^10.1.1", + "react-redux": "8.x.x || 9.x.x", + "reselect": "5.1.1", + "tiny-invariant": "^1.3.3", + "use-sync-external-store": "^1.2.2", + "victory-vendor": "^37.0.2" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-is": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/redux": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/redux/-/redux-5.0.1.tgz", + "integrity": "sha512-M9/ELqF6fy8FwmkpnF0S3YKOqMyoWJ4+CS5Efg2ct3oY9daQvd/Pc71FpGZsVsbl3Cpb+IIcjBDUnnyBdQbq4w==", + "license": "MIT" + }, + "node_modules/redux-thunk": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/redux-thunk/-/redux-thunk-3.1.0.tgz", + "integrity": "sha512-NW2r5T6ksUKXCabzhL9z+h206HQw/NJkcLm1GPImRQ8IzfXwRGqjVhKJGauHirT0DAuyy6hjdnMZaRoAcy0Klw==", + "license": "MIT", + "peerDependencies": { + "redux": "^5.0.0" + } + }, + "node_modules/reselect": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/reselect/-/reselect-5.1.1.tgz", + "integrity": "sha512-K/BG6eIky/SBpzfHZv/dd+9JBFiS4SWV7FIujVyJRux6e45+73RaUHXLmIR1f7WOMaQ0U1km6qwklRQxpJJY0w==", + "license": "MIT" + }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -3804,6 +4158,12 @@ "integrity": "sha512-05PUHKSNE8ou2dwIxTngl4EzcnsCDZGJ/iCLtDflR/SHB/ny14rXc+qU5P4mG9JkusiV7EivzY9Mhm55AzAvCg==", "license": "MIT" }, + "node_modules/tiny-invariant": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/tiny-invariant/-/tiny-invariant-1.3.3.tgz", + "integrity": "sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==", + "license": "MIT" + }, "node_modules/tinyglobby": { "version": "0.2.15", "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.15.tgz", @@ -4039,6 +4399,37 @@ } } }, + "node_modules/use-sync-external-store": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", + "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/victory-vendor": { + "version": "37.3.6", + "resolved": "https://registry.npmjs.org/victory-vendor/-/victory-vendor-37.3.6.tgz", + "integrity": "sha512-SbPDPdDBYp+5MJHhBCAyI7wKM3d5ivekigc2Dk2s7pgbZ9wIgIBYGVw4zGHBml/qTFbexrofXW6Gu4noGxrOwQ==", + "license": "MIT AND ISC", + "dependencies": { + "@types/d3-array": "^3.0.3", + "@types/d3-ease": "^3.0.0", + "@types/d3-interpolate": "^3.0.1", + "@types/d3-scale": "^4.0.2", + "@types/d3-shape": "^3.1.0", + "@types/d3-time": "^3.0.0", + "@types/d3-timer": "^3.0.0", + "d3-array": "^3.1.6", + "d3-ease": "^3.0.1", + "d3-interpolate": "^3.0.1", + "d3-scale": "^4.0.2", + "d3-shape": "^3.1.0", + "d3-time": "^3.0.0", + "d3-timer": "^3.0.1" + } + }, "node_modules/vite": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 6a17218..6a77df9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -21,7 +21,8 @@ "axios": "^1.13.5", "react": "^19.2.0", "react-dom": "^19.2.0", - "react-router-dom": "^7.13.0" + "react-router-dom": "^7.13.0", + "recharts": "^3.8.1" }, "devDependencies": { "@eslint/js": "^9.39.1", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 24df0c9..62a1ebe 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -7,6 +7,7 @@ import { QueryPage } from './pages/QueryPage'; import { ConnectionsPage } from './pages/ConnectionsPage'; import { GlossaryPage } from './pages/GlossaryPage'; import { MetricsPage } from './pages/MetricsPage'; +import { SavedQueriesPage } from './pages/SavedQueriesPage'; import { DictionaryPage } from './pages/DictionaryPage'; import { KnowledgePage } from './pages/KnowledgePage'; import { HistoryPage } from './pages/HistoryPage'; @@ -23,6 +24,7 @@ export default function App() { }> } /> } /> + } /> } /> } /> } /> diff --git a/frontend/src/api/savedQueriesApi.ts b/frontend/src/api/savedQueriesApi.ts new file mode 100644 index 0000000..1f216df --- /dev/null +++ b/frontend/src/api/savedQueriesApi.ts @@ -0,0 +1,62 @@ +import { api } from './client'; +import type { + Chart, + ChartType, + ChartConfig, + SavedQuery, + SavedQueryRunResult, +} from '../types/api'; + +const base = (connectionId: string) => `/connections/${connectionId}/saved-queries`; + +export const savedQueriesApi = { + list: (connectionId: string) => + api.get(base(connectionId)).then((r) => r.data), + get: (connectionId: string, id: string) => + api.get(`${base(connectionId)}/${id}`).then((r) => r.data), + create: (connectionId: string, data: Partial) => + api.post(base(connectionId), data).then((r) => r.data), + update: (connectionId: string, id: string, data: Partial) => + api.put(`${base(connectionId)}/${id}`, data).then((r) => r.data), + delete: (connectionId: string, id: string) => + api.delete(`${base(connectionId)}/${id}`), + clone: (connectionId: string, id: string) => + api.post(`${base(connectionId)}/${id}/clone`).then((r) => r.data), + run: ( + connectionId: string, + id: string, + params: Record = {}, + refresh = false, + ) => + api + .post(`${base(connectionId)}/${id}/run`, { params, refresh }) + .then((r) => r.data), + exportUrl: (connectionId: string, id: string, format: 'csv' | 'json' | 'xlsx') => + `${api.defaults.baseURL}${base(connectionId)}/${id}/export?format=${format}`, +}; + +export const chartsApi = { + list: (connectionId: string, savedQueryId: string) => + api + .get(`${base(connectionId)}/${savedQueryId}/charts`) + .then((r) => r.data), + create: ( + connectionId: string, + savedQueryId: string, + data: { name: string; chart_type: ChartType; config: ChartConfig }, + ) => + api + .post(`${base(connectionId)}/${savedQueryId}/charts`, data) + .then((r) => r.data), + update: ( + connectionId: string, + savedQueryId: string, + chartId: string, + data: Partial<{ name: string; chart_type: ChartType; config: ChartConfig }>, + ) => + api + .put(`${base(connectionId)}/${savedQueryId}/charts/${chartId}`, data) + .then((r) => r.data), + delete: (connectionId: string, savedQueryId: string, chartId: string) => + api.delete(`${base(connectionId)}/${savedQueryId}/charts/${chartId}`), +}; diff --git a/frontend/src/components/charts/ChartView.tsx b/frontend/src/components/charts/ChartView.tsx new file mode 100644 index 0000000..44a4d08 --- /dev/null +++ b/frontend/src/components/charts/ChartView.tsx @@ -0,0 +1,165 @@ +import { Alert } from '@mantine/core'; +import { + Area, + AreaChart, + Bar, + BarChart, + CartesianGrid, + Cell, + Legend, + Line, + LineChart, + Pie, + PieChart, + ResponsiveContainer, + Scatter, + ScatterChart, + Tooltip, + XAxis, + YAxis, + ZAxis, +} from 'recharts'; +import type { ChartType } from '../../types/api'; + +const PALETTE = [ + '#228be6', + '#40c057', + '#fab005', + '#fa5252', + '#7950f2', + '#15aabf', + '#e64980', + '#fd7e14', +]; + +function toNumber(value: unknown): number | null { + if (value === null || value === undefined || value === '') return null; + const n = Number(value); + return Number.isFinite(n) ? n : null; +} + +export interface ChartViewProps { + columns: string[]; + rows: unknown[][]; + chartType: ChartType; + xAxis?: string; + yAxis?: string[]; + height?: number; +} + +/** Renders a query result as a Recharts visualization based on chart config. */ +export function ChartView({ columns, rows, chartType, xAxis, yAxis, height = 360 }: ChartViewProps) { + const xKey = xAxis || columns[0]; + const yKeys = (yAxis && yAxis.length > 0 ? yAxis : columns.slice(1)).filter(Boolean); + + if (!xKey || yKeys.length === 0) { + return Pick an X axis and at least one Y series to render a chart.; + } + + // Project row arrays into objects keyed by column name; coerce Y values to numbers. + const data = rows.map((row) => { + const obj: Record = {}; + columns.forEach((col, i) => { + obj[col] = yKeys.includes(col) ? toNumber(row[i]) : row[i]; + }); + return obj; + }); + + if (chartType === 'pie') { + const valueKey = yKeys[0]; + return ( + + + + + + {data.map((_, i) => ( + + ))} + + + + ); + } + + if (chartType === 'scatter') { + return ( + + + + + + + + + + + + ); + } + + if (chartType === 'area') { + return ( + + + + + + + + {yKeys.map((key, i) => ( + + ))} + + + ); + } + + if (chartType === 'line') { + return ( + + + + + + + + {yKeys.map((key, i) => ( + + ))} + + + ); + } + + // default: bar + return ( + + + + + + + + {yKeys.map((key, i) => ( + + ))} + + + ); +} diff --git a/frontend/src/components/layout/AppLayout.tsx b/frontend/src/components/layout/AppLayout.tsx index b64b047..08ecb41 100644 --- a/frontend/src/components/layout/AppLayout.tsx +++ b/frontend/src/components/layout/AppLayout.tsx @@ -19,6 +19,7 @@ import { IconHistory, IconLogout, IconUserCircle, + IconBookmark, } from '@tabler/icons-react'; import { Outlet, useLocation, useNavigate } from 'react-router-dom'; import { EmbeddingStatusBanner } from '../common/EmbeddingStatusBanner'; @@ -27,6 +28,7 @@ import type { Role } from '../../types/auth'; const NAV_ITEMS = [ { label: 'Query', path: '/query', icon: IconMessageQuestion }, + { label: 'Saved Queries', path: '/saved-queries', icon: IconBookmark }, { label: 'Connections', path: '/connections', icon: IconDatabase }, { label: 'Glossary', path: '/glossary', icon: IconBook }, { label: 'Metrics', path: '/metrics', icon: IconChartBar }, diff --git a/frontend/src/components/query/QueryResultView.tsx b/frontend/src/components/query/QueryResultView.tsx index cef0189..bba0752 100644 --- a/frontend/src/components/query/QueryResultView.tsx +++ b/frontend/src/components/query/QueryResultView.tsx @@ -1,3 +1,4 @@ +import { useState } from 'react'; import { Stack, Paper, @@ -10,13 +11,74 @@ import { CopyButton, ActionIcon, Tooltip, + Button, + Menu, } from '@mantine/core'; -import { IconCopy, IconCheck } from '@tabler/icons-react'; +import { + IconCopy, + IconCheck, + IconDownload, + IconDeviceFloppy, +} from '@tabler/icons-react'; import type { QueryResult } from '../../types/api'; +import { downloadCsv, downloadJson } from '../../utils/exportResult'; +import { SaveQueryModal } from './SaveQueryModal'; + +export function QueryResultView({ + result, + connectionId, +}: { + result: QueryResult; + connectionId?: string | null; +}) { + const [saveOpen, setSaveOpen] = useState(false); + const baseName = result.question || 'query_result'; -export function QueryResultView({ result }: { result: QueryResult }) { return ( + + + + + + + downloadCsv(result.columns, result.rows, baseName)}> + CSV + + downloadJson(result.columns, result.rows, baseName)}> + JSON + + + + {connectionId && ( + + )} + + + {connectionId && ( + setSaveOpen(false)} + connectionId={connectionId} + question={result.question} + sql={result.final_sql} + /> + )} + {result.summary && ( diff --git a/frontend/src/components/query/SaveQueryModal.tsx b/frontend/src/components/query/SaveQueryModal.tsx new file mode 100644 index 0000000..ae86d3d --- /dev/null +++ b/frontend/src/components/query/SaveQueryModal.tsx @@ -0,0 +1,109 @@ +import { + Alert, + Badge, + Button, + Group, + Modal, + Stack, + TextInput, + Textarea, +} from '@mantine/core'; +import { useForm } from '@mantine/form'; +import { notifications } from '@mantine/notifications'; +import { useCreateSavedQuery } from '../../hooks/useSavedQueries'; +import type { ParamDef } from '../../types/api'; + +/** Extract {{name}} placeholders from SQL into string param defs. */ +function detectParams(sql: string): ParamDef[] { + const seen = new Set(); + const defs: ParamDef[] = []; + const re = /\{\{\s*(\w+)\s*\}\}/g; + let m: RegExpExecArray | null; + while ((m = re.exec(sql)) !== null) { + if (!seen.has(m[1])) { + seen.add(m[1]); + defs.push({ name: m[1], type: 'string', label: m[1] }); + } + } + return defs; +} + +export function SaveQueryModal({ + opened, + onClose, + connectionId, + question, + sql, +}: { + opened: boolean; + onClose: () => void; + connectionId: string; + question: string; + sql: string; +}) { + const createMutation = useCreateSavedQuery(connectionId); + const params = detectParams(sql); + + const form = useForm({ + initialValues: { name: question.slice(0, 80) || 'Untitled query', description: '' }, + }); + + const handleSubmit = (values: { name: string; description: string }) => { + createMutation.mutate( + { + name: values.name, + description: values.description || null, + nl_question: question || null, + pinned_sql: sql, + params, + }, + { + onSuccess: () => { + notifications.show({ + title: 'Query saved', + message: `"${values.name}" is now in Saved Queries`, + color: 'green', + }); + form.reset(); + onClose(); + }, + onError: (err) => + notifications.show({ + title: 'Could not save query', + message: (err as Error).message, + color: 'red', + }), + }, + ); + }; + + return ( + +
+ + +