diff --git a/.github/actions/build-model-server-image/action.yml b/.github/actions/build-model-server-image/action.yml index 975a464f0b5..3806c67a8e7 100644 --- a/.github/actions/build-model-server-image/action.yml +++ b/.github/actions/build-model-server-image/action.yml @@ -28,11 +28,15 @@ inputs: description: "ECR registry host for the Docker Hub pull-through cache (the ECR_REGISTRY repo variable)" required: true docker-username: - description: "Docker Hub username (must have access to the DHI catalog on dhi.io)" - required: true + description: >- + Docker Hub username with DHI catalog access. Optional -- without it the image + builds on its public base defaults instead of the hardened ones. + required: false + default: "" docker-token: description: "Docker Hub token" - required: true + required: false + default: "" runs: using: "composite" steps: @@ -59,14 +63,13 @@ runs: with: ecr-registry: ${{ inputs.ecr-registry }} - # Dockerfile.model_server pulls its hardened Python base from dhi.io, authenticated - # with the same Docker account credentials (the account must have DHI catalog access). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # Dockerfile.model_server defaults to the public Python bases. CI builds ship on the + # hardened DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ inputs.docker-username }} - password: ${{ inputs.docker-token }} + docker-username: ${{ inputs.docker-username }} + docker-token: ${{ inputs.docker-token }} - name: Build and push Model Server Docker image uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # ratchet:docker/build-push-action@v6 @@ -77,6 +80,7 @@ runs: platforms: ${{ inputs.platforms }} build-args: | BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }} + ${{ env.DHI_PYTHON_BUILD_ARGS }} tags: ${{ inputs.runs-on-ecr-cache }}:${{ inputs.tag-prefix }}-${{ inputs.run-id }} cache-from: | type=registry,ref=${{ inputs.runs-on-ecr-cache }}:model-server-cache-${{ inputs.github-sha }} diff --git a/.github/actions/dhi-base-images/action.yml b/.github/actions/dhi-base-images/action.yml new file mode 100644 index 00000000000..1bacff551ed --- /dev/null +++ b/.github/actions/dhi-base-images/action.yml @@ -0,0 +1,56 @@ +name: "Docker Hardened Image bases" +description: >- + Logs in to dhi.io and exports the build args that point web/Dockerfile and + backend/Dockerfile.model_server at the pinned Docker Hardened Images (DHI). + Both Dockerfiles default to public Docker Hub images, so without DHI credentials + (fork pull requests) this action exports nothing and the defaults apply. +inputs: + docker-username: + description: "Docker Hub username (the account must have DHI catalog access)" + required: false + default: "" + docker-token: + description: "Docker Hub token" + required: false + default: "" +runs: + using: "composite" + steps: + - name: Check for DHI credentials + id: creds + shell: bash + env: + DOCKER_USERNAME: ${{ inputs.docker-username }} + DOCKER_TOKEN: ${{ inputs.docker-token }} + run: | + if [ -n "${DOCKER_USERNAME}" ] && [ -n "${DOCKER_TOKEN}" ]; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "No DHI credentials; the images build on their public base defaults." + fi + + - name: Login to Docker Hardened Images (dhi.io) + if: steps.creds.outputs.available == 'true' + uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + with: + registry: dhi.io + username: ${{ inputs.docker-username }} + password: ${{ inputs.docker-token }} + + # Single source of truth for the DHI digests. Refresh one with: + # docker buildx imagetools inspect + - name: Export DHI build args + if: steps.creds.outputs.available == 'true' + shell: bash + run: | + { + echo "DHI_NODE_BUILD_ARGS<> "$GITHUB_ENV" diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 3b3db477c5f..93a533264f9 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -535,14 +535,13 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_TOKEN }} - # web/Dockerfile pulls its hardened Node base from dhi.io, authenticated with the same - # Docker account credentials (the account must have access to the DHI catalog). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # web/Dockerfile defaults to the public Node bases. Release builds ship on the + # hardened DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_TOKEN }} + docker-username: ${{ env.DOCKER_USERNAME }} + docker-token: ${{ env.DOCKER_TOKEN }} - name: Build and push AMD64 id: build @@ -554,6 +553,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | ONYX_VERSION=${{ github.ref_name }} + ${{ env.DHI_NODE_BUILD_ARGS }} NODE_OPTIONS=--max-old-space-size=8192 cache-from: | type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge @@ -617,14 +617,13 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_TOKEN }} - # web/Dockerfile pulls its hardened Node base from dhi.io, authenticated with the same - # Docker account credentials (the account must have access to the DHI catalog). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # web/Dockerfile defaults to the public Node bases. Release builds ship on the + # hardened DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_TOKEN }} + docker-username: ${{ env.DOCKER_USERNAME }} + docker-token: ${{ env.DOCKER_TOKEN }} - name: Build and push ARM64 id: build @@ -636,6 +635,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | ONYX_VERSION=${{ github.ref_name }} + ${{ env.DHI_NODE_BUILD_ARGS }} NODE_OPTIONS=--max-old-space-size=8192 cache-from: | type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge @@ -791,14 +791,13 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_TOKEN }} - # web/Dockerfile pulls its hardened Node base from dhi.io, authenticated with the same - # Docker account credentials (the account must have access to the DHI catalog). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # web/Dockerfile defaults to the public Node bases. Release builds ship on the + # hardened DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_TOKEN }} + docker-username: ${{ env.DOCKER_USERNAME }} + docker-token: ${{ env.DOCKER_TOKEN }} - name: Build and push AMD64 id: build @@ -810,6 +809,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | ONYX_VERSION=${{ github.ref_name }} + ${{ env.DHI_NODE_BUILD_ARGS }} NEXT_PUBLIC_CLOUD_ENABLED=true WEB_FRAME_PROTECTION_ENABLED=false NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }} @@ -884,14 +884,13 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_TOKEN }} - # web/Dockerfile pulls its hardened Node base from dhi.io, authenticated with the same - # Docker account credentials (the account must have access to the DHI catalog). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # web/Dockerfile defaults to the public Node bases. Release builds ship on the + # hardened DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_TOKEN }} + docker-username: ${{ env.DOCKER_USERNAME }} + docker-token: ${{ env.DOCKER_TOKEN }} - name: Build and push ARM64 id: build @@ -903,6 +902,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | ONYX_VERSION=${{ github.ref_name }} + ${{ env.DHI_NODE_BUILD_ARGS }} NEXT_PUBLIC_CLOUD_ENABLED=true WEB_FRAME_PROTECTION_ENABLED=false NEXT_PUBLIC_POSTHOG_KEY=${{ secrets.POSTHOG_KEY }} @@ -1351,14 +1351,13 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_TOKEN }} - # Dockerfile.model_server pulls its hardened Python base from dhi.io, authenticated - # with the same Docker account credentials (the account must have DHI catalog access). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # Dockerfile.model_server defaults to the public Python bases. Release builds ship + # on the hardened DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_TOKEN }} + docker-username: ${{ env.DOCKER_USERNAME }} + docker-token: ${{ env.DOCKER_TOKEN }} - name: Build and push AMD64 id: build @@ -1372,6 +1371,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | ONYX_VERSION=${{ github.ref_name }} + ${{ env.DHI_PYTHON_BUILD_ARGS }} cache-from: | type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest @@ -1437,14 +1437,13 @@ jobs: username: ${{ env.DOCKER_USERNAME }} password: ${{ env.DOCKER_TOKEN }} - # Dockerfile.model_server pulls its hardened Python base from dhi.io, authenticated - # with the same Docker account credentials (the account must have DHI catalog access). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # Dockerfile.model_server defaults to the public Python bases. Release builds ship + # on the hardened DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ env.DOCKER_USERNAME }} - password: ${{ env.DOCKER_TOKEN }} + docker-username: ${{ env.DOCKER_USERNAME }} + docker-token: ${{ env.DOCKER_TOKEN }} - name: Build and push ARM64 id: build @@ -1458,6 +1457,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | ONYX_VERSION=${{ github.ref_name }} + ${{ env.DHI_PYTHON_BUILD_ARGS }} cache-from: | type=registry,ref=${{ env.REGISTRY_IMAGE }}:edge type=registry,ref=${{ env.REGISTRY_IMAGE }}:latest diff --git a/.github/workflows/pr-playwright-tests.yml b/.github/workflows/pr-playwright-tests.yml index fb7f9581b3b..ed3b1e576cd 100644 --- a/.github/workflows/pr-playwright-tests.yml +++ b/.github/workflows/pr-playwright-tests.yml @@ -120,6 +120,7 @@ jobs: - '.github/workflows/pr-playwright-tests.yml' - '.github/actions/setup-test-license/**' - '.github/actions/login-ecr-pullthrough-cache/**' + - '.github/actions/dhi-base-images/**' airgap: - 'backend/Dockerfile' - 'backend/Dockerfile.model_server' @@ -147,6 +148,7 @@ jobs: - '.github/workflows/pr-playwright-tests.yml' - '.github/actions/build-model-server-image/**' - '.github/actions/login-ecr-pullthrough-cache/**' + - '.github/actions/dhi-base-images/**' mcp_oauth: - 'backend/onyx/server/features/mcp/**' - 'backend/tests/integration/mock_services/mcp_test_server/**' @@ -196,14 +198,13 @@ jobs: with: ecr-registry: ${{ vars.ECR_REGISTRY }} - # web/Dockerfile pulls its hardened Node base from dhi.io, authenticated with the same - # Docker account credentials (the account must have access to the DHI catalog). - - name: Login to Docker Hardened Images (dhi.io) - uses: docker/login-action@c99871dec2022cc055c062a10cc1a1310835ceb4 + # web/Dockerfile defaults to the public Node bases. CI builds ship on the hardened + # DHI equivalents, passed as build args below. + - name: Resolve Docker Hardened Image bases + uses: ./.github/actions/dhi-base-images with: - registry: dhi.io - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-token: ${{ secrets.DOCKER_TOKEN }} # SKIP_TYPE_CHECK cuts the build time of this image. Types are still checked # by the `typescript-check` prek hook in the Quality Checks PR workflow. @@ -217,6 +218,7 @@ jobs: push: true build-args: | BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }} + ${{ env.DHI_NODE_BUILD_ARGS }} SKIP_TYPE_CHECK=1 cache-from: | type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:web-cache-${{ github.event.pull_request.head.sha || github.sha }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 589b393477b..597df735686 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -306,16 +306,10 @@ If you want to make changes to Onyx and run those changes in Docker, you can als docker compose up -d --build ``` -> **Note:** Building the web image (`web/Dockerfile`) and the model-server image -> (`backend/Dockerfile.model_server`) pulls their bases from Docker Hardened Images (`dhi.io`), -> so you must authenticate first with a Docker account that has access to the DHI catalog: -> -> ```bash -> docker login dhi.io -> ``` -> -> Pulling the pre-built `onyxdotapp/onyx-web-server` / `onyxdotapp/onyx-model-server` images -> (the default `docker compose up -d` without `--build`) does not require this. +> **Note:** Local builds use the public Docker Hub base images, so they need no extra +> registry access. Our release builds override the base images with the Docker Hardened +> Image (`dhi.io`) equivalents, so the published `onyxdotapp/onyx-web-server` and +> `onyxdotapp/onyx-model-server` images differ from a local `--build` in their base layers. > **Note:** `docker-compose.yml`, `docker-compose.prod.yml` and > `docker-compose.prod-no-letsencrypt.yml` are generated from `docker-compose.template.yml` diff --git a/backend/Dockerfile.model_server b/backend/Dockerfile.model_server index 6f888d67b04..39e75947dc3 100644 --- a/backend/Dockerfile.model_server +++ b/backend/Dockerfile.model_server @@ -1,13 +1,18 @@ -# Registry prefix retained for parity with the other Dockerfiles and so the -# BASE_IMAGE_REGISTRY build-arg (passed by docker-bake.hcl / CI) stays valid. -# The hardened Python bases below come from dhi.io -- a separate registry not -# served by the ECR pull-through cache -- so they intentionally bypass this ARG. +# Registry prefix for the base images below. Defaults to Docker Hub; CI overrides it to +# the ECR pull-through cache to dodge rate limits. It only applies to the default images +# -- the DHI overrides below carry their own registry (dhi.io), which the cache does not serve. ARG BASE_IMAGE_REGISTRY=docker.io -# Build stage. The DHI "-dev" variant runs as root and ships a shell, apt, and -# build tooling -- everything needed to install the wheels. -# Refresh the digest with: docker buildx imagetools inspect dhi.io/python:3.13-debian13-dev -FROM dhi.io/python:3.13-debian13-dev@sha256:7933d16e50454c39bca6e935027166d5e5b05c6c03383dc2f58e8fe6e2440b7d AS builder +# Python bases. The defaults are the public slim images, so a plain `docker build` needs no +# extra registry access. CI overrides both with the matching Docker Hardened Images from +# dhi.io, which need a Docker account with DHI catalog access. +# Refresh a digest with: docker buildx imagetools inspect +ARG PYTHON_BUILDER_IMAGE=${BASE_IMAGE_REGISTRY}/library/python:3.13-slim@sha256:b04b5d7233d2ad9c379e22ea8927cd1378cd15c60d4ef876c065b25ea8fb3bf3 +ARG PYTHON_RUNTIME_IMAGE=${BASE_IMAGE_REGISTRY}/library/python:3.13-slim@sha256:b04b5d7233d2ad9c379e22ea8927cd1378cd15c60d4ef876c065b25ea8fb3bf3 + +# Build stage. Needs a root user plus a shell and build tooling to install the wheels, +# which both the default slim image and the DHI "-dev" variant provide. +FROM ${PYTHON_BUILDER_IMAGE} AS builder ENV ONYX_RUNNING_IN_DOCKER="true" \ HF_HOME=/app/.cache/huggingface \ @@ -19,11 +24,11 @@ ENV ONYX_RUNNING_IN_DOCKER="true" \ COPY --from=ghcr.io/astral-sh/uv:0.11.25@sha256:1e3808aa9023d0980e7c15b1fa7c1ac16ff35925780cf5c459858b2d693f01a9 /uv /uvx /bin/ # Install into a self-contained venv rather than the system site-packages. The -# venv lives at a fixed path we control (/app/.venv), so the distroless runtime -# stage can copy it wholesale without depending on the base image's Python layout. +# venv lives at a fixed path we control (/app/.venv), so the runtime stage can +# copy it wholesale without depending on the base image's Python layout. RUN uv venv /app/.venv --python 3.13 -# Pre-create the runtime writable dirs here (the distroless runtime has no shell +# Pre-create the runtime writable dirs here (the DHI runtime has no shell # to mkdir). Left empty in this stage; the model download happens downstream. RUN mkdir -p /app/.cache/huggingface /var/log/onyx @@ -31,7 +36,7 @@ RUN mkdir -p /app/.cache/huggingface /var/log/onyx # pin `-u onyx` / `user: onyx` keep resolving and their 1001-owned volumes stay # writable. The DHI "-dev" image ships a shell but not the `shadow` tools # (groupadd/useradd), so write the passwd/group entries directly; they're copied -# into the distroless runtime below. +# into the runtime stage below. RUN printf 'onyx:x:1001:\n' >> /etc/group && \ printf 'onyx:x:1001:1001::/home/onyx:/usr/sbin/nologin\n' >> /etc/passwd @@ -55,11 +60,11 @@ FROM builder AS embedding-models RUN python -c "from sentence_transformers import SentenceTransformer; \ SentenceTransformer(model_name_or_path='nomic-ai/nomic-embed-text-v1', trust_remote_code=False);" -# Runtime stage. The DHI runtime variant is near-distroless: no shell or package -# manager, runs non-root. We run as the `onyx` user (UID 1001, carried over from -# the previous image; see USER below) and chown everything the runtime writes to it. -# Refresh the digest with: docker buildx imagetools inspect dhi.io/python:3.13-debian13 -FROM dhi.io/python:3.13-debian13@sha256:05827957dafc7b83633d56f24d9281525d3546a1d600eef90385c7212d3def5e AS final +# Runtime stage. We run as the `onyx` user (UID 1001, carried over from the previous +# image; see USER below) and chown everything the runtime writes to it. The DHI runtime +# variant is near-distroless: no shell or package manager, so keep this stage free of +# RUN instructions. +FROM ${PYTHON_RUNTIME_IMAGE} AS final LABEL com.danswer.maintainer="founders@onyx.app" LABEL com.danswer.description="This image is for the Onyx model server which runs all of the \ @@ -76,7 +81,7 @@ ENV ONYX_RUNNING_IN_DOCKER="true" \ VIRTUAL_ENV=/app/.venv \ PATH="/app/.venv/bin:$PATH" -# Bring the `onyx` user (1001) into the distroless runtime. The -dev builder's +# Bring the `onyx` user (1001) into the runtime stage. The -dev builder's # passwd/group are a superset of the runtime's, so copying them preserves the # base's own entries while adding `onyx`, letting `USER onyx` and any runtime # `-u onyx` override resolve the name. diff --git a/backend/alembic/versions/c7f1a9d4e206_add_incognito_admin_settings.py b/backend/alembic/versions/c7f1a9d4e206_add_incognito_admin_settings.py new file mode 100644 index 00000000000..e66cb50ead0 --- /dev/null +++ b/backend/alembic/versions/c7f1a9d4e206_add_incognito_admin_settings.py @@ -0,0 +1,63 @@ +"""add incognito availability, record mode, and the group flag + +Revision ID: c7f1a9d4e206 +Revises: 0ec213a5ffde +Create Date: 2026-08-12 09:40:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op + +from onyx.db.enums import IncognitoRecordMode +from onyx.server.security.models import IncognitoAvailability + +# revision identifiers, used by Alembic. +revision = "c7f1a9d4e206" +down_revision = "0ec213a5ffde" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Both settings are nullable: NULL means the workspace never chose, which + # reads as off and usage_only. + op.add_column( + "security_settings", + sa.Column( + "incognito_availability", + sa.Enum( + IncognitoAvailability, + native_enum=False, + values_callable=lambda x: [e.value for e in x], + ), + nullable=True, + ), + ) + op.add_column( + "security_settings", + sa.Column( + "incognito_record_mode", + sa.Enum( + IncognitoRecordMode, + native_enum=False, + values_callable=lambda x: [e.value for e in x], + ), + nullable=True, + ), + ) + op.add_column( + "user_group", + sa.Column( + "incognito_enabled", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + + +def downgrade() -> None: + op.drop_column("user_group", "incognito_enabled") + op.drop_column("security_settings", "incognito_record_mode") + op.drop_column("security_settings", "incognito_availability") diff --git a/backend/ee/onyx/background/celery/tasks/doc_permission_syncing/tasks.py b/backend/ee/onyx/background/celery/tasks/doc_permission_syncing/tasks.py index e7aa6e2b356..300edcff7e5 100644 --- a/backend/ee/onyx/background/celery/tasks/doc_permission_syncing/tasks.py +++ b/backend/ee/onyx/background/celery/tasks/doc_permission_syncing/tasks.py @@ -414,7 +414,7 @@ def connector_permission_sync_generator_task( LoggerContextVars.reset() - doc_permission_sync_ctx_dict = doc_permission_sync_ctx.get() + doc_permission_sync_ctx_dict = dict(doc_permission_sync_ctx.get()) doc_permission_sync_ctx_dict["cc_pair_id"] = cc_pair_id doc_permission_sync_ctx_dict["request_id"] = self.request.id doc_permission_sync_ctx.set(doc_permission_sync_ctx_dict) diff --git a/backend/ee/onyx/configs/license_enforcement_config.py b/backend/ee/onyx/configs/license_enforcement_config.py index 51d44429012..4704c00cf0e 100644 --- a/backend/ee/onyx/configs/license_enforcement_config.py +++ b/backend/ee/onyx/configs/license_enforcement_config.py @@ -28,6 +28,7 @@ # /settings, /enterprise-settings - View app status and branding # /billing - Unified billing API # /proxy - Self-hosted proxy endpoints (have own license-based auth) +# /mcp/oauth/client-metadata - Public OAuth client identity # /tenants/billing-* - Legacy billing endpoints (backwards compatibility) # /manage/users, /users - User management (needed for seat limit resolution) # /notifications - Needed for UI to load properly @@ -44,6 +45,7 @@ "/admin/billing", # Proxy endpoints for self-hosted billing (no tenant context) "/proxy", + "/mcp/oauth/client-metadata", # Legacy tenant billing endpoints (kept for backwards compatibility) "/tenants/billing-information", "/tenants/create-customer-portal-session", diff --git a/backend/ee/onyx/db/query_history.py b/backend/ee/onyx/db/query_history.py index 16cb9a36424..e882aedd3de 100644 --- a/backend/ee/onyx/db/query_history.py +++ b/backend/ee/onyx/db/query_history.py @@ -1,5 +1,6 @@ from collections.abc import Sequence from datetime import datetime +from uuid import UUID from sqlalchemy import BinaryExpression, ColumnElement, asc, desc, distinct from sqlalchemy.orm import Session, contains_eager, joinedload @@ -8,6 +9,7 @@ from ee.onyx.background.task_name_builders import QUERY_HISTORY_TASK_NAME_PREFIX from onyx.configs.constants import QAFeedbackType +from onyx.db.chat import content_persisting_sessions_filter from onyx.db.models import ChatMessage, ChatMessageFeedback, ChatSession, TaskQueueState from onyx.db.tasks import get_all_tasks_with_prefix @@ -25,7 +27,7 @@ def _build_filter_conditions( feedback_filter: Feedback type to filter by Returns: List of filter conditions """ - conditions = [] + conditions = [content_persisting_sessions_filter()] if start_time is not None: conditions.append(ChatSession.time_created >= start_time) @@ -118,6 +120,27 @@ def get_page_of_chat_sessions( return db_session.scalars(stmt).unique().all() +def fetch_persisting_chat_session_by_id( + chat_session_id: UUID, + db_session: Session, +) -> ChatSession: + """The admin detail read, filtered like the list and the export it belongs to. + + A content-free session is absent rather than refused: whether one exists is + itself metadata the workspace chose not to keep. Deleted sessions stay + visible, which is what the detail view is for. + """ + chat_session = db_session.scalar( + select(ChatSession).where( + ChatSession.id == chat_session_id, + content_persisting_sessions_filter(), + ) + ) + if chat_session is None: + raise ValueError(f"Chat session with id '{chat_session_id}' does not exist.") + return chat_session + + def fetch_chat_sessions_eagerly_by_time( start: datetime, end: datetime, @@ -131,7 +154,8 @@ def fetch_chat_sessions_eagerly_by_time( message_order: UnaryExpression = asc(ChatMessage.id) filters: list[ColumnElement | BinaryExpression] = [ - ChatSession.time_created.between(start, end) + content_persisting_sessions_filter(), + ChatSession.time_created.between(start, end), ] if initial_time: diff --git a/backend/ee/onyx/db/user_group.py b/backend/ee/onyx/db/user_group.py index c8d12b3e6c0..922f74aa919 100644 --- a/backend/ee/onyx/db/user_group.py +++ b/backend/ee/onyx/db/user_group.py @@ -537,6 +537,18 @@ def _add_user_group__cc_pair_relationships__no_commit( return relationships +def set_user_group_incognito( + db_session: Session, user_group_id: int, enabled: bool +) -> UserGroup: + """Flip whether members may use incognito under groups-only availability.""" + group = db_session.scalar(select(UserGroup).where(UserGroup.id == user_group_id)) + if group is None: + raise ValueError(f"UserGroup with id '{user_group_id}' not found") + group.incognito_enabled = enabled + db_session.commit() + return group + + def insert_user_group(db_session: Session, user_group: UserGroupCreate) -> UserGroup: db_user_group = UserGroup( name=user_group.name, diff --git a/backend/ee/onyx/external_permissions/github/utils.py b/backend/ee/onyx/external_permissions/github/utils.py index 6e326e4ba3c..b6a472db64b 100644 --- a/backend/ee/onyx/external_permissions/github/utils.py +++ b/backend/ee/onyx/external_permissions/github/utils.py @@ -1,5 +1,6 @@ from collections.abc import Callable from enum import Enum +from functools import partial from typing import List, Optional, Tuple, TypeVar from github import Github, RateLimitExceededException @@ -151,7 +152,7 @@ def _fetch_repository_teams_detailed( members: PaginatedList[NamedUser] | list[NamedUser] = ( _run_with_retry( - lambda: team.get_members(), + team.get_members, f"get members for team {team.name}", github_client, ) @@ -230,7 +231,7 @@ def _get_collaborators_and_outside_collaborators( if org is not None: org_obj = org membership = _run_with_retry( - lambda: org_obj.has_in_members(collaborator), + partial(org_obj.has_in_members, collaborator), f"check membership for {collaborator.login} in org {org_obj.login}", github_client, ) diff --git a/backend/ee/onyx/server/gateway/api.py b/backend/ee/onyx/server/gateway/api.py index 73b926f5b8e..a578d92b579 100644 --- a/backend/ee/onyx/server/gateway/api.py +++ b/backend/ee/onyx/server/gateway/api.py @@ -1246,7 +1246,7 @@ def emit_thinking_deltas(blocks: list[AnyThinkingBlock]) -> bool: tc for tc in finalized_tool_calls or [] if tc.function.name ] for tool_index, (tool_block, tool_call) in enumerate( - zip(tool_blocks, named_tool_calls), start=next_index + zip(tool_blocks, named_tool_calls, strict=True), start=next_index ): emit( AnthropicContentBlockStartEvent.create( diff --git a/backend/ee/onyx/server/middleware/tenant_tracking.py b/backend/ee/onyx/server/middleware/tenant_tracking.py index 083db7a9ce1..6a81575fec0 100644 --- a/backend/ee/onyx/server/middleware/tenant_tracking.py +++ b/backend/ee/onyx/server/middleware/tenant_tracking.py @@ -186,12 +186,12 @@ async def _get_tenant_id_from_request( # and fall back to the wrong tenant. Fall back only on the normal path. if sys.exc_info()[0] is None: if tenant_id: - return tenant_id + return tenant_id # noqa: B012 # As a final step, check for explicit tenant_id cookie tenant_id_cookie = request.cookies.get(TENANT_ID_COOKIE_NAME) if tenant_id_cookie and is_valid_schema_name(tenant_id_cookie): - return tenant_id_cookie + return tenant_id_cookie # noqa: B012 # If we've reached this point, return the default schema - return POSTGRES_DEFAULT_SCHEMA + return POSTGRES_DEFAULT_SCHEMA # noqa: B012 diff --git a/backend/ee/onyx/server/query_history/api.py b/backend/ee/onyx/server/query_history/api.py index 60721b9d102..1c8b44116fb 100644 --- a/backend/ee/onyx/server/query_history/api.py +++ b/backend/ee/onyx/server/query_history/api.py @@ -10,6 +10,7 @@ from ee.onyx.background.task_name_builders import query_history_task_name from ee.onyx.db.query_history import ( + fetch_persisting_chat_session_by_id, get_all_query_history_export_tasks, get_page_of_chat_sessions, get_total_filtered_chat_sessions_count, @@ -37,7 +38,7 @@ QueryHistoryType, SessionType, ) -from onyx.db.chat import get_chat_session_by_id, get_chat_sessions_by_user +from onyx.db.chat import get_chat_sessions_by_user from onyx.db.engine.sql_engine import get_session from onyx.db.enums import Permission, TaskStatus from onyx.db.file_record import get_query_history_export_files @@ -170,8 +171,14 @@ def admin_get_chat_sessions( ) try: + # Full History incognito is recorded for the workspace and hidden only + # from its own owner, so query history must still return it. chat_sessions = get_chat_sessions_by_user( - user_id=user_id, deleted=False, db_session=db_session, limit=0 + user_id=user_id, + deleted=False, + db_session=db_session, + limit=0, + exclude_content_free=True, ) except ValueError: @@ -248,11 +255,9 @@ def get_chat_session_admin( ) try: - chat_session = get_chat_session_by_id( + chat_session = fetch_persisting_chat_session_by_id( chat_session_id=chat_session_id, - user_id=None, # view chat regardless of user db_session=db_session, - include_deleted=True, ) except ValueError: raise HTTPException( diff --git a/backend/ee/onyx/server/reporting/usage_export_generation.py b/backend/ee/onyx/server/reporting/usage_export_generation.py index c94c3ac6915..a874d9e9859 100644 --- a/backend/ee/onyx/server/reporting/usage_export_generation.py +++ b/backend/ee/onyx/server/reporting/usage_export_generation.py @@ -4,6 +4,7 @@ import zipfile from collections.abc import Iterable from datetime import datetime, timedelta, timezone +from io import BytesIO from fastapi_users_db_sqlalchemy import UUID_ID from sqlalchemy import cast @@ -19,6 +20,9 @@ UsageReportMetadata, UserSkeleton, ) +from ee.onyx.server.reporting.usage_report_branding import load_report_branding +from ee.onyx.server.reporting.usage_report_data import build_usage_report_data +from ee.onyx.server.reporting.usage_report_pdf import render_usage_report_pdf from onyx.configs.constants import FileOrigin from onyx.db.models import User from onyx.db.user_usage import UsageExportRow, iter_usage_export @@ -184,6 +188,31 @@ def generate_usage_breakdown_report( return file_id +def generate_usage_report_pdf( + db_session: Session, + file_store: FileStore, + report_id: str, + period: tuple[datetime, datetime] | None, + rows: list[UsageExportRow], +) -> str: + """Render the review pack PDF and store it. Returns the file id.""" + file_name = f"{report_id}_review_pack" + + # The queried bounds are half-open; only a given period gets the extra day. + display_start, display_end = period if period else _normalize_period(None) + + data = build_usage_report_data(db_session, rows, display_start, display_end) + branding = load_report_branding(file_store) + pdf_bytes = render_usage_report_pdf(data, branding) + + return file_store.save_file( + content=BytesIO(pdf_bytes), + display_name=file_name, + file_origin=FileOrigin.GENERATED_REPORT, + file_type="application/pdf", + ) + + def create_new_usage_report( db_session: Session, user_id: UUID_ID | None, # None = auto-generated @@ -204,12 +233,24 @@ def create_new_usage_report( intermediate_file_ids.append(users_file_id) query_start, query_end = normalized_period - usage_rows = iter_usage_export(db_session, query_start, query_end) + # The CSV and PDF must use the same rows so their totals reconcile. + usage_rows = list(iter_usage_export(db_session, query_start, query_end)) usage_breakdown_file_id = generate_usage_breakdown_report( file_store, report_id, usage_rows ) intermediate_file_ids.append(usage_breakdown_file_id) + # A render failure must not cost the admin their CSV export. + pdf_file_id: str | None = None + try: + pdf_file_id = generate_usage_report_pdf( + db_session, file_store, report_id, period, usage_rows + ) + except Exception: + logger.exception("Failed to render usage report PDF; continuing without it") + else: + intermediate_file_ids.append(pdf_file_id) + # Re-check just before writing the final report: the API-level check # happens before this (async) task runs, so a second request with the # same client-supplied report_id can slip past it while this task is @@ -239,6 +280,12 @@ def create_new_usage_report( ) zip_file.writestr("usage_by_user.csv", usage_breakdown_tmpfile.read()) + if pdf_file_id is not None: + pdf_tmpfile = file_store.read_file( + pdf_file_id, mode="b", use_tempfile=True + ) + zip_file.writestr("usage_report.pdf", pdf_tmpfile.read()) + zip_buffer.seek(0) # store zip blob to file_store diff --git a/backend/ee/onyx/server/reporting/usage_report_branding.py b/backend/ee/onyx/server/reporting/usage_report_branding.py new file mode 100644 index 00000000000..7d97157fa5d --- /dev/null +++ b/backend/ee/onyx/server/reporting/usage_report_branding.py @@ -0,0 +1,79 @@ +"""Branding for the usage report review pack, from enterprise settings.""" + +from pathlib import Path + +from pydantic import BaseModel + +from ee.onyx.server.enterprise_settings.store import ( + get_logo_filename, + get_logotype_filename, + load_runtime_settings, +) +from onyx.configs.constants import ONYX_DEFAULT_APPLICATION_NAME +from onyx.file_store.file_store import FileStore +from onyx.utils.logger import setup_logger + +logger = setup_logger() + +_FALLBACK_LOGO = Path(__file__).parents[4] / "static" / "images" / "logotype.png" + +_SUPPORTED_LOGO_TYPES = ("image/png", "image/jpeg", "image/jpg", "image/gif") + + +class ReportBranding(BaseModel): + application_name: str + # Raster bytes ReportLab can draw. None means the pack sets the name as a + # wordmark instead, which is right when a deployment has a custom logo we + # cannot draw: its own name beats another company's mark. + logo: bytes | None = None + + +def _read_logo(file_store: FileStore, file_id: str) -> bytes | None: + try: + stored = file_store.get_file_with_mime_type(file_id) + except Exception: + logger.exception("Failed to read logo %s for the usage report", file_id) + return None + + if stored is None: + return None + + # ReportLab cannot rasterize SVG, the common upload. + if stored.mime_type not in _SUPPORTED_LOGO_TYPES: + logger.info( + "Usage report cannot draw logo %s of type %s", file_id, stored.mime_type + ) + return None + + return stored.data + + +def load_report_branding(file_store: FileStore) -> ReportBranding: + settings = load_runtime_settings() + name = settings.application_name or ONYX_DEFAULT_APPLICATION_NAME + + has_custom_logo = settings.use_custom_logotype or settings.use_custom_logo + logo: bytes | None = None + if settings.use_custom_logotype: + logo = _read_logo(file_store, get_logotype_filename()) + if logo is None and settings.use_custom_logo: + logo = _read_logo(file_store, get_logo_filename()) + + if logo is None and has_custom_logo: + # Their logo exists but cannot be drawn, so fall through to the + # wordmark. Stamping the bundled mark here would ship our brand on + # their report. + logger.warning( + "Usage report rendering %s as a wordmark: the configured logo is " + "not a raster image ReportLab can draw", + name, + ) + return ReportBranding(application_name=name, logo=None) + + if logo is None: + try: + logo = _FALLBACK_LOGO.read_bytes() + except OSError: + logger.exception("Usage report could not read the fallback logo") + + return ReportBranding(application_name=name, logo=logo) diff --git a/backend/ee/onyx/server/reporting/usage_report_data.py b/backend/ee/onyx/server/reporting/usage_report_data.py new file mode 100644 index 00000000000..d7b9215c9b5 --- /dev/null +++ b/backend/ee/onyx/server/reporting/usage_report_data.py @@ -0,0 +1,171 @@ +"""Aggregates behind the usage report review pack.""" + +from collections import defaultdict +from datetime import datetime + +from pydantic import BaseModel +from sqlalchemy.orm import Session + +from ee.onyx.db.license import user_counts_toward_seats +from onyx.db.api_key import is_api_key_email_address +from onyx.db.user_usage import DELETED_USER_EXPORT_EMAIL, UsageExportRow +from onyx.db.users import get_all_users + +TOP_USER_LIMIT = 10 +TOP_ENTRY_LIMIT = 8 +DORMANT_USER_LIMIT = 25 +UNLABELED_FLOW = "other" + + +class NamedSpend(BaseModel): + name: str + cost_cents: float + input_tokens: int + output_tokens: int + + @property + def total_tokens(self) -> int: + return self.input_tokens + self.output_tokens + + +class DailySpend(BaseModel): + day: str # YYYY-MM-DD + cost_cents: float + active_users: int + + +class UsageReportData(BaseModel): + period_start: datetime + period_end: datetime + + total_cost_cents: float + total_input_tokens: int + total_output_tokens: int + total_cache_read_tokens: int + + licensed_users: int + # Everyone who used it, including people since deactivated, so this can + # exceed licensed_users. `seated_active_users` is the subset holding a seat. + active_users: int + seated_active_users: int + dormant_users: list[str] + + top_users: list[NamedSpend] + by_model: list[NamedSpend] + by_flow: list[NamedSpend] + daily: list[DailySpend] + + @property + def dormant_user_count(self) -> int: + return len(self.dormant_users) + + @property + def cost_per_active_user_cents(self) -> float: + if not self.active_users: + return 0.0 + return self.total_cost_cents / self.active_users + + @property + def has_usage(self) -> bool: + return bool(self.daily) + + +def _top_n(spend_by_name: dict[str, NamedSpend], limit: int) -> list[NamedSpend]: + ordered = sorted(spend_by_name.values(), key=lambda s: s.cost_cents, reverse=True) + if len(ordered) <= limit: + return ordered + + head, tail = ordered[:limit], ordered[limit:] + # Folding a single entry hides a name and saves no space. + if len(tail) == 1: + return ordered + + remainder = NamedSpend( + name=f"Other ({len(tail)})", + cost_cents=sum(s.cost_cents for s in tail), + input_tokens=sum(s.input_tokens for s in tail), + output_tokens=sum(s.output_tokens for s in tail), + ) + return head + [remainder] + + +def build_usage_report_data( + db_session: Session, + rows: list[UsageExportRow], + period_start: datetime, + period_end: datetime, +) -> UsageReportData: + """`rows` is the list written to usage_by_user.csv, so the two cannot + diverge. The period is the admin's requested bounds, for display.""" + by_user: dict[str, NamedSpend] = {} + by_model: dict[str, NamedSpend] = {} + by_flow: dict[str, NamedSpend] = {} + daily_cost: dict[str, float] = defaultdict(float) + daily_users: dict[str, set[str]] = defaultdict(set) + + total_cost = 0.0 + total_input = 0 + total_output = 0 + total_cache_read = 0 + active_emails: set[str] = set() + + for row in rows: + for bucket, key in ( + (by_user, row.email), + (by_model, row.model), + (by_flow, row.flow or UNLABELED_FLOW), + ): + entry = bucket.get(key) + if entry is None: + entry = NamedSpend( + name=key, cost_cents=0.0, input_tokens=0, output_tokens=0 + ) + bucket[key] = entry + entry.cost_cents += row.cost_cents + entry.input_tokens += row.input_tokens + entry.output_tokens += row.output_tokens + + total_cost += row.cost_cents + total_input += row.input_tokens + total_output += row.output_tokens + total_cache_read += row.cache_read_tokens + + daily_cost[row.day] += row.cost_cents + # Their spend still counts toward totals so the pack reconciles with the + # CSV, but neither is a person. + if row.email != DELETED_USER_EXPORT_EMAIL and not is_api_key_email_address( + row.email + ): + active_emails.add(row.email) + daily_users[row.day].add(row.email) + + # Must match license enforcement, or this disagrees with what is billed. + users = get_all_users(db_session, include_api_key_users=False) + seat_emails = {user.email for user in users if user_counts_toward_seats(user)} + dormant = sorted(seat_emails - active_emails) + + daily = [ + DailySpend( + day=day, + cost_cents=daily_cost[day], + active_users=len(daily_users[day]), + ) + for day in sorted(daily_cost) + ] + + return UsageReportData( + period_start=period_start, + period_end=period_end, + total_cost_cents=total_cost, + total_input_tokens=total_input, + total_output_tokens=total_output, + total_cache_read_tokens=total_cache_read, + licensed_users=len(seat_emails), + active_users=len(active_emails), + seated_active_users=len(active_emails & seat_emails), + dormant_users=dormant, + top_users=_top_n(by_user, TOP_USER_LIMIT), + by_model=_top_n(by_model, TOP_ENTRY_LIMIT), + by_flow=_top_n(by_flow, TOP_ENTRY_LIMIT), + daily=daily, + ) diff --git a/backend/ee/onyx/server/reporting/usage_report_pdf.py b/backend/ee/onyx/server/reporting/usage_report_pdf.py new file mode 100644 index 00000000000..414cf40d3d5 --- /dev/null +++ b/backend/ee/onyx/server/reporting/usage_report_pdf.py @@ -0,0 +1,628 @@ +"""Renders the usage report review pack as a PDF. + +The `ty: ignore`s below are load-bearing: ReportLab types `chart.data` from a +sample literal and populates `valueAxis.labels` dynamically, so neither is +resolvable by a static checker. +""" + +from io import BytesIO +from xml.sax.saxutils import escape + +from reportlab.graphics.charts.barcharts import VerticalBarChart +from reportlab.graphics.charts.linecharts import HorizontalLineChart +from reportlab.graphics.shapes import Drawing +from reportlab.lib import colors +from reportlab.lib.enums import TA_LEFT +from reportlab.lib.pagesizes import LETTER +from reportlab.lib.styles import ParagraphStyle, getSampleStyleSheet +from reportlab.lib.units import inch +from reportlab.lib.utils import ImageReader +from reportlab.pdfgen import canvas +from reportlab.platypus import ( + Flowable, + Image, + KeepTogether, + PageBreak, + Paragraph, + SimpleDocTemplate, + Spacer, + Table, + TableStyle, +) + +from ee.onyx.server.reporting.usage_report_branding import ReportBranding +from ee.onyx.server.reporting.usage_report_data import ( + DORMANT_USER_LIMIT, + NamedSpend, + UsageReportData, +) +from onyx.configs.constants import DANSWER_API_KEY_PREFIX, UNNAMED_KEY_PLACEHOLDER +from onyx.db.api_key import is_api_key_email_address +from onyx.utils.logger import setup_logger + +logger = setup_logger() + +_INK = colors.HexColor("#1c1c1c") # onyx-ink-95 +_ACCENT = colors.HexColor("#286df8") # action-selection-05 / blue-50 +_BODY = colors.HexColor("#54545d") # stone-60, 7.5:1 on white +_HAIRLINE = colors.HexColor("#e6e6e9") # stone-10 +_SURFACE = colors.HexColor("#f0f0f1") # stone-05 + +_CONTENT_WIDTH = LETTER[0] - 2 * inch +_MAX_AXIS_LABELS = 12 +_LOGO_MAX_W, _LOGO_MAX_H = 2.0 * inch, 0.5 * inch + + +def _dollars(cents: float) -> str: + return f"${cents / 100:,.2f}" + + +def _thousands(value: int) -> str: + return f"{value:,}" + + +def _display_name(name: str) -> str: + """Render an API key by its name instead of its synthetic address. + + Each API key owns a `User` row whose email is + `API_KEY__@onyxapikey.ai`, so per-key spend already + aggregates correctly. Only the label needs help. A no-op for every other + name, since none of them carry the API-key domain. + """ + if not is_api_key_email_address(name): + return name + + # The key's name can itself contain "@", so split on the last one. + local_part = name.rsplit("@", 1)[0] + # Stored emails are lowercased by a DB check constraint, so the prefix + # cannot be matched case-sensitively against the constant. + if local_part.lower().startswith(DANSWER_API_KEY_PREFIX.lower()): + local_part = local_part[len(DANSWER_API_KEY_PREFIX) :] + return f"{local_part or UNNAMED_KEY_PLACEHOLDER} (API key)" + + +def _styles() -> dict[str, ParagraphStyle]: + base = getSampleStyleSheet() + return { + "cover_title": ParagraphStyle( + "CoverTitle", + parent=base["Title"], + fontName="Helvetica-Bold", + fontSize=32, + leading=36, + textColor=_INK, + alignment=TA_LEFT, + spaceAfter=6, + ), + "wordmark": ParagraphStyle( + "Wordmark", + parent=base["Normal"], + fontName="Helvetica-Bold", + fontSize=19, + leading=23, + textColor=_INK, + ), + "cover_period": ParagraphStyle( + "CoverPeriod", + parent=base["Normal"], + fontName="Helvetica", + fontSize=13, + leading=18, + textColor=_BODY, + ), + "lede": ParagraphStyle( + "Lede", + parent=base["Normal"], + fontName="Helvetica", + fontSize=11.5, + leading=17, + textColor=_INK, + ), + "heading": ParagraphStyle( + "Heading", + parent=base["Heading2"], + fontName="Helvetica-Bold", + fontSize=14, + leading=18, + textColor=_INK, + spaceBefore=22, + spaceAfter=2, + keepWithNext=1, + ), + "subheading": ParagraphStyle( + "Subheading", + parent=base["Normal"], + fontName="Helvetica", + fontSize=9.5, + leading=13, + textColor=_BODY, + spaceAfter=10, + keepWithNext=1, + ), + "note": ParagraphStyle( + "Note", + parent=base["Normal"], + fontName="Helvetica", + fontSize=8.5, + leading=12, + textColor=_BODY, + spaceBefore=4, + ), + } + + +def _logo_flowable(branding: ReportBranding) -> Flowable | None: + if not branding.logo: + return None + try: + reader = ImageReader(BytesIO(branding.logo)) + src_w, src_h = reader.getSize() + except Exception: + logger.exception( + "Usage report could not render the configured logo for %s", + branding.application_name, + ) + return None + if not src_w or not src_h: + return None + + scale = min(_LOGO_MAX_W / src_w, _LOGO_MAX_H / src_h) + return Image( + BytesIO(branding.logo), width=src_w * scale, height=src_h * scale, mask="auto" + ) + + +class _NumberedCanvas(canvas.Canvas): + """Stamps "page N of M" once the total is known. + + The total only exists after the last page is laid out, so pages are held + back until save. The cover is deliberately left unnumbered. + """ + + def __init__(self, *args: object, **kwargs: object) -> None: + super().__init__(*args, **kwargs) + self._pages: list[dict[str, object]] = [] + + def showPage(self) -> None: + self._pages.append(dict(self.__dict__)) + self._startPage() + + def save(self) -> None: + total = len(self._pages) + for number, state in enumerate(self._pages, start=1): + self.__dict__.update(state) + if number > 1: + self._draw_folio(number, total) + super().showPage() + super().save() + + def _draw_folio(self, number: int, total: int) -> None: + width = self._pagesize[0] + self.setFont("Helvetica", 8.5) + self.setFillColor(_BODY) + self.drawRightString(width - inch, 0.6 * inch, f"{number} of {total}") + + +class _Rule(Flowable): + def __init__(self, width: float, color: colors.Color = _HAIRLINE) -> None: + super().__init__() + self.width, self.height, self.color = width, 1, color + + def draw(self) -> None: + self.canv.setStrokeColor(self.color) + self.canv.setLineWidth(1) + self.canv.line(0, 0, self.width, 0) + + +class _SeatMeter(Flowable): + """Seats in use against seats bought.""" + + def __init__(self, active: int, licensed: int, unseated: int, width: float) -> None: + super().__init__() + self.active, self.licensed, self.unseated = active, licensed, unseated + self.width, self.height = width, 54 + + def draw(self) -> None: + c = self.canv + bar_h, bar_y = 14, 20 + ratio = min(1.0, self.active / self.licensed) if self.licensed else 0.0 + + c.setFillColor(_SURFACE) + c.roundRect(0, bar_y, self.width, bar_h, 3, stroke=0, fill=1) + if ratio > 0: + c.setFillColor(_ACCENT) + c.roundRect( + 0, bar_y, max(3.0, self.width * ratio), bar_h, 3, stroke=0, fill=1 + ) + + c.setFillColor(_INK) + c.setFont("Helvetica-Bold", 11) + c.drawString( + 0, bar_y + bar_h + 8, f"{self.active} of {self.licensed} seats active" + ) + + c.setFillColor(_BODY) + c.setFont("Helvetica", 9) + idle = self.licensed - self.active + caption = ( + f"{ratio:.0%} in use · {idle} seats idle" + if idle + else f"{ratio:.0%} of licensed seats in use" + ) + if self.unseated: + caption += f" · {self.unseated} used it without a seat" + c.drawString(0, bar_y - 13, caption) + + +def _headline(data: UsageReportData) -> Table: + figures = [ + (_thousands(data.active_users), "People using it"), + (_dollars(data.total_cost_cents), "Total spend"), + (_dollars(data.cost_per_active_user_cents), "Cost per active person"), + ] + value_style = ParagraphStyle( + "Figure", + fontName="Helvetica-Bold", + fontSize=23, + leading=26, + textColor=_INK, + ) + label_style = ParagraphStyle( + "FigureLabel", + fontName="Helvetica", + fontSize=9, + leading=12, + textColor=_BODY, + ) + row = [ + [Paragraph(v, value_style) for v, _ in figures], + [Paragraph(label, label_style) for _, label in figures], + ] + col = _CONTENT_WIDTH / 3 + table = Table(row, colWidths=[col] * 3, hAlign="LEFT") + table.setStyle( + TableStyle( + [ + ("VALIGN", (0, 0), (-1, -1), "BOTTOM"), + ("TOPPADDING", (0, 0), (-1, 0), 0), + ("BOTTOMPADDING", (0, 0), (-1, 0), 2), + ("TOPPADDING", (0, 1), (-1, 1), 0), + ("LEFTPADDING", (0, 0), (-1, -1), 0), + ("RIGHTPADDING", (0, 0), (-1, -1), 12), + ] + ) + ) + return table + + +def _table(rows: list[list[str]], col_widths: list[float]) -> Table: + table = Table(rows, colWidths=col_widths, repeatRows=1, hAlign="LEFT") + table.setStyle( + TableStyle( + [ + # Header and body typography. + ("FONTNAME", (0, 0), (-1, 0), "Helvetica-Bold"), + ("FONTNAME", (0, 1), (-1, -1), "Helvetica"), + ("FONTSIZE", (0, 0), (-1, -1), 9), + ("TEXTCOLOR", (0, 0), (-1, 0), _INK), + ("TEXTCOLOR", (0, 1), (-1, -1), _BODY), + ("TEXTCOLOR", (0, 1), (0, -1), _INK), + # Text columns left-align; numeric columns right-align. + ("ALIGN", (1, 0), (-1, -1), "RIGHT"), + ("ALIGN", (0, 0), (0, -1), "LEFT"), + # Separate the header and each body row. + ("LINEBELOW", (0, 0), (-1, 0), 1, _INK), + ("LINEBELOW", (0, 1), (-1, -2), 0.5, _HAIRLINE), + # Keep rows readable without changing column width. + ("TOPPADDING", (0, 0), (-1, -1), 7), + ("BOTTOMPADDING", (0, 0), (-1, -1), 7), + ("LEFTPADDING", (0, 0), (-1, -1), 0), + ("VALIGN", (0, 0), (-1, -1), "MIDDLE"), + ] + ) + ) + return table + + +def _axis_labels(days: list[str]) -> list[str]: + """Keep at most `_MAX_AXIS_LABELS` ticks, blanking the rest.""" + step = max(1, (len(days) + _MAX_AXIS_LABELS - 1) // _MAX_AXIS_LABELS) + # Drop the year: the period is already stated on the cover. + return [day[5:] if index % step == 0 else "" for index, day in enumerate(days)] + + +def _style_axes(chart: HorizontalLineChart | VerticalBarChart) -> None: + chart.categoryAxis.labels.fontName = "Helvetica" + chart.categoryAxis.labels.fontSize = 7 + chart.categoryAxis.labels.fillColor = _BODY + chart.categoryAxis.strokeColor = _HAIRLINE + chart.valueAxis.labels.fontName = "Helvetica" # ty: ignore[unresolved-attribute] + chart.valueAxis.labels.fontSize = 7 # ty: ignore[unresolved-attribute] + chart.valueAxis.labels.fillColor = _BODY # ty: ignore[unresolved-attribute] + chart.valueAxis.strokeColor = _HAIRLINE + chart.valueAxis.valueMin = 0 + chart.valueAxis.gridStrokeColor = _HAIRLINE + chart.valueAxis.gridStrokeWidth = 0.5 + chart.valueAxis.visibleGrid = True + + +def _spend_over_time(data: UsageReportData) -> Drawing: + drawing = Drawing(_CONTENT_WIDTH, 168) + chart = HorizontalLineChart() + chart.x, chart.y = 42, 28 + chart.width, chart.height = int(_CONTENT_WIDTH - 56), 122 + chart.data = [[point.cost_cents / 100 for point in data.daily]] # ty: ignore[invalid-assignment] + chart.categoryAxis.categoryNames = _axis_labels([p.day for p in data.daily]) + _style_axes(chart) + chart.lines[0].strokeColor = _ACCENT + chart.lines[0].strokeWidth = 1.6 + drawing.add(chart) + return drawing + + +def _active_users_over_time(data: UsageReportData) -> Drawing: + drawing = Drawing(_CONTENT_WIDTH, 168) + chart = HorizontalLineChart() + chart.x, chart.y = 42, 28 + chart.width, chart.height = int(_CONTENT_WIDTH - 56), 122 + chart.data = [[float(point.active_users) for point in data.daily]] # ty: ignore[invalid-assignment] + chart.categoryAxis.categoryNames = _axis_labels([p.day for p in data.daily]) + _style_axes(chart) + chart.lines[0].strokeColor = _INK + chart.lines[0].strokeWidth = 1.6 + drawing.add(chart) + return drawing + + +def _spend_by_model(data: UsageReportData) -> Drawing: + drawing = Drawing(_CONTENT_WIDTH, 170) + chart = VerticalBarChart() + chart.x, chart.y = 42, 38 + chart.width, chart.height = int(_CONTENT_WIDTH - 56), 112 + chart.data = [[entry.cost_cents / 100 for entry in data.by_model]] # ty: ignore[invalid-assignment] + chart.categoryAxis.categoryNames = [ + _display_name(entry.name) for entry in data.by_model + ] + _style_axes(chart) + chart.categoryAxis.labels.angle = 20 + chart.categoryAxis.labels.dy = -8 + chart.bars[0].fillColor = _ACCENT + chart.bars[0].strokeColor = None + chart.barSpacing = 2 + drawing.add(chart) + return drawing + + +def _cover( + data: UsageReportData, + branding: ReportBranding, + styles: dict[str, ParagraphStyle], +) -> list[Flowable]: + period = ( + f"{data.period_start.date().isoformat()} to " + f"{data.period_end.date().isoformat()} (UTC)" + ) + story: list[Flowable] = [] + + logo = _logo_flowable(branding) + if logo is not None: + logo.hAlign = "LEFT" + story += [logo, Spacer(1, 30)] + else: + story += [ + Paragraph(escape(branding.application_name), styles["wordmark"]), + Spacer(1, 26), + ] + + story += [ + Paragraph("Usage report", styles["cover_title"]), + Paragraph(period, styles["cover_period"]), + Spacer(1, 26), + _Rule(_CONTENT_WIDTH, _INK), + Spacer(1, 22), + _headline(data), + Spacer(1, 30), + ] + + if data.licensed_users: + story += [ + _SeatMeter( + data.seated_active_users, + data.licensed_users, + data.active_users - data.seated_active_users, + _CONTENT_WIDTH, + ) + ] + + story += [ + Spacer(1, 30), + Paragraph(_summary_sentence(data, branding.application_name), styles["lede"]), + ] + + if data.by_model: + story += [ + Spacer(1, 26), + Paragraph("Top models by spend", styles["subheading"]), + _spend_table("Model", data.by_model[:3]), + ] + + return story + + +def _summary_sentence(data: UsageReportData, application_name: str) -> str: + """Returns Paragraph markup, so every interpolated name is escaped.""" + people = "1 person" if data.active_users == 1 else f"{data.active_users} people" + parts = [ + f"{people} used {escape(application_name)} in this period, at a total cost " + f"of {_dollars(data.total_cost_cents)}." + ] + if data.by_flow: + flow = escape(_display_name(data.by_flow[0].name)) + parts.append(f"Most of that ran through {flow}.") + if data.dormant_user_count: + share = ( + data.dormant_user_count / data.licensed_users + if data.licensed_users + else 0.0 + ) + parts.append( + f"{data.dormant_user_count} of {data.licensed_users} licensed seats " + f"({share:.0%}) went unused and are candidates to reassign." + ) + return " ".join(parts) + + +def _spend_table(label: str, entries: list[NamedSpend]) -> Table: + rows: list[list[str]] = [[label, "Spend (USD)", "Tokens"]] + rows += [ + [ + _display_name(entry.name), + _dollars(entry.cost_cents), + _thousands(entry.total_tokens), + ] + for entry in entries + ] + widths = [_CONTENT_WIDTH * 0.5, _CONTENT_WIDTH * 0.25, _CONTENT_WIDTH * 0.25] + return _table(rows, widths) + + +def _section( + heading: str, subheading: str, styles: dict[str, ParagraphStyle] +) -> list[Flowable]: + return [ + Paragraph(heading, styles["heading"]), + Paragraph(subheading, styles["subheading"]), + ] + + +def _charted_section( + heading: str, + subheading: str, + chart: Drawing, + styles: dict[str, ParagraphStyle], +) -> Flowable: + """`keepWithNext` does not reach into a KeepTogether, so the heading must + travel inside the group or it strands at the page foot.""" + return KeepTogether( + [ + Paragraph(heading, styles["heading"]), + Paragraph(subheading, styles["subheading"]), + chart, + ] + ) + + +def render_usage_report_pdf(data: UsageReportData, branding: ReportBranding) -> bytes: + styles = _styles() + buffer = BytesIO() + doc = SimpleDocTemplate( + buffer, + pagesize=LETTER, + leftMargin=inch, + rightMargin=inch, + topMargin=0.9 * inch, + bottomMargin=0.9 * inch, + title=f"{branding.application_name} usage report", + author=branding.application_name, + # Byte-identical output for identical input. + invariant=1, + ) + + story: list[Flowable] = [] + + if not data.has_usage: + logo = _logo_flowable(branding) + if logo is not None: + logo.hAlign = "LEFT" + story += [logo, Spacer(1, 30)] + else: + story += [ + Paragraph(escape(branding.application_name), styles["wordmark"]), + Spacer(1, 26), + ] + story += [ + Paragraph("Usage report", styles["cover_title"]), + Paragraph( + f"{data.period_start.date().isoformat()} to " + f"{data.period_end.date().isoformat()} (UTC)", + styles["cover_period"], + ), + Spacer(1, 24), + Paragraph( + "No recorded usage in this period. If the deployment was active, " + "the usage rollup may have started after the period began.", + styles["lede"], + ), + ] + doc.build(story, canvasmaker=_NumberedCanvas) + return buffer.getvalue() + + story += _cover(data, branding, styles) + story += [PageBreak()] + + story += [ + _charted_section( + "Adoption", + "Distinct people who sent at least one message each day.", + _active_users_over_time(data), + styles, + ), + _charted_section( + "Spend over time", + "Daily cost across every model and surface.", + _spend_over_time(data), + styles, + ), + _charted_section( + "Where the spend goes", + "Cost by model for the period.", + _spend_by_model(data), + styles, + ), + Spacer(1, 6), + _spend_table("Model", data.by_model), + ] + + story += _section("Heaviest users", "The people driving most of the cost.", styles) + story += [_spend_table("User", data.top_users)] + + story += _section("Spend by surface", "Where the work happens.", styles) + story += [_spend_table("Flow", data.by_flow)] + + story += _section( + "Seats not in use", + "Licensed people who sent nothing this period. Reclaim, retrain, or " + "drop them at renewal.", + styles, + ) + if not data.dormant_users: + story.append( + Paragraph("Every licensed seat was used this period.", styles["lede"]) + ) + else: + shown = data.dormant_users[:DORMANT_USER_LIMIT] + rows: list[list[str]] = [["User"]] + [[email] for email in shown] + story.append(_table(rows, [_CONTENT_WIDTH])) + remaining = data.dormant_user_count - len(shown) + if remaining > 0: + story.append( + Paragraph( + f"{remaining} more idle seats. See users.csv for the full list.", + styles["note"], + ) + ) + + story += [ + Spacer(1, 20), + _Rule(_CONTENT_WIDTH), + Spacer(1, 8), + Paragraph( + "Spend from deleted users and API keys is included in every total " + "and attributed separately. Neither counts as a person or a seat. " + "Days are UTC.", + styles["note"], + ), + ] + + doc.build(story, canvasmaker=_NumberedCanvas) + return buffer.getvalue() diff --git a/backend/ee/onyx/server/user_group/api.py b/backend/ee/onyx/server/user_group/api.py index c359d30b955..170239445c9 100644 --- a/backend/ee/onyx/server/user_group/api.py +++ b/backend/ee/onyx/server/user_group/api.py @@ -12,6 +12,7 @@ prepare_user_group_for_deletion, rename_user_group, set_group_permission__no_commit, + set_user_group_incognito, update_user_curator_relationship, update_user_group, ) @@ -25,6 +26,7 @@ UpdateGroupAgentsRequest, UserGroup, UserGroupCreate, + UserGroupIncognitoUpdate, UserGroupRename, UserGroupUpdate, ) @@ -190,6 +192,28 @@ def rename_user_group_endpoint( raise OnyxError(OnyxErrorCode.CONFLICT, msg) +@router.patch("/admin/user-group/{user_group_id}/incognito") +def patch_user_group_incognito( + user_group_id: int, + update: UserGroupIncognitoUpdate, + _: User = Depends(require_permission(Permission.FULL_ADMIN_PANEL_ACCESS)), + db_session: Session = Depends(get_session), +) -> UserGroup: + """Only meaningful while the security setting is groups-only, but always + storable so admins can stage membership before flipping the mode.""" + try: + return UserGroup.from_model( + set_user_group_incognito( + db_session=db_session, + user_group_id=user_group_id, + enabled=update.enabled, + ), + mask_credential_prefix=get_security_settings().mask_credential_prefix, + ) + except ValueError as e: + raise OnyxError(OnyxErrorCode.NOT_FOUND, str(e)) + + @router.patch("/admin/user-group/{user_group_id}") def patch_user_group( user_group_id: int, diff --git a/backend/ee/onyx/server/user_group/models.py b/backend/ee/onyx/server/user_group/models.py index 1efd6c3de80..4e34482b44d 100644 --- a/backend/ee/onyx/server/user_group/models.py +++ b/backend/ee/onyx/server/user_group/models.py @@ -25,6 +25,8 @@ class UserGroup(BaseModel): is_up_to_date: bool is_up_for_deletion: bool is_default: bool + # Members may start incognito chats when availability is groups-only. + incognito_enabled: bool @classmethod def from_model( @@ -87,6 +89,7 @@ def from_model( is_up_to_date=user_group_model.is_up_to_date, is_up_for_deletion=user_group_model.is_up_for_deletion, is_default=user_group_model.is_default, + incognito_enabled=user_group_model.incognito_enabled, ) @@ -115,6 +118,10 @@ class UserGroupUpdate(BaseModel): cc_pair_ids: list[int] +class UserGroupIncognitoUpdate(BaseModel): + enabled: bool + + class AddUsersToUserGroupRequest(BaseModel): user_ids: list[UUID] diff --git a/backend/onyx/auth/users.py b/backend/onyx/auth/users.py index 4e790d19199..68d182a1678 100644 --- a/backend/onyx/auth/users.py +++ b/backend/onyx/auth/users.py @@ -1884,12 +1884,12 @@ async def refresh( # Check if strategy supports refreshing supports_refresh = hasattr(strategy, "refresh_token") and callable( - getattr(strategy, "refresh_token") + getattr(strategy, "refresh_token") # noqa: B009 ) if supports_refresh: try: - refresh_method = getattr(strategy, "refresh_token") + refresh_method = getattr(strategy, "refresh_token") # noqa: B009 new_token = await refresh_method(token, user) logger.info( "Successfully refreshed session token for user %s", diff --git a/backend/onyx/background/celery/tasks/beat_schedule.py b/backend/onyx/background/celery/tasks/beat_schedule.py index e31e9f960fd..b54980bf65a 100644 --- a/backend/onyx/background/celery/tasks/beat_schedule.py +++ b/backend/onyx/background/celery/tasks/beat_schedule.py @@ -49,6 +49,18 @@ "expires": BEAT_EXPIRES_DEFAULT, }, }, + { + "name": "check-for-incognito-file-cleanup", + "task": OnyxCeleryTask.CHECK_FOR_INCOGNITO_FILE_CLEANUP, + "schedule": timedelta(minutes=10), + "options": { + "priority": OnyxCeleryPriority.LOW, + "expires": BEAT_EXPIRES_DEFAULT, + # Run on gated tenants too, their registries hold blob handles. + "skip_gated": False, + "work_gated": True, + }, + }, { "name": "check-for-user-file-project-sync", "task": OnyxCeleryTask.CHECK_FOR_USER_FILE_PROJECT_SYNC, diff --git a/backend/onyx/background/celery/tasks/docfetching/tasks.py b/backend/onyx/background/celery/tasks/docfetching/tasks.py index 0cce6699cc1..54657399aee 100644 --- a/backend/onyx/background/celery/tasks/docfetching/tasks.py +++ b/backend/onyx/background/celery/tasks/docfetching/tasks.py @@ -564,7 +564,7 @@ def docfetching_proxy_task( ) finally: job.release() - break + break # log the memory usage for tracking down memory leaks / connector-specific memory issues pid = job.process.pid diff --git a/backend/onyx/background/celery/tasks/monitoring/tasks.py b/backend/onyx/background/celery/tasks/monitoring/tasks.py index bff20d8b655..7ea1c621160 100644 --- a/backend/onyx/background/celery/tasks/monitoring/tasks.py +++ b/backend/onyx/background/celery/tasks/monitoring/tasks.py @@ -843,7 +843,7 @@ def cloud_check_alembic() -> bool | None: tenant_to_revision[tenant_id] = ALEMBIC_NULL_REVISION # get the total count of each revision - for k, v in tenant_to_revision.items(): + for v in tenant_to_revision.values(): revision_counts[v] = revision_counts.get(v, 0) + 1 # error if any null revision tenants are found diff --git a/backend/onyx/background/celery/tasks/pruning/tasks.py b/backend/onyx/background/celery/tasks/pruning/tasks.py index 9e5ff51a2f9..30ed3793a4d 100644 --- a/backend/onyx/background/celery/tasks/pruning/tasks.py +++ b/backend/onyx/background/celery/tasks/pruning/tasks.py @@ -496,7 +496,7 @@ def connector_pruning_generator_task( LoggerContextVars.reset() - pruning_ctx_dict = pruning_ctx.get() + pruning_ctx_dict = dict(pruning_ctx.get()) pruning_ctx_dict["cc_pair_id"] = cc_pair_id pruning_ctx_dict["request_id"] = self.request.id pruning_ctx.set(pruning_ctx_dict) diff --git a/backend/onyx/background/celery/tasks/user_file_processing/tasks.py b/backend/onyx/background/celery/tasks/user_file_processing/tasks.py index 71c7746cc6b..7402f413f55 100644 --- a/backend/onyx/background/celery/tasks/user_file_processing/tasks.py +++ b/backend/onyx/background/celery/tasks/user_file_processing/tasks.py @@ -16,6 +16,10 @@ ) from onyx.background.celery.celery_utils import httpx_init_vespa_pool from onyx.background.celery.tasks.shared.RetryDocumentIndex import RetryDocumentIndex +from onyx.cache.factory import get_cache_backend +from onyx.chat.chat_processing_checker import is_chat_session_processing +from onyx.chat.incognito import delete_incognito_generated_files +from onyx.chat.incognito_context import incognito_session_ended from onyx.configs.app_configs import ( DISABLE_VECTOR_DB, MANAGED_VESPA, @@ -29,6 +33,7 @@ CELERY_USER_FILE_PROCESSING_TASK_EXPIRES, CELERY_USER_FILE_PROJECT_SYNC_LOCK_TIMEOUT, CELERY_USER_FILE_PROJECT_SYNC_TASK_EXPIRES, + INCOGNITO_FILE_CLEANUP_BATCH, USER_FILE_DELETE_MAX_QUEUE_DEPTH, USER_FILE_PROCESSING_MAX_QUEUE_DEPTH, USER_FILE_PROJECT_SYNC_MAX_QUEUE_DEPTH, @@ -42,6 +47,7 @@ from onyx.connectors.models import Document, HierarchyNode from onyx.db.engine.sql_engine import get_session_with_current_tenant from onyx.db.enums import UserFileStatus +from onyx.db.file_record import get_session_ids_with_incognito_files from onyx.db.models import SearchSettings, UserFile from onyx.db.port_attempt import port_backfill_has_pending_work from onyx.db.port_orphan_candidate import record_port_orphan_candidates_for_user_file @@ -1196,3 +1202,47 @@ def process_single_user_file_project_sync( project_sync_user_file_impl( user_file_id=user_file_id, tenant_id=tenant_id, redis_locking=True ) + + +@shared_task( # ty: ignore[invalid-argument-type] + name=OnyxCeleryTask.CHECK_FOR_INCOGNITO_FILE_CLEANUP, + soft_time_limit=300, + bind=True, + ignore_result=True, +) +def check_for_incognito_file_cleanup(self: Task, *, tenant_id: str) -> None: # noqa: ARG001 + """Retry deletion of tool-generated blobs whose teardown pass failed. + + A blob's own record carries the session that produced it, and deleting the + blob deletes the record, so anything still stamped is what a store failure + left behind.""" + redis_client = get_redis_client(tenant_id=tenant_id) + lock: RedisLock = redis_client.lock( + OnyxRedisLocks.INCOGNITO_FILE_CLEANUP_BEAT_LOCK, + timeout=CELERY_GENERIC_BEAT_LOCK_TIMEOUT, + ) + if not lock.acquire(blocking=False): + return + try: + with get_session_with_current_tenant() as db_session: + cache = get_cache_backend() + for raw_id in get_session_ids_with_incognito_files( + db_session, limit=INCOGNITO_FILE_CLEANUP_BATCH + ): + session_id = UUID(raw_id) + # A turn in flight owns its files, and an evicted context reads + # as ended, so the processing fence decides, not the context. + if is_chat_session_processing(session_id, cache): + continue + if not incognito_session_ended(session_id): + continue + try: + delete_incognito_generated_files(session_id, db_session) + except Exception: + # One unreachable blob must not strand every session behind it. + task_logger.exception( + "Incognito file cleanup failed for session %s", session_id + ) + finally: + if lock.owned(): + lock.release() diff --git a/backend/onyx/chat/chat_state.py b/backend/onyx/chat/chat_state.py index 771a1c265c8..61f963b9f3f 100644 --- a/backend/onyx/chat/chat_state.py +++ b/backend/onyx/chat/chat_state.py @@ -15,8 +15,9 @@ SearchParams, ) from onyx.context.search.models import SearchDoc +from onyx.db.enums import IncognitoRecordMode from onyx.db.memory import UserMemoryContext -from onyx.db.models import ChatMessage, ChatSession, Persona +from onyx.db.models import ChatMessage, Persona from onyx.llm.interfaces import LLM, LLMUserIdentity from onyx.llm.models import ReasoningEffort from onyx.onyxbot.slack.models import SlackContext @@ -183,18 +184,24 @@ class ChatTurnSetup: """Immutable context produced by ``build_chat_turn`` and consumed by ``_run_models``. **Detached-safety contract:** instances of this class travel outside the DB - session that built them. Every ORM object reachable from this dataclass - (``chat_session``, ``persona``, ``user_message``, ``reserved_messages``, - ``llms``) is detached after ``build_chat_turn`` returns. Downstream code - must only read column attributes that were eager-loaded during setup — - do NOT access lazy-loaded relationships (e.g. ``setup.chat_session.messages``, - ``setup.persona.tools[i].some_lazy_field``) or SQLAlchemy will raise - ``DetachedInstanceError`` at runtime.""" + session that built them. The ORM objects still reachable from this dataclass + (``persona``, ``reserved_messages``) are detached after ``build_chat_turn`` + returns. Downstream code must only read column attributes that were + eager-loaded during setup. Do NOT access lazy-loaded relationships + (e.g. ``setup.persona.tools[i].some_lazy_field``) or SQLAlchemy will raise + ``DetachedInstanceError`` at runtime. Closures stored here count: bind the + ids they need, never the rows. + + Session and user-message identity are carried as plain scalars: the turn + needs only their ids and the session's project id.""" new_msg_req: SendMessageRequest - chat_session: ChatSession + chat_session_id: UUID + chat_session_project_id: int | None + # The session's pinned recording policy. None is an ordinary chat. + incognito_record_mode: IncognitoRecordMode | None persona: Persona - user_message: ChatMessage + user_message_id: int user_identity: LLMUserIdentity llms: list[LLM] # length 1 for single-model, N for multi-model model_display_names: list[str] # parallel to llms diff --git a/backend/onyx/chat/chat_utils.py b/backend/onyx/chat/chat_utils.py index e5e086c3c79..771d8fb7d1d 100644 --- a/backend/onyx/chat/chat_utils.py +++ b/backend/onyx/chat/chat_utils.py @@ -8,6 +8,11 @@ from pydantic import BaseModel from sqlalchemy.orm import Session +from onyx.chat.incognito import ( + incognito_allowed_for_user, + resolve_incognito_record_mode, +) +from onyx.chat.incognito_context import incognito_context_available from onyx.chat.models import ( ChatHistoryResult, ChatLoadedFile, @@ -28,7 +33,7 @@ get_chat_messages_by_session, get_or_create_root_message, ) -from onyx.db.enums import UserFileStatus +from onyx.db.enums import IncognitoRecordMode, UserFileStatus from onyx.db.file_record import FileRecordNotFoundError from onyx.db.kg_config import ( get_kg_config_settings, @@ -39,6 +44,8 @@ from onyx.db.persona import user_can_access_persona from onyx.db.projects import check_project_ownership from onyx.db.user_file import get_user_file_by_id +from onyx.error_handling.error_codes import OnyxErrorCode +from onyx.error_handling.exceptions import OnyxError from onyx.file_processing.extract_file_text import extract_file_text from onyx.file_store.file_store import get_default_file_store from onyx.file_store.models import ChatFileType, FileDescriptor @@ -185,12 +192,33 @@ def create_chat_session_from_request( ): raise ValueError("User does not have access to persona") + # Pinned at creation so a later setting change cannot alter a live session. + # Availability decides server-side, never the client flag. A refusal + # errors: degrading would silently persist a believed-incognito chat. + # The capability is checked first so a deployment that cannot hold the + # context says so, rather than reporting it as a permission the admin + # could grant. + incognito_mode: IncognitoRecordMode | None = None + if chat_session_request.incognito: + if not incognito_context_available(): + raise OnyxError( + OnyxErrorCode.DEPLOYMENT_UNSUPPORTED, + "Incognito chat is not supported on this deployment.", + ) + if not incognito_allowed_for_user(user, db_session): + raise OnyxError( + OnyxErrorCode.UNAUTHORIZED, + "Incognito chat is not enabled for this user.", + ) + incognito_mode = resolve_incognito_record_mode() + return create_chat_session( db_session=db_session, description=chat_session_request.description or "", user_id=user.id, persona_id=chat_session_request.persona_id, project_id=chat_session_request.project_id, + incognito_record_mode=incognito_mode, ) diff --git a/backend/onyx/chat/incognito.py b/backend/onyx/chat/incognito.py new file mode 100644 index 00000000000..cba7cea7cf7 --- /dev/null +++ b/backend/onyx/chat/incognito.py @@ -0,0 +1,103 @@ +"""Recording policy for incognito chat turns. + +An incognito chat is an ordinary chat carrying a mode. ``IncognitoRecordMode`` +is the only policy object: behavior is exposed as derived properties on it, so +the legal states are the only representable ones and no caller can assemble an +illegal combination out of loose booleans. + +The contract that enforcement points must honor: an incognito session must pin +its mode on a metadata-only ``chat_session`` row at creation, and downstream +code must read the pinned value, never the live admin setting, so a setting +change cannot alter a session under way. Only FULL_HISTORY may write +conversation content into ``chat_message`` rows. USAGE_ONLY writes +content-free rows and must carry the live conversation outside Postgres +for the length of the session. +""" + +from uuid import UUID + +from sqlalchemy.orm import Session + +from onyx.chat.incognito_context import incognito_context_available +from onyx.db.enums import IncognitoRecordMode, record_mode_persists_content +from onyx.db.file_record import get_incognito_file_ids +from onyx.db.incognito import user_in_incognito_enabled_group +from onyx.db.models import User +from onyx.file_store.file_store import get_default_file_store +from onyx.file_store.models import FileDescriptor +from onyx.server.security.models import IncognitoAvailability +from onyx.server.security.store import get_security_settings, load_effective_uncached +from onyx.utils.logger import setup_logger +from shared_configs.contextvars import get_current_incognito_record_mode + +logger = setup_logger() + + +def current_turn_persists_content() -> bool: + mode = IncognitoRecordMode.from_context_value(get_current_incognito_record_mode()) + return record_mode_persists_content(mode) + + +def incognito_allowed_for_user(user: User, db_session: Session) -> bool: + """Whether this user may start an incognito chat. + + Availability composes the deployment capability (the ephemeral store must + exist) with the admin's security setting, which defaults to off. Anonymous + users never qualify: they share an identity, have no memberships, and + cannot authenticate against the teardown endpoint. + """ + if user.is_anonymous: + return False + if not incognito_context_available(): + return False + availability = get_security_settings().incognito_availability + if availability is IncognitoAvailability.EVERYONE: + return True + if availability is IncognitoAvailability.GROUPS: + return user_in_incognito_enabled_group(db_session, user.id) + return False + + +def resolve_incognito_record_mode() -> IncognitoRecordMode: + """The mode a new incognito session must pin: the workspace's admin + record-mode setting, usage_only by default. + + Reads past the settings cache. Cache invalidation is process-local, so a + second api_server can hold the pre-save mode for the cache TTL, and this + read decides for the whole life of the session: pinning a stale + full_history would persist content the admin has already disallowed. + """ + return load_effective_uncached().incognito_record_mode + + +def content_free_file_descriptors( + file_descriptors: list[FileDescriptor], +) -> list[FileDescriptor]: + """Descriptors safe to persist for a content-free turn. Linkage ids and + type survive for the file-reader tool and teardown. The content-derived + filename does not.""" + return [ + FileDescriptor( + id=fd["id"], type=fd["type"], user_file_id=fd.get("user_file_id") + ) + for fd in file_descriptors + ] + + +def delete_incognito_generated_files( + chat_session_id: UUID, db_session: Session +) -> bool: + """Delete the blobs the session's tools saved. True when none remain. + + The file record carries the session stamp, so deleting the blob deletes the + handle with it. A blob the store refuses keeps both, which is what the + cleanup sweep retries from.""" + file_store = get_default_file_store() + outstanding = False + for file_id in get_incognito_file_ids(str(chat_session_id), db_session): + try: + file_store.delete_file(file_id, error_on_missing=False) + except Exception: + logger.warning("Failed to delete incognito generated file %s", file_id) + outstanding = True + return not outstanding diff --git a/backend/onyx/chat/incognito_context.py b/backend/onyx/chat/incognito_context.py new file mode 100644 index 00000000000..8c40d3cb738 --- /dev/null +++ b/backend/onyx/chat/incognito_context.py @@ -0,0 +1,201 @@ +"""Ephemeral conversation context for incognito chat turns. + +USAGE_ONLY must carry the live conversation outside Postgres, so it lives in +Redis: one value per session, a sliding TTL that starts over on every save, +and explicit teardown when the chat closes. Keys are tenant-prefixed by the +Redis client. Redis may evict or expire the value mid-session: an expired +value loads as empty and the turn continues without earlier context. + +Concurrent turns on one session are possible (the chat processing fence is a +status marker, not admission control), so save is a compare-and-set on a +version. A lost save means a concurrent writer won or the session ended, and +the caller must not retry with the history it loaded. +""" + +from uuid import UUID + +from pydantic import BaseModel, TypeAdapter, ValidationError + +from onyx.cache.interface import CacheBackendType +from onyx.chat.models import ChatMessageSimple +from onyx.chat.stream_buffer import stream_buffer_key_pattern +from onyx.configs import app_configs +from onyx.redis.redis_pool import get_redis_client +from onyx.utils.logger import setup_logger + +logger = setup_logger() + +# Sliding: restarted on every save, so context survives while the page stays +# active and dies within the hour once it goes idle or closes uncleanly. +INCOGNITO_CONTEXT_TTL_SECONDS = 3600 +# Long enough that an in-flight turn cannot resurrect a torn-down context. +_TOMBSTONE_TTL_SECONDS = INCOGNITO_CONTEXT_TTL_SECONDS +# Raw-storage caps. Token budgeting trims context further at prompt build. +# These only bound what one session may hold in Redis. +_MAX_CONTEXT_MESSAGES = 200 +_MAX_CONTEXT_BYTES = 1_000_000 +# 15 digits stay exact in a Lua double, and turn counts never approach it. +_MAX_VERSION_DIGITS = 15 + +_KEY_PREFIX = "incognito_ctx" + +_MESSAGES_ADAPTER: TypeAdapter[list[ChatMessageSimple]] = TypeAdapter( + list[ChatMessageSimple] +) + +# Stored value grammar: ``:``. Lua and Python agree +# only on the digits-before-colon prefix, mirrored by _parse_version_prefix. +# Non-matching values read as version 0. Applies when stored version == ARGV[1]. +_TOMBSTONE = b"tombstone" +_CAS_SCRIPT = """ +local cur = redis.call('GET', KEYS[1]) +if cur == 'tombstone' then + return 0 +end +local cur_version = 0 +if cur then + local v = string.match(cur, '^(%d+):') + if v ~= nil and #v <= 15 then + cur_version = tonumber(v) + end +end +if cur_version ~= tonumber(ARGV[1]) then + return 0 +end +redis.call('SET', KEYS[1], ARGV[2], 'EX', tonumber(ARGV[3])) +return 1 +""" + + +class IncognitoContext(BaseModel): + """A session's history plus the version that makes save a compare-and-set.""" + + version: int + messages: list[ChatMessageSimple] + + +def incognito_context_available() -> bool: + """Whether this deployment can hold incognito context at all. + + USAGE_ONLY content must never reach Postgres, so the Postgres cache + backend (Lite) means the feature is absent rather than degraded. + """ + return app_configs.CACHE_BACKEND == CacheBackendType.REDIS + + +def _context_key(chat_session_id: UUID) -> str: + return f"{_KEY_PREFIX}:{chat_session_id}" + + +def _parse_version_prefix(raw: bytes) -> tuple[int, bytes | None]: + """The value's version and JSON body, or (0, None) for a tombstone or a + value this store did not write. + + Byte-for-byte the same rule as the CAS script: ASCII digits, at most + ``_MAX_VERSION_DIGITS`` of them, immediately followed by a colon. + """ + prefix, sep, body = raw.partition(b":") + if sep and prefix.isdigit() and len(prefix) <= _MAX_VERSION_DIGITS: + return int(prefix), body + return 0, None + + +def load_incognito_context(chat_session_id: UUID) -> IncognitoContext: + """The session's context, messages oldest first. + + Empty messages mean nothing was written, the session was torn down, the + value expired, or its body failed to parse. The turn proceeds with whatever + loads: missing context is degraded recall, never an error. + """ + raw = get_redis_client().get(_context_key(chat_session_id)) + if raw is None: + return IncognitoContext(version=0, messages=[]) + + version, body = _parse_version_prefix(raw) + if body is None: + logger.warning( + "Dropping unreadable incognito context for session %s", chat_session_id + ) + return IncognitoContext(version=0, messages=[]) + try: + messages = _MESSAGES_ADAPTER.validate_json(body) + except ValidationError: + # Corrupt context must end the session cleanly, not fail the turn. + # Keeping the prefix version lets the next save overwrite the value. + logger.warning( + "Dropping unparseable incognito context for session %s", chat_session_id + ) + return IncognitoContext(version=version, messages=[]) + return IncognitoContext(version=version, messages=messages) + + +def save_incognito_context(chat_session_id: UUID, context: IncognitoContext) -> bool: + """Write the full history, bump the version, restart the idle clock. + + Applies only while the stored version still equals ``context.version`` + and the session has not been torn down. False means the write was + discarded. + + Images are stripped: file bytes do not round-trip JSON, and incognito + attachments only live within their own turn. Oldest messages fall off + past the count and byte caps. + """ + trimmed = [ + message.model_copy(update={"image_files": None, "image_token_count": 0}) + for message in context.messages[-_MAX_CONTEXT_MESSAGES:] + ] + body = _MESSAGES_ADAPTER.dump_json(trimmed) + while len(body) > _MAX_CONTEXT_BYTES and len(trimmed) > 1: + trimmed = trimmed[1:] + body = _MESSAGES_ADAPTER.dump_json(trimmed) + payload = f"{context.version + 1}:".encode() + body + + client = get_redis_client() + result = client.eval( + _CAS_SCRIPT, + keys=[_context_key(chat_session_id)], + args=[ + str(context.version).encode(), + payload, + str(INCOGNITO_CONTEXT_TTL_SECONDS).encode(), + ], + ) + return bool(result) + + +def append_incognito_message(chat_session_id: UUID, message: ChatMessageSimple) -> None: + """Append one message to the session's live context, tolerating failure. + + A lost compare-and-set (a concurrent writer or an ended session) or a Redis + blip must degrade the stored context, never fail the turn. + Worst case the next turn is missing this message, which the load contract + treats as ordinary missing context rather than an error. + """ + try: + context = load_incognito_context(chat_session_id) + context.messages.append(message) + if not save_incognito_context(chat_session_id, context): + logger.warning( + "Incognito context save lost the CAS for session %s", chat_session_id + ) + except Exception: + logger.exception( + "Failed to persist incognito context for session %s", chat_session_id + ) + + +def incognito_session_ended(chat_session_id: UUID) -> bool: + """Whether the live context is gone, by teardown or by expiry.""" + raw = get_redis_client().get(_context_key(chat_session_id)) + return raw is None or raw == _TOMBSTONE + + +def teardown_incognito_session(chat_session_id: UUID) -> None: + """End the session now: tombstone the context so an in-flight turn cannot + recreate it (a missing key reads as version zero), and delete the buffered + stream chunks holding the streamed answer NDJSON.""" + client = get_redis_client() + client.set(_context_key(chat_session_id), _TOMBSTONE, ex=_TOMBSTONE_TTL_SECONDS) + buffered = list(client.scan_iter(match=stream_buffer_key_pattern(chat_session_id))) + if buffered: + client.delete(*buffered) diff --git a/backend/onyx/chat/llm_loop.py b/backend/onyx/chat/llm_loop.py index c9006f99ade..d1435e0e19d 100644 --- a/backend/onyx/chat/llm_loop.py +++ b/backend/onyx/chat/llm_loop.py @@ -67,6 +67,7 @@ from onyx.tools.models import ( ChatFile, CustomToolCallSummary, + CustomToolUserFileSnapshot, MemoryToolResponseSnapshot, PythonToolRichResponse, ToolCallInfo, @@ -84,6 +85,7 @@ from onyx.tools.utils import compute_all_tool_tokens from onyx.tracing.framework.create import ChatTraceMetadata, trace from onyx.utils.logger import setup_logger +from shared_configs.contextvars import get_current_incognito_record_mode logger = setup_logger() @@ -1220,35 +1222,60 @@ def run_llm_loop( tool_response.rich_response.generated_files or None ) + # Custom tools save image/CSV blobs and return their ids. + generated_file_ids = None + if isinstance( + tool_response.rich_response, CustomToolCallSummary + ) and isinstance( + tool_response.rich_response.tool_result, CustomToolUserFileSnapshot + ): + generated_file_ids = ( + tool_response.rich_response.tool_result.file_ids or None + ) + # Persist memory if this is a memory tool response memory_snapshot: MemoryToolResponseSnapshot | None = None + incognito_memory_refusal: str | None = None if isinstance(tool_response.rich_response, MemoryToolResponse): - persisted_memory_id: int | None = None - if user_memory_context and user_memory_context.user_id: - if tool_response.rich_response.index_to_replace is not None: - persisted_memory_id = update_memory_at_index( - user_id=user_memory_context.user_id, - index=tool_response.rich_response.index_to_replace, - new_text=tool_response.rich_response.memory_text, - ) - else: - persisted_memory_id = add_memory( - user_id=user_memory_context.user_id, - memory_text=tool_response.rich_response.memory_text, - ) - operation: Literal["add", "update"] = ( - "update" - if tool_response.rich_response.index_to_replace is not None - else "add" - ) - memory_snapshot = MemoryToolResponseSnapshot( - memory_text=tool_response.rich_response.memory_text, - operation=operation, - memory_id=persisted_memory_id, - index=tool_response.rich_response.index_to_replace, - ) + # Any incognito mode refuses memory writes with an explicit + # error, so neither the model nor the user sees a saved + # memory that does not exist. + if get_current_incognito_record_mode() is not None: + incognito_memory_refusal = ( + "Error: memories cannot be saved from an incognito " + "chat. Tell the user their request was not saved." + ) + else: + persisted_memory_id: int | None = None + if user_memory_context and user_memory_context.user_id: + if tool_response.rich_response.index_to_replace is not None: + persisted_memory_id = update_memory_at_index( + user_id=user_memory_context.user_id, + index=tool_response.rich_response.index_to_replace, + new_text=tool_response.rich_response.memory_text, + ) + else: + persisted_memory_id = add_memory( + user_id=user_memory_context.user_id, + memory_text=tool_response.rich_response.memory_text, + ) + operation: Literal["add", "update"] = ( + "update" + if tool_response.rich_response.index_to_replace is not None + else "add" + ) + memory_snapshot = MemoryToolResponseSnapshot( + memory_text=tool_response.rich_response.memory_text, + operation=operation, + memory_id=persisted_memory_id, + index=tool_response.rich_response.index_to_replace, + ) - if memory_snapshot: + if incognito_memory_refusal: + saved_response = incognito_memory_refusal + # The next LLM cycle must see the refusal too. + tool_response.llm_facing_response = incognito_memory_refusal + elif memory_snapshot: saved_response = json.dumps(memory_snapshot.model_dump()) elif isinstance(tool_response.rich_response, CustomToolCallSummary): saved_response = json.dumps( @@ -1272,6 +1299,7 @@ def run_llm_loop( search_docs=displayed_docs or search_docs, generated_images=generated_images, generated_files=generated_files, + generated_file_ids=generated_file_ids, ) # Add to state container for partial save support state_container.add_tool_call(tool_call_info) diff --git a/backend/onyx/chat/llm_step.py b/backend/onyx/chat/llm_step.py index d414b530c0d..26cf6713762 100644 --- a/backend/onyx/chat/llm_step.py +++ b/backend/onyx/chat/llm_step.py @@ -9,6 +9,7 @@ from onyx.chat.chat_state import ChatStateContainer from onyx.chat.citation_processor import DynamicCitationProcessor from onyx.chat.emitter import Emitter +from onyx.chat.incognito import current_turn_persists_content from onyx.chat.models import ChatMessageSimple, LlmStepResult from onyx.chat.tool_call_args_streaming import maybe_emit_argument_delta from onyx.configs.app_configs import ( @@ -1155,7 +1156,7 @@ def _current_placement() -> Placement: llm_msg_history = translate_history_to_llm_format(history, llm.config) has_reasoned = False - if LOG_ONYX_MODEL_INTERACTIONS: + if LOG_ONYX_MODEL_INTERACTIONS and current_turn_persists_content(): logger.debug( "Message history:\n%s", _format_message_history_for_logging(llm_msg_history), @@ -1521,7 +1522,7 @@ def _emit_content_chunk(content_chunk: str) -> Generator[Packet, None, None]: # Note: Content (AgentResponseDelta) doesn't need an explicit end packet - OverallStop handles it # Tool calls are handled by tool execution code and emit their own packets (e.g., SectionEnd) - if LOG_ONYX_MODEL_INTERACTIONS: + if LOG_ONYX_MODEL_INTERACTIONS and current_turn_persists_content(): logger.debug("Accumulated reasoning: %s", accumulated_reasoning) logger.debug("Accumulated answer: %s", accumulated_answer) diff --git a/backend/onyx/chat/models.py b/backend/onyx/chat/models.py index 43ee8dc7bd7..57a34265488 100644 --- a/backend/onyx/chat/models.py +++ b/backend/onyx/chat/models.py @@ -37,6 +37,9 @@ class CustomToolResponse(BaseModel): class CreateChatSessionID(BaseModel): chat_session_id: UUID + # Echoes the pinned mode so the client can verify the server honored an + # incognito request. A server that omits it did not. + incognito: bool = False AnswerStreamPart = ( @@ -93,6 +96,9 @@ class ChatFullResponse(BaseModel): # Metadata message_id: int chat_session_id: UUID | None = None + # Echoes the pinned mode for newly-created sessions, like the streaming + # packet does. A server that omits it did not honor an incognito request. + incognito: bool = False error_msg: str | None = None diff --git a/backend/onyx/chat/process_message.py b/backend/onyx/chat/process_message.py index 07d781153fa..37e02dde81d 100644 --- a/backend/onyx/chat/process_message.py +++ b/backend/onyx/chat/process_message.py @@ -39,6 +39,14 @@ get_compression_params, ) from onyx.chat.emitter import Emitter +from onyx.chat.incognito import ( + content_free_file_descriptors, +) +from onyx.chat.incognito_context import ( + append_incognito_message, + incognito_session_ended, + load_incognito_context, +) from onyx.chat.llm_loop import EmptyLLMResponseError, run_llm_loop from onyx.chat.models import ( AnswerStream, @@ -78,9 +86,9 @@ ) from onyx.db.document_set import filter_document_set_names_by_user_access from onyx.db.engine.sql_engine import get_session_with_current_tenant -from onyx.db.enums import HookPoint +from onyx.db.enums import HookPoint, record_mode_persists_content from onyx.db.memory import get_memories -from onyx.db.models import ChatMessage, Persona, User, UserFile +from onyx.db.models import ChatMessage, ChatSession, Persona, User, UserFile from onyx.db.projects import get_user_files_from_project from onyx.db.tools import get_tools from onyx.deep_research.dr_loop import run_deep_research_llm_loop @@ -142,7 +150,11 @@ from onyx.utils.logger import setup_logger from onyx.utils.telemetry import mt_cloud_telemetry from onyx.utils.timing import log_function_time -from shared_configs.contextvars import get_current_tenant_id +from shared_configs.contextvars import ( + CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR, + CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR, + get_current_tenant_id, +) logger = setup_logger() ERROR_TYPE_CANCELLED = "cancelled" @@ -626,7 +638,10 @@ def build_chat_turn( user=user, db_session=db_session, ) - yield CreateChatSessionID(chat_session_id=chat_session.id) + yield CreateChatSessionID( + chat_session_id=chat_session.id, + incognito=chat_session.incognito_record_mode is not None, + ) chat_session = get_chat_session_by_id( chat_session_id=chat_session.id, user_id=user_id, @@ -741,11 +756,11 @@ def build_chat_turn( if parent_message.message_type == MessageType.USER: user_message = parent_message else: - # New message — run the Query Processing hook before saving to DB. - # Skipped on regeneration: the message already exists and was accepted previously. - # Skip for empty/whitespace-only messages — no meaningful query to process, - # and SendMessageRequest.message has no min_length guard. - if message_text.strip(): + # Runs only for new, non-blank messages: regeneration already processed + # this text, and SendMessageRequest.message has no min_length guard. The + # hook ships the query and user email out, so egress-suppressing modes skip it. + mode = chat_session.incognito_record_mode + if message_text.strip() and (mode is None or mode.fires_hooks): hook_result = execute_hook( db_session=db_session, hook_point=HookPoint.QUERY_PROCESSING, @@ -767,13 +782,22 @@ def build_chat_turn( # assistant/summary rows in save_chat.py) so budget math sums a single # unit even after mid-session model switches. default_tokenizer = get_tokenizer(None, None) + user_token_count = len(default_tokenizer.encode(message_text)) + # Incognito keeps the row for tracking (id, tokens, structure) but its + # text lives in the ephemeral store, never in Postgres. Token count is + # from the real text so usage and budgeting are unaffected. + keeps_content = record_mode_persists_content(mode) user_message = create_new_chat_message( chat_session_id=chat_session.id, parent_message=parent_message, - message=message_text, - token_count=len(default_tokenizer.encode(message_text)), + message=message_text if keeps_content else "", + token_count=user_token_count, message_type=MessageType.USER, - files=new_msg_req.file_descriptors, + files=( + new_msg_req.file_descriptors + if keeps_content + else content_free_file_descriptors(new_msg_req.file_descriptors) + ), db_session=db_session, commit=True, ) @@ -923,7 +947,7 @@ def build_chat_turn( user_message_id=user_message.id, responses=[ ModelResponseSlot(message_id=m.id, model_name=name) - for m, name in zip(reserved_messages, model_display_names) + for m, name in zip(reserved_messages, model_display_names, strict=True) ], ) else: @@ -957,6 +981,26 @@ def build_chat_turn( ) simple_chat_history = chat_history_result.simple_messages + # Incognito rows are content-free, so earlier turns come from the store and + # the current message's text is restored onto convert_chat_history()'s + # blank-row shape. Regeneration uses the store as-is, it already holds the turn. + incognito_mode = chat_session.incognito_record_mode + if not record_mode_persists_content(incognito_mode): + stored_messages = load_incognito_context(chat_session.id).messages + is_new_user_message = parent_message.message_type != MessageType.USER + if ( + is_new_user_message + and simple_chat_history + and simple_chat_history[-1].message_type == MessageType.USER + ): + current_user = simple_chat_history[-1].model_copy( + update={"message": new_msg_req.message} + ) + simple_chat_history = stored_messages + [current_user] + append_incognito_message(chat_session.id, current_user) + else: + simple_chat_history = stored_messages + # Metadata for every text file injected into the history. After context-window # truncation drops older messages, the LLM loop compares surviving file_id tags # against this map to discover "forgotten" files and provide their metadata to @@ -991,8 +1035,12 @@ def build_chat_turn( cache = get_cache_backend() reset_cancel_status(chat_session.id, cache) + # Bind the id, not the row: this closure is stored on ChatTurnSetup and + # would otherwise keep a detached ChatSession reachable for the whole turn. + chat_session_id = chat_session.id + def check_is_connected() -> bool: - return check_stop_signal(chat_session.id, cache) + return check_stop_signal(chat_session_id, cache) set_processing_status( chat_session_id=chat_session.id, @@ -1012,9 +1060,11 @@ def check_is_connected() -> bool: return ChatTurnSetup( new_msg_req=new_msg_req, - chat_session=chat_session, + chat_session_id=chat_session.id, + chat_session_project_id=chat_session.project_id, + incognito_record_mode=chat_session.incognito_record_mode, persona=persona, - user_message=user_message, + user_message_id=user_message.id, user_identity=user_identity, llms=llms, model_display_names=model_display_names, @@ -1247,7 +1297,7 @@ def _run_post_steps() -> None: # "processing", whatever happened to the request generator. try: set_processing_status( - chat_session_id=setup.chat_session.id, + chat_session_id=setup.chat_session_id, cache=setup.cache, value=False, ) @@ -1287,8 +1337,8 @@ def _run_model(model_idx: int) -> None: auto_detect_filters=auto_detect_search_filters, ), custom_tool_config=CustomToolConfig( - chat_session_id=setup.chat_session.id, - message_id=setup.user_message.id, + chat_session_id=setup.chat_session_id, + message_id=setup.user_message_id, additional_headers=setup.custom_tool_additional_headers, mcp_headers=setup.mcp_headers, ), @@ -1312,7 +1362,7 @@ def _run_model(model_idx: int) -> None: # Per-thread copy: run_llm_loop mutates simple_chat_history in-place. if n_models == 1 and setup.new_msg_req.deep_research: - if setup.chat_session.project_id: + if setup.chat_session_project_id: raise RuntimeError("Deep research is not supported for projects") run_deep_research_llm_loop( emitter=model_emitter, @@ -1325,7 +1375,7 @@ def _run_model(model_idx: int) -> None: reasoning_effort=setup.reasoning_effort, skip_clarification=setup.skip_clarification, user_identity=setup.user_identity, - chat_session_id=str(setup.chat_session.id), + chat_session_id=str(setup.chat_session_id), all_injected_file_metadata=setup.all_injected_file_metadata, ) else: @@ -1342,7 +1392,7 @@ def _run_model(model_idx: int) -> None: token_counter=get_llm_token_counter(model_llm), forced_tool_id=setup.forced_tool_id, user_identity=setup.user_identity, - chat_session_id=str(setup.chat_session.id), + chat_session_id=str(setup.chat_session_id), chat_files=setup.chat_files_for_tools, reasoning_effort=setup.reasoning_effort, include_citations=setup.new_msg_req.include_citations, @@ -1371,18 +1421,33 @@ def _save_errored_message(model_idx: int, context: _PersistContext) -> None: ChatMessage, setup.reserved_messages[model_idx].id ) if msg is not None: - info = model_error_info[model_idx] - detail = ( - info.message - if info is not None - else "model encountered an error during generation." - ) - error_text = "Error from %s: %s" % ( - setup.model_display_names[model_idx], - detail, - ) + mode = setup.incognito_record_mode + if not record_mode_persists_content(mode): + # Provider errors can echo prompt fragments, so the + # durable row gets a generic marker. The live stream + # still carries the real error to the user. + error_text = "The model encountered an error." + else: + info = model_error_info[model_idx] + detail = ( + info.message + if info is not None + else "model encountered an error during generation." + ) + error_text = "Error from %s: %s" % ( + setup.model_display_names[model_idx], + detail, + ) msg.message = error_text msg.error = error_text + # The reservation's placeholder count must not survive: + # rows carry the real output count, zero when none emitted. + partial_answer = state_containers[model_idx].get_answer_tokens() + msg.token_count = ( + len(get_tokenizer(None, None).encode(partial_answer)) + if partial_answer + else 0 + ) save_db_session.commit() except Exception: logger.exception( @@ -1416,7 +1481,7 @@ def _drain_to_completion() -> None: last_fence_refresh = now try: set_processing_status( - chat_session_id=setup.chat_session.id, + chat_session_id=setup.chat_session_id, cache=setup.cache, value=True, run_id=setup.processing_run_id, @@ -1554,7 +1619,7 @@ def _read_stream() -> AnswerStream: # the writer thread keeps draining to completion in the background. logger.info( "chat stream reader detached; writer continues for session %s", - setup.chat_session.id, + setup.chat_session_id, ) return _read_stream() @@ -1610,6 +1675,7 @@ def _stream_chat_turn( ) mock_response_token: Token[str | None] | None = None + incognito_mode_flag_set = False setup: ChatTurnSetup | None = None pre_run_packets: list[AnswerStreamPart] = [] run_started = False @@ -1677,10 +1743,29 @@ def _stream_chat_turn( assert setup is not None, ( "build_chat_turn must complete before _run_models is called" ) + # Read at trace start, by the memory gate, and by interaction logging. + # Cleared with a plain set: a Token reset raises when this generator's + # frames resume under a different context. + if setup.incognito_record_mode is not None: + CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.set( + setup.incognito_record_mode.value + ) + incognito_mode_flag_set = True + content_free = not record_mode_persists_content(setup.incognito_record_mode) + if content_free: + # Set for the whole turn so a blob any tool saves carries the + # session on its record, which is what teardown deletes by. + CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR.set(str(setup.chat_session_id)) stream_buffer = StreamBufferWriter( cache=setup.cache, - chat_session_id=setup.chat_session.id, + chat_session_id=setup.chat_session_id, run_id=setup.processing_run_id, + delete_on_done=content_free, + session_ended=( + (lambda: incognito_session_ended(setup.chat_session_id)) + if content_free + else None + ), ) for pre_run_packet in pre_run_packets: stream_buffer.append_line(get_json_line(pre_run_packet.model_dump())) @@ -1769,12 +1854,15 @@ def _stream_chat_turn( finally: if mock_response_token is not None: reset_llm_mock_response(mock_response_token) + if incognito_mode_flag_set: + CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.set(None) + CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR.set(None) try: # Once _run_models started, its writer thread owns the fence — the # run may still be in flight after this generator is closed. if setup is not None and not run_started: set_processing_status( - chat_session_id=setup.chat_session.id, + chat_session_id=setup.chat_session_id, cache=setup.cache, value=False, ) @@ -1923,6 +2011,12 @@ def llm_loop_completion_handle( "ChatMessage %d not found during completion" % assistant_message_id ) + incognito_session = db_session.get(ChatSession, chat_session_id) + incognito_mode = ( + incognito_session.incognito_record_mode if incognito_session else None + ) + keeps_content = record_mode_persists_content(incognito_mode) + save_chat_turn( message_text=final_answer, reasoning_tokens=reasoning_tokens, @@ -1934,8 +2028,22 @@ def llm_loop_completion_handle( is_clarification=is_clarification, emitted_citations=emitted_citations, pre_answer_processing_time=pre_answer_processing_time, + persist_content=keeps_content, ) + # Incognito: the answer lives only in the ephemeral store, and + # compression is skipped since a summary is a durable content row. + if not keeps_content: + append_incognito_message( + chat_session_id, + ChatMessageSimple( + message=final_answer, + token_count=attached_message.token_count, + message_type=MessageType.ASSISTANT, + ), + ) + return + updated_chat_history = create_chat_history_chain( chat_session_id=chat_session_id, db_session=db_session, @@ -2090,6 +2198,7 @@ def gather_stream_full( message_id: int | None = None top_documents: list[SearchDoc] = [] chat_session_id: UUID | None = None + incognito = False for packet in packets: if isinstance(packet, Packet): @@ -2109,6 +2218,7 @@ def gather_stream_full( message_id = packet.reserved_assistant_message_id elif isinstance(packet, CreateChatSessionID): chat_session_id = packet.chat_session_id + incognito = packet.incognito if message_id is None: raise ValueError("Message ID is required") @@ -2141,5 +2251,6 @@ def gather_stream_full( citation_info=citations, message_id=message_id, chat_session_id=chat_session_id, + incognito=incognito, error_msg=error_msg, ) diff --git a/backend/onyx/chat/save_chat.py b/backend/onyx/chat/save_chat.py index fedeb36bed9..c85b9a14447 100644 --- a/backend/onyx/chat/save_chat.py +++ b/backend/onyx/chat/save_chat.py @@ -176,6 +176,7 @@ def save_chat_turn( is_clarification: bool = False, emitted_citations: set[int] | None = None, pre_answer_processing_time: float | None = None, + persist_content: bool = True, ) -> None: """ Save a chat turn by populating the assistant_message and creating related entities. @@ -206,10 +207,20 @@ def save_chat_turn( sanitized_message_text = ( sanitize_string(message_text) if message_text else message_text ) - assistant_message.message = sanitized_message_text - assistant_message.reasoning_tokens = ( - sanitize_string(reasoning_tokens) if reasoning_tokens else reasoning_tokens - ) + # A content-free turn keeps the row and its token count, which comes from + # the real answer, but none of the conversation-derived parts. + if persist_content: + assistant_message.message = sanitized_message_text + assistant_message.reasoning_tokens = ( + sanitize_string(reasoning_tokens) if reasoning_tokens else reasoning_tokens + ) + else: + assistant_message.message = "" + assistant_message.reasoning_tokens = None + tool_calls = [] + citation_to_doc = {} + all_search_docs = {} + emitted_citations = set() assistant_message.is_clarification = is_clarification # Use pre-answer processing time (captured when MESSAGE_START was emitted) diff --git a/backend/onyx/chat/stream_buffer.py b/backend/onyx/chat/stream_buffer.py index 9016a80bc8c..5a1402a572c 100644 --- a/backend/onyx/chat/stream_buffer.py +++ b/backend/onyx/chat/stream_buffer.py @@ -11,6 +11,7 @@ """ import zlib +from collections.abc import Callable from uuid import UUID from pydantic import BaseModel, ValidationError @@ -55,6 +56,11 @@ def _meta_key(chat_session_id: UUID, run_id: int) -> str: return f"{_PREFIX}_{chat_session_id}_{run_id}:meta" +def stream_buffer_key_pattern(chat_session_id: UUID) -> str: + """Glob matching every buffered chunk and meta key of the session's runs.""" + return f"{_PREFIX}_{chat_session_id}_*" + + class StreamBufferWriter: """Append-only writer for one run. Errors never propagate into the stream path — a broken cache downgrades the run to non-resumable (truncated).""" @@ -64,10 +70,20 @@ def __init__( cache: CacheBackend, chat_session_id: UUID, run_id: int, + delete_on_done: bool = False, + session_ended: Callable[[], bool] | None = None, ) -> None: self._cache = cache self._chat_session_id = chat_session_id self._run_id = run_id + # Content-free incognito runs: completion deletes the run's keys, so a + # flush racing the session teardown still cleans itself up. Costs + # post-completion resume. + self._delete_on_done = delete_on_done + # Teardown scans the session's keys once. Without this the run keeps + # writing answer chunks behind it, which then live out the buffer TTL + # if the run never reaches completion. + self._session_ended = session_ended self._meta = StreamBufferMeta() self._pending: list[str] = [] self._pending_bytes = 0 @@ -88,6 +104,11 @@ def append_line(self, line: str) -> None: def flush(self) -> None: if not self._pending or self._meta.truncated or self._meta.done: return + if self._session_ended is not None and self._session_ended(): + self._pending = [] + self._pending_bytes = 0 + self.mark_done() + return payload = zlib.compress("".join(self._pending).encode("utf-8")) self._pending = [] self._pending_bytes = 0 @@ -129,6 +150,23 @@ def flush(self) -> None: ) def mark_done(self) -> None: + if self._delete_on_done: + if self._meta.done: + return + self._meta.done = True + try: + self._cache.delete(_meta_key(self._chat_session_id, self._run_id)) + for chunk_n in range(self._meta.chunk_count): + self._cache.delete( + _chunk_key(self._chat_session_id, self._run_id, chunk_n) + ) + except Exception: + logger.exception( + "stream buffer deletion failed for session %s run %d", + self._chat_session_id, + self._run_id, + ) + return self.flush() if self._meta.done: return diff --git a/backend/onyx/configs/constants.py b/backend/onyx/configs/constants.py index 45ce6dd3c5a..69832cb5df2 100644 --- a/backend/onyx/configs/constants.py +++ b/backend/onyx/configs/constants.py @@ -133,6 +133,10 @@ # NOTE: we use this timeout / 4 in various places to refresh a lock # might be worth separating this timeout into separate timeouts for each situation +# One pass of the incognito cleanup sweep. Leftovers wait for the next pass +# rather than holding the beat lock past its timeout. +INCOGNITO_FILE_CLEANUP_BATCH = 200 + CELERY_GENERIC_BEAT_LOCK_TIMEOUT = 120 CELERY_VESPA_SYNC_BEAT_LOCK_TIMEOUT = 120 @@ -560,6 +564,7 @@ class OnyxRedisLocks: USER_FILE_PROJECT_SYNC_LOCK_PREFIX = "da_lock:user_file_project_sync" USER_FILE_PROJECT_SYNC_QUEUED_PREFIX = "da_lock:user_file_project_sync_queued" USER_FILE_DELETE_BEAT_LOCK = "da_lock:check_user_file_delete_beat" + INCOGNITO_FILE_CLEANUP_BEAT_LOCK = "da_lock:check_incognito_file_cleanup_beat" USER_FILE_DELETE_LOCK_PREFIX = "da_lock:user_file_delete" # Short-lived key set when a delete task is enqueued; cleared when the worker picks it up. # Prevents the beat from re-enqueuing the same file while a delete task is already queued. @@ -643,6 +648,7 @@ class OnyxCeleryTask: PROCESS_SINGLE_USER_FILE_PROJECT_SYNC = "process_single_user_file_project_sync" CHECK_FOR_USER_FILE_DELETE = "check_for_user_file_delete" DELETE_SINGLE_USER_FILE = "delete_single_user_file" + CHECK_FOR_INCOGNITO_FILE_CLEANUP = "check_for_incognito_file_cleanup" # Targeted reindex TARGETED_REINDEX_TASK = "targeted_reindex_task" @@ -743,9 +749,9 @@ class OnyxCeleryTask: # platform where the attribute actually resolves, since ty analyzes one # platform at a time and can't model cross-platform conditional unused-ignores. if platform.system() == "Darwin": - REDIS_SOCKET_KEEPALIVE_OPTIONS[getattr(socket, "TCP_KEEPALIVE")] = 60 + REDIS_SOCKET_KEEPALIVE_OPTIONS[getattr(socket, "TCP_KEEPALIVE")] = 60 # noqa: B009 else: - REDIS_SOCKET_KEEPALIVE_OPTIONS[getattr(socket, "TCP_KEEPIDLE")] = 60 + REDIS_SOCKET_KEEPALIVE_OPTIONS[getattr(socket, "TCP_KEEPIDLE")] = 60 # noqa: B009 class OnyxCallTypes(str, Enum): diff --git a/backend/onyx/connectors/airtable/airtable_connector.py b/backend/onyx/connectors/airtable/airtable_connector.py index 81529b62ca6..800d3c92c51 100644 --- a/backend/onyx/connectors/airtable/airtable_connector.py +++ b/backend/onyx/connectors/airtable/airtable_connector.py @@ -48,7 +48,6 @@ "lookup", "count", "formula", - "date", } @@ -249,7 +248,9 @@ def _extract_field_values( backoff=2, max_delay=10, ) - def get_attachment_with_retry(url: str, record_id: str) -> bytes | None: + def get_attachment_with_retry( + url: str, record_id: str, filename: str + ) -> bytes | None: try: attachment_response = requests.get( url, timeout=REQUEST_TIMEOUT_SECONDS @@ -280,7 +281,7 @@ def get_attachment_with_retry(url: str, record_id: str) -> bytes | None: ) raise - attachment_content = get_attachment_with_retry(url, record_id) + attachment_content = get_attachment_with_retry(url, record_id, filename) if attachment_content: try: file_ext = get_file_ext(filename) diff --git a/backend/onyx/connectors/connector_runner.py b/backend/onyx/connectors/connector_runner.py index b06d73741c4..1ea0d614bd3 100644 --- a/backend/onyx/connectors/connector_runner.py +++ b/backend/onyx/connectors/connector_runner.py @@ -33,7 +33,7 @@ def batched_doc_ids( batch_size: int, ) -> Generator[set[str], None, None]: batch: set[str] = set() - for document, hierarchy_node, failure, next_checkpoint in CheckpointOutputWrapper[ + for document, _hierarchy_node, failure, _next_checkpoint in CheckpointOutputWrapper[ CT ]()(checkpoint_connector_generator): if document is not None: @@ -177,10 +177,13 @@ def run( document, hierarchy_node, failure, - next_checkpoint, + loop_checkpoint, ) in CheckpointOutputWrapper[CT]()( checkpoint_connector_generator # ty: ignore[invalid-argument-type] ): + # Keep the last checkpoint seen; it is yielded after the loop. + next_checkpoint = loop_checkpoint + if document is not None: self.doc_batch.append(document) diff --git a/backend/onyx/connectors/discord/connector.py b/backend/onyx/connectors/discord/connector.py index f59841b9353..847ec51ecd0 100644 --- a/backend/onyx/connectors/discord/connector.py +++ b/backend/onyx/connectors/discord/connector.py @@ -266,12 +266,16 @@ def run_and_yield() -> Iterable[Document]: class DiscordConnector(PollConnector, LoadConnector): def __init__( self, - server_ids: list[str] = [], - channel_names: list[str] = [], + server_ids: list[str] | None = None, + channel_names: list[str] | None = None, # YYYY-MM-DD start_date: str | None = None, batch_size: int = INDEX_BATCH_SIZE, ): + if channel_names is None: + channel_names = [] + if server_ids is None: + server_ids = [] self.batch_size = batch_size self.channel_names: list[str] = channel_names if channel_names else [] self.server_ids: list[int] = ( diff --git a/backend/onyx/connectors/file/connector.py b/backend/onyx/connectors/file/connector.py index 83d8ee53bd7..d0b108ed390 100644 --- a/backend/onyx/connectors/file/connector.py +++ b/backend/onyx/connectors/file/connector.py @@ -238,7 +238,7 @@ def _process_file( ) # Then any extracted images from docx, PDFs, etc. - for idx, (img_data, img_name) in enumerate( + for idx, (img_data, _img_name) in enumerate( extraction_result.embedded_images, start=1 ): # Store each embedded image as a separate file in FileStore diff --git a/backend/onyx/connectors/gitbook/connector.py b/backend/onyx/connectors/gitbook/connector.py index 20e67bc2ab6..b314bde513a 100644 --- a/backend/onyx/connectors/gitbook/connector.py +++ b/backend/onyx/connectors/gitbook/connector.py @@ -146,7 +146,7 @@ def parse_block_node(node: dict[str, Any]) -> str: records.items(), key=lambda x: x[1].get("orderIndex", "") ) - for record_id, record_data in sorted_records: + for _record_id, record_data in sorted_records: values = record_data.get("values", {}) row_cells = [] for col_id in columns: @@ -179,6 +179,22 @@ def parse_block_node(node: dict[str, Any]) -> str: return markdown +def _parse_page_timestamp(raw: str | None) -> datetime | None: + if not raw: + return None + parsed = datetime.fromisoformat(raw) + if parsed.tzinfo is None: + return parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _page_last_modified(page: dict[str, Any]) -> datetime | None: + """updatedAt only exists once a page has been edited; fall back to createdAt.""" + return _parse_page_timestamp(page.get("updatedAt")) or _parse_page_timestamp( + page.get("createdAt") + ) + + def _convert_page_to_document( client: GitbookApiClient, space_id: str, page: dict[str, Any] ) -> Document: @@ -195,13 +211,8 @@ def _convert_page_to_document( ], source=DocumentSource.GITBOOK, semantic_identifier=page.get("title", ""), - doc_updated_at=datetime.fromisoformat(page["updatedAt"]).replace( - tzinfo=timezone.utc - ), - # NOTE: doc_created_at population not yet verified against live data - doc_created_at=datetime.fromisoformat(page["createdAt"]).replace( - tzinfo=timezone.utc - ), + doc_updated_at=_page_last_modified(page), + doc_created_at=_parse_page_timestamp(page.get("createdAt")), metadata={ "path": page.get("path", ""), "type": page.get("type", ""), @@ -250,16 +261,15 @@ def _fetch_all_pages( while pages: page = pages.pop(0) - updated_at_raw = page.get("updatedAt") - if updated_at_raw is None: - # if updatedAt is not present, that means the page has never been edited - continue + # always traverse children, even if this page falls outside the window + pages.extend(page.get("pages", [])) - updated_at = datetime.fromisoformat(updated_at_raw) - if start and updated_at < start: - continue - if end and updated_at > end: - continue + last_modified = _page_last_modified(page) + if last_modified is not None: + if start and last_modified < start: + continue + if end and last_modified > end: + continue current_batch.append( _convert_page_to_document(self.client, self.space_id, page) @@ -269,8 +279,6 @@ def _fetch_all_pages( yield current_batch current_batch = [] - pages.extend(page.get("pages", [])) - if current_batch: yield current_batch diff --git a/backend/onyx/connectors/github/connector.py b/backend/onyx/connectors/github/connector.py index c39b1fc3d30..3c1c5ffbdaa 100644 --- a/backend/onyx/connectors/github/connector.py +++ b/backend/onyx/connectors/github/connector.py @@ -274,7 +274,7 @@ def _get_batch_rate_limited( # this is needed to capture the rate limit exception here (if one occurs) for obj in objs: if hasattr(obj, "raw_data"): - getattr(obj, "raw_data") + _ = obj.raw_data yield from objs except RateLimitExceededException: sleep_after_rate_limit_exception(github_client) @@ -1489,7 +1489,7 @@ def build_dummy_checkpoint(self) -> GithubConnectorCheckpoint: # Run the connector while checkpoint.has_more: - for doc_batch, hierarchy_node_batch, failure, next_checkpoint in runner.run( + for doc_batch, _hierarchy_node_batch, failure, next_checkpoint in runner.run( checkpoint ): if doc_batch: diff --git a/backend/onyx/connectors/salesforce/connector.py b/backend/onyx/connectors/salesforce/connector.py index ffa316eb1f7..449e6180f8f 100644 --- a/backend/onyx/connectors/salesforce/connector.py +++ b/backend/onyx/connectors/salesforce/connector.py @@ -217,9 +217,11 @@ class SalesforceConnector(LoadConnector, PollConnector, SlimConnectorWithPermSyn def __init__( self, batch_size: int = INDEX_BATCH_SIZE, - requested_objects: list[str] = [], + requested_objects: list[str] | None = None, custom_query_config: str | None = None, ) -> None: + if requested_objects is None: + requested_objects = [] self.batch_size = batch_size self._sf_client: OnyxSalesforce | None = None @@ -379,7 +381,7 @@ def _load_csvs_to_db( with open(csv_path, "r", newline="", encoding="utf-8") as f: reader = csv.DictReader(f) - for row in reader: + for _row in reader: num_records += 1 new_ids = sf_db.update_from_csv( @@ -592,7 +594,7 @@ def _full_sync( num_records = 0 with open(csv_path, "r", newline="", encoding="utf-8") as f: reader = csv.DictReader(f) - for row in reader: + for _row in reader: num_records += 1 logger.debug( diff --git a/backend/onyx/connectors/sharepoint/connector.py b/backend/onyx/connectors/sharepoint/connector.py index a313d58a09e..745b630dd4c 100644 --- a/backend/onyx/connectors/sharepoint/connector.py +++ b/backend/onyx/connectors/sharepoint/connector.py @@ -1273,9 +1273,9 @@ class SharepointConnector( def __init__( self, batch_size: int = INDEX_BATCH_SIZE, - sites: list[str] = [], - excluded_sites: list[str] = [], - excluded_paths: list[str] = [], + sites: list[str] | None = None, + excluded_sites: list[str] | None = None, + excluded_paths: list[str] | None = None, include_site_pages: bool = True, include_site_documents: bool = True, treat_sharing_link_as_public: bool = False, @@ -1283,6 +1283,12 @@ def __init__( graph_api_host: str = DEFAULT_GRAPH_API_HOST, sharepoint_domain_suffix: str = DEFAULT_SHAREPOINT_DOMAIN_SUFFIX, ) -> None: + if excluded_paths is None: + excluded_paths = [] + if excluded_sites is None: + excluded_sites = [] + if sites is None: + sites = [] self.batch_size = batch_size self.sites = list(sites) self.excluded_sites = [s for p in excluded_sites if (s := p.strip())] @@ -1377,7 +1383,7 @@ def probe_role_assignments_permission(self) -> None: ) unauthorized_sites: list[str] = [ site_url - for site_url, authorized in zip(sites_to_probe, results) + for site_url, authorized in zip(sites_to_probe, results, strict=True) if authorized is False ] @@ -3687,7 +3693,7 @@ def retrieve_all_slim_docs_perm_sync( # Run the connector while checkpoint.has_more: - for doc_batch, hierarchy_node_batch, failure, next_checkpoint in runner.run( + for doc_batch, _hierarchy_node_batch, failure, next_checkpoint in runner.run( checkpoint ): if doc_batch: diff --git a/backend/onyx/connectors/slack/connector.py b/backend/onyx/connectors/slack/connector.py index 10334021a76..1178e57ec5e 100644 --- a/backend/onyx/connectors/slack/connector.py +++ b/backend/onyx/connectors/slack/connector.py @@ -377,9 +377,7 @@ def thread_to_doc( "group_leave", "group_archive", "group_unarchive", - "channel_leave", "channel_name", - "channel_join", } diff --git a/backend/onyx/connectors/teams/connector.py b/backend/onyx/connectors/teams/connector.py index 242ab9f9c56..37d36959d3a 100644 --- a/backend/onyx/connectors/teams/connector.py +++ b/backend/onyx/connectors/teams/connector.py @@ -2,6 +2,7 @@ import os from collections.abc import Iterator from datetime import datetime, timezone +from functools import partial from typing import Any import msal @@ -72,11 +73,13 @@ def __init__( self, # TODO: (chris) move from "Display Names" to IDs, since display names # are not necessarily guaranteed to be unique - teams: list[str] = [], + teams: list[str] | None = None, max_workers: int = MAX_WORKERS, authority_host: str = DEFAULT_AUTHORITY_HOST, graph_api_host: str = DEFAULT_GRAPH_API_HOST, ) -> None: + if teams is None: + teams = [] self.graph_client: GraphClient | None = None self.msal_app: msal.ConfidentialClientApplication | None = None self.max_workers = max_workers @@ -593,9 +596,7 @@ def _collect_all_teams( if next_url: url = next_url - query.before_execute( - lambda req: _update_request_url(request=req, next_url=url) - ) + query.before_execute(partial(_update_request_url, next_url=url)) team_collection = execute_query_with_retry( query, method_name="_collect_all_teams" @@ -784,9 +785,7 @@ def _collect_all_channels_from_team( ) if next_url: url = next_url - query = query.before_execute( - lambda req: _update_request_url(request=req, next_url=url) - ) + query = query.before_execute(partial(_update_request_url, next_url=url)) channel_collection = execute_query_with_retry( query, method_name="_collect_all_channels_from_team" @@ -870,7 +869,7 @@ def _collect_documents_for_channel( ) teams_connector.validate_connector_settings() - for slim_doc in teams_connector.retrieve_all_slim_docs_perm_sync(): + for _slim_doc in teams_connector.retrieve_all_slim_docs_perm_sync(): ... for doc in load_all_from_connector( diff --git a/backend/onyx/context/search/federated/slack_search.py b/backend/onyx/context/search/federated/slack_search.py index a2b1742dc5f..49071e1d708 100644 --- a/backend/onyx/context/search/federated/slack_search.py +++ b/backend/onyx/context/search/federated/slack_search.py @@ -1233,7 +1233,7 @@ def slack_retrieval( access_token=access_token, team_id=team_id, ) - for slack_message, thread_text in zip(slack_messages, thread_texts): + for slack_message, thread_text in zip(slack_messages, thread_texts, strict=True): slack_message.text = thread_text # get the highlighted texts from shortest to longest diff --git a/backend/onyx/context/search/pipeline.py b/backend/onyx/context/search/pipeline.py index 111ff9c5bbb..06797dc8d26 100644 --- a/backend/onyx/context/search/pipeline.py +++ b/backend/onyx/context/search/pipeline.py @@ -174,7 +174,7 @@ def merge_individual_chunks( chunk_to_section: dict[tuple[str, int], InferenceSection] = {} # Process each document's chunks - for doc_id, doc_chunk_list in doc_chunks.items(): + for _doc_id, doc_chunk_list in doc_chunks.items(): if not doc_chunk_list: continue diff --git a/backend/onyx/db/chat.py b/backend/onyx/db/chat.py index 285be197452..46c83651e22 100644 --- a/backend/onyx/db/chat.py +++ b/backend/onyx/db/chat.py @@ -7,11 +7,13 @@ from sqlalchemy import Row, delete, desc, func, nullsfirst, or_, select, update from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session, joinedload, selectinload +from sqlalchemy.sql.expression import ColumnElement from onyx.configs.chat_configs import HARD_DELETE_CHATS from onyx.configs.constants import MessageType from onyx.context.search.models import InferenceSection, SavedSearchDoc from onyx.context.search.models import SearchDoc as ServerSearchDoc +from onyx.db.enums import IncognitoRecordMode from onyx.db.models import ( ChatMessage, ChatMessage__SearchDoc, @@ -93,6 +95,28 @@ def get_chat_sessions_by_slack_thread_id( return db_session.scalars(stmt).all() +def get_incognito_session_ids_for_user( + user_id: UUID, db_session: Session +) -> list[UUID]: + return list( + db_session.scalars( + select(ChatSession.id).where( + ChatSession.user_id == user_id, + ChatSession.incognito_record_mode.is_not(None), + ) + ) + ) + + +def content_persisting_sessions_filter() -> ColumnElement[bool]: + """Ordinary chats plus incognito modes that persist content. Content-free + sessions have no message content to show on any history surface.""" + persisting = [m for m in IncognitoRecordMode if m.persists_content] + return ChatSession.incognito_record_mode.is_( + None + ) | ChatSession.incognito_record_mode.in_(persisting) + + # Retrieves chat sessions by user # Chat sessions do not include onyxbot flows def get_chat_sessions_by_user( @@ -104,6 +128,8 @@ def get_chat_sessions_by_user( project_id: int | None = None, only_non_project_chats: bool = False, include_failed_chats: bool = False, + exclude_incognito: bool = False, + exclude_content_free: bool = False, ) -> list[ChatSession]: stmt = ( select(ChatSession) @@ -112,6 +138,15 @@ def get_chat_sessions_by_user( .order_by(desc(ChatSession.time_updated)) ) + # The two exclusions are independent because the surfaces differ: the owner + # sees none of their incognito sessions, while a workspace surface keeps the + # full-history ones and drops only those with no content to show. + if exclude_incognito: + stmt = stmt.where(ChatSession.incognito_record_mode.is_(None)) + + if exclude_content_free: + stmt = stmt.where(content_persisting_sessions_filter()) + if deleted is not None: stmt = stmt.where(ChatSession.deleted == deleted) @@ -215,6 +250,7 @@ def create_chat_session( onyxbot_flow: bool = False, slack_thread_id: str | None = None, project_id: int | None = None, + incognito_record_mode: IncognitoRecordMode | None = None, ) -> ChatSession: chat_session = ChatSession( user_id=user_id, @@ -225,6 +261,7 @@ def create_chat_session( onyxbot_flow=onyxbot_flow, slack_thread_id=slack_thread_id, project_id=project_id, + incognito_record_mode=incognito_record_mode, ) db_session.add(chat_session) @@ -250,6 +287,8 @@ def duplicate_chat_session_for_user_from_slack( user_id=None, # Ignore user permissions for this db_session=db_session, ) + if chat_session.incognito_record_mode is not None: + raise ValueError("Incognito chat sessions cannot be duplicated") if not chat_session: raise HTTPException(status_code=400, detail="Invalid Chat Session ID provided") @@ -474,6 +513,9 @@ def add_chats_to_session_from_slack_thread( slack_chat_session_id: UUID, new_chat_session_id: UUID, ) -> None: + source_session = db_session.get(ChatSession, slack_chat_session_id) + if source_session and source_session.incognito_record_mode is not None: + raise ValueError("Incognito chat sessions cannot be duplicated") new_root_message = get_or_create_root_message( chat_session_id=new_chat_session_id, db_session=db_session, diff --git a/backend/onyx/db/chat_search.py b/backend/onyx/db/chat_search.py index f2c132c7361..1e926d5794f 100644 --- a/backend/onyx/db/chat_search.py +++ b/backend/onyx/db/chat_search.py @@ -32,6 +32,7 @@ def search_chat_sessions( stmt = ( select(ChatSession) .where(ChatSession.onyxbot_flow.is_(False)) + .where(ChatSession.incognito_record_mode.is_(None)) .order_by(desc(ChatSession.time_created)) .offset(offset_val) .limit(page_size + 1) @@ -53,7 +54,12 @@ def search_chat_sessions( # Otherwise, proceed with full-text search query = query.strip() - base_conditions: list[ColumnElement[bool]] = [ChatSession.onyxbot_flow.is_(False)] + # Applied to both arms of the union, so an incognito session cannot surface + # through a message body when its description does not match. + base_conditions: list[ColumnElement[bool]] = [ + ChatSession.onyxbot_flow.is_(False), + ChatSession.incognito_record_mode.is_(None), + ] if user_id is not None: base_conditions.append(ChatSession.user_id == user_id) if not include_deleted: diff --git a/backend/onyx/db/document.py b/backend/onyx/db/document.py index c53a9052540..3c93e43613b 100644 --- a/backend/onyx/db/document.py +++ b/backend/onyx/db/document.py @@ -1681,7 +1681,7 @@ def get_base_llm_doc_information( documents = [] - for doc_nr, doc in enumerate(results): + for _doc_nr, doc in enumerate(results): bare_doc = doc[0] documents.append( f"""* [{bare_doc.semantic_id}]({bare_doc.link}) ({bare_doc.doc_updated_at})""" diff --git a/backend/onyx/db/file_record.py b/backend/onyx/db/file_record.py index a9f65af2e1e..a09fa2bf313 100644 --- a/backend/onyx/db/file_record.py +++ b/backend/onyx/db/file_record.py @@ -6,6 +6,8 @@ from onyx.configs.constants import FileOrigin, FileType from onyx.db.enums import IndexingStatus from onyx.db.models import FileRecord, IndexAttempt +from onyx.file_store.constants import INCOGNITO_SESSION_METADATA_KEY +from shared_configs.contextvars import CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR def get_query_history_export_files( @@ -190,7 +192,18 @@ def upsert_filerecord( file_size: int | None = None, ) -> FileRecord: """Atomic upsert using INSERT ... ON CONFLICT DO UPDATE to avoid - race conditions when concurrent calls target the same file_id.""" + race conditions when concurrent calls target the same file_id. + + Every backend writes its record here, so this is also where a blob saved + during a content-free chat turn gets stamped with its session. The stamp + is the only handle cleanup has, and it lands with the record itself. + """ + session_id = CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR.get() + if session_id is not None: + file_metadata = { + **(file_metadata or {}), + INCOGNITO_SESSION_METADATA_KEY: session_id, + } stmt = insert(FileRecord).values( file_id=file_id, display_name=display_name, @@ -216,3 +229,30 @@ def upsert_filerecord( db_session.execute(stmt) return db_session.get(FileRecord, file_id) # ty: ignore[invalid-return-type] + + +def get_incognito_file_ids(session_id: str, db_session: Session) -> list[str]: + """Ids of blobs a content-free session produced and has not deleted yet.""" + return list( + db_session.scalars( + select(FileRecord.file_id).where( + FileRecord.file_metadata[INCOGNITO_SESSION_METADATA_KEY].astext + == session_id + ) + ) + ) + + +def get_session_ids_with_incognito_files( + db_session: Session, limit: int | None = None +) -> list[str]: + """Sessions still holding blobs. Empty in steady state, since teardown + deletes the records, so this only sees what a store failure left.""" + return list( + db_session.scalars( + select(FileRecord.file_metadata[INCOGNITO_SESSION_METADATA_KEY].astext) + .distinct() + .where(FileRecord.file_metadata.has_key(INCOGNITO_SESSION_METADATA_KEY)) + .limit(limit) + ) + ) diff --git a/backend/onyx/db/incognito.py b/backend/onyx/db/incognito.py new file mode 100644 index 00000000000..955295f5b28 --- /dev/null +++ b/backend/onyx/db/incognito.py @@ -0,0 +1,20 @@ +"""Membership query for groups-only incognito availability.""" + +from uuid import UUID + +from sqlalchemy import exists, select +from sqlalchemy.orm import Session + +from onyx.db.models import User__UserGroup, UserGroup + + +def user_in_incognito_enabled_group(db_session: Session, user_id: UUID) -> bool: + """Whether any of the user's groups has its incognito flag set.""" + stmt = select( + exists().where( + User__UserGroup.user_id == user_id, + User__UserGroup.user_group_id == UserGroup.id, + UserGroup.incognito_enabled.is_(True), + ) + ) + return bool(db_session.execute(stmt).scalar()) diff --git a/backend/onyx/db/models.py b/backend/onyx/db/models.py index 1561d9ebf35..7942f659055 100644 --- a/backend/onyx/db/models.py +++ b/backend/onyx/db/models.py @@ -124,7 +124,7 @@ from onyx.kg.models import KGEntityTypeAttributes, KGStage from onyx.llm.models import ReasoningEffort from onyx.llm.override_models import LLMOverride, PromptOverride -from onyx.server.security.models import SSRFProtectionLevel +from onyx.server.security.models import IncognitoAvailability, SSRFProtectionLevel from onyx.tools.tool_implementations.web_search.models import WebContentProviderConfig from onyx.utils.encryption import decrypt_bytes_to_string, encrypt_string_to_bytes from onyx.utils.headers import HeaderItemDict @@ -4608,6 +4608,27 @@ class SecuritySettings(Base): track_external_idp_expiry: Mapped[bool | None] = mapped_column( Boolean, nullable=True, default=None ) + # Stored as the IncognitoAvailability value (e.g. "groups"). None falls + # back to the off-by-default env behavior. + incognito_availability: Mapped[IncognitoAvailability | None] = mapped_column( + Enum( + IncognitoAvailability, + native_enum=False, + values_callable=lambda x: [e.value for e in x], + ), + nullable=True, + default=None, + ) + # What new incognito sessions pin. None falls back to usage_only. + incognito_record_mode: Mapped[IncognitoRecordMode | None] = mapped_column( + Enum( + IncognitoRecordMode, + native_enum=False, + values_callable=lambda x: [e.value for e in x], + ), + nullable=True, + default=None, + ) # Stored as the SSRFProtectionLevel value (e.g. "validate_all"); None falls # back to the level derived from the legacy SSRF env vars. ssrf_protection_level: Mapped[SSRFProtectionLevel | None] = mapped_column( @@ -5016,6 +5037,12 @@ class UserGroup(Base): # whether this is a default group (e.g. "Basic", "Admins") that cannot be deleted is_default: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + # Members may start incognito chats when the workspace availability mode + # is groups-only. Ignored under the other modes. + incognito_enabled: Mapped[bool] = mapped_column( + Boolean, nullable=False, default=False + ) + # Last time a user updated this user group time_last_modified_by_user: Mapped[datetime.datetime] = mapped_column( DateTime(timezone=True), server_default=func.now() @@ -6430,26 +6457,33 @@ class ScheduledTask(Base): back_populates="task", cascade="all, delete-orphan", ) - pre_approved_apps: Mapped[list["ScheduledTaskPreApprovedApp"]] = relationship( - "ScheduledTaskPreApprovedApp", + pre_approved_targets: Mapped[list["ScheduledTaskPreApprovedTarget"]] = relationship( + "ScheduledTaskPreApprovedTarget", back_populates="task", cascade="all, delete-orphan", - order_by="ScheduledTaskPreApprovedApp.id", + order_by="ScheduledTaskPreApprovedTarget.id", ) @property def pre_approved_external_app_ids(self) -> list[int]: - """Granted external-app ids in grant order. MCP-server grants are - excluded — their target ids live in a different id space, and this - property backs the API's external-app-id field (the gate reads all - grants regardless of kind via ``get_running_scheduled_run_grants``). - Set via ``onyx.db.scheduled_task.set_pre_approved_apps``.""" + """Granted external-app ids. MCP-server grants are excluded because + their target ids live in a different id space.""" return [ grant.gated_app.external_app_id - for grant in self.pre_approved_apps + for grant in self.pre_approved_targets if grant.gated_app.external_app_id is not None ] + @property + def pre_approved_mcp_server_ids(self) -> list[int]: + """Granted MCP-server ids. External-app grants are excluded because + their target ids live in a different id space.""" + return [ + grant.gated_app.mcp_server_id + for grant in self.pre_approved_targets + if grant.gated_app.mcp_server_id is not None + ] + __table_args__ = ( # Dispatcher hot path: WHERE status='active' AND deleted=false # AND next_run_at <= now() ORDER BY next_run_at @@ -6540,7 +6574,7 @@ class ScheduledTaskRun(Base): ) -class ScheduledTaskPreApprovedApp(Base): +class ScheduledTaskPreApprovedTarget(Base): """One (task, target) pre-approval grant: the matched target's ASK-gated actions skip the approval park for the task's RUNNING runs. @@ -6550,6 +6584,7 @@ class ScheduledTaskPreApprovedApp(Base): index serves the per-task lookup. """ + # The table name predates MCP support. Keep it to avoid a schema-only rename. __tablename__ = "scheduled_task_pre_approved_app" id: Mapped[int] = mapped_column(Integer, primary_key=True) @@ -6568,11 +6603,10 @@ class ScheduledTaskPreApprovedApp(Base): ) task: Mapped[ScheduledTask] = relationship( - "ScheduledTask", back_populates="pre_approved_apps" + "ScheduledTask", back_populates="pre_approved_targets" ) - # selectin: pre_approved_external_app_ids and set_pre_approved_apps read - # gated_app for every grant, so batch them in one SELECT rather than one - # per grant. + # selectin: the pre-approved target-id properties read gated_app for every + # grant, so batch them in one SELECT rather than one per grant. gated_app: Mapped["GatedApp"] = relationship("GatedApp", lazy="selectin") __table_args__ = ( diff --git a/backend/onyx/db/projects.py b/backend/onyx/db/projects.py index 8188b39ad2e..92a5e9ce369 100644 --- a/backend/onyx/db/projects.py +++ b/backend/onyx/db/projects.py @@ -71,7 +71,7 @@ def create_user_files( id_to_temp_id: dict[str, str] = {} # Pair returned storage paths with the same set of acceptable files we uploaded for file_path, file in zip( - upload_response.file_paths, categorized_files.acceptable + upload_response.file_paths, categorized_files.acceptable, strict=True ): new_id = uuid.uuid4() new_temp_id = ( diff --git a/backend/onyx/db/scheduled_task.py b/backend/onyx/db/scheduled_task.py index c57b3e8747e..47a45767be3 100644 --- a/backend/onyx/db/scheduled_task.py +++ b/backend/onyx/db/scheduled_task.py @@ -31,7 +31,7 @@ from onyx.db.models import ( GatedApp, ScheduledTask, - ScheduledTaskPreApprovedApp, + ScheduledTaskPreApprovedTarget, ScheduledTaskRun, ) from onyx.error_handling.error_codes import OnyxErrorCode @@ -60,6 +60,7 @@ def create_scheduled_task( editor_mode: EditorMode, status: ScheduledTaskStatus = ScheduledTaskStatus.ACTIVE, pre_approved_external_app_ids: list[int] | None = None, + pre_approved_mcp_server_ids: list[int] | None = None, now: datetime | None = None, ) -> ScheduledTask: """Insert a new ``ScheduledTask``. @@ -84,42 +85,60 @@ def create_scheduled_task( status=status, next_run_at=next_run_at, ) - set_pre_approved_apps( + _replace_pre_approved_targets( db_session, task, - GatedAppKind.EXTERNAL_APP, - pre_approved_external_app_ids or [], + external_app_ids=pre_approved_external_app_ids or [], + mcp_server_ids=pre_approved_mcp_server_ids or [], ) db_session.add(task) db_session.flush() return task -def set_pre_approved_apps( +def _replace_pre_approved_targets( db_session: Session, task: ScheduledTask, - kind: GatedAppKind, - target_ids: list[int], + *, + external_app_ids: list[int] | None = None, + mcp_server_ids: list[int] | None = None, ) -> None: - """Replace a task's ``kind`` pre-approval grants with ``target_ids`` - (deduped); grants of other kinds are preserved. Reuses existing grant rows so - re-submitting a granted target is a no-op — recreating it would orphan+reinsert - the same unique key in one flush, which Postgres rejects. Removed grants drop - via the ``delete-orphan`` cascade. + """Replace supplied target kinds and preserve omitted kinds. + + Reuse unchanged rows to avoid deleting and inserting the same unique key + in one flush. The orphan cascade deletes removed grants. """ - wanted_gated_app_ids = [ - get_or_create_gated_app_id(db_session, kind, target_id) - for target_id in dict.fromkeys(target_ids) + replacements = { + kind: target_ids + for kind, target_ids in ( + (GatedAppKind.EXTERNAL_APP, external_app_ids), + (GatedAppKind.MCP_SERVER, mcp_server_ids), + ) + if target_ids is not None + } + if not replacements: + return + + existing_by_target = { + grant.gated_app.target_key: grant for grant in task.pre_approved_targets + } + replacement_grants = [ + existing_by_target.get((kind, target_id)) + or ScheduledTaskPreApprovedTarget( + gated_app_id=get_or_create_gated_app_id(db_session, kind, target_id) + ) + for kind, target_ids in replacements.items() + for target_id in set(target_ids) + ] + retained_grants = [ + grant + for grant in task.pre_approved_targets + if grant.gated_app.kind not in replacements ] - existing = {grant.gated_app_id: grant for grant in task.pre_approved_apps} - other_kind_grants = [ - grant for grant in task.pre_approved_apps if grant.gated_app.kind is not kind + task.pre_approved_targets = [ + *replacement_grants, + *retained_grants, ] - task.pre_approved_apps = [ - existing.get(gated_app_id) - or ScheduledTaskPreApprovedApp(gated_app_id=gated_app_id) - for gated_app_id in wanted_gated_app_ids - ] + other_kind_grants def get_scheduled_task( @@ -175,6 +194,7 @@ def update_scheduled_task( editor_mode: EditorMode | None = None, status: ScheduledTaskStatus | None = None, pre_approved_external_app_ids: list[int] | None = None, + pre_approved_mcp_server_ids: list[int] | None = None, now: datetime | None = None, ) -> ScheduledTask: """Apply a partial update to a scheduled task. @@ -184,8 +204,8 @@ def update_scheduled_task( ``next_run_at`` is recomputed from ``now``. - If ``status`` transitions to PAUSED, ``next_run_at`` is set to NULL. - If ``status`` transitions to ACTIVE, ``next_run_at`` is recomputed. - - ``pre_approved_external_app_ids`` follows normal patch semantics: supplied - replaces the set, omitted leaves it unchanged. + - Each pre-approved target field follows normal patch semantics: supplied + replaces that target kind, and omitted leaves it unchanged. Raises: OnyxError(NOT_FOUND): the task does not exist or is not owned by @@ -200,10 +220,12 @@ def update_scheduled_task( task.name = name if prompt is not None: task.prompt = prompt - if pre_approved_external_app_ids is not None: - set_pre_approved_apps( - db_session, task, GatedAppKind.EXTERNAL_APP, pre_approved_external_app_ids - ) + _replace_pre_approved_targets( + db_session, + task, + external_app_ids=pre_approved_external_app_ids, + mcp_server_ids=pre_approved_mcp_server_ids, + ) if editor_mode is not None: task.editor_mode = editor_mode if cron_expression is not None and cron_expression != task.cron_expression: @@ -540,15 +562,15 @@ def get_live_scheduled_run_grants( if run is None: return None run_id, task_id = run - gated_apps = db_session.scalars( + gated_targets = db_session.scalars( select(GatedApp) .join( - ScheduledTaskPreApprovedApp, - ScheduledTaskPreApprovedApp.gated_app_id == GatedApp.id, + ScheduledTaskPreApprovedTarget, + ScheduledTaskPreApprovedTarget.gated_app_id == GatedApp.id, ) - .where(ScheduledTaskPreApprovedApp.scheduled_task_id == task_id) + .where(ScheduledTaskPreApprovedTarget.scheduled_task_id == task_id) ).all() - granted: set[GrantedTarget] = {ga.target_key for ga in gated_apps} + granted: set[GrantedTarget] = {target.target_key for target in gated_targets} return run_id, granted diff --git a/backend/onyx/db/users.py b/backend/onyx/db/users.py index 30eeb75d339..cfd2fb1b50a 100644 --- a/backend/onyx/db/users.py +++ b/backend/onyx/db/users.py @@ -186,7 +186,7 @@ def get_all_users( def _get_accepted_user_where_clause( email_filter_string: str | None = None, - roles_filter: list[UserRole] = [], + roles_filter: list[UserRole] | None = None, include_external: bool = False, is_active_filter: bool | None = None, ) -> list[ColumnElement[bool]]: @@ -206,6 +206,8 @@ def _get_accepted_user_where_clause( # Access table columns directly via __table__.c to get proper SQLAlchemy column types # This ensures type checking works correctly for SQL operations like ilike, endswith, and is_ + if roles_filter is None: + roles_filter = [] email_col: KeyedColumnElement[Any] = User.__table__.c.email is_active_col: KeyedColumnElement[Any] = User.__table__.c.is_active @@ -258,9 +260,11 @@ def get_page_of_filtered_users( page_num: int, email_filter_string: str | None = None, is_active_filter: bool | None = None, - roles_filter: list[UserRole] = [], + roles_filter: list[UserRole] | None = None, include_external: bool = False, ) -> Sequence[User]: + if roles_filter is None: + roles_filter = [] users_stmt = select(User) where_clause = _get_accepted_user_where_clause( @@ -281,9 +285,11 @@ def get_total_filtered_users_count( db_session: Session, email_filter_string: str | None = None, is_active_filter: bool | None = None, - roles_filter: list[UserRole] = [], + roles_filter: list[UserRole] | None = None, include_external: bool = False, ) -> int: + if roles_filter is None: + roles_filter = [] where_clause = _get_accepted_user_where_clause( email_filter_string=email_filter_string, roles_filter=roles_filter, diff --git a/backend/onyx/document_index/document_index_utils.py b/backend/onyx/document_index/document_index_utils.py index f64e52999d7..917865d9476 100644 --- a/backend/onyx/document_index/document_index_utils.py +++ b/backend/onyx/document_index/document_index_utils.py @@ -160,8 +160,13 @@ def get_uuid_from_chunk_info( def get_uuid_from_chunk_info_old( - *, document_id: str, chunk_id: int, large_chunk_reference_ids: list[int] = [] + *, + document_id: str, + chunk_id: int, + large_chunk_reference_ids: list[int] | None = None, ) -> UUID: + if large_chunk_reference_ids is None: + large_chunk_reference_ids = [] doc_str = document_id # Web parsing URL duplicate catching @@ -188,8 +193,11 @@ def get_uuid_from_chunk(chunk: DocMetadataAwareIndexChunk) -> uuid.UUID: def get_uuid_from_chunk_old( - chunk: DocMetadataAwareIndexChunk, large_chunk_reference_ids: list[int] = [] + chunk: DocMetadataAwareIndexChunk, + large_chunk_reference_ids: list[int] | None = None, ) -> UUID: + if large_chunk_reference_ids is None: + large_chunk_reference_ids = [] return get_uuid_from_chunk_info_old( document_id=chunk.source_document.id, chunk_id=chunk.chunk_id, diff --git a/backend/onyx/error_handling/error_codes.py b/backend/onyx/error_handling/error_codes.py index bf57a783b62..129e49bfd16 100644 --- a/backend/onyx/error_handling/error_codes.py +++ b/backend/onyx/error_handling/error_codes.py @@ -43,6 +43,8 @@ class OnyxErrorCode(Enum): EE_REQUIRED = ("EE_REQUIRED", 403) SINGLE_TENANT_ONLY = ("SINGLE_TENANT_ONLY", 403) ENV_VAR_GATED = ("ENV_VAR_GATED", 403) + # The deployment cannot support the feature at all, so no grant helps. + DEPLOYMENT_UNSUPPORTED = ("DEPLOYMENT_UNSUPPORTED", 403) # -------------------------------------------------------------------------- # Validation / Bad Request (400) diff --git a/backend/onyx/evals/eval.py b/backend/onyx/evals/eval.py index 8c3fc59ce3e..05b7d379852 100644 --- a/backend/onyx/evals/eval.py +++ b/backend/onyx/evals/eval.py @@ -446,8 +446,11 @@ def run_eval( configuration: EvalConfigurationOptions, data: list[dict[str, Any]] | None = None, remote_dataset_name: str | None = None, - provider: EvalProvider = get_provider(), + provider: EvalProvider | None = None, ) -> EvalationAck: + if provider is None: + provider = get_provider() + if data is not None and remote_dataset_name is not None: raise ValueError("Cannot specify both data and remote_dataset_name") diff --git a/backend/onyx/external_apps/providers/actions.py b/backend/onyx/external_apps/providers/actions.py index 588200fae73..06141dd821c 100644 --- a/backend/onyx/external_apps/providers/actions.py +++ b/backend/onyx/external_apps/providers/actions.py @@ -74,12 +74,16 @@ def path_matches(template: str, path: str) -> bool: # Wildcard must swallow >=1 segment and reject empties (`//`), like `{name}`. if not tail or not all(tail): return False - return all(_segment_matches(e, a) for e, a in zip(prefix, actual_segments)) + return all( + _segment_matches(e, a) + for e, a in zip(prefix, actual_segments, strict=False) + ) if len(expected_segments) != len(actual_segments): return False return all( - _segment_matches(e, a) for e, a in zip(expected_segments, actual_segments) + _segment_matches(e, a) + for e, a in zip(expected_segments, actual_segments, strict=True) ) diff --git a/backend/onyx/file_processing/extract_file_text.py b/backend/onyx/file_processing/extract_file_text.py index 2d744ced3cd..95ed91de28a 100644 --- a/backend/onyx/file_processing/extract_file_text.py +++ b/backend/onyx/file_processing/extract_file_text.py @@ -69,7 +69,7 @@ def get_markitdown_converter() -> "MarkItDown": # unindexable. from markitdown.converters._pptx_converter import PptxConverter - setattr( + setattr( # noqa: B010 PptxConverter, "_convert_chart_to_markdown", lambda self, chart: "\n\n[chart omitted]\n\n", # noqa: ARG005 diff --git a/backend/onyx/file_processing/html_utils.py b/backend/onyx/file_processing/html_utils.py index fd0f9971e68..f2f5af3cfe7 100644 --- a/backend/onyx/file_processing/html_utils.py +++ b/backend/onyx/file_processing/html_utils.py @@ -200,7 +200,7 @@ def web_html_cleanup( [ tag.extract() for tag in soup.find_all( - class_=lambda x: x and undesired_element in x.split() + class_=lambda x, cls=undesired_element: x and cls in x.split() ) ] diff --git a/backend/onyx/file_store/constants.py b/backend/onyx/file_store/constants.py index a0845d35ef7..b6836dcd48e 100644 --- a/backend/onyx/file_store/constants.py +++ b/backend/onyx/file_store/constants.py @@ -1,2 +1,6 @@ MAX_IN_MEMORY_SIZE = 30 * 1024 * 1024 # 30MB STANDARD_CHUNK_SIZE = 10 * 1024 * 1024 # 10MB chunks + +# Marks a blob a content-free chat turn produced, so cleanup can find it by the +# record itself rather than by anything that can expire. +INCOGNITO_SESSION_METADATA_KEY = "incognito_session_id" diff --git a/backend/onyx/indexing/chunking/tabular_section_chunker/tabular_section_chunker.py b/backend/onyx/indexing/chunking/tabular_section_chunker/tabular_section_chunker.py index ded935174c0..7ea151add13 100644 --- a/backend/onyx/indexing/chunking/tabular_section_chunker/tabular_section_chunker.py +++ b/backend/onyx/indexing/chunking/tabular_section_chunker/tabular_section_chunker.py @@ -72,7 +72,7 @@ def format_columns_header(headers: list[str]) -> str: def _row_to_pairs(headers: list[str], row: list[str]) -> list[tuple[str, str]]: - return [(h, v) for h, v in zip(headers, row) if v.strip()] + return [(h, v) for h, v in zip(headers, row, strict=False) if v.strip()] def pack_chunk(chunk: str, new_row: str) -> str: diff --git a/backend/onyx/indexing/embedder.py b/backend/onyx/indexing/embedder.py index d65c74d4261..12978d3c10e 100644 --- a/backend/onyx/indexing/embedder.py +++ b/backend/onyx/indexing/embedder.py @@ -178,7 +178,9 @@ def embed_chunks( title_embed_dict.update( { title: vector - for title, vector in zip(chunk_titles_list, title_embeddings) + for title, vector in zip( + chunk_titles_list, title_embeddings, strict=True + ) } ) diff --git a/backend/onyx/indexing/indexing_pipeline.py b/backend/onyx/indexing/indexing_pipeline.py index e45de709de1..e9cf61725bb 100644 --- a/backend/onyx/indexing/indexing_pipeline.py +++ b/backend/onyx/indexing/indexing_pipeline.py @@ -847,7 +847,7 @@ def _summarize(image_data: bytes, context_name: str) -> str: max_workers=MAX_IMAGE_WORKERS, ) - for p, result in zip(pending, results): + for p, result in zip(pending, results, strict=True): p.section.text = result or "[Error processing image]" return indexed_documents diff --git a/backend/onyx/indexing/port_reembed.py b/backend/onyx/indexing/port_reembed.py index 9a74420527c..bb06dfc34ac 100644 --- a/backend/onyx/indexing/port_reembed.py +++ b/backend/onyx/indexing/port_reembed.py @@ -300,7 +300,7 @@ def re_embed_chunks( ] doc_aware_chunks = [ _stored_chunk_to_doc_aware(chunk, embed_input) - for chunk, embed_input in zip(stored_chunks, embed_inputs) + for chunk, embed_input in zip(stored_chunks, embed_inputs, strict=True) ] embedded = embedder.embed_chunks(doc_aware_chunks) # Pair each stored chunk with its OWN vector by identity, not list position. @@ -312,7 +312,7 @@ def re_embed_chunks( content_vector=index_chunk.embeddings.full_embedding, title_vector=index_chunk.title_embedding, ) - for stored, index_chunk in zip(stored_chunks, matched) + for stored, index_chunk in zip(stored_chunks, matched, strict=True) ] @@ -341,7 +341,8 @@ def _reconstruct_source_document( contextual LLM to regenerate summaries — see the module docstring on the accepted imprecision of concatenating overlapping chunks.""" ordered = sorted( - zip(stored_chunks, bare_contents), key=lambda pair: pair[0].chunk_index + zip(stored_chunks, bare_contents, strict=True), + key=lambda pair: pair[0].chunk_index, ) doc_text = " ".join(bare for _, bare in ordered if bare) first = stored_chunks[0] @@ -384,7 +385,7 @@ def _augmentation_reembed( pairs_by_doc: dict[str, list[tuple[DocumentChunkWithoutVectors, str]]] = ( defaultdict(list) ) - for chunk, bare in zip(stored_chunks, bare_contents): + for chunk, bare in zip(stored_chunks, bare_contents, strict=True): pairs_by_doc[chunk.document_id].append((chunk, bare)) source_documents = { doc_id: _reconstruct_source_document( @@ -417,7 +418,7 @@ def _augmentation_reembed( large_chunk_id=None, large_chunk_reference_ids=[], ) - for chunk, bare in zip(stored_chunks, bare_contents) + for chunk, bare in zip(stored_chunks, bare_contents, strict=True) ] if future_rag_on: @@ -441,7 +442,9 @@ def _augmentation_reembed( matched = _match_embeddings_by_identity(stored_chunks, embedded) results: list[DocumentChunk] = [] - for stored, doc_aware, index_chunk in zip(stored_chunks, doc_aware_chunks, matched): + for stored, doc_aware, index_chunk in zip( + stored_chunks, doc_aware_chunks, matched, strict=True + ): fields = dict(stored) # The stored (BM25) content, rebuilt under FUTURE enrichment. fields["content"] = generate_enriched_content_for_chunk_text(doc_aware) diff --git a/backend/onyx/kg/clustering/clustering.py b/backend/onyx/kg/clustering/clustering.py index 0b9041a1aaf..47d09e0a7f5 100644 --- a/backend/onyx/kg/clustering/clustering.py +++ b/backend/onyx/kg/clustering/clustering.py @@ -329,7 +329,7 @@ def kg_clustering( # Cluster and transfer grounded entities sequentially start_time = time.monotonic() i_batch = 0 - for i_batch, untransferred_grounded_entities in enumerate( + for i_batch, untransferred_grounded_entities in enumerate( # noqa: B007 _get_batch_untransferred_grounded_entities( batch_size=processing_chunk_batch_size ) @@ -367,7 +367,7 @@ def kg_clustering( # Transfer the relationship types (no need to do in parallel as there's only a few) start_time = time.monotonic() i_batch = 0 - for i_batch, relationship_types in enumerate( + for i_batch, relationship_types in enumerate( # noqa: B007 _get_batch_untransferred_relationship_types( batch_size=processing_chunk_batch_size ) @@ -390,7 +390,7 @@ def kg_clustering( # Transfer the relationships in parallel start_time = time.monotonic() i_batch = 0 - for i_batch, relationships in enumerate( + for i_batch, relationships in enumerate( # noqa: B007 _get_batch_untransferred_relationships(batch_size=processing_chunk_batch_size) ): run_functions_tuples_in_parallel( @@ -413,7 +413,7 @@ def kg_clustering( # Update vespa for each document start_time = time.monotonic() i_batch = 0 - for i_batch, documents in enumerate( + for i_batch, documents in enumerate( # noqa: B007 _get_batch_kg_processed_documents(batch_size=processing_chunk_batch_size) ): batch_update_requests = run_functions_tuples_in_parallel( @@ -422,7 +422,9 @@ def kg_clustering( for document in documents ] ) - for update_requests, document in zip(batch_update_requests, documents): + for update_requests, document in zip( + batch_update_requests, documents, strict=True + ): try: update_kg_chunks_vespa_info(update_requests, index_name, tenant_id) except Exception as e: diff --git a/backend/onyx/kg/clustering/normalizations.py b/backend/onyx/kg/clustering/normalizations.py index ab9a8fc5296..6a82b704559 100644 --- a/backend/onyx/kg/clustering/normalizations.py +++ b/backend/onyx/kg/clustering/normalizations.py @@ -251,11 +251,11 @@ def normalize_entities( mapping: list[str | None] = run_functions_tuples_in_parallel( [ (_normalize_one_entity, (entity, attributes, allowed_docs_temp_view_name)) - for entity, attributes in zip(raw_entities, entity_attributes) + for entity, attributes in zip(raw_entities, entity_attributes, strict=True) ] ) for entity, attributes, normalized_entity in zip( - raw_entities, entity_attributes, mapping + raw_entities, entity_attributes, mapping, strict=True ): if normalized_entity is not None: normalized_entities.append(normalized_entity) diff --git a/backend/onyx/kg/extractions/extraction_processing.py b/backend/onyx/kg/extractions/extraction_processing.py index 4c9849ed15f..a0d9c914f54 100644 --- a/backend/onyx/kg/extractions/extraction_processing.py +++ b/backend/onyx/kg/extractions/extraction_processing.py @@ -406,8 +406,11 @@ def kg_extraction( batch_deep_extractions: dict[str, KGDocumentDeepExtractionResults] = { document_id: result for document_id, result in zip( - documents_to_process, + # Only the deep-extraction documents have a result. Skipped + # documents are not in `batch_deep_extraction_args`. + [arg[0] for arg in batch_deep_extraction_args], run_functions_tuples_in_parallel(batch_deep_extraction_func_calls), + strict=True, ) } diff --git a/backend/onyx/kg/utils/extraction_utils.py b/backend/onyx/kg/utils/extraction_utils.py index a3bca360040..5e52601bf0b 100644 --- a/backend/onyx/kg/utils/extraction_utils.py +++ b/backend/onyx/kg/utils/extraction_utils.py @@ -398,7 +398,6 @@ def kg_classify_document( return None # prepare prompt - implied_extraction.document_entity company_participants = implied_extraction.company_participant_emails account_participants = implied_extraction.account_participant_emails content = ( diff --git a/backend/onyx/kg/vespa/vespa_interactions.py b/backend/onyx/kg/vespa/vespa_interactions.py index c55d9be5ec7..7e8c6f0dab5 100644 --- a/backend/onyx/kg/vespa/vespa_interactions.py +++ b/backend/onyx/kg/vespa/vespa_interactions.py @@ -53,7 +53,7 @@ def get_document_vespa_contents( # Convert Vespa chunks to KGChunks # kg_chunks: list[KGChunkFormat] = [] - for i, chunk in enumerate(chunks): + for _i, chunk in enumerate(chunks): fields = chunk["fields"] if isinstance(fields.get("metadata", {}), str): fields["metadata"] = json.loads(fields["metadata"]) diff --git a/backend/onyx/llm/utils.py b/backend/onyx/llm/utils.py index bcc6b2b29cb..99f1b203e09 100644 --- a/backend/onyx/llm/utils.py +++ b/backend/onyx/llm/utils.py @@ -103,7 +103,7 @@ def _unwrap_nested_exception(error: Exception) -> Exception: candidate = cause elif ( hasattr(current, "args") - and len(getattr(current, "args")) == 1 + and len(current.args) == 1 and isinstance(current.args[0], Exception) ): candidate = current.args[0] diff --git a/backend/onyx/natural_language_processing/query_embedding_cache.py b/backend/onyx/natural_language_processing/query_embedding_cache.py index 0205d565b67..0f2047f8a2a 100644 --- a/backend/onyx/natural_language_processing/query_embedding_cache.py +++ b/backend/onyx/natural_language_processing/query_embedding_cache.py @@ -227,7 +227,7 @@ def cache_query_embeddings( successes = 0 errors = 0 - for query, embedding in zip(queries, embeddings): + for query, embedding in zip(queries, embeddings, strict=True): key = _build_key(query, search_settings_id) try: packed = _safe_pack_or_none(embedding) diff --git a/backend/onyx/onyxbot/discord/cache.py b/backend/onyx/onyxbot/discord/cache.py index 726083121ad..a4f8c536cd4 100644 --- a/backend/onyx/onyxbot/discord/cache.py +++ b/backend/onyx/onyxbot/discord/cache.py @@ -82,7 +82,7 @@ def load(tenant_id: str) -> TenantDiscordData | None: max_workers=_REFRESH_MAX_WORKERS, ) - for tenant_id, result in zip(tenant_ids, results): + for tenant_id, result in zip(tenant_ids, results, strict=True): if result is None: continue diff --git a/backend/onyx/prompts/prompt_utils.py b/backend/onyx/prompts/prompt_utils.py index 23c6a3beb2f..a516b743935 100644 --- a/backend/onyx/prompts/prompt_utils.py +++ b/backend/onyx/prompts/prompt_utils.py @@ -301,7 +301,7 @@ def drop_messages_history_overflow( final_messages: list[BaseMessage] = [] messages, token_counts = cast( - tuple[list[BaseMessage], list[int]], zip(*messages_with_token_cnts) + tuple[list[BaseMessage], list[int]], zip(*messages_with_token_cnts, strict=True) ) system_msg = ( final_messages[0] diff --git a/backend/onyx/secondary_llm_flows/memory_update.py b/backend/onyx/secondary_llm_flows/memory_update.py index 568bd3c6092..b589ba08663 100644 --- a/backend/onyx/secondary_llm_flows/memory_update.py +++ b/backend/onyx/secondary_llm_flows/memory_update.py @@ -27,7 +27,7 @@ def _format_chat_history(chat_history: list[ChatMinimalTextMessage]) -> str: recent_user_messages = user_messages[-MAX_USER_MESSAGES:] formatted_parts = [] - for i, msg in enumerate(recent_user_messages, start=1): + for _i, msg in enumerate(recent_user_messages, start=1): if len(msg.message) > MAX_CHARS_PER_MESSAGE: truncated_message = msg.message[:MAX_CHARS_PER_MESSAGE] + "[...truncated]" else: diff --git a/backend/onyx/server/auth_check.py b/backend/onyx/server/auth_check.py index 99f67083bdb..693fd3baabc 100644 --- a/backend/onyx/server/auth_check.py +++ b/backend/onyx/server/auth_check.py @@ -68,6 +68,7 @@ # oauth ("/auth/oauth/authorize", {"GET"}), ("/auth/oauth/callback", {"GET"}), + ("/mcp/oauth/client-metadata", {"GET"}), # dedicated mobile google oauth (callback routes to the api_server, not the web app) ("/auth/mobile/oauth/authorize", {"GET"}), ("/auth/mobile/oauth/callback", {"GET"}), diff --git a/backend/onyx/server/documents/connector.py b/backend/onyx/server/documents/connector.py index 6dac5ce9cd9..1d5c28be8ba 100644 --- a/backend/onyx/server/documents/connector.py +++ b/backend/onyx/server/documents/connector.py @@ -507,7 +507,7 @@ def _lookup_size(lookup_file_id: str) -> int | None: ) backfilled_sizes = { file_id: size - for file_id, size in zip(missing_size_ids, looked_up_sizes) + for file_id, size in zip(missing_size_ids, looked_up_sizes, strict=True) if isinstance(size, int) } if backfilled_sizes: @@ -523,7 +523,7 @@ def _lookup_size(lookup_file_id: str) -> int | None: ) files = [] - for file_id, file_name in zip(file_locations, file_names): + for file_id, file_name in zip(file_locations, file_names, strict=False): record = records_by_id.get(file_id) file_size = None upload_date = None @@ -657,7 +657,9 @@ def update_connector_files( remaining_file_names = [] removed_file_names = set() - for file_id, file_name in zip(current_file_locations, current_file_names): + for file_id, file_name in zip( + current_file_locations, current_file_names, strict=False + ): if file_id not in files_to_remove_set: remaining_file_locations.append(file_id) remaining_file_names.append(file_name) diff --git a/backend/onyx/server/features/build/sandbox/opencode/serve_client.py b/backend/onyx/server/features/build/sandbox/opencode/serve_client.py index 85f387bca7a..b13dff4bd21 100644 --- a/backend/onyx/server/features/build/sandbox/opencode/serve_client.py +++ b/backend/onyx/server/features/build/sandbox/opencode/serve_client.py @@ -839,7 +839,7 @@ def _merge_field_meta(event: SandboxEvent, extra: dict[str, Any]) -> None: existing = getattr(event, "field_meta", None) merged: dict[str, Any] = dict(existing) if isinstance(existing, dict) else {} merged.update(extra) - setattr(event, "field_meta", merged) + setattr(event, "field_meta", merged) # noqa: B010 def _emit_tool_events( diff --git a/backend/onyx/server/features/build/scheduled_tasks/api.py b/backend/onyx/server/features/build/scheduled_tasks/api.py index 72e0dc7d0c1..70be6864298 100644 --- a/backend/onyx/server/features/build/scheduled_tasks/api.py +++ b/backend/onyx/server/features/build/scheduled_tasks/api.py @@ -14,7 +14,7 @@ from __future__ import annotations from datetime import datetime, timezone -from typing import Any +from typing import AbstractSet, Any from uuid import UUID from fastapi import APIRouter, Depends, Query, Response @@ -32,6 +32,7 @@ ScheduledTaskTriggerSource, ) from onyx.db.external_app import get_external_apps +from onyx.db.mcp import get_craft_enabled_mcp_servers from onyx.db.models import ScheduledTask, ScheduledTaskRun, User from onyx.db.scheduled_task import ( create_scheduled_task, @@ -122,6 +123,7 @@ class ScheduledTaskCreate(_Forbid): status: ScheduledTaskStatus = ScheduledTaskStatus.ACTIVE run_immediately: bool = False pre_approved_app_ids: list[int] = Field(default_factory=list) + pre_approved_mcp_server_ids: list[int] = Field(default_factory=list) _dispatch = model_validator(mode="before")(_dispatch_editor_payload) @@ -139,6 +141,7 @@ class ScheduledTaskPatch(_Forbid): editor_payload: EditorPayload | None = None status: ScheduledTaskStatus | None = None pre_approved_app_ids: list[int] | None = None + pre_approved_mcp_server_ids: list[int] | None = None _dispatch = model_validator(mode="before")(_dispatch_editor_payload) @@ -210,6 +213,7 @@ class ScheduledTaskDetail(BaseModel): next_runs: list[datetime] last_run: RunSummary | None pre_approved_app_ids: list[int] + pre_approved_mcp_server_ids: list[int] created_at: datetime updated_at: datetime @@ -298,28 +302,53 @@ def _detail( next_runs=next_runs, last_run=RunSummary.from_model(last_run) if last_run is not None else None, pre_approved_app_ids=task.pre_approved_external_app_ids, + pre_approved_mcp_server_ids=task.pre_approved_mcp_server_ids, created_at=task.created_at, updated_at=task.updated_at, ) -def _validated_app_ids(db_session: Session, app_ids: list[int]) -> list[int]: - """Dedupe (order-preserving) and verify each id is a configured app. +def _validate_app_ids(db_session: Session, app_ids: list[int]) -> None: + """Reject unknown external app ids. Existence-only: a grant on an app that never produces an ASK match is inert, so credential / has-ASK-action filtering stays editor-side. """ - deduped = list(dict.fromkeys(app_ids)) - if not deduped: - return [] + if not app_ids: + return known_ids = {app.id for app in get_external_apps(db_session)} - unknown = [app_id for app_id in deduped if app_id not in known_ids] - if unknown: + unknown_ids = sorted(set(app_ids) - known_ids) + if unknown_ids: raise OnyxError( OnyxErrorCode.INVALID_INPUT, - f"Unknown external app id(s): {unknown}", + f"Unknown external app id(s): {unknown_ids}", + ) + + +def _validate_mcp_server_ids( + db_session: Session, + user: User, + server_ids: list[int], + already_approved_server_ids: AbstractSet[int] = frozenset(), +) -> None: + """Reject new MCP server ids unavailable to this user in Craft. + + Existing grants can remain after access changes. The MCP resolver still + blocks use, and users can remove the stale grant from the task editor. + """ + if not server_ids: + return + available_ids = { + server.id for server in get_craft_enabled_mcp_servers(db_session, user) + } + unavailable_ids = sorted( + set(server_ids) - available_ids - already_approved_server_ids + ) + if unavailable_ids: + raise OnyxError( + OnyxErrorCode.INVALID_INPUT, + f"Unknown or unavailable Craft MCP server id(s): {unavailable_ids}", ) - return deduped def _enqueue_executor(run_id: UUID) -> None: @@ -392,6 +421,8 @@ def create_task( and enqueue the executor. Does NOT touch ``next_run_at``. """ cron_expression = compile_to_cron(request.editor_payload) + _validate_app_ids(db_session, request.pre_approved_app_ids) + _validate_mcp_server_ids(db_session, user, request.pre_approved_mcp_server_ids) task = create_scheduled_task( db_session=db_session, @@ -401,9 +432,8 @@ def create_task( cron_expression=cron_expression, editor_mode=request.editor_mode, status=request.status, - pre_approved_external_app_ids=_validated_app_ids( - db_session, request.pre_approved_app_ids - ), + pre_approved_external_app_ids=request.pre_approved_app_ids, + pre_approved_mcp_server_ids=request.pre_approved_mcp_server_ids, ) if request.run_immediately: @@ -455,6 +485,22 @@ def patch_task( # whenever editor_payload is — no runtime check needed here. cron_expression = compile_to_cron(request.editor_payload) + if request.pre_approved_app_ids is not None: + _validate_app_ids(db_session, request.pre_approved_app_ids) + + if request.pre_approved_mcp_server_ids is not None: + existing_task = get_scheduled_task( + db_session=db_session, + task_id=task_id, + user_id=user.id, + ) + _validate_mcp_server_ids( + db_session, + user, + request.pre_approved_mcp_server_ids, + already_approved_server_ids=set(existing_task.pre_approved_mcp_server_ids), + ) + task = update_scheduled_task( db_session=db_session, task_id=task_id, @@ -464,11 +510,8 @@ def patch_task( cron_expression=cron_expression, editor_mode=request.editor_mode, status=request.status, - pre_approved_external_app_ids=( - _validated_app_ids(db_session, request.pre_approved_app_ids) - if request.pre_approved_app_ids is not None - else None - ), + pre_approved_external_app_ids=request.pre_approved_app_ids, + pre_approved_mcp_server_ids=request.pre_approved_mcp_server_ids, ) db_session.commit() db_session.refresh(task) diff --git a/backend/onyx/server/features/default_assistant/models.py b/backend/onyx/server/features/default_assistant/models.py index 3770a573727..c7a15b21e26 100644 --- a/backend/onyx/server/features/default_assistant/models.py +++ b/backend/onyx/server/features/default_assistant/models.py @@ -29,6 +29,3 @@ class DefaultAssistantUpdateRequest(BaseModel): default=None, description="New system prompt (instructions). None resets to default, empty string is allowed.", ) - - -3 diff --git a/backend/onyx/server/features/mcp/api.py b/backend/onyx/server/features/mcp/api.py index 55da69446cd..264fdaddd12 100644 --- a/backend/onyx/server/features/mcp/api.py +++ b/backend/onyx/server/features/mcp/api.py @@ -86,6 +86,12 @@ discover_mcp_tools, log_exception_group, ) +from onyx.server.features.mcp.client_metadata import ( + mcp_oauth_redirect_uri, +) +from onyx.server.features.mcp.client_metadata import ( + router as client_metadata_router, +) from onyx.server.features.mcp.models import ( MCPApiKeyResponse, MCPAuthTemplate, @@ -560,6 +566,7 @@ def _upsert_user_template_config( router = APIRouter(prefix="/mcp") +router.include_router(client_metadata_router) admin_router = APIRouter(prefix="/admin/mcp") HEADER_SUBSTITUTIONS: Literal["header_substitutions"] = "header_substitutions" @@ -621,13 +628,6 @@ def make_pkce_pair() -> tuple[str, str]: return verifier, challenge -MCP_OAUTH_CALLBACK_PATH = "/mcp/oauth/callback" - - -def _mcp_oauth_redirect_uri() -> str: - return f"{WEB_DOMAIN}{MCP_OAUTH_CALLBACK_PATH}" - - def _mcp_known_provider_flow_params( mcp_server: DbMCPServer, client_info: OAuthClientInformationFull, @@ -871,7 +871,7 @@ async def _connect_oauth( oauth_url = build_oauth_authorization_url( _mcp_known_provider_flow_params(mcp_server, client_info), - _mcp_oauth_redirect_uri(), + mcp_oauth_redirect_uri(), state, code_challenge=code_challenge, resource=( @@ -991,7 +991,7 @@ async def process_oauth_callback( token_payload = exchange_oauth_code_for_token( _mcp_known_provider_flow_params(mcp_server, client_info), code, - _mcp_oauth_redirect_uri(), + mcp_oauth_redirect_uri(), code_verifier=state_data.code_verifier, ) except (SSRFException, ValueError) as e: diff --git a/backend/onyx/server/features/mcp/client_metadata.py b/backend/onyx/server/features/mcp/client_metadata.py new file mode 100644 index 00000000000..0d89f41a73e --- /dev/null +++ b/backend/onyx/server/features/mcp/client_metadata.py @@ -0,0 +1,51 @@ +from fastapi import APIRouter, Response +from mcp.client.auth.utils import is_valid_client_metadata_url +from pydantic import AnyUrl + +from onyx.configs.app_configs import WEB_DOMAIN +from onyx.configs.constants import ONYX_DEFAULT_APPLICATION_NAME, PUBLIC_API_TAGS +from onyx.server.features.mcp.models import MCPOAuthClientMetadataDocument + +MCP_OAUTH_CALLBACK_PATH = "/mcp/oauth/callback" +MCP_OAUTH_CLIENT_METADATA_ROUTE = "/oauth/client-metadata" +MCP_OAUTH_CLIENT_METADATA_PUBLIC_PATH = "/api/mcp/oauth/client-metadata" +MCP_OAUTH_CLIENT_METADATA_CACHE_CONTROL = "public, max-age=3600" + +AUTHORIZATION_CODE_GRANT = "authorization_code" +REFRESH_TOKEN_GRANT = "refresh_token" +CODE_RESPONSE_TYPE = "code" +PUBLIC_CLIENT_AUTH_METHOD = "none" + +router = APIRouter() + + +def mcp_oauth_redirect_uri() -> str: + return f"{WEB_DOMAIN.rstrip('/')}{MCP_OAUTH_CALLBACK_PATH}" + + +def mcp_oauth_client_metadata_url() -> str: + return f"{WEB_DOMAIN.rstrip('/')}{MCP_OAUTH_CLIENT_METADATA_PUBLIC_PATH}" + + +def validated_mcp_oauth_client_metadata_url() -> str | None: + metadata_url = mcp_oauth_client_metadata_url() + return metadata_url if is_valid_client_metadata_url(metadata_url) else None + + +def build_mcp_oauth_client_metadata() -> MCPOAuthClientMetadataDocument: + return MCPOAuthClientMetadataDocument( + client_id=AnyUrl(mcp_oauth_client_metadata_url()), + client_name=ONYX_DEFAULT_APPLICATION_NAME, + redirect_uris=[AnyUrl(mcp_oauth_redirect_uri())], + grant_types=[AUTHORIZATION_CODE_GRANT, REFRESH_TOKEN_GRANT], + response_types=[CODE_RESPONSE_TYPE], + token_endpoint_auth_method=PUBLIC_CLIENT_AUTH_METHOD, + ) + + +@router.get(MCP_OAUTH_CLIENT_METADATA_ROUTE, tags=PUBLIC_API_TAGS) +def get_mcp_oauth_client_metadata( + response: Response, +) -> MCPOAuthClientMetadataDocument: + response.headers["Cache-Control"] = MCP_OAUTH_CLIENT_METADATA_CACHE_CONTROL + return build_mcp_oauth_client_metadata() diff --git a/backend/onyx/server/features/mcp/models.py b/backend/onyx/server/features/mcp/models.py index 7dd6a9520ad..ce6c77f92ef 100644 --- a/backend/onyx/server/features/mcp/models.py +++ b/backend/onyx/server/features/mcp/models.py @@ -1,11 +1,11 @@ import datetime import re from enum import Enum -from typing import Any, List, NotRequired, Optional, TypedDict +from typing import Any, List, Literal, NotRequired, Optional, TypedDict from uuid import UUID from mcp.types import Tool as MCPLibTool -from pydantic import BaseModel, Field, model_validator +from pydantic import AnyUrl, BaseModel, Field, model_validator from onyx.db.enums import ( EndpointPolicy, @@ -348,9 +348,8 @@ def validate_auth_configuration(self) -> "MCPToolCreateRequest": "admin_credentials is required when auth_performer is 'per_user'" ) - # OAuth client ID/secret are optional. If provided, they will seed the - # OAuth client info; otherwise, the MCP client will attempt dynamic - # client registration. + # OAuth client ID/secret are optional. Without them, auto-discovery + # attempts CIMD before falling back to dynamic client registration. if self.auth_type != MCPAuthenticationType.OAUTH: self.oauth_provider_mode = MCPOAuthProviderMode.AUTO_DISCOVERY self.oauth_authorization_endpoint = None @@ -491,10 +490,10 @@ class MCPUserOAuthConnectRequest(BaseModel): description="Ignore stored OAuth tokens and start a fresh authorization flow", ) oauth_client_id: str | None = Field( - None, description="OAuth client ID (optional for DCR)" + None, description="OAuth client ID (optional for CIMD or DCR)" ) oauth_client_secret: str | None = Field( - None, description="OAuth client secret (optional for DCR)" + None, description="OAuth client secret (optional for CIMD or DCR)" ) oauth_client_id_changed: bool = Field( default=False, @@ -541,6 +540,15 @@ class MCPOAuthCallbackResponse(BaseModel): redirect_url: str +class MCPOAuthClientMetadataDocument(BaseModel): + client_id: AnyUrl + client_name: str + redirect_uris: list[AnyUrl] + grant_types: list[Literal["authorization_code", "refresh_token"]] + response_types: list[Literal["code"]] + token_endpoint_auth_method: Literal["none"] + + class MCPDynamicClientRegistrationRequest(BaseModel): """Request for dynamic client registration per RFC 7591""" diff --git a/backend/onyx/server/features/mcp/oauth.py b/backend/onyx/server/features/mcp/oauth.py index 8b08a02e945..0b4abb0e12c 100644 --- a/backend/onyx/server/features/mcp/oauth.py +++ b/backend/onyx/server/features/mcp/oauth.py @@ -33,7 +33,6 @@ from onyx.cache.interface import CacheLockAcquisitionError from onyx.cache.locks import cache_shared_lock -from onyx.configs.app_configs import WEB_DOMAIN from onyx.db.engine.sql_engine import get_session_with_current_tenant from onyx.db.enums import MCPOAuthProviderMode, MCPTransport from onyx.db.mcp import ( @@ -46,6 +45,10 @@ from onyx.error_handling.exceptions import OnyxError from onyx.redis.redis_pool import get_redis_client from onyx.server.features.mcp.client import initialize_mcp_client +from onyx.server.features.mcp.client_metadata import ( + mcp_oauth_redirect_uri, + validated_mcp_oauth_client_metadata_url, +) from onyx.server.features.mcp.models import ( DENYLISTED_MCP_HEADERS, MCPConnectionData, @@ -911,12 +914,17 @@ async def callback_handler() -> tuple[str, str | None]: refresh_log_context, load_stored_tokens=load_stored_tokens, ) + client_metadata_url = ( + validated_mcp_oauth_client_metadata_url() + if mcp_server.oauth_provider_mode is MCPOAuthProviderMode.AUTO_DISCOVERY + else None + ) provider = OnyxOAuthClientProvider( refresh_log_context=refresh_log_context, server_url=mcp_server.server_url, client_metadata=OAuthClientMetadata( client_name=f"Onyx - {mcp_server.name}", - redirect_uris=[AnyUrl(f"{WEB_DOMAIN}/mcp/oauth/callback")], + redirect_uris=[AnyUrl(mcp_oauth_redirect_uri())], grant_types=["authorization_code", "refresh_token"], response_types=["code"], scope=REQUESTED_SCOPE, # TODO(evan): do we need to pass this in? maybe make configurable @@ -925,6 +933,7 @@ async def callback_handler() -> tuple[str, str | None]: storage=storage, redirect_handler=redirect_handler, callback_handler=callback_handler, + client_metadata_url=client_metadata_url, ) # A fresh provider per tool call starts with an empty context, so the SDK diff --git a/backend/onyx/server/features/projects/api.py b/backend/onyx/server/features/projects/api.py index cd0a9338005..1a984737866 100644 --- a/backend/onyx/server/features/projects/api.py +++ b/backend/onyx/server/features/projects/api.py @@ -31,6 +31,8 @@ get_project_token_count, upload_files_to_user_files_with_indexing, ) +from onyx.error_handling.error_codes import OnyxErrorCode +from onyx.error_handling.exceptions import OnyxError from onyx.server.features.projects.models import ( CategorizedFilesSnapshot, ChatSessionRequest, @@ -46,6 +48,19 @@ router = APIRouter(prefix="/user/projects") +# `user_project.name` and `user_project.description` are both varchar(255). Longer +# values make Postgres reject the write, which surfaces as a 500. +_MAX_PROJECT_FIELD_LENGTH = 255 + + +def _validate_project_field_length(value: str, field_name: str) -> None: + if len(value) > _MAX_PROJECT_FIELD_LENGTH: + raise OnyxError( + OnyxErrorCode.INVALID_INPUT, + f"Project {field_name} cannot be longer than " + f"{_MAX_PROJECT_FIELD_LENGTH} characters", + ) + class UserFileDeleteResult(BaseModel): has_associations: bool @@ -119,6 +134,7 @@ def create_project( ) -> UserProjectSnapshot: if name == "": raise HTTPException(status_code=400, detail="Project name cannot be empty") + _validate_project_field_length(name, "name") user_id = user.id project = UserProject(name=name, user_id=user_id) db_session.add(project) @@ -405,8 +421,10 @@ def update_project( raise HTTPException(status_code=404, detail="Project not found") if body.name is not None: + _validate_project_field_length(body.name, "name") project.name = body.name if body.description is not None: + _validate_project_field_length(body.description, "description") project.description = body.description db_session.commit() diff --git a/backend/onyx/server/features/projects/models.py b/backend/onyx/server/features/projects/models.py index 4c3dba40728..ca14a8ef5a0 100644 --- a/backend/onyx/server/features/projects/models.py +++ b/backend/onyx/server/features/projects/models.py @@ -28,8 +28,10 @@ class UserFileSnapshot(BaseModel): @classmethod def from_model( - cls, model: UserFile, temp_id_map: dict[str, str] = {} + cls, model: UserFile, temp_id_map: dict[str, str] | None = None ) -> "UserFileSnapshot": + if temp_id_map is None: + temp_id_map = {} return cls( id=model.id, temp_id=temp_id_map.get(str(model.id)), @@ -95,10 +97,12 @@ def from_model(cls, model: UserProject) -> "UserProjectSnapshot": created_at=model.created_at, user_id=model.user_id, instructions=model.instructions, + # A project lists its sessions by title, so an incognito chat would + # surface here the same way it would in the sidebar. chat_sessions=[ ChatSessionDetails.from_model(chat) for chat in model.chat_sessions - if not chat.deleted + if not chat.deleted and chat.incognito_record_mode is None ], ) diff --git a/backend/onyx/server/manage/voice/websocket_api.py b/backend/onyx/server/manage/voice/websocket_api.py index a8dfb1e059a..dfc8f3502c3 100644 --- a/backend/onyx/server/manage/voice/websocket_api.py +++ b/backend/onyx/server/manage/voice/websocket_api.py @@ -114,7 +114,7 @@ def trim_pcm16_silence(audio: bytes) -> bytes: speech_frame_indices = [ idx - for idx, frame_rms in zip(frame_offsets, frame_rms_values) + for idx, frame_rms in zip(frame_offsets, frame_rms_values, strict=True) if frame_rms >= threshold ] if not speech_frame_indices: diff --git a/backend/onyx/server/query_and_chat/chat_backend.py b/backend/onyx/server/query_and_chat/chat_backend.py index 8f2afca4ea1..05796e8a53c 100644 --- a/backend/onyx/server/query_and_chat/chat_backend.py +++ b/backend/onyx/server/query_and_chat/chat_backend.py @@ -27,6 +27,11 @@ create_chat_session_from_request, extract_headers, ) +from onyx.chat.incognito import ( + delete_incognito_generated_files, + incognito_allowed_for_user, +) +from onyx.chat.incognito_context import teardown_incognito_session from onyx.chat.models import ChatFullResponse, CreateChatSessionID from onyx.chat.process_message import ( gather_stream_full, @@ -53,6 +58,7 @@ get_chat_messages_by_session, get_chat_session_by_id, get_chat_sessions_by_user, + get_incognito_session_ids_for_user, set_as_latest_chat_message, set_preferred_response, translate_db_message_to_chat_message_detail, @@ -60,7 +66,7 @@ ) from onyx.db.chat_search import search_chat_sessions from onyx.db.engine.sql_engine import get_session, get_session_with_current_tenant -from onyx.db.enums import Permission +from onyx.db.enums import Permission, record_mode_persists_content from onyx.db.feedback import create_chat_message_feedback, remove_chat_message_feedback from onyx.db.llm import fetch_default_chat_naming_model from onyx.db.models import ChatMessage, ChatSessionSharedStatus, Persona, User @@ -79,6 +85,7 @@ ) from onyx.llm.override_models import LLMOverride from onyx.secondary_llm_flows.chat_session_naming import ( + DEFAULT_CHAT_SESSION_NAME, generate_chat_session_name, get_fallback_chat_session_name, ) @@ -199,6 +206,8 @@ def get_user_chat_sessions( project_id=project_id, only_non_project_chats=only_non_project_chats, include_failed_chats=include_failed_chats, + # The owner's own history is the one surface incognito hides from. + exclude_incognito=True, limit=page_size + 1, before=before_dt, ) @@ -433,6 +442,7 @@ def get_chat_session( # Packets are now directly serialized as Packet Pydantic models packets=replay_packet_lists, current_run=current_run, + incognito=chat_session.incognito_record_mode is not None, ) @@ -450,6 +460,9 @@ def create_new_chat_session( user=user, db_session=db_session, ) + except OnyxError: + # Carries its own status and detail (e.g. incognito refused). + raise except ValueError as e: # Project or persona access denied raise HTTPException(status_code=403, detail=str(e)) @@ -457,7 +470,10 @@ def create_new_chat_session( logger.exception(e) raise HTTPException(status_code=400, detail="Invalid Persona provided.") - return CreateChatSessionID(chat_session_id=new_chat_session.id) + return CreateChatSessionID( + chat_session_id=new_chat_session.id, + incognito=new_chat_session.incognito_record_mode is not None, + ) def _generate_or_fallback_chat_session_name( @@ -541,6 +557,11 @@ def rename_chat_session( db_session=db_session, eager_load_persona=True, ) + # Auto-naming derives a title from the conversation and writes it to the + # session row. A non-persisting incognito mode keeps no content in + # Postgres, so it keeps the fallback name and skips the LLM call. + if not record_mode_persists_content(chat_session.incognito_record_mode): + return RenameChatSessionResponse(new_name=DEFAULT_CHAT_SESSION_NAME) full_history = create_chat_history_chain( chat_session_id=chat_session_id, db_session=db_session, @@ -594,15 +615,37 @@ def patch_chat_session( return None +def _teardown_incognito_after_delete(chat_session_id: UUID) -> None: + """The rows are already gone, so a failed teardown is logged rather than + failing a delete the caller cannot retry. The context TTL is the backstop.""" + try: + teardown_incognito_session(chat_session_id) + except Exception: + logger.exception("Incognito teardown failed for session %s", chat_session_id) + + @router.delete("/delete-all-chat-sessions", tags=PUBLIC_API_TAGS) def delete_all_chat_sessions( user: User = Depends(require_permission(Permission.BASIC_ACCESS)), db_session: Session = Depends(get_session), ) -> None: + incognito_session_ids = get_incognito_session_ids_for_user(user.id, db_session) + # Blobs first, and nothing is deleted while any remain: their ids live on + # the rows this is about to drop, so the other order strands them. + if not all( + delete_incognito_generated_files(incognito_id, db_session) + for incognito_id in incognito_session_ids + ): + raise OnyxError( + OnyxErrorCode.SERVICE_UNAVAILABLE, + "Some generated files could not be deleted yet. Try again shortly.", + ) try: delete_all_chat_sessions_for_user(user=user, db_session=db_session) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + for incognito_id in incognito_session_ids: + _teardown_incognito_after_delete(incognito_id) @router.delete("/delete-chat-session/{session_id}", tags=PUBLIC_API_TAGS) @@ -614,6 +657,17 @@ def delete_chat_session_by_id( ) -> None: user_id = user.id try: + session = get_chat_session_by_id( + chat_session_id=session_id, user_id=user_id, db_session=db_session + ) + is_incognito = session.incognito_record_mode is not None + if is_incognito and not delete_incognito_generated_files( + session_id, db_session + ): + raise OnyxError( + OnyxErrorCode.SERVICE_UNAVAILABLE, + "Some generated files could not be deleted yet. Try again shortly.", + ) # Use the provided hard_delete parameter if specified, otherwise use the default config actual_hard_delete = ( hard_delete if hard_delete is not None else HARD_DELETE_CHATS @@ -623,6 +677,47 @@ def delete_chat_session_by_id( ) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) + if is_incognito: + _teardown_incognito_after_delete(session_id) + + +class IncognitoAvailabilityResponse(BaseModel): + available: bool + + +@router.get("/incognito-availability") +def get_incognito_availability( + user: User = Depends(require_permission(Permission.BASIC_ACCESS)), + db_session: Session = Depends(get_session), +) -> IncognitoAvailabilityResponse: + """Whether the acting user may start an incognito chat, so the client can + hide the toggle. The create endpoint enforces the same rule regardless.""" + return IncognitoAvailabilityResponse( + available=incognito_allowed_for_user(user, db_session) + ) + + +@router.post("/end-incognito-session/{session_id}", tags=PUBLIC_API_TAGS) +def end_incognito_session( + session_id: UUID, + user: User = Depends(require_permission(Permission.BASIC_ACCESS)), + db_session: Session = Depends(get_session), +) -> None: + """Drop an incognito session's live context the moment the chat closes. + + The context TTL is only the backstop for when this never arrives, such as + a hard tab close. + """ + chat_session = get_chat_session_by_id( + chat_session_id=session_id, user_id=user.id, db_session=db_session + ) + if chat_session.incognito_record_mode is not None: + teardown_incognito_session(session_id) + if not delete_incognito_generated_files(session_id, db_session): + raise OnyxError( + OnyxErrorCode.SERVICE_UNAVAILABLE, + "Some generated files could not be deleted yet and will be retried.", + ) # NOTE: This endpoint is extremely central to the application, any changes to it should be reviewed and approved by an experienced @@ -674,7 +769,12 @@ def handle_send_chat_message( Returns: StreamingResponse | ChatFullResponse: Either streams or returns complete response. """ - logger.debug("Received new chat message: %s", chat_message_req.message) + # Session id only: the session's incognito mode isn't loaded yet, and a + # verbatim prompt in the debug log would be exactly the durable message + # log incognito must never leave behind. + logger.debug( + "Received new chat message for session %s", chat_message_req.chat_session_id + ) tenant_id = get_current_tenant_id() mt_cloud_telemetry( diff --git a/backend/onyx/server/query_and_chat/models.py b/backend/onyx/server/query_and_chat/models.py index e8831083cd4..92340101165 100644 --- a/backend/onyx/server/query_and_chat/models.py +++ b/backend/onyx/server/query_and_chat/models.py @@ -79,6 +79,9 @@ class ChatSessionCreationRequest(BaseModel): persona_id: int = 0 description: str | None = None project_id: int | None = None + # Start the session incognito. Refused with an error when incognito is + # unavailable, never silently downgraded to an ordinary chat. + incognito: bool = False class ChatFeedbackRequest(BaseModel): @@ -237,8 +240,8 @@ def model_dump( # ty: ignore[invalid-method-override] self, *args: list, **kwargs: dict[str, Any] ) -> dict[str, Any]: initial_dict = super().model_dump( - mode="json", *args, + mode="json", **kwargs, # ty: ignore[invalid-argument-type] ) initial_dict["time_sent"] = self.time_sent.isoformat() @@ -274,6 +277,9 @@ class ChatSessionDetailResponse(BaseModel): # Set while a run is in flight and resumable: cursor-0 replay+tail is # available at /chat-session/{id}/resume-stream. current_run: CurrentRunInfo | None = None + # True for sessions pinned to an incognito record mode, so a reload can + # restore the incognito UI state. + incognito: bool = False class AdminSearchRequest(BaseModel): diff --git a/backend/onyx/server/security/models.py b/backend/onyx/server/security/models.py index 799f3fe8629..0851c72d08e 100644 --- a/backend/onyx/server/security/models.py +++ b/backend/onyx/server/security/models.py @@ -4,6 +4,8 @@ from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from typing_extensions import Self +from onyx.db.enums import IncognitoRecordMode + class SSRFProtectionLevel(str, Enum): """How aggressively outbound HTTP requests are validated against private / @@ -93,6 +95,16 @@ def _tenant_editable() -> dict[str, bool]: return {_OPERATOR_LOCKED_MARKER: False} +class IncognitoAvailability(str, Enum): + """Who may start incognito chats. Secure default is OFF: the feature is + invisible until an admin turns it on.""" + + OFF = "off" + EVERYONE = "everyone" + # Only members of user groups whose incognito_enabled flag is set. + GROUPS = "groups" + + class SecuritySettingsOverrides(BaseModel): """Wire/storage shape. Absent / None on any field means "use env default".""" @@ -107,6 +119,12 @@ class SecuritySettingsOverrides(BaseModel): track_external_idp_expiry: bool | None = Field( default=None, json_schema_extra=_tenant_editable() ) + incognito_availability: IncognitoAvailability | None = Field( + default=None, json_schema_extra=_tenant_editable() + ) + incognito_record_mode: IncognitoRecordMode | None = Field( + default=None, json_schema_extra=_tenant_editable() + ) ssrf_protection_level: SSRFProtectionLevel | None = Field( default=None, json_schema_extra=_operator_locked() ) @@ -188,6 +206,8 @@ class SecuritySettings(BaseModel): user_directory_admin_only: bool track_external_idp_expiry: bool + incognito_availability: IncognitoAvailability + incognito_record_mode: IncognitoRecordMode ssrf_protection_level: SSRFProtectionLevel mask_credential_prefix: bool llm_custom_config_env_injection: bool diff --git a/backend/onyx/server/security/store.py b/backend/onyx/server/security/store.py index de68095288f..510ac1544d4 100644 --- a/backend/onyx/server/security/store.py +++ b/backend/onyx/server/security/store.py @@ -11,6 +11,7 @@ from onyx.configs import app_configs as _cfg from onyx.configs.constants import KV_PASSWORD_AUTH_ENABLED_KEY, OnyxRedisLocks from onyx.db.engine.sql_engine import get_session_with_current_tenant +from onyx.db.enums import IncognitoRecordMode from onyx.db.security_settings import load_overrides as _db_load_overrides from onyx.db.security_settings import upsert_overrides as _db_upsert_overrides from onyx.db.sso_provider import fetch_sso_providers @@ -20,6 +21,7 @@ from onyx.key_value_store.interface import KvKeyNotFoundError from onyx.server.security.models import ( OPERATOR_LOCKED_FIELDS, + IncognitoAvailability, SecuritySettings, SecuritySettingsOverrides, SSRFProtectionLevel, @@ -89,6 +91,9 @@ def _build_env_defaults() -> SecuritySettings: return SecuritySettings( user_directory_admin_only=_cfg.USER_DIRECTORY_ADMIN_ONLY, track_external_idp_expiry=_cfg.TRACK_EXTERNAL_IDP_EXPIRY, + # No env knob on purpose: incognito is off until an admin enables it. + incognito_availability=IncognitoAvailability.OFF, + incognito_record_mode=IncognitoRecordMode.USAGE_ONLY, ssrf_protection_level=_derive_ssrf_level_from_env(), mask_credential_prefix=_cfg.MASK_CREDENTIAL_PREFIX, llm_custom_config_env_injection=not MULTI_TENANT, diff --git a/backend/onyx/skills/builtin/pptx/scripts/office/validators/pptx.py b/backend/onyx/skills/builtin/pptx/scripts/office/validators/pptx.py index c030d63cbdc..71e1dd6d3ce 100644 --- a/backend/onyx/skills/builtin/pptx/scripts/office/validators/pptx.py +++ b/backend/onyx/skills/builtin/pptx/scripts/office/validators/pptx.py @@ -252,7 +252,7 @@ def validate_notes_slide_references(self): errors.append( f" Notes slide '{target}' is referenced by multiple slides: {', '.join(slide_names)}" ) - for slide_name, rels_file in references: + for _slide_name, rels_file in references: errors.append(f" - {rels_file.relative_to(self.unpacked_dir)}") if errors: diff --git a/backend/onyx/tools/fake_tools/research_agent.py b/backend/onyx/tools/fake_tools/research_agent.py index f96c04e684b..2e0c322756e 100644 --- a/backend/onyx/tools/fake_tools/research_agent.py +++ b/backend/onyx/tools/fake_tools/research_agent.py @@ -694,7 +694,7 @@ def run_research_agent_calls( ), ) for research_agent_call, parent_tool_call_id in zip( - research_agent_calls, parent_tool_call_ids + research_agent_calls, parent_tool_call_ids, strict=False ) ] diff --git a/backend/onyx/tools/models.py b/backend/onyx/tools/models.py index 999defbd901..3bbef6ec084 100644 --- a/backend/onyx/tools/models.py +++ b/backend/onyx/tools/models.py @@ -274,6 +274,8 @@ class ToolCallInfo(BaseModel): search_docs: list[SearchDoc] | None = None generated_images: list[GeneratedImage] | None = None generated_files: list[PythonExecutionFile] | None = None + # File-store ids of blobs custom tools saved during the call. + generated_file_ids: list[str] | None = None CHAT_SESSION_ID_PLACEHOLDER = "CHAT_SESSION_ID" diff --git a/backend/onyx/tools/tool_implementations/images/image_generation_tool.py b/backend/onyx/tools/tool_implementations/images/image_generation_tool.py index 51265c55095..1bccc8c215f 100644 --- a/backend/onyx/tools/tool_implementations/images/image_generation_tool.py +++ b/backend/onyx/tools/tool_implementations/images/image_generation_tool.py @@ -400,7 +400,7 @@ def generate_all_images() -> None: revised_prompt=img.revised_prompt, shape=shape.value, ) - for img, file_id in zip(image_generation_responses, file_ids) + for img, file_id in zip(image_generation_responses, file_ids, strict=True) ] # Emit final packet with generated images diff --git a/backend/onyx/tools/tool_implementations/memory/memory_tool.py b/backend/onyx/tools/tool_implementations/memory/memory_tool.py index 67917d1bd8b..3bafad4ba5b 100644 --- a/backend/onyx/tools/tool_implementations/memory/memory_tool.py +++ b/backend/onyx/tools/tool_implementations/memory/memory_tool.py @@ -12,6 +12,7 @@ from typing_extensions import override from onyx.chat.emitter import Emitter +from onyx.chat.incognito import current_turn_persists_content from onyx.llm.interfaces import LLM from onyx.secondary_llm_flows.memory_update import process_memory_update from onyx.server.query_and_chat.placement import Placement @@ -134,7 +135,8 @@ def run( user_role=override_kwargs.user_role, ) - logger.info("New memory to be added: %s", memory_text) + if current_turn_persists_content(): + logger.info("New memory to be added: %s", memory_text) operation: Literal["add", "update"] = ( "update" if index_to_replace is not None else "add" diff --git a/backend/onyx/tools/tool_implementations/open_url/open_url_tool.py b/backend/onyx/tools/tool_implementations/open_url/open_url_tool.py index 33027cfa6ec..344c47a0a55 100644 --- a/backend/onyx/tools/tool_implementations/open_url/open_url_tool.py +++ b/backend/onyx/tools/tool_implementations/open_url/open_url_tool.py @@ -305,7 +305,7 @@ def _resolve_urls_to_document_ids( def _estimate_result_chars(result: dict[str, Any]) -> int: """Estimate character count from document fields in a result dict.""" total = 0 - for key, value in result.items(): + for _key, value in result.items(): if value is not None: total += len(str(value)) return total diff --git a/backend/onyx/tools/tool_implementations/python/python_tool.py b/backend/onyx/tools/tool_implementations/python/python_tool.py index e1a17b7bcba..bf4b0c0da8b 100644 --- a/backend/onyx/tools/tool_implementations/python/python_tool.py +++ b/backend/onyx/tools/tool_implementations/python/python_tool.py @@ -163,7 +163,7 @@ def _select_files_for_staging( ) over_budget = False - for idx, content in zip(batch, contents): + for idx, content in zip(batch, contents, strict=True): if content is None: logger.warning( "Failed to read file for Python execution: %s", @@ -327,7 +327,7 @@ def _upload_and_stage( allow_failures=True, max_workers=CODE_INTERPRETER_STAGING_CONCURRENCY, ) - for plan, ci_file_id in zip(misses, upload_results): + for plan, ci_file_id in zip(misses, upload_results, strict=True): if ci_file_id is None: logger.warning( "Failed to upload file for Python execution: %s", plan.file_name diff --git a/backend/onyx/tools/tool_implementations/search/search_utils.py b/backend/onyx/tools/tool_implementations/search/search_utils.py index b1df713336e..8053b99b414 100644 --- a/backend/onyx/tools/tool_implementations/search/search_utils.py +++ b/backend/onyx/tools/tool_implementations/search/search_utils.py @@ -84,7 +84,9 @@ def weighted_reciprocal_rank_fusion( id_to_source_rank: dict[str, int] = {} # Compute weighted RRF scores - for source_idx, (result_list, weight) in enumerate(zip(ranked_results, weights)): + for source_idx, (result_list, weight) in enumerate( + zip(ranked_results, weights, strict=True) + ): for rank, item in enumerate(result_list, start=1): item_id = id_extractor(item) @@ -241,7 +243,7 @@ def merge_overlapping_sections( merged_sections: dict[tuple[str, int], InferenceSection] = {} # Process each document's sections - for doc_id, doc_section_list in doc_sections.items(): + for _doc_id, doc_section_list in doc_sections.items(): if not doc_section_list: continue diff --git a/backend/onyx/tools/tool_implementations/web_search/web_search_tool.py b/backend/onyx/tools/tool_implementations/web_search/web_search_tool.py index 121e0d0cf60..dbd6ef99a35 100644 --- a/backend/onyx/tools/tool_implementations/web_search/web_search_tool.py +++ b/backend/onyx/tools/tool_implementations/web_search/web_search_tool.py @@ -249,7 +249,9 @@ def run( valid_results: list[list[WebSearchResult]] = [] failed_queries: dict[str, str] = {} - for query, (results, error) in zip(queries, search_results_with_errors): + for query, (results, error) in zip( + queries, search_results_with_errors, strict=True + ): if error is not None: failed_queries[query] = error elif results is not None: diff --git a/backend/onyx/tools/tool_runner.py b/backend/onyx/tools/tool_runner.py index 5cc1723449e..c7f6edb7184 100644 --- a/backend/onyx/tools/tool_runner.py +++ b/backend/onyx/tools/tool_runner.py @@ -246,7 +246,7 @@ def run_tool_calls( # Files from the chat session to pass to tools like PythonTool chat_files: list[ChatFile] | None = None, # A map of url -> summary for passing web results to open url tool - url_snippet_map: dict[str, str] = {}, + url_snippet_map: dict[str, str] | None = None, # When False, don't pass memory context to search tools for query expansion # (but still pass it to the memory tool for persistence) inject_memories_in_prompt: bool = True, @@ -289,6 +289,8 @@ def run_tool_calls( - `updated_citation_mapping`: The updated citation mapping dictionary. """ # Merge tool calls for SearchTool, WebSearchTool, and OpenURLTool + if url_snippet_map is None: + url_snippet_map = {} merged_tool_calls = _merge_tool_calls(tool_calls) if not merged_tool_calls: diff --git a/backend/onyx/tracing/braintrust_tracing_processor.py b/backend/onyx/tracing/braintrust_tracing_processor.py index 43917a8e818..ff791c00371 100644 --- a/backend/onyx/tracing/braintrust_tracing_processor.py +++ b/backend/onyx/tracing/braintrust_tracing_processor.py @@ -6,6 +6,7 @@ from onyx.llm.cost import compute_cost_cents from onyx.tracing.flows import IMAGE_FLOWS +from onyx.tracing.incognito import suppresses_external_traces from .framework.processor_interface import TracingProcessor from .framework.span_data import ( @@ -72,8 +73,16 @@ def __init__(self, logger: Optional[braintrust.Logger] = None): self._last_output: Dict[str, Any] = {} self._trace_metadata: Dict[str, Dict[str, Any]] = {} self._span_names: Dict[str, str] = {} + # Traces suppressed at start. Membership decides every later callback, + # so an incognito flag change mid-trace cannot mismatch start/end state. + self._suppressed_traces: set[str] = set() def on_trace_start(self, trace: Trace) -> None: + # Incognito turns must leave no content in external tracing, so the + # whole trace is dropped, spans included. + if suppresses_external_traces(): + self._suppressed_traces.add(trace.trace_id) + return trace_meta = trace.export() or {} metadata = trace_meta.get("metadata") or {} if metadata: @@ -102,6 +111,9 @@ def on_trace_start(self, trace: Trace) -> None: self._span_names[trace.trace_id] = trace.name def on_trace_end(self, trace: Trace) -> None: + if trace.trace_id in self._suppressed_traces: + self._suppressed_traces.discard(trace.trace_id) + return span: Any = self._spans.pop(trace.trace_id) self._trace_metadata.pop(trace.trace_id, None) self._span_names.pop(trace.trace_id, None) @@ -227,6 +239,8 @@ def _log_data(self, span: Span[Any]) -> Dict[str, Any]: return {} def on_span_start(self, span: Span[SpanData]) -> None: + if span.trace_id in self._suppressed_traces: + return parent: Any = ( self._spans[span.parent_id] if span.parent_id is not None @@ -253,6 +267,8 @@ def on_span_start(self, span: Span[SpanData]) -> None: created_span.set_current() def on_span_end(self, span: Span[SpanData]) -> None: + if span.trace_id in self._suppressed_traces: + return s: Any = self._spans.pop(span.span_id) self._span_names.pop(span.span_id, None) event = dict(error=span.error, **self._log_data(span)) diff --git a/backend/onyx/tracing/incognito.py b/backend/onyx/tracing/incognito.py new file mode 100644 index 00000000000..c500223cb0f --- /dev/null +++ b/backend/onyx/tracing/incognito.py @@ -0,0 +1,9 @@ +from onyx.db.enums import IncognitoRecordMode +from shared_configs.contextvars import get_current_incognito_record_mode + + +def suppresses_external_traces() -> bool: + """External tracing processors must check this at trace start and drop the + whole trace, spans included, when True.""" + mode = IncognitoRecordMode.from_context_value(get_current_incognito_record_mode()) + return mode is not None and not mode.emits_external_traces diff --git a/backend/onyx/tracing/langfuse_tracing_processor.py b/backend/onyx/tracing/langfuse_tracing_processor.py index c4db920d4dd..9799405b7c9 100644 --- a/backend/onyx/tracing/langfuse_tracing_processor.py +++ b/backend/onyx/tracing/langfuse_tracing_processor.py @@ -20,6 +20,7 @@ ) from onyx.tracing.framework.spans import Span from onyx.tracing.framework.traces import Trace +from onyx.tracing.incognito import suppresses_external_traces logger = logging.getLogger(__name__) @@ -64,6 +65,8 @@ def __init__( self._langfuse_span_ids: dict[ str, str ] = {} # framework_span_id -> langfuse_span.id + # Membership decides every later callback, immune to mid-trace changes. + self._suppressed_traces: set[str] = set() def _get_client(self) -> Langfuse: """Get or create Langfuse client.""" @@ -125,6 +128,10 @@ def _calculate_cost(self, data: GenerationSpanData) -> Optional[float]: def on_trace_start(self, trace: Trace) -> None: """Called when a trace is started.""" + if suppresses_external_traces(): + with self._lock: + self._suppressed_traces.add(trace.trace_id) + return try: client = self._get_client() trace_meta = trace.export() or {} @@ -162,6 +169,10 @@ def on_trace_start(self, trace: Trace) -> None: def on_trace_end(self, trace: Trace) -> None: """Called when a trace is finished.""" + with self._lock: + if trace.trace_id in self._suppressed_traces: + self._suppressed_traces.discard(trace.trace_id) + return try: with self._lock: langfuse_span = self._trace_spans.pop(trace.trace_id, None) @@ -191,6 +202,9 @@ def on_span_start(self, span: Span[SpanData]) -> None: agents run in parallel threads, and calling methods on span objects created in other threads can cause OpenTelemetry context issues. """ + with self._lock: + if span.trace_id in self._suppressed_traces: + return try: data = span.span_data # Declare as Any since different code paths return different observation types diff --git a/backend/onyx/utils/logger.py b/backend/onyx/utils/logger.py index 3d7f024ed1c..1d784fd273a 100644 --- a/backend/onyx/utils/logger.py +++ b/backend/onyx/utils/logger.py @@ -26,12 +26,15 @@ logging.addLevelName(logging.INFO + 5, "NOTICE") +# The shared default dicts are only ever read, never mutated in place: writers +# copy, update, then `set()` a new dict. pruning_ctx: contextvars.ContextVar[dict[str, Any]] = contextvars.ContextVar( - "pruning_ctx", default=dict() + "pruning_ctx", + default=dict(), # noqa: B039 ) doc_permission_sync_ctx: contextvars.ContextVar[dict[str, Any]] = ( - contextvars.ContextVar("doc_permission_sync_ctx", default=dict()) + contextvars.ContextVar("doc_permission_sync_ctx", default=dict()) # noqa: B039 ) diff --git a/backend/requirements/default.txt b/backend/requirements/default.txt index 4c0bd13d0d4..b71eb75ba64 100644 --- a/backend/requirements/default.txt +++ b/backend/requirements/default.txt @@ -531,6 +531,7 @@ charset-normalizer==3.4.4 \ # htmldate # markitdown # pdfminer-six + # reportlab # requests # trafilatura # unstructured @@ -2021,7 +2022,9 @@ pillow==12.3.0 \ --hash=sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5 \ --hash=sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d \ --hash=sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198 - # via python-pptx + # via + # python-pptx + # reportlab platformdirs==4.5.0 \ --hash=sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312 \ --hash=sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3 @@ -2672,6 +2675,9 @@ regex==2025.11.3 \ # dateparser # nltk # tiktoken +reportlab==5.0.0 \ + --hash=sha256:9d5a3affa84919e1111ede580031266a570e93b1ce388219621347965ff1d93c \ + --hash=sha256:e4494a0c6623ae213bb856fba523171b2b54a7bf629fda02d5e525a7b899a784 requests==2.33.0 \ --hash=sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b \ --hash=sha256:c7ebc5e8b0f21837386ad0e1c8fe8b829fa5f544d8df3b2253bff14ef29d7652 diff --git a/backend/requirements/dev.txt b/backend/requirements/dev.txt index cd1db146b02..73aa19fa336 100644 --- a/backend/requirements/dev.txt +++ b/backend/requirements/dev.txt @@ -1177,13 +1177,13 @@ numpy==2.4.1 \ # contourpy # matplotlib # voyageai -onyx-devtools==0.10.6 \ - --hash=sha256:197261d5676e89dc2cf1ec6313d293c43766e9dbf0ef80e4ff7483d6a0ab1374 \ - --hash=sha256:1d8ea1058de6ba3f2f9726972a17c4c1471f8ac460421b021481d0f86ac80819 \ - --hash=sha256:2405164cb2bfd379a40c7c69c320bb0244ef0163154e00ba62e4c673e514102c \ - --hash=sha256:b7a5ebbe08e18b76bca4cf8afa9820049278a735ded9efaeb75a36f062796f00 \ - --hash=sha256:be5ada8204563223b57535659563058fd16bf6c8251dfd9b2519f563cd51cd69 \ - --hash=sha256:ca208e1768f2ee39ef52763caac377ccb6e8ebda35aecfd55856e3c9c9e6e636 +onyx-devtools==0.11.0 \ + --hash=sha256:73c28107648d201ac77c5ddba34808f7ce7f7033ad1d20dec01db6061283b590 \ + --hash=sha256:7af864b1bb61c553aad8e8ca4db085e83bcf2e722d41f6ea64e7a6b9b8b40497 \ + --hash=sha256:802ba607f1bc2e87eafbf37454eafc6d38f6e1a711083550494fa74abb92bee9 \ + --hash=sha256:81821c3b65a288c01feb66d5399df99a7ddf79602cc0b50f0335939499c915ff \ + --hash=sha256:cb374c82eb2984f617e44fd5295bf5a407a88c0df37ab974714056bcd32a0ec2 \ + --hash=sha256:e49721ed1434d3781f322695e579f3427e1f85da466b5f597617f9618debf9d0 openai==2.38.0 \ --hash=sha256:798694c6cf74145541fda94325b6f8f72d8e1fd0262cc137c8d728177a6a4ce3 \ --hash=sha256:ec6661c57b2dcc47414a767e6e3335c7ed3d19c9696999283a3c82e95c756a3c diff --git a/backend/scripts/build_docx_preview.py b/backend/scripts/build_docx_preview.py index 389d944a3d9..f55c03ba0ec 100644 --- a/backend/scripts/build_docx_preview.py +++ b/backend/scripts/build_docx_preview.py @@ -121,7 +121,7 @@ def _paragraph_alignment_score( """Fraction of position-aligned paragraphs whose text AND style both match.""" if not a and not b: return 1.0 - matches = sum(1 for pa, pb in zip(a, b) if pa == pb) + matches = sum(1 for pa, pb in zip(a, b, strict=False) if pa == pb) return matches / max(len(a), len(b)) @@ -129,7 +129,7 @@ def _text_alignment_score(a: list[tuple[str, str]], b: list[tuple[str, str]]) -> """Fraction of position-aligned paragraphs whose text matches (ignoring style).""" if not a and not b: return 1.0 - matches = sum(1 for (_, ta), (_, tb) in zip(a, b) if ta == tb) + matches = sum(1 for (_, ta), (_, tb) in zip(a, b, strict=False) if ta == tb) return matches / max(len(a), len(b)) @@ -175,7 +175,7 @@ def compare( ) shown = 0 for index, (ref_p, cand_p) in enumerate( - zip(reference.paragraphs, candidate.paragraphs) + zip(reference.paragraphs, candidate.paragraphs, strict=False) ): if ref_p == cand_p: continue diff --git a/backend/scripts/orphan_doc_cleanup_script.py b/backend/scripts/orphan_doc_cleanup_script.py index 7f923f68bd3..c3f7656abe8 100644 --- a/backend/scripts/orphan_doc_cleanup_script.py +++ b/backend/scripts/orphan_doc_cleanup_script.py @@ -80,7 +80,9 @@ def main() -> None: # Process documents in parallel using ThreadPoolExecutor with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor: - def process_doc(doc_id: str) -> str | None: + def process_doc( + doc_id: str, vespa_index: VespaDocumentIndex = vespa_index + ) -> str | None: document = get_document(doc_id, db_session) if not document: return None diff --git a/backend/scripts/sources_selection_analysis.py b/backend/scripts/sources_selection_analysis.py index d3615dc13be..4ed59c2ee39 100644 --- a/backend/scripts/sources_selection_analysis.py +++ b/backend/scripts/sources_selection_analysis.py @@ -176,7 +176,9 @@ def _identify_diff(self, content_key: str) -> list[dict]: pos if content_key == "score" else { - "x": k for k, v in new_content.items() if v == data + "x": k # noqa: B035 + for k, v in new_content.items() + if v == data }.get("x", "not_ranked") ), "document_id": self._previous_content[pos]["document_id"], @@ -318,8 +320,8 @@ class SelectionAnalysis: def __init__( self, exectype: str, - analysisfiles: list = [], - queries: list = [], + analysisfiles: list | None = None, + queries: list | None = None, threshold: float = 0.0, web_port: int = 3000, auth_cookie: str = "", @@ -339,6 +341,10 @@ def __init__( wait (int, optional): The waiting time (in seconds) to respect between queries. It is helpful to avoid hitting the Generative AI rate limiting. """ + if queries is None: + queries = [] + if analysisfiles is None: + analysisfiles = [] self._exectype = exectype self._analysisfiles = analysisfiles self._queries = queries diff --git a/backend/shared_configs/contextvars.py b/backend/shared_configs/contextvars.py index 8b900d71605..67783b96f27 100644 --- a/backend/shared_configs/contextvars.py +++ b/backend/shared_configs/contextvars.py @@ -36,6 +36,19 @@ "current_user_id", default=None ) +# IncognitoRecordMode value of the streaming turn's session, None outside +# incognito. A plain string keeps this layer free of onyx imports. +CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR: contextvars.ContextVar[str | None] = ( + contextvars.ContextVar("current_incognito_record_mode", default=None) +) + +# Session id of a content-free turn, and only of a content-free turn: a blob +# saved while this is set is conversation-derived and must die with the +# session, so the file store stamps it on the record at creation. +CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR: contextvars.ContextVar[str | None] = ( + contextvars.ContextVar("current_content_free_session_id", default=None) +) + class UsageCredentialIdentity(NamedTuple): credential_type: UsageCredentialType @@ -71,5 +84,10 @@ def get_current_user_id() -> str | None: return CURRENT_USER_ID_CONTEXTVAR.get() +def get_current_incognito_record_mode() -> str | None: + """The incognito record-mode value of the current turn, None outside one.""" + return CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.get() + + def get_current_usage_credential() -> UsageCredentialIdentity | None: return CURRENT_USAGE_CREDENTIAL_CONTEXTVAR.get() diff --git a/backend/tests/daily/connectors/airtable/test_airtable_basic.py b/backend/tests/daily/connectors/airtable/test_airtable_basic.py index 684b92f30aa..c5976dc21ff 100644 --- a/backend/tests/daily/connectors/airtable/test_airtable_basic.py +++ b/backend/tests/daily/connectors/airtable/test_airtable_basic.py @@ -185,7 +185,7 @@ def compare_documents( f"Number of sections mismatch for document {doc_id}" ) for i, (actual_section, expected_section) in enumerate( - zip(actual.sections, expected.sections) + zip(actual.sections, expected.sections, strict=True) ): assert actual_section.text == expected_section.text, ( f"Section {i} text mismatch for document {doc_id}" diff --git a/backend/tests/daily/connectors/google_drive/test_service_acct.py b/backend/tests/daily/connectors/google_drive/test_service_acct.py index f84a8d16961..65dbb15ede8 100644 --- a/backend/tests/daily/connectors/google_drive/test_service_acct.py +++ b/backend/tests/daily/connectors/google_drive/test_service_acct.py @@ -701,7 +701,7 @@ def test_slim_retrieval_does_not_call_permissions_list( "onyx.connectors.google_drive.connector.execute_paginated_retrieval", wraps=execute_paginated_retrieval, ) as mock_paginated: - for batch in connector.retrieve_all_slim_docs(): + for _batch in connector.retrieve_all_slim_docs(): pass permissions_calls = [ diff --git a/backend/tests/daily/connectors/slack/test_slack_connector.py b/backend/tests/daily/connectors/slack/test_slack_connector.py index 3190fdb4a49..b28591ec263 100644 --- a/backend/tests/daily/connectors/slack/test_slack_connector.py +++ b/backend/tests/daily/connectors/slack/test_slack_connector.py @@ -121,7 +121,7 @@ def test_indexing_channels_that_dont_exist( ValueError, match=r"Channel '.*' not found in workspace.*", ): - load_all_from_connector( + _ = load_all_from_connector( connector=slack_connector, start=0.0, end=time.time(), diff --git a/backend/tests/external_dependency_unit/answer/stream_test_assertions.py b/backend/tests/external_dependency_unit/answer/stream_test_assertions.py index a789a36cc60..2f8c8b1ca95 100644 --- a/backend/tests/external_dependency_unit/answer/stream_test_assertions.py +++ b/backend/tests/external_dependency_unit/answer/stream_test_assertions.py @@ -69,7 +69,7 @@ def _are_search_docs_equal( received.sort(key=lambda x: x.document_id) expected.sort(key=lambda x: x.document_id) - for received_document, expected_document in zip(received, expected): + for received_document, expected_document in zip(received, expected, strict=True): if received_document.document_id != expected_document.document_id: return False if received_document.link != expected_document.link: @@ -147,7 +147,9 @@ def is_image_generation_final_equal( if len(received.images) != len(expected.images): return False - for received_image, expected_image in zip(received.images, expected.images): + for received_image, expected_image in zip( + received.images, expected.images, strict=True + ): if received_image.url != f"/api/chat/file/{received_image.file_id}": return False if received_image.revised_prompt != expected_image.revised_prompt: diff --git a/backend/tests/external_dependency_unit/chat/test_incognito_availability.py b/backend/tests/external_dependency_unit/chat/test_incognito_availability.py new file mode 100644 index 00000000000..240090ad512 --- /dev/null +++ b/backend/tests/external_dependency_unit/chat/test_incognito_availability.py @@ -0,0 +1,104 @@ +"""Guards who may start an incognito chat. + +The availability rule composes the admin security setting (default off) with +group membership under groups-only mode. Runs against real Postgres because +the membership query is a real join, and a mocked session would return +whatever it is told. +""" + +from collections.abc import Generator, Iterator +from contextlib import contextmanager +from unittest.mock import MagicMock, patch + +import pytest +from sqlalchemy.orm import Session + +from onyx.chat.incognito import incognito_allowed_for_user +from onyx.db.models import User, User__UserGroup, UserGroup +from onyx.server.security.models import IncognitoAvailability +from tests.external_dependency_unit.conftest import create_test_user + +GROUP_NAME_PREFIX = "incognito-avail-" + + +@pytest.fixture +def owner(db_session: Session) -> Generator[User, None, None]: + user = create_test_user(db_session, "incognito-avail") + yield user + db_session.rollback() + db_session.query(User__UserGroup).filter( + User__UserGroup.user_id == user.id + ).delete() + db_session.query(UserGroup).filter( + UserGroup.name.like(f"{GROUP_NAME_PREFIX}%") + ).delete(synchronize_session=False) + db_session.delete(user) + db_session.commit() + + +def _make_group(db_session: Session, user: User, incognito_enabled: bool) -> None: + group = UserGroup( + name=f"{GROUP_NAME_PREFIX}{incognito_enabled}", + incognito_enabled=incognito_enabled, + ) + db_session.add(group) + db_session.flush() + db_session.add(User__UserGroup(user_group_id=group.id, user_id=user.id)) + db_session.commit() + + +@contextmanager +def _workspace( + mode: IncognitoAvailability, store_available: bool = True +) -> Iterator[None]: + with ( + patch( + "onyx.chat.incognito.incognito_context_available", + return_value=store_available, + ), + patch( + "onyx.chat.incognito.get_security_settings", + return_value=MagicMock(incognito_availability=mode), + ), + ): + yield + + +@pytest.mark.parametrize( + "mode,group_enabled,allowed", + [ + (IncognitoAvailability.OFF, None, False), + (IncognitoAvailability.EVERYONE, None, True), + (IncognitoAvailability.GROUPS, True, True), + (IncognitoAvailability.GROUPS, False, False), + # No group at all, which is the case a membership join can get wrong. + (IncognitoAvailability.GROUPS, None, False), + ], +) +def test_availability_setting_decides( + db_session: Session, + owner: User, + mode: IncognitoAvailability, + group_enabled: bool | None, + allowed: bool, +) -> None: + if group_enabled is not None: + _make_group(db_session, owner, group_enabled) + + with _workspace(mode): + assert incognito_allowed_for_user(owner, db_session) is allowed + + +def test_unavailable_store_refuses_what_the_setting_allows( + db_session: Session, owner: User +) -> None: + """The setting cannot grant what the deployment cannot hold.""" + with _workspace(IncognitoAvailability.EVERYONE, store_available=False): + assert not incognito_allowed_for_user(owner, db_session) + + +def test_anonymous_user_is_refused(db_session: Session) -> None: + """Anonymous users share an identity and cannot call teardown.""" + anonymous = MagicMock(is_anonymous=True) + with _workspace(IncognitoAvailability.EVERYONE): + assert not incognito_allowed_for_user(anonymous, db_session) diff --git a/backend/tests/external_dependency_unit/chat/test_incognito_persistence.py b/backend/tests/external_dependency_unit/chat/test_incognito_persistence.py new file mode 100644 index 00000000000..571bcd497d4 --- /dev/null +++ b/backend/tests/external_dependency_unit/chat/test_incognito_persistence.py @@ -0,0 +1,266 @@ +"""Guards the incognito persistence seams against real Postgres and Redis. + +Two behaviors the feature rests on: save_chat_turn keeps the assistant row for +tracking but writes no text when content is not persisted, and the ephemeral +store round-trips a turn's messages so the next turn has its context. Run here +rather than as unit tests because both only mean something against the real +stores. +""" + +from collections.abc import Generator +from io import BytesIO +from unittest.mock import patch +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy.orm import Session + +from onyx.chat.incognito import delete_incognito_generated_files +from onyx.chat.incognito_context import ( + append_incognito_message, + load_incognito_context, + teardown_incognito_session, +) +from onyx.chat.models import ChatMessageSimple +from onyx.chat.save_chat import save_chat_turn +from onyx.configs.constants import DocumentSource, FileOrigin, MessageType +from onyx.context.search.models import SearchDoc +from onyx.db.chat import ( + create_chat_session, + get_or_create_root_message, + reserve_message_id, +) +from onyx.db.file_record import ( + get_incognito_file_ids, + get_session_ids_with_incognito_files, +) +from onyx.db.models import ChatMessage, ChatSession, User +from onyx.file_store.file_store import get_default_file_store +from onyx.redis.redis_pool import get_redis_client +from onyx.tools.models import ToolCallInfo +from shared_configs.contextvars import CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR +from tests.external_dependency_unit.conftest import create_test_user + + +@pytest.fixture +def owner(db_session: Session) -> Generator[User, None, None]: + user = create_test_user(db_session, "incognito-persist") + yield user + db_session.rollback() + db_session.query(ChatSession).filter(ChatSession.user_id == user.id).delete() + db_session.delete(user) + db_session.commit() + + +def _new_session(db_session: Session, user_id: UUID) -> ChatSession: + return create_chat_session( + db_session=db_session, + description="incognito", + user_id=user_id, + persona_id=None, + ) + + +def _reserve_assistant(db_session: Session, session_id: UUID) -> ChatMessage: + root = get_or_create_root_message(chat_session_id=session_id, db_session=db_session) + return reserve_message_id( + db_session=db_session, + chat_session_id=session_id, + parent_message=root.id, + ) + + +def _search_doc(document_id: str) -> SearchDoc: + return SearchDoc( + document_id=document_id, + chunk_ind=0, + semantic_identifier="secret doc", + blurb="confidential excerpt", + source_type=DocumentSource.WEB, + boost=0, + hidden=False, + metadata={}, + match_highlights=["confidential"], + ) + + +def test_save_chat_turn_keeps_the_row_but_writes_no_text( + db_session: Session, owner: User +) -> None: + session = _new_session(db_session, owner.id) + assistant = _reserve_assistant(db_session, session.id) + + doc = _search_doc("secret-doc-1") + save_chat_turn( + message_text="the acquisition target is confidential", + reasoning_tokens="secret reasoning", + tool_calls=[ + ToolCallInfo( + parent_tool_call_id=None, + turn_index=0, + tab_index=0, + tool_name="run_search", + tool_call_id="call-1", + tool_id=1, + reasoning_tokens=None, + tool_call_arguments={"query": "the confidential query"}, + tool_call_response="retrieved excerpt text", + search_docs=[doc], + ) + ], + citation_to_doc={1: doc}, + all_search_docs={doc.document_id: doc}, + db_session=db_session, + assistant_message=assistant, + emitted_citations={1}, + persist_content=False, + ) + db_session.commit() + + stored = db_session.get(ChatMessage, assistant.id) + assert stored is not None + # Row survives for tracking, with a real token count, but no text. + assert stored.message == "" + assert stored.reasoning_tokens is None + assert stored.token_count > 0 + # Conversation-derived artifacts stay out too: no tool calls, no search + # docs, no citations for a content-free turn. + assert not stored.tool_calls + assert not stored.search_docs + assert not stored.citations + + +def test_save_chat_turn_persists_text_by_default( + db_session: Session, owner: User +) -> None: + session = _new_session(db_session, owner.id) + assistant = _reserve_assistant(db_session, session.id) + + save_chat_turn( + message_text="an ordinary answer", + reasoning_tokens=None, + tool_calls=[], + citation_to_doc={}, + all_search_docs={}, + db_session=db_session, + assistant_message=assistant, + ) + db_session.commit() + + stored = db_session.get(ChatMessage, assistant.id) + assert stored is not None + assert stored.message == "an ordinary answer" + + +def test_turn_round_trips_through_the_store() -> None: + """A turn appends the user message then the answer. The next turn loads both + in order so the model sees its own context.""" + session_id = uuid4() + try: + append_incognito_message( + session_id, + ChatMessageSimple( + message="what is our runway", + token_count=4, + message_type=MessageType.USER, + ), + ) + append_incognito_message( + session_id, + ChatMessageSimple( + message="eighteen months", + token_count=2, + message_type=MessageType.ASSISTANT, + ), + ) + + history = load_incognito_context(session_id).messages + assert [(m.message_type, m.message) for m in history] == [ + (MessageType.USER, "what is our runway"), + (MessageType.ASSISTANT, "eighteen months"), + ] + finally: + teardown_incognito_session(session_id) + + +def test_teardown_ends_the_session_immediately() -> None: + session_id = uuid4() + append_incognito_message( + session_id, + ChatMessageSimple( + message="secret", token_count=1, message_type=MessageType.USER + ), + ) + assert load_incognito_context(session_id).messages + + teardown_incognito_session(session_id) + + assert load_incognito_context(session_id).messages == [] + + +def test_teardown_clears_buffered_stream_chunks() -> None: + """The stream buffer holds the streamed answer NDJSON, so teardown must + delete it with the context instead of leaving it to the TTL.""" + session_id = uuid4() + client = get_redis_client() + chunk_key = f"chatstream_{session_id}_1:0" + client.set(chunk_key, b"buffered answer text", ex=600) + assert client.get(chunk_key) is not None + + teardown_incognito_session(session_id) + + assert client.get(chunk_key) is None + + +def _content_free_blob(session_id: UUID) -> str: + """Save a blob the way a tool does inside a content-free turn.""" + token = CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR.set(str(session_id)) + try: + return get_default_file_store().save_file( + content=BytesIO(b"generated chart bytes"), + display_name="chart.png", + file_origin=FileOrigin.CHAT_IMAGE_GEN, + file_type="image/png", + ) + finally: + CURRENT_CONTENT_FREE_SESSION_ID_CONTEXTVAR.reset(token) + + +def test_a_blob_is_stamped_when_it_is_saved(db_session: Session) -> None: + """The stamp lands with the record, so no window exists where a blob is + durable but unfindable.""" + session_id = uuid4() + file_id = _content_free_blob(session_id) + + assert get_incognito_file_ids(str(session_id), db_session) == [file_id] + + +def test_teardown_deletes_the_stamped_blobs(db_session: Session) -> None: + session_id = uuid4() + file_id = _content_free_blob(session_id) + file_store = get_default_file_store() + + assert delete_incognito_generated_files(session_id, db_session) + + with pytest.raises(Exception): + file_store.read_file(file_id) + assert get_incognito_file_ids(str(session_id), db_session) == [] + + +def test_a_refused_deletion_keeps_the_stamp(db_session: Session) -> None: + """A store outage must leave the blob findable for the sweep.""" + session_id = uuid4() + file_id = _content_free_blob(session_id) + file_store = get_default_file_store() + + with patch.object( + type(file_store), "delete_file", side_effect=RuntimeError("store blip") + ): + assert not delete_incognito_generated_files(session_id, db_session) + + assert get_incognito_file_ids(str(session_id), db_session) == [file_id] + assert str(session_id) in get_session_ids_with_incognito_files(db_session) + + assert delete_incognito_generated_files(session_id, db_session) + with pytest.raises(Exception): + file_store.read_file(file_id) diff --git a/backend/tests/external_dependency_unit/craft/test_scheduled_task_pre_approvals.py b/backend/tests/external_dependency_unit/craft/test_scheduled_task_pre_approvals.py index 7e934472a39..8a83ea5195b 100644 --- a/backend/tests/external_dependency_unit/craft/test_scheduled_task_pre_approvals.py +++ b/backend/tests/external_dependency_unit/craft/test_scheduled_task_pre_approvals.py @@ -23,13 +23,12 @@ ScheduledTaskStatus, ScheduledTaskTriggerSource, ) -from onyx.db.gated_app import get_or_create_gated_app_id from onyx.db.models import ( BuildSession, ExternalApp, MCPServer, ScheduledTask, - ScheduledTaskPreApprovedApp, + ScheduledTaskPreApprovedTarget, User, ) from onyx.db.scheduled_task import ( @@ -58,11 +57,25 @@ def _make_app(db_session: Session) -> int: return app.id +def _make_mcp_server(db_session: Session, user: User) -> int: + server = MCPServer( + owner=user.email, + name=f"pre_approval_mcp_{uuid4().hex[:8]}", + server_url="https://example.com/mcp", + available_in_craft=True, + is_public=False, + ) + db_session.add(server) + db_session.flush() + return server.id + + def _seed_task( db_session: Session, user: User, *, pre_approved_external_app_ids: list[int] | None = None, + pre_approved_mcp_server_ids: list[int] | None = None, prompt: str = "Summarise yesterday's events", ) -> ScheduledTask: task = create_scheduled_task( @@ -74,6 +87,7 @@ def _seed_task( editor_mode="advanced", status=ScheduledTaskStatus.ACTIVE, pre_approved_external_app_ids=pre_approved_external_app_ids, + pre_approved_mcp_server_ids=pre_approved_mcp_server_ids, ) db_session.commit() db_session.refresh(task) @@ -93,7 +107,13 @@ def test_grants_returned_for_running_run( user = make_user(db_session) bs = build_session_with_user(user=user) app_a, app_b = _make_app(db_session), _make_app(db_session) - task = _seed_task(db_session, user, pre_approved_external_app_ids=[app_a, app_b]) + server_id = _make_mcp_server(db_session, user) + task = _seed_task( + db_session, + user, + pre_approved_external_app_ids=[app_a, app_b], + pre_approved_mcp_server_ids=[server_id], + ) run = insert_run( db_session=db_session, task_id=task.id, @@ -115,6 +135,7 @@ def test_grants_returned_for_running_run( assert granted == { (GatedAppKind.EXTERNAL_APP, app_a), (GatedAppKind.EXTERNAL_APP, app_b), + (GatedAppKind.MCP_SERVER, server_id), } @@ -306,33 +327,18 @@ def test_mcp_grants_survive_external_app_replacement( tenant_context: None, # noqa: ARG001 build_session_with_user: Callable[..., BuildSession], ) -> None: - """``set_pre_approved_apps`` replaces only the given kind's grants: an MCP-server - grant (seeded directly — no API writes these yet) survives a wholesale - external-app replacement, stays out of ``pre_approved_external_app_ids``, and reaches - the gate through ``get_live_scheduled_run_grants`` as its (kind, id) target. - """ + """Each target kind has independent replacement semantics and reaches the gate.""" user = make_user(db_session) bs = build_session_with_user(user=user) app_a, app_b = _make_app(db_session), _make_app(db_session) - task = _seed_task(db_session, user, pre_approved_external_app_ids=[app_a]) - - server = MCPServer( - owner=user.email, - name=f"pre_approval_mcp_{uuid4().hex[:8]}", - server_url="https://example.com/mcp", - is_public=False, - ) - db_session.add(server) - db_session.flush() - mcp_gated_app_id = get_or_create_gated_app_id( - db_session, GatedAppKind.MCP_SERVER, server.id - ) - db_session.add( - ScheduledTaskPreApprovedApp( - scheduled_task_id=task.id, gated_app_id=mcp_gated_app_id - ) + server_a = _make_mcp_server(db_session, user) + server_b = _make_mcp_server(db_session, user) + task = _seed_task( + db_session, + user, + pre_approved_external_app_ids=[app_a], + pre_approved_mcp_server_ids=[server_a], ) - db_session.commit() updated = update_scheduled_task( db_session=db_session, @@ -343,10 +349,18 @@ def test_mcp_grants_survive_external_app_replacement( db_session.commit() assert updated.pre_approved_external_app_ids == [app_b] # MCP grant excluded - assert {g.gated_app.target_key for g in updated.pre_approved_apps} == { - (GatedAppKind.EXTERNAL_APP, app_b), - (GatedAppKind.MCP_SERVER, server.id), - } + assert updated.pre_approved_mcp_server_ids == [server_a] + + updated = update_scheduled_task( + db_session=db_session, + task_id=task.id, + user_id=user.id, + pre_approved_mcp_server_ids=[server_b], + ) + db_session.commit() + + assert updated.pre_approved_external_app_ids == [app_b] + assert updated.pre_approved_mcp_server_ids == [server_b] run = insert_run( db_session=db_session, @@ -365,23 +379,31 @@ def test_mcp_grants_survive_external_app_replacement( assert grants is not None assert grants[1] == { (GatedAppKind.EXTERNAL_APP, app_b), - (GatedAppKind.MCP_SERVER, server.id), + (GatedAppKind.MCP_SERVER, server_b), } -def test_create_persists_grants( +def test_create_stores_each_grant_once( db_session: Session, tenant_context: None, # noqa: ARG001 ) -> None: user = make_user(db_session) app_a, app_b = _make_app(db_session), _make_app(db_session) - assert app_a < app_b # ids autoincrement, so the higher id is created last - # Insertion order is preserved (not sorted): pass the higher id first. - task = _seed_task(db_session, user, pre_approved_external_app_ids=[app_b, app_a]) - assert task.pre_approved_external_app_ids == [app_b, app_a] + server_id = _make_mcp_server(db_session, user) + task = _seed_task( + db_session, + user, + pre_approved_external_app_ids=[app_b, app_a, app_b], + pre_approved_mcp_server_ids=[server_id, server_id], + ) + assert set(task.pre_approved_external_app_ids) == {app_a, app_b} + assert len(task.pre_approved_external_app_ids) == 2 + assert set(task.pre_approved_mcp_server_ids) == {server_id} + assert len(task.pre_approved_mcp_server_ids) == 1 bare = _seed_task(db_session, user) assert bare.pre_approved_external_app_ids == [] + assert bare.pre_approved_mcp_server_ids == [] # --------------------------------------------------------------------------- @@ -405,8 +427,8 @@ def test_deleting_app_drops_grants( remaining = ( db_session.execute( - select(ScheduledTaskPreApprovedApp).where( - ScheduledTaskPreApprovedApp.scheduled_task_id == task.id + select(ScheduledTaskPreApprovedTarget).where( + ScheduledTaskPreApprovedTarget.scheduled_task_id == task.id ) ) .scalars() @@ -451,14 +473,12 @@ def test_deleting_app_nulls_action_approval_fk( # --------------------------------------------------------------------------- -def test_validated_app_ids_rejects_unknown_and_dedupes( +def test_validate_app_ids_accepts_known_duplicates_and_reports_unknown_ids_once( db_session: Session, tenant_context: None, # noqa: ARG001 monkeypatch: pytest.MonkeyPatch, ) -> None: - """Dedupe is order-preserving; any id outside the tenant's apps raises - INVALID_INPUT. Apps are stubbed — only the validation logic is under - test, not ``get_external_apps``'s SQL.""" + """Duplicates are valid, but unknown ids raise INVALID_INPUT.""" class _App: def __init__(self, app_id: int) -> None: @@ -470,9 +490,10 @@ def __init__(self, app_id: int) -> None: lambda _db: [_App(7), _App(9)], ) - assert scheduled_tasks_api._validated_app_ids(db_session, []) == [] - assert scheduled_tasks_api._validated_app_ids(db_session, [9, 7, 9]) == [9, 7] + scheduled_tasks_api._validate_app_ids(db_session, []) + scheduled_tasks_api._validate_app_ids(db_session, [9, 7, 9]) with pytest.raises(OnyxError) as exc_info: - scheduled_tasks_api._validated_app_ids(db_session, [7, 123]) + scheduled_tasks_api._validate_app_ids(db_session, [456, 7, 123, 456]) assert exc_info.value.error_code == OnyxErrorCode.INVALID_INPUT + assert exc_info.value.detail == "Unknown external app id(s): [123, 456]" diff --git a/backend/tests/external_dependency_unit/db/test_incognito_history_exclusion.py b/backend/tests/external_dependency_unit/db/test_incognito_history_exclusion.py new file mode 100644 index 00000000000..9b06ac8b9f3 --- /dev/null +++ b/backend/tests/external_dependency_unit/db/test_incognito_history_exclusion.py @@ -0,0 +1,314 @@ +"""Guards that an incognito session stays out of every surface its owner sees. + +Covers the three that list a user's own sessions: history, search, and the +chat list a project carries. Exercises the real WHERE clauses against Postgres, +since a mocked session would happily return rows the SQL would have filtered. +""" + +from collections.abc import Generator +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import pytest +from sqlalchemy.orm import Session + +from ee.onyx.db.query_history import ( + fetch_chat_sessions_eagerly_by_time, + fetch_persisting_chat_session_by_id, + get_page_of_chat_sessions, +) +from onyx.configs.constants import MessageType +from onyx.db.chat import ( + create_chat_session, + create_new_chat_message, + get_chat_sessions_by_user, + get_or_create_root_message, +) +from onyx.db.chat_search import search_chat_sessions +from onyx.db.enums import IncognitoRecordMode +from onyx.db.models import ChatSession, User, UserProject +from onyx.server.features.projects.models import UserProjectSnapshot +from tests.external_dependency_unit.conftest import create_test_user + + +@pytest.fixture +def owner(db_session: Session) -> Generator[User, None, None]: + """A user whose rows are deleted afterwards rather than rolled back. + + ``create_chat_session`` commits, so a rollback cannot reach these rows, and + the filter under test hides them from the UI that would otherwise clean them + up. Left alone they accumulate as sessions nobody can see or remove. + """ + user = create_test_user(db_session, "incognito-history") + yield user + + db_session.rollback() + # Sessions before projects: chat_session.project_id references user_project. + db_session.query(ChatSession).filter(ChatSession.user_id == user.id).delete() + db_session.query(UserProject).filter(UserProject.user_id == user.id).delete() + db_session.delete(user) + db_session.commit() + + +def _make_session( + db_session: Session, + user_id: UUID, + description: str, + mode: IncognitoRecordMode | None, + project_id: int | None = None, +) -> ChatSession: + chat_session = create_chat_session( + db_session=db_session, + description=description, + user_id=user_id, + persona_id=None, + project_id=project_id, + ) + chat_session.incognito_record_mode = mode + # Commit rather than flush: the next create_chat_session would otherwise be + # what commits this assignment, leaving the last one written only in memory. + db_session.commit() + return chat_session + + +def _sessions_by_user( + db_session: Session, + user_id: UUID, + exclude_incognito: bool = False, + exclude_content_free: bool = False, +) -> set[UUID]: + return { + session.id + for session in get_chat_sessions_by_user( + user_id=user_id, + deleted=None, + db_session=db_session, + include_failed_chats=True, + exclude_incognito=exclude_incognito, + exclude_content_free=exclude_content_free, + ) + } + + +def _history_ids(db_session: Session, user_id: UUID) -> set[UUID]: + """The owner's own history, which is the call site that opts out.""" + return _sessions_by_user(db_session, user_id, exclude_incognito=True) + + +def test_history_excludes_incognito_by_default( + db_session: Session, owner: User +) -> None: + ordinary = _make_session(db_session, owner.id, "ordinary chat", None) + incognito = _make_session( + db_session, owner.id, "incognito chat", IncognitoRecordMode.FULL_HISTORY + ) + + returned_ids = _history_ids(db_session, owner.id) + assert ordinary.id in returned_ids + assert incognito.id not in returned_ids + + +def test_workspace_surface_keeps_full_history_and_drops_content_free( + db_session: Session, owner: User +) -> None: + """The exclusion the query-history endpoint uses, which is the weaker one: + a full-history session is recorded for the workspace and only its owner is + kept from seeing it.""" + full_history = _make_session( + db_session, owner.id, "full history chat", IncognitoRecordMode.FULL_HISTORY + ) + usage_only = _make_session( + db_session, owner.id, "usage only chat", IncognitoRecordMode.USAGE_ONLY + ) + + returned_ids = _sessions_by_user(db_session, owner.id, exclude_content_free=True) + assert full_history.id in returned_ids + assert usage_only.id not in returned_ids + + +def test_admin_query_history_page_hides_content_free_sessions( + db_session: Session, owner: User +) -> None: + """Content-free sessions must not appear as blank rows in the table.""" + ordinary = _make_session(db_session, owner.id, "ordinary chat", None) + full_history = _make_session( + db_session, owner.id, "full history chat", IncognitoRecordMode.FULL_HISTORY + ) + usage_only = _make_session( + db_session, owner.id, "usage only chat", IncognitoRecordMode.USAGE_ONLY + ) + + page_ids = { + session.id + for session in get_page_of_chat_sessions( + start_time=None, + end_time=None, + db_session=db_session, + page_num=0, + page_size=1000, + ) + } + assert ordinary.id in page_ids + assert full_history.id in page_ids + assert usage_only.id not in page_ids + + +def test_query_history_export_hides_content_free_sessions( + db_session: Session, owner: User +) -> None: + ordinary = _make_session(db_session, owner.id, "ordinary chat", None) + usage_only = _make_session( + db_session, owner.id, "usage only chat", IncognitoRecordMode.USAGE_ONLY + ) + + window = timedelta(minutes=5) + now = datetime.now(timezone.utc) + export_ids = { + session.id + for session in fetch_chat_sessions_eagerly_by_time( + start=now - window, + end=now + window, + db_session=db_session, + limit=None, + ) + } + assert ordinary.id in export_ids + assert usage_only.id not in export_ids + + +def test_query_history_detail_hides_content_free_sessions( + db_session: Session, owner: User +) -> None: + """Knowing the id must not be a way around the list filter: the detail + route is the one surface that takes an id straight from the caller.""" + ordinary = _make_session(db_session, owner.id, "ordinary chat", None) + full_history = _make_session( + db_session, owner.id, "full history chat", IncognitoRecordMode.FULL_HISTORY + ) + usage_only = _make_session( + db_session, owner.id, "usage only chat", IncognitoRecordMode.USAGE_ONLY + ) + + for visible in (ordinary, full_history): + assert fetch_persisting_chat_session_by_id(visible.id, db_session).id == ( + visible.id + ) + + with pytest.raises(ValueError): + fetch_persisting_chat_session_by_id(usage_only.id, db_session) + + +def test_every_mode_is_excluded_from_history(db_session: Session, owner: User) -> None: + """Every mode must stay out of the owner's history. + + Iterates the enum so a newly added mode is covered without editing this + test. + """ + sessions = { + mode: _make_session(db_session, owner.id, f"chat {mode.value}", mode) + for mode in IncognitoRecordMode + } + + returned_ids = _history_ids(db_session, owner.id) + for mode, chat_session in sessions.items(): + assert chat_session.id not in returned_ids, f"{mode.value} leaked" + + +def test_search_excludes_incognito_without_a_query( + db_session: Session, owner: User +) -> None: + ordinary = _make_session(db_session, owner.id, "quarterly revenue notes", None) + incognito = _make_session( + db_session, + owner.id, + "quarterly revenue secrets", + IncognitoRecordMode.FULL_HISTORY, + ) + + sessions, _ = search_chat_sessions(user_id=owner.id, db_session=db_session) + returned_ids = {session.id for session in sessions} + assert ordinary.id in returned_ids + assert incognito.id not in returned_ids + + +def test_search_excludes_incognito_matching_the_query( + db_session: Session, owner: User +) -> None: + """The description arm of the union must not surface an incognito session.""" + ordinary = _make_session(db_session, owner.id, "penguin migration notes", None) + incognito = _make_session( + db_session, + owner.id, + "penguin migration secrets", + IncognitoRecordMode.FULL_HISTORY, + ) + + sessions, _ = search_chat_sessions( + user_id=owner.id, db_session=db_session, query="penguin" + ) + returned_ids = {session.id for session in sessions} + assert ordinary.id in returned_ids + assert incognito.id not in returned_ids + + +def test_project_does_not_list_its_incognito_sessions( + db_session: Session, owner: User +) -> None: + """A project lists sessions by title, which is the thing incognito hides.""" + project = UserProject(name="incognito-project", user_id=owner.id) + db_session.add(project) + db_session.commit() + + ordinary = _make_session( + db_session, owner.id, "ordinary chat", None, project_id=project.id + ) + incognito = _make_session( + db_session, + owner.id, + "incognito chat", + IncognitoRecordMode.FULL_HISTORY, + project_id=project.id, + ) + + db_session.expire(project) + listed = UserProjectSnapshot.from_model(project).chat_sessions + listed_ids = {session.id for session in listed} + assert ordinary.id in listed_ids + assert incognito.id not in listed_ids + assert all(session.name != "incognito chat" for session in listed) + + +def test_search_excludes_incognito_matching_only_in_a_message( + db_session: Session, owner: User +) -> None: + """The message-body arm of the union is filtered too. + + Both arms carry base_conditions, so a hit on message text must not surface + a session whose description never matched. + """ + incognito = _make_session( + db_session, owner.id, "untitled", IncognitoRecordMode.FULL_HISTORY + ) + ordinary = _make_session(db_session, owner.id, "untitled", None) + + for chat_session in (incognito, ordinary): + create_new_chat_message( + chat_session_id=chat_session.id, + parent_message=get_or_create_root_message( + chat_session_id=chat_session.id, db_session=db_session + ), + message="the aardvark budget is confidential", + token_count=7, + message_type=MessageType.USER, + db_session=db_session, + ) + db_session.commit() + + sessions, _ = search_chat_sessions( + user_id=owner.id, db_session=db_session, query="aardvark" + ) + returned_ids = {session.id for session in sessions} + # The ordinary control proves the query actually matches message bodies, + # so the incognito assertion is not passing for want of any hit at all. + assert ordinary.id in returned_ids + assert incognito.id not in returned_ids diff --git a/backend/tests/external_dependency_unit/document_index/conftest.py b/backend/tests/external_dependency_unit/document_index/conftest.py index 92447797010..e6183a51864 100644 --- a/backend/tests/external_dependency_unit/document_index/conftest.py +++ b/backend/tests/external_dependency_unit/document_index/conftest.py @@ -98,7 +98,7 @@ def make_indexing_metadata( old_chunk_cnt=old, new_chunk_cnt=new, ) - for doc_id, old, new in zip(doc_ids, old_counts, new_counts) + for doc_id, old, new in zip(doc_ids, old_counts, new_counts, strict=True) } ) diff --git a/backend/tests/external_dependency_unit/document_index/test_document_index.py b/backend/tests/external_dependency_unit/document_index/test_document_index.py index a263c0b897f..dde9002d119 100644 --- a/backend/tests/external_dependency_unit/document_index/test_document_index.py +++ b/backend/tests/external_dependency_unit/document_index/test_document_index.py @@ -271,7 +271,7 @@ def test_index_accepts_generator( doc_id = f"test_gen_{uuid.uuid4().hex[:8]}" metadata = make_indexing_metadata([doc_id], old_counts=[0], new_counts=[3]) - def chunk_gen() -> Iterator[DocMetadataAwareIndexChunk]: + def chunk_gen(doc_id: str = doc_id) -> Iterator[DocMetadataAwareIndexChunk]: for i in range(3): yield make_chunk(doc_id, chunk_id=i) diff --git a/backend/tests/external_dependency_unit/file_store/test_postgres_file_store_non_mocked.py b/backend/tests/external_dependency_unit/file_store/test_postgres_file_store_non_mocked.py index 524d5687185..4c7937aec65 100644 --- a/backend/tests/external_dependency_unit/file_store/test_postgres_file_store_non_mocked.py +++ b/backend/tests/external_dependency_unit/file_store/test_postgres_file_store_non_mocked.py @@ -11,6 +11,7 @@ from io import BytesIO, StringIO from typing import Any, Dict, List +import psycopg2 import pytest from sqlalchemy.orm import Session @@ -314,7 +315,7 @@ def test_overwrite_file(self, pg_file_store: PostgresBackedFileStore) -> None: assert new_oid != old_oid raw_conn = _get_raw_connection(session) - with pytest.raises(Exception): + with pytest.raises(psycopg2.Error): _read_large_object(raw_conn, old_oid) # ── change_file_id ───────────────────────────────────────────── diff --git a/backend/tests/external_dependency_unit/indexing/test_persistent_indexing.py b/backend/tests/external_dependency_unit/indexing/test_persistent_indexing.py index e0477949d79..911f3d28ccd 100644 --- a/backend/tests/external_dependency_unit/indexing/test_persistent_indexing.py +++ b/backend/tests/external_dependency_unit/indexing/test_persistent_indexing.py @@ -372,7 +372,7 @@ def test_threshold_default_aborts_attempt( try: mock_app = MagicMock() - with pytest.raises(Exception): + with pytest.raises(RuntimeError, match="too many errors"): run_docfetching_entrypoint( app=mock_app, index_attempt_id=attempt_id, diff --git a/backend/tests/external_dependency_unit/indexing/test_port_reembed.py b/backend/tests/external_dependency_unit/indexing/test_port_reembed.py index 5caa18f1a4d..5a538ef21c0 100644 --- a/backend/tests/external_dependency_unit/indexing/test_port_reembed.py +++ b/backend/tests/external_dependency_unit/indexing/test_port_reembed.py @@ -571,7 +571,7 @@ def embed_chunks(self, chunks: list[DocAwareChunk]) -> list[IndexChunk]: ] # ...and each chunk carries ITS OWN content's vector despite the reversed output # (a positional zip would give c0 the vector of c2, etc.). - for result, stored in zip(results, [c0, c1, c2]): + for result, stored in zip(results, [c0, c1, c2], strict=True): assert result.content_vector == _vec(stored.content) diff --git a/backend/tests/external_dependency_unit/llm/test_llm_provider_api_base.py b/backend/tests/external_dependency_unit/llm/test_llm_provider_api_base.py index 2130569ba6a..b3dde00d56b 100644 --- a/backend/tests/external_dependency_unit/llm/test_llm_provider_api_base.py +++ b/backend/tests/external_dependency_unit/llm/test_llm_provider_api_base.py @@ -610,7 +610,7 @@ def capture_test_llm(llm: LLM) -> str: # Check inside the database and check that custom_config is the same as the original db_provider = fetch_existing_llm_provider(name=name, db_session=db_session) if not db_provider: - assert False, "Provider not found in the database" + raise AssertionError("Provider not found in the database") assert db_provider.custom_config == custom_config, ( f"Expected custom_config {custom_config}, but got {db_provider.custom_config}" diff --git a/backend/tests/external_dependency_unit/mock_llm.py b/backend/tests/external_dependency_unit/mock_llm.py index 5aa69352bb5..d68d648ecb4 100644 --- a/backend/tests/external_dependency_unit/mock_llm.py +++ b/backend/tests/external_dependency_unit/mock_llm.py @@ -326,7 +326,7 @@ def stream( if not self.stream_controller: return - for idx, item in enumerate(self.stream_controller): + for _idx, item in enumerate(self.stream_controller): yield ModelResponseStream( id="chatcmp-123", created="1", diff --git a/backend/tests/external_dependency_unit/opensearch/test_opensearch_client.py b/backend/tests/external_dependency_unit/opensearch/test_opensearch_client.py index f07082d51fa..0317010e6bc 100644 --- a/backend/tests/external_dependency_unit/opensearch/test_opensearch_client.py +++ b/backend/tests/external_dependency_unit/opensearch/test_opensearch_client.py @@ -59,6 +59,14 @@ ) from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA +_PUBLIC_DOCUMENT_ACCESS = DocumentAccess.build( + user_emails=[], + user_groups=[], + external_user_emails=[], + external_user_group_ids=[], + is_public=True, +) + def _patch_global_tenant_state(monkeypatch: pytest.MonkeyPatch, state: bool) -> None: """Patches MULTI_TENANT wherever necessary for this test file. @@ -138,13 +146,7 @@ def _create_test_document_chunk( title: str | None = None, title_vector: list[float] | None = None, hidden: bool = False, - document_access: DocumentAccess = DocumentAccess.build( - user_emails=[], - user_groups=[], - external_user_emails=[], - external_user_group_ids=[], - is_public=True, - ), + document_access: DocumentAccess = _PUBLIC_DOCUMENT_ACCESS, source_type: DocumentSource = DocumentSource.FILE, last_updated: datetime | None = None, created_at: datetime | None = None, @@ -1257,7 +1259,7 @@ def test_bulk_update_documents( # Postcondition. # Retrieve each document and verify updates were applied. - for doc, doc_chunk_id in zip(docs, doc_chunk_ids): + for doc, doc_chunk_id in zip(docs, doc_chunk_ids, strict=True): updated_doc = test_client.get_document(document_chunk_id=doc_chunk_id) assert updated_doc.hidden is True assert updated_doc.global_boost == 7 diff --git a/backend/tests/external_dependency_unit/permission_sync/test_doc_permission_sync_attempt.py b/backend/tests/external_dependency_unit/permission_sync/test_doc_permission_sync_attempt.py index 31e91a4e411..5a0982c7697 100644 --- a/backend/tests/external_dependency_unit/permission_sync/test_doc_permission_sync_attempt.py +++ b/backend/tests/external_dependency_unit/permission_sync/test_doc_permission_sync_attempt.py @@ -197,7 +197,7 @@ def test_get_recent_doc_permission_sync_attempts_for_cc_pair( # Create multiple attempts attempt_ids = [] - for i in range(5): + for _ in range(5): attempt_id = create_doc_permission_sync_attempt(cc_pair.id, db_session) attempt_ids.append(attempt_id) diff --git a/backend/tests/external_dependency_unit/permission_sync/test_external_group_permission_sync_attempt.py b/backend/tests/external_dependency_unit/permission_sync/test_external_group_permission_sync_attempt.py index e7b2b513d7d..128abbd2809 100644 --- a/backend/tests/external_dependency_unit/permission_sync/test_external_group_permission_sync_attempt.py +++ b/backend/tests/external_dependency_unit/permission_sync/test_external_group_permission_sync_attempt.py @@ -246,7 +246,7 @@ def test_get_recent_external_group_sync_attempts_for_cc_pair( # Create multiple attempts for the cc_pair attempt_ids = [] - for i in range(5): + for _ in range(5): attempt_id = create_external_group_sync_attempt(cc_pair.id, db_session) attempt_ids.append(attempt_id) @@ -293,7 +293,7 @@ def test_get_recent_global_external_group_sync_attempts( # Create multiple global attempts global_attempt_ids = [] - for i in range(3): + for _ in range(3): attempt_id = create_external_group_sync_attempt(None, db_session) # Global global_attempt_ids.append(attempt_id) diff --git a/backend/tests/external_dependency_unit/redis/test_incognito_context.py b/backend/tests/external_dependency_unit/redis/test_incognito_context.py new file mode 100644 index 00000000000..42bffebf51e --- /dev/null +++ b/backend/tests/external_dependency_unit/redis/test_incognito_context.py @@ -0,0 +1,230 @@ +"""Guards the incognito context store's Redis contract. + +Round trip, the compare-and-set that guards against concurrent turns, the +sliding TTL, teardown, corruption degrading to an ended session, image +stripping, and the storage caps, all against a real Redis. Each test runs +under a unique tenant so runs cannot collide, mirroring test_tenant_redis.py. +""" + +import time +from collections.abc import Generator +from unittest.mock import patch +from uuid import UUID, uuid4 + +import pytest + +from onyx.cache.interface import CacheBackendType +from onyx.chat.incognito_context import ( + INCOGNITO_CONTEXT_TTL_SECONDS, + IncognitoContext, + _context_key, + incognito_context_available, + load_incognito_context, + save_incognito_context, + teardown_incognito_session, +) +from onyx.chat.models import ChatLoadedFile, ChatMessageSimple, ToolCallSimple +from onyx.configs.constants import MessageType +from onyx.file_store.models import ChatFileType +from onyx.redis.redis_pool import get_raw_redis_client, get_redis_client +from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR + + +@pytest.fixture(autouse=True) +def isolated_tenant() -> Generator[str, None, None]: + tenant = f"tenant_test_{uuid4().hex[:12]}" + token = CURRENT_TENANT_ID_CONTEXTVAR.set(tenant) + yield tenant + CURRENT_TENANT_ID_CONTEXTVAR.reset(token) + raw = get_raw_redis_client() + keys = list(raw.scan_iter(match=f"{tenant}:*")) + if keys: + raw.delete(*keys) + + +def _message( + text: str, message_type: MessageType = MessageType.USER +) -> ChatMessageSimple: + return ChatMessageSimple( + message=text, token_count=len(text), message_type=message_type + ) + + +def _save( + chat_session_id: UUID, messages: list[ChatMessageSimple], version: int = 0 +) -> bool: + return save_incognito_context( + chat_session_id, IncognitoContext(version=version, messages=messages) + ) + + +def test_missing_key_loads_empty_version_zero() -> None: + context = load_incognito_context(uuid4()) + assert context.messages == [] + assert context.version == 0 + + +def test_stale_version_save_is_discarded() -> None: + """A concurrent turn that loaded the same version must not roll the + winner's write back.""" + session_id = uuid4() + assert _save(session_id, [_message("turn one")], version=0) + + # A racing writer that also loaded version 0 loses. + assert not _save(session_id, [_message("stale rollback")], version=0) + + loaded = load_incognito_context(session_id) + assert loaded.version == 1 + assert loaded.messages[0].message == "turn one" + + +def test_sequential_turns_chain_versions() -> None: + session_id = uuid4() + assert _save(session_id, [_message("one")], version=0) + + first = load_incognito_context(session_id) + assert _save(session_id, first.messages + [_message("two")], first.version) + + second = load_incognito_context(session_id) + assert second.version == 2 + assert [m.message for m in second.messages] == ["one", "two"] + + +def test_corrupt_value_degrades_and_is_overwritable() -> None: + session_id = uuid4() + get_redis_client().set(_context_key(session_id), b"not json at all") + + context = load_incognito_context(session_id) + assert context.messages == [] + assert context.version == 0 + + # The load/save pair recovers: expecting version 0 overwrites the garbage. + assert _save(session_id, [_message("fresh start")], version=0) + assert load_incognito_context(session_id).messages[0].message == "fresh start" + + +def test_ttl_is_set_and_slides_on_save() -> None: + session_id = uuid4() + client = get_redis_client() + + assert _save(session_id, [_message("first")]) + ttl_after_first = client.ttl(_context_key(session_id)) + assert 0 < ttl_after_first <= INCOGNITO_CONTEXT_TTL_SECONDS + + time.sleep(2) + first = load_incognito_context(session_id) + assert _save(session_id, first.messages + [_message("second")], first.version) + ttl_after_second = client.ttl(_context_key(session_id)) + # A non-sliding TTL would have decayed by the sleep. A fresh save restarts it. + assert ttl_after_second > INCOGNITO_CONTEXT_TTL_SECONDS - 2 + + +def test_teardown_ends_the_context_and_fences_writers() -> None: + session_id = uuid4() + assert _save(session_id, [_message("secret plans")]) + context = load_incognito_context(session_id) + assert context.messages + + teardown_incognito_session(session_id) + + # Loads empty, and the tombstone refuses any save from an in-flight turn. + assert load_incognito_context(session_id).messages == [] + assert not _save(session_id, [_message("resurrected")]) + assert load_incognito_context(session_id).messages == [] + + +def test_images_are_stripped_before_storage() -> None: + """File bytes do not round-trip JSON, so save must drop them rather than + fail the turn or store binary content.""" + session_id = uuid4() + image = ChatLoadedFile( + file_id="f1", + content=b"\x89PNG\r\n", + file_type=ChatFileType.IMAGE, + filename="chart.png", + content_text=None, + token_count=0, + ) + message = ChatMessageSimple( + message="see attached", + token_count=100, + message_type=MessageType.USER, + image_files=[image], + image_token_count=85, + ) + + assert _save(session_id, [message]) + (loaded,) = load_incognito_context(session_id).messages + + assert loaded.image_files is None + assert loaded.image_token_count == 0 + assert loaded.message == "see attached" + + +def test_tool_calls_round_trip() -> None: + """Assistant tool calls and tool responses are part of history and must + survive storage intact.""" + session_id = uuid4() + call = ChatMessageSimple( + message="", + token_count=12, + message_type=MessageType.ASSISTANT, + tool_calls=[ + ToolCallSimple( + tool_call_id="call_1", + tool_name="run_search", + tool_arguments={"query": "churn", "limit": 5, "nested": {"a": [1]}}, + token_count=12, + ) + ], + ) + response = ChatMessageSimple( + message="3 documents found", + token_count=4, + message_type=MessageType.TOOL_CALL_RESPONSE, + tool_call_id="call_1", + ) + + assert _save(session_id, [call, response]) + loaded = load_incognito_context(session_id).messages + + assert loaded == [call, response] + + +def test_message_count_cap_keeps_the_newest() -> None: + session_id = uuid4() + history = [_message(f"m{i}") for i in range(205)] + + assert _save(session_id, history) + loaded = load_incognito_context(session_id).messages + + assert len(loaded) == 200 + assert loaded[0].message == "m5" + assert loaded[-1].message == "m204" + + +def test_byte_cap_drops_oldest_but_keeps_an_oversized_singleton() -> None: + session_id = uuid4() + big = "x" * 600_000 + oversized = "y" * 1_200_000 + + assert _save(session_id, [_message(big), _message(big + "newer")]) + loaded = load_incognito_context(session_id).messages + assert len(loaded) == 1 + assert loaded[0].message.endswith("newer") + + # One message alone over the cap is stored anyway: an empty save would + # read as session-ended on the next turn. + singleton_session = uuid4() + assert _save(singleton_session, [_message(oversized)]) + assert len(load_incognito_context(singleton_session).messages) == 1 + + +def test_availability_follows_the_cache_backend() -> None: + """USAGE_ONLY content must never reach Postgres, so the Postgres cache + backend (Lite) means the feature is absent.""" + with patch("onyx.chat.incognito_context.app_configs") as mock_configs: + mock_configs.CACHE_BACKEND = CacheBackendType.REDIS + assert incognito_context_available() + mock_configs.CACHE_BACKEND = CacheBackendType.POSTGRES + assert not incognito_context_available() diff --git a/backend/tests/external_dependency_unit/server/security/test_security_settings_store.py b/backend/tests/external_dependency_unit/server/security/test_security_settings_store.py index 0af7de7780d..a5627c4feef 100644 --- a/backend/tests/external_dependency_unit/server/security/test_security_settings_store.py +++ b/backend/tests/external_dependency_unit/server/security/test_security_settings_store.py @@ -13,7 +13,10 @@ from onyx.db.engine.sql_engine import get_session_with_current_tenant from onyx.db.models import SecuritySettings as SecuritySettingsRow from onyx.server.security import store as security_store -from onyx.server.security.models import SecuritySettingsOverrides +from onyx.server.security.models import ( + IncognitoAvailability, + SecuritySettingsOverrides, +) from onyx.server.security.store import ( _build_env_defaults, _install_cache_for_test, @@ -73,6 +76,27 @@ def test_partial_overrides_only_overrides_specified_fields() -> None: assert effective.password_require_uppercase == env.password_require_uppercase +def test_every_override_field_has_a_column_or_kv_backing() -> None: + """upsert_overrides only writes fields with a matching row column, so an + override without one echoes from PUT but vanishes on the next read.""" + kv_backed = {"password_auth_enabled"} + missing = [ + name + for name in SecuritySettingsOverrides.model_fields + if name not in kv_backed and not hasattr(SecuritySettingsRow, name) + ] + assert not missing + + +def test_incognito_availability_round_trips_through_the_store() -> None: + apply_patch( + SecuritySettingsOverrides(incognito_availability=IncognitoAvailability.GROUPS), + present_keys={"incognito_availability"}, + ) + effective = get_security_settings() + assert effective.incognito_availability is IncognitoAvailability.GROUPS + + def test_cache_hits_avoid_db_reads() -> None: """Repeated loader calls within the TTL must hit the DB once.""" get_security_settings() # warm cache diff --git a/backend/tests/integration/tests/craft/test_scheduled_tasks_api.py b/backend/tests/integration/tests/craft/test_scheduled_tasks_api.py index 5bcdbeeb454..21f6c6d77f5 100644 --- a/backend/tests/integration/tests/craft/test_scheduled_tasks_api.py +++ b/backend/tests/integration/tests/craft/test_scheduled_tasks_api.py @@ -17,6 +17,7 @@ ScheduledTaskStatus, ScheduledTaskTriggerSource, ) +from onyx.db.mcp import create_mcp_server__no_commit, update_mcp_server__no_commit from onyx.db.models import ScheduledTask, ScheduledTaskRun from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA_STANDARD_VALUE from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR @@ -51,6 +52,7 @@ def _create_task( editor_payload: dict[str, Any] | None = None, status: ScheduledTaskStatus = ScheduledTaskStatus.ACTIVE, run_immediately: bool = False, + pre_approved_mcp_server_ids: list[int] | None = None, ) -> httpx.Response: body: dict[str, Any] = { "name": name or f"task-{uuid4().hex[:8]}", @@ -60,6 +62,8 @@ def _create_task( "status": status.value, "run_immediately": run_immediately, } + if pre_approved_mcp_server_ids is not None: + body["pre_approved_mcp_server_ids"] = pre_approved_mcp_server_ids return client.post( _url(), json=body, @@ -135,6 +139,28 @@ def _get_runs_for_task(task_id: UUID) -> list[ScheduledTaskRun]: ) +def _create_mcp_server(owner: str, *, available_in_craft: bool) -> int: + with get_session_with_current_tenant() as db_session: + server = create_mcp_server__no_commit( + owner_email=owner, + name=f"scheduled-task-mcp-{uuid4().hex[:8]}", + description=None, + server_url="https://example.com/mcp", + auth_type=None, + transport=None, + auth_performer=None, + db_session=db_session, + is_public=True, + ) + update_mcp_server__no_commit( + server_id=server.id, + db_session=db_session, + available_in_craft=available_in_craft, + ) + db_session.commit() + return server.id + + def test_create_task_compiles_cron(admin_user: DATestUser) -> None: response = _create_task( admin_user, @@ -152,6 +178,82 @@ def test_create_task_compiles_cron(admin_user: DATestUser) -> None: assert row.cron_expression == cron +def test_create_and_patch_mcp_pre_approvals(admin_user: DATestUser) -> None: + server_a = _create_mcp_server(admin_user.email, available_in_craft=True) + server_b = _create_mcp_server(admin_user.email, available_in_craft=True) + unavailable_server = _create_mcp_server(admin_user.email, available_in_craft=False) + + create_response = _create_task( + admin_user, + pre_approved_mcp_server_ids=[server_b, server_a, server_b], + ) + create_response.raise_for_status() + created = create_response.json() + task_id = UUID(created["id"]) + created_mcp_server_ids = created["pre_approved_mcp_server_ids"] + assert len(created_mcp_server_ids) == 2 + assert set(created_mcp_server_ids) == {server_a, server_b} + + patch_response = _patch_task( + admin_user, + task_id, + {"pre_approved_mcp_server_ids": [server_a]}, + ) + patch_response.raise_for_status() + assert patch_response.json()["pre_approved_mcp_server_ids"] == [server_a] + + rejected_response = _patch_task( + admin_user, + task_id, + {"pre_approved_mcp_server_ids": [unavailable_server]}, + ) + assert rejected_response.status_code == 400 + + detail_response = client.get( + _url(str(task_id)), + headers=admin_user.headers, + cookies=admin_user.cookies, + ) + detail_response.raise_for_status() + assert detail_response.json()["pre_approved_mcp_server_ids"] == [server_a] + + +def test_patch_retains_and_removes_existing_unavailable_mcp_pre_approval( + admin_user: DATestUser, +) -> None: + server_id = _create_mcp_server(admin_user.email, available_in_craft=True) + create_response = _create_task( + admin_user, + pre_approved_mcp_server_ids=[server_id], + ) + create_response.raise_for_status() + task_id = UUID(create_response.json()["id"]) + + with get_session_with_current_tenant() as db_session: + update_mcp_server__no_commit( + server_id=server_id, + db_session=db_session, + available_in_craft=False, + ) + db_session.commit() + + retain_response = _patch_task( + admin_user, + task_id, + {"pre_approved_mcp_server_ids": [server_id]}, + ) + retain_response.raise_for_status() + assert retain_response.json()["pre_approved_mcp_server_ids"] == [server_id] + + remove_response = _patch_task( + admin_user, + task_id, + {"pre_approved_mcp_server_ids": []}, + ) + remove_response.raise_for_status() + assert remove_response.json()["pre_approved_mcp_server_ids"] == [] + + def test_create_task_interval_days_requires_time_of_day(admin_user: DATestUser) -> None: response = _create_task( admin_user, diff --git a/backend/tests/integration/tests/indexing/test_repeated_error_state.py b/backend/tests/integration/tests/indexing/test_repeated_error_state.py index 9bed382d742..a8ff8f6999a 100644 --- a/backend/tests/integration/tests/indexing/test_repeated_error_state.py +++ b/backend/tests/integration/tests/indexing/test_repeated_error_state.py @@ -133,7 +133,9 @@ def test_repeated_error_state_detection_and_recovery( break if time.monotonic() - start_time > 90: - assert False, "CC pair did not enter repeated error state within 90 seconds" + raise AssertionError( + "CC pair did not enter repeated error state within 90 seconds" + ) time.sleep(2) diff --git a/backend/tests/integration/tests/projects/test_projects.py b/backend/tests/integration/tests/projects/test_projects.py index 9478a1f82ff..6dd3da11168 100644 --- a/backend/tests/integration/tests/projects/test_projects.py +++ b/backend/tests/integration/tests/projects/test_projects.py @@ -1,5 +1,6 @@ from typing import List +import httpx import pytest from onyx.db.engine.sql_engine import get_session_with_current_tenant @@ -194,7 +195,7 @@ def test_projects_flow( assert len(remaining_files) == 2 # Case 7: Edge cases - with pytest.raises(Exception): + with pytest.raises(httpx.HTTPStatusError): ProjectManager.create( name="", user_performing_action=basic_user, @@ -207,14 +208,14 @@ def test_projects_flow( ) assert not deletion_success - with pytest.raises(Exception): + with pytest.raises(httpx.HTTPStatusError): ProjectManager.set_instructions( project_id=non_existent_id, instructions="Test instructions", user_performing_action=basic_user, ) - with pytest.raises(Exception): + with pytest.raises(httpx.HTTPStatusError): ProjectManager.upload_files( project_id=non_existent_id, files=[("test.txt", b"content")], @@ -222,7 +223,7 @@ def test_projects_flow( ) long_name = "a" * 1000 - with pytest.raises(Exception): + with pytest.raises(httpx.HTTPStatusError): ProjectManager.create( name=long_name, user_performing_action=basic_user, diff --git a/backend/tests/integration/tests/query_history/test_usage_reports.py b/backend/tests/integration/tests/query_history/test_usage_reports.py index 1a835f20806..8fea1e8f561 100644 --- a/backend/tests/integration/tests/query_history/test_usage_reports.py +++ b/backend/tests/integration/tests/query_history/test_usage_reports.py @@ -23,7 +23,7 @@ def test_usage_reports(reset: None) -> None: # noqa: ARG001 count = 0 for entry_batch in get_all_empty_chat_message_entries(db_session, period): - for entry in entry_batch: + for _entry in entry_batch: count += 1 assert count == EXPECTED_MESSAGES @@ -37,7 +37,7 @@ def test_usage_reports(reset: None) -> None: # noqa: ARG001 count = 0 for entry_batch in get_all_empty_chat_message_entries(db_session, period): - for entry in entry_batch: + for _entry in entry_batch: count += 1 lower = EXPECTED_MESSAGES // 3 - (EXPECTED_MESSAGES // (3 * 3)) diff --git a/backend/tests/integration/tests/reporting/test_usage_export_api.py b/backend/tests/integration/tests/reporting/test_usage_export_api.py index e3420181cdf..ca1996b697a 100644 --- a/backend/tests/integration/tests/reporting/test_usage_export_api.py +++ b/backend/tests/integration/tests/reporting/test_usage_export_api.py @@ -290,6 +290,13 @@ def test_read_usage_report( assert "chat_messages.csv" in file_names assert "users.csv" in file_names assert "usage_by_user.csv" in file_names + assert "usage_report.pdf" in file_names + + with zip_file.open("usage_report.pdf") as pdf_file: + pdf_bytes = pdf_file.read() + assert pdf_bytes.startswith(b"%PDF-") + assert len(pdf_bytes) > 1000 + # Verify usage_by_user.csv has the expected columns. The seeded # chat history doesn't record UserUsage rows, so there's no data # to assert on, just the header shape. @@ -426,7 +433,7 @@ def test_concurrent_report_generation( # Generate multiple reports concurrently num_reports = 3 - for i in range(num_reports): + for _i in range(num_reports): response = client.post( f"{API_SERVER_URL}/admin/usage-report", json={}, diff --git a/backend/tests/regression/search_quality/run_search_eval.py b/backend/tests/regression/search_quality/run_search_eval.py index 16658f0f4f0..568c4c1883d 100644 --- a/backend/tests/regression/search_quality/run_search_eval.py +++ b/backend/tests/regression/search_quality/run_search_eval.py @@ -346,7 +346,7 @@ def generate_chart(self, export_path: Path) -> None: plt.grid(axis="y", alpha=0.3) # add value labels on top of each bar - for bar, count in zip(bars, counts): + for bar, count in zip(bars, counts, strict=True): if count > 0: plt.text( bar.get_x() + bar.get_width() / 2, diff --git a/backend/tests/unit/background/celery/test_celery_utils.py b/backend/tests/unit/background/celery/test_celery_utils.py index ba9e9742e97..3037233345a 100644 --- a/backend/tests/unit/background/celery/test_celery_utils.py +++ b/backend/tests/unit/background/celery/test_celery_utils.py @@ -62,7 +62,7 @@ def test_recorded_on_exception(self) -> None: connector_type="confluence" )._sum.get() - with pytest.raises(Exception): + with pytest.raises(Exception, match="unexpected error"): extract_ids_from_runnable_connector(connector, connector_type="confluence") after = PRUNING_ENUMERATION_DURATION.labels( @@ -116,7 +116,7 @@ def test_rate_limit_detection_is_case_insensitive(self) -> None: connector = _raising_connector("RATE LIMIT exceeded") before = PRUNING_RATE_LIMIT_ERRORS.labels(connector_type="jira")._value.get() - with pytest.raises(Exception): + with pytest.raises(Exception, match="RATE LIMIT exceeded"): extract_ids_from_runnable_connector(connector, connector_type="jira") after = PRUNING_RATE_LIMIT_ERRORS.labels(connector_type="jira")._value.get() @@ -131,7 +131,7 @@ def test_connector_type_label_matches_input(self) -> None: connector_type="jira" )._value.get() - with pytest.raises(Exception): + with pytest.raises(Exception, match="rate limit exceeded"): extract_ids_from_runnable_connector( connector, connector_type="google_drive" ) @@ -149,7 +149,7 @@ def test_defaults_to_unknown_connector_type(self) -> None: connector = _raising_connector("rate limit exceeded") before = PRUNING_RATE_LIMIT_ERRORS.labels(connector_type="unknown")._value.get() - with pytest.raises(Exception): + with pytest.raises(Exception, match="rate limit exceeded"): extract_ids_from_runnable_connector(connector) after = PRUNING_RATE_LIMIT_ERRORS.labels(connector_type="unknown")._value.get() diff --git a/backend/tests/unit/ee/onyx/server/middleware/test_license_enforcement.py b/backend/tests/unit/ee/onyx/server/middleware/test_license_enforcement.py index 07621012f23..857a651d76b 100644 --- a/backend/tests/unit/ee/onyx/server/middleware/test_license_enforcement.py +++ b/backend/tests/unit/ee/onyx/server/middleware/test_license_enforcement.py @@ -51,6 +51,7 @@ def test_allowed_path_prefix_matching(self) -> None: """Subpaths of allowed prefixes should also be allowed.""" assert _is_path_allowed("/auth/callback/google") is True assert _is_path_allowed("/admin/billing/checkout") is True + assert _is_path_allowed("/mcp/oauth/client-metadata") is True def test_custom_api_prefix_allowlist(self, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(api_prefix, "APP_API_PREFIX", "v2") diff --git a/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_branding.py b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_branding.py new file mode 100644 index 00000000000..9010c38d61d --- /dev/null +++ b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_branding.py @@ -0,0 +1,88 @@ +"""Which logo the review pack renders under.""" + +from unittest.mock import MagicMock, patch + +from ee.onyx.server.enterprise_settings.models import EnterpriseSettings +from ee.onyx.server.reporting.usage_report_branding import ( + ReportBranding, + load_report_branding, +) +from onyx.utils.file import FileWithMimeType + +_LOGOTYPE = b"logotype-bytes" +_LOGO = b"logo-bytes" + + +def _file_store(stored: dict[str, FileWithMimeType]) -> MagicMock: + store = MagicMock() + store.get_file_with_mime_type.side_effect = lambda file_id: stored.get(file_id) + return store + + +def _load( + settings: EnterpriseSettings, stored: dict[str, FileWithMimeType] +) -> ReportBranding: + with patch( + "ee.onyx.server.reporting.usage_report_branding.load_runtime_settings", + return_value=settings, + ): + return load_report_branding(_file_store(stored)) + + +def test_logotype_wins_over_the_square_mark() -> None: + branding = _load( + EnterpriseSettings( + application_name="Acme", use_custom_logo=True, use_custom_logotype=True + ), + { + "__logotype__": FileWithMimeType(data=_LOGOTYPE, mime_type="image/png"), + "__logo__": FileWithMimeType(data=_LOGO, mime_type="image/png"), + }, + ) + + assert branding.logo == _LOGOTYPE + assert branding.application_name == "Acme" + + +def test_falls_back_to_the_square_mark_when_no_logotype() -> None: + branding = _load( + EnterpriseSettings(use_custom_logo=True, use_custom_logotype=True), + {"__logo__": FileWithMimeType(data=_LOGO, mime_type="image/png")}, + ) + + assert branding.logo == _LOGO + + +def test_an_undrawable_logo_becomes_a_wordmark_not_our_mark() -> None: + """ReportLab cannot draw SVG. Their own name beats stamping the Onyx mark + on a report they forward to their leadership.""" + branding = _load( + EnterpriseSettings(application_name="Acme", use_custom_logotype=True), + {"__logotype__": FileWithMimeType(data=b"", mime_type="image/svg+xml")}, + ) + + assert branding.logo is None + assert branding.application_name == "Acme" + + +def test_bundled_logo_is_used_when_nothing_is_uploaded() -> None: + branding = _load(EnterpriseSettings(), {}) + + assert branding.logo is not None + assert branding.logo.startswith(b"\x89PNG") + + +def test_a_file_store_failure_falls_back_to_the_wordmark() -> None: + """A configured custom logo we cannot read still must not become our mark.""" + store = MagicMock() + store.get_file_with_mime_type.side_effect = RuntimeError("object storage down") + + with patch( + "ee.onyx.server.reporting.usage_report_branding.load_runtime_settings", + return_value=EnterpriseSettings( + application_name="Acme", use_custom_logotype=True + ), + ): + branding = load_report_branding(store) + + assert branding.logo is None diff --git a/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_data.py b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_data.py new file mode 100644 index 00000000000..331b497b1f2 --- /dev/null +++ b/backend/tests/unit/ee/onyx/server/reporting/test_usage_report_data.py @@ -0,0 +1,281 @@ +"""Aggregation behind the usage report review pack.""" + +from datetime import datetime, timezone +from io import BytesIO +from unittest.mock import MagicMock, patch + +import pytest +from PIL import Image as PILImage +from pypdf import PdfReader + +from ee.onyx.server.reporting.usage_report_branding import ReportBranding +from ee.onyx.server.reporting.usage_report_data import ( + TOP_USER_LIMIT, + UsageReportData, + build_usage_report_data, +) +from ee.onyx.server.reporting.usage_report_pdf import ( + _axis_labels, + _display_name, + render_usage_report_pdf, +) +from onyx.db.enums import AccountType +from onyx.db.models import User +from onyx.db.user_usage import DELETED_USER_EXPORT_EMAIL, UsageExportRow + +_BRANDING = ReportBranding(application_name="Acme Intelligence", logo=None) + +PERIOD_START = datetime(2026, 7, 1, tzinfo=timezone.utc) +PERIOD_END = datetime(2026, 7, 31, tzinfo=timezone.utc) + + +def _row( + email: str, + cost: float = 10.0, + day: str = "2026-07-01", + flow: str = "chat", +) -> UsageExportRow: + return UsageExportRow( + email=email, + model="gpt-5", + flow=flow, + provider="openai", + day=day, + input_tokens=100, + output_tokens=50, + cache_read_tokens=10, + cost_cents=cost, + ) + + +def _user( + email: str, + is_active: bool = True, + account_type: AccountType = AccountType.STANDARD, +) -> User: + user = User() + user.email = email + user.is_active = is_active + user.account_type = account_type + return user + + +def _build(rows: list[UsageExportRow], users: list[User]) -> UsageReportData: + with patch( + "ee.onyx.server.reporting.usage_report_data.get_all_users", return_value=users + ): + return build_usage_report_data( + db_session=MagicMock(), + rows=rows, + period_start=PERIOD_START, + period_end=PERIOD_END, + ) + + +def test_totals_reconcile_with_the_rows() -> None: + """The pack's totals must equal the CSV's, including deleted-user spend.""" + rows = [ + _row("a@x.com", 10.0), + _row("b@x.com", 5.5), + _row(DELETED_USER_EXPORT_EMAIL, 4.5), + ] + + data = _build(rows, [_user("a@x.com"), _user("b@x.com")]) + + assert data.total_cost_cents == pytest.approx(20.0) + assert sum(e.cost_cents for e in data.by_model) == pytest.approx(20.0) + assert sum(e.cost_cents for e in data.by_flow) == pytest.approx(20.0) + assert sum(e.cost_cents for e in data.top_users) == pytest.approx(20.0) + assert data.total_input_tokens == 300 + + +def test_deleted_user_counts_toward_spend_but_is_not_a_person() -> None: + rows = [_row("a@x.com"), _row(DELETED_USER_EXPORT_EMAIL)] + + data = _build(rows, [_user("a@x.com")]) + + assert data.active_users == 1 + assert data.daily[0].active_users == 1 + assert data.total_cost_cents == pytest.approx(20.0) + + +def test_api_key_usage_is_not_an_active_user() -> None: + """Seats exclude API-key users, so activity must exclude them too, or + active_users can exceed licensed_users.""" + rows = [_row("a@x.com"), _row("somekey@onyxapikey.ai")] + + data = _build(rows, [_user("a@x.com")]) + + assert data.active_users == 1 + assert data.active_users <= data.licensed_users + + +def test_unlabeled_flow_is_grouped_as_other() -> None: + data = _build([_row("a@x.com", flow="")], [_user("a@x.com")]) + + assert [(entry.name, entry.cost_cents) for entry in data.by_flow] == [ + ("other", 10.0) + ] + + +def test_service_accounts_do_not_hold_a_seat() -> None: + users = [ + _user("human@x.com"), + _user("bot@x.com", account_type=AccountType.SERVICE_ACCOUNT), + _user("gone@x.com", is_active=False), + ] + + data = _build([_row("human@x.com")], users) + + assert data.licensed_users == 1 + assert data.seated_active_users == 1 + assert data.dormant_users == [] + + +def test_a_deactivated_user_is_active_but_holds_no_seat() -> None: + """Someone who used it mid-period and was deactivated before the report ran + counts as a person, not as an occupied seat. The meter must not read + "2 of 1 seats active".""" + rows = [_row("stayed@x.com"), _row("departed@x.com")] + users = [_user("stayed@x.com"), _user("departed@x.com", is_active=False)] + + data = _build(rows, users) + + assert data.active_users == 2 + assert data.licensed_users == 1 + assert data.seated_active_users == 1 + assert data.seated_active_users <= data.licensed_users + + +def test_dormant_seats_are_named() -> None: + users = [_user("active@x.com"), _user("idle@x.com")] + + data = _build([_row("active@x.com")], users) + + assert data.licensed_users == 2 + assert data.active_users == 1 + assert data.dormant_users == ["idle@x.com"] + + +def test_top_users_folds_the_tail_and_preserves_the_total() -> None: + rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(TOP_USER_LIMIT + 3)] + + data = _build(rows, []) + + assert len(data.top_users) == TOP_USER_LIMIT + 1 + assert data.top_users[-1].name == "Other (3)" + assert sum(e.cost_cents for e in data.top_users) == pytest.approx( + data.total_cost_cents + ) + + +def test_a_single_extra_user_is_named_rather_than_folded() -> None: + """Folding one entry hides a name and saves no space.""" + rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(TOP_USER_LIMIT + 1)] + + data = _build(rows, []) + + assert not any(e.name.startswith("Other") for e in data.top_users) + + +def test_api_key_spend_is_labeled_by_key_name() -> None: + """Per-key spend stays in the breakdown; only the label is cleaned up.""" + # A DB check constraint lowercases stored emails, so this is the real shape. + assert ( + _display_name("api_key__nightly-sync@2f3c8a10-uuid.onyxapikey.ai") + == "nightly-sync (API key)" + ) + assert ( + _display_name("API_KEY__nightly-sync@2f3c8a10-uuid.onyxapikey.ai") + == "nightly-sync (API key)" + ) + assert ( + _display_name("api_key__sync@corp@2f3c8a10-uuid.onyxapikey.ai") + == "sync@corp (API key)" + ) + assert _display_name("human@corp.com") == "human@corp.com" + assert _display_name("gpt-5") == "gpt-5" + assert _display_name(DELETED_USER_EXPORT_EMAIL) == DELETED_USER_EXPORT_EMAIL + + +def test_api_key_spend_still_reconciles_after_relabeling() -> None: + rows = [_row("a@x.com", 10.0), _row("api_key__bot@uuid.onyxapikey.ai", 90.0)] + + data = _build(rows, [_user("a@x.com")]) + + assert data.total_cost_cents == pytest.approx(100.0) + assert sum(e.cost_cents for e in data.top_users) == pytest.approx(100.0) + assert data.active_users == 1 + + +def test_zero_usage_still_renders_a_pdf() -> None: + data = _build([], [_user("idle@x.com")]) + + assert not data.has_usage + assert data.cost_per_active_user_cents == 0.0 + assert render_usage_report_pdf(data, _BRANDING).startswith(b"%PDF-") + + +def test_application_name_is_not_parsed_as_markup() -> None: + """ReportLab Paragraph parses a markup subset: an unescaped name with a tag + silently drops text, injects formatting, or raises and loses the PDF.""" + data = _build([_row("a@x.com")], [_user("a@x.com")]) + + for name in ["Tools Us", "Acme Corp", 'X Y']: + branding = ReportBranding(application_name=name, logo=None) + pdf = render_usage_report_pdf(data, branding) + assert pdf.startswith(b"%PDF-") + + +def _png(width: int = 40, height: int = 10) -> bytes: + buffer = BytesIO() + PILImage.new("RGBA", (width, height), (0, 0, 0, 255)).save(buffer, format="PNG") + return buffer.getvalue() + + +def test_the_cover_is_unnumbered_and_later_pages_are_not() -> None: + """A folio on the cover, or an off-by-one, corrupts every generated pack.""" + rows = [_row(f"u{i}@x.com", cost=float(i + 1)) for i in range(12)] + data = _build(rows, [_user(f"u{i}@x.com") for i in range(12)]) + + reader = PdfReader(BytesIO(render_usage_report_pdf(data, _BRANDING))) + total = len(reader.pages) + + assert total > 1 + assert f"1 of {total}" not in reader.pages[0].extract_text() + for number in range(2, total + 1): + assert f"{number} of {total}" in reader.pages[number - 1].extract_text() + + +def test_render_is_deterministic() -> None: + data = _build([_row("a@x.com")], [_user("a@x.com")]) + + assert render_usage_report_pdf(data, _BRANDING) == render_usage_report_pdf( + data, _BRANDING + ) + + +def test_render_is_deterministic_with_an_embedded_logo() -> None: + """The default path embeds a logo, so determinism must hold with one.""" + data = _build([_row("a@x.com")], [_user("a@x.com")]) + branding = ReportBranding(application_name="Acme", logo=_png()) + + assert render_usage_report_pdf(data, branding) == render_usage_report_pdf( + data, branding + ) + + +def test_unreadable_logo_still_produces_a_pdf() -> None: + """A corrupt upload must cost the branding, not the report.""" + data = _build([_row("a@x.com")], [_user("a@x.com")]) + + for logo in (b"not-an-image", b""): + branding = ReportBranding(application_name="Acme", logo=logo) + assert render_usage_report_pdf(data, branding).startswith(b"%PDF-") + + +def test_axis_labels_never_exceed_the_display_limit() -> None: + for day_count in (0, 1, 12, 13, 23, 24, 25, 30, 365): + days = [f"2026-07-{day + 1:02d}" for day in range(day_count)] + + assert sum(bool(label) for label in _axis_labels(days)) <= 12 diff --git a/backend/tests/unit/onyx/auth/test_user_registration.py b/backend/tests/unit/onyx/auth/test_user_registration.py index 00b815dea4e..4df49b4172a 100644 --- a/backend/tests/unit/onyx/auth/test_user_registration.py +++ b/backend/tests/unit/onyx/auth/test_user_registration.py @@ -66,8 +66,8 @@ async def __aexit__( def _mock_user_manager_methods(user_manager: UserManager) -> None: - setattr(user_manager, "validate_password", AsyncMock()) - setattr(user_manager, "_assign_default_pinned_assistants", AsyncMock()) + user_manager.validate_password = AsyncMock() + user_manager._assign_default_pinned_assistants = AsyncMock() class TestDisposableEmailValidation: @@ -593,11 +593,9 @@ async def test_oauth_create_does_not_block_dotted_gmail( user_manager = UserManager(MagicMock()) _mock_user_manager_methods(user_manager) - setattr(user_manager, "on_after_register", AsyncMock()) - setattr( - user_manager, - "get_by_oauth_account", - AsyncMock(side_effect=exceptions.UserNotExists()), + user_manager.on_after_register = AsyncMock() + user_manager.get_by_oauth_account = AsyncMock( + side_effect=exceptions.UserNotExists() ) created_user = MagicMock(id="test-id", email=dotted_email) @@ -657,11 +655,9 @@ def _unclaimed(**attrs: object) -> MagicMock: def _manager_with_existing(existing_user: MagicMock) -> UserManager: user_manager = UserManager(MagicMock()) _mock_user_manager_methods(user_manager) - setattr(user_manager, "on_after_register", AsyncMock()) - setattr( - user_manager, - "get_by_oauth_account", - AsyncMock(side_effect=exceptions.UserNotExists()), + user_manager.on_after_register = AsyncMock() + user_manager.get_by_oauth_account = AsyncMock( + side_effect=exceptions.UserNotExists() ) mock_user_db = MagicMock() mock_user_db.get_by_email = AsyncMock(return_value=existing_user) diff --git a/backend/tests/unit/onyx/chat/test_incognito_record_mode.py b/backend/tests/unit/onyx/chat/test_incognito_record_mode.py new file mode 100644 index 00000000000..15b97458cc0 --- /dev/null +++ b/backend/tests/unit/onyx/chat/test_incognito_record_mode.py @@ -0,0 +1,105 @@ +"""Guards the incognito recording policy: which mode permits which sink. + +The mode enum is the only policy object, so these tests pin the full +mode x sink matrix. A behavior change that is not also a deliberate edit +here is a policy regression. +""" + +from unittest.mock import MagicMock, patch + +import pytest + +from onyx.chat.incognito import ( + content_free_file_descriptors, + resolve_incognito_record_mode, +) +from onyx.db.enums import IncognitoRecordMode +from onyx.file_store.models import ChatFileType, FileDescriptor + + +class TestModeSinkMatrix: + @pytest.mark.parametrize( + "mode,persists_content,emits_external_traces,fires_hooks", + [ + (IncognitoRecordMode.FULL_HISTORY, True, True, True), + (IncognitoRecordMode.USAGE_ONLY, False, False, False), + ], + ) + def test_matrix( + self, + mode: IncognitoRecordMode, + persists_content: bool, + emits_external_traces: bool, + fires_hooks: bool, + ) -> None: + assert mode.persists_content is persists_content + assert mode.emits_external_traces is emits_external_traces + assert mode.fires_hooks is fires_hooks + + def test_only_full_history_persists_content(self) -> None: + """The guarantee: no other mode may write conversation content.""" + persisting = [m for m in IncognitoRecordMode if m.persists_content] + assert persisting == [IncognitoRecordMode.FULL_HISTORY] + + def test_no_mode_emits_external_traces_without_persisting_content(self) -> None: + """External egress never outlives the decision to record.""" + for mode in IncognitoRecordMode: + if mode.emits_external_traces: + assert mode.persists_content + + +class TestResolver: + @pytest.mark.parametrize("mode", list(IncognitoRecordMode)) + def test_resolves_to_the_admin_setting(self, mode: IncognitoRecordMode) -> None: + with patch( + "onyx.chat.incognito.load_effective_uncached", + return_value=MagicMock(incognito_record_mode=mode), + ): + assert resolve_incognito_record_mode() is mode + + def test_reads_past_the_settings_cache(self) -> None: + """The pin is durable, so a cached pre-save mode must never reach it.""" + with ( + patch( + "onyx.chat.incognito.load_effective_uncached", + return_value=MagicMock( + incognito_record_mode=IncognitoRecordMode.USAGE_ONLY + ), + ) as uncached, + patch( + "onyx.chat.incognito.get_security_settings", + return_value=MagicMock( + incognito_record_mode=IncognitoRecordMode.FULL_HISTORY + ), + ) as cached, + ): + assert resolve_incognito_record_mode() is IncognitoRecordMode.USAGE_ONLY + assert uncached.called + assert not cached.called + + def test_unknown_context_value_fails_closed(self) -> None: + """A corrupt contextvar must never read as content-persisting.""" + resolved = IncognitoRecordMode.from_context_value("garbage") + assert resolved is IncognitoRecordMode.USAGE_ONLY + assert IncognitoRecordMode.from_context_value(None) is None + + +class TestContentFreeFileDescriptors: + """The persisted descriptor of a content-free turn keeps linkage, never + the filename.""" + + def test_strips_name_and_keeps_linkage(self) -> None: + scrubbed = content_free_file_descriptors( + [ + FileDescriptor( + id="file-1", + type=ChatFileType.DOC, + name="acquisition_target.pdf", + user_file_id="uf-1", + ) + ] + ) + assert scrubbed == [ + FileDescriptor(id="file-1", type=ChatFileType.DOC, user_file_id="uf-1") + ] + assert "name" not in scrubbed[0] diff --git a/backend/tests/unit/onyx/chat/test_multi_model_streaming.py b/backend/tests/unit/onyx/chat/test_multi_model_streaming.py index 88d556b546f..72b4a4e942d 100644 --- a/backend/tests/unit/onyx/chat/test_multi_model_streaming.py +++ b/backend/tests/unit/onyx/chat/test_multi_model_streaming.py @@ -276,8 +276,9 @@ def _make_setup(n_models: int = 1) -> MagicMock: setup.available_files.chat_file_ids = [] setup.forced_tool_id = None setup.simple_chat_history = [] - setup.chat_session.id = uuid4() - setup.user_message.id = None + setup.chat_session_id = uuid4() + setup.chat_session_project_id = None + setup.user_message_id = None setup.custom_tool_additional_headers = None setup.mcp_headers = None return setup diff --git a/backend/tests/unit/onyx/connectors/box/test_box_connector.py b/backend/tests/unit/onyx/connectors/box/test_box_connector.py index 7ccfc91ab70..0612f16ebbc 100644 --- a/backend/tests/unit/onyx/connectors/box/test_box_connector.py +++ b/backend/tests/unit/onyx/connectors/box/test_box_connector.py @@ -652,7 +652,7 @@ def test_load_credentials_defers_clients_and_impersonation_lookup() -> None: assert connector._content_client is None assert connector._user_email == "user@example.com" - connector.enterprise_client + _ = connector.enterprise_client assert connector._enterprise_client is not None assert connector._content_client is None diff --git a/backend/tests/unit/onyx/connectors/braintrust/__init__.py b/backend/tests/unit/onyx/connectors/braintrust/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/backend/tests/unit/onyx/connectors/jira/test_jira_bulk_fetch.py b/backend/tests/unit/onyx/connectors/jira/test_jira_bulk_fetch.py index 5a052bb4b0e..2cdf1d340c4 100644 --- a/backend/tests/unit/onyx/connectors/jira/test_jira_bulk_fetch.py +++ b/backend/tests/unit/onyx/connectors/jira/test_jira_bulk_fetch.py @@ -104,7 +104,7 @@ def test_bulk_fetch_non_json_error_propagates() -> None: try: bulk_fetch_issues(client, ["1"]) - assert False, "Expected ValueError to propagate" + raise AssertionError("Expected ValueError to propagate") except ValueError: pass diff --git a/backend/tests/unit/onyx/connectors/mediawiki/test_wiki.py b/backend/tests/unit/onyx/connectors/mediawiki/test_wiki.py index 262de64ed5b..33fb7cfdcbd 100644 --- a/backend/tests/unit/onyx/connectors/mediawiki/test_wiki.py +++ b/backend/tests/unit/onyx/connectors/mediawiki/test_wiki.py @@ -101,7 +101,7 @@ def test_get_doc_from_page( ) assert len(doc.sections) == 3 for section, expected_section in zip( - doc.sections, test_page._sections_helper + [test_page.header] + doc.sections, test_page._sections_helper + [test_page.header], strict=True ): assert ( section.text is not None diff --git a/backend/tests/unit/onyx/connectors/salesforce/test_salesforce_custom_config.py b/backend/tests/unit/onyx/connectors/salesforce/test_salesforce_custom_config.py index 15d14149f9f..cc4320e2ab4 100644 --- a/backend/tests/unit/onyx/connectors/salesforce/test_salesforce_custom_config.py +++ b/backend/tests/unit/onyx/connectors/salesforce/test_salesforce_custom_config.py @@ -85,7 +85,9 @@ def test_validation() -> None: for i, invalid_config in enumerate(invalid_configs): try: _validate_custom_query_config(invalid_config) - assert False, f"Should have raised ValueError for invalid_config[{i}]" + raise AssertionError( + f"Should have raised ValueError for invalid_config[{i}]" + ) except ValueError: print(f"✅ Correctly rejected invalid config {i}") diff --git a/backend/tests/unit/onyx/connectors/test_connector_factory.py b/backend/tests/unit/onyx/connectors/test_connector_factory.py index 83dd6309d25..e352e343b64 100644 --- a/backend/tests/unit/onyx/connectors/test_connector_factory.py +++ b/backend/tests/unit/onyx/connectors/test_connector_factory.py @@ -262,7 +262,8 @@ def test_instantiate_connector_loads_class_lazily(self) -> None: # This should trigger lazy loading but will fail on actual instantiation # due to missing real configuration - that's expected - with pytest.raises(Exception): # We expect some kind of error due to mock data + # We expect some kind of error due to mock data + with pytest.raises(Exception): # noqa: B017 instantiate_connector( mock_session, DocumentSource.WEB, # Simple connector diff --git a/backend/tests/unit/onyx/document_index/vespa/test_vespa_batch_flush.py b/backend/tests/unit/onyx/document_index/vespa/test_vespa_batch_flush.py index 5af8d8b8b88..8f08c0228eb 100644 --- a/backend/tests/unit/onyx/document_index/vespa/test_vespa_batch_flush.py +++ b/backend/tests/unit/onyx/document_index/vespa/test_vespa_batch_flush.py @@ -80,7 +80,7 @@ def _make_indexing_metadata( old_chunk_cnt=old, new_chunk_cnt=new, ) - for doc_id, old, new in zip(doc_ids, old_counts, new_counts) + for doc_id, old, new in zip(doc_ids, old_counts, new_counts, strict=True) } ) diff --git a/backend/tests/unit/onyx/llm/test_tracing_wrap.py b/backend/tests/unit/onyx/llm/test_tracing_wrap.py index 95f831ab479..2fc223a53d3 100644 --- a/backend/tests/unit/onyx/llm/test_tracing_wrap.py +++ b/backend/tests/unit/onyx/llm/test_tracing_wrap.py @@ -211,7 +211,7 @@ def test_outer_guard_false_for_finished_span_leaked_into_contextvar() -> None: def test_extract_prompt_reads_positional_arg() -> None: llm = _FakeLLM() - sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) + sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) # noqa: B009 prompt, tools = _extract_prompt_and_tools(sig, llm, ("hi",), {}) assert prompt == "hi" assert tools is None @@ -219,7 +219,7 @@ def test_extract_prompt_reads_positional_arg() -> None: def test_extract_prompt_reads_keyword_arg() -> None: llm = _FakeLLM() - sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) + sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) # noqa: B009 prompt, tools = _extract_prompt_and_tools(sig, llm, (), {"prompt": "hi"}) assert prompt == "hi" assert tools is None @@ -227,7 +227,7 @@ def test_extract_prompt_reads_keyword_arg() -> None: def test_extract_tools_reads_keyword_arg() -> None: llm = _FakeLLM() - sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) + sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) # noqa: B009 tool_defs = [{"type": "function", "function": {"name": "search"}}] prompt, tools = _extract_prompt_and_tools( sig, llm, (), {"prompt": "hi", "tools": tool_defs} @@ -240,7 +240,7 @@ def test_extract_prompt_returns_none_on_signature_mismatch() -> None: """Unknown keyword arguments don't match the signature → bind fails → extraction returns (None, None) rather than raising.""" llm = _FakeLLM() - sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) + sig = _validate_prompt_param(getattr(_FakeLLM.invoke, "__wrapped__")) # noqa: B009 assert _extract_prompt_and_tools(sig, llm, (), {"not_a_real_param": "hi"}) == ( None, None, diff --git a/backend/tests/unit/onyx/server/features/craft/sandbox/test_serve_client_401_reload.py b/backend/tests/unit/onyx/server/features/craft/sandbox/test_serve_client_401_reload.py index 636551e3373..b4bc7fcac39 100644 --- a/backend/tests/unit/onyx/server/features/craft/sandbox/test_serve_client_401_reload.py +++ b/backend/tests/unit/onyx/server/features/craft/sandbox/test_serve_client_401_reload.py @@ -54,7 +54,7 @@ def handler(_: httpx.Request) -> httpx.Response: # 401 with unchanged password → no retry, surfaces as HTTPStatusError. try: client.ensure_session(None, directory="/workspace/sessions/x") - assert False, "expected HTTPStatusError" + raise AssertionError("expected HTTPStatusError") except httpx.HTTPStatusError: pass assert n == 1, n diff --git a/backend/tests/unit/onyx/server/features/mcp/test_mcp_oauth_client_metadata_document.py b/backend/tests/unit/onyx/server/features/mcp/test_mcp_oauth_client_metadata_document.py new file mode 100644 index 00000000000..46ab49ee037 --- /dev/null +++ b/backend/tests/unit/onyx/server/features/mcp/test_mcp_oauth_client_metadata_document.py @@ -0,0 +1,60 @@ +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from onyx.server.auth_check import PUBLIC_ENDPOINT_SPECS, is_route_in_spec_list +from onyx.server.features.mcp import client_metadata + +TEST_WEB_DOMAIN = "https://onyx.example.com" +METADATA_ROUTE = "/mcp/oauth/client-metadata" + + +def _build_test_app() -> FastAPI: + app = FastAPI() + app.include_router(client_metadata.router, prefix="/mcp") + return app + + +def test_mcp_oauth_client_metadata_document_is_public_and_cacheable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(client_metadata, "WEB_DOMAIN", f"{TEST_WEB_DOMAIN}/") + app = _build_test_app() + + response = TestClient(app).get(METADATA_ROUTE) + + assert response.status_code == 200 + assert response.headers["Cache-Control"] == "public, max-age=3600" + assert response.json() == { + "client_id": f"{TEST_WEB_DOMAIN}/api/mcp/oauth/client-metadata", + "client_name": "Onyx", + "redirect_uris": [f"{TEST_WEB_DOMAIN}/mcp/oauth/callback"], + "grant_types": ["authorization_code", "refresh_token"], + "response_types": ["code"], + "token_endpoint_auth_method": "none", + } + + metadata_route = next( + route for route in app.routes if getattr(route, "path", "") == METADATA_ROUTE + ) + assert is_route_in_spec_list(metadata_route, PUBLIC_ENDPOINT_SPECS) + + +@pytest.mark.parametrize( + ("web_domain", "expected_url"), + [ + ( + "https://onyx.example.com", + "https://onyx.example.com/api/mcp/oauth/client-metadata", + ), + ("http://localhost:3000", None), + ], +) +def test_mcp_oauth_client_metadata_url_requires_https( + monkeypatch: pytest.MonkeyPatch, + web_domain: str, + expected_url: str | None, +) -> None: + monkeypatch.setattr(client_metadata, "WEB_DOMAIN", web_domain) + + assert client_metadata.validated_mcp_oauth_client_metadata_url() == expected_url diff --git a/backend/tests/unit/onyx/server/features/mcp/test_mcp_oauth_refresh.py b/backend/tests/unit/onyx/server/features/mcp/test_mcp_oauth_refresh.py index bbaf5c92736..0dabf333681 100644 --- a/backend/tests/unit/onyx/server/features/mcp/test_mcp_oauth_refresh.py +++ b/backend/tests/unit/onyx/server/features/mcp/test_mcp_oauth_refresh.py @@ -487,14 +487,15 @@ def test_form_encoded_refresh_error_is_logged_without_secrets( for record in caplog.records if record.getMessage() == "mcp_oauth.refresh.started" ) - assert getattr(failed_record, "oauth_error") == "bad_refresh_token" + assert getattr(failed_record, "oauth_error") == "bad_refresh_token" # noqa: B009 assert ( - getattr(failed_record, "response_content_type") + getattr(failed_record, "response_content_type") # noqa: B009 == "application/x-www-form-urlencoded" ) - assert getattr(failed_record, "response_body_format") == "form" - assert getattr(failed_record, "refresh_attempt_id") == getattr( - started_record, "refresh_attempt_id" + assert getattr(failed_record, "response_body_format") == "form" # noqa: B009 + assert getattr(failed_record, "refresh_attempt_id") == getattr( # noqa: B009 + started_record, + "refresh_attempt_id", # noqa: B009 ) assert "OLD_ACCESS_TOKEN" not in caplog.text assert "ROTATING_REFRESH_TOKEN" not in caplog.text diff --git a/backend/tests/unit/onyx/server/security/test_models.py b/backend/tests/unit/onyx/server/security/test_models.py index da7e62ed3c2..f6fc3e35dbf 100644 --- a/backend/tests/unit/onyx/server/security/test_models.py +++ b/backend/tests/unit/onyx/server/security/test_models.py @@ -5,10 +5,12 @@ import pytest from pydantic import ValidationError +from onyx.db.enums import IncognitoRecordMode from onyx.server.security.models import ( OPERATOR_LOCKED_FIELDS, PASSWORD_LENGTH_CAP, PASSWORD_MAX_LENGTH_FLOOR, + IncognitoAvailability, SecuritySettings, SecuritySettingsOverrides, SSRFProtectionLevel, @@ -20,6 +22,8 @@ _VALID_EFFECTIVE_KWARGS: dict[str, Any] = { "user_directory_admin_only": False, "track_external_idp_expiry": False, + "incognito_availability": IncognitoAvailability.OFF, + "incognito_record_mode": IncognitoRecordMode.USAGE_ONLY, "ssrf_protection_level": SSRFProtectionLevel.VALIDATE_LLM, "mask_credential_prefix": True, "llm_custom_config_env_injection": True, diff --git a/backend/tests/unit/onyx/server/test_mcp_known_provider_oauth_helpers.py b/backend/tests/unit/onyx/server/test_mcp_known_provider_oauth_helpers.py index 5a7972936c8..50d43dd7f6f 100644 --- a/backend/tests/unit/onyx/server/test_mcp_known_provider_oauth_helpers.py +++ b/backend/tests/unit/onyx/server/test_mcp_known_provider_oauth_helpers.py @@ -8,6 +8,7 @@ import httpx import pytest from mcp.client.auth import OAuthClientProvider +from mcp.client.auth.utils import should_use_client_metadata_url from mcp.shared.auth import OAuthClientInformationFull, OAuthMetadata, OAuthToken from pydantic import AnyHttpUrl, AnyUrl @@ -250,6 +251,17 @@ def _build_provider(provider_mode: MCPOAuthProviderMode) -> OAuthClientProvider: ) +def _oauth_metadata_with_cimd(supported: bool) -> OAuthMetadata: + return OAuthMetadata( + issuer=cast(AnyHttpUrl, "https://accounts.example.com"), + authorization_endpoint=cast( + AnyHttpUrl, "https://accounts.example.com/authorize" + ), + token_endpoint=cast(AnyHttpUrl, "https://accounts.example.com/token"), + client_id_metadata_document_supported=supported, + ) + + def _patch_config_read( monkeypatch: pytest.MonkeyPatch, config_data: dict[str, object] ) -> None: @@ -283,15 +295,53 @@ def test_make_oauth_provider_sets_known_provider_metadata_and_binds_storage() -> assert storage._oauth_context is provider.context -def test_make_oauth_provider_auto_discovery_leaves_metadata_unset() -> None: +def test_make_oauth_provider_auto_discovery_leaves_metadata_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + mcp_oauth, "validated_mcp_oauth_client_metadata_url", lambda: None + ) provider = _build_provider(MCPOAuthProviderMode.AUTO_DISCOVERY) assert provider.context.oauth_metadata is None assert provider.context.token_expiry_time is None + assert provider.context.client_metadata_url is None -def test_make_oauth_provider_auto_discovery_requests_public_pkce_client() -> None: +def test_make_oauth_provider_auto_discovery_requests_public_pkce_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + metadata_url = "https://onyx.example.com/api/mcp/oauth/client-metadata" + monkeypatch.setattr( + mcp_oauth, + "validated_mcp_oauth_client_metadata_url", + lambda: metadata_url, + ) provider = _build_provider(MCPOAuthProviderMode.AUTO_DISCOVERY) + assert provider.context.client_metadata.token_endpoint_auth_method == "none" + assert provider.context.client_metadata_url == metadata_url + assert should_use_client_metadata_url( + _oauth_metadata_with_cimd(True), + provider.context.client_metadata_url, + ) + assert not should_use_client_metadata_url( + _oauth_metadata_with_cimd(False), + provider.context.client_metadata_url, + ) + + +def test_make_oauth_provider_known_provider_ignores_client_metadata_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + mcp_oauth, + "validated_mcp_oauth_client_metadata_url", + lambda: "https://onyx.example.com/api/mcp/oauth/client-metadata", + ) + + provider = _build_provider(MCPOAuthProviderMode.KNOWN_PROVIDER) + + assert provider.context.client_metadata_url is None def test_get_tokens_hydrates_expiry_and_invalidates_expired_token( @@ -529,8 +579,9 @@ def test_proactive_refresh_targets_configured_endpoint_and_persists( for record in caplog.records if record.getMessage() == "mcp_oauth.refresh.persisted" ) - assert getattr(persisted_record, "refresh_attempt_id") == getattr( - started_record, "refresh_attempt_id" + assert getattr(persisted_record, "refresh_attempt_id") == getattr( # noqa: B009 + started_record, + "refresh_attempt_id", # noqa: B009 ) caplog.clear() diff --git a/backend/tests/unit/onyx/tracing/test_braintrust_incognito_suppression.py b/backend/tests/unit/onyx/tracing/test_braintrust_incognito_suppression.py new file mode 100644 index 00000000000..9284e0a6c4a --- /dev/null +++ b/backend/tests/unit/onyx/tracing/test_braintrust_incognito_suppression.py @@ -0,0 +1,80 @@ +"""Incognito turns must leave no content in Braintrust: the processor drops +the whole trace, spans included, keyed on membership recorded at trace start.""" + +from collections.abc import Generator +from unittest.mock import MagicMock + +import pytest + +from onyx.tracing.braintrust_tracing_processor import BraintrustTracingProcessor +from shared_configs.contextvars import CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR + + +@pytest.fixture +def incognito_context() -> Generator[None, None, None]: + token = CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.set("usage_only") + yield + CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.reset(token) + + +def _fake_trace(trace_id: str) -> MagicMock: + trace = MagicMock() + trace.trace_id = trace_id + trace.name = "run_llm_loop" + trace.export.return_value = {} + return trace + + +def _fake_span(trace_id: str, span_id: str) -> MagicMock: + span = MagicMock() + span.trace_id = trace_id + span.span_id = span_id + span.parent_id = None + return span + + +def test_incognito_trace_is_fully_suppressed( + incognito_context: None, # noqa: ARG001 (requested for the flag side-effect) +) -> None: + logger = MagicMock() + processor = BraintrustTracingProcessor(logger=logger) + trace = _fake_trace("t1") + span = _fake_span("t1", "s1") + + processor.on_trace_start(trace) + processor.on_span_start(span) + processor.on_span_end(span) + processor.on_trace_end(trace) + + logger.start_span.assert_not_called() + assert processor._spans == {} + assert processor._suppressed_traces == set() + + +def test_suppression_holds_even_if_flag_clears_mid_trace() -> None: + """Membership at trace start decides, so a reset contextvar cannot leak + the tail of an incognito trace.""" + logger = MagicMock() + processor = BraintrustTracingProcessor(logger=logger) + trace = _fake_trace("t1") + span = _fake_span("t1", "s1") + + token = CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.set("usage_only") + processor.on_trace_start(trace) + CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.reset(token) + + processor.on_span_start(span) + processor.on_span_end(span) + processor.on_trace_end(trace) + + logger.start_span.assert_not_called() + assert processor._suppressed_traces == set() + + +def test_regular_trace_still_logs() -> None: + logger = MagicMock() + processor = BraintrustTracingProcessor(logger=logger) + + processor.on_trace_start(_fake_trace("t2")) + + logger.start_span.assert_called_once() diff --git a/backend/tests/unit/onyx/tracing/test_langfuse_incognito_suppression.py b/backend/tests/unit/onyx/tracing/test_langfuse_incognito_suppression.py new file mode 100644 index 00000000000..e47221c9066 --- /dev/null +++ b/backend/tests/unit/onyx/tracing/test_langfuse_incognito_suppression.py @@ -0,0 +1,60 @@ +"""Incognito turns must leave no content in Langfuse: the processor drops the +whole trace, spans included, keyed on membership recorded at trace start.""" + +from collections.abc import Generator +from unittest.mock import MagicMock + +import pytest + +from onyx.tracing.langfuse_tracing_processor import LangfuseTracingProcessor +from shared_configs.contextvars import CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR + + +@pytest.fixture +def incognito_context() -> Generator[None, None, None]: + token = CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.set("usage_only") + yield + CURRENT_INCOGNITO_RECORD_MODE_CONTEXTVAR.reset(token) + + +def _fake_trace(trace_id: str) -> MagicMock: + trace = MagicMock() + trace.trace_id = trace_id + trace.name = "run_llm_loop" + trace.export.return_value = {} + return trace + + +def _fake_span(trace_id: str, span_id: str) -> MagicMock: + span = MagicMock() + span.trace_id = trace_id + span.span_id = span_id + span.parent_id = None + return span + + +def test_incognito_trace_is_fully_suppressed( + incognito_context: None, # noqa: ARG001 (requested for the flag side-effect) +) -> None: + client = MagicMock() + processor = LangfuseTracingProcessor(client=client) + trace = _fake_trace("t1") + span = _fake_span("t1", "s1") + + processor.on_trace_start(trace) + processor.on_span_start(span) + processor.on_span_end(span) + processor.on_trace_end(trace) + + client.start_observation.assert_not_called() + assert processor._suppressed_traces == set() + + +def test_ordinary_trace_still_exports() -> None: + client = MagicMock() + processor = LangfuseTracingProcessor(client=client) + trace = _fake_trace("t2") + + processor.on_trace_start(trace) + + client.start_observation.assert_called_once() diff --git a/backend/tests/unit/onyx/utils/test_threadpool_contextvars.py b/backend/tests/unit/onyx/utils/test_threadpool_contextvars.py index c087259e177..8d0d54819bc 100644 --- a/backend/tests/unit/onyx/utils/test_threadpool_contextvars.py +++ b/backend/tests/unit/onyx/utils/test_threadpool_contextvars.py @@ -46,7 +46,7 @@ def test_run_functions_in_parallel_preserves_contextvar() -> None: # Run in parallel and verify all results have the correct value results = run_functions_in_parallel(function_calls) - for result_id, value in results.items(): + for _result_id, value in results.items(): assert value == "parallel_test" diff --git a/backend/tests/unit/sandbox_proxy/test_gate.py b/backend/tests/unit/sandbox_proxy/test_gate.py index 28e99bfdb51..fbf85ae55f2 100644 --- a/backend/tests/unit/sandbox_proxy/test_gate.py +++ b/backend/tests/unit/sandbox_proxy/test_gate.py @@ -154,6 +154,15 @@ def _assert_403(flow: http.HTTPFlow, expected_code: SandboxProxyError) -> None: _MATCH = make_matched_actions(payload={"text": "hi"}) +_MATCH_MCP = AllMatchedActions( + actions=_MATCH.actions, + target=GatedTarget( + kind=GatedAppKind.MCP_SERVER, + id=73, + app_name="Linear MCP", + ), + payload=_MATCH.payload, +) _MATCH_MULTI_ASK = AllMatchedActions( actions=( MatchedAction( @@ -561,16 +570,19 @@ async def test_ask_denied_blocks( _RUN_ID = UUID("55555555-5555-5555-5555-555555555555") # make_matched_actions defaults external_app_id=42. _GRANTED_APP_ID = 42 +_GRANTED_MCP_SERVER_ID = 73 def _stub_grants( monkeypatch: pytest.MonkeyPatch, result: tuple[UUID, list[int]] | None | Exception, + *, + kind: GatedAppKind = GatedAppKind.EXTERNAL_APP, ) -> list[UUID]: """Stub the gate's grant lookup; returns the recorded session_ids. - Callers pass granted external-app ids; the stub wraps them as the - ``(kind, id)`` targets the real lookup now returns.""" + The stub wraps target ids as the ``(kind, id)`` keys returned by the real + lookup.""" calls: list[UUID] = [] def _lookup( @@ -583,8 +595,8 @@ def _lookup( raise result if result is None: return None - run_id, app_ids = result - return run_id, {(GatedAppKind.EXTERNAL_APP, app_id) for app_id in app_ids} + run_id, target_ids = result + return run_id, {(kind, target_id) for target_id in target_ids} monkeypatch.setattr(gate, "get_live_scheduled_run_grants", _lookup) return calls @@ -643,17 +655,27 @@ def set( @pytest.mark.asyncio +@pytest.mark.parametrize( + ("matched_actions", "target_kind", "target_id"), + [ + (_MATCH, GatedAppKind.EXTERNAL_APP, _GRANTED_APP_ID), + (_MATCH_MCP, GatedAppKind.MCP_SERVER, _GRANTED_MCP_SERVER_ID), + ], + ids=["external-app", "mcp-server"], +) async def test_pre_approved_scheduled_run_skips_park( monkeypatch: pytest.MonkeyPatch, + matched_actions: AllMatchedActions, + target_kind: GatedAppKind, + target_id: int, ) -> None: - """Granted app on a RUNNING scheduled run: forwarded immediately with a - pre-decided APPROVED row — the park pipeline never runs.""" + """A granted target skips the approval park during a scheduled run.""" user_id = uuid4() sandbox = make_resolved_sandbox(user_id=user_id) resolver = StubResolver(sandbox=sandbox, session_by_id=UUID(_TAG_UUID)) - addon = _build(resolver=resolver, matcher=_StubMatcher(result=_MATCH)) + addon = _build(resolver=resolver, matcher=_StubMatcher(result=matched_actions)) spy = _spy_pipeline(addon, monkeypatch) - _stub_grants(monkeypatch, (_RUN_ID, [_GRANTED_APP_ID])) + _stub_grants(monkeypatch, (_RUN_ID, [target_id]), kind=target_kind) inserted = _spy_pre_approve_insert(monkeypatch) notified: list[dict[str, Any]] = [] monkeypatch.setattr(gate, "create_notification", lambda **kw: notified.append(kw)) @@ -664,17 +686,17 @@ async def test_pre_approved_scheduled_run_skips_park( assert flow.response is None # forwarded assert not spy.approval_ran # park pipeline skipped assert spy.awaited == [] - assert spy.dispatched == [(_MATCH, user_id, sandbox.tenant_id)] + assert spy.dispatched == [(matched_actions, user_id, sandbox.tenant_id)] assert len(inserted) == 1 assert inserted[0]["decision"] == ApprovalDecision.APPROVED assert inserted[0]["decided_via"] == ApprovalDecidedVia.PRE_APPROVAL - assert inserted[0]["target"] == (GatedAppKind.EXTERNAL_APP, _GRANTED_APP_ID) - # Dedup contract: additional_data is exactly the stable (run, app) pair. + assert inserted[0]["target"] == (target_kind, target_id) + # Dedup contract: additional_data is exactly the stable run and target. assert len(notified) == 1 assert notified[0]["additional_data"] == { "run_id": str(_RUN_ID), - "target_kind": GatedAppKind.EXTERNAL_APP.value, - "target_id": _GRANTED_APP_ID, + "target_kind": target_kind.value, + "target_id": target_id, } diff --git a/backend/tests/unit/sandbox_proxy/test_response_streaming.py b/backend/tests/unit/sandbox_proxy/test_response_streaming.py index 733fc6432f5..e6095cfb4e1 100644 --- a/backend/tests/unit/sandbox_proxy/test_response_streaming.py +++ b/backend/tests/unit/sandbox_proxy/test_response_streaming.py @@ -188,7 +188,11 @@ def _start_proxy( holder: dict[str, Any] = {} ready = threading.Event() - async def _amain(bind_port: int) -> None: + async def _amain( + bind_port: int, + holder: dict[str, Any] = holder, + ready: threading.Event = ready, + ) -> None: options = Options( listen_host="127.0.0.1", listen_port=bind_port, @@ -207,7 +211,9 @@ async def _amain(bind_port: int) -> None: ) thread.start() - def _stop() -> None: + def _stop( + holder: dict[str, Any] = holder, thread: threading.Thread = thread + ) -> None: loop = holder.get("loop") master = holder.get("master") if loop is not None and master is not None: diff --git a/docs/craft/features/egress-proxy-and-approvals/README.md b/docs/craft/features/egress-proxy-and-approvals/README.md index a0d2c4ae59e..03dc6c9d5eb 100644 --- a/docs/craft/features/egress-proxy-and-approvals/README.md +++ b/docs/craft/features/egress-proxy-and-approvals/README.md @@ -42,7 +42,7 @@ Proxy runtime: Approval persistence and API: - `backend/onyx/db/models.py` defines `ActionApproval`, - `ExternalAppPolicy`, and `ScheduledTaskPreApprovedApp`. + `ExternalAppPolicy`, and `ScheduledTaskPreApprovedTarget`. - `backend/onyx/db/enums.py` defines `EndpointPolicy`, `ApprovalDecision`, and `ApprovalDecidedVia`. - `backend/onyx/server/features/build/db/action_approval.py` owns approval DB @@ -451,8 +451,8 @@ Partial coverage still parks. ### Scheduled Task Pre-Approvals Scheduled task pre-approvals are another grant source inside the same proxy -approval path. A task can store a set of pre-approved external app ids in -`scheduled_task_pre_approved_app`. +approval path. A task can store external-app and MCP-server grants in the +legacy-named `scheduled_task_pre_approved_app` table. For an `ASK` request, the gate checks grant sources in this order: @@ -463,7 +463,7 @@ A scheduled-task grant applies only when: - The `BuildSession` has a `ScheduledTaskRun` row. - That run is currently `RUNNING`. -- The task has a grant for the matched `external_app_id`. +- The task has a grant for the matched `(target kind, target id)`. When it applies, the proxy inserts an `action_approval` row already `APPROVED` with `decided_via=PRE_APPROVAL`, emits a deduped scheduled-task diff --git a/docs/craft/features/scheduled-tasks/pre-approvals.md b/docs/craft/features/scheduled-tasks/pre-approvals.md index 833bc7a98b5..392ef5c3237 100644 --- a/docs/craft/features/scheduled-tasks/pre-approvals.md +++ b/docs/craft/features/scheduled-tasks/pre-approvals.md @@ -125,20 +125,22 @@ unguarded after an `APPROVED` row is already committed. ## Data Model -New table `scheduled_task_pre_approved_app` — one row per `(task, app)` -grant: +The legacy-named `scheduled_task_pre_approved_app` table stores one row per +`(task, target)` grant: -- `scheduled_task_id` → `scheduled_task.id` (`ON DELETE CASCADE`) and - `external_app_id` → `external_app.id` (`ON DELETE CASCADE`), with a - `UNIQUE(scheduled_task_id, external_app_id)` constraint that keeps - grants idempotent and serves the per-task lookup. The FKs give real - referential integrity — a grant can't point at a removed app, and - removing either side drops the grant. `ScheduledTask.pre_approved_apps` - is the ORM collection; `pre_approved_app_ids` is a read-only accessor - over it, so the API contract (`list[int]`) is unchanged. The write - path replaces the whole set (`set_pre_approved_apps`), validated - against the configured apps (via the tenant-scoped session) and - deduped order-preserving. +- `scheduled_task_id` references `scheduled_task.id` with `ON DELETE CASCADE`. +- `gated_app_id` references the polymorphic `gated_app.id` with + `ON DELETE CASCADE`. Each `gated_app` row identifies exactly one external app + or MCP server. +- `UNIQUE(scheduled_task_id, gated_app_id)` keeps grants idempotent and serves + the per-task lookup. +- `ScheduledTask.pre_approved_targets` is the ORM collection of + `ScheduledTaskPreApprovedTarget` rows. The + `pre_approved_external_app_ids` and `pre_approved_mcp_server_ids` properties + project each target kind into its API field. +- `_replace_pre_approved_targets` replaces each supplied target kind and + preserves omitted kinds. It deduplicates submitted IDs and reuses unchanged + rows. - `action_approval.decided_via` — nullable (`user | pre_approval`, NULL for legacy/expired rows): the audit marker distinguishing a human click from a pre-approval. Kept separate from `decision` so @@ -160,10 +162,10 @@ pre-decided inserts go through `insert_action_approval` in - `ScheduledTaskCreate` / `ScheduledTaskPatch` gain `pre_approved_app_ids: list[int]`; `ScheduledTaskDetail` returns it. - The write path validates ids via `_validated_app_ids` and dedupes - (order-preserving) — existence only; a credential / ≥1-`ASK` filter is - editor-side advisory, since a grant on a no-`ASK` app is inert and - never consulted. + The write path validates ids via `_validate_app_ids` and stores each grant + once. Grant order has no meaning. Validation checks existence only; a + credential / ≥1-`ASK` filter is editor-side advisory because a grant on a + no-`ASK` app is inert and never consulted. - New `NotificationType.SCHEDULED_TASK_PRE_APPROVED_ACTION`, emitted per `(run, app)` on the first unattended forward so chatty tasks don't flood the bell. Dedup rides `create_notification`'s existing @@ -254,7 +256,7 @@ The grant-source seam means future modes drop in as new - Grant patch semantics: a prompt edit preserves grants, supplied `pre_approved_app_ids` replaces the set, and re-submitting an existing grant is idempotent (no unique-key collision). - - Create persistence + `_validated_app_ids` dedupe and unknown-id + - Create persistence, duplicate grant normalization, and unknown-id rejection. - **Unit** (gate, stubbed DB): `backend/tests/unit/sandbox_proxy/test_gate.py` diff --git a/docs/usage/usage-reports.md b/docs/usage/usage-reports.md new file mode 100644 index 00000000000..553ecfbe475 --- /dev/null +++ b/docs/usage/usage-reports.md @@ -0,0 +1,273 @@ +# Usage reports: what they are for + +This document defines what an Onyx usage report must tell an organization admin, +and why. It starts from the admin's job, not from the data we happen to store. + +The rule that governs every decision here: **a number belongs in the report only +if the admin can finish the sentence "so I will...".** If no action follows, cut +the number. + +## Where we are today + +`create_new_usage_report` builds a zip with three CSVs and a PDF review pack: + +| File | Contents | +| -------------------- | ------------------------------------------------------------------------------------- | +| `chat_messages.csv` | One row per message: session, user, flow, time, agent, email, tokens, model | +| `users.csv` | `user_id`, `is_active` | +| `usage_by_user.csv` | Per user, per day, per model/flow/provider: tokens, cache reads, cost | +| `usage_report.pdf` | Summary of spend, adoption, seats, and usage attribution | + +The raw CSV files are still a data dump. They have three problems: + +1. **It answers no question.** The admin must build every pivot. +2. **It has no dimensions the admin budgets by.** No team, no agent, no source. +3. **It joins badly.** `users.csv` has no email, so it cannot join to the other files. + +The raw export is still valuable. It is just the wrong artifact for every job. + +## The admin's job + +An org admin is accountable for three things: + +1. **Money.** They signed the contract. They own the spend. +2. **Adoption.** They championed the rollout. Someone will ask if it worked. +3. **Risk.** If the tool leaks or misbehaves, it lands on them. + +Every useful metric comes from one of these. The sections below derive the +metrics from the decisions. + +### Decision: how many seats do I buy at renewal? + +The admin needs a seat ledger, not a message count. + +- Licensed seats, provisioned seats, and active seats. +- A named list of users who hold a seat and did not use it in 30 days. +- The inverse list: users who hit rate limits. + +The named lists are the deliverable. The admin acts on them directly. They +reclaim a seat, retrain the person, or drop the seat at renewal. + +### Decision: am I overspending, and what can I cut? + +Total cost drives no action. Cost **concentration** does. + +- Share of spend from the top 5 users, the top 3 agents, and the top model. +- Cost per active user per month. This is the one number finance accepts. +- Spend sent to an expensive model for work a cheap model handles. +- Money already saved by prompt caching. We store `cache_read_tokens` today. + +### Decision: who pays for it? + +Cost split by team or user group. Most orgs must charge the cost back, or at +least explain the invoice internally. An admin who cannot attribute cost to a +cost center cannot grow the deployment. This blocks expansion. + +`User__UserGroup` already gives us the join. + +### Decision: did the rollout work? + +Message volume is a trap. It rises when three people go heavy. + +Measure breadth first, then habit, then depth: + +- Distinct humans who used Onyx in the period. +- How many use it every week, and whether that count rises. +- Multi-turn sessions versus one-question-and-leave. +- The funnel: invited, first message, five messages, weekly habit. + +The drop-off point in the funnel tells the admin what to fix. A drop before the +first message means onboarding. A drop after it means answer quality. + +### Decision: is the quality good? + +The admin cannot read conversations. Give them proxies, and give them trends. +Nobody knows what a good absolute thumbs-down rate is. + +- Negative feedback rate. +- Regeneration rate. +- Sessions abandoned after one answer. +- Answers where retrieval returned nothing. + +### Decision: what do I do next to improve it? + +This section is the most actionable, and it does not exist today. + +- Connectors that are indexed but never cited. +- Topics that people ask often and Onyx answers badly. +- Agents that nobody uses. + +Each item maps to one concrete action: add a source, write a document, delete +an agent. + +### Decision: am I exposed? + +Admins do not want a security console here. They want a short "look at this" +list, with names on it. + +- A user whose usage jumped 10x. +- Access to sensitive sources. +- Traffic from a service account or API key, not a human. +- Activity from an employee who left and kept an account. + +## The flagship: the knowledge gap report + +Every vendor can report spend and logins. Only Onyx knows **what the +organization tries to learn and fails to find.** + +A monthly artifact that lists the top unanswered questions, clustered by topic, +with the missing sources named, is worth more than the full cost breakdown. It +tells the admin something about their own company that they cannot get anywhere +else. It also makes the strongest renewal argument that exists. + +Treat this as the headline of the report, not an extra tab. + +## Three artifacts, not one zip + +The current report tries to be one thing. The admin needs three, at three +cadences. This is the main structural change. + +| Artifact | Cadence | Form | Purpose | +| ----------------------- | --------- | --------------------------------- | -------------------------------- | +| **Pulse** | Monthly | Pushed to email or Slack, no download | Tell the admin if anything changed | +| **Review pack** | Quarterly | A PDF the admin forwards to their boss | Defend the spend and the rollout | +| **Investigation export**| On demand | The raw CSVs we build today | Answer a specific question, feed BI | + +The pulse must be pushed. An artifact that needs a download and a spreadsheet +gets read once. + +## The one screen + +If the pulse were a single screen, it holds eight items: + +1. Active users, and the change. +2. Spend, and the change. +3. Cost per active user. +4. Percent of seats dormant, with the list. +5. Top 3 cost concentrations. +6. Quality trend, one line. +7. Top 5 unanswered topics. +8. One anomaly callout. + +Everything else lives one click deeper. + +## The PDF review pack + +The review pack must be a PDF. An admin forwards it to a VP or a CFO. Those +people do not open a zip of CSVs, and they do not log in to an admin panel. The +PDF is the artifact that travels. + +### Build it in pure Python with ReportLab + +Use **ReportLab** (BSD licensed). Write the document in Python. Do not template +markdown, do not render HTML, and do not drive a browser. + +ReportLab supplies everything the pack needs: + +- `platypus` flows content into pages. It splits long tables across pages and + repeats the header row. +- `graphics.charts` draws bar, line, and pie charts as native PDF vectors. +- The PDF base-14 fonts (Helvetica, Times) need no embedding. A brand font + works too, but vendor the TTF in the repo. Never fetch a font at render time. + +Measured on a representative pack (40-row table across 2 pages, one line chart, +one bar chart): **13 ms, 4.4 KB**. No subprocess, no browser. + +### The shared layer is the data model, not the text + +Build one typed aggregate object, and give each output its own renderer: + +| Output | Renderer | +| ------------ | --------------------------------- | +| PDF | ReportLab document builder | +| Email digest | HTML with inlined styles | +| Slack digest | Slack blocks | +| CSV rollups | The existing writers | + +All four read the same aggregate object, so the numbers always agree. Sharing a +data model is stronger than sharing a markdown string. Markdown cannot express a +chart, a page break, or a repeated table header, so it would have leaked layout +concerns into the shared layer anyway. + +### Pipeline + +1. Query the aggregates into the typed object. Same queries as the CSV rollups. +2. Build the ReportLab document from that object. +3. Save the PDF to the file store next to the zip, under the same `report_id`. + +This runs in the existing Celery report task. + +### Alternatives considered + +- **Chromium through Playwright.** It works, and it needs no new dependency, + because the image already installs Chromium (`backend/Dockerfile:124`) and + already launches it for the web connector. It also renders correctly with no + network (verified below). It still loses: a browser launch costs about 470 ms + and a few hundred MB of RSS, versus 13 ms for ReportLab, and it drags a + browser process, a template layer, and hand-written SVG into the report path. +- **WeasyPrint.** Adds a Python dependency plus pango system libraries, and + still makes us write CSS to control pagination. +- **Markdown plus Jinja2.** A lossy intermediate format. It cannot express + charts or pagination, so every hard part would still need solving elsewhere. + +### Air-gap status + +Verified inside the shipped image with `docker run --network none`: + +- Chromium launches and prints a PDF with tables, inline SVG, and real fonts. + Text extracts correctly. So the browser path is air-gap safe if we ever want + it. +- ReportLab needs no network by construction, and the base-14 fonts live inside + the PDF spec. + +Note that the air-gap CI job only starts `api_server`, `inference_model_server`, +and `minio`. It does not exercise the background worker or a browser. Any +air-gap claim for a new render path needs its own check. + +### Constraints to respect + +- Determinism. The same period must produce the same PDF. Do not stamp a + render timestamp inside the content body. +- Size. Cap the page count. The pack is a summary, not the export. +- Anonymized mode. The PDF must honor it, the same as the CSVs. +- Failure isolation. A PDF failure must not fail the zip. Generate them + independently. + +## Anti-metrics + +Do not put these in a report. They look informative and drive no decision: + +- Total messages, total tokens, total sessions. +- Average response time. +- Any cumulative all-time count. + +## What the raw export still needs + +The investigation export stays. Fix its defects: + +- Add `summary.csv`, `manifest.json` (schema version, period, timezone, row + counts, generator version), and a `README.md` that defines every column. +- Add pre-built rollups: by team, by agent, by model, by day. +- Fix `users.csv`: email, role, groups, created date, last active, seat state. +- State the units. Name the currency. Snapshot the price table, so the numbers + stay reproducible after prices change. +- Warn when the period is incomplete. If the usage rollup started after the + period start, say so on the report. Otherwise a partial month reads as a + full one. +- Support an anonymized mode. Some EU customers cannot legally receive per + person usage. Hash the email with a per-report salt. + +## Build order + +Ordered by value per unit of work. The first two are independent of each other. + +1. **Seat ledger.** Fix `users.csv` and derive the dormant-seat list. Small + change, and it makes every other file joinable. +2. **Knowledge gap report.** The differentiated artifact. +3. **Team and agent dimensions** on the cost breakdown. Unblocks chargeback. +4. **Summary, manifest, and README** on the export. +5. **Extend the PDF review pack.** The shipped pack summarizes spend and + adoption by person, model, and flow. Add the missing dimensions from steps + 1 to 3 as they become available. +6. **Scheduled pulse** to email or Slack. The Celery beat already exists, and + the existing typed aggregate object supplies the content. diff --git a/pyproject.toml b/pyproject.toml index 40d8265304a..64345c85b87 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -99,6 +99,8 @@ backend = [ "pywikibot==11.4.2", "readerwriterlock==1.0.9", "redis==5.0.8", + # PDF generation for the usage report review pack. + "reportlab==5.0.0", "requests==2.33.0", "requests-oauthlib==2.0.0", "simple-salesforce==1.12.6", @@ -140,7 +142,7 @@ dev = [ "manygo==0.2.0", "matplotlib==3.10.8", "ty==0.0.63", - "onyx-devtools==0.10.6", + "onyx-devtools==0.11.0", "openapi-generator-cli==7.17.0", "pre-commit==3.2.2", "pytest-alembic==0.12.1", @@ -285,9 +287,24 @@ ignore = [ # Tracked in kanban ticket #491 — to be removed once existing violations are # cleaned up: "S113", # request-without-timeout (~463 existing violations). + # flake8-bugbear (B) rule deferred — to be removed once existing violations + # are cleaned up: + "B904", # raise-without-from-inside-except (~465 existing violations). ] # G004: f-strings in logging break Sentry's message-pattern grouping. -select = ["ARG", "E", "F", "G004", "I", "S", "W"] +select = ["ARG", "B", "E", "F", "G004", "I", "S", "W"] + +[tool.ruff.lint.flake8-bugbear] +# FastAPI dependency-injection markers are the intended way to write these +# defaults, so B008 must not flag them. +extend-immutable-calls = [ + "fastapi.Body", + "fastapi.Depends", + "fastapi.File", + "fastapi.Form", + "fastapi.Query", + "onyx.auth.permissions.require_permission", +] [tool.ruff.lint.isort] known-first-party = ["onyx", "ee", "tests", "shared_configs", "model_server"] diff --git a/tools/ods/cmd/cherry-pick.go b/tools/ods/cmd/cherry-pick.go index d38b430a5e9..40d4d2378cf 100644 --- a/tools/ods/cmd/cherry-pick.go +++ b/tools/ods/cmd/cherry-pick.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "regexp" + "sort" "strconv" "strings" @@ -45,7 +46,8 @@ with fewer than 6 digits is treated as a PR number and resolved to its merge commit automatically. This command will: - 1. Find the nearest stable version tag + 1. Detect the newest release branch that does not already contain the commit + (unless --release is given) 2. Fetch the corresponding release branch(es) 3. Create a hotfix branch with the cherry-picked commit(s) 4. Push and create a PR using the GitHub CLI @@ -172,11 +174,11 @@ func runCherryPick(cmd *cobra.Command, args []string, opts *CherryPickOptions) { } log.Debugf("Using specified release versions: %v", releases) } else { - // Find the nearest stable tag using the first commit - version, err := findNearestStableTag(commitSHAs[0]) + // Find the newest release branch missing the first commit. + version, err := findTargetReleaseVersion(commitSHAs[0]) if err != nil { git.RestoreStash(stashResult) - log.Fatalf("Failed to find nearest stable tag: %v", err) + log.Fatalf("Failed to auto-detect the target release: %v", err) } // Prompt user for confirmation @@ -388,7 +390,7 @@ func cherryPickToRelease(commitSHAs, commitMessages []string, branchSuffix, vers // Fetch the release branch log.Infof("Fetching release branch: %s", releaseBranch) - if err := git.RunCommand("fetch", "--prune", "--quiet", "origin", releaseBranch); err != nil { + if err := git.RunCommand("fetch", "--prune", "--quiet", "origin", releaseBranchRefspec(releaseBranch)); err != nil { return "", fmt.Errorf("failed to fetch release branch %s: %w", releaseBranch, err) } @@ -570,26 +572,129 @@ func extractPRNumbers(commitMsg string) []string { return matches } -// findNearestStableTag finds the nearest tag matching v*.*.* pattern and returns major.minor -func findNearestStableTag(commitSHA string) (string, error) { - // Get tags that are ancestors of the commit, sorted by version - cmd := exec.Command("git", "describe", "--tags", "--abbrev=0", "--match", "v*.*.*", commitSHA) +// releaseBranchPattern matches maintained release branch names, e.g. +// "release/v4.5". Ad-hoc branches such as "release/v3.0-qa-f1df36e" are +// deliberately excluded. +var releaseBranchPattern = regexp.MustCompile(`^release/v(\d+)\.(\d+)$`) + +// releaseBranchRefspec returns a forced fetch refspec that creates or updates +// the origin/ tracking ref even in clones whose configured fetch +// refspec does not cover release branches (e.g. single-branch clones), where a +// plain "git fetch origin " only writes FETCH_HEAD. +func releaseBranchRefspec(releaseBranch string) string { + return fmt.Sprintf("+refs/heads/%s:refs/remotes/origin/%s", releaseBranch, releaseBranch) +} + +// releaseVersion is the parsed version of a "release/vX.Y" branch. +type releaseVersion struct { + major int + minor int +} + +// String returns the version with its 'v' prefix, e.g. "v4.5". +func (v releaseVersion) String() string { + return fmt.Sprintf("v%d.%d", v.major, v.minor) +} + +// parseReleaseVersions extracts "release/vX.Y" versions from branch names and +// returns them sorted newest first. Names that do not match the pattern are +// ignored. +func parseReleaseVersions(branchNames []string) []releaseVersion { + versions := []releaseVersion{} + for _, name := range branchNames { + matches := releaseBranchPattern.FindStringSubmatch(name) + if matches == nil { + continue + } + major, err := strconv.Atoi(matches[1]) + if err != nil { + continue + } + minor, err := strconv.Atoi(matches[2]) + if err != nil { + continue + } + versions = append(versions, releaseVersion{major: major, minor: minor}) + } + sort.Slice(versions, func(i, j int) bool { + if versions[i].major != versions[j].major { + return versions[i].major > versions[j].major + } + return versions[i].minor > versions[j].minor + }) + return versions +} + +// listRemoteReleaseBranches returns the names (e.g. "release/v4.5") of all +// release branches on origin. +func listRemoteReleaseBranches() ([]string, error) { + cmd := exec.Command("git", "ls-remote", "--heads", "origin", "release/*") output, err := cmd.Output() if err != nil { - return "", fmt.Errorf("git describe failed: %w", err) + if exitErr, ok := err.(*exec.ExitError); ok { + return nil, fmt.Errorf("git ls-remote failed: %w: %s", err, string(exitErr.Stderr)) + } + return nil, fmt.Errorf("git ls-remote failed: %w", err) } - tag := strings.TrimSpace(string(output)) - log.Debugf("Found tag: %s", tag) + branches := []string{} + for _, line := range strings.Split(string(output), "\n") { + // Each line is "\trefs/heads/". + _, ref, found := strings.Cut(line, "\t") + if !found { + continue + } + branches = append(branches, strings.TrimPrefix(strings.TrimSpace(ref), "refs/heads/")) + } + return branches, nil +} + +// findTargetReleaseVersion returns the version (e.g. "v4.5") of the newest +// "release/vX.Y" branch on origin that does not already contain commitSHA. A +// commit merged to main after the latest branch cut targets the newest branch; +// a commit that predates the cut (and is therefore already part of the newer +// branches) falls back to the newest branch actually missing it. Tags are +// deliberately not consulted: release tag names on main (e.g. "vX.Y.0-cloud.N") +// roll over to a new version asynchronously from the branch cut, so the nearest +// tag can disagree with the newest branch. +func findTargetReleaseVersion(commitSHA string) (string, error) { + // A shallow clone cannot answer ancestry truthfully: history beyond the + // shallow boundary makes contained commits look uncontained, silently + // routing them to the wrong branch. Fail loudly instead. + shallow, err := git.IsShallowRepository() + if err != nil { + return "", err + } + if shallow { + return "", fmt.Errorf("this is a shallow clone, so release auto-detection cannot check branch ancestry; pass --release explicitly") + } - // Extract major.minor with v prefix from tag (e.g., v1.2.3 -> v1.2) - re := regexp.MustCompile(`^(v\d+\.\d+)\.\d+`) - matches := re.FindStringSubmatch(tag) - if len(matches) < 2 { - return "", fmt.Errorf("tag %s does not match expected format v*.*.* ", tag) + branchNames, err := listRemoteReleaseBranches() + if err != nil { + return "", err + } + versions := parseReleaseVersions(branchNames) + if len(versions) == 0 { + return "", fmt.Errorf("no release/vX.Y branches found on origin") + } + + for _, version := range versions { + releaseBranch := fmt.Sprintf("release/%s", version) + // Fetch so the ancestry check runs against the branch's current tip. + if err := git.RunCommand("fetch", "--quiet", "origin", releaseBranchRefspec(releaseBranch)); err != nil { + return "", fmt.Errorf("failed to fetch %s: %w", releaseBranch, err) + } + contained, err := git.IsAncestor(commitSHA, fmt.Sprintf("origin/%s", releaseBranch)) + if err != nil { + return "", err + } + if !contained { + return version.String(), nil + } + log.Infof("Commit %s is already contained in %s, checking the next older release branch", commitSHA, releaseBranch) } - return matches[1], nil + return "", fmt.Errorf("commit %s is already contained in every release branch; pass --release explicitly", commitSHA) } // createCherryPickPR creates a pull request for cherry-picks using the GitHub CLI diff --git a/tools/ods/cmd/cherry-pick_test.go b/tools/ods/cmd/cherry-pick_test.go new file mode 100644 index 00000000000..e1239390da8 --- /dev/null +++ b/tools/ods/cmd/cherry-pick_test.go @@ -0,0 +1,189 @@ +package cmd + +import ( + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestParseReleaseVersions_sortsNewestFirstIgnoringNonMatching(t *testing.T) { + // Precondition. + branchNames := []string{ + "release/v4.4", + "release/v3.0-qa-f1df36e", + "release/v4.10", + "main", + "release/v4.5", + "release/v10.0", + } + + // Under test. + versions := parseReleaseVersions(branchNames) + + // Postcondition. + got := make([]string, len(versions)) + for i, version := range versions { + got[i] = version.String() + } + want := []string{"v10.0", "v4.10", "v4.5", "v4.4"} + if !slices.Equal(got, want) { + t.Errorf("expected %v, got %v", want, got) + } +} + +func TestParseReleaseVersions_emptyWhenNothingMatches(t *testing.T) { + // Under test and postcondition. + if versions := parseReleaseVersions([]string{"main", "hotfix/abc-v4.4"}); len(versions) != 0 { + t.Errorf("expected no versions, got %v", versions) + } +} + +// gitIn runs a git command in dir, failing the test on error. +func gitIn(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// commitIn creates a file and commits it in dir, returning the commit SHA. +func commitIn(t *testing.T, dir, filename string) string { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, filename), []byte(filename), 0644); err != nil { + t.Fatal(err) + } + gitIn(t, dir, "add", filename) + gitIn(t, dir, "commit", "-m", "add "+filename) + return gitIn(t, dir, "rev-parse", "HEAD") +} + +// setupReleaseBranchRepo creates a bare origin holding main, release/v4.4, and +// release/v4.5, with a local work repo as the current directory. It returns +// three main-line commit SHAs: the v4.4 cut point (ancestor of both release +// branches), the v4.5 cut point (only on release/v4.5), and a post-cut commit +// (on neither release branch). +func setupReleaseBranchRepo(t *testing.T) (preCutSHA, cutSHA, postCutSHA string) { + t.Helper() + + origin := t.TempDir() + gitIn(t, origin, "init", "--bare", "-b", "main") + + work := t.TempDir() + gitIn(t, work, "init", "-b", "main") + gitIn(t, work, "config", "user.email", "test@test.com") + gitIn(t, work, "config", "user.name", "Test") + gitIn(t, work, "config", "commit.gpgsign", "false") + gitIn(t, work, "remote", "add", "origin", origin) + // Narrow the fetch refspec to main only, like a single-branch clone, so the + // tests also pin that detection fetches release branches with an explicit + // refspec (a plain "git fetch origin " would only write FETCH_HEAD + // here and never create origin/release/vX.Y). + gitIn(t, work, "config", "remote.origin.fetch", "+refs/heads/main:refs/remotes/origin/main") + + preCutSHA = commitIn(t, work, "a.txt") + gitIn(t, work, "branch", "release/v4.4", preCutSHA) + cutSHA = commitIn(t, work, "b.txt") + gitIn(t, work, "branch", "release/v4.5", cutSHA) + postCutSHA = commitIn(t, work, "c.txt") + + gitIn(t, work, "push", "--quiet", "origin", "main", "release/v4.4", "release/v4.5") + + // Tags named after the previous release must not influence detection + // (tag-anchored detection was the original misrouting bug). Mirror the + // incident topology: a stable v4.4 tag at the v4.4 cut point, plus a v4.4 + // pre-release tag minted on main after the v4.5 cut, which is the exact + // shape that misrouted real cherry-picks to release/v4.4. + gitIn(t, work, "tag", "v4.4.2", preCutSHA) + gitIn(t, work, "tag", "v4.4.0-cloud.9", postCutSHA) + + // Drop the local release branches and the remote-tracking refs the push + // created, so the fixture looks like a clone that has never fetched the + // release branches; detection must create origin/* itself via fetch. + gitIn(t, work, "branch", "-D", "release/v4.4", "release/v4.5") + gitIn(t, work, "update-ref", "-d", "refs/remotes/origin/release/v4.4") + gitIn(t, work, "update-ref", "-d", "refs/remotes/origin/release/v4.5") + + // The functions under test run git in the process working directory. + t.Chdir(work) + + return preCutSHA, cutSHA, postCutSHA +} + +func TestFindTargetReleaseVersion_postCutCommitTargetsNewestBranch(t *testing.T) { + // Precondition. + _, _, postCutSHA := setupReleaseBranchRepo(t) + + // Under test. + version, err := findTargetReleaseVersion(postCutSHA) + + // Postcondition. + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if version != "v4.5" { + t.Errorf("expected v4.5, got %s", version) + } +} + +func TestFindTargetReleaseVersion_preCutCommitFallsBackToOlderBranch(t *testing.T) { + // Precondition. + _, cutSHA, _ := setupReleaseBranchRepo(t) + + // Under test. + version, err := findTargetReleaseVersion(cutSHA) + + // Postcondition. + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if version != "v4.4" { + t.Errorf("expected v4.4, got %s", version) + } +} + +func TestFindTargetReleaseVersion_commitOnAllBranchesErrors(t *testing.T) { + // Precondition. + preCutSHA, _, _ := setupReleaseBranchRepo(t) + + // Under test. + _, err := findTargetReleaseVersion(preCutSHA) + + // Postcondition. + if err == nil || !strings.Contains(err.Error(), "already contained in every release branch") { + t.Errorf("expected already-contained error, got %v", err) + } +} + +func TestFindTargetReleaseVersion_shallowCloneErrors(t *testing.T) { + // Precondition: a shallow clone, where ancestry cannot be answered. + origin := t.TempDir() + gitIn(t, origin, "init", "--bare", "-b", "main") + seed := t.TempDir() + gitIn(t, seed, "init", "-b", "main") + gitIn(t, seed, "config", "user.email", "test@test.com") + gitIn(t, seed, "config", "user.name", "Test") + gitIn(t, seed, "config", "commit.gpgsign", "false") + gitIn(t, seed, "remote", "add", "origin", origin) + sha := commitIn(t, seed, "a.txt") + gitIn(t, seed, "branch", "release/v4.5") + gitIn(t, seed, "push", "--quiet", "origin", "main", "release/v4.5") + shallow := filepath.Join(t.TempDir(), "shallow") + // Depth flags are ignored for plain local-path clones, hence file://. + gitIn(t, t.TempDir(), "clone", "--quiet", "--depth", "1", "--no-single-branch", "file://"+origin, shallow) + t.Chdir(shallow) + + // Under test. + _, err := findTargetReleaseVersion(sha) + + // Postcondition. + if err == nil || !strings.Contains(err.Error(), "shallow clone") { + t.Errorf("expected shallow-clone error, got %v", err) + } +} diff --git a/tools/ods/go.mod b/tools/ods/go.mod index a0cd05321de..ccfa2c66d62 100644 --- a/tools/ods/go.mod +++ b/tools/ods/go.mod @@ -70,7 +70,7 @@ require ( github.com/go-errors/errors v1.5.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect - github.com/go-git/go-git/v5 v5.19.1 // indirect + github.com/go-git/go-git/v5 v5.19.2 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect @@ -146,16 +146,16 @@ require ( go.uber.org/zap v1.28.0 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v4 v4.0.0-rc.4 // indirect - golang.org/x/crypto v0.52.0 // indirect - golang.org/x/mod v0.36.0 // indirect - golang.org/x/net v0.55.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 // indirect - golang.org/x/term v0.43.0 // indirect - golang.org/x/text v0.37.0 // indirect - golang.org/x/tools v0.45.0 // indirect + golang.org/x/sync v0.21.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect + golang.org/x/term v0.44.0 // indirect + golang.org/x/text v0.39.0 // indirect + golang.org/x/tools v0.47.0 // indirect golang.org/x/vuln v1.3.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260511170946-3700d4141b60 // indirect diff --git a/tools/ods/go.sum b/tools/ods/go.sum index 65fb68ce051..ae2ca7ba3b3 100644 --- a/tools/ods/go.sum +++ b/tools/ods/go.sum @@ -167,8 +167,8 @@ github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmm github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= -github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -450,8 +450,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a h1:+3jdDGGB8NGb1Zktc737jlt3/A5f6UlwSzmvqUuufxw= golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -461,8 +461,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -477,8 +477,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -490,8 +490,8 @@ golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -515,11 +515,11 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -528,8 +528,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -540,8 +540,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -552,8 +552,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= diff --git a/tools/ods/internal/git/git.go b/tools/ods/internal/git/git.go index d20a4bae707..96a4735f7ff 100644 --- a/tools/ods/internal/git/git.go +++ b/tools/ods/internal/git/git.go @@ -2,6 +2,7 @@ package git import ( "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -118,6 +119,39 @@ func RestoreStash(result *StashResult) { } } +// IsAncestor reports whether ancestor is an ancestor of (or equal to) +// descendant. Both arguments may be any commit-ish. +func IsAncestor(ancestor, descendant string) (bool, error) { + cmd := exec.Command("git", "merge-base", "--is-ancestor", ancestor, descendant) + var stderr strings.Builder + cmd.Stderr = &stderr + err := cmd.Run() + if err == nil { + return true, nil + } + // Exit code 1 is the documented "not an ancestor" result; anything else + // (e.g. an unknown revision) is a real error. + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return false, nil + } + if diagnostic := strings.TrimSpace(stderr.String()); diagnostic != "" { + return false, fmt.Errorf("git merge-base --is-ancestor %s %s failed: %w: %s", ancestor, descendant, err, diagnostic) + } + return false, fmt.Errorf("git merge-base --is-ancestor %s %s failed: %w", ancestor, descendant, err) +} + +// IsShallowRepository reports whether the current repository is a shallow +// clone. +func IsShallowRepository() (bool, error) { + cmd := exec.Command("git", "rev-parse", "--is-shallow-repository") + output, err := cmd.Output() + if err != nil { + return false, fmt.Errorf("git rev-parse --is-shallow-repository failed: %w", err) + } + return strings.TrimSpace(string(output)) == "true", nil +} + // CommitExistsOnBranch checks if a commit exists on a branch func CommitExistsOnBranch(commitSHA, branchName string) bool { cmd := exec.Command("git", "branch", "--contains", commitSHA, "--list", branchName) diff --git a/tools/ods/internal/git/git_test.go b/tools/ods/internal/git/git_test.go index 156039a1902..0ef696c51a1 100644 --- a/tools/ods/internal/git/git_test.go +++ b/tools/ods/internal/git/git_test.go @@ -182,3 +182,25 @@ func TestIsCommitAppliedOnBranch_NoFalsePositiveFromBody(t *testing.T) { t.Error("should NOT match when subject only appears in body of another commit") } } + +// --- IsAncestor tests --- + +func TestIsAncestor_distinguishesFalseFromError(t *testing.T) { + // Precondition. + r := newTestRepo(t) + first := r.HEAD() + second := r.Commit("second commit", "second.txt", "content") + + // Under test and postcondition: ancestor, non-ancestor, and error cases. + contained, err := IsAncestor(first, second) + if err != nil || !contained { + t.Errorf("expected (true, nil) for ancestor, got (%v, %v)", contained, err) + } + contained, err = IsAncestor(second, first) + if err != nil || contained { + t.Errorf("expected (false, nil) for non-ancestor, got (%v, %v)", contained, err) + } + if _, err = IsAncestor("deadbeefdeadbeefdeadbeefdeadbeefdeadbeef", first); err == nil { + t.Error("expected an error for an unknown revision, got nil") + } +} diff --git a/uv.lock b/uv.lock index d27dfd39bd0..0d5fd9cb88b 100644 --- a/uv.lock +++ b/uv.lock @@ -4350,6 +4350,7 @@ backend = [ { name = "rapidfuzz" }, { name = "readerwriterlock" }, { name = "redis" }, + { name = "reportlab" }, { name = "requests" }, { name = "requests-oauthlib" }, { name = "sendgrid" }, @@ -4524,6 +4525,7 @@ backend = [ { name = "rapidfuzz", specifier = "==3.14.5" }, { name = "readerwriterlock", specifier = "==1.0.9" }, { name = "redis", specifier = "==5.0.8" }, + { name = "reportlab", specifier = "==5.0.0" }, { name = "requests", specifier = "==2.33.0" }, { name = "requests-oauthlib", specifier = "==2.0.0" }, { name = "sendgrid", specifier = "==6.12.5" }, @@ -4550,7 +4552,7 @@ dev = [ { name = "ipykernel", specifier = "==6.29.5" }, { name = "manygo", specifier = "==0.2.0" }, { name = "matplotlib", specifier = "==3.10.8" }, - { name = "onyx-devtools", specifier = "==0.10.6" }, + { name = "onyx-devtools", specifier = "==0.11.0" }, { name = "openapi-generator-cli", specifier = "==7.17.0" }, { name = "pre-commit", specifier = "==3.2.2" }, { name = "pytest", specifier = "==9.0.3" }, @@ -4600,19 +4602,19 @@ zizmor = [{ name = "zizmor", specifier = "==1.25.2" }] [[package]] name = "onyx-devtools" -version = "0.10.6" +version = "0.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "fastapi" }, { name = "openapi-generator-cli" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/45/92/99ed336a50be5ae2007b922c9bfa8a7c8ca54e80d251c8e52cab4c219b96/onyx_devtools-0.10.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:be5ada8204563223b57535659563058fd16bf6c8251dfd9b2519f563cd51cd69", size = 17812517, upload-time = "2026-07-31T17:50:45.924Z" }, - { url = "https://files.pythonhosted.org/packages/9c/bc/bace482d18563b7f2da7dde98d8819097d2109325022a02bf1756bd8ef30/onyx_devtools-0.10.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:ca208e1768f2ee39ef52763caac377ccb6e8ebda35aecfd55856e3c9c9e6e636", size = 16461951, upload-time = "2026-07-31T17:50:48.584Z" }, - { url = "https://files.pythonhosted.org/packages/eb/27/a46036a9c727e3f475cb39ce60ab81c951b432eb28cad3b1737e0a3aeeba/onyx_devtools-0.10.6-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:b7a5ebbe08e18b76bca4cf8afa9820049278a735ded9efaeb75a36f062796f00", size = 15917906, upload-time = "2026-07-31T17:50:51.052Z" }, - { url = "https://files.pythonhosted.org/packages/d5/5c/5224a051b1db3a32774ed0b492b749c41ccd8ecc066b979bef2582208681/onyx_devtools-0.10.6-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:1d8ea1058de6ba3f2f9726972a17c4c1471f8ac460421b021481d0f86ac80819", size = 17668246, upload-time = "2026-07-31T17:50:53.32Z" }, - { url = "https://files.pythonhosted.org/packages/64/c0/cce5b2a119be1123dc60fd77efe6624d9ecea47d807073a85fc60bbda494/onyx_devtools-0.10.6-py3-none-win_amd64.whl", hash = "sha256:197261d5676e89dc2cf1ec6313d293c43766e9dbf0ef80e4ff7483d6a0ab1374", size = 17881924, upload-time = "2026-07-31T17:50:55.822Z" }, - { url = "https://files.pythonhosted.org/packages/f8/5c/99053e6d3e40a62cd61efbb79ac6f63aa159854b495e93c3a85acd65250e/onyx_devtools-0.10.6-py3-none-win_arm64.whl", hash = "sha256:2405164cb2bfd379a40c7c69c320bb0244ef0163154e00ba62e4c673e514102c", size = 15960758, upload-time = "2026-07-31T17:50:58.292Z" }, + { url = "https://files.pythonhosted.org/packages/5b/80/5fc8c0a242bfcfc1451bafde6bd95fb38c5b4da87a72577824875446d456/onyx_devtools-0.11.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:73c28107648d201ac77c5ddba34808f7ce7f7033ad1d20dec01db6061283b590", size = 17831480, upload-time = "2026-08-12T18:34:23.793Z" }, + { url = "https://files.pythonhosted.org/packages/33/ee/43d1b9ce96abbca72cc494987a3d3f8ac9309e9f8aa8023f9130e2c45f41/onyx_devtools-0.11.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e49721ed1434d3781f322695e579f3427e1f85da466b5f597617f9618debf9d0", size = 16481186, upload-time = "2026-08-12T18:34:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/2c/63/f96f76d07da43937c1bde8d46c355682dcc3118b788c9606ea7d6248e72e/onyx_devtools-0.11.0-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:81821c3b65a288c01feb66d5399df99a7ddf79602cc0b50f0335939499c915ff", size = 15940233, upload-time = "2026-08-12T18:34:28.294Z" }, + { url = "https://files.pythonhosted.org/packages/9a/db/83212e6632949f53168253ae33a981cd49f736987981e73574060b4212a9/onyx_devtools-0.11.0-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:7af864b1bb61c553aad8e8ca4db085e83bcf2e722d41f6ea64e7a6b9b8b40497", size = 17692680, upload-time = "2026-08-12T18:34:30.406Z" }, + { url = "https://files.pythonhosted.org/packages/56/2a/db98c4b7b8a5c9020cf417ff55e0e7c6e73f752b0e16cc48b37109d678e7/onyx_devtools-0.11.0-py3-none-win_amd64.whl", hash = "sha256:802ba607f1bc2e87eafbf37454eafc6d38f6e1a711083550494fa74abb92bee9", size = 17907861, upload-time = "2026-08-12T18:34:32.626Z" }, + { url = "https://files.pythonhosted.org/packages/6c/ea/8f3da5045e04ab39b9e6b0bd0d35e204381a5082c8682ce09e58eeaf3467/onyx_devtools-0.11.0-py3-none-win_arm64.whl", hash = "sha256:cb374c82eb2984f617e44fd5295bf5a407a88c0df37ab974714056bcd32a0ec2", size = 15978499, upload-time = "2026-08-12T18:34:34.763Z" }, ] [[package]] @@ -6271,6 +6273,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dd/6f/832c2023a8bd8414c93452bd8b43bf61cedfa5b9575f70c06fb911e51a29/release_tag-0.5.2-py3-none-win_arm64.whl", hash = "sha256:5f26b008e0be0c7a122acd8fcb1bb5c822f38e77fed0c0bf6c550cc226c6bf14", size = 1203191, upload-time = "2026-03-11T00:27:29.789Z" }, ] +[[package]] +name = "reportlab" +version = "5.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/d6/4b7b0cf56880eb96533e607967be6a939e344675601e033d113a0bfa1f4e/reportlab-5.0.0.tar.gz", hash = "sha256:e4494a0c6623ae213bb856fba523171b2b54a7bf629fda02d5e525a7b899a784", size = 3701928, upload-time = "2026-06-18T11:34:31.145Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/07/70085c17a369605f15e301d10ab902115019b1126c7253d964afc230c7d6/reportlab-5.0.0-py3-none-any.whl", hash = "sha256:9d5a3affa84919e1111ede580031266a570e93b1ce388219621347965ff1d93c", size = 1956710, upload-time = "2026-06-18T11:34:29.07Z" }, +] + [[package]] name = "requests" version = "2.33.0" diff --git a/web/Dockerfile b/web/Dockerfile index cfa15db7648..a6210719131 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,15 +1,22 @@ -# Registry prefix for the bun binary source below. Defaults to Docker Hub; CI overrides it to -# the ECR pull-through cache to dodge rate limits. Applies only to `bun_source` -- the DHI Node -# bases come from dhi.io, a separate registry not served by the cache (see the FROM lines below). +# Registry prefix for the base images below. Defaults to Docker Hub; CI overrides it to the +# ECR pull-through cache to dodge rate limits. It only applies to the default images -- the +# DHI overrides below carry their own registry (dhi.io), which the cache does not serve. ARG BASE_IMAGE_REGISTRY=docker.io +# Node bases. The defaults are the public Debian slim images, so a plain `docker build` needs +# no extra registry access. CI overrides both with the matching Docker Hardened Images from +# dhi.io, which need a Docker account with DHI catalog access. +# Refresh a digest with: docker buildx imagetools inspect +ARG NODE_BUILDER_IMAGE=${BASE_IMAGE_REGISTRY}/library/node:24-trixie-slim@sha256:0711b541c1c33a8a530ac4f0d391baa9a15b3d804695b1b24a47daa5fb60e74d +ARG NODE_RUNTIME_IMAGE=${BASE_IMAGE_REGISTRY}/library/node:24-trixie-slim@sha256:0711b541c1c33a8a530ac4f0d391baa9a15b3d804695b1b24a47daa5fb60e74d + # bun binary source, in its own stage because buildx only expands the registry ARG in a FROM, -# not in `COPY --from=`. Use the glibc (Debian) build so it runs on the glibc DHI node images. +# not in `COPY --from=`. Use the glibc (Debian) build so it runs on the glibc node images. FROM ${BASE_IMAGE_REGISTRY}/oven/bun:1@sha256:e10577f0db68676a7024391c6e5cb4b879ebd17188ab750cf10024a6d700e5c4 AS bun_source -# Build stage. The DHI "-dev" variant runs as root and ships a shell, apt, and npm/npx. -# Refresh the digest with: docker buildx imagetools inspect dhi.io/node:24-debian13-dev -FROM dhi.io/node:24-debian13-dev@sha256:25a83f18150669e9ce3c9437327add4d75ae4ea26cbdb5e8747b66d3b74a567a AS builder +# Build stage. Needs a root user plus a shell and npm/npx, which both the default slim image +# and the DHI "-dev" variant provide. +FROM ${NODE_BUILDER_IMAGE} AS builder COPY --from=bun_source /usr/local/bin/bun /usr/local/bin/bun COPY --from=bun_source /usr/local/bin/bunx /usr/local/bin/bunx WORKDIR /app @@ -100,10 +107,10 @@ RUN --mount=type=secret,id=sentry_auth_token \ fi && \ NODE_OPTIONS="${NODE_OPTIONS}" npx next build -# Runtime stage. The DHI runtime variant is near-distroless: no shell or package manager, runs -# as the non-root `node` user (uid 1000) with WORKDIR /app pre-set. -# Refresh the digest with: docker buildx imagetools inspect dhi.io/node:24-debian13 -FROM dhi.io/node:24-debian13@sha256:805278f24c1146c6d3c96577b6256f8f97c43196fff88315fe3291a1ce118ddd AS runner +# Runtime stage. Runs as the non-root `node` user (uid 1000), which both the default slim +# image and the DHI runtime variant ship. The DHI variant is near-distroless: no shell or +# package manager, so keep this stage free of RUN instructions. +FROM ${NODE_RUNTIME_IMAGE} AS runner LABEL com.onyx.maintainer="founders@onyx.app" LABEL com.onyx.description="This image is the web/frontend container of Onyx which \ @@ -193,5 +200,9 @@ ENV ONYX_VERSION=${ONYX_VERSION} # name only — breaking loopback healthchecks. Force 0.0.0.0 instead. ENV HOSTNAME="0.0.0.0" +# Don't run production as root. The DHI runtime already defaults to this user; the default +# slim image does not. +USER node + # The DHI runtime image sets no default ENTRYPOINT, so invoke node explicitly. CMD ["node", "server.js"] diff --git a/web/lib/opal/src/components/buttons/sidebar-tab/SidebarTab.test.tsx b/web/lib/opal/src/components/buttons/sidebar-tab/SidebarTab.test.tsx new file mode 100644 index 00000000000..c0a6012215c --- /dev/null +++ b/web/lib/opal/src/components/buttons/sidebar-tab/SidebarTab.test.tsx @@ -0,0 +1,55 @@ +// SidebarTab reads its fold state from the enclosing sidebar, not from the +// app-wide sidebar state. SidebarTab is also used for page-level tab +// navigation (e.g. the settings page), and those tabs must stay expanded when +// the app sidebar folds. +import { render, screen } from "@tests/setup/test-utils"; +import { SidebarTab } from "@opal/components"; +import { SidebarLayouts, SidebarStateProvider } from "@opal/layouts"; +import { renderSidebarLogo } from "@/lib/sidebar/utils"; + +jest.mock("@opal/hooks/useScreenSize", () => ({ + __esModule: true, + default: () => ({ isMobile: false, isSmallScreen: false }), +})); + +function FoldedSidebar({ foldable }: { foldable?: boolean }) { + return ( + + + + Settings + + + + ); +} + +it("collapses the label inside a folded foldable sidebar", () => { + render(); + expect(screen.queryByText("Settings")).not.toBeInTheDocument(); +}); + +it("keeps the label in a non-foldable sidebar, even when app state is folded", () => { + render(); + expect(screen.getByText("Settings")).toBeInTheDocument(); +}); + +it("keeps the label outside a sidebar, even when app state is folded", () => { + render( + + Settings + + ); + expect(screen.getByText("Settings")).toBeInTheDocument(); +}); + +it("still honors an explicit folded prop as an override", () => { + render( + + + Settings + + + ); + expect(screen.queryByText("Settings")).not.toBeInTheDocument(); +}); diff --git a/web/lib/opal/src/components/buttons/sidebar-tab/components.tsx b/web/lib/opal/src/components/buttons/sidebar-tab/components.tsx index 2f4b94bd4ef..b36c352ca4b 100644 --- a/web/lib/opal/src/components/buttons/sidebar-tab/components.tsx +++ b/web/lib/opal/src/components/buttons/sidebar-tab/components.tsx @@ -5,6 +5,7 @@ import type { ButtonType, IconFunctionComponent, RichStr } from "@opal/types"; import type { Route } from "next"; import { Interactive, type InteractiveStatefulVariant } from "@opal/core"; import { ContentAction } from "@opal/layouts"; +import { useSidebarFolded } from "@opal/layouts/sidebar/context"; import { Text, Tooltip } from "@opal/components"; import Link from "next/link"; @@ -13,7 +14,10 @@ import Link from "next/link"; // --------------------------------------------------------------------------- interface SidebarTabProps { - /** Collapses the label, showing only the icon. */ + /** + * Collapses the label, showing only the icon. Defaults to the enclosing + * sidebar's fold state, so tabs inside a sidebar never need to pass this. + */ folded?: boolean; /** Marks this tab as the currently active/selected item. */ @@ -59,7 +63,7 @@ interface SidebarTabProps { * `rightChildren` for inline actions, and folded mode with an auto-tooltip. */ function SidebarTab({ - folded, + folded: foldedProp, selected, variant = "sidebar-heavy", nested, @@ -73,6 +77,9 @@ function SidebarTab({ tooltip, children, }: SidebarTabProps) { + const foldedFromSidebar = useSidebarFolded(); + const folded = foldedProp ?? foldedFromSidebar; + const Icon = icon ?? (nested diff --git a/web/lib/opal/src/components/cards/card/Card.stories.tsx b/web/lib/opal/src/components/cards/card/Card.stories.tsx index cb1bae0a6c6..5a237dc51dc 100644 --- a/web/lib/opal/src/components/cards/card/Card.stories.tsx +++ b/web/lib/opal/src/components/cards/card/Card.stories.tsx @@ -4,7 +4,7 @@ import { Button, Card } from "@opal/components"; const BACKGROUND_VARIANTS = ["none", "light", "heavy"] as const; const BORDER_VARIANTS = ["none", "dashed", "solid"] as const; -const PADDING_VARIANTS = ["fit", "2xs", "xs", "sm", "md", "lg"] as const; +const PADDING_VARIANTS = [0, 0.5, 1, 2, 4, 6] as const; const ROUNDING_VARIANTS = ["xs", "sm", "md", "lg"] as const; const meta: Meta = { diff --git a/web/lib/opal/src/components/cards/card/README.md b/web/lib/opal/src/components/cards/card/README.md index c3fceeb55c6..279453527fb 100644 --- a/web/lib/opal/src/components/cards/card/README.md +++ b/web/lib/opal/src/components/cards/card/README.md @@ -14,7 +14,7 @@ Default behavior — a plain container. ```tsx import { Card } from "@opal/components"; - +

Hello

 ``` @@ -23,7 +23,7 @@ import { Card } from "@opal/components"; | Prop | Type | Default | Description | |------|------|---------|-------------| -| `padding` | `PaddingVariants` | `"md"` | Padding preset | +| `padding` | `Spacing` | `4` | Padding, as a spacing step (`N / 4` rem) | | `rounding` | `RoundingVariants` | `"md"` | Border-radius preset | | `background` | `"none" \| "light" \| "heavy"` | `"light"` | Background fill intensity | | `border` | `"none" \| "dashed" \| "solid"` | `"none"` | Border style | @@ -33,14 +33,8 @@ import { Card } from "@opal/components"; ### Padding scale -| `padding` | Class | -|-----------|---------| -| `"lg"` | `p-6` | -| `"md"` | `p-4` | -| `"sm"` | `p-2` | -| `"xs"` | `p-1` | -| `"2xs"` | `p-0.5` | -| `"fit"` | `p-0` | +`padding` is a spacing step, not a preset: `N` is `N / 4` rem, the same scale Tailwind +uses. So `padding={2}` is the same distance as `p-2`, and the default `4` is `1rem`. ### Rounding scale @@ -113,7 +107,7 @@ Because Card doesn't own the trigger, it also doesn't generate IDs or ARIA attri ```ts type CardBaseProps = { - padding?: PaddingVariants; + padding?: Spacing; rounding?: RoundingVariants; background?: "none" | "light" | "heavy"; border?: "none" | "dashed" | "solid"; diff --git a/web/lib/opal/src/components/cards/card/components.tsx b/web/lib/opal/src/components/cards/card/components.tsx index dd0a80d0348..fca64bc6d59 100644 --- a/web/lib/opal/src/components/cards/card/components.tsx +++ b/web/lib/opal/src/components/cards/card/components.tsx @@ -3,17 +3,17 @@ import "@opal/components/cards/card/styles.css"; import type { BackgroundVariants, BorderVariants, - PaddingVariants, + Spacing, RoundingVariants, ShadowVariants, SizeVariants, StatusVariants, } from "@opal/types"; import { - paddingVariants, cardRoundingVariants, cardTopRoundingVariants, cardBottomRoundingVariants, + spacingToRem, } from "@opal/shared"; import { cn } from "@opal/utils"; @@ -26,24 +26,17 @@ import { cn } from "@opal/utils"; */ type CardBaseProps = { /** - * Padding preset. + * Padding. * - * | Value | Class | - * |---------|---------| - * | `"lg"` | `p-6` | - * | `"md"` | `p-4` | - * | `"sm"` | `p-2` | - * | `"xs"` | `p-1` | - * | `"2xs"` | `p-0.5` | - * | `"fit"` | `p-0` | + * A spacing step: `N` is `N / 4` rem, so `4` is `1rem`. * * In expandable mode, applied **only** to the header region. The * `expandedContent` slot has no intrinsic padding — callers own any padding * inside the content they pass in. * - * @default "md" + * @default 4 */ - padding?: PaddingVariants; + padding?: Spacing; /** * Border-radius preset. @@ -182,7 +175,7 @@ type CardProps = CardPlainProps | CardExpandableProps; * * @example Plain * ```tsx - * + * *

Hello

* * ``` @@ -202,7 +195,7 @@ type CardProps = CardPlainProps | CardExpandableProps; */ function Card(props: CardProps) { const { - padding: paddingProp = "md", + padding: paddingProp = 4, rounding: roundingProp = "md", background = "light", border = "none", @@ -212,14 +205,15 @@ function Card(props: CardProps) { children, } = props; - const padding = paddingVariants[paddingProp]; + const paddingStyle = { padding: spacingToRem(paddingProp) }; // Plain mode — unchanged behavior if (!props.expandable) { return (
= { title: "opal/components/EmptyMessageCard", diff --git a/web/lib/opal/src/components/cards/empty-message-card/README.md b/web/lib/opal/src/components/cards/empty-message-card/README.md index c9d29583dd8..9c1a98b8cd8 100644 --- a/web/lib/opal/src/components/cards/empty-message-card/README.md +++ b/web/lib/opal/src/components/cards/empty-message-card/README.md @@ -13,7 +13,7 @@ A pre-configured Card for empty states. Renders a transparent card with a dashed | `sizePreset` | `"secondary" \| "main-ui"` | `"secondary"` | Controls layout and text sizing | | `icon` | `IconFunctionComponent` | `SvgEmpty` | Icon displayed alongside the title | | `title` | `string \| RichStr` | — | Primary message text (required) | -| `padding` | `PaddingVariants` | `"md"` | Padding preset for the card | +| `padding` | `Spacing` | `4` | Padding, as a spacing step (`N / 4` rem) | | `ref` | `React.Ref` | — | Ref forwarded to the root div | ### `sizePreset="main-ui"` only @@ -45,5 +45,5 @@ import { SvgSparkle, SvgFileText, SvgActions } from "@opal/icons"; /> // Custom padding - + ``` diff --git a/web/lib/opal/src/components/cards/empty-message-card/components.tsx b/web/lib/opal/src/components/cards/empty-message-card/components.tsx index 0d2d9c40298..12301aae58a 100644 --- a/web/lib/opal/src/components/cards/empty-message-card/components.tsx +++ b/web/lib/opal/src/components/cards/empty-message-card/components.tsx @@ -1,11 +1,7 @@ import { Card } from "@opal/components/cards/card/components"; import { Content } from "@opal/layouts"; import { SvgEmpty } from "@opal/icons"; -import type { - IconFunctionComponent, - PaddingVariants, - RichStr, -} from "@opal/types"; +import type { IconFunctionComponent, Spacing, RichStr } from "@opal/types"; // --------------------------------------------------------------------------- // Types @@ -19,7 +15,7 @@ type EmptyMessageCardBaseProps = { title: string | RichStr; /** Padding preset for the card. @default "md" */ - padding?: PaddingVariants; + padding?: Spacing; /** Ref forwarded to the root Card div. */ ref?: React.Ref; @@ -45,7 +41,7 @@ function EmptyMessageCard(props: EmptyMessageCardProps) { sizePreset = "secondary", icon = SvgEmpty, title, - padding = "md", + padding = 4, ref, } = props; diff --git a/web/lib/opal/src/components/cards/message-card/README.md b/web/lib/opal/src/components/cards/message-card/README.md index 20b7c7f0dee..4bc4b79ca98 100644 --- a/web/lib/opal/src/components/cards/message-card/README.md +++ b/web/lib/opal/src/components/cards/message-card/README.md @@ -14,8 +14,8 @@ and border colors. | `icon` | `IconFunctionComponent` | per variant | Override the default variant icon | | `title` | `string \| RichStr` | — | Main title text | | `description` | `string \| RichStr` | — | Description below the title | -| `padding` | `"sm" \| "xs"` | `"sm"` | Padding preset for the outer card | -| `headerPadding` | `PaddingVariants` | `"fit"` | Padding around the header Content area. `"fit"` → no padding; `"sm"` → `p-2`. | +| `padding` | `1 \| 2` | `2` | Padding around the outer card, as a spacing step (`N / 4` rem). Narrowed to two densities. | +| `headerPadding` | `Spacing` | `0` | Padding around the header Content area, as a spacing step (`N / 4` rem) | | `bottomChildren` | `ReactNode` | — | Content below a divider, under the main content | | `rightChildren` | `ReactNode` | — | Content on the right side. Mutually exclusive with `onClose`. | | `onClose` | `() => void` | — | Close button callback. When omitted, no close button is rendered. | diff --git a/web/lib/opal/src/components/cards/message-card/components.tsx b/web/lib/opal/src/components/cards/message-card/components.tsx index ba32459df42..ed436ca1fcd 100644 --- a/web/lib/opal/src/components/cards/message-card/components.tsx +++ b/web/lib/opal/src/components/cards/message-card/components.tsx @@ -3,11 +3,11 @@ import "@opal/components/cards/message-card/styles.css"; import { cn } from "@opal/utils"; import type { IconFunctionComponent, - PaddingVariants, + Spacing, RichStr, StatusVariants, } from "@opal/types"; -import { paddingVariants } from "@opal/shared"; +import { spacingToRem } from "@opal/shared"; import { ContentAction } from "@opal/layouts"; import { Button, Divider } from "@opal/components"; import { @@ -39,11 +39,16 @@ interface MessageCardBaseProps { /** Clamp the title to N lines with ellipsis. Default: `1`. Pass `undefined` to wrap freely. */ titleMaxLines?: number; - /** Padding preset. @default "sm" */ - padding?: Extract; + /** + * Padding, as a spacing step (`N / 4` rem). Narrowed on purpose — a message + * card is a fixed-density surface, so only these two densities are offered. + * + * @default 2 + */ + padding?: 1 | 2; - /** Padding around the header Content area. @default "fit" */ - headerPadding?: PaddingVariants; + /** Padding around the header Content area, as a spacing step. @default 0 */ + headerPadding?: Spacing; /** * Content rendered below a divider, under the main content area. @@ -130,8 +135,8 @@ function MessageCard({ title, description, titleMaxLines, - padding = "sm", - headerPadding = "fit", + padding = 2, + headerPadding = 0, bottomChildren, rightChildren, onClose, @@ -154,12 +159,13 @@ function MessageCard({ return (
-
+
( @@ -176,7 +182,7 @@ function MessageCard({ {bottomChildren && ( <> - + {bottomChildren} )} diff --git a/web/lib/opal/src/components/cards/select-card/README.md b/web/lib/opal/src/components/cards/select-card/README.md index c0ea64430db..d2d1b0ce03b 100644 --- a/web/lib/opal/src/components/cards/select-card/README.md +++ b/web/lib/opal/src/components/cards/select-card/README.md @@ -38,7 +38,7 @@ Inherits **all** props from `InteractiveStatefulProps` (except `variant`, which | Prop | Type | Default | Description | |---|---|---|---| -| `padding` | `PaddingVariants` | `"md"` | Padding preset | +| `padding` | `Spacing` | `4` | Padding, as a spacing step (`N / 4` rem) | | `rounding` | `RoundingVariants` | `"md"` | Border-radius preset | | `border` | `BorderVariants` | `"solid"` | Border style (`"none"` \| `"dashed"` \| `"solid"`) | | `ref` | `React.Ref` | — | Ref forwarded to the root div | @@ -46,14 +46,8 @@ Inherits **all** props from `InteractiveStatefulProps` (except `variant`, which ### Padding scale -| `padding` | Class | -|-----------|---------| -| `"lg"` | `p-6` | -| `"md"` | `p-4` | -| `"sm"` | `p-2` | -| `"xs"` | `p-1` | -| `"2xs"` | `p-0.5` | -| `"fit"` | `p-0` | +`padding` is a spacing step, not a preset: `N` is `N / 4` rem, the same scale Tailwind +uses. So `padding={2}` is the same distance as `p-2`, and the default `4` is `1rem`. ### Rounding scale diff --git a/web/lib/opal/src/components/cards/select-card/SelectCard.stories.tsx b/web/lib/opal/src/components/cards/select-card/SelectCard.stories.tsx index 7cce958bc3d..e29ce15c8eb 100644 --- a/web/lib/opal/src/components/cards/select-card/SelectCard.stories.tsx +++ b/web/lib/opal/src/components/cards/select-card/SelectCard.stories.tsx @@ -13,7 +13,7 @@ import { import { Interactive } from "@opal/core"; const STATES = ["empty", "filled", "selected"] as const; -const PADDING_VARIANTS = ["fit", "2xs", "xs", "sm", "md", "lg"] as const; +const PADDING_VARIANTS = [0, 0.5, 1, 2, 4, 6] as const; const ROUNDING_VARIANTS = ["xs", "sm", "md", "lg"] as const; const meta = { diff --git a/web/lib/opal/src/components/cards/select-card/components.tsx b/web/lib/opal/src/components/cards/select-card/components.tsx index e3c4a9872d8..20a06239aff 100644 --- a/web/lib/opal/src/components/cards/select-card/components.tsx +++ b/web/lib/opal/src/components/cards/select-card/components.tsx @@ -1,10 +1,6 @@ import "@opal/components/cards/select-card/styles.css"; -import type { - BorderVariants, - PaddingVariants, - RoundingVariants, -} from "@opal/types"; -import { paddingVariants, cardRoundingVariants } from "@opal/shared"; +import type { BorderVariants, Spacing, RoundingVariants } from "@opal/types"; +import { cardRoundingVariants, spacingToRem } from "@opal/shared"; import { cn } from "@opal/utils"; import { Interactive, type InteractiveStatefulProps } from "@opal/core"; @@ -14,20 +10,13 @@ import { Interactive, type InteractiveStatefulProps } from "@opal/core"; type SelectCardProps = Omit & { /** - * Padding preset. + * Padding. * - * | Value | Class | - * |---------|---------| - * | `"lg"` | `p-6` | - * | `"md"` | `p-4` | - * | `"sm"` | `p-2` | - * | `"xs"` | `p-1` | - * | `"2xs"` | `p-0.5` | - * | `"fit"` | `p-0` | + * A spacing step: `N` is `N / 4` rem, so `4` is `1rem`. * - * @default "md" + * @default 4 */ - padding?: PaddingVariants; + padding?: Spacing; /** * Border-radius preset. @@ -86,21 +75,22 @@ type SelectCardProps = Omit & { * ``` */ function SelectCard({ - padding: paddingProp = "md", + padding: paddingProp = 4, rounding: roundingProp = "md", border = "solid", ref, children, ...statefulProps }: SelectCardProps) { - const padding = paddingVariants[paddingProp]; + const paddingStyle = { padding: spacingToRem(paddingProp) }; const rounding = cardRoundingVariants[roundingProp]; return (
{children} diff --git a/web/lib/opal/src/components/divider/Divider.stories.tsx b/web/lib/opal/src/components/divider/Divider.stories.tsx index 79781b7a793..82e1d027ca7 100644 --- a/web/lib/opal/src/components/divider/Divider.stories.tsx +++ b/web/lib/opal/src/components/divider/Divider.stories.tsx @@ -28,11 +28,11 @@ export const Vertical: Story = { }; export const NoPadding: Story = { - render: () => , + render: () => , }; export const CustomPadding: Story = { - render: () => , + render: () => , }; export const VerticalNoPadding: Story = { @@ -43,8 +43,8 @@ export const VerticalNoPadding: Story = { Left Right
diff --git a/web/lib/opal/src/components/divider/README.md b/web/lib/opal/src/components/divider/README.md index a42cc2415b4..3f3001da353 100644 --- a/web/lib/opal/src/components/divider/README.md +++ b/web/lib/opal/src/components/divider/README.md @@ -15,8 +15,8 @@ A plain line with no title or description. | Prop | Type | Default | Description | |---|---|---|---| | `orientation` | `"horizontal" \| "vertical"` | `"horizontal"` | Direction of the line | -| `paddingParallel` | `PaddingVariants` | `"sm"` | Padding along the line direction (0.5rem) | -| `paddingPerpendicular` | `PaddingVariants` | `"xs"` | Padding perpendicular to the line (0.25rem) | +| `paddingParallel` | `Spacing` | `2` | Padding along the line direction (0.5rem) | +| `paddingPerpendicular` | `Spacing` | `1` | Padding perpendicular to the line (0.25rem) | ### Titled divider @@ -53,10 +53,10 @@ import { Divider } from "@opal/components"; // No padding - + // Custom padding - + // With title diff --git a/web/lib/opal/src/components/divider/components.tsx b/web/lib/opal/src/components/divider/components.tsx index 312babfebc5..0c8424a6682 100644 --- a/web/lib/opal/src/components/divider/components.tsx +++ b/web/lib/opal/src/components/divider/components.tsx @@ -2,16 +2,12 @@ import "@opal/components/divider/styles.css"; import { useState, useCallback } from "react"; -import type { - OrientationVariants, - PaddingVariants, - RichStr, -} from "@opal/types"; +import type { OrientationVariants, Spacing, RichStr } from "@opal/types"; import { Button, Text } from "@opal/components"; import { SvgChevronRight } from "@opal/icons"; import { Interactive } from "@opal/core"; import { cn } from "@opal/utils"; -import { paddingXVariants, paddingYVariants } from "@opal/shared"; +import { spacingToRem } from "@opal/shared"; // --------------------------------------------------------------------------- // Types @@ -38,10 +34,10 @@ type DividerBareProps = Omit< > & { /** Orientation of the line. Default: `"horizontal"`. */ orientation?: OrientationVariants; - /** Padding along the line direction. Default: `"sm"` (0.5rem). */ - paddingParallel?: PaddingVariants; - /** Padding perpendicular to the line. Default: `"xs"` (0.25rem). */ - paddingPerpendicular?: PaddingVariants; + /** Padding along the line direction, as a spacing step. Default: `2` (0.5rem). */ + paddingParallel?: Spacing; + /** Padding perpendicular to the line, as a spacing step. Default: `1` (0.25rem). */ + paddingPerpendicular?: Spacing; }; /** Line with a title to the left. */ @@ -93,19 +89,19 @@ function Divider(props: DividerProps) { title, description, orientation = "horizontal", - paddingParallel = "sm", - paddingPerpendicular = "xs", + paddingParallel = 2, + paddingPerpendicular = 1, } = props; if (orientation === "vertical") { return (
@@ -115,11 +111,11 @@ function Divider(props: DividerProps) { return (
{title && ( diff --git a/web/lib/opal/src/components/end-of-list/components.tsx b/web/lib/opal/src/components/end-of-list/components.tsx index b9c6996a412..a63ecf5bd66 100644 --- a/web/lib/opal/src/components/end-of-list/components.tsx +++ b/web/lib/opal/src/components/end-of-list/components.tsx @@ -11,11 +11,11 @@ interface EndOfListProps { function EndOfList({ title }: EndOfListProps) { return (
- + {title} - +
); } diff --git a/web/lib/opal/src/components/inputs/input-select/components.tsx b/web/lib/opal/src/components/inputs/input-select/components.tsx index 5dbaecca3ac..f94ac6c52bf 100644 --- a/web/lib/opal/src/components/inputs/input-select/components.tsx +++ b/web/lib/opal/src/components/inputs/input-select/components.tsx @@ -8,7 +8,7 @@ import { cn } from "@opal/utils"; import type { IconFunctionComponent, InputVariants, - PaddingVariants, + Spacing, RichStr, WithoutStyles, } from "@opal/types"; @@ -434,8 +434,8 @@ function InputSelectLabel({ } interface InputSelectSeparatorProps { - paddingParallel?: PaddingVariants; - paddingPerpendicular?: PaddingVariants; + paddingParallel?: Spacing; + paddingPerpendicular?: Spacing; } function InputSelectSeparator({ diff --git a/web/lib/opal/src/components/modal/components.tsx b/web/lib/opal/src/components/modal/components.tsx index 6095da86e9b..eedfab1c8bb 100644 --- a/web/lib/opal/src/components/modal/components.tsx +++ b/web/lib/opal/src/components/modal/components.tsx @@ -359,13 +359,13 @@ function ModalHeader({ ); return ( -
+
{closeButton}
@@ -416,7 +416,7 @@ function ModalBody({ className="opal-modal-body" {...(twoTone && { "data-two-tone": "" })} > -
+
{children}
@@ -433,8 +433,8 @@ function ModalFooter({ ref, ...props }: ModalFooterProps) { ref={ref} flexDirection="row" justifyContent="end" - gap={0.5} - padding={1} + gap={2} + padding={4} height="fit" {...props} /> @@ -469,7 +469,7 @@ function BasicModalFooter({ left, cancel, submit }: BasicModalFooterProps) { <> {left &&
{left}
} {(cancel || submit) && ( -
+
{cancel} {submit}
diff --git a/web/lib/opal/src/components/popover/components.tsx b/web/lib/opal/src/components/popover/components.tsx index c2ac62be3c5..85b39260b71 100644 --- a/web/lib/opal/src/components/popover/components.tsx +++ b/web/lib/opal/src/components/popover/components.tsx @@ -165,7 +165,7 @@ const Popover = Object.assign(PopoverRoot, { // ============================================================================ function SeparatorHelper() { - return ; + return ; } /** diff --git a/web/lib/opal/src/components/tabs/README.md b/web/lib/opal/src/components/tabs/README.md index 3f49b556187..5023f5c8cf8 100644 --- a/web/lib/opal/src/components/tabs/README.md +++ b/web/lib/opal/src/components/tabs/README.md @@ -109,7 +109,7 @@ When tabs overflow the available width, show navigation arrows: ### Content padding ```tsx - + Padded content ``` @@ -150,4 +150,4 @@ Forwards all [Radix Tabs.Root](https://www.radix-ui.com/docs/primitives/componen | Prop | Type | Default | Description | |---|---|---|---| | `value` | `string` | **required** | Must match a `Tabs.Trigger` value | -| `padding` | `number` | `0` | Additional inner padding in rem units | +| `padding` | `Spacing` | `0` | Additional inner padding, as a spacing step (`N / 4` rem) | diff --git a/web/lib/opal/src/components/tabs/Tabs.stories.tsx b/web/lib/opal/src/components/tabs/Tabs.stories.tsx index efc41c70d5b..c0b0388e349 100644 --- a/web/lib/opal/src/components/tabs/Tabs.stories.tsx +++ b/web/lib/opal/src/components/tabs/Tabs.stories.tsx @@ -190,7 +190,7 @@ export const ContentPadding: Story = { Padded Flush - +
Inner content with 1rem padding
diff --git a/web/lib/opal/src/components/tabs/components.tsx b/web/lib/opal/src/components/tabs/components.tsx index 41cf5935256..a48b5871c8b 100644 --- a/web/lib/opal/src/components/tabs/components.tsx +++ b/web/lib/opal/src/components/tabs/components.tsx @@ -4,7 +4,12 @@ import "@opal/components/tabs/styles.css"; import React, { useRef, useState, useEffect, useMemo } from "react"; import * as TabsPrimitive from "@radix-ui/react-tabs"; import { mergeRefs } from "@opal/utils"; -import { IconFunctionComponent, type WithoutStyles } from "@opal/types"; +import { + IconFunctionComponent, + type Spacing, + type WithoutStyles, +} from "@opal/types"; +import { spacingToRem } from "@opal/shared"; import { SvgChevronLeft, SvgChevronRight } from "@opal/icons"; import { Tooltip, Text, Button } from "@opal/components"; import { @@ -283,15 +288,15 @@ function TabsTrigger({ interface TabsContentProps extends WithoutStyles< React.ComponentProps > { - /** Additional inner padding in rem. @default 0 */ - padding?: number; + /** Additional inner padding, as a {@link Spacing} step (`N / 4` rem). @default 0 */ + padding?: Spacing; } function TabsContent({ padding, children, ...props }: TabsContentProps) { return ( {padding ? ( -
{children}
+
{children}
) : ( children )} diff --git a/web/lib/opal/src/components/tooltip/Tooltip.stories.tsx b/web/lib/opal/src/components/tooltip/Tooltip.stories.tsx index 70dba3a5a8b..f4a547d20d7 100644 --- a/web/lib/opal/src/components/tooltip/Tooltip.stories.tsx +++ b/web/lib/opal/src/components/tooltip/Tooltip.stories.tsx @@ -35,7 +35,7 @@ export const Sides: Story = { export const OnCard: Story = { render: () => ( - +

Hover this card

 diff --git a/web/lib/opal/src/core/disabled/Disabled.stories.tsx b/web/lib/opal/src/core/disabled/Disabled.stories.tsx index c9bbafab566..cd64e49752c 100644 --- a/web/lib/opal/src/core/disabled/Disabled.stories.tsx +++ b/web/lib/opal/src/core/disabled/Disabled.stories.tsx @@ -12,7 +12,7 @@ export default meta; type Story = StoryObj; const SampleContent = () => ( - +

Card Title

Some content that can be disabled.

@@ -63,7 +63,7 @@ export const TooltipSides: Story = { tooltip={`Tooltip on ${side}`} tooltipSide={side} > - +

tooltipSide: {side}

 @@ -76,7 +76,7 @@ export const WithAllowClick: Story = { render: () => (
- +

Disabled visuals, but pointer events are still active.

diff --git a/web/lib/opal/src/hooks/useScreenSize.ts b/web/lib/opal/src/hooks/useScreenSize.ts index 271b1a856d9..400bd96636e 100644 --- a/web/lib/opal/src/hooks/useScreenSize.ts +++ b/web/lib/opal/src/hooks/useScreenSize.ts @@ -14,6 +14,12 @@ export interface ScreenSize { isMobile: boolean; isSmallScreen: boolean; isMediumScreen: boolean; + /** + * `false` until the hook mounts. Before that the size flags all report + * desktop, to keep the first client render equal to the server render. + * Gate on this in effects that must not act on the desktop default. + */ + isMounted: boolean; } export default function useScreenSize(): ScreenSize { @@ -37,5 +43,6 @@ export default function useScreenSize(): ScreenSize { isMobile: isMounted && sizes.width < SMALL_BREAKPOINT_PX, isSmallScreen: isMounted && sizes.width < MEDIUM_BREAKPOINT_PX, isMediumScreen: isMounted && sizes.width < LARGE_BREAKPOINT_PX, + isMounted, }; } diff --git a/web/lib/opal/src/layouts/auth/components.tsx b/web/lib/opal/src/layouts/auth/components.tsx index 0aa171ff0c8..e27014c0f8d 100644 --- a/web/lib/opal/src/layouts/auth/components.tsx +++ b/web/lib/opal/src/layouts/auth/components.tsx @@ -49,7 +49,7 @@ function Card({ }: CardProps) { return (
- +
diff --git a/web/lib/opal/src/layouts/general/README.md b/web/lib/opal/src/layouts/general/README.md index 60e6d755978..ca3b761f616 100644 --- a/web/lib/opal/src/layouts/general/README.md +++ b/web/lib/opal/src/layouts/general/README.md @@ -4,7 +4,10 @@ A flexbox container primitive for grouping related content. Configurable direction, alignment, spacing, and dimensions. Defaults to a full-width / full-height column with centered children -and a 1rem gap. +and a gap of `4` (1rem). + +`gap` and `padding` are spacing steps, not raw lengths: `N` is `N / 4` rem, the same scale +Tailwind uses. So `gap={2}` is the same distance as `gap-2`. ## Props @@ -15,8 +18,8 @@ and a 1rem gap. | `alignItems` | `"start" \| "center" \| "end" \| "stretch"` | `"center"` | Cross-axis alignment | | `width` | `"auto" \| "fit" \| "full" \| number` | `"full"` | Width. `number` = rem. | | `height` | `"auto" \| "fit" \| "full" \| number` | `"full"` | Height. `number` = rem. | -| `gap` | `number` | `1` | Gap between children, in rem | -| `padding` | `number` | `0` | Padding, in rem | +| `gap` | `Spacing` | `4` | Gap between children, as a spacing step (`N / 4` rem) | +| `padding` | `Spacing` | `0` | Padding, as a spacing step (`N / 4` rem) | | `wrap` | `boolean` | `false` | Enables `flex-wrap` | | `dbg` | `boolean` | `false` | Adds a red debug border | | `className` | `string` | — | Additional classes | @@ -40,7 +43,7 @@ import { Section } from "@opal/layouts";
// Tighter gap, custom width -
+
One Two
diff --git a/web/lib/opal/src/layouts/general/components.tsx b/web/lib/opal/src/layouts/general/components.tsx index 30ca086885a..84ff26961d0 100644 --- a/web/lib/opal/src/layouts/general/components.tsx +++ b/web/lib/opal/src/layouts/general/components.tsx @@ -2,7 +2,8 @@ import React from "react"; import { cn } from "@opal/utils"; -import type { WithoutStyles } from "@opal/types"; +import type { Spacing, WithoutStyles } from "@opal/types"; +import { spacingToRem } from "@opal/shared"; type FlexDirection = "row" | "column"; type JustifyContent = "start" | "center" | "end" | "between"; @@ -46,8 +47,10 @@ interface SectionProps extends WithoutStyles< width?: Length; height?: Length; - gap?: number; - padding?: number; + /** Spacing between children, as a {@link Spacing} step (`N / 4` rem). @default 4 */ + gap?: Spacing; + /** Inner padding, as a {@link Spacing} step (`N / 4` rem). @default 0 */ + padding?: Spacing; wrap?: boolean; ref?: React.Ref; @@ -60,7 +63,7 @@ function Section({ alignItems = "center", width = "full", height = "full", - gap = 1, + gap = 4, padding = 0, wrap, ref, @@ -83,8 +86,8 @@ function Section({ className )} style={{ - gap: `${gap}rem`, - padding: `${padding}rem`, + gap: spacingToRem(gap), + padding: spacingToRem(padding), ...(typeof width === "number" && { width: `${width}rem` }), ...(typeof height === "number" && { height: `${height}rem` }), }} diff --git a/web/lib/opal/src/layouts/index.ts b/web/lib/opal/src/layouts/index.ts index 77d9c3a82e1..59381f88dc6 100644 --- a/web/lib/opal/src/layouts/index.ts +++ b/web/lib/opal/src/layouts/index.ts @@ -68,6 +68,7 @@ export type { /* SidebarLayouts */ export * as SidebarLayouts from "@opal/layouts/sidebar/components"; export { type SidebarRootProps } from "@opal/layouts/sidebar/components"; +export { useSidebarFolded } from "@opal/layouts/sidebar/context"; /* AuthLayouts */ export * as AuthLayouts from "@opal/layouts/auth/components"; diff --git a/web/lib/opal/src/layouts/inputs/components.tsx b/web/lib/opal/src/layouts/inputs/components.tsx index 854397f6185..6eecb5b0249 100644 --- a/web/lib/opal/src/layouts/inputs/components.tsx +++ b/web/lib/opal/src/layouts/inputs/components.tsx @@ -120,7 +120,7 @@ function Vertical({ ); const content = ( -
+
{titleRow} {children} {fieldName && } @@ -182,7 +182,7 @@ function Horizontal({ typeof withLabelProp === "string" ? withLabelProp : undefined; const content = ( -
+
; + return ; } // --------------------------------------------------------------------------- diff --git a/web/lib/opal/src/layouts/root/components.tsx b/web/lib/opal/src/layouts/root/components.tsx index 30181353e11..df8f95b5551 100644 --- a/web/lib/opal/src/layouts/root/components.tsx +++ b/web/lib/opal/src/layouts/root/components.tsx @@ -17,8 +17,10 @@ import type { WithoutStyles } from "@opal/types"; // --------------------------------------------------------------------------- // Sidebar state — raw fold state + setter, owned here as the single source -// of truth. SidebarRoot (in sidebar/components.tsx) derives contentFolded -// and provides RootLayoutFoldedContext from this. +// of truth. SidebarRoot (in sidebar/components.tsx) derives the effective +// fold state from this and provides it via SidebarFoldedContext. Components +// inside a sidebar should read that (`useSidebarFolded`) rather than the raw +// state here, which is app-wide and true even outside a sidebar. // --------------------------------------------------------------------------- export interface SidebarStateContextType { diff --git a/web/lib/opal/src/layouts/settings/components.tsx b/web/lib/opal/src/layouts/settings/components.tsx index 9ccfc31a5c4..51e934c5d1c 100644 --- a/web/lib/opal/src/layouts/settings/components.tsx +++ b/web/lib/opal/src/layouts/settings/components.tsx @@ -147,7 +147,7 @@ function SettingsHeader({ {divider ? ( <> - + ) : ( diff --git a/web/lib/opal/src/layouts/sidebar/components.tsx b/web/lib/opal/src/layouts/sidebar/components.tsx index 796a1036055..9a53ea8a471 100644 --- a/web/lib/opal/src/layouts/sidebar/components.tsx +++ b/web/lib/opal/src/layouts/sidebar/components.tsx @@ -17,6 +17,7 @@ import { Disabled, Hoverable, Interactive } from "@opal/core"; import { SvgSidebar } from "@opal/icons"; import type { IconFunctionComponent, RichStr } from "@opal/types"; import { useSidebarState } from "@opal/layouts/root/components"; +import { SidebarFoldedContext } from "@opal/layouts/sidebar/context"; import useScreenSize from "@opal/hooks/useScreenSize"; // --------------------------------------------------------------------------- @@ -44,19 +45,37 @@ interface SidebarRootProps { } function SidebarRoot({ foldable = false, children }: SidebarRootProps) { - const { isMobile, isSmallScreen } = useScreenSize(); + const { isMobile, isSmallScreen, isMounted } = useScreenSize(); const { folded, setFolded } = useSidebarState(); const closeSidebar = useCallback(() => setFolded(true), [setFolded]); useEffect(() => { - if (!isMobile && !isSmallScreen && !foldable) { + // Before mount the screen size reports desktop. Act on the real size only, + // or every mount unfolds the overlay on mobile and small screens. + if (!isMounted) return; + + if (isMobile || isSmallScreen) { + // The overlay hides the page behind it, so it starts closed. + setFolded(true); + } else if (!foldable) { + // A non-foldable desktop sidebar is a column that is always open. setFolded(false); } - }, [isMobile, isSmallScreen, foldable, setFolded]); + }, [isMounted, isMobile, isSmallScreen, foldable, setFolded]); const foldedAttr = String(folded); - const inner =
{children}
; + + // The overlays always fold; a desktop column only folds when `foldable`. + // Tabs read this derived value, not the app-wide raw state, so tabs outside + // a sidebar (and inside a non-foldable one) never collapse. + const effectiveFolded = (isMobile || isSmallScreen || foldable) && folded; + + const inner = ( + +
{children}
+ + ); if (isMobile) { return ( diff --git a/web/lib/opal/src/layouts/sidebar/context.ts b/web/lib/opal/src/layouts/sidebar/context.ts new file mode 100644 index 00000000000..b10c448cc39 --- /dev/null +++ b/web/lib/opal/src/layouts/sidebar/context.ts @@ -0,0 +1,21 @@ +"use client"; + +import { createContext, useContext } from "react"; + +/** + * Effective fold state of the enclosing sidebar, provided by `SidebarRoot`. + * + * This is the value `SidebarRoot` derives from the raw `useSidebarState` + * fold state — it accounts for the mobile and small-screen overlays, and for + * a non-foldable sidebar, which never folds on desktop. + * + * The default is `false`, so a component outside any `SidebarRoot` reads as + * unfolded. `SidebarTab` is used for page-level tab navigation as well as in + * sidebars, and those tabs must not collapse when the app sidebar folds. + */ +export const SidebarFoldedContext = createContext(false); + +/** Fold state of the enclosing sidebar. `false` outside a `SidebarRoot`. */ +export function useSidebarFolded(): boolean { + return useContext(SidebarFoldedContext); +} diff --git a/web/lib/opal/src/layouts/toast/components.tsx b/web/lib/opal/src/layouts/toast/components.tsx index 171d8bd4ade..7a7a92bd4b8 100644 --- a/web/lib/opal/src/layouts/toast/components.tsx +++ b/web/lib/opal/src/layouts/toast/components.tsx @@ -123,7 +123,7 @@ function ToastContainer({ errorAppendix }: ToastContainerProps) { variant={t.level ?? "info"} title={truncatedTitle} description={buildDescription(t, errorAppendix)} - padding="xs" + padding={1} onClose={t.dismissible ? () => handleClose(t.id) : undefined} bottomChildren={ isExpanded ? : undefined diff --git a/web/lib/opal/src/shared.ts b/web/lib/opal/src/shared.ts index b959086929b..17513807215 100644 --- a/web/lib/opal/src/shared.ts +++ b/web/lib/opal/src/shared.ts @@ -13,8 +13,8 @@ import type { OverridableExtremaSizeVariants, ContainerSizeVariants, ExtremaSizeVariants, - PaddingVariants, RoundingVariants, + Spacing, } from "@opal/types"; /** @@ -120,32 +120,15 @@ const heightVariants: Record = { // - SelectCard (padding, rounding) // --------------------------------------------------------------------------- -const paddingVariants: Record = { - lg: "p-6", - md: "p-4", - sm: "p-2", - xs: "p-1", - "2xs": "p-0.5", - fit: "p-0", -}; - -const paddingXVariants: Record = { - lg: "px-6", - md: "px-4", - sm: "px-2", - xs: "px-1", - "2xs": "px-0.5", - fit: "px-0", -}; - -const paddingYVariants: Record = { - lg: "py-6", - md: "py-4", - sm: "py-2", - xs: "py-1", - "2xs": "py-0.5", - fit: "py-0", -}; +/** + * Converts a spacing step to a CSS length: `N` is `N / 4` rem. + * + * Kept as a function rather than a class lookup so the scale stays open — + * Tailwind cannot build a class name from a runtime value, but arithmetic can. + */ +function spacingToRem(spacing: Spacing): string { + return `${spacing / 4}rem`; +} const cardRoundingVariants: Record = { xl: "rounded-20", @@ -176,10 +159,9 @@ export { type ContainerSizeVariants, type OverridableExtremaSizeVariants, type SizeVariants, + type Spacing, containerSizeVariants, - paddingVariants, - paddingXVariants, - paddingYVariants, + spacingToRem, cardRoundingVariants, cardTopRoundingVariants, cardBottomRoundingVariants, diff --git a/web/lib/opal/src/types.ts b/web/lib/opal/src/types.ts index 9ce49ad3793..94b97dbc695 100644 --- a/web/lib/opal/src/types.ts +++ b/web/lib/opal/src/types.ts @@ -45,23 +45,6 @@ export type SizeVariants = */ export type ContainerSizeVariants = Exclude; -/** - * Padding size variants. - * - * | Variant | Class | - * |---------|---------| - * | `lg` | `p-6` | - * | `md` | `p-4` | - * | `sm` | `p-2` | - * | `xs` | `p-1` | - * | `2xs` | `p-0.5` | - * | `fit` | `p-0` | - */ -export type PaddingVariants = Extract< - SizeVariants, - "fit" | "lg" | "md" | "sm" | "xs" | "2xs" ->; - /** * Rounding size variants. * @@ -85,6 +68,24 @@ export type RoundingVariants = Extract< */ export type ExtremaSizeVariants = Extract; +// --------------------------------------------------------------------------- +// Spacing Scale +// --------------------------------------------------------------------------- + +/** + * A spacing step. `N` is `N / 4` rem, so `4` is `1rem` and `2` is `0.5rem`. + * + * This borrows Tailwind's scale as an interface, not as an implementation — a + * step reads the same here as in a class name, so a `padding` of `2` is the same + * distance as `p-2`. The value is converted with {@link spacingToRem} rather + * than looked up as a class, which keeps the scale open: any step works, + * including ones Tailwind does not ship. + * + * Replaces the named scales. `PaddingVariants` meant one distance on a card and + * a different one on a container; a number cannot be ambiguous that way. + */ +export type Spacing = number; + /** * Shadow depth variants. * diff --git a/web/src/app/admin/billing/BillingDetailsView.tsx b/web/src/app/admin/billing/BillingDetailsView.tsx index 5b4b1cd88f9..bb119abfb2e 100644 --- a/web/src/app/admin/billing/BillingDetailsView.tsx +++ b/web/src/app/admin/billing/BillingDetailsView.tsx @@ -299,7 +299,7 @@ function SubscriptionCard({ alignItems="start" height="auto" > -
+
{planName} @@ -310,7 +310,7 @@ function SubscriptionCard({
@@ -547,7 +547,7 @@ function SeatsCard({ flexDirection="row" alignItems="center" justifyContent="between" - padding={1} + padding={4} height="auto" > {isAdding ? ( @@ -595,7 +595,7 @@ function SeatsCard({ alignItems="center" height="auto" > -
+
{totalSeats} Seats @@ -606,7 +606,7 @@ function SeatsCard({
Payment -
+
+
{/* Renewal fetched on arrival while expired. The page renders regardless: billing is the one route a lapsed instance must always reach. */} {isGraceSyncing && ( diff --git a/web/src/app/admin/billing/CheckoutView.tsx b/web/src/app/admin/billing/CheckoutView.tsx index b896c6c0897..a4c13f106ba 100644 --- a/web/src/app/admin/billing/CheckoutView.tsx +++ b/web/src/app/admin/billing/CheckoutView.tsx @@ -42,7 +42,7 @@ function BillingOption({ >
@@ -185,8 +185,8 @@ export default function CheckoutView({ onAdjustPlan }: CheckoutViewProps) {
{/* Billing Cycle */} @@ -197,7 +197,7 @@ export default function CheckoutView({ onAdjustPlan }: CheckoutViewProps) { >
- + {/* Seats */} {error ? ( diff --git a/web/src/app/admin/billing/LicenseActivationCard.tsx b/web/src/app/admin/billing/LicenseActivationCard.tsx index 65619575587..9579f3f223c 100644 --- a/web/src/app/admin/billing/LicenseActivationCard.tsx +++ b/web/src/app/admin/billing/LicenseActivationCard.tsx @@ -86,7 +86,7 @@ export default function LicenseActivationCard({ // License status view (when license exists and not editing) if (hasLicense && !showInput) { return ( - +
@@ -118,7 +118,7 @@ export default function LicenseActivationCard({ )}
-
+
@@ -137,7 +137,7 @@ export default function LicenseActivationCard({ return ( {/* Header */} -
+
{success && (
@@ -197,7 +197,7 @@ export default function LicenseActivationCard({ flexDirection="row" alignItems="center" justifyContent="start" - gap={0.25} + gap={1} height="auto" >
@@ -221,7 +221,7 @@ export default function LicenseActivationCard({
{/* Footer */} -
+
{index < items.length - 1 && ( - + )}
))} @@ -257,7 +257,7 @@ export function ConfigDisplay({ />
{index < entries.length - 1 && ( - + )}
))} diff --git a/web/src/app/admin/connector/[ccPairId]/DocPermissionSyncAttemptsTable.tsx b/web/src/app/admin/connector/[ccPairId]/DocPermissionSyncAttemptsTable.tsx index 16d7213ab52..46109fcb2dc 100644 --- a/web/src/app/admin/connector/[ccPairId]/DocPermissionSyncAttemptsTable.tsx +++ b/web/src/app/admin/connector/[ccPairId]/DocPermissionSyncAttemptsTable.tsx @@ -187,7 +187,7 @@ export function DocPermissionSyncAttemptsTable({ /> )} -
+
)} -
+
-
+
@@ -137,7 +132,7 @@ export function IndexAttemptsTable({ alignItems="center" width="fit" height="fit" - gap={0.25} + gap={1} // Stack above the row-wide trace overlay button so // the metrics button stays clickable on rows with // a full exception trace. diff --git a/web/src/app/admin/connector/[ccPairId]/stage-metrics/AttemptOverhead.tsx b/web/src/app/admin/connector/[ccPairId]/stage-metrics/AttemptOverhead.tsx index f90af7e2846..d112a8af009 100644 --- a/web/src/app/admin/connector/[ccPairId]/stage-metrics/AttemptOverhead.tsx +++ b/web/src/app/admin/connector/[ccPairId]/stage-metrics/AttemptOverhead.tsx @@ -27,7 +27,7 @@ export default function AttemptOverhead({ }, [attemptStages]); return ( -
+
+
{ @@ -154,7 +154,7 @@ interface IntervalEditorProps { function IntervalEditor({ payload, onChange }: IntervalEditorProps) { return ( -
+
Every @@ -211,7 +211,7 @@ function DailyWeeklyEditor({ payload, onChange }: DailyWeeklyEditorProps) { ? "Runs every day" : `Runs on ${selectedDays.join(", ")}`; return ( -
+
At diff --git a/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.test.tsx b/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.test.tsx index 6425a22d651..1b73d024fc3 100644 --- a/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.test.tsx +++ b/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.test.tsx @@ -1,15 +1,24 @@ import { render, screen, setupUser, waitFor } from "@tests/setup/test-utils"; import ScheduleTaskForm, { defaultFormInitial, + type ScheduleTaskFormInitial, } from "@/app/craft/v1/tasks/components/ScheduleTaskForm"; import type { PickerEntry } from "@/lib/skills/picker"; const mockRouterPush = jest.fn(); +const mockCreateScheduledTask = jest.fn(); +const mockUpdateScheduledTask = jest.fn(); +const mockMutate = jest.fn(); jest.mock("next/navigation", () => ({ useRouter: () => ({ push: mockRouterPush }), })); +jest.mock("swr", () => ({ + ...jest.requireActual("swr"), + useSWRConfig: () => ({ mutate: mockMutate }), +})); + jest.mock("@/hooks/useUserSkills", () => ({ __esModule: true, default: () => ({ data: { builtins: [], customs: [] } }), @@ -20,6 +29,15 @@ jest.mock("@/hooks/useUserExternalApps", () => ({ default: () => ({ data: [] }), })); +jest.mock("@/lib/tools/hooks", () => ({ + useCraftMcpServers: () => ({ data: { mcp_servers: [] } }), +})); + +jest.mock("@/app/craft/v1/tasks/api", () => ({ + createScheduledTask: (...args: unknown[]) => mockCreateScheduledTask(...args), + updateScheduledTask: (...args: unknown[]) => mockUpdateScheduledTask(...args), +})); + jest.mock("@/app/craft/v1/tasks/components/ScheduleEditor", () => ({ __esModule: true, default: () => null, @@ -27,7 +45,22 @@ jest.mock("@/app/craft/v1/tasks/components/ScheduleEditor", () => ({ jest.mock("@/app/craft/v1/tasks/components/PreApprovalPicker", () => ({ __esModule: true, - default: () => null, + default: ({ + onAppChange, + onMcpServerChange, + }: { + onAppChange: (ids: number[]) => void; + onMcpServerChange: (ids: number[]) => void; + }) => ( +
+ + +
+ ), })); jest.mock("@/sections/input/EntryPickerPopover", () => ({ @@ -73,18 +106,34 @@ jest.mock("@/sections/input/EntryPickerPopover", () => ({ ) : null, })); -function renderForm() { +function renderForm({ + initial = defaultFormInitial(), + isEdit = false, +}: { + initial?: ScheduleTaskFormInitial; + isEdit?: boolean; +} = {}) { render( ); } describe("ScheduleTaskForm app picker", () => { + beforeEach(() => { + mockRouterPush.mockReset(); + mockCreateScheduledTask.mockReset(); + mockUpdateScheduledTask.mockReset(); + mockMutate.mockReset(); + mockCreateScheduledTask.mockResolvedValue({ id: "task-id" }); + mockUpdateScheduledTask.mockResolvedValue({ id: "task-id" }); + mockMutate.mockResolvedValue(undefined); + }); + it("inserts an authenticated app into the task prompt", async () => { const user = setupUser(); renderForm(); @@ -112,4 +161,60 @@ describe("ScheduleTaskForm app picker", () => { expect(mockRouterPush).toHaveBeenCalledWith("/craft/v1/apps?connect=2"); expect(prompt).toHaveValue("/"); }); + + it("saves app and MCP server pre-approvals independently", async () => { + const user = setupUser(); + renderForm(); + + await user.type(screen.getByTestId("task-name-input"), "Daily report"); + await user.type(screen.getByTestId("task-prompt-input"), "Make a report"); + await user.click(screen.getByRole("button", { name: "Pre-approve app" })); + await user.click( + screen.getByRole("button", { name: "Pre-approve MCP server" }) + ); + await user.click(screen.getByRole("button", { name: /^Save$/ })); + + await waitFor(() => + expect(mockCreateScheduledTask).toHaveBeenCalledWith( + expect.objectContaining({ + pre_approved_app_ids: [11], + pre_approved_mcp_server_ids: [22], + }) + ) + ); + expect(mockMutate).toHaveBeenCalledWith("/api/build/scheduled-tasks"); + }); + + it("preserves both grant kinds when an existing task is edited", async () => { + const user = setupUser(); + renderForm({ + isEdit: true, + initial: { + ...defaultFormInitial(), + taskId: "task-id", + name: "Daily report", + prompt: "Make a report", + preApprovedAppIds: [11], + preApprovedMcpServerIds: [22], + }, + }); + + await user.click(screen.getByRole("button", { name: "Save changes" })); + + await waitFor(() => + expect(mockUpdateScheduledTask).toHaveBeenCalledWith( + "task-id", + expect.objectContaining({ + pre_approved_app_ids: [11], + pre_approved_mcp_server_ids: [22], + }) + ) + ); + expect(mockMutate).toHaveBeenCalledWith( + "/api/build/scheduled-tasks/task-id", + { id: "task-id" }, + { revalidate: false } + ); + expect(mockRouterPush).toHaveBeenCalledWith("/craft/v1/tasks/task-id"); + }); }); diff --git a/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.tsx b/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.tsx index abe5fb28bf6..b9e0b25b5c0 100644 --- a/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.tsx +++ b/web/src/app/craft/v1/tasks/components/ScheduleTaskForm.tsx @@ -2,6 +2,7 @@ import { useCallback, useMemo, useRef, useState } from "react"; import { useRouter } from "next/navigation"; +import { useSWRConfig } from "swr"; import { Button, Divider, @@ -42,6 +43,7 @@ import { updateScheduledTask, } from "@/app/craft/v1/tasks/api"; import { TASKS_PATH, taskDetailPath } from "@/app/craft/v1/tasks/constants"; +import { SWR_KEYS } from "@/lib/swr-keys"; export interface ScheduleTaskFormInitial { /** ``null`` for create. */ @@ -51,6 +53,7 @@ export interface ScheduleTaskFormInitial { mode: EditorMode; payload: EditorPayload; preApprovedAppIds: number[]; + preApprovedMcpServerIds: number[]; } interface ScheduleTaskFormProps { @@ -73,6 +76,7 @@ export default function ScheduleTaskForm({ onBack, }: ScheduleTaskFormProps) { const router = useRouter(); + const { mutate } = useSWRConfig(); const [name, setName] = useState(initial.name); const [prompt, setPrompt] = useState(initial.prompt); const [mode, setMode] = useState(initial.mode); @@ -80,6 +84,9 @@ export default function ScheduleTaskForm({ const [preApprovedAppIds, setPreApprovedAppIds] = useState( initial.preApprovedAppIds ); + const [preApprovedMcpServerIds, setPreApprovedMcpServerIds] = useState< + number[] + >(initial.preApprovedMcpServerIds); const [saving, setSaving] = useState(false); const [nameTouched, setNameTouched] = useState(false); const [promptTouched, setPromptTouched] = useState(false); @@ -206,11 +213,16 @@ export default function ScheduleTaskForm({ editor_mode: mode, editor_payload: storagePayload, pre_approved_app_ids: preApprovedAppIds, + pre_approved_mcp_server_ids: preApprovedMcpServerIds, }; const updated: ScheduledTaskDetail = await updateScheduledTask( initial.taskId, body ); + await mutate(SWR_KEYS.scheduledTask(updated.id), updated, { + revalidate: false, + }); + await mutate(SWR_KEYS.scheduledTasks); toast.success("Scheduled task updated."); router.push(taskDetailPath(updated.id)); } else { @@ -221,8 +233,10 @@ export default function ScheduleTaskForm({ editor_payload: storagePayload, run_immediately: runImmediately, pre_approved_app_ids: preApprovedAppIds, + pre_approved_mcp_server_ids: preApprovedMcpServerIds, }; await createScheduledTask(body); + await mutate(SWR_KEYS.scheduledTasks); toast.success( runImmediately ? "Scheduled task created and queued." @@ -243,8 +257,10 @@ export default function ScheduleTaskForm({ isEdit, initial.taskId, mode, + mutate, payload, preApprovedAppIds, + preApprovedMcpServerIds, router, trimmedName, trimmedPrompt, @@ -361,7 +377,7 @@ export default function ScheduleTaskForm({ - + @@ -375,16 +391,18 @@ export default function ScheduleTaskForm({ - + @@ -401,5 +419,6 @@ export function defaultFormInitial(): ScheduleTaskFormInitial { mode: "interval", payload: { unit: "hours", every: 1 }, preApprovedAppIds: [], + preApprovedMcpServerIds: [], }; } diff --git a/web/src/app/craft/v1/tasks/interfaces.ts b/web/src/app/craft/v1/tasks/interfaces.ts index 2331127eee9..839d678ab41 100644 --- a/web/src/app/craft/v1/tasks/interfaces.ts +++ b/web/src/app/craft/v1/tasks/interfaces.ts @@ -79,6 +79,7 @@ export interface ScheduledTaskDetail { next_runs: string[]; last_run: ScheduledRunSummary | null; pre_approved_app_ids: number[]; + pre_approved_mcp_server_ids: number[]; created_at: string; updated_at: string; } @@ -91,6 +92,7 @@ export interface ScheduledTaskCreateBody { status?: ScheduledTaskStatus; run_immediately?: boolean; pre_approved_app_ids?: number[]; + pre_approved_mcp_server_ids?: number[]; } export interface ScheduledTaskPatchBody { @@ -100,6 +102,7 @@ export interface ScheduledTaskPatchBody { editor_payload?: EditorPayload; status?: ScheduledTaskStatus; pre_approved_app_ids?: number[]; + pre_approved_mcp_server_ids?: number[]; } export interface ScheduledTaskListResponse { diff --git a/web/src/app/craft/v1/tasks/page.tsx b/web/src/app/craft/v1/tasks/page.tsx index 25b3d2679fc..9dec5658fcf 100644 --- a/web/src/app/craft/v1/tasks/page.tsx +++ b/web/src/app/craft/v1/tasks/page.tsx @@ -209,7 +209,7 @@ export default function ScheduledTasksListPage() {
) : error ? ( -
+
Failed to load scheduled tasks. diff --git a/web/src/app/ee/admin/performance/analytics/page.tsx b/web/src/app/ee/admin/performance/analytics/page.tsx new file mode 100644 index 00000000000..2f361401783 --- /dev/null +++ b/web/src/app/ee/admin/performance/analytics/page.tsx @@ -0,0 +1 @@ +export { default } from "@/views/admin/WorkspaceAnalyticsPage"; diff --git a/web/src/app/ee/admin/performance/lib.ts b/web/src/app/ee/admin/performance/lib.ts deleted file mode 100644 index 58c67df3572..00000000000 --- a/web/src/app/ee/admin/performance/lib.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { errorHandlingFetcher } from "@/lib/fetcher"; -import useSWR, { mutate } from "swr"; -import { OnyxBotAnalytics, QueryAnalytics, UserAnalytics } from "./usage/types"; -import { useState } from "react"; -import { buildApiPath } from "@/lib/urlBuilder"; - -import { - convertDateToEndOfDay, - convertDateToStartOfDay, - getXDaysAgo, -} from "../../../../components/dateRangeSelectors/dateUtils"; -import { THIRTY_DAYS } from "../../../../components/dateRangeSelectors/AdminDateRangeSelector"; -import { DateRangePickerValue } from "@/components/dateRangeSelectors/AdminDateRangeSelector"; - -export const useTimeRange = () => { - return useState({ - to: new Date(), - from: getXDaysAgo(30), - selectValue: THIRTY_DAYS, - }); -}; - -export const useQueryAnalytics = (timeRange: DateRangePickerValue) => { - const url = buildApiPath("/api/analytics/admin/query", { - start: convertDateToStartOfDay(timeRange.from)?.toISOString(), - end: convertDateToEndOfDay(timeRange.to)?.toISOString(), - }); - const swrResponse = useSWR(url, errorHandlingFetcher); - - return { - ...swrResponse, - refreshQueryAnalytics: () => mutate(url), - }; -}; - -export const useUserAnalytics = (timeRange: DateRangePickerValue) => { - const url = buildApiPath("/api/analytics/admin/user", { - start: convertDateToStartOfDay(timeRange.from)?.toISOString(), - end: convertDateToEndOfDay(timeRange.to)?.toISOString(), - }); - const swrResponse = useSWR(url, errorHandlingFetcher); - - return { - ...swrResponse, - refreshUserAnalytics: () => mutate(url), - }; -}; - -export const useOnyxBotAnalytics = (timeRange: DateRangePickerValue) => { - const url = buildApiPath("/api/analytics/admin/onyxbot", { - start: convertDateToStartOfDay(timeRange.from)?.toISOString(), - end: convertDateToEndOfDay(timeRange.to)?.toISOString(), - }); - const swrResponse = useSWR(url, errorHandlingFetcher); // TODO - - return { - ...swrResponse, - refreshOnyxBotAnalytics: () => mutate(url), - }; -}; - -export function getDatesList(startDate: Date): string[] { - const datesList: string[] = []; - const endDate = new Date(); // current date - - for (let d = new Date(startDate); d <= endDate; d.setDate(d.getDate() + 1)) { - const dateStr = d.toISOString().split("T")[0]; // convert date object to 'YYYY-MM-DD' format - if (dateStr !== undefined) { - datesList.push(dateStr); - } - } - - return datesList; -} - -export interface PersonaMessageAnalytics { - total_messages: number; - date: string; - persona_id: number; -} - -export interface PersonaSnapshot { - id: number; - name: string; - description: string; - is_listed: boolean; - is_public: boolean; -} - -export const usePersonaMessages = ( - personaId: number | undefined, - timeRange: DateRangePickerValue -) => { - const url = buildApiPath(`/api/analytics/admin/persona/messages`, { - persona_id: personaId?.toString(), - start: convertDateToStartOfDay(timeRange.from)?.toISOString(), - end: convertDateToEndOfDay(timeRange.to)?.toISOString(), - }); - - const { data, error, isLoading } = useSWR( - personaId !== undefined ? url : null, - errorHandlingFetcher - ); - - return { - data, - error, - isLoading, - refreshPersonaMessages: () => mutate(url), - }; -}; - -export interface PersonaUniqueUserAnalytics { - unique_users: number; - date: string; - persona_id: number; -} - -export const usePersonaUniqueUsers = ( - personaId: number | undefined, - timeRange: DateRangePickerValue -) => { - const url = buildApiPath(`/api/analytics/admin/persona/unique-users`, { - persona_id: personaId?.toString(), - start: convertDateToStartOfDay(timeRange.from)?.toISOString(), - end: convertDateToEndOfDay(timeRange.to)?.toISOString(), - }); - - const { data, error, isLoading } = useSWR( - personaId !== undefined ? url : null, - errorHandlingFetcher - ); - - return { - data, - error, - isLoading, - refreshPersonaUniqueUsers: () => mutate(url), - }; -}; diff --git a/web/src/app/ee/admin/performance/query-history/KickoffCSVExport.tsx b/web/src/app/ee/admin/performance/query-history/KickoffCSVExport.tsx index 8e202887e03..050c357ace5 100644 --- a/web/src/app/ee/admin/performance/query-history/KickoffCSVExport.tsx +++ b/web/src/app/ee/admin/performance/query-history/KickoffCSVExport.tsx @@ -1,7 +1,7 @@ import { toast } from "@opal/layouts"; import Button from "@/refresh-components/buttons/Button"; import { useRef, useState } from "react"; -import { DateRange } from "../../../../../components/dateRangeSelectors/AdminDateRangeSelector"; +import type { DateRange } from "@/refresh-components/DateRangePicker"; import { withRequestId, withDateRange } from "./utils"; import { CHECK_QUERY_HISTORY_EXPORT_STATUS_URL, diff --git a/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx b/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx index e3a563d38e0..91697e05c7f 100644 --- a/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx +++ b/web/src/app/ee/admin/performance/query-history/QueryHistoryTable.tsx @@ -16,8 +16,8 @@ import { Dispatch, SetStateAction, useCallback, useState } from "react"; import { Feedback, TaskStatus } from "@/lib/types"; import { DateRange, - AdminDateRangeSelector, -} from "@/components/dateRangeSelectors/AdminDateRangeSelector"; + DateRangePicker, +} from "@/refresh-components/DateRangePicker"; import { PageSelector } from "@/components/PageSelector"; import Link from "next/link"; import type { Route } from "next"; @@ -101,7 +101,7 @@ function SelectFeedbackType({ onValueChange: (value: Feedback | "all") => void; }) { return ( -
+
Feedback Type @@ -316,7 +316,7 @@ export function QueryHistoryTable() { }} /> - diff --git a/web/src/app/ee/admin/performance/query-history/utils.ts b/web/src/app/ee/admin/performance/query-history/utils.ts index b33336074b5..9e6a50811d4 100644 --- a/web/src/app/ee/admin/performance/query-history/utils.ts +++ b/web/src/app/ee/admin/performance/query-history/utils.ts @@ -1,4 +1,4 @@ -import { DateRange } from "../../../../../components/dateRangeSelectors/AdminDateRangeSelector"; +import type { DateRange } from "@/refresh-components/DateRangePicker"; import { START_QUERY_HISTORY_EXPORT_URL } from "./constants"; export const withRequestId = (url: string, requestId: string): string => diff --git a/web/src/app/ee/admin/performance/usage/page.tsx b/web/src/app/ee/admin/performance/usage/page.tsx index ea6075692f9..56ef0caa857 100644 --- a/web/src/app/ee/admin/performance/usage/page.tsx +++ b/web/src/app/ee/admin/performance/usage/page.tsx @@ -1,7 +1,7 @@ "use client"; -import { AdminDateRangeSelector } from "@/components/dateRangeSelectors/AdminDateRangeSelector"; -import { useTimeRange } from "@/app/ee/admin/performance/lib"; +import { DateRangePicker } from "@/refresh-components/DateRangePicker"; +import { useTimeRange } from "@/lib/usage/hooks"; import PerUserUsagePanel from "@/views/admin/PerUserUsagePanel"; import { ADMIN_ROUTES } from "@/lib/admin-routes"; import { Divider } from "@opal/components"; @@ -25,7 +25,7 @@ export default function UsagePage() { setTimeRange(value as any)} /> diff --git a/web/src/app/ee/admin/performance/usage/types.ts b/web/src/app/ee/admin/performance/usage/types.ts index 7c20155940b..52789294fc9 100644 --- a/web/src/app/ee/admin/performance/usage/types.ts +++ b/web/src/app/ee/admin/performance/usage/types.ts @@ -1,23 +1,5 @@ import { Feedback, SessionType } from "@/lib/types"; -export interface QueryAnalytics { - total_queries: number; - total_likes: number; - total_dislikes: number; - date: string; -} - -export interface UserAnalytics { - total_active_users: number; - date: string; -} - -export interface OnyxBotAnalytics { - total_queries: number; - auto_resolved: number; - date: string; -} - export interface AbridgedSearchDoc { document_id: string; semantic_identifier: string; diff --git a/web/src/app/ee/agents/stats/[id]/AgentStats.tsx b/web/src/app/ee/agents/stats/[id]/AgentStats.tsx index f3a1e853393..3e3bcfe5298 100644 --- a/web/src/app/ee/agents/stats/[id]/AgentStats.tsx +++ b/web/src/app/ee/agents/stats/[id]/AgentStats.tsx @@ -1,30 +1,61 @@ "use client"; -import SvgSimpleLoader from "@opal/icons/simple-loader"; -import { getDatesList } from "@/app/ee/admin/performance/lib"; import { useEffect, useState, useMemo } from "react"; +import { Card, Text } from "@opal/components"; +import { Section } from "@opal/layouts"; import { - AdminDateRangeSelector, + DateRangePicker, DateRange, -} from "@/components/dateRangeSelectors/AdminDateRangeSelector"; +} from "@/refresh-components/DateRangePicker"; import { useAgents } from "@/lib/agents/hooks"; import AgentAvatar from "@/refresh-components/avatars/AgentAvatar"; -import { Card, CardContent, CardHeader } from "@/components/ui/card"; -import { AreaChartDisplay } from "@/components/ui/areaChart"; +import { + AnalyticsChart, + chartSeries, + resolveChartState, +} from "@/sections/usage/AnalyticsChart"; +import { ChartState } from "@/sections/usage/interfaces"; -type AgentDailyUsageEntry = { +interface AgentDailyUsageEntry { date: string; total_messages: number; total_unique_users: number; -}; +} -type AgentStatsResponse = { +interface AgentStatsResponse { daily_stats: AgentDailyUsageEntry[]; total_messages: number; total_unique_users: number; -}; +} + +interface SummaryMetricProps { + label: string; + value: number; +} + +function SummaryMetric({ label, value }: SummaryMetricProps) { + return ( +
+ + {label} + + {value.toLocaleString()} +
+ ); +} + +interface AgentStatsProps { + agentId: number; +} -export function AgentStats({ agentId }: { agentId: number }) { +export function AgentStats({ agentId }: AgentStatsProps) { const [agentStats, setAgentStats] = useState(null); const { agents } = useAgents(); const [isLoading, setIsLoading] = useState(false); @@ -71,120 +102,108 @@ export function AgentStats({ agentId }: { agentId: number }) { fetchStats(); }, [agentId, dateRange]); - const chartData = useMemo(() => { - if (!agentStats?.daily_stats?.length || !dateRange) { - return null; - } - - const initialDate = - dateRange.from || - new Date( - Math.min( - ...agentStats.daily_stats.map((entry) => - new Date(entry.date).getTime() - ) - ) - ); - const endDate = dateRange.to || new Date(); - - const dateRangeList = getDatesList(initialDate); - - const statsMap = new Map( - agentStats.daily_stats.map((entry) => [entry.date, entry]) - ); - - return dateRangeList - .filter((date) => new Date(date) <= endDate) - .map((dateStr) => { - const dayData = statsMap.get(dateStr); - return { - Day: dateStr, - Messages: dayData?.total_messages || 0, - "Unique Users": dayData?.total_unique_users || 0, - }; + const state: ChartState = error + ? { status: "error", message: error } + : resolveChartState({ + isLoading: isLoading || !agent, + error: null, + errorMessage: "Failed to fetch agent stats.", + emptyMessage: + "No data found for this agent in the selected date range.", + series: [ + chartSeries( + "Messages", + agentStats?.daily_stats, + (entry) => entry.total_messages + ), + chartSeries( + "Unique Users", + agentStats?.daily_stats, + (entry) => entry.total_unique_users + ), + ], }); - }, [agentStats, dateRange]); - - const totalMessages = agentStats?.total_messages ?? 0; - const totalUniqueUsers = agentStats?.total_unique_users ?? 0; - let content; - if (isLoading || !agent) { - content = ( -
- -
- ); - } else if (error) { - content = ( -
-

{error}

-
- ); - } else if (!agentStats?.daily_stats?.length) { - content = ( -
-

- No data found for this agent in the selected date range -

+ return ( +
+ {/* sm:flex-row / sm:items-center / sm:justify-between have no Section equivalent, kept as a raw div */} +
+ Agent Analytics +
- ); - } else if (chartData) { - content = ( - - ); - } - return ( - - -

Agent Analytics

- - - -
- - -
- {agent && } -
-

{agent?.name}

-

{agent?.description}

-
-
- +
+
+ +
+ {agent && } +
+ {agent?.name ?? ""} + + {agent?.description ?? ""} + +
+
- - -
-
-

- Total Messages -

-

{totalMessages}

-
-
-

- Total Unique Users -

-

{totalUniqueUsers}

-
-
- +
+
+ +
+ + +
- {content} - - +
+ + +
); } diff --git a/web/src/app/mcp/oauth/callback/page.tsx b/web/src/app/mcp/oauth/callback/page.tsx index 15875cb6ee0..5289aa4d62d 100644 --- a/web/src/app/mcp/oauth/callback/page.tsx +++ b/web/src/app/mcp/oauth/callback/page.tsx @@ -185,7 +185,7 @@ export default function MCPOAuthCallbackPage() { return (
- +
{state.phase === "processing" && (
diff --git a/web/src/components/dateRangeSelectors/SearchDateRangeSelector.tsx b/web/src/components/dateRangeSelectors/SearchDateRangeSelector.tsx index e8cdd92e2e3..f6d0bba50e1 100644 --- a/web/src/components/dateRangeSelectors/SearchDateRangeSelector.tsx +++ b/web/src/components/dateRangeSelectors/SearchDateRangeSelector.tsx @@ -1,4 +1,4 @@ -import { DateRangePickerValue } from "@/components/dateRangeSelectors/AdminDateRangeSelector"; +import { DateRangePickerValue } from "@/refresh-components/DateRangePicker"; import { FiCalendar, FiChevronDown, FiXCircle } from "react-icons/fi"; import { CustomDropdown } from "../Dropdown"; import { timeRangeValues } from "@/app/config/timeRange"; diff --git a/web/src/components/dateRangeSelectors/dateUtils.ts b/web/src/components/dateRangeSelectors/dateUtils.ts deleted file mode 100644 index caffdd999ab..00000000000 --- a/web/src/components/dateRangeSelectors/dateUtils.ts +++ /dev/null @@ -1,26 +0,0 @@ -export function getXDaysAgo(daysAgo: number) { - const today = new Date(); - const daysAgoDate = new Date(today); - daysAgoDate.setDate(today.getDate() - daysAgo); - return daysAgoDate; -} - -export function convertDateToEndOfDay(date?: Date | null) { - if (!date) { - return date; - } - - const dateCopy = new Date(date); - dateCopy.setHours(23, 59, 59, 999); - return dateCopy; -} - -export function convertDateToStartOfDay(date?: Date | null) { - if (!date) { - return date; - } - - const dateCopy = new Date(date); - dateCopy.setHours(0, 0, 0, 0); - return dateCopy; -} diff --git a/web/src/components/filters/SourceSelector.tsx b/web/src/components/filters/SourceSelector.tsx index fef863692a3..083e5162d1e 100644 --- a/web/src/components/filters/SourceSelector.tsx +++ b/web/src/components/filters/SourceSelector.tsx @@ -3,7 +3,7 @@ import { DocumentSetSummary, Tag, ValidSources } from "@/lib/types"; import { SourceMetadata } from "@/lib/search/interfaces"; import { FiBook, FiBookmark, FiMap, FiX } from "react-icons/fi"; import { SearchDateRangeSelector } from "@/components/dateRangeSelectors/SearchDateRangeSelector"; -import { DateRangePickerValue } from "@/components/dateRangeSelectors/AdminDateRangeSelector"; +import { DateRangePickerValue } from "@/refresh-components/DateRangePicker"; import { listSourceMetadata } from "@/lib/sources"; import { SourceIcon } from "@/components/SourceIcon"; import { FilterDropdown } from "@/components/search/filtering/FilterDropdown"; diff --git a/web/src/components/ui/areaChart.tsx b/web/src/components/ui/areaChart.tsx deleted file mode 100644 index 5cb2221c6cd..00000000000 --- a/web/src/components/ui/areaChart.tsx +++ /dev/null @@ -1,120 +0,0 @@ -"use client"; - -import React from "react"; -import { - Area, - AreaChart as ReChartsAreaChart, - CartesianGrid, - ResponsiveContainer, - Tooltip, - XAxis, - YAxis, -} from "recharts"; - -import { - Card, - CardContent, - CardDescription, - CardHeader, - CardTitle, -} from "@/components/ui/card"; - -interface AreaChartProps { - data?: Array>; - categories?: string[]; - index?: string; - colors?: string[]; - showXAxis?: boolean; - showYAxis?: boolean; - yAxisWidth?: number; - showAnimation?: boolean; - showTooltip?: boolean; - showGridLines?: boolean; - connectNulls?: boolean; - allowDecimals?: boolean; - className?: string; - title?: string; - description?: string; - xAxisFormatter?: (value: string) => string; - yAxisFormatter?: (value: number) => string; - stacked?: boolean; -} - -export function AreaChartDisplay({ - data = [], - categories = [], - index, - colors = ["indigo", "fuchsia"], - showXAxis = true, - showYAxis = true, - yAxisWidth = 56, - showAnimation = true, - showTooltip = true, - showGridLines = true, - connectNulls = false, - allowDecimals = true, - className, - title, - description, - xAxisFormatter = (dateStr: string) => dateStr, - yAxisFormatter = (number: number) => number.toString(), - stacked = false, -}: AreaChartProps) { - return ( - - - {title && {title}} - {description && {description}} - - -
- - - {showGridLines && } - {showXAxis && ( - xAxisFormatter(value)} - /> - )} - {showYAxis && ( - yAxisFormatter(value)} - allowDecimals={allowDecimals} - /> - )} - {showTooltip && } - {categories.map((category, ind) => ( - - ))} - - -
- - - ); -} diff --git a/web/src/ee/sections/SearchCard.tsx b/web/src/ee/sections/SearchCard.tsx index 2ab3c782bfe..42fb3bd82fe 100644 --- a/web/src/ee/sections/SearchCard.tsx +++ b/web/src/ee/sections/SearchCard.tsx @@ -57,13 +57,13 @@ export default function SearchCard({ return ( -
+
{/* Title Row */}
{isWebSource && document.link ? ( @@ -78,9 +78,9 @@ export default function SearchCard({ {/* Body Row */}
-
+
{/* Metadata */} -
+
{(document.primary_owners ?? []).map((owner, index) => ( {owner} ))} diff --git a/web/src/ee/sections/SearchUI.tsx b/web/src/ee/sections/SearchUI.tsx index 6c30fd0c705..c76a5550582 100644 --- a/web/src/ee/sections/SearchUI.tsx +++ b/web/src/ee/sections/SearchUI.tsx @@ -312,7 +312,7 @@ export default function SearchUI({ onDocumentClick }: SearchResultsProps) {
- +
{!showEmpty && ( @@ -323,7 +323,7 @@ export default function SearchUI({ onDocumentClick }: SearchResultsProps) {
- +
)} @@ -368,7 +368,7 @@ export default function SearchUI({ onDocumentClick }: SearchResultsProps) { {!showEmpty && (
-
+
{sourcesWithMeta.map(({ source, meta, count }) => (
diff --git a/web/src/ee/views/admin/HooksPage/HookLogsModal.tsx b/web/src/ee/views/admin/HooksPage/HookLogsModal.tsx index c32feeb0feb..199b89a675b 100644 --- a/web/src/ee/views/admin/HooksPage/HookLogsModal.tsx +++ b/web/src/ee/views/admin/HooksPage/HookLogsModal.tsx @@ -47,7 +47,7 @@ function LogRow({ log, group }: { log: HookExecutionRecord; group: string }) { flexDirection="row" justifyContent="start" alignItems="start" - gap={0.5} + gap={2} height="fit" className="py-2" > @@ -157,7 +157,7 @@ export default function HookLogsModal({ hook, spec }: HookLogsModalProps) { flexDirection="row" justifyContent="between" alignItems="center" - padding={0.5} + padding={2} className="bg-background-tint-01" > @@ -167,8 +167,8 @@ export default function HookLogsModal({ hook, spec }: HookLogsModalProps) { flexDirection="row" alignItems="center" width="fit" - gap={0.25} - padding={0.25} + gap={1} + padding={1} className="rounded-xl bg-background-tint-00" > diff --git a/web/src/ee/views/admin/HooksPage/HookStatusPopover.tsx b/web/src/ee/views/admin/HooksPage/HookStatusPopover.tsx index 70ec4e07be0..bfd8db10dd7 100644 --- a/web/src/ee/views/admin/HooksPage/HookStatusPopover.tsx +++ b/web/src/ee/views/admin/HooksPage/HookStatusPopover.tsx @@ -39,8 +39,8 @@ function ErrorLogRow({ flexDirection="column" justifyContent="start" alignItems="start" - gap={0.25} - padding={0.25} + gap={1} + padding={1} height="fit" >
{isLoading ? (
@@ -250,14 +250,14 @@ export default function HookStatusPopover({ {topErrors.length > 0 ? ( <> - +
{topErrors.map((log, idx) => ( @@ -270,7 +270,7 @@ export default function HookStatusPopover({
) : ( - + )}
- + {/* Log rows — at most 3, timestamp first then error message */}
{recentErrors.slice(0, 3).map((log, idx) => ( @@ -352,7 +352,7 @@ export default function HookStatusPopover({ />
- + {/* View Older Errors */} +
{/* TODO(@raunakab): Modify the background colour (by using `SelectCard disabled={...}` [when it lands]) to indicate when the card is "disconnected". */} - +
( - SWR_KEYS.buildExternalApps, + enabled ? SWR_KEYS.buildExternalApps : null, errorHandlingFetcher ); diff --git a/web/src/layouts/chromes/AdminChrome.tsx b/web/src/layouts/chromes/AdminChrome.tsx index c1bd48d9781..91389072774 100644 --- a/web/src/layouts/chromes/AdminChrome.tsx +++ b/web/src/layouts/chromes/AdminChrome.tsx @@ -20,6 +20,10 @@ export interface AdminChromeProps { children: React.ReactNode; } +// The create-connector page (`/admin/connectors/`) renders its own +// sidebar. Routes below it do not. +const CUSTOM_SIDEBAR_ROUTE = /^\/admin\/connectors\/[^/]+\/?$/; + // Lets a page render its own sidebar into the chrome as a sibling of the main // content column — i.e. *outside* the scrollable region — so it stays pinned // while the page scrolls. The page keeps ownership (and React context) of the @@ -48,13 +52,15 @@ export default function AdminChrome({ children }: AdminChromeProps) { // Certain admin panels have their own custom sidebar. // For those pages, we skip rendering the default `AdminSidebar` and let those individual pages render their own. - const hasCustomSidebar = pathname.startsWith("/admin/connectors"); + // The OAuth callback / finalize interstitials below the create-connector page + // render no sidebar of their own, so they keep the default one. + const hasCustomSidebar = CUSTOM_SIDEBAR_ROUTE.test(pathname); let content = children; if (isVectorDbRequiredRoute(pathname)) { if (isLoading) { content = ( -
+
); @@ -90,7 +96,9 @@ export default function AdminChrome({ children }: AdminChromeProps) { )} - {isMobile && !hasCustomSidebar && ( + {/* On mobile every sidebar is an off-screen overlay, so the main + column always needs a control to bring it back. */} + {isMobile && (
diff --git a/web/src/sections/modals/PreviewModal/variants/docxVariant.tsx b/web/src/sections/modals/PreviewModal/variants/docxVariant.tsx index 2f7bab3d1be..339370ff846 100644 --- a/web/src/sections/modals/PreviewModal/variants/docxVariant.tsx +++ b/web/src/sections/modals/PreviewModal/variants/docxVariant.tsx @@ -107,7 +107,7 @@ function DocxPreview({ fileUrl, onLoad }: DocxPreviewProps) { if (error) { return ( -
+
{error} @@ -160,7 +160,7 @@ export const docxVariant: PreviewVariant = { if (isLegacyDoc(ctx.fileName)) { lastDocxResult = null; return ( -
+
Legacy .doc format cannot be previewed. Download the file to view it. diff --git a/web/src/sections/modals/PreviewModal/variants/xlsxVariant.tsx b/web/src/sections/modals/PreviewModal/variants/xlsxVariant.tsx index 64a327da429..5628c59675f 100644 --- a/web/src/sections/modals/PreviewModal/variants/xlsxVariant.tsx +++ b/web/src/sections/modals/PreviewModal/variants/xlsxVariant.tsx @@ -38,7 +38,7 @@ export const xlsxVariant: PreviewVariant = { const preview = parseSpreadsheetPreview(ctx.fileContent); if (!preview || preview.sheets.length === 0) { return ( -
+
Unable to preview this spreadsheet. diff --git a/web/src/sections/modals/ShareAgentModal.tsx b/web/src/sections/modals/ShareAgentModal.tsx index 6f9a5a01f92..7b9ba76a116 100644 --- a/web/src/sections/modals/ShareAgentModal.tsx +++ b/web/src/sections/modals/ShareAgentModal.tsx @@ -568,7 +568,7 @@ export default function ShareAgentModal({ } /> - + {/* Admins always appear and always hold edit access (ENG-4175); on vacant agents this row carries the transfer affordance */} diff --git a/web/src/sections/modals/ShareChatSessionModal.tsx b/web/src/sections/modals/ShareChatSessionModal.tsx index 5a9f1408e89..1d9653af89c 100644 --- a/web/src/sections/modals/ShareChatSessionModal.tsx +++ b/web/src/sections/modals/ShareChatSessionModal.tsx @@ -63,7 +63,7 @@ function PrivacyOption({ return ( +
{displayedUnavailableReason && ( -
+
Instructions diff --git a/web/src/sections/modals/UserFilesModal.tsx b/web/src/sections/modals/UserFilesModal.tsx index b5a8e0c80f8..b014d5e9824 100644 --- a/web/src/sections/modals/UserFilesModal.tsx +++ b/web/src/sections/modals/UserFilesModal.tsx @@ -178,7 +178,7 @@ export default function UserFilesModal({ > {/* Search bar section */} -
+
{/* File display section */} @@ -263,7 +263,7 @@ export default function UserFilesModal({ {/* Left side: file count and controls */} {onPickRecent && ( -
+
{selectedCount} {selectedCount === 1 ? "file" : "files"}{" "} selected diff --git a/web/src/sections/modals/languageModels/BedrockModal.tsx b/web/src/sections/modals/languageModels/BedrockModal.tsx index fd0532ddbc0..e06c53a5b85 100644 --- a/web/src/sections/modals/languageModels/BedrockModal.tsx +++ b/web/src/sections/modals/languageModels/BedrockModal.tsx @@ -129,7 +129,7 @@ function BedrockModalInternals({ return ( <> -
+
{authMethod === AUTH_METHOD_ACCESS_KEY && ( - -
+ +
-
+ +
) : ( - + )}
-
+
{undismissedCount !== 0 && ( {`${undismissedCount} unread`} diff --git a/web/src/sections/sidebar/StepSidebarWrapper.tsx b/web/src/sections/sidebar/StepSidebarWrapper.tsx deleted file mode 100644 index 818320e1e07..00000000000 --- a/web/src/sections/sidebar/StepSidebarWrapper.tsx +++ /dev/null @@ -1,40 +0,0 @@ -"use client"; - -import { ReactNode } from "react"; -import type { IconProps } from "@opal/types"; -import { SidebarLayouts } from "@opal/layouts"; -import { SidebarTab } from "@opal/components"; -import { renderSidebarLogo } from "@/lib/sidebar/utils"; -import { useShowLogoWhenFolded } from "@/lib/sidebar/hooks"; - -export interface StepSidebarProps { - children: ReactNode; - buttonName: string; - buttonIcon: React.FunctionComponent; - buttonHref: string; -} - -export default function StepSidebar({ - children, - buttonName, - buttonIcon, - buttonHref, -}: StepSidebarProps) { - const showLogoWhenFolded = useShowLogoWhenFolded(); - - return ( - - - - {buttonName} - - - - {children} - - - ); -} diff --git a/web/src/sections/usage/AnalyticsChart.tsx b/web/src/sections/usage/AnalyticsChart.tsx new file mode 100644 index 00000000000..2820d274771 --- /dev/null +++ b/web/src/sections/usage/AnalyticsChart.tsx @@ -0,0 +1,207 @@ +import React from "react"; +import { Card, EmptyMessageCard, MessageCard, Text } from "@opal/components"; +import { SvgX } from "@opal/icons"; +import { PageLoader, Section } from "@opal/layouts"; +import type { RichStr } from "@opal/types"; +import AreaChart from "@/refresh-components/AreaChart"; +import { getDatesList } from "@/lib/usage/utils"; +import { DateRange } from "@/refresh-components/DateRangePicker"; +import { ChartSeries, ChartState } from "@/sections/usage/interfaces"; + +const CHART_BODY_HEIGHT = 20; + +// "YYYY-MM-DD" parses as UTC midnight, which renders a day early west of UTC, +// so build the Date from the parts instead. +function formatDay(dateStr: string): string { + const [year, month, day] = dateStr.split("-").map(Number); + if (year === undefined || month === undefined || day === undefined) { + return dateStr; + } + return new Date(year, month - 1, day).toLocaleDateString("en-US", { + month: "short", + day: "numeric", + }); +} + +/** Keeps the underlying failure in the console when a chart shows its error card. */ +export function useLoggedChartError(label: string, error: unknown): void { + React.useEffect(() => { + if (error) console.error(`${label} analytics request failed:`, error); + }, [label, error]); +} + +export function chartSeries( + label: string, + data: T[] | undefined, + value: (entry: T) => number +): ChartSeries { + const entries = data ?? []; + const byDate = new Map(entries.map((entry) => [entry.date, entry])); + const dates = entries.map((entry) => entry.date).sort(); + + return { + label, + isEmpty: entries.length === 0, + firstDate: dates[0], + valueForDate: (date) => { + const entry = byDate.get(date); + return entry === undefined ? 0 : value(entry); + }, + }; +} + +interface ResolveChartStateArgs { + isLoading: boolean; + error: unknown; + series: ChartSeries[]; + errorMessage: string; + emptyMessage: string; +} + +export function resolveChartState({ + isLoading, + error, + series, + errorMessage, + emptyMessage, +}: ResolveChartStateArgs): ChartState { + if (error) return { status: "error", message: errorMessage }; + if (isLoading) return { status: "loading" }; + if (series.every((entry) => entry.isEmpty)) { + return { status: "empty", message: emptyMessage }; + } + return { status: "ready", series }; +} + +interface ChartBodyProps { + state: ChartState; + timeRange: DateRange; + stacked: boolean; + allowDecimals: boolean; + xAxisFormatter: (value: string) => string; + yAxisFormatter?: (value: number) => string; +} + +function ChartBody({ + state, + timeRange, + stacked, + allowDecimals, + xAxisFormatter, + yAxisFormatter, +}: ChartBodyProps) { + if (state.status === "error") { + return ; + } + + if (state.status === "loading") { + return ( +
+ +
+ ); + } + + if (state.status === "empty") { + return ; + } + + const earliest = state.series + .map((entry) => entry.firstDate) + .filter((date): date is string => date !== undefined) + .sort()[0]; + const dateRange = getDatesList( + timeRange?.from ?? new Date(earliest ?? Date.now()), + timeRange?.to + ); + + return ( + + state.series.reduce>( + (row, entry) => { + row[entry.label] = entry.valueForDate(date); + return row; + }, + { Day: date } + ) + )} + categories={state.series.map((entry) => entry.label)} + index="Day" + yAxisWidth={60} + stacked={stacked} + allowDecimals={allowDecimals} + xAxisFormatter={xAxisFormatter} + {...(yAxisFormatter && { yAxisFormatter })} + /> + ); +} + +interface AnalyticsChartProps { + title: string | RichStr; + description: string | RichStr; + timeRange: DateRange; + state: ChartState; + headerChildren?: React.ReactNode; + stacked?: boolean; + allowDecimals?: boolean; + xAxisFormatter?: (value: string) => string; + yAxisFormatter?: (value: number) => string; +} + +export function AnalyticsChart({ + title, + description, + timeRange, + state, + headerChildren, + stacked = false, + allowDecimals = true, + xAxisFormatter = formatDay, + yAxisFormatter, +}: AnalyticsChartProps) { + return ( + +
+ {/* sm:flex-row / sm:items-center / sm:justify-between have no Section equivalent, kept as a raw div */} +
+
+ {title} + + {description} + +
+ {headerChildren} +
+ +
+ + ); +} diff --git a/web/src/sections/usage/SpendByUserTable.tsx b/web/src/sections/usage/SpendByUserTable.tsx index 85d642e3149..a9e5e5759e6 100644 --- a/web/src/sections/usage/SpendByUserTable.tsx +++ b/web/src/sections/usage/SpendByUserTable.tsx @@ -186,7 +186,7 @@ export default function SpendByUserTable({ flexDirection="column" justifyContent="start" alignItems="stretch" - gap={0.5} + gap={2} width="full" height="fit" > diff --git a/web/src/sections/usage/UserUsageDetailModal.tsx b/web/src/sections/usage/UserUsageDetailModal.tsx index 3171572fe23..29c22195329 100644 --- a/web/src/sections/usage/UserUsageDetailModal.tsx +++ b/web/src/sections/usage/UserUsageDetailModal.tsx @@ -90,7 +90,7 @@ function BreakdownList({ title, slices, totalCostCents }: BreakdownListProps) { flexDirection="column" justifyContent="start" alignItems="stretch" - gap={0.5} + gap={2} width="full" height="fit" > @@ -101,7 +101,7 @@ function BreakdownList({ title, slices, totalCostCents }: BreakdownListProps) { flexDirection="column" justifyContent="start" alignItems="stretch" - gap={0.625} + gap={2.5} width="full" height="fit" > @@ -114,7 +114,7 @@ function BreakdownList({ title, slices, totalCostCents }: BreakdownListProps) { flexDirection="column" justifyContent="start" alignItems="stretch" - gap={0.25} + gap={1} width="full" height="fit" > @@ -157,7 +157,7 @@ function DailySpendStrip({ days }: { days: DailySpend[] }) { flexDirection="column" justifyContent="start" alignItems="stretch" - gap={0.25} + gap={1} width="full" height="fit" > @@ -175,7 +175,7 @@ function DailySpendStrip({ days }: { days: DailySpend[] }) { flexDirection="row" justifyContent="start" alignItems="end" - gap={0.125} + gap={0.5} width="full" height={3.5} > @@ -260,7 +260,7 @@ export default function UserUsageDetailModal({ onClose={() => onOpenChange(false)} /> -
+
diff --git a/web/src/sections/usage/interfaces.ts b/web/src/sections/usage/interfaces.ts new file mode 100644 index 00000000000..cb6442ccaf1 --- /dev/null +++ b/web/src/sections/usage/interfaces.ts @@ -0,0 +1,12 @@ +export interface ChartSeries { + label: string; + isEmpty: boolean; + firstDate: string | undefined; + valueForDate: (date: string) => number; +} + +export type ChartState = + | { status: "loading" } + | { status: "error"; message: string } + | { status: "empty"; message: string } + | { status: "ready"; series: ChartSeries[] }; diff --git a/web/src/views/AgentEditorPage.tsx b/web/src/views/AgentEditorPage.tsx index 9325a2e0955..48cd1de595f 100644 --- a/web/src/views/AgentEditorPage.tsx +++ b/web/src/views/AgentEditorPage.tsx @@ -333,21 +333,21 @@ function MCPServerCard({ if (isLoading) { cardContent = (
- +
); } else if (hasTools) { cardContent = ( - + {filteredTools.map((tool) => { const toolDisabled = !tool.isAvailable || !getFieldMeta(`${serverFieldName}.enabled`).value; return ( - + + {(arrayHelpers) => ( - + {Array.from({ length: visibleCount }, (_, i) => ( deleteAgentModal.toggle(false)} > - + Anyone using this agent will no longer be able to access it. Deletion cannot be undone. @@ -1373,7 +1373,7 @@ export default function AgentEditorPage({ @@ -1406,10 +1406,7 @@ export default function AgentEditorPage({ - + - + - + @@ -1539,7 +1530,7 @@ export default function AgentEditorPage({ )} - + - + 0 || openApiTools.length > 0) && ( )} {/* MCP tools */} {mcpServersWithVisibleTools.length > 0 && ( {mcpServersWithVisibleTools.map( @@ -1710,7 +1695,7 @@ export default function AgentEditorPage({ {/* OpenAPI tools */} {openApiTools.length > 0 && ( - + {openApiTools.map((tool) => ( - + - + diff --git a/web/src/views/AppPage.tsx b/web/src/views/AppPage.tsx index c110bc155aa..c5b5c0e7851 100644 --- a/web/src/views/AppPage.tsx +++ b/web/src/views/AppPage.tsx @@ -855,7 +855,7 @@ export default function AppPage({ firstMessage }: ChatPageProps) {
} > -
+
} > -
+
All your chat sessions and history will be permanently deleted. Deletion cannot be undone. @@ -464,8 +464,8 @@ function GeneralSettings() { )} -
-
+
+
-
+
- + -
+
{shortcuts.length > 0 && ( -
+
{shortcuts.map((shortcut, index) => { const isEmpty = !shortcut.prompt.trim() && !shortcut.content.trim(); const isExisting = !shortcut.isNew; @@ -1028,8 +1028,8 @@ function ChatPreferencesSettings() { ); return ( -
-
+
+
-
+
-
+
-
+
} > -
+
{`Any application using the token ${tokenToDelete.name} (${tokenToDelete.token_display}) will lose access to Onyx. This action cannot be undone.`} @@ -1616,8 +1616,8 @@ function AccountsAccessSettings() { setShowPasswordModal(false); }} > -
-
+
+
-
+
-
+
)} -
-
+
+
{showTokensSection && ( -
+
{canCreateTokens ? ( - +
-
+
{pats.length === 0 ? (
@@ -1748,7 +1748,7 @@ function AccountsAccessSettings() {
-
+
{filteredPats.map((pat) => { const now = new Date(); const createdDate = new Date(pat.created_at); @@ -1907,7 +1907,7 @@ function FederatedConnectorCard({ } > -
+
{`Onyx will no longer be able to access or search content from your ${sourceMetadata.displayName} account.`} @@ -1918,7 +1918,7 @@ function FederatedConnectorCard({ )} - + 0 || federatedConnectors.length > 0; return ( -
-
+
+
-
- -
+
+ +
- + )} @@ -641,7 +641,7 @@ export default function SkillEditorPage({
- +
@@ -657,7 +657,7 @@ export default function SkillEditorPage({ />
- + {instructionsDisplayMode === "raw" ? (
- + -
+
- + -
+
- +
diff --git a/web/src/views/admin/AgentsPage/AgentsTable.tsx b/web/src/views/admin/AgentsPage/AgentsTable.tsx index 552579a43db..fe17fdd4d6f 100644 --- a/web/src/views/admin/AgentsPage/AgentsTable.tsx +++ b/web/src/views/admin/AgentsPage/AgentsTable.tsx @@ -144,14 +144,14 @@ export default function AgentsTable() { return (
-
+
setSearchTerm(e.target.value)} placeholder="Search agents..." searchIcon /> -
+
{filterBar}
diff --git a/web/src/views/admin/ChatPreferencesPage.tsx b/web/src/views/admin/ChatPreferencesPage.tsx index 655e03e21ca..1e661b98337 100644 --- a/web/src/views/admin/ChatPreferencesPage.tsx +++ b/web/src/views/admin/ChatPreferencesPage.tsx @@ -128,10 +128,10 @@ function MCPServerCard({ expanded={expanded} border="solid" rounding="lg" - padding="sm" + padding={2} expandedContent={ hasContent ? ( -
+
{filteredTools.map((tool) => ( 0 ? ( -
+
+
} > -
+
{markdown( `LLM call traces will no longer be sent to **${target.label}**. Traces already sent are unaffected.` diff --git a/web/src/views/admin/UsersPage/EditUserModal.tsx b/web/src/views/admin/UsersPage/EditUserModal.tsx index 35c6674588e..6f444e86e38 100644 --- a/web/src/views/admin/UsersPage/EditUserModal.tsx +++ b/web/src/views/admin/UsersPage/EditUserModal.tsx @@ -189,8 +189,8 @@ export default function EditUserModal({
{user.role && ( <> - + + - + This is a synced SCIM user managed by your identity provider. @@ -150,7 +150,7 @@ export default function UserRowActions({ > Reset Password - + Reset Password - + openModal(Modal.ACTIVATE)} > Activate User - +
+ 0; const statsCard = ( - +
{statsCard} {rightCard} diff --git a/web/src/views/admin/VoicePage/index.tsx b/web/src/views/admin/VoicePage/index.tsx index 492009839fa..b2814de1eb9 100644 --- a/web/src/views/admin/VoicePage/index.tsx +++ b/web/src/views/admin/VoicePage/index.tsx @@ -234,8 +234,8 @@ export default function VoicePage() { divider /> -
-
+
+
)} -
+
{STT_MODELS.map((model) => (
-
+
)} -
+
{TTS_PROVIDER_GROUPS.map((group) => (
-
+
{providerType === "azure" && ( } > -
+
{markdown( `**${disconnectTarget.providerLabel}** models will no longer be used for speech-to-text or text-to-speech, and it will no longer be your default. Session history will be preserved.` diff --git a/web/src/views/admin/WebSearchPage/WebSearchDisconnectModal.tsx b/web/src/views/admin/WebSearchPage/WebSearchDisconnectModal.tsx index c0cb15adb6a..1dd20acd82f 100644 --- a/web/src/views/admin/WebSearchPage/WebSearchDisconnectModal.tsx +++ b/web/src/views/admin/WebSearchPage/WebSearchDisconnectModal.tsx @@ -82,7 +82,7 @@ export function WebSearchDisconnectModal({ } > -
+
{isSearch ? ( <> diff --git a/web/src/views/admin/WorkspaceAnalyticsPage/AnalyticsCharts.tsx b/web/src/views/admin/WorkspaceAnalyticsPage/AnalyticsCharts.tsx new file mode 100644 index 00000000000..add1946ea63 --- /dev/null +++ b/web/src/views/admin/WorkspaceAnalyticsPage/AnalyticsCharts.tsx @@ -0,0 +1,105 @@ +import { + useOnyxBotAnalytics, + useQueryAnalytics, + useUserAnalytics, +} from "@/lib/usage/hooks"; +import { + AnalyticsChart, + chartSeries, + resolveChartState, + useLoggedChartError, +} from "@/sections/usage/AnalyticsChart"; +import { DateRangePickerValue } from "@/refresh-components/DateRangePicker"; + +interface TimeRangeProps { + timeRange: DateRangePickerValue; +} + +function formatCount(value: number): string { + return new Intl.NumberFormat("en-US", { + notation: "standard", + maximumFractionDigits: 0, + }).format(value); +} + +export function UsageChart({ timeRange }: TimeRangeProps) { + const queryAnalytics = useQueryAnalytics(timeRange); + const userAnalytics = useUserAnalytics(timeRange); + + useLoggedChartError("Query", queryAnalytics.error); + useLoggedChartError("Active user", userAnalytics.error); + + return ( + e.total_queries), + chartSeries( + "Unique Users", + userAnalytics.data, + (e) => e.total_active_users + ), + ], + })} + allowDecimals={false} + yAxisFormatter={formatCount} + /> + ); +} + +export function FeedbackChart({ timeRange }: TimeRangeProps) { + const { data, isLoading, error } = useQueryAnalytics(timeRange); + + useLoggedChartError("Feedback", error); + + return ( + e.total_likes), + chartSeries("Negative Feedback", data, (e) => e.total_dislikes), + ], + })} + /> + ); +} + +export function SlackChannelChart({ timeRange }: TimeRangeProps) { + const { data, isLoading, error } = useOnyxBotAnalytics(timeRange); + + useLoggedChartError("OnyxBot", error); + + return ( + e.total_queries), + chartSeries("Automatically Resolved", data, (e) => e.auto_resolved), + ], + })} + /> + ); +} diff --git a/web/src/views/admin/WorkspaceAnalyticsPage/PersonaMessagesChart.tsx b/web/src/views/admin/WorkspaceAnalyticsPage/PersonaMessagesChart.tsx new file mode 100644 index 00000000000..78a22270810 --- /dev/null +++ b/web/src/views/admin/WorkspaceAnalyticsPage/PersonaMessagesChart.tsx @@ -0,0 +1,196 @@ +import React, { useEffect, useMemo, useState } from "react"; +import { + InputTypeIn, + LineItemButton, + Popover, + PopoverMenu, + SelectButton, + Text, +} from "@opal/components"; +import { SvgOnyxOctagon } from "@opal/icons"; +import { Section } from "@opal/layouts"; +import { usePersonaMessages, usePersonaUniqueUsers } from "@/lib/usage/hooks"; +import { useAdminAgents } from "@/lib/agents/hooks"; +import { + AnalyticsChart, + chartSeries, + resolveChartState, +} from "@/sections/usage/AnalyticsChart"; +import { DateRangePickerValue } from "@/refresh-components/DateRangePicker"; +import { Agent } from "@/lib/agents/types"; + +interface PersonaPickerProps { + agents: Agent[]; + selectedAgent: Agent | undefined; + onSelect: (agentId: number) => void; +} + +function PersonaPicker({ + agents, + selectedAgent, + onSelect, +}: PersonaPickerProps) { + const [open, setOpen] = useState(false); + const [search, setSearch] = useState(""); + + const matches = useMemo(() => { + const query = search.trim().toLowerCase(); + if (query === "") return agents; + return agents.filter((agent) => agent.name.toLowerCase().includes(query)); + }, [agents, search]); + + return ( + { + setOpen(nextOpen); + if (!nextOpen) setSearch(""); + }} + > + + + {selectedAgent?.name ?? "Select an agent to display"} + + + + + {[ + setSearch(event.target.value)} + />, + ...matches.map((agent) => ( + + onSelect(agent.id)} + /> + + )), + ...(matches.length === 0 + ? [ +
+ + No agents match that search + +
, + ] + : []), + ]} + + + + ); +} + +interface PersonaMessagesChartProps { + timeRange: DateRangePickerValue; +} + +export function PersonaMessagesChart({ timeRange }: PersonaMessagesChartProps) { + const [selectedPersonaId, setSelectedPersonaId] = useState< + number | undefined + >(undefined); + + const { + agents, + error: agentsError, + isLoading: agentsLoading, + } = useAdminAgents(); + + const { + data: personaMessagesData, + isLoading: isPersonaMessagesLoading, + error: personaMessagesError, + } = usePersonaMessages(selectedPersonaId, timeRange); + + const { + data: personaUniqueUsersData, + isLoading: isPersonaUniqueUsersLoading, + error: personaUniqueUsersError, + } = usePersonaUniqueUsers(selectedPersonaId, timeRange); + + useEffect(() => { + if (agentsError) { + console.error("Failed to fetch admin agents:", agentsError); + } + if (personaMessagesError) { + console.error("Failed to fetch agent messages:", personaMessagesError); + } + if (personaUniqueUsersError) { + console.error( + "Failed to fetch agent unique users:", + personaUniqueUsersError + ); + } + }, [agentsError, personaMessagesError, personaUniqueUsersError]); + + const selectedAgent = agents.find((agent) => agent.id === selectedPersonaId); + + const series = [ + chartSeries( + "Messages", + personaMessagesData, + (entry) => entry.total_messages + ), + chartSeries( + "Unique Users", + personaUniqueUsersData, + (entry) => entry.unique_users + ), + ]; + + return ( + + } + /> + ); +} diff --git a/web/src/views/admin/WorkspaceAnalyticsPage/UsageReports.tsx b/web/src/views/admin/WorkspaceAnalyticsPage/UsageReports.tsx new file mode 100644 index 00000000000..0f4afe981b7 --- /dev/null +++ b/web/src/views/admin/WorkspaceAnalyticsPage/UsageReports.tsx @@ -0,0 +1,468 @@ +"use client"; + +import React, { useEffect, useMemo, useRef, useState } from "react"; +import { format, startOfDay, subDays } from "date-fns"; +import useSWR from "swr"; +import { + Button, + Calendar, + LineItemButton, + MessageCard, + Pagination, + Popover, + Text, +} from "@opal/components"; +import { ContentAction, PageLoader, Section, toast } from "@opal/layouts"; +import { + SvgCalendar, + SvgDownload, + SvgDownloadCloud, + SvgSimpleLoader, + SvgSpreadsheetFile, + SvgX, +} from "@opal/icons"; +import { humanReadableFormat, humanReadableFormatWithTime } from "@opal/time"; +import type { IconFunctionComponent, RichStr } from "@opal/types"; +import { errorHandlingFetcher } from "@/lib/fetcher"; +import { SWR_KEYS } from "@/lib/swr-keys"; +import { UsageReport } from "@/app/ee/admin/performance/usage/types"; +import { + PendingReport, + ReportPeriod, +} from "@/views/admin/WorkspaceAnalyticsPage/interfaces"; +import { + generateUsageReport, + usageReportDownloadUrl, +} from "@/views/admin/WorkspaceAnalyticsPage/svc"; + +const PRESET_DAYS: { label: string; days: number }[] = [ + { label: "Today", days: 1 }, + { label: "Last 7 days", days: 7 }, + { label: "Last 30 days", days: 30 }, + { label: "Last 3 months", days: 90 }, +]; + +function presetPeriod(label: string, days: number): ReportPeriod { + const to = startOfDay(new Date()); + return { label, range: { from: subDays(to, days - 1), to } }; +} + +const PAGE_SIZE = 8; +const POLL_INTERVAL_MS = 3_000; +const SLOW_REPORT_AFTER_MS = 20_000; +const REPORT_TIMEOUT_MS = 5 * 60_000; + +function periodLabel(report: UsageReport): string { + return report.period_from + ? `${humanReadableFormat(report.period_from)} – ${humanReadableFormat( + report.period_to! + )}` + : "All time"; +} + +interface PendingReportRowProps { + rangeLabel: string; + slow: boolean; +} + +function PendingReportRow({ rangeLabel, slow }: PendingReportRowProps) { + return ( +
+ + } + /> +
+ ); +} + +interface ReportRowProps { + report: UsageReport; + justArrived: boolean; +} + +function ReportRow({ report, justArrived }: ReportRowProps) { + return ( +
+ + } + /> +
+ ); +} + +interface PeriodMenuItemProps { + title: string | RichStr; + onClick: () => void; + icon?: IconFunctionComponent; +} + +function PeriodMenuItem({ title, onClick, icon }: PeriodMenuItemProps) { + return ( + + ); +} + +interface GenerateReportMenuProps { + disabled: boolean; + pending: boolean; + onGenerate: (period: ReportPeriod) => void; +} + +function GenerateReportMenu({ + disabled, + pending, + onGenerate, +}: GenerateReportMenuProps) { + const [open, setOpen] = useState(false); + const [view, setView] = useState<"presets" | "calendar">("presets"); + const [pendingStart, setPendingStart] = useState(undefined); + const [draftRange, setDraftRange] = useState< + { from: Date; to?: Date } | undefined + >(undefined); + + function reset() { + setView("presets"); + setPendingStart(undefined); + setDraftRange(undefined); + } + + return ( + { + setOpen(nextOpen); + if (!nextOpen) reset(); + }} + > + + + + + {view === "presets" ? ( + // Children must stay a flat array: Popover.Menu filters over it and + // renders each `null` as a divider. + + {[ + ...PRESET_DAYS.map((preset) => ( + + + onGenerate(presetPeriod(preset.label, preset.days)) + } + /> + + )), + + onGenerate({ label: "All time" })} + /> + , + null, + setView("calendar")} + />, + ]} + + ) : ( +
+ + {pendingStart + ? "Pick the end of the period" + : "Pick the start of the period"} + + { + if (!pendingStart) { + setDraftRange({ from: day }); + setPendingStart(day); + return; + } + const from = day < pendingStart ? day : pendingStart; + const to = day < pendingStart ? pendingStart : day; + onGenerate({ + label: `${format(from, "MMM d, y")} – ${format(to, "MMM d, y")}`, + range: { from, to }, + }); + setOpen(false); + reset(); + }} + numberOfMonths={1} + disabled={(date) => date > new Date()} + /> +
+ )} + + + ); +} + +export default function UsageReports() { + const [page, setPage] = useState(1); + const [requesting, setRequesting] = useState(false); + const [pendingReport, setPendingReport] = useState( + null + ); + const [arrivedReportName, setArrivedReportName] = useState( + null + ); + const slowTimerRef = useRef | null>(null); + const reportTimeoutRef = useRef | null>(null); + const abortRef = useRef(null); + + const pending = pendingReport !== null; + const { + data: reports, + error: listError, + isLoading: listLoading, + mutate, + } = useSWR(SWR_KEYS.usageReport, errorHandlingFetcher, { + refreshInterval: pending ? POLL_INTERVAL_MS : 0, + }); + + useEffect(() => { + if (listError) console.error("Failed to load usage reports:", listError); + }, [listError]); + + function clearPendingTimers() { + if (slowTimerRef.current) { + clearTimeout(slowTimerRef.current); + slowTimerRef.current = null; + } + if (reportTimeoutRef.current) { + clearTimeout(reportTimeoutRef.current); + reportTimeoutRef.current = null; + } + } + + useEffect(() => { + if (!reports || !pendingReport) return; + const completed = reports.find((report) => + report.report_name.includes(pendingReport.id) + ); + if (completed) { + setPendingReport(null); + setArrivedReportName(completed.report_name); + setPage(1); + toast.success("Usage report ready."); + clearPendingTimers(); + } + }, [reports, pendingReport]); + + useEffect( + () => () => { + clearPendingTimers(); + abortRef.current?.abort(); + }, + [] + ); + + async function requestReport(period: ReportPeriod): Promise { + setRequesting(true); + const abort = new AbortController(); + abortRef.current = abort; + try { + const reportId = await generateUsageReport(period, abort.signal); + setPendingReport({ id: reportId, label: period.label, slow: false }); + setArrivedReportName(null); + slowTimerRef.current = setTimeout( + () => + setPendingReport((current) => + current ? { ...current, slow: true } : current + ), + SLOW_REPORT_AFTER_MS + ); + reportTimeoutRef.current = setTimeout(() => { + setPendingReport(null); + toast.error( + "Report generation is taking too long. Try again or check back later." + ); + reportTimeoutRef.current = null; + }, REPORT_TIMEOUT_MS); + } catch (error) { + if (abort.signal.aborted) return; + console.error("Failed to start usage report generation:", error); + const message = error instanceof Error ? error.message : "unknown error"; + toast.error(`Failed to start report generation: ${message}`); + return; + } finally { + if (!abort.signal.aborted) setRequesting(false); + } + + // Generation already succeeded, so a failed revalidation must not surface + // as "failed to start report generation". + void mutate().catch((error: unknown) => { + console.error("Failed to refresh the usage report list:", error); + }); + } + + const orderedReports = useMemo( + () => + [...(reports ?? [])].sort( + (left, right) => + new Date(right.time_created).getTime() - + new Date(left.time_created).getTime() + ), + [reports] + ); + const totalPages = Math.max(1, Math.ceil(orderedReports.length / PAGE_SIZE)); + const pageReports = orderedReports.slice( + PAGE_SIZE * (page - 1), + PAGE_SIZE * page + ); + + return ( +
+ {/* sm:flex-row / sm:items-center / sm:justify-between have no Section equivalent, kept as a raw div */} +
+
+ Usage reports + + Export per-user usage as a ZIP of CSV files. Reports build in the + background and stay available here. + +
+ void requestReport(period)} + /> +
+ + {listLoading ? ( + + ) : listError ? ( + + ) : ( +
+ {pending && page === 1 && ( + + )} + {orderedReports.length === 0 && !pending ? ( +
+ + No reports yet. Pick a period and generate your first one. + +
+ ) : ( + pageReports.map((report) => ( + + )) + )} + {totalPages > 1 && ( +
+ +
+ )} +
+ )} +
+ ); +} diff --git a/web/src/views/admin/WorkspaceAnalyticsPage/index.tsx b/web/src/views/admin/WorkspaceAnalyticsPage/index.tsx new file mode 100644 index 00000000000..bbf316eada6 --- /dev/null +++ b/web/src/views/admin/WorkspaceAnalyticsPage/index.tsx @@ -0,0 +1,51 @@ +"use client"; + +import { DateRangePicker } from "@/refresh-components/DateRangePicker"; +import { useTimeRange } from "@/lib/usage/hooks"; +import { + FeedbackChart, + SlackChannelChart, + UsageChart, +} from "@/views/admin/WorkspaceAnalyticsPage/AnalyticsCharts"; +import { PersonaMessagesChart } from "@/views/admin/WorkspaceAnalyticsPage/PersonaMessagesChart"; +import UsageReports from "@/views/admin/WorkspaceAnalyticsPage/UsageReports"; +import { ADMIN_ROUTES } from "@/lib/admin-routes"; +import { Divider } from "@opal/components"; +import { Section, SettingsLayouts } from "@opal/layouts"; + +const route = ADMIN_ROUTES.WORKSPACE_ANALYTICS; + +export default function WorkspaceAnalyticsPage() { + const [timeRange, setTimeRange] = useTimeRange(); + + return ( + + + +
+ + setTimeRange((previous) => + range + ? { ...range, selectValue: previous.selectValue } + : previous + ) + } + /> +
+ + + + + + + + + ); +} diff --git a/web/src/views/admin/WorkspaceAnalyticsPage/interfaces.ts b/web/src/views/admin/WorkspaceAnalyticsPage/interfaces.ts new file mode 100644 index 00000000000..c5dd16048c3 --- /dev/null +++ b/web/src/views/admin/WorkspaceAnalyticsPage/interfaces.ts @@ -0,0 +1,10 @@ +export interface ReportPeriod { + label: string; + range?: { from: Date; to: Date }; +} + +export interface PendingReport { + id: string; + label: string; + slow: boolean; +} diff --git a/web/src/views/admin/WorkspaceAnalyticsPage/svc.ts b/web/src/views/admin/WorkspaceAnalyticsPage/svc.ts new file mode 100644 index 00000000000..d47f348b7bb --- /dev/null +++ b/web/src/views/admin/WorkspaceAnalyticsPage/svc.ts @@ -0,0 +1,39 @@ +/** API helpers for the Workspace Analytics page. */ + +import { SWR_KEYS } from "@/lib/swr-keys"; +import { ReportPeriod } from "@/views/admin/WorkspaceAnalyticsPage/interfaces"; + +const USAGE_REPORT_URL = SWR_KEYS.usageReport; + +export function usageReportDownloadUrl(reportName: string): string { + return `${USAGE_REPORT_URL}/${reportName}`; +} + +/** Starts a report build and returns the id used to recognise it in the list. */ +export async function generateUsageReport( + period: ReportPeriod, + signal: AbortSignal +): Promise { + const reportId = crypto.randomUUID(); + const res = await fetch(USAGE_REPORT_URL, { + method: "POST", + credentials: "include", + signal, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + period_from: period.range ? period.range.from.toISOString() : null, + period_to: period.range ? period.range.to.toISOString() : null, + report_id: reportId, + }), + }); + if (!res.ok) { + const detail = await res.json().catch((parseError: unknown) => { + console.error("Usage report error response was not JSON:", parseError); + return null; + }); + throw new Error( + detail?.detail ?? `Failed to start report generation: ${res.statusText}` + ); + } + return reportId; +} diff --git a/web/tests/e2e/admin/admin_mobile_sidebar.spec.ts b/web/tests/e2e/admin/admin_mobile_sidebar.spec.ts new file mode 100644 index 00000000000..1be1bd0a8e0 --- /dev/null +++ b/web/tests/e2e/admin/admin_mobile_sidebar.spec.ts @@ -0,0 +1,34 @@ +import { test } from "@playwright/test"; +import { AdminChromePage } from "@tests/e2e/pages/AdminChromePage"; + +test.use({ + storageState: "admin_auth.json", + viewport: { width: 390, height: 800 }, +}); +test.describe.configure({ mode: "parallel" }); + +// The create-connector page renders its own sidebar instead of the default one. +// It must still offer the same way back to it as every other admin page. +const ADMIN_PAGES = [ + { name: "an admin page", path: "/admin/indexing/status" }, + { name: "the create-connector page", path: "/admin/connectors/web" }, +]; + +for (const { name, path } of ADMIN_PAGES) { + test(`Sidebar starts folded and can be re-opened on ${name} – mobile`, async ({ + page, + }) => { + const adminChrome = new AdminChromePage(page); + + // The sidebar is an overlay on mobile, so it must not cover the page. + await adminChrome.goto(path); + await adminChrome.expectSidebarFolded(); + + await adminChrome.expectOpenSidebarButtonVisible(); + await adminChrome.openSidebar(); + await adminChrome.expectSidebarUnfolded(); + + await adminChrome.closeSidebar(); + await adminChrome.expectSidebarFolded(); + }); +} diff --git a/web/tests/e2e/pages/AdminChromePage.ts b/web/tests/e2e/pages/AdminChromePage.ts new file mode 100644 index 00000000000..8eb307a09a4 --- /dev/null +++ b/web/tests/e2e/pages/AdminChromePage.ts @@ -0,0 +1,54 @@ +/** + * Page Object Model for the admin chrome (`AdminChrome`) — the sidebar column + * and the main content column that wrap every `/admin/*` page. + * + * Covers the mobile layout, where the sidebar is an off-screen overlay and the + * main column carries the control that brings it back. + */ + +import { expect, type Locator, type Page } from "@playwright/test"; + +export class AdminChromePage { + readonly page: Page; + + // The sidebar column has no role or accessible name of its own; `data-folded` + // is the only signal for whether it sits on- or off-screen. + private readonly sidebar: Locator; + private readonly openSidebarButton: Locator; + private readonly closeSidebarButton: Locator; + + constructor(page: Page) { + this.page = page; + this.sidebar = page.locator(".opal-sidebar-root__overlay"); + // Scoped to the main column, so it never matches the sidebar's own control. + this.openSidebarButton = page + .locator("[data-main-container]") + .getByLabel("Open Sidebar"); + this.closeSidebarButton = this.sidebar.getByLabel("Close Sidebar"); + } + + async goto(path: string): Promise { + await this.page.goto(path); + await this.page.waitForLoadState("networkidle"); + } + + async openSidebar(): Promise { + await this.openSidebarButton.click(); + } + + async closeSidebar(): Promise { + await this.closeSidebarButton.click(); + } + + async expectOpenSidebarButtonVisible(): Promise { + await expect(this.openSidebarButton).toBeVisible(); + } + + async expectSidebarFolded(): Promise { + await expect(this.sidebar).toHaveAttribute("data-folded", "true"); + } + + async expectSidebarUnfolded(): Promise { + await expect(this.sidebar).toHaveAttribute("data-folded", "false"); + } +} diff --git a/web/tests/e2e/scheduled-tasks/ScheduledTasksPage.ts b/web/tests/e2e/pages/ScheduledTasksPage.ts similarity index 66% rename from web/tests/e2e/scheduled-tasks/ScheduledTasksPage.ts rename to web/tests/e2e/pages/ScheduledTasksPage.ts index 3a071cc6a20..9ef8d5330a7 100644 --- a/web/tests/e2e/scheduled-tasks/ScheduledTasksPage.ts +++ b/web/tests/e2e/pages/ScheduledTasksPage.ts @@ -2,10 +2,11 @@ * Page Object Model for the Onyx Craft Scheduled Tasks surface * (/craft/v1/tasks, /craft/v1/tasks/new, /craft/v1/tasks/[id]). * - * Encapsulates all locators and interactions so specs remain declarative. + * Keeps locators and interactions out of declarative specs. */ import { type Page, type Locator, expect } from "@playwright/test"; +import { appFixture, mcpServerFixture } from "@/lib/skills/__fixtures__/picker"; const TASKS_LIST_PATH = "/craft/v1/tasks"; const NEW_TASK_PATH = "/craft/v1/tasks/new"; @@ -25,6 +26,9 @@ export class ScheduledTasksPage { readonly intervalEveryInput: Locator; readonly intervalUnitTrigger: Locator; readonly saveAndRunNowButton: Locator; + readonly preApprovalPicker: Locator; + readonly appsGroup: Locator; + readonly mcpServersGroup: Locator; constructor(page: Page) { this.page = page; @@ -39,6 +43,9 @@ export class ScheduledTasksPage { // on the new-task form. this.intervalUnitTrigger = page.getByRole("combobox").first(); this.saveAndRunNowButton = page.getByTestId("save-and-run-now"); + this.preApprovalPicker = page.getByTestId("pre-approval-picker"); + this.appsGroup = page.getByRole("region", { name: "Apps" }); + this.mcpServersGroup = page.getByRole("region", { name: "MCP servers" }); } // --------------------------------------------------------------------------- @@ -127,6 +134,100 @@ export class ScheduledTasksPage { await this.saveAndRunNowButton.click(); } + async mockPreApprovalOptions(): Promise { + await this.page.route("**/api/build/apps", async (route) => { + await route.fulfill({ + json: [ + appFixture({ + id: 11, + name: "Acme CRM", + app_type: "CUSTOM", + }), + appFixture({ + id: 12, + name: "Acme Support", + app_type: "CUSTOM", + }), + ], + }); + }); + await this.page.route("**/api/mcp/servers/craft", async (route) => { + await route.fulfill({ + json: { + mcp_servers: [ + mcpServerFixture({ + id: 22, + name: "Acme MCP", + server_url: "https://mcp.example.com/mcp", + }), + ], + }, + }); + }); + } + + async expectResponsivePreApprovalLayout(): Promise { + await expect(this.preApprovalPicker).toBeVisible(); + await expect(this.appsGroup).toBeVisible(); + await expect(this.mcpServersGroup).toBeVisible(); + await this.expectPreApprovalGroupsFillContainer(); + + const appOptions = this.appsGroup.getByTestId(/^pre-approval-app-/); + await expect(appOptions).toHaveCount(2); + await expect + .poll(async () => { + const [first, second] = await Promise.all([ + appOptions.nth(0).boundingBox(), + appOptions.nth(1).boundingBox(), + ]); + return Boolean( + first && + second && + Math.abs(first.y - second.y) <= 1 && + second.x > first.x + ); + }) + .toBe(true); + + await this.page.setViewportSize({ width: 390, height: 844 }); + await this.expectPreApprovalGroupsFillContainer(); + await expect + .poll(async () => { + const [first, second] = await Promise.all([ + appOptions.nth(0).boundingBox(), + appOptions.nth(1).boundingBox(), + ]); + return Boolean( + first && + second && + Math.abs(first.x - second.x) <= 1 && + second.y > first.y + ); + }) + .toBe(true); + } + + private async expectPreApprovalGroupsFillContainer(): Promise { + const container = this.preApprovalPicker.locator(".."); + await expect + .poll(async () => { + const [containerBox, pickerBox, appsBox, mcpBox] = await Promise.all([ + container.boundingBox(), + this.preApprovalPicker.boundingBox(), + this.appsGroup.boundingBox(), + this.mcpServersGroup.boundingBox(), + ]); + if (!containerBox || !pickerBox || !appsBox || !mcpBox) return false; + + return ( + Math.abs(containerBox.width - pickerBox.width) <= 1 && + Math.abs(pickerBox.width - appsBox.width) <= 1 && + Math.abs(pickerBox.width - mcpBox.width) <= 1 + ); + }) + .toBe(true); + } + // --------------------------------------------------------------------------- // List page // --------------------------------------------------------------------------- diff --git a/web/tests/e2e/scheduled-tasks/scheduled-tasks.spec.ts b/web/tests/e2e/scheduled-tasks/scheduled-tasks.spec.ts index 07ac85165ce..012058d26bf 100644 --- a/web/tests/e2e/scheduled-tasks/scheduled-tasks.spec.ts +++ b/web/tests/e2e/scheduled-tasks/scheduled-tasks.spec.ts @@ -1,8 +1,25 @@ import { test } from "@playwright/test"; import { loginAsWorkerUser } from "@tests/e2e/utils/auth"; -import { ScheduledTasksPage } from "@tests/e2e/scheduled-tasks/ScheduledTasksPage"; +import { ScheduledTasksPage } from "@tests/e2e/pages/ScheduledTasksPage"; test.describe("Scheduled Tasks", () => { + test("pre-approval groups fill the form on desktop and mobile", async ({ + page, + }, testInfo) => { + await loginAsWorkerUser(page, testInfo.workerIndex); + + const scheduledTasks = new ScheduledTasksPage(page); + await scheduledTasks.mockPreApprovalOptions(); + await scheduledTasks.gotoList(); + test.skip( + !scheduledTasks.isCraftEnabled(), + "Onyx Craft is disabled in this environment (settings.onyx_craft_enabled !== true)" + ); + + await scheduledTasks.openCreateForm(); + await scheduledTasks.expectResponsivePreApprovalLayout(); + }); + test("create, run, and verify a run row exists", async ({ page, }, testInfo) => {