diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000000..1d8c668eff3 --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,5 @@ +{ + "recommendations": [ + "anysphere.remote-containers" + ] +} \ No newline at end of file diff --git a/backend/ee/onyx/server/log_export/api.py b/backend/ee/onyx/server/log_export/api.py index 0cf5b374f8a..22e23e6d8fb 100644 --- a/backend/ee/onyx/server/log_export/api.py +++ b/backend/ee/onyx/server/log_export/api.py @@ -26,6 +26,8 @@ from onyx import __version__ from onyx.auth.permissions import require_permission from onyx.background.celery.versioned_apps.client import app as client_app +from onyx.cache.interface import CacheBackendType +from onyx.configs.app_configs import CACHE_BACKEND from onyx.configs.constants import ( OnyxCeleryPriority, OnyxCeleryQueues, @@ -137,9 +139,9 @@ def start_log_export( Starts an export: fans out one collector task per worker type, collects the api_server's logs inline, and returns the export ID to poll. - Fan-out failures (e.g. deployments with no broker or workers, like the - onyx-lite overlay) degrade the export to just the api_server's logs instead - of failing. + The fan-out is skipped on deployments that have no celery broker (the + onyx-lite overlay); a failing broker degrades the export to the workers + already enqueued instead of failing it. """ if MULTI_TENANT: raise OnyxError( @@ -165,33 +167,44 @@ def start_log_export( # Fan out before the inline collection below so workers get the full # window before ``expires=`` discards their tasks, and their collection - # overlaps the api_server's. + # overlaps the api_server's. When redis is absent by design (the + # onyx-lite overlay), there is no broker behind ``send_task`` and no + # workers to collect from, so the fan-out is skipped outright + # (``maybe_schedule_license_reclaim`` applies the same rule); a broker + # that exists but is down degrades per-send below instead. enqueued_worker_names: list[str] = [] - for worker_name, queue in WORKER_COLLECT_QUEUES.items(): - try: - client_app.send_task( - OnyxCeleryTask.EXPORT_LOGS_COLLECT_TASK, - priority=OnyxCeleryPriority.HIGHEST, - queue=queue, - expires=deadline, - kwargs={ - "export_id": export_id, - "worker_name": worker_name, - }, - ) - except Exception as e: - # All sends share one broker, so the first failure means the - # rest would fail too. Only the workers already enqueued are - # awaited. - logger.warning( - "Log export fan-out failed while enqueueing %s; continuing " - "with %s: %s", - worker_name, - enqueued_worker_names, - e, - ) - break - enqueued_worker_names.append(worker_name) + if CACHE_BACKEND != CacheBackendType.REDIS: + logger.info( + "Log export fan-out skipped: this deployment has no celery " + "broker (CACHE_BACKEND=%s).", + CACHE_BACKEND.value, + ) + else: + for worker_name, queue in WORKER_COLLECT_QUEUES.items(): + try: + client_app.send_task( + OnyxCeleryTask.EXPORT_LOGS_COLLECT_TASK, + priority=OnyxCeleryPriority.HIGHEST, + queue=queue, + expires=deadline, + kwargs={ + "export_id": export_id, + "worker_name": worker_name, + }, + ) + except Exception as e: + # All sends share one broker, so the first failure means the + # rest would fail too. Only the workers already enqueued are + # awaited. + logger.warning( + "Log export fan-out failed while enqueueing %s; " + "continuing with %s: %s", + worker_name, + enqueued_worker_names, + e, + ) + break + enqueued_worker_names.append(worker_name) manifest = LogExportManifest( export_id=export_id, diff --git a/backend/ee/onyx/server/reporting/usage_export_generation.py b/backend/ee/onyx/server/reporting/usage_export_generation.py index 8dd71938221..c94c3ac6915 100644 --- a/backend/ee/onyx/server/reporting/usage_export_generation.py +++ b/backend/ee/onyx/server/reporting/usage_export_generation.py @@ -2,6 +2,7 @@ import tempfile import uuid import zipfile +from collections.abc import Iterable from datetime import datetime, timedelta, timezone from fastapi_users_db_sqlalchemy import UUID_ID @@ -20,33 +21,37 @@ ) from onyx.configs.constants import FileOrigin from onyx.db.models import User +from onyx.db.user_usage import UsageExportRow, iter_usage_export from onyx.db.users import get_all_users from onyx.file_store.constants import MAX_IN_MEMORY_SIZE from onyx.file_store.file_store import FileStore, get_default_file_store from onyx.utils.csv_utils import sanitize_csv_cell_or_none +from onyx.utils.logger import setup_logger + +logger = setup_logger() + + +def _normalize_period( + period: tuple[datetime, datetime] | None, +) -> tuple[datetime, datetime]: + if period is None: + return ( + datetime.fromtimestamp(0, tz=timezone.utc), + datetime.now(tz=timezone.utc), + ) + # time-picker sends a time which is at the beginning of the day + # so we need to add one day to the end time to make it inclusive + return (period[0], period[1] + timedelta(days=1)) def generate_chat_messages_report( db_session: Session, file_store: FileStore, report_id: str, - period: tuple[datetime, datetime] | None, + period: tuple[datetime, datetime], ) -> str: file_name = f"{report_id}_chat_sessions" - if period is None: - period = ( - datetime.fromtimestamp(0, tz=timezone.utc), - datetime.now(tz=timezone.utc), - ) - else: - # time-picker sends a time which is at the beginning of the day - # so we need to add one day to the end time to make it inclusive - period = ( - period[0], - period[1] + timedelta(days=1), - ) - with tempfile.SpooledTemporaryFile( max_size=MAX_IN_MEMORY_SIZE, mode="w+" ) as temp_file: @@ -128,6 +133,57 @@ def generate_user_report( return file_id +def generate_usage_breakdown_report( + file_store: FileStore, + report_id: str, + rows: Iterable[UsageExportRow], +) -> str: + file_name = f"{report_id}_usage_by_user" + + with tempfile.SpooledTemporaryFile( + max_size=MAX_IN_MEMORY_SIZE, mode="w+" + ) as temp_file: + csvwriter = csv.writer(temp_file, delimiter=",") + csvwriter.writerow( + [ + "user_email", + "day", + "model", + "flow", + "provider", + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cost_cents", + ] + ) + for row in rows: + # User-controlled strings: formula-injection guard. + csvwriter.writerow( + [ + sanitize_csv_cell_or_none(row.email), + row.day, + sanitize_csv_cell_or_none(row.model), + sanitize_csv_cell_or_none(row.flow), + sanitize_csv_cell_or_none(row.provider), + row.input_tokens, + row.output_tokens, + row.cache_read_tokens, + row.cost_cents, + ] + ) + + temp_file.seek(0) + file_id = file_store.save_file( + content=temp_file, + display_name=file_name, + file_origin=FileOrigin.GENERATED_REPORT, + file_type="text/csv", + ) + + return file_id + + def create_new_usage_report( db_session: Session, user_id: UUID_ID | None, # None = auto-generated @@ -136,47 +192,72 @@ def create_new_usage_report( ) -> UsageReportMetadata: report_id = report_id or str(uuid.uuid4()) file_store = get_default_file_store() + normalized_period = _normalize_period(period) - messages_file_id = generate_chat_messages_report( - db_session, file_store, report_id, period - ) - users_file_id = generate_user_report(db_session, file_store, report_id) - - # Re-check just before writing the final report: the API-level check - # happens before this (async) task runs, so a second request with the - # same client-supplied report_id can slip past it while this task is - # still generating the first report. - if usage_report_id_in_use(db_session, uuid.UUID(report_id)): - raise ValueError(f"report_id {report_id} is already in use") - - with tempfile.SpooledTemporaryFile(max_size=MAX_IN_MEMORY_SIZE) as zip_buffer: - with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED) as zip_file: - # write messages - chat_messages_tmpfile = file_store.read_file( - messages_file_id, mode="b", use_tempfile=True - ) - zip_file.writestr( - "chat_messages.csv", - chat_messages_tmpfile.read(), - ) + intermediate_file_ids: list[str] = [] + try: + messages_file_id = generate_chat_messages_report( + db_session, file_store, report_id, normalized_period + ) + intermediate_file_ids.append(messages_file_id) + users_file_id = generate_user_report(db_session, file_store, report_id) + intermediate_file_ids.append(users_file_id) - # write users - users_tmpfile = file_store.read_file( - users_file_id, mode="b", use_tempfile=True - ) - zip_file.writestr("users.csv", users_tmpfile.read()) + query_start, query_end = normalized_period + usage_rows = iter_usage_export(db_session, query_start, query_end) + usage_breakdown_file_id = generate_usage_breakdown_report( + file_store, report_id, usage_rows + ) + intermediate_file_ids.append(usage_breakdown_file_id) - zip_buffer.seek(0) + # Re-check just before writing the final report: the API-level check + # happens before this (async) task runs, so a second request with the + # same client-supplied report_id can slip past it while this task is + # still generating the first report. + if usage_report_id_in_use(db_session, uuid.UUID(report_id)): + raise ValueError(f"report_id {report_id} is already in use") - # store zip blob to file_store - report_name = f"{datetime.now(tz=timezone.utc).strftime('%Y-%m-%d')}_{report_id}_usage_report.zip" - file_store.save_file( - content=zip_buffer, - display_name=report_name, - file_origin=FileOrigin.GENERATED_REPORT, - file_type="application/zip", - file_id=report_name, - ) + with tempfile.SpooledTemporaryFile(max_size=MAX_IN_MEMORY_SIZE) as zip_buffer: + with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED) as zip_file: + # write messages + chat_messages_tmpfile = file_store.read_file( + messages_file_id, mode="b", use_tempfile=True + ) + zip_file.writestr( + "chat_messages.csv", + chat_messages_tmpfile.read(), + ) + + # write users + users_tmpfile = file_store.read_file( + users_file_id, mode="b", use_tempfile=True + ) + zip_file.writestr("users.csv", users_tmpfile.read()) + + usage_breakdown_tmpfile = file_store.read_file( + usage_breakdown_file_id, mode="b", use_tempfile=True + ) + zip_file.writestr("usage_by_user.csv", usage_breakdown_tmpfile.read()) + + zip_buffer.seek(0) + + # store zip blob to file_store + report_name = f"{datetime.now(tz=timezone.utc).strftime('%Y-%m-%d')}_{report_id}_usage_report.zip" + file_store.save_file( + content=zip_buffer, + display_name=report_name, + file_origin=FileOrigin.GENERATED_REPORT, + file_type="application/zip", + file_id=report_name, + ) + finally: + for file_id in intermediate_file_ids: + try: + file_store.delete_file(file_id, error_on_missing=False) + except Exception: + logger.exception( + "Failed to delete temporary usage report file %s", file_id + ) # add report after zip file is written new_report = write_usage_report(db_session, report_name, user_id, period) diff --git a/backend/onyx/db/user_usage.py b/backend/onyx/db/user_usage.py index 96f740d46fe..8392faf985d 100644 --- a/backend/onyx/db/user_usage.py +++ b/backend/onyx/db/user_usage.py @@ -4,7 +4,7 @@ model, flow, provider), not an append-only per-call ledger.""" from collections import defaultdict -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from datetime import datetime, timedelta from math import ceil from typing import Any, cast @@ -210,12 +210,11 @@ def get_user_usage_by_day_and_model( ] -def get_usage_export( - db_session: Session, +def _get_usage_export_query( start: datetime, end: datetime, model: str | None = None, -) -> list[UsageExportRow]: +) -> Any: utc_day = func.date(func.timezone("UTC", UserUsage.window_start)) # Deleted users/API keys leave user_id NULL but keep their spend. An inner # join would hide that spend here while the tenant-wide totals still count @@ -254,10 +253,22 @@ def get_usage_export( if model is not None: query = query.where(UserUsage.model == model) - rows = db_session.execute(query).all() + return query - return [ - UsageExportRow( + +def iter_usage_export( + db_session: Session, + start: datetime, + end: datetime, + model: str | None = None, +) -> Iterator[UsageExportRow]: + result = db_session.execute( + _get_usage_export_query(start, end, model).execution_options( + stream_results=True + ) + ).yield_per(1000) + for email, mdl, flow, provider, day, in_tok, out_tok, cache_tok, cost in result: + yield UsageExportRow( email=str(email), model=mdl, flow=flow, @@ -268,8 +279,15 @@ def get_usage_export( cache_read_tokens=int(cache_tok or 0), cost_cents=float(cost or 0.0), ) - for email, mdl, flow, provider, day, in_tok, out_tok, cache_tok, cost in rows - ] + + +def get_usage_export( + db_session: Session, + start: datetime, + end: datetime, + model: str | None = None, +) -> list[UsageExportRow]: + return list(iter_usage_export(db_session, start, end, model)) def get_usage_reset_window_start( diff --git a/backend/onyx/sandbox_proxy/server.py b/backend/onyx/sandbox_proxy/server.py index b78f3ccc288..b9603145ed3 100644 --- a/backend/onyx/sandbox_proxy/server.py +++ b/backend/onyx/sandbox_proxy/server.py @@ -35,6 +35,7 @@ SANDBOX_NAMESPACE, SANDBOX_PROXY_HEALTHZ_PORT, SANDBOX_PROXY_LISTEN_PORT, + SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA, ) from onyx.utils.logger import setup_logger from onyx.utils.variable_functionality import set_is_ee_based_on_env_variable @@ -152,6 +153,7 @@ def _build_mitm_options() -> Options: confdir=_MITM_CONFDIR, mode=["regular"], ssl_insecure=False, + ssl_verify_upstream_trusted_ca=SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA, ) diff --git a/backend/onyx/server/features/build/configs.py b/backend/onyx/server/features/build/configs.py index f29a562c582..2e2750e1d55 100644 --- a/backend/onyx/server/features/build/configs.py +++ b/backend/onyx/server/features/build/configs.py @@ -106,6 +106,13 @@ class SandboxBackend(str, Enum): # read container env), so a compose change here desyncs the probe. SANDBOX_PROXY_HEALTHZ_PORT = int(os.environ.get("SANDBOX_PROXY_HEALTHZ_PORT", "8081")) +# Optional additive CA bundle used by mitmproxy when it verifies HTTPS origins. +# The Helm chart writes it into the proxy's writable confdir because the proxy +# container intentionally runs non-root with a read-only root filesystem. +SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA = ( + os.environ.get("SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA", "").strip() or None +) + # The CA Secret lives here; the CA ConfigMap is projected into SANDBOX_NAMESPACE # so sandboxes can mount it (K8s does not allow cross-namespace ConfigMap # mounts). diff --git a/backend/scripts/env_inventory_baseline.txt b/backend/scripts/env_inventory_baseline.txt index dfa2f0debfa..61f57b5e155 100644 --- a/backend/scripts/env_inventory_baseline.txt +++ b/backend/scripts/env_inventory_baseline.txt @@ -410,6 +410,7 @@ SANDBOX_PROXY_HEALTHZ_PORT SANDBOX_PROXY_LISTEN_PORT SANDBOX_PROXY_MITM_CONFDIR SANDBOX_PROXY_NAMESPACE +SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA SECONDARY_LLM_FLOW_TIMEOUT_S SENDGRID_API_KEY SEND_USER_METADATA_TO_LLM_PROVIDER diff --git a/backend/tests/external_dependency_unit/ee/onyx/server/log_export/test_start_export.py b/backend/tests/external_dependency_unit/ee/onyx/server/log_export/test_start_export.py index 67a5d180639..9f2603f90fa 100644 --- a/backend/tests/external_dependency_unit/ee/onyx/server/log_export/test_start_export.py +++ b/backend/tests/external_dependency_unit/ee/onyx/server/log_export/test_start_export.py @@ -23,6 +23,7 @@ derive_export_state, read_export_snapshot, ) +from onyx.cache.interface import CacheBackendType from onyx.error_handling.error_codes import OnyxErrorCode from onyx.error_handling.exceptions import OnyxError @@ -62,9 +63,9 @@ def _admin_user() -> MagicMock: def test_fanout_failure_degrades_to_api_server_only() -> None: # Precondition. - # No broker is reachable, as in the onyx-lite overlay. + # A broker is configured but unreachable. with patch(f"{_API_MODULE}.client_app") as celery_client: - celery_client.send_task.side_effect = OSError("no broker") + celery_client.send_task.side_effect = OSError("broker down") # Under test. response = start_log_export(user=_admin_user()) @@ -79,6 +80,26 @@ def test_fanout_failure_degrades_to_api_server_only() -> None: assert derive_export_state(snapshot, now=now) is LogExportState.READY +def test_lite_deployment_skips_fanout(monkeypatch: pytest.MonkeyPatch) -> None: + # Precondition. + # Redis is absent by design (the onyx-lite overlay), so there is no celery + # broker and there are no workers. + monkeypatch.setattr(log_export_api, "CACHE_BACKEND", CacheBackendType.POSTGRES) + with patch(f"{_API_MODULE}.client_app") as celery_client: + # Under test. + response = start_log_export(user=_admin_user()) + + # Postcondition. + # The broker is never touched, and the manifest awaits only the api_server, + # whose inline receipt already exists. + celery_client.send_task.assert_not_called() + snapshot = read_export_snapshot(response.export_id) + assert snapshot is not None + assert snapshot.manifest.worker_names == [API_SERVER_WORKER_NAME] + now = datetime.now(tz=timezone.utc) + assert derive_export_state(snapshot, now=now) is LogExportState.READY + + def test_failed_start_releases_lock() -> None: # Precondition. with ( diff --git a/backend/tests/integration/tests/reporting/test_usage_export_api.py b/backend/tests/integration/tests/reporting/test_usage_export_api.py index c20564eb765..e3420181cdf 100644 --- a/backend/tests/integration/tests/reporting/test_usage_export_api.py +++ b/backend/tests/integration/tests/reporting/test_usage_export_api.py @@ -289,6 +289,28 @@ def test_read_usage_report( file_names = zip_file.namelist() assert "chat_messages.csv" in file_names assert "users.csv" in file_names + assert "usage_by_user.csv" in file_names + # Verify usage_by_user.csv has the expected columns. The seeded + # chat history doesn't record UserUsage rows, so there's no data + # to assert on, just the header shape. + with zip_file.open("usage_by_user.csv") as csv_file: + csv_content = csv_file.read().decode("utf-8") + csv_reader = csv.DictReader(StringIO(csv_content)) + expected_columns = { + "user_email", + "day", + "model", + "flow", + "provider", + "input_tokens", + "output_tokens", + "cache_read_tokens", + "cost_cents", + } + actual_columns = set(csv_reader.fieldnames or []) + assert expected_columns == actual_columns, ( + f"Expected columns {expected_columns}, but got {actual_columns}" + ) # Verify chat_messages.csv has the expected columns with zip_file.open("chat_messages.csv") as csv_file: diff --git a/backend/tests/unit/sandbox_proxy/test_server.py b/backend/tests/unit/sandbox_proxy/test_server.py new file mode 100644 index 00000000000..89a2c857806 --- /dev/null +++ b/backend/tests/unit/sandbox_proxy/test_server.py @@ -0,0 +1,34 @@ +"""Unit tests for sandbox-proxy process configuration.""" + +import pytest + +from onyx.sandbox_proxy import server + + +def test_mitm_options_use_custom_upstream_ca_when_configured( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + server, + "SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA", + "/var/run/sandbox-proxy/upstream-ca-bundle.crt", + ) + + options = server._build_mitm_options() + + assert ( + options.ssl_verify_upstream_trusted_ca + == "/var/run/sandbox-proxy/upstream-ca-bundle.crt" + ) + assert options.ssl_insecure is False + + +def test_mitm_options_keep_default_trust_store_without_custom_ca( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(server, "SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA", None) + + options = server._build_mitm_options() + + assert options.ssl_verify_upstream_trusted_ca is None + assert options.ssl_insecure is False diff --git a/deployment/helm/charts/onyx/Chart.yaml b/deployment/helm/charts/onyx/Chart.yaml index 1c001ddfcbe..e94d8342901 100644 --- a/deployment/helm/charts/onyx/Chart.yaml +++ b/deployment/helm/charts/onyx/Chart.yaml @@ -5,7 +5,7 @@ home: https://www.onyx.app/ sources: - "https://github.com/onyx-dot-app/onyx" type: application -version: 0.8.13 +version: 0.8.14 appVersion: latest annotations: category: Productivity @@ -16,6 +16,11 @@ annotations: - name: background image: docker.io/onyxdotapp/onyx-backend:latest artifacthub.io/changes: | + - kind: added + description: PostgreSQL TLS can now be configured with postgresTls, which + mounts a CA certificate and sets verified POSTGRES_SSL* defaults across + Onyx backend workloads. Craft's sandbox proxy now trusts customCACerts for + outbound HTTPS while retaining the public CA bundle. - kind: fixed description: Route /scim to the api server in ingress mode (ingress.enabled=true). The ingress templates only routed /api to the api server, so SCIM requests fell diff --git a/deployment/helm/charts/onyx/templates/_helpers.tpl b/deployment/helm/charts/onyx/templates/_helpers.tpl index db0241b8c91..bb50d0369c5 100644 --- a/deployment/helm/charts/onyx/templates/_helpers.tpl +++ b/deployment/helm/charts/onyx/templates/_helpers.tpl @@ -170,8 +170,9 @@ checksum/pginto: {{ include (print $.Template.BasePath "/tooling-pginto-configma {{- define "onyx.renderVolumeMounts" -}} {{- $pginto := include "onyx.pgInto.volumeMount" .ctx -}} {{- $ca := include "onyx.customCACerts.volumeMount" .ctx -}} +{{- $postgresTls := include "onyx.postgresTls.volumeMount" .ctx -}} {{- $existing := .volumeMounts -}} -{{- if or $pginto $ca $existing -}} +{{- if or $pginto $ca $postgresTls $existing -}} volumeMounts: {{- if $pginto }} {{ $pginto | nindent 2 }} @@ -182,14 +183,18 @@ volumeMounts: {{- if $ca }} {{ $ca | nindent 2 }} {{- end }} +{{- if $postgresTls }} +{{ $postgresTls | nindent 2 }} +{{- end }} {{- end -}} {{- end }} {{- define "onyx.renderVolumes" -}} {{- $pginto := include "onyx.pgInto.volume" .ctx -}} {{- $ca := include "onyx.customCACerts.volume" .ctx -}} +{{- $postgresTls := include "onyx.postgresTls.volume" .ctx -}} {{- $existing := .volumes -}} -{{- if or $pginto $ca $existing -}} +{{- if or $pginto $ca $postgresTls $existing -}} volumes: {{- if $pginto }} {{ $pginto | nindent 2 }} @@ -200,6 +205,9 @@ volumes: {{- if $ca }} {{ $ca | nindent 2 }} {{- end }} +{{- if $postgresTls }} +{{ $postgresTls | nindent 2 }} +{{- end }} {{- end -}} {{- end }} @@ -326,6 +334,54 @@ Emits a single line ending with a comma. {{- end -}} {{- end }} +{{/* +"true" when PostgreSQL TLS settings and a CA certificate source are configured. +*/}} +{{- define "onyx.postgresTls.enabled" -}} +{{- if and .Values.postgresTls .Values.postgresTls.enabled }}true{{- end -}} +{{- end }} + +{{/* +Volume sourcing the PostgreSQL server CA. The backend validates the configured +path at startup, so mount exactly the configured key at its expected filename. +*/}} +{{- define "onyx.postgresTls.volume" -}} +{{- if include "onyx.postgresTls.enabled" . -}} +{{- $tls := .Values.postgresTls -}} +{{- if and $tls.caSecretName $tls.caConfigMapName -}} +{{- fail "postgresTls.caSecretName and postgresTls.caConfigMapName are mutually exclusive; set exactly one" -}} +{{- end -}} +{{- $caPath := $tls.caMountPath | default "/etc/postgres-ca/ca.crt" -}} +{{- $caKey := $tls.caKey | default "ca.crt" -}} +- name: postgres-ca + {{- if $tls.caSecretName }} + secret: + secretName: {{ $tls.caSecretName }} + items: + - key: {{ $caKey }} + path: {{ base $caPath }} + {{- else if $tls.caConfigMapName }} + configMap: + name: {{ $tls.caConfigMapName }} + items: + - key: {{ $caKey }} + path: {{ base $caPath }} + {{- else -}} + {{- fail "postgresTls.enabled is true but neither postgresTls.caSecretName nor postgresTls.caConfigMapName is set" -}} + {{- end }} +{{- end -}} +{{- end }} + +{{/* Mount for the PostgreSQL server CA. */}} +{{- define "onyx.postgresTls.volumeMount" -}} +{{- if include "onyx.postgresTls.enabled" . -}} +{{- $caPath := .Values.postgresTls.caMountPath | default "/etc/postgres-ca/ca.crt" -}} +- name: postgres-ca + mountPath: {{ dir $caPath }} + readOnly: true +{{- end -}} +{{- end }} + {{/* Model-server variant of the custom-CA env. The model servers run on a distroless image with no shell to run update-ca-certificates, so instead of pointing at the @@ -371,14 +427,33 @@ volumeMounts: {{- end -}} {{- end }} +{{/* +Render a volumes block combining pod-specific volumes with the model-server +custom-CA volume. Usage: include "onyx.modelServer.volumesWithCA" (dict "ctx" . "volumes" ) +*/}} +{{- define "onyx.modelServer.volumesWithCA" -}} +{{- $ca := include "onyx.customCACerts.volume" .ctx -}} +{{- $existing := .volumes -}} +{{- if or $ca $existing -}} +volumes: +{{- if $existing }} +{{ toYaml $existing | nindent 2 }} +{{- end }} +{{- if $ca }} +{{ $ca | nindent 2 }} +{{- end }} +{{- end -}} +{{- end }} + {{/* Render a volumeMounts block combining pod-specific mounts with the custom CA mount. Usage: include "onyx.volumeMountsWithCA" (dict "ctx" . "volumeMounts" ) */}} {{- define "onyx.volumeMountsWithCA" -}} {{- $ca := include "onyx.customCACerts.volumeMount" .ctx -}} +{{- $postgresTls := include "onyx.postgresTls.volumeMount" .ctx -}} {{- $existing := .volumeMounts -}} -{{- if or $ca $existing -}} +{{- if or $ca $postgresTls $existing -}} volumeMounts: {{- if $existing }} {{ toYaml $existing | nindent 2 }} @@ -386,6 +461,9 @@ volumeMounts: {{- if $ca }} {{ $ca | nindent 2 }} {{- end }} +{{- if $postgresTls }} +{{ $postgresTls | nindent 2 }} +{{- end }} {{- end -}} {{- end }} @@ -395,8 +473,9 @@ volume. Usage: include "onyx.volumesWithCA" (dict "ctx" . "volumes" ) */}} {{- define "onyx.volumesWithCA" -}} {{- $ca := include "onyx.customCACerts.volume" .ctx -}} +{{- $postgresTls := include "onyx.postgresTls.volume" .ctx -}} {{- $existing := .volumes -}} -{{- if or $ca $existing -}} +{{- if or $ca $postgresTls $existing -}} volumes: {{- if $existing }} {{ toYaml $existing | nindent 2 }} @@ -404,5 +483,8 @@ volumes: {{- if $ca }} {{ $ca | nindent 2 }} {{- end }} +{{- if $postgresTls }} +{{ $postgresTls | nindent 2 }} +{{- end }} {{- end -}} {{- end }} diff --git a/deployment/helm/charts/onyx/templates/configmap.yaml b/deployment/helm/charts/onyx/templates/configmap.yaml index 06b4a318b03..95f4e4e8fb2 100755 --- a/deployment/helm/charts/onyx/templates/configmap.yaml +++ b/deployment/helm/charts/onyx/templates/configmap.yaml @@ -36,10 +36,15 @@ data: {{- end }} {{- range $key, $value := .Values.configMap }} {{- $skipCraftProxyKey := and $craftEnabled (or (eq $key "SANDBOX_PROXY_HOST") (eq $key "SANDBOX_PROXY_PORT")) }} -{{- if and (not (empty $value)) (not $skipCraftProxyKey) }} +{{- $skipPostgresTlsKey := and (eq (include "onyx.postgresTls.enabled" $) "true") (or (eq $key "POSTGRES_SSLMODE") (eq $key "POSTGRES_SSLROOTCERT")) }} +{{- if and (not (empty $value)) (not $skipCraftProxyKey) (not $skipPostgresTlsKey) }} {{ $key }}: "{{ $value }}" {{- end }} {{- end }} + {{- if eq (include "onyx.postgresTls.enabled" .) "true" }} + POSTGRES_SSLMODE: {{ .Values.postgresTls.sslMode | default "verify-ca" | quote }} + POSTGRES_SSLROOTCERT: {{ .Values.postgresTls.caMountPath | default "/etc/postgres-ca/ca.crt" | quote }} + {{- end }} {{- if .Values.minio.enabled }} S3_ENDPOINT_URL: "http://{{ .Release.Name }}-minio:{{ default 9000 .Values.minio.service.port }}" {{- end }} diff --git a/deployment/helm/charts/onyx/templates/indexing-model-deployment.yaml b/deployment/helm/charts/onyx/templates/indexing-model-deployment.yaml index 848ce5f632c..a224d8c7c00 100644 --- a/deployment/helm/charts/onyx/templates/indexing-model-deployment.yaml +++ b/deployment/helm/charts/onyx/templates/indexing-model-deployment.yaml @@ -97,7 +97,7 @@ spec: {{- with include "onyx.modelServer.volumeMountsWithCA" (dict "ctx" . "volumeMounts" .Values.indexCapability.volumeMounts) }} {{- . | nindent 8 }} {{- end }} - {{- with include "onyx.volumesWithCA" (dict "ctx" . "volumes" .Values.indexCapability.volumes) }} + {{- with include "onyx.modelServer.volumesWithCA" (dict "ctx" . "volumes" .Values.indexCapability.volumes) }} {{- . | nindent 6 }} {{- end }} {{- end }} diff --git a/deployment/helm/charts/onyx/templates/inference-model-deployment.yaml b/deployment/helm/charts/onyx/templates/inference-model-deployment.yaml index 8368b0897da..b1b0a63c8f8 100644 --- a/deployment/helm/charts/onyx/templates/inference-model-deployment.yaml +++ b/deployment/helm/charts/onyx/templates/inference-model-deployment.yaml @@ -86,7 +86,7 @@ spec: {{- with include "onyx.modelServer.volumeMountsWithCA" (dict "ctx" . "volumeMounts" .Values.inferenceCapability.volumeMounts) }} {{- . | nindent 8 }} {{- end }} - {{- with include "onyx.volumesWithCA" (dict "ctx" . "volumes" .Values.inferenceCapability.volumes) }} + {{- with include "onyx.modelServer.volumesWithCA" (dict "ctx" . "volumes" .Values.inferenceCapability.volumes) }} {{- . | nindent 6 }} {{- end }} {{- end }} diff --git a/deployment/helm/charts/onyx/templates/sandbox-proxy/deployment.yaml b/deployment/helm/charts/onyx/templates/sandbox-proxy/deployment.yaml index 0bc9d467ea8..b8c6ecb4cb0 100644 --- a/deployment/helm/charts/onyx/templates/sandbox-proxy/deployment.yaml +++ b/deployment/helm/charts/onyx/templates/sandbox-proxy/deployment.yaml @@ -60,6 +60,42 @@ spec: app.kubernetes.io/component: sandbox-proxy {{- end }} {{- end }} + {{- if include "onyx.customCACerts.enabled" . }} + # The proxy is intentionally non-root with a read-only root filesystem, + # so it cannot run update-ca-certificates like the other backend pods. + # Build an additive bundle in its writable confdir instead: replacing the + # image bundle with only private roots would break normal public HTTPS. + initContainers: + - name: build-upstream-ca-bundle + image: "{{ .Values.celery_shared.image.repository }}:{{ .Values.celery_shared.image.tag | default .Values.global.version }}" + imagePullPolicy: {{ .Values.global.pullPolicy }} + command: + - /bin/sh + - -ec + - | + cp /etc/ssl/certs/ca-certificates.crt /var/run/sandbox-proxy/upstream-ca-bundle.crt + # Secret/ConfigMap mounts expose top-level keys as symlinks. Keep + # the search shallow so the backing hidden data directory is not + # included again, and print each PEM with a trailing newline. + find /usr/local/share/ca-certificates -maxdepth 1 \( -type f -o -type l \) -name '*.crt' -exec awk '1' {} + >> /var/run/sandbox-proxy/upstream-ca-bundle.crt + chmod 0444 /var/run/sandbox-proxy/upstream-ca-bundle.crt + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: {{ .Values.sandboxProxy.runAsUser }} + seccompProfile: + type: RuntimeDefault + capabilities: + drop: + - ALL + volumeMounts: + - name: confdir + mountPath: /var/run/sandbox-proxy + {{- with include "onyx.customCACerts.volumeMount" . }} + {{- . | nindent 12 }} + {{- end }} + {{- end }} containers: - name: sandbox-proxy image: "{{ .Values.celery_shared.image.repository }}:{{ .Values.celery_shared.image.tag | default .Values.global.version }}" @@ -90,6 +126,10 @@ spec: value: {{ $proxyPort | quote }} - name: SANDBOX_PROXY_HEALTHZ_PORT value: {{ $healthzPort | quote }} + {{- if include "onyx.customCACerts.enabled" . }} + - name: SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA + value: /var/run/sandbox-proxy/upstream-ca-bundle.crt + {{- end }} {{- include "onyx.envSecrets" . | nindent 12}} envFrom: - configMapRef: @@ -133,10 +173,19 @@ spec: # one writable mount on an otherwise read-only root. - name: confdir mountPath: /var/run/sandbox-proxy + {{- with include "onyx.postgresTls.volumeMount" . }} + {{- . | nindent 12 }} + {{- end }} volumes: - name: confdir emptyDir: sizeLimit: {{ .Values.sandboxProxy.confdirSizeLimit }} + {{- with include "onyx.customCACerts.volume" . }} + {{- . | nindent 8 }} + {{- end }} + {{- with include "onyx.postgresTls.volume" . }} + {{- . | nindent 8 }} + {{- end }} {{- with .Values.sandboxProxy.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/deployment/helm/charts/onyx/values.yaml b/deployment/helm/charts/onyx/values.yaml index d7ef0b3bc32..d7f3f2dc8c7 100644 --- a/deployment/helm/charts/onyx/values.yaml +++ b/deployment/helm/charts/onyx/values.yaml @@ -22,8 +22,8 @@ global: # -- Custom (private/internal) CA certificates for outbound TLS. # When enabled, every pod that makes outbound TLS connections mounts the # referenced Secret/ConfigMap and trusts its roots. Keys must be PEM certs -# ending in `.crt` (model servers also accept `.pem`). Two mechanisms, by pod -# type: +# ending in `.crt` (model servers also accept `.pem`). The handling varies by +# pod type: # - Shell-based pods (api server, celery workers, bots, MCP server) mount at # /usr/local/share/ca-certificates, run `update-ca-certificates` at startup, # and get REQUESTS_CA_BUNDLE / SSL_CERT_FILE pointed at the merged system @@ -32,6 +32,10 @@ global: # - Model servers run on a distroless image with no shell, so they mount at # /etc/onyx/certs and merge the mounted roots with certifi's public roots in # Python at startup (additive — public roots stay trusted). No root required. +# - The Craft sandbox proxy runs non-root on a read-only root filesystem. An +# init container builds an additive bundle in its writable confdir and +# configures mitmproxy through SANDBOX_PROXY_SSL_VERIFY_UPSTREAM_TRUSTED_CA +# to use that bundle for upstream TLS verification. # Version coupling: chart >= 0.7.0 pairs with the distroless model-server image # and can't run older ones — the model servers now run nonroot (1001, see their # securityContext below) and read ONYX_CUSTOM_CA_CERTS_DIR. Upgrade the chart and @@ -44,6 +48,22 @@ customCACerts: secretName: "" configMapName: "" +# -- TLS verification for PostgreSQL connections. +# When enabled, the chart mounts one CA certificate into each backend workload +# that connects to PostgreSQL and configures libpq-compatible POSTGRES_SSL* +# settings. The certificate source must already exist in the release namespace. +postgresTls: + enabled: false + # verify-ca validates the server certificate without requiring POSTGRES_HOST + # to be one of the certificate's DNS names. Use verify-full when it is. + sslMode: verify-ca + # Exactly one of caSecretName / caConfigMapName must be set when enabled. + caSecretName: "" + caConfigMapName: "" + caKey: ca.crt + # Full path exposed to POSTGRES_SSLROOTCERT. + caMountPath: /etc/postgres-ca/ca.crt + # Uncomment to skip the bundled CNPG operator and its CRDs, and let an existing # cluster-managed CloudNativePG operator reconcile the Cluster CR below (keep # postgresql.enabled: true). Two operators/CRD copies on one cluster drift in diff --git a/web/src/app/ee/admin/export-logs/page.tsx b/web/src/app/ee/admin/export-logs/page.tsx index 32990c7e0ed..7ec7865576f 100644 --- a/web/src/app/ee/admin/export-logs/page.tsx +++ b/web/src/app/ee/admin/export-logs/page.tsx @@ -17,6 +17,9 @@ const route = ADMIN_ROUTES.EXPORT_LOGS; const DESCRIPTION = "Download a zip of server log files to attach to an Onyx support thread."; const EXPORT_URL = "/api/admin/log-export"; +const EXPORT_ID_QUERY_PARAM = "export"; +// Export ids are uuid4().hex values; anything else found in the URL is noise. +const EXPORT_ID_PATTERN = /^[0-9a-f]{32}$/; const FALLBACK_FILENAME = "onyx_logs.zip"; const POLL_INTERVAL_MS = 2_000; // Give up on a poll that fails this many times in a row (~30s at the poll @@ -52,6 +55,18 @@ function extractFilename(response: Response): string { return match?.[1]?.trim() ?? FALLBACK_FILENAME; } +// Mirrors the export id into the URL (shallow, no navigation) so a refresh or +// shared tab can re-attach to the export. +function writeExportIdToUrl(exportId: string | null): void { + const url = new URL(window.location.href); + if (exportId === null) { + url.searchParams.delete(EXPORT_ID_QUERY_PARAM); + } else { + url.searchParams.set(EXPORT_ID_QUERY_PARAM, exportId); + } + window.history.replaceState({}, "", url.toString()); +} + function receiptLabel(receipt: LogExportReceipt): string { switch (receipt.status) { case "uploaded": @@ -97,12 +112,32 @@ export default function ExportLogsPage() { const [isStarting, setIsStarting] = useState(false); const [isDownloading, setIsDownloading] = useState(false); const downloadedExportIdRef = useRef(null); + // The export eligible for auto-download: one this tab started or watched + // collecting. A page that loads onto an already-finished export (restored + // URL) only offers the manual button, so page loads never trigger downloads. + const armedExportIdRef = useRef(null); // Pending deferred revocation: cancelling the timer must also revoke the URL. const pendingRevokeRef = useRef<{ timer: ReturnType; url: string; } | null>(null); + // Re-attach to an in-flight export after a refresh or navigation; the id is + // mirrored into the URL when an export starts. Malformed ids are scrubbed. + useEffect(() => { + const fromUrl = new URL(window.location.href).searchParams.get( + EXPORT_ID_QUERY_PARAM + ); + if (fromUrl === null) { + return; + } + if (EXPORT_ID_PATTERN.test(fromUrl)) { + setExportId(fromUrl); + } else { + writeExportIdToUrl(null); + } + }, []); + const { data: status, error: statusError } = useSWR( exportId === null ? null : SWR_KEYS.logExportStatus(exportId), errorHandlingFetcher, @@ -133,7 +168,11 @@ export default function ExportLogsPage() { statusError.status >= 400 && statusError.status < 500; if (isTerminal4xx) { - toast.error("Lost access to the running export. Start a new one."); + // An id restored from the URL can be stale (export already swept, or + // never real); it has no status yet, so scrub it without a toast. + if (status !== undefined) { + toast.error("Lost access to the running export. Start a new one."); + } } else if ( consecutivePollFailuresRef.current >= MAX_CONSECUTIVE_POLL_FAILURES ) { @@ -144,8 +183,9 @@ export default function ExportLogsPage() { return; } consecutivePollFailuresRef.current = 0; + writeExportIdToUrl(null); setExportId(null); - }, [statusError]); + }, [status, statusError]); useEffect(() => { return () => { @@ -198,11 +238,19 @@ export default function ExportLogsPage() { } }, []); - // Download exactly once per export, as soon as it is ready. + // Download exactly once per export, as soon as it is ready. Arming happens + // while the export is still collecting (or at start, in handleExport), so + // attaching to an already-finished export never auto-fires. useEffect(() => { + if (exportId === null || status === undefined) { + return; + } + if (status.state === "collecting") { + armedExportIdRef.current = exportId; + return; + } if ( - exportId === null || - status?.state !== "ready" || + armedExportIdRef.current !== exportId || downloadedExportIdRef.current === exportId ) { return; @@ -224,7 +272,9 @@ export default function ExportLogsPage() { ); } const body: { export_id: string } = await response.json(); + armedExportIdRef.current = body.export_id; setExportId(body.export_id); + writeExportIdToUrl(body.export_id); } catch (error) { console.error("Error starting log export:", error); toast.error("Failed to start the log export.");