Improve ingestion & query service

.gitignore

@@ -138,3 +138,7 @@ usermanagement_service
 Network Trash Folder
 Temporary Items
 .apdisk
+
+# Auto-generated SSO signing key (do not commit)
+secrets/
+env_copy

Dockerfile.unified

@@ -30,29 +30,130 @@ RUN apt-get update && apt-get install -y \
 # Create directories for all services
 WORKDIR /app
 
+# DEPENDENCY LAYERING
+#
+# Each service copies ONLY requirements.txt, installs, and copies its source
+# afterwards. Previously every service did `COPY <service>/` before `pip install`,
+# so editing any .py file invalidated that service's pip layer and reinstalled
+# everything — for ml_service that means torch (via
+# synthscholar[semantic] -> sentence-transformers), which is 1-2.5 GB and the reason
+# a rebuild took ~8 minutes on that step alone.
+#
+# With this ordering the pip layers are keyed on requirements.txt alone: a code-only
+# change reuses them, and a dependency change invalidates them automatically. That
+# also means `--no-cache` is no longer needed to pick up a pin change.
+
 # Copy APItokenmanager
-COPY APItokenmanager/ /app/APItokenmanager/
+COPY APItokenmanager/requirements.txt /app/APItokenmanager/requirements.txt
 WORKDIR /app/APItokenmanager
 RUN pip install --upgrade pip && \
     pip install -r requirements.txt && \
     pip install python-decouple gunicorn
+COPY APItokenmanager/ /app/APItokenmanager/
 
 # Copy query_service
-COPY query_service/ /app/query_service/
+COPY query_service/requirements.txt /app/query_service/requirements.txt
 WORKDIR /app/query_service
 RUN pip install -r requirements.txt
+COPY query_service/ /app/query_service/
 
-# Copy ml_service
-COPY ml_service/ /app/ml_service/
+# Copy ml_service. requirements.txt first — see DEPENDENCY LAYERING above. This is
+# the layer that matters most: it installs torch.
+COPY ml_service/requirements.txt /app/ml_service/requirements.txt
 WORKDIR /app/ml_service
-RUN pip install --use-deprecated=legacy-resolver "structsense==0.0.4" || \
-    pip install --use-deprecated=legacy-resolver --no-deps "structsense==0.0.4" && \
-    pip install --use-deprecated=legacy-resolver -r requirements.txt
+# The bug here was NOT operator precedence — `A || B && C` already groups as
+# `(A || B) && C`, which is what was intended. The parentheses below are only to
+# make that grouping obvious to the next reader.
+#
+# The bug is that the fallback is unverified. When the resolver fails, B installs
+# structsense with `--no-deps`, so crewai/litellm never arrive — and ml_service's
+# own requirements.txt lists neither, so nothing downstream fills the gap. The RUN
+# still exits 0, producing a GREEN BUILD shipping an ml_service whose every
+# gunicorn worker dies on `from structsense import kickoff`: exit 3, nothing bound
+# to 8007, supervisor giving up after 4 tries.
+#
+# The `--no-deps` fallback stays (the legacy resolver does genuinely fail on this
+# tree) but the import check now fails the BUILD rather than deferring the failure
+# to a deploy.
+# The legacy resolver is used ONLY for structsense, which genuinely needs it. It is
+# deliberately NOT used for requirements.txt — that difference is why ml_service
+# worked in the standalone ml_service/Dockerfile build and died on the unified one:
+#
+#   ml_service/Dockerfile : pip install -r requirements.txt                  (works)
+#   this file, previously  : pip install --use-deprecated=legacy-resolver -r ...
+#
+# The legacy resolver does not backtrack and does not verify consistency, so a later
+# requirement silently downgrades an earlier one. `aiohttp>=3.10` in requirements.txt
+# was therefore not enough: structsense==0.0.4 drags an old crewai/litellm that
+# tolerates aiohttp 3.8.x, that got installed last, and openai's vendored
+# httpx_aiohttp then failed on import with
+#   AttributeError: module aiohttp has no attribute SocketTimeoutError
+# taking out /api/synth-scholar entirely (and extraction, silently, behind its
+# guard). The modern resolver either produces a consistent set or fails the build.
+#
+# The explicit aiohttp upgrade is a safety net for that one regression, not the fix
+# — nothing in the tree wants aiohttp <3.10 (litellm requires >=3.14.2). Keep it
+# after the requirements install so nothing can pull it back down.
+# structsense is NOT installed. It is pinned at 0.0.4, which drags an old
+# crewai/litellm that holds aiohttp below 3.10 — and openai's vendored httpx_aiohttp
+# references aiohttp.SocketTimeoutError at import time, so the whole LLM chain fails
+# with an AttributeError. That took out synthscholar (and therefore every
+# /api/synth-scholar route, including the public reviews the UI reads) as collateral
+# damage, because both stacks import openai.
+#
+# Dropping it is safe: core/shared.py guards the import and
+# core/routers/structsense.py registers the three extraction WebSocket routes only
+# when the package is present. Everything else in that router — GET /ner,
+# GET /structured-resource, the save endpoints, GET /job/{task_id} — reads stored
+# data and keeps working, which is what /knowledge-base/ner needs.
+#
+# To re-enable, restore the install below and add "structsense" back to CHECKS in
+# scripts/verify_imports.py. It will need a structsense release whose crewai/litellm
+# accept aiohttp>=3.10, or the two stacks will keep fighting over it.
+#   RUN ( pip install --use-deprecated=legacy-resolver "structsense==0.0.4" \
+#         || pip install --use-deprecated=legacy-resolver --no-deps "structsense==0.0.4" ) \
+#       && ...
+# Two pip solves, not one. Together on this empty base image, pip gives up:
+#
+#   error: resolution-too-deep
+#   × Dependency resolution exceeded maximum depth
+#
+# synthscholar drags pydantic-ai -> openai and sentence-transformers -> torch; added
+# to the exact `==` pins in requirements.txt, the graph exceeds what pip will search.
+# (The standalone ml_service/Dockerfile does it in one solve only because its tiangolo
+# base preinstalls fastapi/uvicorn/pydantic, which constrains the search. python:3.11-
+# slim starts empty, so pip explores everything — which is why this worked locally.)
+#
+# Order matters: requirements.txt first so its pins are established, then synthscholar
+# resolves against an already-populated environment.
+RUN pip install -r requirements.txt \
+    && pip install "synthscholar[fulltext,semantic]==0.0.11"
+COPY ml_service/ /app/ml_service/
 
 # Copy usermanagement_service
-COPY usermanagement_service/ /app/usermanagement_service/
+COPY usermanagement_service/requirements.txt /app/usermanagement_service/requirements.txt
 WORKDIR /app/usermanagement_service
 RUN pip install -r requirements.txt
+COPY usermanagement_service/ /app/usermanagement_service/
+
+# ---------------------------------------------------------------------------
+# FINAL dependency reconciliation + verification. Must stay LAST, after every
+# service's pip step.
+#
+# All four services share ONE site-packages in this image (supervisor runs them as
+# programs in a single container), so the last `pip install` wins. usermanagement's
+# `aiohttp==3.9.1` was silently downgrading the `aiohttp>=3.10` that ml_service needs
+# for aiohttp.SocketTimeoutError — which openai's vendored httpx_aiohttp imports —
+# and that disabled every /api/synth-scholar route at runtime.
+#
+# The pins are now compatible (see the notes in usermanagement/chat requirements), so
+# this upgrade should be a no-op. It stays as a backstop, and the verification is the
+# real point: it previously ran inside the ml_service block, i.e. BEFORE usermanagement
+# downgraded aiohttp, so the build passed while the runtime was broken. Verifying here
+# checks the environment the container actually runs.
+WORKDIR /app/ml_service
+RUN pip install --upgrade "aiohttp>=3.10,<4" \
+    && python scripts/verify_imports.py
 
 
 # Create supervisor configuration
@@ -210,6 +311,32 @@ PYTHON_SCRIPT
     echo "Django migrations completed"
 fi
 
+# --- Auto-provision the SSO signing key (auth Phase 2, RS256 + JWKS) ---
+# usermanagement signs SSO tokens with an RS256 private key and publishes the
+# public half at /.well-known/jwks.json. An explicit key
+# (USERMANAGEMENT_JWT_PRIVATE_KEY_PEM or _FILE) always wins. Otherwise we
+# generate a persistent key here, ONCE, so the JWKS `kid` is stable across all
+# gunicorn workers and restarts (a per-worker ephemeral key would break
+# cross-worker verification). Persist it on the /app/secrets volume so it also
+# survives redeploys. Exported env is inherited by supervisord's children.
+if [ -z "${USERMANAGEMENT_JWT_PRIVATE_KEY_PEM}" ] && [ -z "${USERMANAGEMENT_JWT_PRIVATE_KEY_FILE}" ]; then
+    SSO_KEY_DIR="${USERMANAGEMENT_JWT_KEY_DIR:-/app/secrets}"
+    SSO_KEY_FILE="${SSO_KEY_DIR}/um_jwt_private.pem"
+    mkdir -p "${SSO_KEY_DIR}"
+    if [ ! -s "${SSO_KEY_FILE}" ]; then
+        echo "Generating SSO RS256 signing key at ${SSO_KEY_FILE}..."
+        if openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${SSO_KEY_FILE}" 2>/dev/null; then
+            chmod 600 "${SSO_KEY_FILE}"
+            echo "SSO signing key generated."
+        else
+            echo "WARNING: openssl key generation failed; usermanagement will fall back to a shared ephemeral key."
+        fi
+    else
+        echo "Using existing SSO RS256 signing key at ${SSO_KEY_FILE}."
+    fi
+    [ -s "${SSO_KEY_FILE}" ] && export USERMANAGEMENT_JWT_PRIVATE_KEY_FILE="${SSO_KEY_FILE}"
+fi
+
 # Ensure supervisor socket directory exists and is writable (in case /var/run is tmpfs)
 mkdir -p /var/run
 chmod 755 /var/run

README.unified-docker.md

@@ -551,7 +551,7 @@ All environment variables are loaded from the `.env` file in the project root. T
 - **JWT**: `*_SERVICE_JWT_SECRET_KEY` (service-specific keys), `JWT_POSTGRES_*` (all JWT-related variables)
 - **Oxigraph**: `OXIGRAPH_USER`, `OXIGRAPH_PASSWORD`
 - **Ports**: `API_TOKEN_PORT`, `QUERY_SERVICE_PORT`, `ML_SERVICE_PORT`, `USERMANAGEMENT_SERVICE_PORT`, `OXIGRAPH_PORT`, `PGADMIN_PORT`
-- **User Management OAuth**: `USERMANAGEMENT_SERVICE_JWT_SECRET_KEY`, `USERMANAGEMENT_PUBLIC_BASE_URL`, `USERMANAGEMENT_FRONTEND_CALLBACK_URL`, `USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY`, `USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS`, `GITHUB_CLIENT_ID/SECRET`, `ORCID_CLIENT_ID/SECRET`, `GLOBUS_CLIENT_ID/SECRET`
+- **User Management OAuth**: `USERMANAGEMENT_SERVICE_JWT_SECRET_KEY`, `USERMANAGEMENT_PUBLIC_BASE_URL`, `USERMANAGEMENT_FRONTEND_CALLBACK_URL`, `USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY`, `USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS` (comma-separated), `USERMANAGEMENT_WEB_SESSION_TTL_MIN` / `USERMANAGEMENT_WEB_REFRESH_TTL_MIN` (web access/refresh token TTLs), `GITHUB_CLIENT_ID/SECRET`, `ORCID_CLIENT_ID/SECRET`, `GLOBUS_CLIENT_ID/SECRET`
 - **Ollama**: `OLLAMA_MODEL`, `OLLAMA_PORT`, `OLLAMA_API_ENDPOINT`
 - **ML Service**: `MONGO_DB_URL`, `WEAVIATE_*`, etc.
 - **SynthScholar** (PRISMA reviews, lives in ml_service): `OPENROUTER_API_KEY` (operator fallback — UI normally forwards a per-user or admin-shared key), `NCBI_API_KEY` (optional, raises PubMed rate limits), `SEMANTIC_SCHOLAR_API_KEY` / `CORE_API_KEY` / `SYNTHSCHOLAR_EMAIL` (all optional). Database is shared — no separate DSN.

chat_service/core/main.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from contextlib import asynccontextmanager
 
 # logging
@@ -44,13 +45,29 @@ async def lifespan(app: FastAPI):
 app = FastAPI(lifespan=lifespan)
 logger = logging.getLogger(__name__)
 
-origins = [
+# Browser origins allowed to call this service.
+#
+# This list was missing https://brainkb.org and https://www.brainkb.org entirely —
+# so the production UI was blocked from the chat service, on the main domain, while
+# beta and sandbox worked. It also had "http://127.0.0.1:300" (a typo for :3000) and
+# a schemeless "localhost:3000", which can never match: the browser's Origin header
+# always carries a scheme.
+#
+# The four BrainKB services each keep their own copy of this list and they had
+# drifted apart. CORS_ALLOWED_ORIGINS (comma-separated) adds to the defaults so a
+# new domain does not need a code change in four places.
+_DEFAULT_ORIGINS = [
+    "https://brainkb.org",
+    "https://www.brainkb.org",
     "https://beta.brainkb.org",
     "https://sandbox.brainkb.org",
-    "localhost:3000",
     "http://localhost:3000",
-    "http://127.0.0.1:300",
+    "http://127.0.0.1:3000",
 ]
+origins = sorted({
+    *_DEFAULT_ORIGINS,
+    *(o.strip() for o in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if o.strip()),
+})
 
 app.add_middleware(
     CORSMiddleware,

chat_service/requirements.txt

@@ -4,7 +4,9 @@ uvicorn==0.29.0
 gunicorn==21.2.0
 
 # HTTP requests
-aiohttp==3.9.1
+# Shared site-packages with the other services — see the note in
+# usermanagement_service/requirements.txt. Must not pin below ml_service's >=3.10.
+aiohttp>=3.10,<4
 
 # Logging
 rich==13.9.4

docker-compose.unified.yml

@@ -37,6 +37,9 @@ services:
 
     volumes:
       - ./logs:/var/log/supervisor
+      # Persist the auto-generated SSO RS256 signing key across redeploys so the
+      # JWKS `kid` stays stable (see Dockerfile.unified start.sh + AUTH_UNIFICATION.md).
+      - ./secrets:/app/secrets
     networks:
       - brainkb-network
     depends_on: