diff --git a/.gitignore b/.gitignore index 63289b7..54ce914 100644 --- a/.gitignore +++ b/.gitignore @@ -138,3 +138,7 @@ usermanagement_service Network Trash Folder Temporary Items .apdisk + +# Auto-generated SSO signing key (do not commit) +secrets/ +env_copy diff --git a/Dockerfile.unified b/Dockerfile.unified index e2f9fd8..de17e5e 100644 --- a/Dockerfile.unified +++ b/Dockerfile.unified @@ -30,29 +30,130 @@ RUN apt-get update && apt-get install -y \ # Create directories for all services WORKDIR /app +# DEPENDENCY LAYERING +# +# Each service copies ONLY requirements.txt, installs, and copies its source +# afterwards. Previously every service did `COPY /` before `pip install`, +# so editing any .py file invalidated that service's pip layer and reinstalled +# everything — for ml_service that means torch (via +# synthscholar[semantic] -> sentence-transformers), which is 1-2.5 GB and the reason +# a rebuild took ~8 minutes on that step alone. +# +# With this ordering the pip layers are keyed on requirements.txt alone: a code-only +# change reuses them, and a dependency change invalidates them automatically. That +# also means `--no-cache` is no longer needed to pick up a pin change. + # Copy APItokenmanager -COPY APItokenmanager/ /app/APItokenmanager/ +COPY APItokenmanager/requirements.txt /app/APItokenmanager/requirements.txt WORKDIR /app/APItokenmanager RUN pip install --upgrade pip && \ pip install -r requirements.txt && \ pip install python-decouple gunicorn +COPY APItokenmanager/ /app/APItokenmanager/ # Copy query_service -COPY query_service/ /app/query_service/ +COPY query_service/requirements.txt /app/query_service/requirements.txt WORKDIR /app/query_service RUN pip install -r requirements.txt +COPY query_service/ /app/query_service/ -# Copy ml_service -COPY ml_service/ /app/ml_service/ +# Copy ml_service. requirements.txt first — see DEPENDENCY LAYERING above. This is +# the layer that matters most: it installs torch. +COPY ml_service/requirements.txt /app/ml_service/requirements.txt WORKDIR /app/ml_service -RUN pip install --use-deprecated=legacy-resolver "structsense==0.0.4" || \ - pip install --use-deprecated=legacy-resolver --no-deps "structsense==0.0.4" && \ - pip install --use-deprecated=legacy-resolver -r requirements.txt +# The bug here was NOT operator precedence — `A || B && C` already groups as +# `(A || B) && C`, which is what was intended. The parentheses below are only to +# make that grouping obvious to the next reader. +# +# The bug is that the fallback is unverified. When the resolver fails, B installs +# structsense with `--no-deps`, so crewai/litellm never arrive — and ml_service's +# own requirements.txt lists neither, so nothing downstream fills the gap. The RUN +# still exits 0, producing a GREEN BUILD shipping an ml_service whose every +# gunicorn worker dies on `from structsense import kickoff`: exit 3, nothing bound +# to 8007, supervisor giving up after 4 tries. +# +# The `--no-deps` fallback stays (the legacy resolver does genuinely fail on this +# tree) but the import check now fails the BUILD rather than deferring the failure +# to a deploy. +# The legacy resolver is used ONLY for structsense, which genuinely needs it. It is +# deliberately NOT used for requirements.txt — that difference is why ml_service +# worked in the standalone ml_service/Dockerfile build and died on the unified one: +# +# ml_service/Dockerfile : pip install -r requirements.txt (works) +# this file, previously : pip install --use-deprecated=legacy-resolver -r ... +# +# The legacy resolver does not backtrack and does not verify consistency, so a later +# requirement silently downgrades an earlier one. `aiohttp>=3.10` in requirements.txt +# was therefore not enough: structsense==0.0.4 drags an old crewai/litellm that +# tolerates aiohttp 3.8.x, that got installed last, and openai's vendored +# httpx_aiohttp then failed on import with +# AttributeError: module aiohttp has no attribute SocketTimeoutError +# taking out /api/synth-scholar entirely (and extraction, silently, behind its +# guard). The modern resolver either produces a consistent set or fails the build. +# +# The explicit aiohttp upgrade is a safety net for that one regression, not the fix +# — nothing in the tree wants aiohttp <3.10 (litellm requires >=3.14.2). Keep it +# after the requirements install so nothing can pull it back down. +# structsense is NOT installed. It is pinned at 0.0.4, which drags an old +# crewai/litellm that holds aiohttp below 3.10 — and openai's vendored httpx_aiohttp +# references aiohttp.SocketTimeoutError at import time, so the whole LLM chain fails +# with an AttributeError. That took out synthscholar (and therefore every +# /api/synth-scholar route, including the public reviews the UI reads) as collateral +# damage, because both stacks import openai. +# +# Dropping it is safe: core/shared.py guards the import and +# core/routers/structsense.py registers the three extraction WebSocket routes only +# when the package is present. Everything else in that router — GET /ner, +# GET /structured-resource, the save endpoints, GET /job/{task_id} — reads stored +# data and keeps working, which is what /knowledge-base/ner needs. +# +# To re-enable, restore the install below and add "structsense" back to CHECKS in +# scripts/verify_imports.py. It will need a structsense release whose crewai/litellm +# accept aiohttp>=3.10, or the two stacks will keep fighting over it. +# RUN ( pip install --use-deprecated=legacy-resolver "structsense==0.0.4" \ +# || pip install --use-deprecated=legacy-resolver --no-deps "structsense==0.0.4" ) \ +# && ... +# Two pip solves, not one. Together on this empty base image, pip gives up: +# +# error: resolution-too-deep +# × Dependency resolution exceeded maximum depth +# +# synthscholar drags pydantic-ai -> openai and sentence-transformers -> torch; added +# to the exact `==` pins in requirements.txt, the graph exceeds what pip will search. +# (The standalone ml_service/Dockerfile does it in one solve only because its tiangolo +# base preinstalls fastapi/uvicorn/pydantic, which constrains the search. python:3.11- +# slim starts empty, so pip explores everything — which is why this worked locally.) +# +# Order matters: requirements.txt first so its pins are established, then synthscholar +# resolves against an already-populated environment. +RUN pip install -r requirements.txt \ + && pip install "synthscholar[fulltext,semantic]==0.0.11" +COPY ml_service/ /app/ml_service/ # Copy usermanagement_service -COPY usermanagement_service/ /app/usermanagement_service/ +COPY usermanagement_service/requirements.txt /app/usermanagement_service/requirements.txt WORKDIR /app/usermanagement_service RUN pip install -r requirements.txt +COPY usermanagement_service/ /app/usermanagement_service/ + +# --------------------------------------------------------------------------- +# FINAL dependency reconciliation + verification. Must stay LAST, after every +# service's pip step. +# +# All four services share ONE site-packages in this image (supervisor runs them as +# programs in a single container), so the last `pip install` wins. usermanagement's +# `aiohttp==3.9.1` was silently downgrading the `aiohttp>=3.10` that ml_service needs +# for aiohttp.SocketTimeoutError — which openai's vendored httpx_aiohttp imports — +# and that disabled every /api/synth-scholar route at runtime. +# +# The pins are now compatible (see the notes in usermanagement/chat requirements), so +# this upgrade should be a no-op. It stays as a backstop, and the verification is the +# real point: it previously ran inside the ml_service block, i.e. BEFORE usermanagement +# downgraded aiohttp, so the build passed while the runtime was broken. Verifying here +# checks the environment the container actually runs. +WORKDIR /app/ml_service +RUN pip install --upgrade "aiohttp>=3.10,<4" \ + && python scripts/verify_imports.py # Create supervisor configuration @@ -210,6 +311,32 @@ PYTHON_SCRIPT echo "Django migrations completed" fi +# --- Auto-provision the SSO signing key (auth Phase 2, RS256 + JWKS) --- +# usermanagement signs SSO tokens with an RS256 private key and publishes the +# public half at /.well-known/jwks.json. An explicit key +# (USERMANAGEMENT_JWT_PRIVATE_KEY_PEM or _FILE) always wins. Otherwise we +# generate a persistent key here, ONCE, so the JWKS `kid` is stable across all +# gunicorn workers and restarts (a per-worker ephemeral key would break +# cross-worker verification). Persist it on the /app/secrets volume so it also +# survives redeploys. Exported env is inherited by supervisord's children. +if [ -z "${USERMANAGEMENT_JWT_PRIVATE_KEY_PEM}" ] && [ -z "${USERMANAGEMENT_JWT_PRIVATE_KEY_FILE}" ]; then + SSO_KEY_DIR="${USERMANAGEMENT_JWT_KEY_DIR:-/app/secrets}" + SSO_KEY_FILE="${SSO_KEY_DIR}/um_jwt_private.pem" + mkdir -p "${SSO_KEY_DIR}" + if [ ! -s "${SSO_KEY_FILE}" ]; then + echo "Generating SSO RS256 signing key at ${SSO_KEY_FILE}..." + if openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out "${SSO_KEY_FILE}" 2>/dev/null; then + chmod 600 "${SSO_KEY_FILE}" + echo "SSO signing key generated." + else + echo "WARNING: openssl key generation failed; usermanagement will fall back to a shared ephemeral key." + fi + else + echo "Using existing SSO RS256 signing key at ${SSO_KEY_FILE}." + fi + [ -s "${SSO_KEY_FILE}" ] && export USERMANAGEMENT_JWT_PRIVATE_KEY_FILE="${SSO_KEY_FILE}" +fi + # Ensure supervisor socket directory exists and is writable (in case /var/run is tmpfs) mkdir -p /var/run chmod 755 /var/run diff --git a/README.unified-docker.md b/README.unified-docker.md index 423b21d..b9bdb0a 100644 --- a/README.unified-docker.md +++ b/README.unified-docker.md @@ -551,7 +551,7 @@ All environment variables are loaded from the `.env` file in the project root. T - **JWT**: `*_SERVICE_JWT_SECRET_KEY` (service-specific keys), `JWT_POSTGRES_*` (all JWT-related variables) - **Oxigraph**: `OXIGRAPH_USER`, `OXIGRAPH_PASSWORD` - **Ports**: `API_TOKEN_PORT`, `QUERY_SERVICE_PORT`, `ML_SERVICE_PORT`, `USERMANAGEMENT_SERVICE_PORT`, `OXIGRAPH_PORT`, `PGADMIN_PORT` -- **User Management OAuth**: `USERMANAGEMENT_SERVICE_JWT_SECRET_KEY`, `USERMANAGEMENT_PUBLIC_BASE_URL`, `USERMANAGEMENT_FRONTEND_CALLBACK_URL`, `USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY`, `USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS`, `GITHUB_CLIENT_ID/SECRET`, `ORCID_CLIENT_ID/SECRET`, `GLOBUS_CLIENT_ID/SECRET` +- **User Management OAuth**: `USERMANAGEMENT_SERVICE_JWT_SECRET_KEY`, `USERMANAGEMENT_PUBLIC_BASE_URL`, `USERMANAGEMENT_FRONTEND_CALLBACK_URL`, `USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY`, `USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS` (comma-separated), `USERMANAGEMENT_WEB_SESSION_TTL_MIN` / `USERMANAGEMENT_WEB_REFRESH_TTL_MIN` (web access/refresh token TTLs), `GITHUB_CLIENT_ID/SECRET`, `ORCID_CLIENT_ID/SECRET`, `GLOBUS_CLIENT_ID/SECRET` - **Ollama**: `OLLAMA_MODEL`, `OLLAMA_PORT`, `OLLAMA_API_ENDPOINT` - **ML Service**: `MONGO_DB_URL`, `WEAVIATE_*`, etc. - **SynthScholar** (PRISMA reviews, lives in ml_service): `OPENROUTER_API_KEY` (operator fallback — UI normally forwards a per-user or admin-shared key), `NCBI_API_KEY` (optional, raises PubMed rate limits), `SEMANTIC_SCHOLAR_API_KEY` / `CORE_API_KEY` / `SYNTHSCHOLAR_EMAIL` (all optional). Database is shared — no separate DSN. diff --git a/chat_service/core/main.py b/chat_service/core/main.py index c94a23e..72e5be2 100644 --- a/chat_service/core/main.py +++ b/chat_service/core/main.py @@ -1,4 +1,5 @@ import logging +import os from contextlib import asynccontextmanager # logging @@ -44,13 +45,29 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan) logger = logging.getLogger(__name__) -origins = [ +# Browser origins allowed to call this service. +# +# This list was missing https://brainkb.org and https://www.brainkb.org entirely — +# so the production UI was blocked from the chat service, on the main domain, while +# beta and sandbox worked. It also had "http://127.0.0.1:300" (a typo for :3000) and +# a schemeless "localhost:3000", which can never match: the browser's Origin header +# always carries a scheme. +# +# The four BrainKB services each keep their own copy of this list and they had +# drifted apart. CORS_ALLOWED_ORIGINS (comma-separated) adds to the defaults so a +# new domain does not need a code change in four places. +_DEFAULT_ORIGINS = [ + "https://brainkb.org", + "https://www.brainkb.org", "https://beta.brainkb.org", "https://sandbox.brainkb.org", - "localhost:3000", "http://localhost:3000", - "http://127.0.0.1:300", + "http://127.0.0.1:3000", ] +origins = sorted({ + *_DEFAULT_ORIGINS, + *(o.strip() for o in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if o.strip()), +}) app.add_middleware( CORSMiddleware, diff --git a/chat_service/requirements.txt b/chat_service/requirements.txt index 843f9ab..266b63a 100644 --- a/chat_service/requirements.txt +++ b/chat_service/requirements.txt @@ -4,7 +4,9 @@ uvicorn==0.29.0 gunicorn==21.2.0 # HTTP requests -aiohttp==3.9.1 +# Shared site-packages with the other services — see the note in +# usermanagement_service/requirements.txt. Must not pin below ml_service's >=3.10. +aiohttp>=3.10,<4 # Logging rich==13.9.4 diff --git a/docker-compose.unified.yml b/docker-compose.unified.yml index feb90ea..6df0303 100644 --- a/docker-compose.unified.yml +++ b/docker-compose.unified.yml @@ -37,6 +37,9 @@ services: volumes: - ./logs:/var/log/supervisor + # Persist the auto-generated SSO RS256 signing key across redeploys so the + # JWKS `kid` stays stable (see Dockerfile.unified start.sh + AUTH_UNIFICATION.md). + - ./secrets:/app/secrets networks: - brainkb-network depends_on: diff --git a/env.template b/env.template index 1d2807d..ecdc3dc 100644 --- a/env.template +++ b/env.template @@ -75,16 +75,110 @@ USERMANAGEMENT_PUBLIC_BASE_URL=http://localhost:8004 # e.g. http://localhost:3000/auth/callback USERMANAGEMENT_FRONTEND_CALLBACK_URL=http://localhost:3000/auth/callback +# Extra browser origins allowed to call the APIs, comma-separated. ADDED to each +# service's built-in list (brainkb.org, www/beta/sandbox, localhost:3000), so leave +# it unset unless you serve the UI from another domain. Read by all four services — +# ml_service, query_service, usermanagement_service, chat_service. +# +# Must be a scheme + host (+ port if non-default), exactly as the browser sends the +# Origin header: "https://ui.example.org", not "ui.example.org" and no trailing +# slash. A schemeless entry silently never matches. +# +# Note: a fetch failing with "No Access-Control-Allow-Origin" is not always a CORS +# problem — the load balancer's own 502 page carries no CORS headers, so a service +# that is down looks identical to one that is misconfigured. Check the service +# responds at all before editing this. +# CORS_ALLOWED_ORIGINS=https://ui.example.org,https://another.example.org + # Fernet key for encrypting OAuth access/refresh tokens at rest. # Generate once: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" USERMANAGEMENT_OAUTH_TOKEN_ENC_KEY=Z85D3iJe4XCfJ5f8DExKXW3DfznyE4HzJ7XmmaOYtUQ= -# Comma-separated emails that are bootstrapped as SuperAdmin on first startup. +# Comma-separated emails bootstrapped as SuperAdmin on first startup — supports +# MULTIPLE (each email in the list is seeded as SuperAdmin+Admin, and every one is +# honored by require_admin even before its profile exists). Each must be a +# Globus-verifiable account: it goes live when that person logs in via Globus. # SuperAdmins also receive the regular Admin role for permissions, but the # SuperAdmin marker protects them from being banned, deleted, or having that # role stripped via the admin UI/API. Regular Admins (assigned through the # admin UI) remain fully manageable. -USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS=tekraj@mit.edu +# You can also add more SuperAdmins at runtime: an existing SuperAdmin assigns the +# SuperAdmin role to another user (SuperAdmin-only action). +# Example (multiple, comma-separated — no spaces needed): +# USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS=admin@example.org,lead@example.org,pi@lab.example.edu +USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS=admin@example.org,second-admin@example.org + +# ---------------------------------------------------------------------------- +# Single-issuer SSO (auth Phase 2 — RS256 + JWKS). See query_service/AUTH_UNIFICATION.md +# ---------------------------------------------------------------------------- +# usermanagement is the sole token issuer. A login (/api/auth/login) mints a +# short-lived refresh token; clients exchange it (/api/auth/exchange) for narrow +# per-service access tokens (aud=). Services verify via the published +# JWKS (/.well-known/jwks.json) and require their own audience, so a token minted +# for one service cannot be replayed against another. Legacy HS256 tokens above +# still work during migration. +# +# RS256 private key: AUTO-GENERATED — do NOT set it here. The unified container's +# start.sh creates a persistent key at /app/secrets/um_jwt_private.pem (mounted +# from ./secrets) on first boot, so the JWKS `kid` is stable across the 4 gunicorn +# workers and redeploys. Because it is auto-generated, it is intentionally NOT a +# variable in this template or in .env. +# To OVERRIDE with your own key (optional), add one of these lines yourself: +# USERMANAGEMENT_JWT_PRIVATE_KEY_PEM= +# USERMANAGEMENT_JWT_PRIVATE_KEY_FILE= +# Generate one with: openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out sso_key.pem +# Token issuer (`iss`). query_service's QUERY_SERVICE_SSO_ISSUER must match this. +USERMANAGEMENT_JWT_ISSUER=brainkb-usermanagement +# Token lifetimes (minutes). These control how long a login lasts before the user +# must re-authenticate — logins are NOT forever. +# - ACCESS: short-lived per-service token (exchanged on demand). Keep small. +# - REFRESH: the session lifetime from one login. When it expires, the user is +# asked to log in again. 720 = 12h; lower it (e.g. 240 = 4h, 60 = 1h) to force +# more frequent re-login, raise it for longer sessions. +# The MCP/skill caps its cached session to the refresh token's expiry (and to its +# own MCP_SESSION_TTL_MIN) — see brainkb_mcp/.env.example. +USERMANAGEMENT_ACCESS_TOKEN_TTL_MIN=15 +USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN=720 +# TTL (minutes) of the web-session ACCESS JWT the UI receives at OAuth login +# (?token=). The UI silently refreshes it using the web refresh token below, so +# this can stay short-ish. 720 = 12h. +USERMANAGEMENT_WEB_SESSION_TTL_MIN=720 +# TTL (minutes) of the web REFRESH token (?refresh=) the UI stores to renew its +# access token without re-login (exchanged at /api/auth/exchange). Governs the +# overall web session length before the user must sign in again. 10080 = 7 days. +USERMANAGEMENT_WEB_REFRESH_TTL_MIN=10080 +# Services a refresh token may be exchanged for (valid `aud` values). +USERMANAGEMENT_TOKEN_AUDIENCES=usermanagement,query_service,ml_service,chat_service + +# OAuth CLI/skill paste-code length (raw chars, before dash grouping). The code +# is short-lived (~10 min) + single-use, but kept high-entropy: 20 chars over a +# 30-symbol alphabet ≈ 98 bits. Clamped to 24 to fit the storage column. +USERMANAGEMENT_CLI_CODE_LEN=20 + +# Personal Access Tokens (PATs) — browser-free CLI/MCP auth. A user mints a PAT +# once (while logged in), sets it as BRAINKB_TOKEN in their MCP/skill config, and +# authenticates with it thereafter (no login/browser) until it expires or is +# revoked. PATs are opaque + hashed at rest + revocable instantly; on use they +# are exchanged for the same short-lived per-service token the login flow issues. +# DEFAULT_DAYS also acts as the SLIDING/idle window (see PAT_SLIDING): an +# actively-used token keeps rolling forward and never re-prompts; leave it unused +# this many days and it expires. +USERMANAGEMENT_PAT_DEFAULT_DAYS=3 # default lifetime + idle window when `days` omitted +USERMANAGEMENT_PAT_MAX_DAYS=365 # absolute ceiling — a PAT can't roll past created_at + this +USERMANAGEMENT_PAT_MAX_PER_USER=20 # max active PATs one user may hold +# Sliding expiry: each use pushes expiry to now + DEFAULT_DAYS (capped at +# created_at + MAX_DAYS). true = "stay logged in while you keep using it, expire +# when idle"; false = fixed lifetime from creation. +USERMANAGEMENT_PAT_SLIDING=true + +# query_service verifies RS256 access tokens against the issuer's JWKS. +# In the unified container usermanagement is on localhost:8004; in a split +# deployment point this at the usermanagement service URL. +QUERY_SERVICE_SSO_JWKS_URL=http://127.0.0.1:8004/.well-known/jwks.json +# Must match USERMANAGEMENT_JWT_ISSUER above. +QUERY_SERVICE_SSO_ISSUER=brainkb-usermanagement +# This service's audience — tokens must carry aud == this value. +QUERY_SERVICE_SSO_AUDIENCE=query_service # GitHub OAuth App GITHUB_CLIENT_ID= @@ -96,7 +190,13 @@ ORCID_CLIENT_SECRET= # Use https://sandbox.orcid.org for dev, https://orcid.org for prod ORCID_BASE_URL=https://orcid.org -# Globus OAuth (register at https://app.globus.org/settings/developers) +# Globus OAuth (register at https://developers.globus.org). +# IMPORTANT: use an app that supports REDIRECTS (not a Service Account), and add +# the redirect URL ${USERMANAGEMENT_PUBLIC_BASE_URL}/api/auth/globus/callback +# local: http://localhost:8004/api/auth/globus/callback +# deploy: https:///api/auth/globus/callback +# Same callback serves web + skill (paste-code) login. See usermanagement_service/README.md +# ("OAuth provider setup"). GitHub/ORCID use the same /api/auth//callback pattern. GLOBUS_CLIENT_ID= GLOBUS_CLIENT_SECRET= diff --git a/ml_service/core/configuration.py b/ml_service/core/configuration.py index 862ecbb..24d118a 100644 --- a/ml_service/core/configuration.py +++ b/ml_service/core/configuration.py @@ -61,6 +61,12 @@ def load_environment(env_name="env"): "JWT_POSTGRES_DATABASE_NAME": os.getenv("JWT_POSTGRES_DATABASE_NAME"), "JWT_ALGORITHM": os.getenv("JWT_ALGORITHM", "HS256"), "JWT_SECRET_KEY": os.getenv("ML_SERVICE_JWT_SECRET_KEY"), + # Phase 2 SSO: verify RS256 access tokens minted by usermanagement. + # Signature checked against the issuer's JWKS; token `aud` must equal + # SSO_AUDIENCE. Legacy HS256 tokens still validate too. + "SSO_JWKS_URL": os.getenv("ML_SERVICE_SSO_JWKS_URL", "http://127.0.0.1:8004/.well-known/jwks.json"), + "SSO_ISSUER": os.getenv("ML_SERVICE_SSO_ISSUER", "brainkb-usermanagement"), + "SSO_AUDIENCE": os.getenv("ML_SERVICE_SSO_AUDIENCE", "ml_service"), # Ingestion specific environment "RABBITMQ_USERNAME": os.getenv("RABBITMQ_USERNAME"), diff --git a/ml_service/core/jwks.py b/ml_service/core/jwks.py new file mode 100644 index 0000000..aec9d06 --- /dev/null +++ b/ml_service/core/jwks.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +"""JWKS-based verification of Phase 2 SSO access tokens (RS256). + +Verifies tokens minted by usermanagement (the single issuer). The signature is +checked against the issuer's published JWKS (fetched + cached), and the token's +`aud` MUST equal this service's audience — a token minted for another service is +rejected here (containment enforced by `aud`, not shared secrets). + +Synchronous on purpose: the sync `require_scopes` dependency must be able to call +it. The network fetch only happens on a cache miss / key rotation (10-min TTL). +""" +import logging +import threading +import time +from typing import Dict, Optional + +import httpx +from jose import jwt + +from core.configuration import load_environment + +logger = logging.getLogger(__name__) + +_env = load_environment() +JWKS_URL = _env["SSO_JWKS_URL"] +SSO_ISSUER = _env["SSO_ISSUER"] +SSO_AUDIENCE = _env["SSO_AUDIENCE"] + +_CACHE_TTL = 600 # seconds +_keys: Dict[str, dict] = {} +_fetched_at: float = 0.0 +_lock = threading.Lock() + + +def _refresh(force: bool = False) -> None: + global _keys, _fetched_at + with _lock: + if not force and _keys and (time.monotonic() - _fetched_at) < _CACHE_TTL: + return + try: + with httpx.Client(timeout=5.0) as client: + resp = client.get(JWKS_URL) + resp.raise_for_status() + data = resp.json() + _keys = {k["kid"]: k for k in data.get("keys", []) if k.get("kid")} + _fetched_at = time.monotonic() + except Exception as e: + logger.warning(f"JWKS fetch failed from {JWKS_URL}: {e}") + + +def _get_key(kid: Optional[str]) -> Optional[dict]: + if not kid: + return None + if kid in _keys and (time.monotonic() - _fetched_at) < _CACHE_TTL: + return _keys[kid] + _refresh() + if kid not in _keys: + _refresh(force=True) + return _keys.get(kid) + + +def verify_access_token(token: str) -> Optional[Dict]: + """Verify an RS256 SSO access token. Returns claims on success, or None if the + token is not RS256 / fails verification / issuer unreachable. Never raises — + callers fall back to legacy HS256.""" + try: + header = jwt.get_unverified_header(token) + except Exception: + return None + if header.get("alg") != "RS256": + return None + key = _get_key(header.get("kid")) + if not key: + return None + try: + payload = jwt.decode( + token, key, algorithms=["RS256"], + audience=SSO_AUDIENCE, issuer=SSO_ISSUER, + ) + except Exception as e: + logger.info(f"RS256 token rejected: {e}") + return None + if payload.get("typ") not in (None, "access"): + return None + return payload diff --git a/ml_service/core/main.py b/ml_service/core/main.py index 360f70c..82e4874 100644 --- a/ml_service/core/main.py +++ b/ml_service/core/main.py @@ -22,12 +22,16 @@ # SynthScholar (PRISMA literature review). Imports are lazy-guarded so a # missing `synthscholar` library doesn't crash the rest of ml_service — # the router simply won't mount and the /api/synth-scholar/* surface returns 404. +# `except Exception`, not `except ImportError`: a transitive dependency conflict +# surfaces as AttributeError/RuntimeError, not ImportError. The structsense chain +# took every worker down that way (aiohttp.SocketTimeoutError missing under the +# aiohttp==3.8.6 pin), and this guard would have let the same thing through. try: from core.synth_scholar.routes import router as synth_scholar_router from core.synth_scholar.database import init_db as init_synth_scholar_db, close_db as close_synth_scholar_db from core.synth_scholar.store import fix_stuck_reviews as fix_synth_scholar_stuck_reviews _SYNTH_SCHOLAR_AVAILABLE = True -except ImportError as _exc: +except Exception as _exc: # noqa: BLE001 - see comment above _SYNTH_SCHOLAR_AVAILABLE = False _SYNTH_SCHOLAR_IMPORT_ERROR = _exc @@ -166,14 +170,26 @@ async def lifespan(app: FastAPI): env_state = env.get("ENV_STATE", "production").lower() # CORS Configuration -origins = [ +# Browser origins allowed to call this service. Kept identical across the four +# BrainKB services, which each hold their own copy and had drifted apart. +# CORS_ALLOWED_ORIGINS (comma-separated) adds to these without a code change. +# +# Worth knowing when a fetch to this service reports "No Access-Control-Allow-Origin": +# check whether the service is actually up first. The ALB's own 502 page carries no +# CORS headers, so an ml_service that failed to boot presents in the browser as a +# CORS misconfiguration — which is exactly how the aiohttp 3.8.6 outage looked. +_DEFAULT_ORIGINS = [ + "https://brainkb.org", + "https://www.brainkb.org", "https://beta.brainkb.org", - "http://localhost", + "https://sandbox.brainkb.org", "http://localhost:3000", - "http://localhost:3001", "http://127.0.0.1:3000", - "http://127.0.0.1:3001", ] +origins = sorted({ + *_DEFAULT_ORIGINS, + *(o.strip() for o in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if o.strip()), +}) app.add_middleware( CORSMiddleware, diff --git a/ml_service/core/routers/jwt_auth.py b/ml_service/core/routers/jwt_auth.py index 60621fd..fbda53a 100644 --- a/ml_service/core/routers/jwt_auth.py +++ b/ml_service/core/routers/jwt_auth.py @@ -11,30 +11,23 @@ router = APIRouter() -@router.post("/register", status_code=201) +@router.post("/register", include_in_schema=False) async def register(user: UserIn): - """ - Register a new user. Uses proper connection management to avoid connection leaks. - - Note: The unique constraint check is handled atomically by the database. - This prevents race conditions where two concurrent requests could both pass - a pre-check and then both attempt to insert the same email. - """ - async with get_db_connection() as conn: - hashed_password = await get_password_hash(user.password) - - # Let the database enforce uniqueness atomically - no pre-check needed - # insert_data will catch UniqueViolationError and return a user-friendly message - return await insert_data( - conn=conn, fullname=user.full_name, email=user.email, password=hashed_password - ) + """Self-registration is DISABLED — no separate register step. Users onboard by + signing in with Globus / ORCID / GitHub (profile auto-created on first login).""" + raise HTTPException( + status_code=status.HTTP_405_METHOD_NOT_ALLOWED, + detail=("Self-registration is disabled. Sign in with Globus / ORCID / GitHub " + "— your account is created automatically on first login."), + ) -@router.post("/token") +@router.post("/login") +@router.post("/token", include_in_schema=False) # deprecated alias of /login async def login(user: LoginUserIn): """ - Authenticate user and return JWT token. Uses proper connection management to avoid connection leaks. - Reuses the same database connection for both authentication and scope retrieval to minimize connection pool usage. + Authenticate (password) and return a JWT. Primary path is `/api/login`; + `/api/token` is a deprecated alias. """ async with get_db_connection() as conn: authenticated_user = await authenticate_user(user.email, user.password, conn) diff --git a/ml_service/core/routers/structsense.py b/ml_service/core/routers/structsense.py index 99f22b0..c9f021c 100644 --- a/ml_service/core/routers/structsense.py +++ b/ml_service/core/routers/structsense.py @@ -15,7 +15,7 @@ # @File : structsense.py # @Software: PyCharm from fastapi import Request -from fastapi import APIRouter, File, UploadFile, HTTPException, Form, Depends, WebSocket +from fastapi import APIRouter, File, UploadFile, HTTPException, Form, Depends, WebSocket, Query from fastapi.responses import JSONResponse import logging from typing import Annotated @@ -23,12 +23,16 @@ from core.security import get_current_user, require_scopes, authenticate_websocket from core.shared import parse_yaml_or_json, upsert_structured_resources from core.pydantic_models import AgentConfig, TaskConfig, EmbedderConfig, SearchKeyConfig -from structsense import kickoff +# NOTE: `kickoff` is deliberately NOT imported here. It was, and it was unused — +# this module reaches structsense only through core.shared.run_kickoff_with_config, +# which now imports it defensively. A hard import here defeated that guard and took +# every endpoint in this router down with it, including the read-only /ner surface. import os from datetime import datetime, timezone from core.shared import upsert_ner_annotations from core.shared import (_is_safe_path, run_kickoff_with_config, JobStatus, _job_storage, - _handle_websocket_connection, _get_job) + _handle_websocket_connection, _get_job, + STRUCTSENSE_AVAILABLE, _STRUCTSENSE_IMPORT_ERROR) from core.configuration import load_environment from motor.motor_asyncio import AsyncIOMotorClient from typing import Optional @@ -71,10 +75,52 @@ def serialize_mongo_document(doc): router = APIRouter(tags=["Multi-agent Systems"]) +# The three extraction WebSocket endpoints below are the only routes in this module +# that need the structsense package (via core.shared.run_kickoff_with_config). +# Everything else — GET /ner, GET /structured-resource, the save endpoints, +# GET /job/{task_id} — only reads and writes stored data, and must keep working: +# /knowledge-base/ner in the web UI depends on GET /ner. +# +# So registration is conditional. When structsense is unimportable (or deliberately +# not installed — see Dockerfile.unified) those paths do not exist at all, which is +# a better contract than accepting a WebSocket upgrade and then failing mid-session +# once the client has already uploaded a PDF. +def _extraction_ws(path: str): + """Register a WebSocket route only if the extraction stack is usable.""" + def _decorator(fn): + if STRUCTSENSE_AVAILABLE: + return router.websocket(path)(fn) + logger.warning( + "Extraction endpoint %s NOT registered — structsense unavailable (%s)", + path, _STRUCTSENSE_IMPORT_ERROR, + ) + return fn + return _decorator + + @router.get("/ws-info") async def ws_info(): + # Report availability first: the extraction WebSocket routes are not registered + # when structsense is unavailable, so a client that trusts this document would + # otherwise be told to connect to paths that 404. + if not STRUCTSENSE_AVAILABLE: + return JSONResponse({ + "extraction_available": False, + "reason": ( + "The structsense extraction stack is not installed on this server, " + "so /ws/ner, /ws/extract-resources and /ws/pdf2reproschema are not " + "registered. Stored annotations remain readable via GET /api/ner and " + "GET /api/structured-resource." + ), + "disabled_endpoints": [ + "/ws/ner/{client_id}", + "/ws/extract-resources/{client_id}", + "/ws/pdf2reproschema/{client_id}", + ], + }) return JSONResponse({ + "extraction_available": True, "connect_to": "/ws/{client_id}/ner or /ws/{client_id}/resource", "protocol": [ {"type": "message", "text": "string (required, non-empty)"}, @@ -111,7 +157,7 @@ async def ws_info(): }) -@router.websocket("/ws/ner/{client_id}") +@_extraction_ws("/ws/ner/{client_id}") async def websocket_endpoint_ner(websocket: WebSocket, client_id: str): """WebSocket endpoint for NER processing with JWT authentication.""" try: @@ -149,7 +195,7 @@ async def websocket_endpoint_ner(websocket: WebSocket, client_id: str): except Exception: pass -@router.websocket("/ws/extract-resources/{client_id}") +@_extraction_ws("/ws/extract-resources/{client_id}") async def websocket_endpoint_ner(websocket: WebSocket, client_id: str): """WebSocket endpoint for NER processing with JWT authentication.""" try: @@ -187,7 +233,7 @@ async def websocket_endpoint_ner(websocket: WebSocket, client_id: str): except Exception: pass -@router.websocket("/ws/pdf2reproschema/{client_id}") +@_extraction_ws("/ws/pdf2reproschema/{client_id}") async def websocket_endpoint_ner(websocket: WebSocket, client_id: str): """WebSocket endpoint for NER processing with JWT authentication.""" try: @@ -555,4 +601,73 @@ async def get_structured_resources( raise except Exception as e: logger.error(f"Error retrieving structured resources: {str(e)}", exc_info=True) - raise HTTPException(status_code=500, detail=f"Failed to retrieve structured resources: {str(e)}") \ No newline at end of file + raise HTTPException(status_code=500, detail=f"Failed to retrieve structured resources: {str(e)}") + + +# --------------------------------------------------------------------------- +# Public (unauthenticated) read surface +# +# Backs https://brainkb.org/knowledge-base/ner and .../knowledge-base/resources, +# which are public pages. They were reading the authenticated endpoints above, so an +# anonymous visitor got 403 {"detail":"Not authenticated"} — get_current_user rejects +# for having no credential at all, before require_scopes(["read"]) is ever consulted. +# +# These serve the same documents as the authenticated routes. That is deliberate and +# safe here: saved annotations carry documentName, processedAt, sourceType, +# sourceContent and the extracted entities, with no submitter identity in the write +# path (see upsert_ner_annotations / upsert_structured_resources) — so there is +# nothing per-user to leak and no visibility flag to honour. If per-record visibility +# is ever added, filter it HERE first. +# +# `limit` is capped lower than the authenticated routes: these are open to the +# internet and the payload includes sourceContent, which can be a whole paper. +# --------------------------------------------------------------------------- + +PUBLIC_MAX_LIMIT = 200 + + +@router.get("/public/ner", + summary="Get saved NER annotations (public, no auth)", + description="Unauthenticated read of saved NER annotations. Backs the " + "public /knowledge-base/ner page.") +async def get_public_ner_annotations( + client: AsyncIOMotorClient = Depends(get_mongo_client), + document_name: Optional[str] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + limit: int = Query(default=100, ge=1, le=PUBLIC_MAX_LIMIT), + skip: int = Query(default=0, ge=0), +): + env = load_environment() + db_name = env.get("NER_DATABASE") + collection_name = env.get("NER_COLLECTION") + if not db_name or not collection_name: + raise HTTPException(status_code=500, detail="MongoDB configuration not found") + return await _get_documents_from_collection( + client=client, db_name=db_name, collection_name=collection_name, + document_name=document_name, start_date=start_date, end_date=end_date, + limit=limit, skip=skip, + ) + + +@router.get("/public/structured-resource", + summary="Get saved structured resources (public, no auth)", + description="Unauthenticated read of saved structured resources. Backs " + "the public /knowledge-base/resources page.") +async def get_public_structured_resources( + client: AsyncIOMotorClient = Depends(get_mongo_client), + document_name: Optional[str] = None, + start_date: Optional[str] = None, + end_date: Optional[str] = None, + limit: int = Query(default=100, ge=1, le=PUBLIC_MAX_LIMIT), + skip: int = Query(default=0, ge=0), +): + env = load_environment() + db_name = env.get("NER_DATABASE") + if not db_name: + raise HTTPException(status_code=500, detail="MongoDB configuration not found") + return await _get_documents_from_collection( + client=client, db_name=db_name, collection_name="structured_resource", + document_name=document_name, start_date=start_date, end_date=end_date, + limit=limit, skip=skip, + ) \ No newline at end of file diff --git a/ml_service/core/security.py b/ml_service/core/security.py index d8c2695..dfe6f15 100644 --- a/ml_service/core/security.py +++ b/ml_service/core/security.py @@ -29,6 +29,7 @@ from core.configuration import load_environment from core.database import get_user +from core import jwks logger = logging.getLogger(__name__) @@ -77,10 +78,20 @@ async def authenticate_user(email, password, conn): return user +def decode_token_any(token: str) -> dict: + """Decode a bearer token, newest scheme first: a Phase 2 SSO RS256 access + token (verified via the issuer's JWKS and required to carry + ``aud == this service``), else a legacy HS256 token signed with this + service's own secret. Raises jose errors if neither validates.""" + payload = jwks.verify_access_token(token) + if payload is not None: + return payload + return jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + + def decode_jwt(token: str): try: - decoded_token = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) - return decoded_token + return decode_token_any(token) except JWTError: raise HTTPException(status_code=403, detail="Could not validate credentials") @@ -89,7 +100,7 @@ async def get_current_user( token: Annotated[str, Depends(oauth2_scheme)], ): try: - payload = decode_jwt(token) + payload = decode_token_any(token) email = payload.get("sub") if email is None: raise credentials_exception @@ -108,7 +119,10 @@ async def get_current_user( def verify_scopes(required_scopes: List[str], token: str) -> bool: - decoded_token = decode_jwt(token) + try: + decoded_token = decode_token_any(token) + except (ExpiredSignatureError, JWTError): + raise HTTPException(status_code=403, detail="Could not validate credentials") token_scopes = decoded_token.get("scopes", []) return all(scope in token_scopes for scope in required_scopes) @@ -156,9 +170,10 @@ async def authenticate_websocket(websocket: WebSocket, required_scopes: Optional logger.warning("No JWT token provided in WebSocket connection") return None - # Decode and validate JWT token (same logic as get_current_user) + # Decode and validate JWT token (same logic as get_current_user): + # RS256 SSO access token (aud-checked) first, then legacy HS256. try: - payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + payload = decode_token_any(token) except ExpiredSignatureError: logger.warning("JWT token has expired") return None diff --git a/ml_service/core/shared.py b/ml_service/core/shared.py index a92faa8..117ea4f 100644 --- a/ml_service/core/shared.py +++ b/ml_service/core/shared.py @@ -36,7 +36,46 @@ from io import BytesIO import asyncio from enum import Enum -from structsense import kickoff +# Guarded like the SynthScholar import in core/main.py, and for the same reason: +# an unimportable extraction library must not take the whole service down. +# +# This was a hard import, and it is the only thing `structsense` is needed for. +# When the package or one of its heavy dependencies (crewai, litellm) is missing, +# the ImportError propagated through core.routers.structsense to core.main, so +# every gunicorn worker died before the app object existed — gunicorn exit 3, +# nothing bound to 8007, and supervisor eventually giving up. Endpoints that never +# touch kickoff (GET /api/ner and the rest of the saved-annotation surface, the +# whole /api/synth-scholar tree) were collateral damage. +# +# Now they keep working and only the extraction endpoints report the problem, via +# run_kickoff_with_config below. +# `except Exception`, not `except ImportError`, on purpose. The failure this was +# written for is an AttributeError, not an ImportError: +# +# openai/_vendor/httpx_aiohttp/transport.py: aiohttp.SocketTimeoutError +# AttributeError: module aiohttp has no attribute SocketTimeoutError +# +# A four-package import chain (structsense -> crewai -> litellm -> openai) can fail +# in any number of ways, and every one of them must cost the extraction endpoints +# rather than the process. Narrowing this to ImportError would re-open the exact +# outage it exists to prevent. +try: + from structsense import kickoff + _STRUCTSENSE_IMPORT_ERROR = None +except Exception as _exc: # noqa: BLE001 - see comment above + kickoff = None + _STRUCTSENSE_IMPORT_ERROR = _exc + logger.error( + "structsense is not importable (%s: %s) — extraction endpoints are " + "disabled. Usually a dependency version conflict in the image rather than a " + "missing package; check aiohttp/openai/litellm.", + type(_exc).__name__, _exc, + ) + +# Read by core/routers/structsense.py to decide whether to register the extraction +# WebSocket routes at all. Exported from here rather than recomputed there, so +# there is one source of truth for "is the extraction stack usable". +STRUCTSENSE_AVAILABLE = kickoff is not None from pathlib import Path from motor.motor_asyncio import AsyncIOMotorClient from datetime import datetime, timezone @@ -1198,6 +1237,18 @@ async def run_kickoff_with_config( api_key: str, chunking: bool, ): + # The one place that needs the structsense package. Fail here, per request, + # rather than at import time — see the guarded import at the top of this file. + if kickoff is None: + raise HTTPException( + status_code=503, + detail=( + "Extraction is unavailable: the structsense package failed to import " + f"on this server ({_STRUCTSENSE_IMPORT_ERROR}). Other endpoints are " + "unaffected." + ), + ) + # Load all sections from your config file all_config = await load_config(config_path, "all") diff --git a/ml_service/core/synth_scholar/backfill_oxigraph_push.py b/ml_service/core/synth_scholar/backfill_oxigraph_push.py index 8ef062f..9a0442e 100644 --- a/ml_service/core/synth_scholar/backfill_oxigraph_push.py +++ b/ml_service/core/synth_scholar/backfill_oxigraph_push.py @@ -1,6 +1,13 @@ """One-shot backfill — push every completed review's RDF to Oxigraph. -Use cases: +LEGACY. This is the direct-to-Oxigraph path, which writes triples without an ingest +job, without PROV-O provenance and without a search-index row — a named graph that +belongs to no space and that only an Admin SPARQL query can see. Reviews now reach +BrainKB by ingesting their TTL export through query_service (see the `brainkb` skill, +"Ingest a SynthScholar review"), which attributes the write to a real user. This +script refuses to run unless SYNTH_SCHOLAR_PUSH_TO_GRAPHDB is explicitly enabled. + +Use cases (all predate the ingest path): * You ran reviews **before** the auto-push (mark_completed → oxigraph_push) was wired in. Their result_json is sitting in Postgres but no triples @@ -35,6 +42,7 @@ import argparse import asyncio import logging +import os import sys from typing import Any, Optional @@ -42,7 +50,7 @@ from .database import async_session from .db_models import ReviewRow -from .oxigraph_push import push_review_to_oxigraph +from .oxigraph_push import _truthy, push_review_to_oxigraph logger = logging.getLogger("backfill_oxigraph_push") @@ -137,6 +145,23 @@ async def main_async(argv: list[str] | None = None) -> int: format="%(asctime)s %(levelname)-7s %(name)s: %(message)s", ) + # The direct push is off by default now (reviews go through the query_service + # ingest pipeline instead, so they get provenance). Without this guard the + # backfill would report "pushed N reviews" while _make_config returned None for + # every one of them and nothing left the process — a silent no-op is the worst + # possible outcome for a one-shot repair script. + if not _truthy(os.getenv("SYNTH_SCHOLAR_PUSH_TO_GRAPHDB"), default=False): + logger.error( + "SYNTH_SCHOLAR_PUSH_TO_GRAPHDB is not enabled, so every push would be " + "skipped. Reviews now reach BrainKB by ingesting their TTL export " + "through query_service, which is what records provenance — see the " + "`brainkb` skill, 'Ingest a SynthScholar review'. To run this legacy " + "direct push anyway, set SYNTH_SCHOLAR_PUSH_TO_GRAPHDB=true, and make " + "sure nothing else writes the same named graph (ingest is append-only " + "and the export's blank nodes do not de-duplicate)." + ) + return 2 + rows = await _select_reviews(args.review_id or None) if args.limit is not None: rows = rows[: args.limit] diff --git a/ml_service/core/synth_scholar/oxigraph_push.py b/ml_service/core/synth_scholar/oxigraph_push.py index e8acf2c..91586d4 100644 --- a/ml_service/core/synth_scholar/oxigraph_push.py +++ b/ml_service/core/synth_scholar/oxigraph_push.py @@ -22,8 +22,11 @@ ``GRAPHDATABASE_TYPE`` Informational; only ``OXIGRAPH`` triggers the GSP path. Default ``OXIGRAPH``. ``SYNTH_SCHOLAR_PUSH_TO_GRAPHDB`` Feature flag (``true``/``false``). - Default ``true`` — set to ``false`` to - disable the push without unsetting creds. + **Default ``false``** — reviews now + reach BrainKB through the query_service + ingest pipeline instead, which is what + gives them provenance. See the note in + ``_make_config`` before enabling. ``SYNTH_SCHOLAR_GRAPHDB_PATH`` Endpoint path (default ``/store`` for GSP, use ``/update`` for SPARQL Update). ``SYNTH_SCHOLAR_GRAPHDB_NAMED_GRAPH_PREFIX`` IRI prefix for review-specific named graphs. @@ -93,7 +96,23 @@ def _make_config(): optional ``synthscholar`` import fails, or when no usable endpoint can be composed. """ - if not _truthy(os.getenv("SYNTH_SCHOLAR_PUSH_TO_GRAPHDB"), default=True): + # Default OFF. This path writes to Oxigraph directly, which means it produces no + # ingest job, no PROV-O provenance and no search-index row — the triples land in + # a named graph that belongs to no space, so nothing except an Admin SPARQL query + # can see them. Reviews now reach BrainKB the same way all other RDF does: the + # TTL export is ingested through query_service (see the `brainkb` skill, + # "Ingest a SynthScholar review"), which attributes the write to a real user. + # + # Leaving this on alongside that ingest gives every review TWO graphs, not one. + # Review ids are unique, so reviews never collide with each other — the problem + # is that the two writers disagree about the IRI by one character. + # _named_graph_for below returns prefix + review_id with no trailing slash, while + # the ingest path registers and writes ...// (query_service normalises + # the registry lookup but writes the IRI verbatim). The result is a governed graph + # plus an unregistered shadow copy that only an Admin SPARQL query can see, which + # is worse than either outcome alone: search and provenance describe one of them + # and the store holds both. + if not _truthy(os.getenv("SYNTH_SCHOLAR_PUSH_TO_GRAPHDB"), default=False): return None try: diff --git a/ml_service/core/synth_scholar/routes.py b/ml_service/core/synth_scholar/routes.py index 73c935f..3d6c57d 100644 --- a/ml_service/core/synth_scholar/routes.py +++ b/ml_service/core/synth_scholar/routes.py @@ -1247,18 +1247,29 @@ async def cancel_review( ) +_EXPORT_FORMAT_PATTERN = ( + r"^(markdown|json|bibtex|ttl|jsonld|rubric_markdown|rubric_json|charting_markdown" + r"|charting_json|appraisal_markdown|appraisal_json|narrative_summary_markdown" + r"|narrative_summary_json)$" +) + + @router.get("/synth-scholar/reviews/{review_id}/export", tags=["SynthScholar — export"]) async def export_review( review_id: str, user: Annotated[dict, Depends(get_current_user)], - format: str = Query( - default="markdown", - pattern=r"^(markdown|json|bibtex|ttl|jsonld|rubric_markdown|rubric_json|charting_markdown|charting_json|appraisal_markdown|appraisal_json|narrative_summary_markdown|narrative_summary_json)$", - ), + format: str = Query(default="markdown", pattern=_EXPORT_FORMAT_PATTERN), model: Optional[str] = Query(default=None, description="Compare-mode only: export a single model's result by model_name"), ): """Export a completed review in the requested format.""" session = await _session_or_404(review_id, user) + return await _export_session(session, format, model) + + +# Everything past the access check is identical for the authenticated route above +# and the public one further down, so it lives here once — otherwise the two +# drift, and a format added for signed-in users silently 400s on public reviews. +async def _export_session(session: ReviewSession, format: str, model: Optional[str]): if session.status != ReviewStatus.COMPLETED or not session.result: raise HTTPException( status_code=400, @@ -1440,6 +1451,89 @@ async def get_pipeline_log( } +# --------------------------------------------------------------------------- +# Public (unauthenticated) read surface +# +# Backs /knowledge-base/synth-scholar in the web UI, which is reachable without +# signing in. Those pages previously called the authenticated routes above, which +# cannot work for an anonymous visitor: the UI has no credential to present, so +# the token exchange fails before any request is sent ("ML service requires a +# signed-in session"). Nor could they work for a signed-in visitor, because +# GET /reviews is owner-scoped — the "public listing" showed the viewer their own +# reviews, and a public review's detail page 404'd for everyone but its author. +# +# So `is_public` had no reader. These routes are it. Three rules hold throughout: +# * no `Depends(get_current_user)` — that is the point; +# * every lookup goes through review_store.get_public / list_public, which +# require is_public AND completed, so an unpublished review is invisible; +# * a review that is not published answers 404, never 403 — a 403 would confirm +# the id exists. +# Nothing here is owner-scoped, because published means published to everyone. +# --------------------------------------------------------------------------- + + +async def _public_session_or_404(review_id: str) -> ReviewSession: + session = await review_store.get_public(review_id) + if not session: + raise HTTPException(status_code=404, detail=f"Review '{review_id}' not found") + return session + + +@router.get( + "/synth-scholar/public/reviews", + response_model=list[ReviewSummaryResponse], + tags=["SynthScholar — public"], +) +async def list_public_reviews(): + """List every completed review its author marked Public. No auth.""" + sessions = await review_store.list_public() + return [_to_summary_response(s) for s in sessions] + + +@router.get( + "/synth-scholar/public/reviews/{review_id}", + response_model=ReviewDetailResponse, + tags=["SynthScholar — public"], +) +async def get_public_review(review_id: str): + """Full detail for one published review. 404 if it is not published.""" + return _to_detail_response(await _public_session_or_404(review_id)) + + +@router.get("/synth-scholar/public/reviews/{review_id}/log", tags=["SynthScholar — public"]) +async def get_public_review_log(review_id: str): + """Pipeline log for a published review — the provenance timeline reads this. + + Same content the author sees: the log records which pipeline step ran when, + which is exactly the provenance a published review is meant to carry. + """ + session = await _public_session_or_404(review_id) + log_entries = list(session.pipeline_log) + return { + "review_id": review_id, + "status": session.status.value, + "step_count": session.progress_step, + "log": log_entries, + "log_events": [ + {"step": i + 1, "message": msg, "timestamp": ts} + for i, (ts, msg) in enumerate(_parse_log_entry(e) for e in log_entries) + ], + } + + +@router.get("/synth-scholar/public/reviews/{review_id}/export", tags=["SynthScholar — public"]) +async def export_public_review( + review_id: str, + format: str = Query(default="markdown", pattern=_EXPORT_FORMAT_PATTERN), + model: Optional[str] = Query( + default=None, description="Compare-mode only: export a single model's result by model_name" + ), +): + """Export a published review. Same formats as the authenticated route.""" + session = await _public_session_or_404(review_id) + return await _export_session(session, format, model) + + @router.patch( "/synth-scholar/reviews/{review_id}/visibility", response_model=ReviewSummaryResponse, diff --git a/ml_service/core/synth_scholar/store.py b/ml_service/core/synth_scholar/store.py index 2bb6769..201284c 100644 --- a/ml_service/core/synth_scholar/store.py +++ b/ml_service/core/synth_scholar/store.py @@ -723,6 +723,37 @@ async def list_for_owner(self, owner_email: Optional[str]) -> list[ReviewSession sessions.append(s) return sessions + async def list_public(self) -> list[ReviewSession]: + """List every review its author published — `is_public` and completed. + + Read by the unauthenticated public routes, so the filter lives in SQL + rather than in the caller: a client-side filter over a full listing would + mean shipping other people's unpublished reviews to the browser first. + Runtime (in-flight) state is deliberately not merged in — a completed + review has none, and consulting it would only leak progress for rows that + are being re-run. + """ + async with async_session() as db: + result = await db.execute( + select(ReviewRow) + .where(ReviewRow.is_public.is_(True)) + .where(ReviewRow.status == ReviewStatus.COMPLETED.value) + .order_by(ReviewRow.created_at.desc()) + ) + rows = result.scalars().all() + return [_row_to_session(row) for row in rows] + + async def get_public(self, review_id: str) -> Optional[ReviewSession]: + """Fetch a review only if its author published it. Returns None otherwise + — callers turn that into a 404 so an unpublished review's existence is not + disclosed by the status code.""" + session = await self.get(review_id) + if not session: + return None + if not session.is_public or session.status != ReviewStatus.COMPLETED: + return None + return session + async def delete(self, review_id: str) -> bool: self._runtime.pop(review_id, None) async with async_session() as db: diff --git a/ml_service/requirements.txt b/ml_service/requirements.txt index dc518ce..1d4d8f7 100644 --- a/ml_service/requirements.txt +++ b/ml_service/requirements.txt @@ -3,7 +3,17 @@ fastapi==0.115.3 uvicorn==0.29.0 gunicorn==21.2.0 -aiohttp==3.8.6 +# Was pinned to 3.8.6, which is what killed ml_service on boot. Dockerfile.unified +# installs this file AFTER structsense, so the pin DOWNGRADED the aiohttp that +# structsense -> crewai -> litellm -> openai had just pulled in. Recent openai +# vendors httpx_aiohttp, which references aiohttp.SocketTimeoutError — added in +# aiohttp 3.10 — so importing openai raised +# AttributeError: module aiohttp has no attribute SocketTimeoutError +# and every gunicorn worker died before the app existed (exit 3, nothing on 8007). +# +# A floor rather than a pin, so pip can reconcile with whatever openai/litellm want. +# The other services pin 3.9.1; they do not import openai, so they are unaffected. +aiohttp>=3.10,<4 async-timeout==4.0.3 # Logging rich==13.9.4 @@ -47,9 +57,41 @@ grpcio-health-checking==1.60.2 PyMuPDF==1.26.5 ollama==0.6.0 -# SynthScholar (PRISMA literature review). The `synthscholar` package owns -# the AI pipeline; the local module under core/synth_scholar/ is just the -# orchestration layer (sessions, SSE, exports). SQLAlchemy 2.0 async backs -# the review tables, separate from ml_service's raw-asyncpg pool. -synthscholar[fulltext,semantic]==0.0.11 +# Imported DIRECTLY by core/shared.py and core/configuration.py, but never declared — +# they arrived as transitive dependencies of structsense. Dropping structsense took +# them with it, and ml_service died on boot with +# ModuleNotFoundError: No module named 'bs4' +# (gunicorn exit 3, nothing bound to 8007). Declared explicitly so what this service +# imports no longer depends on what another package happens to pull in. +# +# Found by parsing every import in core/ and diffing against this file, not one crash +# at a time. Deliberately NOT added: pydantic and starlette (guaranteed by the fastapi +# pin, and adding unbounded entries risks the resolution-too-deep problem again), +# pytest (test-only), structsense (guarded, intentionally absent), synthscholar +# (installed in its own pip step — see the note below). +beautifulsoup4>=4.12 +python-dotenv>=1.0 +rdflib>=6.0 +PyYAML>=6.0 + +# SynthScholar (PRISMA literature review). The `synthscholar` package owns the AI +# pipeline; the local module under core/synth_scholar/ is just the orchestration layer +# (sessions, SSE, exports). SQLAlchemy 2.0 async backs the review tables, separate +# from ml_service's raw-asyncpg pool. +# +# synthscholar itself is NOT listed here — it is installed as a separate pip step in +# Dockerfile.unified. Solved together with this file on an empty base image, pip gives +# up: +# +# error: resolution-too-deep +# × Dependency resolution exceeded maximum depth +# +# It drags pydantic-ai -> openai plus sentence-transformers -> torch, and combined +# with the exact `==` pins here the graph is too large for one solve. Two smaller +# solves succeed where one large one does not. (The standalone ml_service/Dockerfile +# gets away with one solve only because its tiangolo base image preinstalls +# fastapi/uvicorn/pydantic, which constrains the search.) +# +# Keep sqlalchemy here: core/synth_scholar/ imports it directly, independently of the +# synthscholar package. sqlalchemy[asyncio]>=2.0.30 \ No newline at end of file diff --git a/ml_service/scripts/verify_imports.py b/ml_service/scripts/verify_imports.py new file mode 100644 index 0000000..6fc077c --- /dev/null +++ b/ml_service/scripts/verify_imports.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Build-time gate: fail the image build if ml_service's optional AI stacks are +unimportable, instead of shipping and discovering it in production. + +Run from Dockerfile.unified after the pip steps for ml_service. + +Why this exists +--------------- +Both AI stacks are imported behind guards at runtime (core/main.py for +synthscholar, core/shared.py for structsense) so one broken dependency cannot take +the whole service down. That is the right runtime behaviour, but it means a broken +install is INVISIBLE: the container starts, /api/health returns 200, and the +affected routers are simply never mounted. The symptom reaches you hours later as +404s on endpoints that used to exist, or a browser CORS error that is really a +missing route. + +The specific failure this was written for: + + openai/_vendor/httpx_aiohttp/transport.py: aiohttp.SocketTimeoutError + AttributeError: module aiohttp has no attribute SocketTimeoutError + +aiohttp gained SocketTimeoutError in 3.10. `pip install --use-deprecated= +legacy-resolver` does not backtrack, so a later requirement silently downgrades an +earlier one — pinning aiohttp in requirements.txt is not sufficient, which is why +the Dockerfile forces it explicitly right before this script runs. + +Note it is an AttributeError, not an ImportError. Guards written as +`except ImportError` do not catch it; both are `except Exception` for this reason. +""" +from __future__ import annotations + +import importlib +import sys +import traceback +from pathlib import Path + +# Running this as `python scripts/verify_imports.py` puts scripts/ on sys.path, not the +# service root, so `import core.main` would fail with a misleading ModuleNotFoundError +# regardless of what is installed. Add the service root explicitly. +_SERVICE_ROOT = Path(__file__).resolve().parent.parent +if str(_SERVICE_ROOT) not in sys.path: + sys.path.insert(0, str(_SERVICE_ROOT)) + +# (module, what breaks if it is unimportable) +# +# structsense is deliberately absent: it is NOT installed by Dockerfile.unified, +# because structsense==0.0.4 holds aiohttp below 3.10 via an old crewai/litellm and +# that breaks openai's import for synthscholar too. Adding it back here without +# restoring the install would fail every build. See the comment on the ml_service +# pip step in Dockerfile.unified. +CHECKS = [ + ("synthscholar", "the whole /api/synth-scholar tree, incl. public reviews"), + # The package importing is not enough. core/main.py mounts the router inside a + # try/except, so a failure in THIS module disables every /api/synth-scholar route + # while the service still starts and /api/health still returns 200 — which is + # exactly how the outage stayed invisible. Checked separately, and fatally, + # because synthscholar is deliberately installed: if it is present but the router + # cannot import, that is a defect, not a configuration choice. + ("core.synth_scholar.routes", "every /api/synth-scholar route (the router itself)"), +] + +# Minimum aiohttp that provides SocketTimeoutError, which openai's vendored +# httpx_aiohttp transport references at import time. +AIOHTTP_MIN = (3, 10) + + +def _fail(*lines: str) -> None: + # Flush stdout first: it is block-buffered when the build log is a pipe, while + # stderr is not, so without this the failure block prints BEFORE the progress + # lines it refers to. + sys.stdout.flush() + print("", file=sys.stderr) + print("=" * 72, file=sys.stderr) + print("BUILD FAILED - ml_service import verification", file=sys.stderr) + print("=" * 72, file=sys.stderr) + for line in lines: + print(line, file=sys.stderr) + print("", file=sys.stderr) + sys.exit(1) + + +def check_aiohttp() -> str | None: + """Return an error string, or None if aiohttp is usable.""" + try: + import aiohttp + except Exception as exc: + return f"aiohttp itself is unimportable: {type(exc).__name__}: {exc}" + + version = getattr(aiohttp, "__version__", "unknown") + print(f" aiohttp {version}") + + # Check the attribute rather than parsing the version: the attribute is what + # openai actually touches, and it is the real contract. + if not hasattr(aiohttp, "SocketTimeoutError"): + return ( + f"aiohttp {version} has no SocketTimeoutError (needs " + f">={'.'.join(map(str, AIOHTTP_MIN))}). Something downgraded it AFTER " + "the explicit upgrade in Dockerfile.unified - check whether a " + "requirement added since then pins an older aiohttp." + ) + return None + + +def check_app() -> str | None: + """Import the real ASGI app, the way gunicorn does. + + This is the check that matters most, and it was missing. Verifying only the two + optional AI stacks let a build ship with a dependency the service's OWN code + imports directly: bs4 arrived transitively via structsense, dropping structsense + took it away, and ml_service died on boot with + `ModuleNotFoundError: No module named 'bs4'` — through core/shared.py, a module + this script never touched. + + Only ModuleNotFoundError is fatal. Anything else here (missing env vars, no + database) is expected at build time and says nothing about the image: importing + core.main builds the app and mounts routers but opens no connections — that + happens in the lifespan, at runtime. + """ + try: + importlib.import_module("core.main") + except ModuleNotFoundError as exc: + return ( + f"core.main cannot be imported: {exc}\n" + " A package the service imports directly is not installed. Add it to " + "requirements.txt rather than relying on another package to pull it in." + ) + except Exception as exc: + # Not a dependency problem — report and continue. + print(f" core.main imported with a non-import error ({type(exc).__name__}: " + f"{exc}) - expected at build time if it needs env/database") + return None + print(" core.main OK (the ASGI app gunicorn loads)") + return None + + +def main() -> None: + print("Verifying ml_service AI stack imports...") + + problems = [] + + err = check_aiohttp() + if err: + problems.append(err) + + for module, consequence in CHECKS: + try: + mod = importlib.import_module(module) + except Exception as exc: + # Deliberately broad: this chain (structsense -> crewai -> litellm -> + # openai, synthscholar -> pydantic-ai -> openai) raises AttributeError + # and RuntimeError as readily as ImportError. + # + # Print the FULL traceback, not just the message. When core/main.py + # swallowed these, diagnosis meant a docker exec into a running container + # to reproduce the import by hand; the frame that actually fails is the + # only thing that identifies the bad dependency. + problems.append( + f"{module} is unimportable - would disable {consequence}\n" + f" {type(exc).__name__}: {exc}" + ) + print(f" {module} FAILED - traceback follows:", flush=True) + traceback.print_exc() + else: + print(f" {module} {getattr(mod, '__version__', '?')} OK") + + err = check_app() + if err: + problems.append(err) + + if problems: + _fail(*(f" * {p}" for p in problems)) + + print("All ml_service imports verified.") + + +if __name__ == "__main__": + main() diff --git a/pgadmin-init/servers.json b/pgadmin-init/servers.json new file mode 100644 index 0000000..f7014e9 --- /dev/null +++ b/pgadmin-init/servers.json @@ -0,0 +1,15 @@ +{ + "Servers": { + "1": { + "Name": "BrainKB PostgreSQL", + "Group": "Servers", + "Host": "postgres", + "Port": 5432, + "MaintenanceDB": "brainkb", + "Username": "postgres", + "Password": "postgres", + "SSLMode": "prefer", + "Comment": "BrainKB PostgreSQL Database" + } + } +} diff --git a/query_service/AUTH_UNIFICATION.md b/query_service/AUTH_UNIFICATION.md new file mode 100644 index 0000000..a416d92 --- /dev/null +++ b/query_service/AUTH_UNIFICATION.md @@ -0,0 +1,643 @@ +# BrainKB Authentication & Identity — Unification Design + +Status: **Phase 1 + Phase 2 implemented & verified** (branch `auth-unification`). +Single-issuer RS256 + JWKS SSO with per-audience tokens is live-verified for +`query_service`, `usermanagement_service` (its own routes), and `ml_service`; the +`brainkb_mcp` is migrated to single sign-on. Legacy HS256 tokens still validate +during migration. `chat_service` deferred (not in use). The only step not +exercisable in the dev sandbox is the actual Globus browser consent. + +Several further decisions were taken **during** implementation — onboarding via +OAuth (no self-registration), `/api/token` → `/api/login`, role/group-level +capability grants, SuperAdmin-over-Admin, ban-not-delete, OAuth login via the +skill (paste-code), and expiring sessions. Each is recorded with its problem and +rationale in **§9. Implementation decisions log**. +Audience: BrainKB maintainers +Scope: `query_service`, `usermanagement_service`, `APItokenmanager` (Django), and downstream services (`ml_service`, `chat_service`, `brainkb_mcp`). + +--- + +## 0. Current authentication flow (as implemented) + +usermanagement is the **single issuer**. One login mints a short-lived **refresh +token**; clients **exchange** it for narrow, per-service **access tokens** +(`aud=`). Each service verifies against the issuer's **JWKS** and requires +its own audience, so a token minted for one service can't be replayed against +another (containment via `aud`, not shared secrets). Legacy HS256 per-service +tokens still validate during migration. + +### A. Password / SSO login + per-service calls (via the MCP) + +![Auth flow A — login, exchange, per-service access](docs/auth/flow-a.png) + +
Diagram source (Mermaid) + +```mermaid +sequenceDiagram + actor U as User + participant MCP as brainkb_mcp / skill + participant UM as usermanagement issuer + JWKS + participant QS as query_service + participant ML as ml_service + U->>MCP: brainkb_login(email, password) + MCP->>UM: POST /api/auth/login + UM-->>MCP: refresh token, aud=brainkb-auth + Note over MCP: cache refresh per session, expires with token + U->>MCP: a KG tool (ingest / search) + MCP->>UM: POST /api/auth/exchange audience=query_service + UM-->>MCP: access token, aud=query_service, ~15m + MCP->>QS: request + Bearer access token + QS->>UM: GET /.well-known/jwks.json (cached ~10m) + QS-->>MCP: 200 verify RS256 + iss + aud=query_service + U->>MCP: an admin tool (list users) + MCP->>UM: POST /api/auth/exchange audience=usermanagement + UM-->>MCP: access token, aud=usermanagement + MCP->>UM: admin call + Bearer, verify aud=usermanagement + Note over MCP,ML: same exchange for aud=ml_service. a query_service token is rejected elsewhere +``` +
+ +### B. OAuth login via the skill (Globus / ORCID / GitHub — paste-code) + +The browser consent is unavoidable (only the user can approve at the provider), +but the result is picked up out-of-band — no web UI needed. + +![Auth flow B — OAuth paste-code login via the skill](docs/auth/flow-b.png) + +
Diagram source (Mermaid) + +```mermaid +sequenceDiagram + actor U as User + participant MCP as brainkb_mcp / skill + participant BR as Browser + participant UM as usermanagement + participant P as Globus / ORCID / GitHub + U->>MCP: brainkb_globus_login() + MCP->>UM: POST /api/auth/cli/start provider + UM-->>MCP: authorize_url, state.mode=cli + MCP-->>U: open this URL + U->>BR: open URL, sign in + BR->>P: consent + P->>UM: GET /api/auth/provider/callback with code + state + UM->>UM: provision profile + default role, mint refresh, store behind short CODE + UM-->>BR: minimal page shows CODE + U->>MCP: brainkb_finish_login(CODE) + MCP->>UM: POST /api/auth/cli/exchange code + UM-->>MCP: refresh token, single-use code + Note over MCP: now exchanges per service as in flow A +``` +
+ +### C. Trust / containment overview + +![Auth flow C — single issuer, per-audience tokens, JWKS verification](docs/auth/flow-c.png) + +
Diagram source (Mermaid) + +```mermaid +flowchart LR + subgraph Clients + MCP[brainkb_mcp / skill] + WEB[Web UI] + end + UM["usermanagement
issuer - RS256 private key
/.well-known/jwks.json
login - exchange - OAuth"] + QS[query_service
aud=query_service] + ML[ml_service
aud=ml_service] + MCP -- login / exchange --> UM + WEB -- OAuth / login --> UM + MCP -- "Bearer aud=query_service" --> QS + MCP -- "Bearer aud=ml_service" --> ML + MCP -- "Bearer aud=usermanagement" --> UM + QS -- "fetch public keys (JWKS)" --> UM + ML -- "fetch public keys (JWKS)" --> UM +``` +
+ +Sessions are **not forever**: a cached login lasts until its refresh token expires +(`USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN`, default 12h; MCP additionally caps via +`MCP_SESSION_TTL_MIN`). On expiry the MCP forgets the credentials and prompts a new +login. + +--- + +## 1. Problem statement + +BrainKB today authenticates users through **two overlapping systems**: + +1. **JWT / scope system** — credentials in `Web_jwtuser` (email, password, `is_active`, + `read`/`write`/`admin` scopes). Tokens are issued/validated by `query_service` + (`/api/token`) and the user/scope records are administered by the Django + `APItokenmanager`. This layer answers **"can this client call this API?"** + +2. **Web / RBAC system** — identity in `Web_user_profile` + + `Web_oauth_identity` (Globus / ORCID / GitHub), with authorization in + `Web_user_role`. Tokens are issued by `usermanagement_service` + (`create_access_token_v2`, roles embedded) and OAuth auto-provisions users. + This layer answers **"who is this person and what are they allowed to do?"** + +Both live in the **same Postgres database** and are **joined by email** +(`query_service` RBAC already reads roles from the profile side by email). So this +is not two isolated silos — it is **one user split across two tables**: +`Web_jwtuser` (credentials) vs `Web_user_profile` (identity + roles). + +That split is the real source of the inconsistencies observed in practice: + +| Symptom | Cause | +|---|---| +| OAuth user cannot password-login to `query_service` | OAuth path creates a `Web_jwtuser` shell with a random password | +| Password `Web_jwtuser` has no roles → treated as public/no-role | No matching `Web_user_profile` row | +| Two token shapes | `query_service` token = scopes only; `v2` token = roles + scopes + profile | +| MCP must log in twice | One login for `query_service`, another for `usermanagement` | +| "Where do I add a user?" is ambiguous | Two admin surfaces (Django token-manager vs usermanagement) | + +--- + +## 2. Design principles (constraints we must respect) + +- **P1 — Preserve per-service token containment.** Per-service tokens were a + deliberate security decision: a token leaked from one service must not be + replayable against another. Any unification **must not** regress to a single + shared bearer token that works everywhere. +- **P2 — One identity, one authorization source.** "Who is this user and what can + they do" must be answerable exactly one way, from one source of truth. +- **P3 — OAuth is the primary onboarding path.** Globus/ORCID/GitHub provisioning + already exists and is where new users come from; password login remains a + supported credential, not a separate user universe. +- **P4 — Incremental, reversible migration.** No flag-day cutover; each phase must + be shippable and independently valuable. + +--- + +## 3. Key insight: separate the two decisions + +"Merge the auth systems" conflates two independent choices. Treat them separately: + +- **Decision A — Identity model.** One canonical user vs. the current + `jwtuser`-shell + `profile` split. +- **Decision B — Token issuance.** How many issuers, what token shape, and how + cross-service replay is prevented. + +Merging identity (A) is high-value and low-risk. Collapsing tokens (B) is where the +security tradeoff lives and must be chosen deliberately. + +--- + +## 4. Recommendation + +### 4.1 Decision A — Unify the identity model (do this) + +Converge on **one canonical user**, the profile, keyed by email: + +- `Web_user_profile` is the **single user record**. Credentials, OAuth identities, + roles, and API scopes all hang off it. +- `Web_jwtuser` is **demoted to a 1:1 credential record** for a profile (or removed + entirely, with password hash + `is_active` folded into the profile). It is no + longer a parallel "user." +- Every service resolves **identity + roles + scopes from this single source** + (`query_service` already reads roles by email — this generalizes that pattern). +- OAuth provisioning stops creating a "shell" user; it creates/links **the** profile. + A password can be set on the same profile later, so OAuth and password login + address the same identity. + +**Outcome:** the four-row table of symptoms in §1 disappears. There is one user, +one place roles/scopes live, one answer to "who is this." + +### 4.2 Decision B — Token strategy (choose one, deliberately) + +**Option B1 — Unified identity, per-service tokens (recommended first step).** +Keep each service issuing and validating its **own** token (own secret). The token +stays a thin proof-of-identity; **roles/scopes are read from the unified identity +source**, not trusted from a cross-service token. This fully preserves P1 +(containment) and requires no JWKS/audience machinery. It is the smallest change +that removes the identity fragmentation. + +**Option B2 — Single issuer + audience-scoped tokens (proper SSO; later).** +Make `usermanagement_service` the **auth authority** (it already issues role-bearing +v2 tokens and owns OAuth). Publish signing keys via **JWKS**; stamp every token with +an **`aud` (audience)** claim; each service accepts **only** tokens minted for its +own audience. This gives *one login* while still preventing cross-service replay — +containment is enforced by `aud` validation instead of separate secrets. This is the +clean long-term target but is a real project: JWKS rollout, `aud` enforcement in +every service, and token/user migration. + +**Do not** merge into a single shared-secret token that works everywhere — it +violates P1. + +### 4.3 Direction of travel + +`usermanagement_service` is the newer, richer identity service (OAuth + roles + v2 +tokens) and should become the **identity/auth authority**. The Django +`APItokenmanager`'s separate `jwtuser` notion is the **legacy piece to fold in**, +not the foundation to build on. + +--- + +## 5. Phased migration plan + +### Phase 0 — Freeze the contract (no behavior change) +- Document the canonical claims a BrainKB token carries: `sub` (email/profile id), + `roles`, `scopes`, `iss`, `exp`, and (reserved for Phase 2) `aud`. +- Confirm every service reads roles/scopes from the shared source, not only from the + token, so tokens can shrink safely later. + +### Phase 1 — Unify identity (Decision A + Option B1) +1. Make `Web_user_profile` the single user; ensure every `Web_jwtuser` has a matching + profile (backfill by email) and every profile can hold a password + `is_active`. +2. Repoint OAuth provisioning to create/link the profile (no shell user). +3. Repoint `query_service` login and RBAC to the unified record; keep its own token + issuance/validation (containment preserved). +4. Update `brainkb_mcp` + `brainkb_skills`: one login concept, one identity; the MCP + still obtains a per-service token where it calls each service, but from **one** + set of user credentials. +5. Migration safety: keep `Web_jwtuser` readable during transition; dual-read, then + deprecate. + +**Ship here.** This resolves the reported inconsistencies. Stop unless SSO is wanted. + +#### Phase 1 — what was actually implemented (branch `auth-unification`) + +Decision A (unify identity) + Decision B Option B1 (per-service tokens), verified +live against the running stack: + +- **Explicit credential→profile link.** Added `Web_jwtuser.profile_id` + (FK → `Web_user_profile.id`, `ON DELETE SET NULL`, indexed) to the ORM model + and as an idempotent inline migration in `usermanagement bootstrap`, plus a + **case-insensitive email backfill** for pre-existing rows. The credential row + is now a 1:1 record for the canonical profile, not an email-only sibling. + *(Verified: migration applied cleanly; 2/3 existing credentials backfilled.)* +- **Single provisioning path.** New `provision_identity(...)` in + `usermanagement/core/database.py` is the one place that ensures + profile + linked credential + default role (`Curator`) + bootstrap-superadmin + elevation. Idempotent; caller owns the transaction. +- **OAuth callback refactored** onto `provision_identity` — removed the ad-hoc + `_ensure_jwt_user_shell` + default-role + bootstrap duplication; OAuth now + sets `profile_id`. *(Verified: usermanagement boots clean, `/api/token` OK.)* +- **Password registration now provisions identity.** `query_service /api/register` + ensures a canonical profile, assigns the default role, and links the + credential (`_provision_profile_for_registration`, best-effort so it never + blocks account creation). This fixes the "password user has no roles" symptom. + *(Verified: fresh register → profile + `profile_id` link + `Curator` role.)* +- **Standardized token claims.** `query_service` tokens now carry + `sub`/`scopes`/`user_id`/`profile_id`/`roles`/`auth_source` — identical in + shape to usermanagement's v2 token — while still signed with query_service's + **own** secret (containment preserved). Roles remain informational; + authorization keeps re-reading roles from the DB via `core.rbac`. + *(Verified: both services' `/api/token` return the same claim shape; existing + protected endpoints still authorize.)* + +Not in Phase 1 (unchanged, deliberate): per-service secrets/token isolation; +`require_admin` still trusts the token `roles` claim in usermanagement (noted for +Phase 2); no shared issuer / JWKS / `aud`. + +### Phase 2 — Optional single-issuer SSO (Decision B → Option B2) +1. `usermanagement_service` becomes the sole issuer; expose **JWKS**. +2. Add `aud` to tokens; enforce per-service audience validation everywhere + (`query_service`, `ml_service`, `chat_service`, MCP targets). +3. Migrate services from local secret validation to JWKS + `aud`. +4. Retire per-service `/api/token` login endpoints in favor of the central one; + `APItokenmanager`'s user store is fully folded in. + +#### Phase 2 — what was actually implemented (branch `auth-unification`) + +Single-issuer, per-audience SSO with containment preserved via `aud` (Decision B +Option B2). Additive: legacy HS256 tokens keep working, so this is a safe +migration, not a flag-day cutover. + +- **usermanagement is the issuer (RS256 + JWKS).** New `core/tokens_rs256.py`: + loads an RS256 private key from `USERMANAGEMENT_JWT_PRIVATE_KEY_PEM`/`_FILE`, + or generates a **process-shared** ephemeral key persisted to a file (so all + uvicorn workers agree — a per-worker ephemeral key breaks cross-worker + verification). Publishes `GET /.well-known/jwks.json`. +- **Login → refresh → exchange.** New `core/routers/sso.py`: + `POST /api/auth/login` (`{email,password}`) returns a short-lived **refresh + token** (`aud=brainkb-auth`, not accepted by any service); + `POST /api/auth/exchange` (Bearer refresh, `{audience}`) returns a narrow + **access token** for a single service (`aud=`). Roles/scopes are + **re-read fresh from the DB at exchange time**, so a stale refresh token can't + carry stale authorization; active-credential + ban checks run here too. +- **query_service verifies via JWKS + `aud`.** New `core/jwks.py` (sync, so the + sync `require_scopes` dependency can use it): fetches + caches the JWKS, + verifies RS256, and **requires `aud == query_service`** and the configured + issuer. A new `decode_token_any()` tries RS256 (SSO) first, then falls back to + legacy HS256; it is wired into `get_current_user`, `get_current_user_optional`, + `verify_scopes`/`require_scopes`, and the websocket auth path. +- **Containment preserved.** Tokens are audience-scoped: a `query_service` token + cannot be replayed against `ml_service`. Enforcement is by `aud` validation, + not shared secrets — query_service keeps its own HS256 secret for legacy + tokens and never learns the issuer's private key. + +Deployment env (set before/at the fresh deploy): + +- usermanagement: `USERMANAGEMENT_JWT_PRIVATE_KEY_PEM` **or** `_FILE` — normally + **not needed**: the unified container's `start.sh` auto-generates a persistent + RS256 key at `/app/secrets/um_jwt_private.pem` (mounted from `./secrets`) on + first boot, giving a stable `kid` across the 4 gunicorn workers and redeploys. + Set `_PEM`/`_FILE` only to supply your own key. (If key generation is somehow + unavailable, the service falls back to a shared ephemeral key and logs a + warning.) Plus `USERMANAGEMENT_JWT_ISSUER` (default `brainkb-usermanagement`), + `USERMANAGEMENT_ACCESS_TOKEN_TTL_MIN` (15), `USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN` + (720), `USERMANAGEMENT_TOKEN_AUDIENCES` (`query_service,ml_service,chat_service`). + Web sign-in uses its own pair: `USERMANAGEMENT_WEB_SESSION_TTL_MIN` (720 — the + access JWT the UI gets as `?token=`) and `USERMANAGEMENT_WEB_REFRESH_TTL_MIN` + (10080 — the `?refresh=` token the UI exchanges for silent renew), so the overall + web session lasts 7 days without re-login. +- query_service: `QUERY_SERVICE_SSO_JWKS_URL` (default + `http://127.0.0.1:8004/.well-known/jwks.json`; in a split deployment point at + the usermanagement service URL), `QUERY_SERVICE_SSO_ISSUER` (must match the + issuer), `QUERY_SERVICE_SSO_AUDIENCE` (`query_service`). + +Verified before deploy: JWKS endpoint serves a key; `/api/auth/login` returns a +refresh token; RS256→JWK verification roundtrip validates `aud`+`iss`; jose +accepts JWK dicts. Full login→exchange→query_service acceptance, wrong-`aud` +rejection, and legacy-HS256 coexistence to be confirmed on the fresh deployment. + +Phase 2 rollout — services now verifying RS256 (aud-scoped, JWKS): +- **query_service** (`aud=query_service`) — verified live. +- **usermanagement** (`aud=usermanagement`) — its own protected routes now + accept SSO tokens via `verify_token` (local public-key verify, since it is the + issuer); `usermanagement` added to the exchangeable audiences. Verified live: + usermanagement-aud → 200, query_service-aud → 401, legacy v2 → 200. +- **ml_service** (`aud=ml_service`) — `core/jwks.py` verifier + `decode_token_any` + wired into `get_current_user`, `verify_scopes`/`require_scopes`, `decode_jwt` + (covers SSE), and the websocket path. Verified live: ml-aud → 200, + query_service-aud → 401, legacy HS256 → 200. + +- **brainkb_mcp** — migrated to single sign-on: `brainkb_login` mints a refresh + token (cached per session) that the MCP exchanges on demand for per-service + access tokens (`query_service`, `usermanagement`). One login now covers both KG + and admin tools — the old two-login logic is gone. Legacy `/api/token` remains + an automatic fallback. A header caller can pass a refresh token to unlock all + services. Verified live: login → exchange(query_service|usermanagement) → both + services accept their token; a query_service token is rejected at usermanagement. + +Remaining Phase 2 rollout (not yet done): +- **chat_service**: same `core/jwks.py` pattern (using `requests`, no httpx) — + deferred; service not currently in use. +- Once clients have migrated, retire the legacy HS256 `/api/token` paths and + fold in `APItokenmanager`; tighten `require_admin` to re-read roles from the DB. + +--- + +## 6. `brainkb_mcp` — how it authenticates (implemented) + +The MCP is migrated to single sign-on (see the diagrams in §0). Summary of the +implemented behavior: + +- **One login, per-service exchange.** `brainkb_login(email, password)` mints a + refresh token cached for the session; `_token_for(audience)` exchanges it on + demand for a `query_service` or `usermanagement` access token. The old + two-login model (`_um_login` / separate `_UM_TOKENS`) is gone — one login now + covers both KG and admin tools. +- **OAuth via the skill** (Globus/ORCID/GitHub): `brainkb_globus_login()` → + authorize URL → user signs in → browser shows a short code → + `brainkb_finish_login(code)` (backend `/api/auth/cli/start` + `/cli/exchange`, + flow B in §0). No web UI required. +- **Personal Access Token (browser-free, recommended).** Set `BRAINKB_TOKEN` to a + `brainkb_pat_…` (minted once via `brainkb_create_token`) — `_token_for` recognizes + the prefix and exchanges it at `/api/auth/pat/exchange` per service, no login or + browser afterward. `brainkb_use_token(pat)` does the same for one session. + Manage with `brainkb_list_tokens` / `brainkb_revoke_token`. See §9.11. +- **Header pass-through (stateless, multi-user remote):** a caller may send + `Authorization: Bearer `. A **refresh** token or a **PAT** unlocks all + services (the MCP exchanges it per service); a single **service access token** + is used as-is. +- **Sessions expire.** The cached session lasts until its refresh token expires + (`USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN`), hard-capped by `MCP_SESSION_TTL_MIN`. + On lapse the MCP forgets the credentials and asks the user to log in again; + `brainkb_whoami` reports `session_expires_in_min`. +- **Legacy fallback.** If the backend has no SSO, the MCP falls back to the + per-service `/api/token` login automatically, so it keeps working during + migration. +- **Abuse protection.** Per-caller (source-IP) rate limiting; login/register are + the strict `auth` bucket. See `brainkb_mcp/README.md`. + +--- + +## 7. Risks & mitigations + +| Risk | Mitigation | +|---|---| +| User/token migration errors | Backfill by email; dual-read `jwtuser`↔`profile` before deprecating | +| Losing containment (P1) | Phase 1 keeps per-service tokens; Phase 2 uses `aud`, not shared secrets | +| Multi-service coordination | Phase 2 only; roll out JWKS+`aud` service by service behind a validation shim | +| OAuth vs password ambiguity | Single profile owns both; password becomes an attribute of the identity | +| Downstream breakage (`ml_service`, `chat_service`) | Phase 0 makes services read roles from source, so token shape can change safely | + +--- + +## 8. Recommendation summary + +- **Do now:** unify the **identity model** (§4.1) with **per-service tokens** + (§4.2 Option B1). High value, preserves containment, small blast radius. +- **Do later, deliberately:** single-issuer **audience-scoped SSO** (§4.2 Option B2) + when ready to operate JWKS + `aud` properly. +- **Do not:** collapse to one shared-secret token usable across all services. + +--- + +## 9. Implementation decisions log (problems → decisions) + +Decisions taken while building Phases 1–2. Each: the problem, the decision, and +where it lives. All are live-verified except the Globus browser consent (dev +sandbox has no browser); the mechanics around it are verified. + +### 9.1 Onboarding: no self-registration — OAuth first-login creates the user +- **Problem.** Two onboarding paths existed: a password `/api/register` (created a + `Web_jwtuser`, initially role-less / inactive, needing admin activation) *and* + OAuth. The password path produced role-less "orphan" accounts and an extra + activation step, and duplicated identity creation. +- **Decision.** A user is created **only** on first Globus/ORCID/GitHub login, + which auto-provisions + links the profile and assigns a default role + (`provision_identity`). Self-registration is **disabled**: `/api/register` → + `405` on `query_service` and `ml_service`; the MCP `brainkb_register` tool was + removed. +- **Trade-off.** No API path to create a *password* account anymore (Globus is the + identity source). Existing/seeded password accounts still log in. + +### 9.2 Endpoint naming: `/api/token` → `/api/login` +- **Problem.** "token" was ambiguous next to the SSO refresh/exchange tokens, and + read as an issuance detail rather than "log in". +- **Decision.** Password login is **`/api/login`** on `query_service`, + `usermanagement`, `ml_service`; **`/api/token` kept as a hidden deprecated + alias** (same handler) so existing clients don't break. MCP prefers `/api/login` + and falls back to `/api/token`. + +### 9.3 Authorization: grant capabilities to a whole group/role +- **Problem.** Capabilities could be granted per-**user** (`user_capability_grants`) + or scoped to a space (access rules), but a **custom group/role** (e.g. + `uk_collaborator`) could only ever get the hardcoded `read_private` — no way to + give a whole group `ingest`/`create_private_space`, etc. +- **Decision.** New `role_capability_grants` table + `grant/revoke_role_capability` + and `/admin/capabilities/grant-role|revoke-role|role|available` endpoints. + Effective caps = role-derived ∪ **role/group grants** ∪ per-user grants. Only the + delegatable set is grantable (`grant`/`sparql_admin` stay admin-intrinsic — no + escalation). Also: a **space write access rule now GRANTS ingest** to a group + (previously rules could only restrict). + +### 9.4 Admin hierarchy: SuperAdmin-over-Admin (and who creates whom) +- **Problem.** Any Admin could assign/remove the `Admin` role on, or ban, another + Admin — no real hierarchy; a peer/rogue Admin could lock others out. +- **Decision.** Assigning/removing the `Admin` (or `SuperAdmin`) role and banning + an Admin are **SuperAdmin-only**. `SuperAdmin` stays bootstrap-seeded and + protected (never removable/bannable). Regular Admins manage non-admin users. +- **Who creates whom.** **SuperAdmin** is bootstrapped at deployment via + `USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS` (seeded on first login) and can grant + `SuperAdmin`/`Admin` to others. **Admin** is created by a SuperAdmin assigning the + `Admin` role. Admin and SuperAdmin have the same KG capabilities; the difference + is that SuperAdmin is the protected, admin-managing tier. + +### 9.4a `manage_team_space` is scoped, not blanket +- **Problem.** A `manage_team_space` holder could manage **every** team space — + effectively a platform-wide admin power leaking through a delegatable capability. +- **Decision.** A non-admin manages a team space **only** if they **own** it + (created it), are matched by a per-space `manage` access rule, or hold + `manage_team_space` **and are a member of that space** — i.e. only spaces they + created or were assigned to. **Admin/SuperAdmin** still manage all. (query_service + `_can_manage`.) Verified: non-member holder → 403; owner/member-holder/admin → OK. + +### 9.5 Removal: ban, never hard-delete +- **Problem.** Hard-deleting a user destroys provenance/audit history and is + irreversible. +- **Decision.** **We don't delete.** `DELETE /api/admin/users/{id}` → `405`; + removal is a reversible **ban** (`/ban` + `DELETE /ban` to lift), which preserves + history. `deactivate` toggles login access. + +### 9.6 OAuth login through the skill (paste-code) +- **Problem.** OAuth needs a browser consent the MCP can't perform, and the normal + callback redirects to the **web UI** — so "Globus login via skill" seemed to + require the website. +- **Decision.** An out-of-band **paste-code** flow: `POST /api/auth/cli/start` + (state marked `cli`) → user signs in → the callback mints an SSO refresh token, + stores it behind a short one-time **code**, and shows a minimal page (no SPA) → + `POST /api/auth/cli/exchange {code}` returns the refresh token (single-use). MCP: + `brainkb_globus_login` → `brainkb_finish_login(code)`. + +### 9.7 Sessions expire (logins are not forever) +- **Problem.** The MCP cached credentials/refresh, so a login effectively lasted + forever (a cached password could silently re-login). +- **Decision.** A cached session lives only until its **refresh token expires** + (`USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN`, default 12h), hard-capped by + `MCP_SESSION_TTL_MIN`. On lapse the MCP forgets the credentials and prompts a new + login; `brainkb_whoami` reports `session_expires_in_min`. + +### 9.8 Operational decisions +- **SSO signing key** auto-provisioned at container start to `/app/secrets` (a + persistent volume) so the JWKS `kid` is stable across the 4 gunicorn workers and + redeploys — fixing a real multi-worker "MissingGreenlet"/per-worker-key failure + found in testing. An explicit `USERMANAGEMENT_JWT_PRIVATE_KEY_PEM/_FILE` overrides. +- **Rate limiting** (per-caller, source-IP) in the MCP; `login`/OAuth-start use a + strict bucket. Large **file** ingest is not byte-capped and uses no read/write + timeout (raw-text ingest is capped) so ~5 GB TTL/JSON-LD uploads aren't aborted. + +### 9.9 `require_admin` re-reads roles from the DB (done) +- **Problem.** `require_admin` trusted the token's `roles` claim, so a revoked/ + demoted admin kept access until their token expired. +- **Decision.** `require_admin` (and the `_is_superadmin` gate on admin-tier + actions) now re-read **active roles from the DB** (`_current_roles_from_db` by + profile_id/email); the bootstrap-superadmin allowlist is still honored for first + sign-in. Verified: an old token claiming `roles=[Admin]` is accepted while the + role exists, and rejected (403) the moment the role is removed in the DB. + +### 9.10 Web UI migrated off the password service account (SSO session-exchange) +- **Problem.** The `brainkb-ui` audit showed user login is already OAuth-only, but + ML/query calls used a shared **service-account password** on `/api/token` + (`NEXT_PUBLIC_JWT_USER/PASSWORD`), and two NER routes took form-entered + credentials. That was the last thing blocking password-login retirement. +- **Decision.** New `POST /api/auth/session-exchange` (usermanagement): swap an + authenticated **session token** (the UI's usermanagement JWT) for a short-lived + `aud=` access token, **scopes derived from roles** (RBAC authoritative — + removes the need for the Django scope manager). The UI now session-exchanges for + `ml_service`/`query_service` tokens; the service-account password is a deprecated + fallback only. Verified backend-side (session token → aud token → 200 at + query_service); the UI change needs deploy testing. + +### 9.11 Personal Access Tokens (PATs) — browser-free CLI/MCP auth +- **Problem.** CLI/MCP auth required either a password login or a one-time Globus + browser paste-code **every session** — and both mint an RS256 refresh token that + depends on the SSO key material (the per-worker key-file we had to persist). Users + wanted a "generate once, paste into the skill config, no browser afterward" + credential. +- **Decision.** Add an **opaque, DB-backed Personal Access Token**. A user mints one + while logged in (`POST /api/auth/tokens` → shown once as `brainkb_pat_…`), sets it + as `BRAINKB_TOKEN` in the MCP config, and every call thereafter exchanges it at + `POST /api/auth/pat/exchange` for the same short-lived `aud=` access token + the refresh flow issues. No browser/login after the one-time mint. +- **Why opaque (not a signed/long-lived JWT).** (1) **Instantly revocable** — a + signed JWT lives until it expires; an opaque token is a DB row we flip. (2) **No + key material exposed to the user** — they handle one string, never a key; this is + exactly the "no RSA/key for CLI" ask. (3) **Roles re-read live** at exchange time, + so a ban/demotion takes effect immediately. Only the SHA-256 hash is stored, so a + DB leak yields no usable tokens. +- **Why services need no change.** The PAT is validated only at usermanagement; the + token it *exchanges into* is the ordinary RS256 per-service token, so query/ml + verify it via JWKS unchanged — containment (`aud`) preserved. +- **Model.** `Web_personal_access_token` (token_hash unique, prefix, name, + profile_id, jwt_user_id, email, revoked, expires_at, last_used_at). Endpoints: + create / list / revoke (session-auth) + `pat/exchange` (PAT-auth). Env: + `USERMANAGEMENT_PAT_DEFAULT_DAYS` (**3** — short-lived by default; a user may + still request up to the cap), `_MAX_DAYS` (365), `_MAX_PER_USER` (20). +- **MCP.** `BRAINKB_TOKEN` env + `brainkb_use_token` set a PAT; `_token_for` + recognizes the `brainkb_pat_` prefix and PAT-exchanges (header, session, or env). + Tools: `brainkb_create_token`, `brainkb_list_tokens`, `brainkb_revoke_token`. +- **Verified live** (unified container): create → list → exchange → **200** at + query_service → revoke → exchange **401** (instant revocation); exchange re-read + roles fresh from the DB (`auth_source=pat`). +- **Note (signing scheme).** Considered switching the internal SSO tokens to an + HS256 shared secret. **Decision: no change for now** — stay on RS256/JWKS (full + reasoning in §9.12). The PAT is independent of this (opaque, DB-validated); it + exchanges into whatever the issuer mints. + +### 9.12 Why the SSO access tokens are RS256 (asymmetric) and not a shared HS256 secret +- **Question raised.** For a four-service deployment, wouldn't a single shared + secret (HS256) be simpler than RSA — no JWKS, no public-key distribution, no + per-worker key generation? (It would; the shared secret *is* the key. The + question is what that simplicity costs.) +- **The core property RS256 buys: a verifier that cannot forge.** + usermanagement is the **sole issuer**; query/ml/chat only **verify**. + - **RS256 (what we use).** The issuer holds the **private** key (signs); every + other service holds only the **public** key (published at + `/.well-known/jwks.json`) and can *verify without holding any secret capable of + minting*. If ml_service is compromised, the attacker gets a public key — they + **still cannot mint tokens** for any service. + - **HS256 (shared secret).** Signing and verifying use the **same** secret. Every + service that verifies must hold a secret that can equally **forge**. Compromise + of *any one* service (or a leaked env/log) lets the attacker mint tokens for + **all** services with arbitrary `sub`/`roles`/`scopes`/`aud`. The other services + cannot distinguish the forgery from a genuine token. +- **Containment is the whole point of Phase 2.** Per-`aud` tokens give crypto-enforced + containment: a token minted for `query_service` is provably unusable at + `ml_service`. Under HS256 that containment degrades to *trust-based* — any + secret-holder can set `aud` to anything — which undoes a property we deliberately + built (see §4.2 / Phase 2). +- **Other HS256 costs (not removed, just relocated).** A shared secret still has to + be distributed to every service, environment, and worker, and **rotated + everywhere simultaneously**; its compromise radius is all four services at once. + So HS256 removes *asymmetric-key* management but not *secret* management. +- **Why RSA's operational pain is acceptable.** The one real downside we hit was + provisioning the private key across gunicorn workers (fixed by persisting one key + to a shared file; production sets `USERMANAGEMENT_JWT_PRIVATE_KEY_PEM/_FILE` + explicitly). That is a one-time deploy concern, not a per-request cost — RS256 + verification is local and stateless (no introspection call), same as HS256. +- **When we *would* switch.** If the four services ever collapse into a context where + asymmetry provably buys nothing (and stays that way) *and* the key-provisioning + overhead outweighs containment, HS256 is defensible — but only with: strict + `algorithms=["HS256"]` (never let the JWT header pick the alg), enforced + `iss`+`aud`, a ≥256-bit secret from a secret manager (not `.env` in Git), separate + secrets per environment, and the internet-facing MCP holding **no** secret (it + only relays, so it never becomes a forger). Until there's a concrete reason, + RS256's "verifier ≠ issuer" property is worth its modest operational cost. +- **If reversing:** keep it a **config** switch, not a rewrite — retain RS256 verify + paths so a future distributed/split deployment can re-enable asymmetric signing + without new code. + +### Still open (deliberately deferred) +- **Retire the legacy HS256 `/api/login`(`/token`) paths + fold in + `APItokenmanager`.** Now unblocked for the web UI (migrated to session-exchange, + §9.10) and the MCP (SSO). **Do after** confirming, on a real deploy, that the UI + works via session-exchange — then: (1) restrict/remove password login (keep a + SuperAdmin break-glass if wanted), (2) remove the `api_tokenmanager` Django + program from `Dockerfile.unified` + change the container healthcheck off `:8000` + + drop its start.sh migration steps (`Web_jwtuser`/`Web_scope` tables are created + by usermanagement `create_all`). +- `chat_service` RS256 verification (same `core/jwks.py` pattern) — not in use. diff --git a/query_service/PROVENANCE_MODEL.md b/query_service/PROVENANCE_MODEL.md new file mode 100644 index 0000000..7dd3d60 --- /dev/null +++ b/query_service/PROVENANCE_MODEL.md @@ -0,0 +1,198 @@ +# BrainKB Provenance Model (PROV-O in Oxigraph) + +Status: implemented on branch `improve-ingestion-query-service`. + +BrainKB stores knowledge in Oxigraph (a triplestore), so provenance is tracked +**natively as W3C PROV-O triples in the graph database** and queried with SPARQL — +not in a relational side-table. Postgres continues to hold job *execution state* +(`jobs`, `job_results`, `job_processing_log`); Oxigraph is the provenance +*source of truth*. + +This replaces the previous approach, which embedded PROV triples directly into +each uploaded file's domain data via `attach_provenance()`. That polluted domain +graphs, generated per-file provenance with random UUIDs unlinked to the job, and +forced a full parse+re-serialize of every file (a memory/latency bottleneck). + +## Where provenance lives + +All provenance is written to one dedicated named graph: + +``` +https://brainkb.org/provenance/ +``` + +Domain data uploaded during ingestion is **no longer modified** — files land in +their target named graph exactly as provided. + +## Vocabulary + +| Prefix | IRI | +|-----------|------------------------------------------------| +| `prov` | `http://www.w3.org/ns/prov#` | +| `dcterms` | `http://purl.org/dc/terms/` | +| `xsd` | `http://www.w3.org/2001/XMLSchema#` | +| `brainkb` | `https://brainkb.org/vocab/` (custom terms) | + +Instance IRIs are minted under `https://brainkb.org/prov/`: + +- Activity: `…/prov/activity/{job_id}` +- Ingested bundle (entity): `…/prov/bundle/{job_id}` +- Per-file entity: `…/prov/file/{job_id}/{urlencoded filename}` +- Agent: `…/prov/agent/{urlencoded id}`; system agent: `…/prov/agent/system` + +## Tracked activities + +Data-mutation actions become a `prov:Activity` in the provenance graph. +Named-graph registration is **not** duplicated here — it is recorded on the +registry graph (see §2). + +### 1. Ingestion — `brainkb:IngestionActivity` (agent: user) + +Written when a job reaches a terminal state (`done` / `error` / partial). + +```turtle +GRAPH { + <…/prov/activity/{job_id}> + a prov:Activity, brainkb:IngestionActivity ; + prov:startedAtTime "…"^^xsd:dateTime ; + prov:endedAtTime "…"^^xsd:dateTime ; + prov:wasAssociatedWith <…/prov/agent/{user}> ; + brainkb:targetGraph <{named_graph_iri}> ; + brainkb:jobStatus "done" ; + brainkb:totalFiles 20 ; + brainkb:successCount 19 ; + brainkb:failCount 1 . + + <…/prov/bundle/{job_id}> + a prov:Entity ; + prov:wasGeneratedBy <…/prov/activity/{job_id}> ; + prov:wasAttributedTo <…/prov/agent/{user}> ; + prov:generatedAtTime "…"^^xsd:dateTime ; + dcterms:isPartOf <{named_graph_iri}> . + + <…/prov/file/{job_id}/data_009.ttl> + a prov:Entity ; + prov:wasGeneratedBy <…/prov/activity/{job_id}> ; + dcterms:isPartOf <…/prov/bundle/{job_id}> ; + brainkb:fileName "data_009.ttl" ; + brainkb:uploadStatus "success" ; + brainkb:httpStatus 204 ; + brainkb:sizeBytes 41943040 . + + <…/prov/agent/{user}> a prov:Agent, prov:Person . +} +``` + +### 2. Named-graph registration (recorded in the registry graph — no duplication) + +Registration is **not** written as a separate activity in the provenance graph, +because the named-graph **registry** graph +(`https://brainkb.org/metadata/named-graph`) already records each registration as +a PROV entity. We simply attribute it to the registering user on that same entry, +so registration facts live in exactly one place: + +```turtle +GRAPH { + <{named_graph_url}> + a prov:Entity ; + prov:generatedAtTime "…"^^xsd:dateTime ; + dcterms:description "…" ; + prov:wasAttributedTo <…/prov/agent/{user}> . +} +``` + +`GET /api/query/registered-named-graphs` returns `registered_by` alongside the +description and timestamp. + +### 3. Crash recovery — `brainkb:RecoveryActivity` (agent: system) + +Written when `recover_stuck_jobs()` marks a stuck job as `error`. + +```turtle +<…/prov/activity/rec-{job_id}-{ts}> + a prov:Activity, brainkb:RecoveryActivity ; + prov:startedAtTime "…"^^xsd:dateTime ; + prov:wasAssociatedWith <…/prov/agent/system> ; + prov:used <…/prov/activity/{job_id}> ; + dcterms:description "{cause}" . +<…/prov/agent/system> a prov:Agent, prov:SoftwareAgent . +``` + +## Writing + +Provenance triples are serialized to Turtle and appended to the provenance graph +via Oxigraph's Graph Store HTTP protocol (`POST {endpoint}/store?graph=…`, which +*merges* rather than replaces). Writes are best-effort: a provenance failure is +logged but never fails the underlying job/registration. + +## Incremental change tracking (triple-level deltas) + +Provenance above records *who/what/when* at the activity level. To also track +*which triples changed*, each ingestion job stages its data in a **per-job delta +graph** before it reaches the target: + +``` +https://brainkb.org/provenance/delta/{job_id} +``` + +Flow (when `TRACK_TRIPLE_DELTAS=true`, the default): + +1. Each uploaded file is written to the job's delta graph (not the target). +2. When the job finalizes, the delta graph is merged into the target graph + server-side via SPARQL `ADD SILENT TO ` (a set union, so + re-ingesting identical triples is idempotent — no duplicates). +3. The delta graph is **preserved** as the immutable record of exactly what that + job added, and a PROV-O `brainkb:IngestionDelta` entity is written: + +```turtle +<…/prov/delta/{job_id}> + a prov:Entity, brainkb:IngestionDelta ; + prov:wasGeneratedBy <…/prov/activity/{job_id}> ; + prov:wasDerivedFrom <{target_graph}> ; + brainkb:changeType "addition" ; + brainkb:targetGraph <{target_graph}> ; + brainkb:deltaGraph ; + brainkb:addedTripleCount 1234 ; + prov:generatedAtTime "…"^^xsd:dateTime ; + dcterms:isPartOf <…/prov/bundle/{job_id}> . +``` + +Set `TRACK_TRIPLE_DELTAS=false` to upload directly to the target graph (no delta +graphs). Trade-off: delta graphs persist, so this roughly doubles stored triples +for ingested data — the cost of full triple-level history and diffing. + +Current scope is **additions** (ingestion is append-only). Removals/updates would +add a `removed` delta graph per activity; that is a future extension. + +## Retrieval (JSON-LD) + +Read endpoints return a PROV-O bundle as `application/ld+json` via SPARQL +`CONSTRUCT` against the provenance graph: + +- `GET /api/provenance/job?job_id=…&user_id=…` — provenance for one job + (access-controlled with `verify_user_access`). +- `GET /api/provenance/named-graph?iri=…` — all ingestion activity that targeted + a given named graph. (Registration attribution is on the registry graph; see + `GET /api/query/registered-named-graphs`.) + +Delta / change endpoints: + +- `GET /api/provenance/delta?job_id=…&user_id=…` — the exact triples a job added + (its delta graph) as JSON-LD (access-controlled). +- `GET /api/provenance/delta/history?iri=…` — the ordered change history of a + named graph: one entry per delta (job, added triple count, timestamp, status), + newest first. +- `GET /api/provenance/delta/compare?job_id_a=…&job_id_b=…&user_id=…` — compare + the triples added by two jobs; returns counts (A-only / B-only / shared) and the + differing triples as JSON-LD (access-controlled). + +Because everything is in Oxigraph, arbitrary provenance questions can also be +asked directly over SPARQL, e.g. "all graphs ingested by agent X since T". + +## Backward compatibility + +- The `skip_provenance` query parameter on the ingestion endpoints is retained + but no longer controls domain-data embedding (which is removed). Graph-level + PROV-O tracking always happens. +- `attach_provenance()` / `process_file_with_provenance()` remain in the codebase + but are no longer on the ingestion hot path. diff --git a/query_service/RBAC_MODEL.md b/query_service/RBAC_MODEL.md new file mode 100644 index 0000000..d9b1be7 --- /dev/null +++ b/query_service/RBAC_MODEL.md @@ -0,0 +1,120 @@ +# BrainKB Authorization (RBAC) + +Status: implemented on branch `improve-ingestion-query-service`. + +**JWT = authentication / API access. Roles = authorization (what you may do).** +The query_service authenticates via JWT (scopes gate *API access* only) and then +authorizes actions from the user's **roles**, mapped to **capabilities**, plus +per-resource **space membership**. + +## Where roles come from + +Roles live in the Django-owned RBAC tables and join to a JWT user **by email**: + +``` +Web_jwtuser.email == Web_user_profile.email +Web_user_profile.id -> Web_user_role (is_active, not expired) -> role name +``` + +query_service only **reads** roles (see `core/rbac.py`). It never assigns roles — +creating/removing Admins is **role assignment**, owned by the usermanagement/Django +side. This keeps the two systems consistent and means the KG API cannot be used to +escalate privileges. + +## Role hierarchy + +``` +SuperAdmin >= Admin > write roles > read roles > (no role) +``` + +- **SuperAdmin** — ultimate authority, **bootstrapped at deployment**. Can do + everything Admin can; the SuperAdmin-vs-Admin difference (managing admins) is + role assignment, handled outside query_service. +- **Admin** — all KG capabilities, incl. granting delegatable capabilities. +- **write roles** — Curator, Lab Member, Submitter, Annotator, Mapper, + Knowledge Contributor: create their own private spaces + ingest. +- **read roles** — Reviewer, Validator, Moderator, etc.: read member content. +- **no role** — a JWT user not linked to a profile/role gets **public content + only** (read), nothing else. + +## Capabilities + +| Capability | Granted by | Gates | +|---|---|---| +| `create_private_space` | write roles, admins | create an individual/private space | +| `create_team_space` | admins (or delegated) | create a team space | +| `manage_team_space` | admins (or delegated) | manage a team space's members/visibility/graphs | +| `ingest` | write roles, admins | ingest into a graph (also needs space owner/editor) | +| `recover` | write roles, admins | recover stuck/errored jobs | +| `read_private` | any role | read non-public content you're a member of | +| `sparql_admin` | admins only | run arbitrary SPARQL | +| `grant` | admins only | grant/revoke delegatable capabilities | + +**Delegated upgrades:** Admin/SuperAdmin can grant a specific user extra +capabilities via `POST /api/admin/capabilities/grant` — e.g. let a Curator or Lab +Member create/manage **team** spaces. Only **delegatable** caps may be granted +(everything except `grant` and `sparql_admin`), so the grant endpoint can't turn a +non-admin into an admin. + +## Enforcement points (layered) + +For each action: **JWT scope** (API access) → **capability** (role) → **space +membership** (resource). + +| Action | Capability required | + resource check | +|---|---|---| +| Create individual/private space | `create_private_space` | — | +| Create team space | `create_team_space` | — | +| Manage space (members/visibility/graphs) | owner, or `manage_team_space`/admin | space ownership | +| Ingest (`/insert/*`) | `ingest` | space owner/editor of the target graph | +| Recover jobs | `recover` | own jobs | +| Arbitrary SPARQL (`/query/sparql/`) | `sparql_admin` | — | +| Read space / data | `read_private` for private; public = anyone (anon) | membership for private | + +## Space types + +- **individual** — a personal/private workspace; any write-capable user can create + one (`create_private_space`). +- **team** — a shared workspace; only Admin/SuperAdmin (or a user granted + `create_team_space`) can create/manage it. + +## Admin capability endpoints + +- `GET /api/admin/capabilities?member=` — a user's roles, effective + capabilities, and grants (admin only). +- `POST /api/admin/capabilities/grant` `{member, capability}` — delegate a cap. +- `POST /api/admin/capabilities/revoke` `{member, capability}`. + +## Fine-grained per-space access rules + +On top of capabilities + owner/editor/viewer membership, a space can carry +**access rules** that restrict a specific action to specific subjects — e.g. "in +this space, only Admins may write", "only these Lab Members may read", "let this +member manage". + +- Rule = `(action, subject_type, subject_value)`: + - `action` ∈ `read` | `write` | `manage` + - `subject_type` ∈ `global_role` (e.g. `Admin`, `Lab Member`) | `member` (an + email) | `space_role` (`viewer`|`editor`|`owner`, matched as ">=") +- Semantics per action: + - **Owner** of the space and **global Admin/SuperAdmin** always pass (no lockout). + - `read`/`write`: if **no** rules exist for the action, the normal + capability/membership/visibility gates apply; if rules **do** exist, the caller + must also **match at least one**. + - `manage`: owner/admin/`manage_team_space` by default; a `manage` rule can + additionally **grant** management to a matched subject. +- Rules layer *on top of* the global gates — e.g. ingest still needs the `ingest` + capability and space owner/editor membership; a write rule then narrows *which* + of those writers may actually ingest here. + +Endpoints (space manager only, except GET which is member/manager): + +- `GET /api/spaces/{slug}/access-rules` +- `POST /api/spaces/{slug}/access-rules` `{action, subject_type, subject_value}` +- `DELETE /api/spaces/{slug}/access-rules/{rule_id}` + +## Notes + +- JWT scopes (`read`/`write`/`admin`) remain as an API-access layer for defense in + depth; the authoritative "who can do what" is the role/capability + per-space + rule layer above. diff --git a/query_service/README.md b/query_service/README.md index 6a1dc22..6fe6260 100644 --- a/query_service/README.md +++ b/query_service/README.md @@ -1,14 +1,235 @@ # Query Service -This service provides the endpoints and the functionalities necessary for querying (and updating) the knowledge graphs from the graph database. -## Features Implemented -- [x] Logging -- [ ] Endpoints to query the data from the graph database +FastAPI service for querying **and** ingesting BrainKB knowledge graphs in the +graph database (Oxigraph), with W3C PROV-O provenance and team-owned +private/public spaces. +## Features +- [x] Structured logging (with correlation IDs) +- [x] JWT auth with per-endpoint scopes (`read` / `write` / `admin`) +- [x] SPARQL query endpoints (registered graphs, taxonomy, arbitrary query — admin only) +- [x] Bulk RDF ingestion into named graphs (file + raw), run as background jobs +- [x] Job tracking: live status, processing history, progress, crash recovery +- [x] Native **PROV-O provenance** in Oxigraph (ingestion / recovery activities) +- [x] **Triple-level delta tracking** — per-job delta graphs + query/compare endpoints +- [x] **Spaces** — owner-controlled, private/public containers of named graphs +- [x] **Search** — hybrid Postgres-locator + Oxigraph-data, access-filtered by space +## Auth & scopes +Two token schemes are accepted (see `AUTH_UNIFICATION.md`): +- **Single sign-on (RS256)** — access tokens minted by usermanagement (the single + issuer) and verified here against its published **JWKS**; the token's `aud` must + equal `query_service`, so a token minted for another service is rejected + (containment). Configure with `QUERY_SERVICE_SSO_JWKS_URL` / + `QUERY_SERVICE_SSO_ISSUER` / `QUERY_SERVICE_SSO_AUDIENCE`. +- **Legacy HS256** — this service's own password login at **`/api/login`** + (`/api/token` is a deprecated alias), signed with its own secret. Still accepted + during migration; both schemes work side by side. + +Scope policy (same for either scheme): + +- **GET (reads)** → `read` +- **Mutations** (ingest, register/attach graph, recover, create/modify space) → `write` +- **Arbitrary SPARQL** (`/query/sparql/`) → `admin` +- **Public-space reads** → no token required (anonymous), see Spaces below +- `/login` (and deprecated alias `/token`) → public + +**Onboarding is via Globus/ORCID/GitHub sign-in — there is no self-registration.** +`POST /register` is **disabled** (returns 405). A user's profile + default role are +auto-created/linked on first OAuth login (usermanagement). Authorization is +role-based (see `RBAC_MODEL.md`) and read from the DB, not just the token. + +Users may only act on their own `user_id` (enforced), and job-scoped endpoints are +owner-only. + +## Capabilities & roles (RBAC) + +Two independent layers apply to every mutating call: + +1. **JWT scope** (`read`/`write`/`admin`) — API-access gate at the endpoint. +2. **Capability** — *who is allowed to do what*, derived from the user's **role(s)** + (read from the DB, not just the token) plus any admin-delegated grants. + +### Capabilities — what each one means + +| Capability | Meaning | +|---|---| +| `create_private_space` | Create your own individual/private space | +| `create_team_space` | Create a **team** (shared) space | +| `manage_team_space` | Manage a team space's members/visibility/graphs/rules — **only for team spaces you own or are assigned to** (a member of), *not* all team spaces (Admin/SuperAdmin manage all) | +| `ingest` | Ingest data into a graph — **also** needs per-space write (owner/editor membership **or** a space write access rule; see below) | +| `recover` | Recover stuck/errored ingest jobs | +| `read_private` | Read non-public content you're a member of | +| `sparql_admin` | Run arbitrary SPARQL (`/query/sparql/`) | +| `grant` | Grant/revoke capabilities to other users | + +### Which roles get which capabilities + +| Role tier | Capabilities | +|---|---| +| **SuperAdmin / Admin** | **all** of the above | +| **Write roles** — Curator, Lab Member, Submitter, Annotator, Mapper, Knowledge Contributor | `create_private_space`, `ingest`, `recover`, `read_private` | +| **Any other active role** (Reviewer, Validator, Moderator, …) | `read_private` | +| **No role** | public reads only — no create/ingest/private read | + +**Delegation (Admin/SuperAdmin only):** the *grantable* capabilities — +`create_private_space`, `create_team_space`, `manage_team_space`, `ingest`, +`recover`, `read_private` — can be granted to either: + +- **an individual** — `POST /admin/capabilities/grant` `{member, capability}` + (revoke: `/admin/capabilities/revoke`); or +- **a whole role/group** — `POST /admin/capabilities/grant-role` + `{role, capability}` (revoke: `/admin/capabilities/revoke-role`; inspect: + `GET /admin/capabilities/role?role=`). This gives every member of a role/group + (including a custom group like `uk_collaborator`) the capability. + +`GET /admin/capabilities/available` lists the full catalog and which are +delegatable. `grant` and `sparql_admin` are **not** delegatable (they come only +from an Admin/SuperAdmin role), so grants can't escalate a non-admin into an admin. +Effective capabilities = role-derived ∪ role/group grants ∪ per-user grants. + +**SuperAdmin vs Admin — who they are and who creates them:** +- **SuperAdmin** is the top authority, **bootstrapped at deployment** from + `USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS` (seeded on that user's first login). + It's a protected marker — can't be banned, deleted, or role-stripped. Only a + **SuperAdmin** can grant the `Admin` (or `SuperAdmin`) role to others. +- **Admin** is created by a **SuperAdmin** assigning the `Admin` role + (`brainkb_assign_role(email, "Admin")` — SuperAdmin-only). Admins have the same + KG capabilities but are themselves manageable (a SuperAdmin can demote/ban them). +- **Scope of "manage all":** only Admin/SuperAdmin manage *every* team space. + A non-admin (even with `manage_team_space`) manages **only** the team spaces they + **created (own)** or were **assigned to** (a member, or matched by a per-space + `manage` rule). Role *assignment* itself is owned by usermanagement, not + query_service. + +### Giving a whole group ingest access to a team space + +Ingesting into a space-mapped graph needs the `ingest` capability **and** write +authorization on the space. Write is granted by any of: global Admin, owner/editor +membership, **or a per-space write access rule**. So to let a whole group ingest +without adding each person as a member, an admin (or space manager) adds a rule: + +``` +action=write, subject_type=global_role, subject_value="" # e.g. "Lab Member" +``` + +Every user in that group can then ingest into the space's graphs (they still need +a write-capable role for the `ingest` capability). Rules can also target a single +`member` (email) or a `space_role`. Remove the rule to revoke. See +`RBAC_MODEL.md` and `SPACES_MODEL.md` for the full model. + +### Creating a custom group and giving it powers + +Creating a group and assigning it always works; a **brand-new custom role starts +with no powers** (only `read_private`) — you then **grant** it capabilities. Full +flow (Admin/SuperAdmin): + +``` +# 1. create the group/category (usermanagement) +POST /api/admin/roles {name: "uk_collaborator", category, description} +# 2. give the group a power (query_service) — global capability +POST /api/admin/capabilities/grant-role {role: "uk_collaborator", capability: "ingest"} +# 3. put a user in the group (usermanagement) +POST /api/admin/users/{profile_id}/roles {role: "uk_collaborator"} +# → every uk_collaborator can now ingest +``` + +**Global vs per-space** — two ways to let a group ingest: +- **Global** (§ grant-role above): the group can ingest into any space where it + has write authorization (membership/owner/admin) — a broad, workspace-wide power. +- **Per-space** (§ access rule above): the group can ingest into **one** named + space only. Narrower; preferred when scoping a group to a specific workspace. + +### Identity & admin actions catalog (usermanagement `/api/admin`) + +Beyond the KG capabilities above, these are the management actions and who may do +them (Admin unless noted): + +| Action | Endpoint / tool | +|---|---| +| Create / list custom groups (roles) | `POST/GET /api/admin/roles` · `brainkb_create_role`, `brainkb_available_roles` | +| Assign / remove a role on a user | `POST/DELETE /api/admin/users/{id}/roles` · `brainkb_assign_role` / `brainkb_remove_role` *(Admin role = **SuperAdmin-only**)* | +| Grant / revoke a capability to a **user** | `brainkb_grant_capability` / `brainkb_revoke_capability` | +| Grant / revoke a capability to a **group** | `brainkb_grant_role_capability` / `brainkb_revoke_role_capability` | +| List capability catalog / a group's caps | `brainkb_list_capabilities` · `brainkb_role_capabilities` | +| Create / list permissions (resource·action) | `POST/GET /api/admin/permissions` · `brainkb_create_permission`, `brainkb_list_permissions` | +| Per-space access rules (read/write/manage) | `brainkb_add_access_rule` / `brainkb_remove_access_rule` | +| Activate / deactivate login | `brainkb_activate_user` / `brainkb_deactivate_user` | +| Ban / unban (removal — **no delete**) | `brainkb_ban_user` / `brainkb_unban_user` *(banning an Admin = **SuperAdmin-only**)* | + +Onboarding is via Globus/ORCID/GitHub sign-in (auto-creates the profile + default +role); there is **no self-registration** (`/api/register` → 405). + +## Endpoints (prefix `/api`) + +### Query +- `GET /query/registered-named-graphs` — registry/catalog of graphs (visibility-filtered) +- `GET /query/taxonomy` — taxonomy view +- `GET /query/sparql/` — arbitrary SPARQL (**admin**) + +### Ingestion & jobs +- `POST /insert/raw/knowledge-graph-triples` — ingest raw triples (background job) +- `POST /insert/files/knowledge-graph-triples` — ingest uploaded RDF files +- `POST /register-named-graph` — register a named graph +- `GET /insert/jobs`, `GET /insert/user/jobs/detail` — job listing / detail +- `GET /insert/jobs/check-recoverable`, `POST /insert/jobs/recover` — crash recovery + +Ingestion is **submit-and-forget**: the request saves data to disk, creates a +`pending` job, and returns immediately with a `job_id`; processing runs in the +background (concurrent file uploads + batched DB writes) and the client polls job +status. A per-worker **resource-safety cap** (`MAX_CONCURRENT_INGEST_JOBS`, default +3) bounds how many jobs process at once so a burst of submissions can't exhaust +memory / the DB pool / Oxigraph and crash the process — excess jobs simply wait as +`pending` until a slot frees (backpressure, no queue rework). Search indexing is +then queued separately in the background. + +### Provenance (PROV-O, JSON-LD) +- `GET /provenance/job` — full bundle for one job +- `GET /provenance/named-graph` — ingestion/activity history of a graph +- `GET /provenance/delta` — exact triples a job added +- `GET /provenance/delta/history` — a graph's change history +- `GET /provenance/delta/compare` — diff two jobs' deltas + +See [PROVENANCE_MODEL.md](PROVENANCE_MODEL.md). + +### Spaces (private/public) +- `POST /spaces`, `GET /spaces`, `GET /spaces/{slug}` +- `PATCH /spaces/{slug}/visibility` — flip private/public (owner) +- `POST/DELETE /spaces/{slug}/members[/{member}]` — membership (owner) +- `POST /spaces/{slug}/graphs` — register + bind a graph to a space (owner/editor) +- `GET /spaces/{slug}/data` — read space RDF (public = anonymous) + +**public** = readable by anyone, including unauthenticated clients; **private** = +members only; **write/ingest** = space owner/editor only. See +[SPACES_MODEL.md](SPACES_MODEL.md). + +### Search +- `GET /search?q=…[&space={slug}][&limit&offset]` — full-text search, access-filtered. +- `POST /search/reindex` (**admin**) — queue a background backfill/rebuild of the index. +- `GET /search/index-tasks[?task_id=…]` — background indexing task status. + +Hybrid design: **Postgres** holds a full-text **locator index** (`graph_search_index`: +subject + text + named graph + owning space), populated at ingest. A search runs in +Postgres (fast, filtered by space visibility/membership), then the matched subjects' +triples are fetched from **Oxigraph** (the source of truth). Anonymous → public +spaces only; authenticated → public + own/member spaces (+ legacy). Pass `space` to +scope to one workspace, omit for a full search. Private data is never returned to +non-members — the filter is enforced in the locator query. + +**Indexing is asynchronous.** It never blocks ingestion: ingest enqueues an indexing +task on an in-process async queue (durable `index_tasks` table, background consumer, +atomic cross-worker claim, restart recovery) and the job completes immediately. The +`/search/reindex` backfill uses the same queue. Poll `/search/index-tasks` for status. + +## Architecture notes + +- **Postgres** holds identity/teams/enforcement (JWT users, jobs, spaces/members/graphs). +- **Oxigraph** holds all knowledge-graph data, PROV-O provenance, per-job delta + graphs, and a mirror of each space manifest — the graph database is the source of + truth for graph data and provenance. ### Acknowledgements Special thanks to the authors of the resources below who helped with some best practices. @@ -17,4 +238,4 @@ Special thanks to the authors of the resources below who helped with some best p - FastAPI official documentation ### License -[MIT](https://github.com/git/git-scm.com/blob/main/MIT-LICENSE.txt) \ No newline at end of file +[MIT](https://github.com/git/git-scm.com/blob/main/MIT-LICENSE.txt) diff --git a/query_service/SPACES_MODEL.md b/query_service/SPACES_MODEL.md new file mode 100644 index 0000000..b57434a --- /dev/null +++ b/query_service/SPACES_MODEL.md @@ -0,0 +1,95 @@ +# BrainKB Spaces (private/public, team-owned) + +Status: implemented on branch `improve-ingestion-query-service`. + +Spaces let a user or team create their own **owner-controlled container** of named +graphs, keep it **private**, or publish it **publicly** for anyone — including +unauthenticated clients — to read. This supports a decentralized model where each +space is a sovereign, IRI-addressable pod (`https://brainkb.org/space/{slug}`). + +## Storage split (confirmed architecture) + +- **Postgres** — identity/teams/enforcement only: JWT users, and the space tables + (`spaces`, `space_members`, `space_graphs`). This is the source of truth for + authorization and is what every request checks (fast, joinable with the JWT user). +- **Oxigraph (graph DB)** — all knowledge-graph data AND provenance AND a + best-effort **RDF mirror** of each space manifest (in the spaces metadata graph + `https://brainkb.org/metadata/spaces/`), so spaces are portable/queryable via SPARQL. + +The RDF mirror is best-effort: a mirror failure never fails the enforcing Postgres +write. + +## Model + +- `spaces(space_id, slug, name, description, owner, visibility, …)` — + `visibility ∈ {private, public}`. +- `space_members(space_id, member, role)` — `role ∈ {owner, editor, viewer}`; + `member` is the user email (= the PROV agent id). +- `space_graphs(space_id, named_graph_iri)` — a named graph belongs to exactly one + space. + +Roles: **owner** manages members/visibility/graphs; **editor** may ingest; +**viewer** may read a private space; **public** grants read to everyone. + +## Authorization (enforced on every request) + +`spaces.authorize(named_graph_iri, member, need)`: + +| Space state | read | write (ingest) | +|-------------|------|----------------| +| public | **anyone (even anonymous)** | owner/editor only | +| private | owner/editor/viewer | owner/editor | +| *unmapped (legacy graph)* | falls through to endpoint scope | falls through to endpoint scope | + +Backward-compat: graphs never attached to a space (e.g. pre-existing graphs) are +"unmapped" and keep their previous scope-only behavior; only space-mapped graphs +are governed by space ACL. + +**Public = anonymous**: public reads require no token. Read endpoints that serve +space data use an optional-auth dependency (`get_current_user_optional`) — a token +is used if present, but its absence is not an error for public spaces. + +## Endpoints (`/api`) + +| Method | Path | Auth | Purpose | +|--------|------|------|---------| +| POST | `/spaces` | write | Create a space (caller = owner) | +| GET | `/spaces` | optional | List visible spaces (public + own; anon → public only) | +| GET | `/spaces/{slug}` | optional | Space manifest (public to anyone, private to members) | +| PATCH | `/spaces/{slug}/visibility` | owner | Flip private/public | +| POST | `/spaces/{slug}/members` | owner | Add/update a member (role) | +| DELETE | `/spaces/{slug}/members/{member}` | owner | Remove a member | +| POST | `/spaces/{slug}/graphs` | owner/editor | Register + bind a named graph to the space | +| GET | `/spaces/{slug}/data` | optional | Read the space's RDF (JSON-LD); public = anonymous | + +Ingestion (`/insert/{raw,files}/knowledge-graph-triples`) now also checks space +write-authorization for the target graph (in addition to the `write` scope and the +user-identity check). + +## RDF manifest (mirror) + +```turtle +GRAPH { + + a brainkb:Space ; + brainkb:slug "{slug}" ; schema:name "…" ; dcterms:description "…" ; + brainkb:visibility "public" ; + brainkb:owner ; + brainkb:editor <…/agent/{editor}> ; + brainkb:viewer <…/agent/{viewer}> ; + brainkb:containsGraph <{named_graph_iri}> . +} +``` + +## Notes + +- `registered-named-graphs` is **visibility-filtered**: graphs in a private space + the caller isn't a member of are omitted, so private graph existence is not + leaked. Public-space and legacy (unmapped) graphs remain listed. The endpoint + requires authentication (read scope); anonymous discovery of public spaces is via + `GET /api/spaces`. +- Job-scoped provenance endpoints are **intentionally owner-restricted** (by + `user_id` = authenticated identity) and stay that way — job provenance is not made + public even for public spaces. Ingestion is likewise restricted to activated JWT + users with valid credentials (and space owner/editor membership); it is never + anonymous. Only *reads* of public spaces are anonymous. diff --git a/query_service/VERIFICATION_QUERIES.md b/query_service/VERIFICATION_QUERIES.md new file mode 100644 index 0000000..82edc7b --- /dev/null +++ b/query_service/VERIFICATION_QUERIES.md @@ -0,0 +1,208 @@ +# BrainKB — Verification Queries + +SPARQL (Oxigraph) and SQL (Postgres) queries to verify data written by the +BrainKB skill / MCP: ingested graphs, provenance, deltas, spaces, and the search +index. Replace `my-lab` / `JOB_ID` with your values. + +## Fixed graph IRIs + +| Purpose | Named graph | +|---|---| +| Graph registry (catalog) | `https://brainkb.org/metadata/named-graph` | +| Spaces manifest | `https://brainkb.org/metadata/spaces/` | +| Provenance | `https://brainkb.org/provenance/` | +| Per-job delta | `https://brainkb.org/provenance/delta/{job_id}` | +| Your data | e.g. `https://brainkb.org/graph/my-lab/` | + +## Prefixes + +```sparql +PREFIX prov: +PREFIX dcterms: +PREFIX schema: +PREFIX brainkb: +PREFIX rdfs: +``` + +--- + +## SPARQL (Oxigraph) + +### 1. All named graphs + triple counts (what exists) + +```sparql +SELECT ?g (COUNT(*) AS ?triples) WHERE { GRAPH ?g { ?s ?p ?o } } +GROUP BY ?g ORDER BY DESC(?triples) +``` + +### 2. Your ingested data + +```sparql +SELECT ?s ?p ?o WHERE { GRAPH { ?s ?p ?o } } LIMIT 200 +``` + +### 3. Registry — which graphs are registered + by whom + +```sparql +PREFIX prov: +PREFIX dcterms: +SELECT ?graph ?description ?registered_at ?registered_by WHERE { + GRAPH { + ?graph dcterms:description ?description ; prov:generatedAtTime ?registered_at . + OPTIONAL { ?graph prov:wasAttributedTo ?registered_by } + } +} +``` + +### 4. Spaces manifest — visibility, owner, contained graphs + +```sparql +PREFIX schema: +PREFIX brainkb: +SELECT ?space ?name ?visibility ?owner + (GROUP_CONCAT(DISTINCT STR(?graph); SEPARATOR=", ") AS ?graphs) WHERE { + GRAPH { + ?space a brainkb:Space ; schema:name ?name ; + brainkb:visibility ?visibility ; brainkb:owner ?owner . + OPTIONAL { ?space brainkb:containsGraph ?graph } + } +} GROUP BY ?space ?name ?visibility ?owner +``` + +Members (owner/editor/viewer): + +```sparql +PREFIX brainkb: +SELECT ?space ?role ?agent WHERE { + GRAPH { + VALUES ?role { brainkb:owner brainkb:editor brainkb:viewer } + ?space ?role ?agent . + } +} +``` + +### 5. Provenance — ingestion activities (who/when/status) + +```sparql +PREFIX prov: +PREFIX brainkb: +SELECT ?activity ?agent ?targetGraph ?status ?start ?end ?success ?fail WHERE { + GRAPH { + ?activity a brainkb:IngestionActivity ; + prov:wasAssociatedWith ?agent ; + brainkb:targetGraph ?targetGraph ; + brainkb:jobStatus ?status ; + prov:startedAtTime ?start . + OPTIONAL { ?activity prov:endedAtTime ?end } + OPTIONAL { ?activity brainkb:successCount ?success } + OPTIONAL { ?activity brainkb:failCount ?fail } + } +} ORDER BY DESC(?start) +``` + +### 6. Change history + deltas for a graph + +```sparql +PREFIX prov: +PREFIX brainkb: +SELECT ?delta ?deltaGraph ?added ?time WHERE { + GRAPH { + ?delta a brainkb:IngestionDelta ; + brainkb:targetGraph ; + brainkb:deltaGraph ?deltaGraph ; + brainkb:addedTripleCount ?added ; + prov:generatedAtTime ?time . + } +} ORDER BY DESC(?time) +``` + +The exact triples one job added: + +```sparql +SELECT ?s ?p ?o WHERE { GRAPH { ?s ?p ?o } } +``` + +### 7. Per-file results for a job + +```sparql +PREFIX prov: +PREFIX brainkb: +SELECT ?name ?status ?http ?size WHERE { + GRAPH { + ?file prov:wasGeneratedBy ; + brainkb:fileName ?name ; brainkb:uploadStatus ?status . + OPTIONAL { ?file brainkb:httpStatus ?http } + OPTIONAL { ?file brainkb:sizeBytes ?size } + } +} +``` + +### 8. Find a term across everything (what search matched) + +```sparql +PREFIX rdfs: +SELECT ?g ?s ?label WHERE { + GRAPH ?g { ?s rdfs:label ?label . FILTER(CONTAINS(LCASE(STR(?label)), "purkinje")) } +} +``` + +--- + +## How to run the SPARQL + +### Via the API (needs `admin` scope) + +```bash +Q='SELECT ?g (COUNT(*) AS ?n) WHERE { GRAPH ?g {?s ?p ?o} } GROUP BY ?g' +curl -s "http://localhost:8010/api/query/sparql/?sparql_query=$(python3 -c "import urllib.parse,sys;print(urllib.parse.quote(sys.argv[1]))" "$Q")" \ + -H "Authorization: Bearer $TOKEN" +``` + +Get `$TOKEN`: + +```bash +TOKEN=$(curl -s -X POST http://localhost:8010/api/token -H 'Content-Type: application/json' \ + -d '{"email":"you@example.com","password":"***"}' \ + | python3 -c 'import sys,json;print(json.load(sys.stdin)["access_token"])') +``` + +### Directly against Oxigraph (nginx proxy on :7878, HTTP Basic auth) + +```bash +curl -s "http://localhost:7878/query" \ + --data-urlencode 'query=SELECT ?g (COUNT(*) AS ?n) WHERE { GRAPH ?g {?s ?p ?o} } GROUP BY ?g' \ + -u admin:"$OXIGRAPH_PASSWORD" -H 'Accept: application/sparql-results+json' +``` + +(`OXIGRAPH_USER` / `OXIGRAPH_PASSWORD` are in `.env`.) + +--- + +## SQL (Postgres — not in Oxigraph) + +Jobs, space ACL, the search locator index, and indexing tasks live in Postgres. + +```sql +-- spaces & membership & graph bindings +SELECT slug, name, visibility, owner FROM spaces; +SELECT space_id, member, role FROM space_members; +SELECT space_id, named_graph_iri FROM space_graphs; + +-- ingest jobs +SELECT job_id, status, total_files, success_count, fail_count, start_time, end_time +FROM jobs ORDER BY start_time DESC LIMIT 10; + +-- search locator index (subject text per graph/space) +SELECT named_graph_iri, subject, left(text, 60) AS text FROM graph_search_index; + +-- background indexing tasks +SELECT task_id, kind, target, status, subjects_indexed, graphs_done, graphs_total +FROM index_tasks ORDER BY created_at DESC LIMIT 10; +``` + +Run against the Docker Postgres: + +```bash +docker exec -e PGPASSWORD="$JWT_POSTGRES_DATABASE_PASSWORD" brainkb-postgres \ + psql -U postgres -d brainkb -c "SELECT slug, visibility, owner FROM spaces;" +``` diff --git a/query_service/core/configuration.py b/query_service/core/configuration.py index 7683d55..aae7101 100644 --- a/query_service/core/configuration.py +++ b/query_service/core/configuration.py @@ -78,6 +78,13 @@ def load_environment(env_name="production"): "JWT_POSTGRES_DATABASE_NAME": os.getenv("JWT_POSTGRES_DATABASE_NAME"), "JWT_ALGORITHM": os.getenv("JWT_ALGORITHM", "HS256"), "JWT_SECRET_KEY": os.getenv("QUERY_SERVICE_JWT_SECRET_KEY"), + # Phase 2 SSO: verify RS256 access tokens minted by usermanagement. + # Signature is checked against the issuer's published JWKS; the token's + # `aud` must equal SSO_AUDIENCE (this service) — a token minted for + # another service is rejected. Legacy HS256 tokens still validate too. + "SSO_JWKS_URL": os.getenv("QUERY_SERVICE_SSO_JWKS_URL", "http://127.0.0.1:8004/.well-known/jwks.json"), + "SSO_ISSUER": os.getenv("QUERY_SERVICE_SSO_ISSUER", "brainkb-usermanagement"), + "SSO_AUDIENCE": os.getenv("QUERY_SERVICE_SSO_AUDIENCE", "query_service"), # service specific "GRAPHDATABASE_USERNAME": os.getenv("GRAPHDATABASE_USERNAME"), "GRAPHDATABASE_PASSWORD": os.getenv("GRAPHDATABASE_PASSWORD"), diff --git a/query_service/core/database.py b/query_service/core/database.py index 1621156..a9e249a 100644 --- a/query_service/core/database.py +++ b/query_service/core/database.py @@ -39,6 +39,13 @@ table_name_scope = load_environment()["JWT_POSTGRES_TABLE_SCOPE"] table_relation = load_environment()["JWT_POSTGRES_TABLE_USER_SCOPE_REL"] +# Identity unification (Phase 1): a password-registered user is a first-class +# identity, not a role-less credential orphan. On registration we ensure a +# canonical Web_user_profile, link the credential to it (Web_jwtuser.profile_id), +# and assign this default role so rbac (which reads roles via the profile) has +# something to work with. Kept in sync with usermanagement's OAuth default. +DEFAULT_REGISTRATION_ROLE = "Curator" + # Global connection pool pool = None @@ -211,6 +218,45 @@ async def debug_pool_status(): # Refactored Database Functions - Using Context Manager Pattern # ============================================================================ +async def _provision_profile_for_registration(connection, jwt_user_id: int, email: str, fullname: str) -> None: + """Ensure a canonical Web_user_profile + default role for a freshly + registered credential, and link them via Web_jwtuser.profile_id. + + Mirrors usermanagement's provision_identity for the password path so both + onboarding routes converge on the same shape (profile + linked credential + + at least one role). Best-effort: any failure is logged and swallowed so + registration itself never fails on it. rbac reads roles via the profile, so + this is what makes a password user more than public/no-role.""" + try: + async with connection.transaction(): + profile_id = await connection.fetchval( + 'SELECT id FROM "Web_user_profile" WHERE lower(email) = lower($1)', email + ) + if profile_id is None: + profile_id = await connection.fetchval( + '''INSERT INTO "Web_user_profile" (name, email, created_at, updated_at) + VALUES ($1, $2, $3, $4) RETURNING id''', + fullname or email.split("@")[0], email, datetime.utcnow(), datetime.utcnow(), + ) + await connection.execute( + f'UPDATE "{table_name_user}" SET profile_id = $1, updated_at = $2 WHERE id = $3', + profile_id, datetime.utcnow(), jwt_user_id, + ) + has_role = await connection.fetchval( + 'SELECT 1 FROM "Web_user_role" WHERE profile_id = $1 AND is_active IS TRUE LIMIT 1', + profile_id, + ) + if not has_role: + await connection.execute( + '''INSERT INTO "Web_user_role" (profile_id, role, is_active, assigned_at, updated_at) + VALUES ($1, $2, TRUE, $3, $4) + ON CONFLICT (profile_id, role) DO NOTHING''', + profile_id, DEFAULT_REGISTRATION_ROLE, datetime.utcnow(), datetime.utcnow(), + ) + except Exception as e: + logger.warning(f"Profile/role provisioning skipped for {email}: {e}") + + async def insert_data(fullname: str, email: str, password: str, conn: Optional[asyncpg.Connection] = None): """ Insert a new user with default 'read' scope. @@ -218,64 +264,43 @@ async def insert_data(fullname: str, email: str, password: str, conn: Optional[a Otherwise, manages its own connection. """ async def _insert_logic(connection): - # Use a transaction to ensure all operations succeed or fail together + # Credential + scope go in one transaction (all-or-nothing). async with connection.transaction(): - scope_exist_id = await select_scope_id(connection) - - if not scope_exist_id: - # First insert the default read access - scope_query = f""" - INSERT INTO \"{table_name_scope}\" (name, description, created_at, updated_at) - VALUES ($1, $2, $3, $4) RETURNING id""" - - new_scope_id = await connection.fetchval( - scope_query, + scope_id = await select_scope_id(connection) + if not scope_id: + # Seed the default 'read' scope the first time anyone registers. + scope_id = await connection.fetchval( + f"""INSERT INTO \"{table_name_scope}\" (name, description, created_at, updated_at) + VALUES ($1, $2, $3, $4) RETURNING id""", "read", "This allows read access", datetime.utcnow(), datetime.utcnow(), ) - user_query = f""" - INSERT INTO \"{table_name_user}\" (full_name, email, password, is_active, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6) RETURNING id - """ - jwt_user_id = await connection.fetchval( - user_query, - fullname, - email, - password, - False, - datetime.utcnow(), - datetime.utcnow(), - ) + jwt_user_id = await connection.fetchval( + f"""INSERT INTO \"{table_name_user}\" (full_name, email, password, is_active, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING id""", + fullname, + email, + password, + False, + datetime.utcnow(), + datetime.utcnow(), + ) - # Connect with relationship - await connection.execute( - f"""INSERT INTO \"{table_relation}\" (jwtuser_id, scope_id) VALUES ($1, $2)""", - jwt_user_id, - new_scope_id, - ) - else: - user_query = f""" - INSERT INTO \"{table_name_user}\" (full_name, email, password, is_active, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6) RETURNING id - """ - jwt_user_id = await connection.fetchval( - user_query, - fullname, - email, - password, - False, - datetime.utcnow(), - datetime.utcnow(), - ) + await connection.execute( + f"""INSERT INTO \"{table_relation}\" (jwtuser_id, scope_id) VALUES ($1, $2)""", + jwt_user_id, + scope_id, + ) - await connection.execute( - f"""INSERT INTO \"{table_relation}\" (jwtuser_id, scope_id) VALUES ($1, $2)""", - jwt_user_id, - scope_exist_id, - ) + # Identity unification: ensure a canonical profile + default role and + # link this credential to it. Best-effort in its own transaction so a + # hiccup here (e.g. the profile_id column not migrated yet) never blocks + # account creation — the email backfill in usermanagement bootstrap can + # still repair the link later. + await _provision_profile_for_registration(connection, jwt_user_id, email, fullname) return { "detail": "Registration completed successfully! Admin will activate your account after verification." @@ -380,14 +405,23 @@ async def get_scopes_by_user(user_id: int, conn: Optional[asyncpg.Connection] = raise HTTPException(status_code=400, detail=str(e)) -async def get_user(email: str, conn: Optional[asyncpg.Connection] = None): +async def get_user(email: str, conn: Optional[asyncpg.Connection] = None, + include_inactive: bool = False): """ Get an active user by email. Returns the user row if found and active, False otherwise. + + `include_inactive=True` drops the is_active filter. Needed for OAuth callers: + usermanagement provisions them a credential SHELL row with is_active=False + (they have no usable password; the row exists only to carry a stable user_id), + so an active-only lookup rejects every Globus/ORCID/GitHub user even though + their token verifies. Do NOT use it on the password path — there is_active is + the deactivation switch that POST /api/admin/users/deactivate flips. """ + active_filter = "" if include_inactive else "AND is_active = True" query = f""" - SELECT * FROM \"{table_name_user}\" - WHERE email = $1 AND is_active = True + SELECT * FROM \"{table_name_user}\" + WHERE email = $1 {active_filter} LIMIT 1 """ diff --git a/query_service/core/indexing.py b/query_service/core/indexing.py new file mode 100644 index 0000000..5928921 --- /dev/null +++ b/query_service/core/indexing.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# DISCLAIMER: This software is provided "as is" without any warranty, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose, and non-infringement. +# ----------------------------------------------------------------------------- + +# @Author : Tek Raj Chhetri +# @Email : tekraj@mit.edu +# @File : indexing.py + +""" +Async search-indexing task queue. + +Search indexing (building the Postgres locator rows from Oxigraph) can be slow for +large graphs, so it must not run inline with ingestion. This module provides a +lightweight in-process asyncio queue with a single background consumer, backed by a +durable `index_tasks` table for observability and cross-restart recovery. + +Ingest enqueues an 'ingest' task (fast) and returns immediately; a 'backfill' task +reindexes every graph. Tasks are claimed atomically in the DB so multiple gunicorn +workers never process the same task twice. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +import uuid +from typing import Any, Dict, List, Optional + +from core.database import get_db_connection +from core.search import index_graph_subjects, _sparql_select + +logger = logging.getLogger(__name__) + +# Graphs that are infrastructure, not user data — never indexed for search. +_SKIP_GRAPHS = { + "https://brainkb.org/metadata/named-graph", + "https://brainkb.org/metadata/spaces/", + "https://brainkb.org/provenance/", +} + +_queue: Optional[asyncio.Queue] = None +_consumer: Optional[asyncio.Task] = None + + +def _get_queue() -> asyncio.Queue: + global _queue + if _queue is None: + _queue = asyncio.Queue() + return _queue + + +async def enqueue_ingest(named_graph_iri: str, delta_graph: Optional[str] = None) -> str: + """Queue an indexing task for a single graph (called by ingest). Non-blocking.""" + task_id = uuid.uuid4().hex + async with get_db_connection() as conn: + await conn.execute( + """ + INSERT INTO index_tasks (task_id, kind, target, delta_graph, status, created_at) + VALUES ($1, 'ingest', $2, $3, 'queued', $4) + """, + task_id, named_graph_iri, delta_graph, time.time(), + ) + _get_queue().put_nowait({"task_id": task_id, "kind": "ingest", + "target": named_graph_iri, "delta_graph": delta_graph}) + return task_id + + +async def enqueue_backfill() -> str: + """Queue a full reindex of every user graph. Non-blocking.""" + task_id = uuid.uuid4().hex + async with get_db_connection() as conn: + await conn.execute( + """ + INSERT INTO index_tasks (task_id, kind, target, status, created_at) + VALUES ($1, 'backfill', 'ALL', 'queued', $2) + """, + task_id, time.time(), + ) + _get_queue().put_nowait({"task_id": task_id, "kind": "backfill", "target": "ALL"}) + return task_id + + +async def _claim(task_id: str) -> bool: + """Atomically move a task queued->running. Returns False if already claimed + (by another worker) or not claimable — prevents double processing.""" + async with get_db_connection() as conn: + row = await conn.fetchrow( + """ + UPDATE index_tasks SET status = 'running', started_at = $2 + WHERE task_id = $1 AND status = 'queued' + RETURNING task_id + """, + task_id, time.time(), + ) + return row is not None + + +async def _finish(task_id: str, status: str, subjects: int = 0, message: str = "", + graphs_total: Optional[int] = None, graphs_done: int = 0) -> None: + async with get_db_connection() as conn: + await conn.execute( + """ + UPDATE index_tasks + SET status = $2, subjects_indexed = $3, message = $4, + graphs_total = COALESCE($5, graphs_total), graphs_done = $6, ended_at = $7 + WHERE task_id = $1 + """, + task_id, status, subjects, message[:500] if message else None, + graphs_total, graphs_done, time.time(), + ) + + +async def _all_user_graphs() -> List[str]: + rows = await _sparql_select("SELECT DISTINCT ?g WHERE { GRAPH ?g { ?s ?p ?o } }") + graphs = [] + for b in rows: + g = b.get("g", {}).get("value") + if g and g not in _SKIP_GRAPHS and "/provenance/delta/" not in g: + graphs.append(g) + return graphs + + +async def _run(task: Dict[str, Any]) -> None: + task_id = task["task_id"] + if not await _claim(task_id): + return # another worker already handling it, or not queued + try: + if task["kind"] == "ingest": + n = await index_graph_subjects(task["target"], task.get("delta_graph")) + await _finish(task_id, "done", subjects=n, message=f"indexed {n} subject(s)") + elif task["kind"] == "backfill": + graphs = await _all_user_graphs() + total, done, subs = len(graphs), 0, 0 + # record the total up front (status stays 'running') + async with get_db_connection() as conn: + await conn.execute("UPDATE index_tasks SET graphs_total=$2 WHERE task_id=$1", task_id, total) + for g in graphs: + subs += await index_graph_subjects(g) + done += 1 + async with get_db_connection() as conn: + await conn.execute( + "UPDATE index_tasks SET graphs_done=$2, subjects_indexed=$3 WHERE task_id=$1", + task_id, done, subs, + ) + await _finish(task_id, "done", subjects=subs, graphs_total=total, graphs_done=done, + message=f"reindexed {done}/{total} graph(s), {subs} subject(s)") + except Exception as e: + logger.error(f"[indexing] task {task_id} failed: {e}", exc_info=True) + await _finish(task_id, "error", message=str(e)) + + +async def _consume() -> None: + q = _get_queue() + while True: + task = await q.get() + try: + await _run(task) + except Exception as e: + logger.error(f"[indexing] consumer error: {e}", exc_info=True) + finally: + q.task_done() + + +async def start_worker() -> None: + """Start the background consumer and recover tasks from a previous run. + Called once on application startup.""" + global _consumer + # Recovery: any task left 'running' by a crashed process is re-queued. + try: + async with get_db_connection() as conn: + await conn.execute("UPDATE index_tasks SET status = 'queued' WHERE status = 'running'") + pending = await conn.fetch( + "SELECT task_id, kind, target, delta_graph FROM index_tasks WHERE status = 'queued'" + ) + except Exception as e: + logger.warning(f"[indexing] recovery query failed: {e}") + pending = [] + + if _consumer is None or _consumer.done(): + _consumer = asyncio.create_task(_consume()) + logger.info("[indexing] background consumer started") + + for p in pending: + _get_queue().put_nowait({ + "task_id": p["task_id"], "kind": p["kind"], + "target": p["target"], "delta_graph": p["delta_graph"], + }) + if pending: + logger.info(f"[indexing] re-queued {len(pending)} pending task(s) from previous session") + + +async def get_task(task_id: str) -> Optional[Dict[str, Any]]: + async with get_db_connection() as conn: + row = await conn.fetchrow("SELECT * FROM index_tasks WHERE task_id = $1", task_id) + return dict(row) if row else None + + +async def list_tasks(limit: int = 50) -> List[Dict[str, Any]]: + async with get_db_connection() as conn: + rows = await conn.fetch( + "SELECT * FROM index_tasks ORDER BY created_at DESC LIMIT $1", limit + ) + return [dict(r) for r in rows] diff --git a/query_service/core/jwks.py b/query_service/core/jwks.py new file mode 100644 index 0000000..ac4ab88 --- /dev/null +++ b/query_service/core/jwks.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +"""JWKS-based verification of Phase 2 SSO access tokens (RS256). + +Verifies tokens minted by usermanagement (the single issuer). The signature is +checked against the issuer's published JWKS (fetched + cached), and the token's +`aud` MUST equal this service's audience — so a token minted for another service +is rejected here (containment enforced by `aud`, not shared secrets). + +Deliberately SYNCHRONOUS: query_service's scope dependency (`require_scopes`) is +a sync FastAPI dependency, so verification must be callable from sync code. The +network fetch only happens on a cache miss / key rotation (default 10-min TTL); +FastAPI runs sync dependencies in a threadpool, so the occasional blocking fetch +does not stall the event loop. +""" +import logging +import threading +import time +from typing import Dict, Optional + +import httpx +from jose import jwt + +from core.configuration import load_environment + +logger = logging.getLogger(__name__) + +_env = load_environment() +JWKS_URL = _env["SSO_JWKS_URL"] +SSO_ISSUER = _env["SSO_ISSUER"] +SSO_AUDIENCE = _env["SSO_AUDIENCE"] + +_CACHE_TTL = 600 # seconds +_keys: Dict[str, dict] = {} +_fetched_at: float = 0.0 +_lock = threading.Lock() + + +def _refresh(force: bool = False) -> None: + global _keys, _fetched_at + with _lock: + if not force and _keys and (time.monotonic() - _fetched_at) < _CACHE_TTL: + return + try: + with httpx.Client(timeout=5.0) as client: + resp = client.get(JWKS_URL) + resp.raise_for_status() + data = resp.json() + _keys = {k["kid"]: k for k in data.get("keys", []) if k.get("kid")} + _fetched_at = time.monotonic() + except Exception as e: + logger.warning(f"JWKS fetch failed from {JWKS_URL}: {e}") + + +def _get_key(kid: Optional[str]) -> Optional[dict]: + if not kid: + return None + if kid in _keys and (time.monotonic() - _fetched_at) < _CACHE_TTL: + return _keys[kid] + _refresh() + if kid not in _keys: + # Unknown kid — the issuer may have rotated keys. Force one refresh. + _refresh(force=True) + return _keys.get(kid) + + +def verify_access_token(token: str) -> Optional[Dict]: + """Verify an RS256 SSO access token. Returns claims on success, or None if + the token is not RS256 / fails verification / issuer is unreachable. Never + raises — callers fall back to legacy HS256 verification.""" + try: + header = jwt.get_unverified_header(token) + except Exception: + return None + if header.get("alg") != "RS256": + return None # not an SSO token; let the caller try HS256 + key = _get_key(header.get("kid")) + if not key: + return None + try: + payload = jwt.decode( + token, + key, + algorithms=["RS256"], + audience=SSO_AUDIENCE, + issuer=SSO_ISSUER, + ) + except Exception as e: + logger.info(f"RS256 token rejected: {e}") + return None + # Only access tokens are usable at a service; refresh tokens are not. + if payload.get("typ") not in (None, "access"): + return None + return payload diff --git a/query_service/core/main.py b/query_service/core/main.py index 1978718..1396ec7 100644 --- a/query_service/core/main.py +++ b/query_service/core/main.py @@ -1,4 +1,5 @@ import logging +import os # logging from asgi_correlation_id import CorrelationIdMiddleware @@ -12,6 +13,8 @@ from core.routers.query import router as query_router from core.routers.rapid_release import router as rapid_release from core.routers.insert import router as insert_router +from core.routers.spaces import router as spaces_router +from core.routers.search import router as search_router from core.configuration import load_environment from core.database import init_db_pool from core.graph_database_connection_manager import initialize_metadata_graph @@ -21,13 +24,21 @@ environment = load_environment()["ENV_STATE"] -origins = [ +# Browser origins allowed to call this service. Kept identical across the four +# BrainKB services, which each hold their own copy and had drifted apart. +# CORS_ALLOWED_ORIGINS (comma-separated) adds to these without a code change. +_DEFAULT_ORIGINS = [ + "https://brainkb.org", + "https://www.brainkb.org", "https://beta.brainkb.org", -"https://sandbox.brainkb.org", - "http://localhost:3000/", + "https://sandbox.brainkb.org", "http://localhost:3000", - "http://127.0.0.1:3000:" + "http://127.0.0.1:3000", ] +origins = sorted({ + *_DEFAULT_ORIGINS, + *(o.strip() for o in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if o.strip()), +}) if environment == "prods": app = FastAPI(docs_url=None, redoc_url=None) @@ -50,6 +61,8 @@ app.include_router(jwt_router, prefix="/api") app.include_router(query_router, prefix="/api") app.include_router(insert_router,prefix="/api") +app.include_router(spaces_router, prefix="/api", tags=["Spaces"]) +app.include_router(search_router, prefix="/api", tags=["Search"]) # rapid-release app.include_router(rapid_release, prefix="/api/rapid-release", tags=["Rapid release"]) @@ -153,6 +166,175 @@ async def startup_event(): except Exception: pass # Indexes may already exist logger.info("Job tracking tables initialized") + + # Spaces: owner-controlled containers of named graphs with + # private/public visibility and team membership (see spaces.py). + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS spaces ( + space_id TEXT PRIMARY KEY, + slug TEXT UNIQUE NOT NULL, + name TEXT NOT NULL, + description TEXT, + owner TEXT NOT NULL, + visibility TEXT NOT NULL DEFAULT 'private', + created_at DOUBLE PRECISION, + updated_at DOUBLE PRECISION + ) + """ + ) + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS space_members ( + id SERIAL PRIMARY KEY, + space_id TEXT NOT NULL REFERENCES spaces(space_id) ON DELETE CASCADE, + member TEXT NOT NULL, + role TEXT NOT NULL, + added_at DOUBLE PRECISION, + UNIQUE (space_id, member) + ) + """ + ) + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS space_graphs ( + id SERIAL PRIMARY KEY, + space_id TEXT NOT NULL REFERENCES spaces(space_id) ON DELETE CASCADE, + named_graph_iri TEXT NOT NULL UNIQUE, + added_at DOUBLE PRECISION + ) + """ + ) + try: + await conn.execute("CREATE INDEX IF NOT EXISTS idx_space_members_space ON space_members(space_id)") + await conn.execute("CREATE INDEX IF NOT EXISTS idx_space_members_member ON space_members(member)") + await conn.execute("CREATE INDEX IF NOT EXISTS idx_space_graphs_space ON space_graphs(space_id)") + except Exception: + pass + # Space type: 'individual' (personal) or 'team' (created by + # Admin/SuperAdmin or a user granted create_team_space). + try: + await conn.execute("ALTER TABLE spaces ADD COLUMN IF NOT EXISTS space_type TEXT NOT NULL DEFAULT 'individual'") + except Exception: + pass + logger.info("Spaces tables initialized") + + # RBAC: capabilities granted directly to a user (delegated + # upgrades by Admin/SuperAdmin), on top of role-derived caps. + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS user_capability_grants ( + id SERIAL PRIMARY KEY, + member TEXT NOT NULL, + capability TEXT NOT NULL, + granted_by TEXT, + created_at DOUBLE PRECISION, + UNIQUE (member, capability) + ) + """ + ) + try: + await conn.execute("CREATE INDEX IF NOT EXISTS idx_capability_grants_member ON user_capability_grants(member)") + except Exception: + pass + # Role/group-level capability grants: attach a delegatable KG + # capability to a whole role/group (e.g. a custom "uk_collaborator" + # group), so every member of that role gains it — without a + # per-user grant. + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS role_capability_grants ( + id SERIAL PRIMARY KEY, + role TEXT NOT NULL, + capability TEXT NOT NULL, + granted_by TEXT, + created_at DOUBLE PRECISION, + UNIQUE (role, capability) + ) + """ + ) + try: + await conn.execute("CREATE INDEX IF NOT EXISTS idx_role_capability_grants_role ON role_capability_grants(role)") + except Exception: + pass + logger.info("RBAC capability-grants tables initialized") + + # Fine-grained per-space access rules: restrict a space action + # (read/write/manage) to a global role, a space role, or specific + # members. Owner + global Admin/SuperAdmin always bypass. + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS space_access_rules ( + id SERIAL PRIMARY KEY, + space_id TEXT NOT NULL REFERENCES spaces(space_id) ON DELETE CASCADE, + action TEXT NOT NULL, + subject_type TEXT NOT NULL, + subject_value TEXT NOT NULL, + created_by TEXT, + created_at DOUBLE PRECISION, + UNIQUE (space_id, action, subject_type, subject_value) + ) + """ + ) + try: + await conn.execute("CREATE INDEX IF NOT EXISTS idx_space_access_rules_space ON space_access_rules(space_id, action)") + except Exception: + pass + logger.info("Space access-rules table initialized") + + # Search locator index (hybrid search): Postgres full-text index that + # locates subjects/subgraphs (carrying graph + workspace/space), then the + # actual triples are fetched from Oxigraph. Access is filtered by space + # visibility/membership at query time. + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS graph_search_index ( + id BIGSERIAL PRIMARY KEY, + named_graph_iri TEXT NOT NULL, + space_id TEXT, + subject TEXT NOT NULL, + text TEXT NOT NULL, + tsv tsvector, + updated_at DOUBLE PRECISION, + UNIQUE (named_graph_iri, subject) + ) + """ + ) + try: + await conn.execute("CREATE INDEX IF NOT EXISTS idx_gsi_tsv ON graph_search_index USING GIN(tsv)") + await conn.execute("CREATE INDEX IF NOT EXISTS idx_gsi_graph ON graph_search_index(named_graph_iri)") + await conn.execute("CREATE INDEX IF NOT EXISTS idx_gsi_space ON graph_search_index(space_id)") + except Exception: + pass + logger.info("Search index table initialized") + + # Async indexing task queue: search indexing runs in the + # background (not inline with ingest) so large graphs don't + # block jobs. Task status is durable here for observability + # and cross-restart recovery. + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS index_tasks ( + task_id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + target TEXT, + delta_graph TEXT, + status TEXT NOT NULL, + subjects_indexed INTEGER DEFAULT 0, + graphs_total INTEGER, + graphs_done INTEGER DEFAULT 0, + message TEXT, + created_at DOUBLE PRECISION, + started_at DOUBLE PRECISION, + ended_at DOUBLE PRECISION + ) + """ + ) + try: + await conn.execute("CREATE INDEX IF NOT EXISTS idx_index_tasks_status ON index_tasks(status)") + except Exception: + pass + logger.info("Index task table initialized") break # Success, exit retry loop finally: await pool.release(conn) @@ -207,6 +389,14 @@ async def startup_event(): except Exception as e: logger.warning(f"Failed to recover stuck jobs: {str(e)}. Continuing anyway...") + # Start the background search-indexing consumer and recover any pending tasks + logger.info("Starting background search-indexing worker...") + try: + from core.indexing import start_worker + await start_worker() + except Exception as e: + logger.warning(f"Failed to start indexing worker: {str(e)}. Continuing anyway...") + logger.info("FastAPI startup completed successfully") diff --git a/query_service/core/provenance.py b/query_service/core/provenance.py new file mode 100644 index 0000000..b8528bd --- /dev/null +++ b/query_service/core/provenance.py @@ -0,0 +1,479 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# DISCLAIMER: This software is provided "as is" without any warranty, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose, and non-infringement. +# +# In no event shall the authors or copyright holders be liable for any +# claim, damages, or other liability, whether in an action of contract, +# tort, or otherwise, arising from, out of, or in connection with the +# software or the use or other dealings in the software. +# ----------------------------------------------------------------------------- + +# @Author : Tek Raj Chhetri +# @Email : tekraj@mit.edu +# @Web : https://tekrajchhetri.com/ +# @File : provenance.py +# @Software: PyCharm + +""" +Native PROV-O provenance tracking in Oxigraph. + +BrainKB stores knowledge in a triplestore, so provenance is tracked as W3C +PROV-O triples written to a dedicated provenance named graph and queried with +SPARQL. See PROVENANCE_MODEL.md for the full design. + +This module is intentionally self-contained and best-effort: helpers here build +and persist provenance, but callers must ensure a provenance failure never fails +the underlying operation (wrap calls in try/except). +""" + +from __future__ import annotations + +import datetime +import json +import logging +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +import httpx +from rdflib import Graph, Literal, Namespace, URIRef, RDF, XSD, DCTERMS + +from core.shared import get_oxigraph_endpoint, get_oxigraph_auth +from core.graph_database_connection_manager import _get_endpoint + +logger = logging.getLogger(__name__) + +# Dedicated named graph holding all provenance. +PROVENANCE_GRAPH = "https://brainkb.org/provenance/" + +# Per-job delta graphs live under this base. Each ingestion job stages its +# triples in its own delta graph, giving an exact, queryable record of what that +# job added (triple-level change tracking) before/after merging into the target. +PROVENANCE_DELTA_BASE = "https://brainkb.org/provenance/delta/" + +# Namespaces +PROV = Namespace("http://www.w3.org/ns/prov#") +BRAINKB = Namespace("https://brainkb.org/vocab/") # custom vocabulary +PROV_BASE = Namespace("https://brainkb.org/prov/") # instance IRIs + + +def _now_iso() -> str: + """Current UTC time as an ISO-8601 string.""" + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _new_graph() -> Graph: + g = Graph() + g.bind("prov", PROV) + g.bind("brainkb", BRAINKB) + g.bind("dcterms", DCTERMS) + return g + + +def agent_ref(agent_id: str) -> URIRef: + """Mint the agent IRI for a user/system identifier.""" + return URIRef(PROV_BASE[f"agent/{quote(str(agent_id), safe='')}"]) + + +def activity_ref(job_id: str) -> URIRef: + return URIRef(PROV_BASE[f"activity/{quote(str(job_id), safe='')}"]) + + +def delta_graph_for(job_id: str) -> str: + """The per-job delta graph IRI (holds exactly the triples this job added).""" + return f"{PROVENANCE_DELTA_BASE}{quote(str(job_id), safe='')}" + + +# --------------------------------------------------------------------------- +# Builders — each returns an rdflib Graph of PROV-O triples +# --------------------------------------------------------------------------- + +def build_ingestion_provenance( + *, + job_id: str, + agent_id: str, + named_graph_iri: str, + started_at: str, + ended_at: str, + status: str, + total_files: int, + success_count: int, + fail_count: int, + results: Optional[List[Dict[str, Any]]] = None, + agent_type: str = "user", + delta_graph: Optional[str] = None, + added_triple_count: Optional[int] = None, +) -> Graph: + """Build the PROV-O bundle for a completed (terminal) ingestion job. + + When ``delta_graph`` is given, a ``brainkb:IngestionDelta`` entity is added + describing the incremental change: which named graph it derives from, the + delta graph holding the exact added triples, and (optionally) the count. + """ + g = _new_graph() + + activity = activity_ref(job_id) + bundle = URIRef(PROV_BASE[f"bundle/{quote(str(job_id), safe='')}"]) + agent = agent_ref(agent_id) + + # Agent + g.add((agent, RDF.type, PROV.Agent)) + g.add((agent, RDF.type, PROV.SoftwareAgent if agent_type == "system" else PROV.Person)) + + # Activity + g.add((activity, RDF.type, PROV.Activity)) + g.add((activity, RDF.type, BRAINKB.IngestionActivity)) + g.add((activity, PROV.startedAtTime, Literal(started_at, datatype=XSD.dateTime))) + g.add((activity, PROV.endedAtTime, Literal(ended_at, datatype=XSD.dateTime))) + g.add((activity, PROV.wasAssociatedWith, agent)) + g.add((activity, BRAINKB.targetGraph, URIRef(named_graph_iri))) + g.add((activity, BRAINKB.jobStatus, Literal(status))) + g.add((activity, BRAINKB.totalFiles, Literal(int(total_files), datatype=XSD.integer))) + g.add((activity, BRAINKB.successCount, Literal(int(success_count), datatype=XSD.integer))) + g.add((activity, BRAINKB.failCount, Literal(int(fail_count), datatype=XSD.integer))) + + # Ingested bundle entity + g.add((bundle, RDF.type, PROV.Entity)) + g.add((bundle, PROV.wasGeneratedBy, activity)) + g.add((bundle, PROV.wasAttributedTo, agent)) + g.add((bundle, PROV.generatedAtTime, Literal(ended_at, datatype=XSD.dateTime))) + g.add((bundle, DCTERMS.isPartOf, URIRef(named_graph_iri))) + + # Per-file entities + for r in results or []: + fname = str(r.get("file", "unknown")) + file_entity = URIRef(PROV_BASE[f"file/{quote(str(job_id), safe='')}/{quote(fname, safe='')}"]) + g.add((file_entity, RDF.type, PROV.Entity)) + g.add((file_entity, PROV.wasGeneratedBy, activity)) + g.add((file_entity, DCTERMS.isPartOf, bundle)) + g.add((file_entity, BRAINKB.fileName, Literal(fname))) + g.add((file_entity, BRAINKB.uploadStatus, Literal("success" if r.get("success") else "failed"))) + if r.get("http_status") is not None: + g.add((file_entity, BRAINKB.httpStatus, Literal(int(r["http_status"]), datatype=XSD.integer))) + if r.get("size_bytes") is not None: + g.add((file_entity, BRAINKB.sizeBytes, Literal(int(r["size_bytes"]), datatype=XSD.integer))) + + # Incremental-change (delta) entity: an isolated, queryable record of exactly + # the triples this job contributed to the target graph. + if delta_graph: + delta = URIRef(PROV_BASE[f"delta/{quote(str(job_id), safe='')}"]) + g.add((delta, RDF.type, PROV.Entity)) + g.add((delta, RDF.type, BRAINKB.IngestionDelta)) + g.add((delta, PROV.wasGeneratedBy, activity)) + # The change derives from (is applied to) the target named graph + g.add((delta, PROV.wasDerivedFrom, URIRef(named_graph_iri))) + g.add((delta, BRAINKB.changeType, Literal("addition"))) + g.add((delta, BRAINKB.targetGraph, URIRef(named_graph_iri))) + g.add((delta, BRAINKB.deltaGraph, URIRef(delta_graph))) + g.add((delta, DCTERMS.isPartOf, bundle)) + g.add((delta, PROV.generatedAtTime, Literal(ended_at, datatype=XSD.dateTime))) + if added_triple_count is not None: + g.add((delta, BRAINKB.addedTripleCount, Literal(int(added_triple_count), datatype=XSD.integer))) + + return g + + +def build_recovery_provenance( + *, + job_id: str, + at: Optional[str] = None, + cause: str = "", +) -> Graph: + """Build PROV-O for an automated crash-recovery action (agent: system).""" + g = _new_graph() + at = at or _now_iso() + ts = at.replace(":", "").replace("-", "").replace(".", "") + activity = URIRef(PROV_BASE[f"activity/rec-{quote(str(job_id), safe='')}-{quote(ts, safe='')}"]) + system_agent = agent_ref("system") + + g.add((system_agent, RDF.type, PROV.Agent)) + g.add((system_agent, RDF.type, PROV.SoftwareAgent)) + g.add((activity, RDF.type, PROV.Activity)) + g.add((activity, RDF.type, BRAINKB.RecoveryActivity)) + g.add((activity, PROV.startedAtTime, Literal(at, datatype=XSD.dateTime))) + g.add((activity, PROV.wasAssociatedWith, system_agent)) + # Link the recovery activity to the ingestion activity it acted upon + g.add((activity, PROV.used, activity_ref(job_id))) + if cause: + g.add((activity, DCTERMS.description, Literal(cause))) + return g + + +# --------------------------------------------------------------------------- +# Persistence + retrieval +# --------------------------------------------------------------------------- + +async def write_provenance(graph: Graph) -> bool: + """ + Append PROV-O triples to the provenance named graph via Oxigraph's Graph + Store HTTP protocol (POST merges into the graph). Best-effort: returns True + on success, False on failure (never raises). + """ + if graph is None or len(graph) == 0: + return True + try: + payload = graph.serialize(format="turtle") + endpoint = get_oxigraph_endpoint() # .../store + url = f"{endpoint}?graph={quote(PROVENANCE_GRAPH, safe='')}" + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(60.0, connect=15.0)) as client: + resp = await client.post( + url, + content=payload.encode("utf-8"), + headers={"Content-Type": "text/turtle"}, + auth=auth, + ) + if resp.status_code in (200, 201, 204): + return True + logger.warning( + "[provenance] Failed to write provenance (HTTP %s): %s", + resp.status_code, (resp.text or "")[:500], + ) + return False + except Exception as e: + logger.warning(f"[provenance] Error writing provenance: {e}", exc_info=True) + return False + + +async def merge_delta_into_target(delta_graph: str, target_graph: str) -> bool: + """ + Copy all triples from a per-job delta graph into the target named graph via + SPARQL Update ADD (server-side; the delta graph is preserved as the change + record). Best-effort: returns True on success, False otherwise. + """ + try: + update = f"ADD SILENT <{delta_graph}> TO <{target_graph}>" + endpoint = _get_endpoint("post") # .../update for OXIGRAPH + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(600.0, connect=15.0)) as client: + resp = await client.post(endpoint, data={"update": update}, auth=auth) + if resp.status_code in (200, 204): + return True + logger.warning( + "[provenance] Delta merge failed (HTTP %s): %s", + resp.status_code, (resp.text or "")[:500], + ) + return False + except Exception as e: + logger.warning(f"[provenance] Error merging delta graph: {e}", exc_info=True) + return False + + +async def count_graph_triples(graph_iri: str) -> Optional[int]: + """Count triples in a named graph. Returns None on error.""" + try: + query = f"SELECT (COUNT(*) AS ?n) WHERE {{ GRAPH <{graph_iri}> {{ ?s ?p ?o }} }}" + endpoint = _get_endpoint("get") # .../query for OXIGRAPH + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(30.0, connect=10.0)) as client: + resp = await client.post( + endpoint, + data={"query": query}, + headers={"Accept": "application/sparql-results+json"}, + auth=auth, + ) + if resp.status_code != 200: + return None + bindings = resp.json().get("results", {}).get("bindings", []) + return int(bindings[0]["n"]["value"]) if bindings else 0 + except Exception as e: + logger.warning(f"[provenance] Error counting triples in {graph_iri}: {e}", exc_info=True) + return None + + +async def query_provenance_jsonld(construct_query: str) -> Optional[str]: + """ + Execute a SPARQL CONSTRUCT against Oxigraph and return JSON-LD text. + Returns None on error. + + Oxigraph does not serialize CONSTRUCT results as JSON-LD (it offers + Turtle / N-Triples / N-Quads / RDF-XML), so we request Turtle and convert to + JSON-LD locally with rdflib. This keeps the JSON-LD API contract independent + of the triplestore's supported output formats. + """ + try: + endpoint = _get_endpoint("get") # .../query for OXIGRAPH + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(30.0, connect=10.0)) as client: + resp = await client.post( + endpoint, + data={"query": construct_query}, + headers={"Accept": "text/turtle"}, + auth=auth, + ) + if resp.status_code != 200: + logger.warning( + "[provenance] CONSTRUCT query failed (HTTP %s): %s", + resp.status_code, (resp.text or "")[:500], + ) + return None + g = Graph() + g.parse(data=resp.text, format="turtle") + g.bind("prov", PROV) + g.bind("brainkb", BRAINKB) + g.bind("dcterms", DCTERMS) + return g.serialize(format="json-ld", auto_compact=True) + except Exception as e: + logger.warning(f"[provenance] Error querying provenance: {e}", exc_info=True) + return None + + +def construct_for_job(job_id: str) -> str: + """SPARQL CONSTRUCT for all provenance about a single job (activity, bundle, + per-file entities, and the connected agent).""" + activity = str(activity_ref(job_id)) + bundle = str(URIRef(PROV_BASE[f"bundle/{quote(str(job_id), safe='')}"])) + delta = str(URIRef(PROV_BASE[f"delta/{quote(str(job_id), safe='')}"])) + file_prefix = f"{str(PROV_BASE)}file/{quote(str(job_id), safe='')}/" + return f""" + CONSTRUCT {{ ?s ?p ?o }} + WHERE {{ + GRAPH <{PROVENANCE_GRAPH}> {{ + {{ <{activity}> ?p ?o . BIND(<{activity}> AS ?s) }} + UNION + {{ <{bundle}> ?p ?o . BIND(<{bundle}> AS ?s) }} + UNION + {{ <{delta}> ?p ?o . BIND(<{delta}> AS ?s) }} + UNION + {{ ?s ?p ?o . FILTER(STRSTARTS(STR(?s), "{file_prefix}")) }} + UNION + {{ <{activity}> (<{PROV.wasAssociatedWith}>|<{PROV.used}>) ?s . ?s ?p ?o }} + UNION + {{ <{bundle}> <{PROV.wasAttributedTo}> ?s . ?s ?p ?o }} + UNION + {{ ?s <{PROV.used}> <{activity}> . ?s ?p ?o }} + UNION + {{ ?rec <{PROV.used}> <{activity}> ; <{PROV.wasAssociatedWith}> ?s . ?s ?p ?o }} + }} + }} + """ + + +def construct_delta_content(job_id: str) -> str: + """SPARQL CONSTRUCT returning the exact triples a job added (its delta graph).""" + dg = delta_graph_for(job_id) + return f"CONSTRUCT {{ ?s ?p ?o }} WHERE {{ GRAPH <{dg}> {{ ?s ?p ?o }} }}" + + +async def _fetch_construct_graph(construct_query: str) -> Optional[Graph]: + """Run a CONSTRUCT and return the result parsed into an rdflib Graph.""" + try: + endpoint = _get_endpoint("get") + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(60.0, connect=10.0)) as client: + resp = await client.post( + endpoint, + data={"query": construct_query}, + headers={"Accept": "text/turtle"}, + auth=auth, + ) + if resp.status_code != 200: + logger.warning( + "[provenance] CONSTRUCT fetch failed (HTTP %s): %s", + resp.status_code, (resp.text or "")[:500], + ) + return None + g = Graph() + g.parse(data=resp.text, format="turtle") + return g + except Exception as e: + logger.warning(f"[provenance] Error fetching graph: {e}", exc_info=True) + return None + + +async def delta_history_for_graph(named_graph_iri: str) -> Optional[List[Dict[str, Any]]]: + """ + Return the change history for a named graph: one entry per ingestion delta + (job, added triple count, timestamp, status), newest first. + """ + query = f""" + PREFIX prov: <{str(PROV)}> + PREFIX brainkb: <{str(BRAINKB)}> + SELECT ?delta ?activity ?count ?time ?status ?deltaGraph + WHERE {{ + GRAPH <{PROVENANCE_GRAPH}> {{ + ?delta a brainkb:IngestionDelta ; + brainkb:targetGraph <{named_graph_iri}> ; + prov:wasGeneratedBy ?activity . + OPTIONAL {{ ?delta brainkb:addedTripleCount ?count }} + OPTIONAL {{ ?delta prov:generatedAtTime ?time }} + OPTIONAL {{ ?delta brainkb:deltaGraph ?deltaGraph }} + OPTIONAL {{ ?activity brainkb:jobStatus ?status }} + }} + }} + ORDER BY DESC(?time) + """ + try: + endpoint = _get_endpoint("get") + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(30.0, connect=10.0)) as client: + resp = await client.post( + endpoint, + data={"query": query}, + headers={"Accept": "application/sparql-results+json"}, + auth=auth, + ) + if resp.status_code != 200: + return None + out = [] + for b in resp.json().get("results", {}).get("bindings", []): + out.append({ + "delta": b.get("delta", {}).get("value"), + "activity": b.get("activity", {}).get("value"), + "added_triple_count": int(b["count"]["value"]) if "count" in b else None, + "generated_at": b.get("time", {}).get("value"), + "status": b.get("status", {}).get("value"), + "delta_graph": b.get("deltaGraph", {}).get("value"), + }) + return out + except Exception as e: + logger.warning(f"[provenance] Error fetching delta history: {e}", exc_info=True) + return None + + +async def compare_deltas(job_id_a: str, job_id_b: str) -> Optional[Dict[str, Any]]: + """ + Compare the triples added by two jobs. Returns counts and the differing + triples (A-only / B-only) as JSON-LD, plus the shared-triple count. + """ + ga = await _fetch_construct_graph(construct_delta_content(job_id_a)) + gb = await _fetch_construct_graph(construct_delta_content(job_id_b)) + if ga is None or gb is None: + return None + + only_a = ga - gb # in A, not in B + only_b = gb - ga # in B, not in A + common = ga & gb # in both + + def _jsonld(graph: Graph): + graph.bind("prov", PROV) + graph.bind("brainkb", BRAINKB) + graph.bind("dcterms", DCTERMS) + return json.loads(graph.serialize(format="json-ld", auto_compact=True)) + + return { + "job_a": job_id_a, + "job_b": job_id_b, + "a_total": len(ga), + "b_total": len(gb), + "only_in_a_count": len(only_a), + "only_in_b_count": len(only_b), + "common_count": len(common), + "only_in_a": _jsonld(only_a), + "only_in_b": _jsonld(only_b), + } + + +def construct_for_named_graph(named_graph_iri: str) -> str: + """SPARQL CONSTRUCT for all activity that targeted a given named graph.""" + return f""" + CONSTRUCT {{ ?activity ?p ?o . ?ent ?ep ?eo }} + WHERE {{ + GRAPH <{PROVENANCE_GRAPH}> {{ + ?activity <{BRAINKB.targetGraph}> <{named_graph_iri}> ; + ?p ?o . + OPTIONAL {{ ?ent <{PROV.wasGeneratedBy}> ?activity ; ?ep ?eo . }} + }} + }} + """ diff --git a/query_service/core/rbac.py b/query_service/core/rbac.py new file mode 100644 index 0000000..1fdfd15 --- /dev/null +++ b/query_service/core/rbac.py @@ -0,0 +1,230 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# DISCLAIMER: This software is provided "as is" without any warranty, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose, and non-infringement. +# ----------------------------------------------------------------------------- + +# @Author : Tek Raj Chhetri +# @Email : tekraj@mit.edu +# @File : rbac.py + +""" +Role-based access control for BrainKB. + +Separation of concerns: + * JWT -> authentication + API access (who you are; can you call the API). + * Roles -> authorization (what you may DO): create spaces, ingest, admin, etc. + +Roles live in the Django-owned RBAC tables and are joined to a JWT user by email: + Web_jwtuser.email == Web_user_profile.email + Web_user_profile.id -> Web_user_role (active, non-expired) -> role name + +Roles map to capabilities via the policy below. Admins/SuperAdmins can also grant +extra capabilities to individual users (delegated upgrades) via +`user_capability_grants` — e.g. letting a Curator create/manage team spaces. + +A JWT user with NO active role is treated as having NO capabilities: read-only +access to PUBLIC content, nothing else. +""" + +from __future__ import annotations + +import logging +import time +from typing import Optional, Set + +from core.database import get_db_connection + +logger = logging.getLogger(__name__) + +# ---- capabilities ---------------------------------------------------------- +CREATE_PRIVATE_SPACE = "create_private_space" # make your own individual/private space +CREATE_TEAM_SPACE = "create_team_space" # create a team space (Admin/SuperAdmin or granted) +MANAGE_TEAM_SPACE = "manage_team_space" # manage members/visibility/graphs of team spaces +INGEST = "ingest" # ingest data (still needs per-space write membership) +RECOVER = "recover" # recover stuck/errored jobs +SPARQL_ADMIN = "sparql_admin" # run arbitrary SPARQL +GRANT = "grant" # grant/revoke capabilities to other users +READ_PRIVATE = "read_private" # read non-public content you're a member of + +ALL_CAPS = { + CREATE_PRIVATE_SPACE, CREATE_TEAM_SPACE, MANAGE_TEAM_SPACE, INGEST, + RECOVER, SPARQL_ADMIN, GRANT, READ_PRIVATE, +} + +# Capabilities an admin may DELEGATE to another user via the grant endpoint. +# Admin-intrinsic caps (grant, sparql_admin) are intentionally NOT delegatable: +# they come only from an Admin/SuperAdmin role, so the grant endpoint cannot be +# used to escalate a non-admin into an admin. +GRANTABLE_CAPS = { + CREATE_PRIVATE_SPACE, CREATE_TEAM_SPACE, MANAGE_TEAM_SPACE, INGEST, + RECOVER, READ_PRIVATE, +} + +# ---- role hierarchy / policy ---------------------------------------------- +# Hierarchy (highest first): SuperAdmin >= Admin > write roles > read roles > none. +# SuperAdmin is the ultimate authority and is bootstrapped at deployment. Both +# SuperAdmin and Admin get every KG capability here; the *difference* between them +# (e.g. SuperAdmin creating/removing Admins) is ROLE ASSIGNMENT, which is owned by +# the usermanagement/Django side — query_service never assigns roles, it only +# reads them and grants delegatable KG capabilities. This keeps the two systems +# consistent and prevents privilege escalation through the KG API. +SUPERADMIN_ROLE = "SuperAdmin" +ADMIN_ROLES = {"Admin", "SuperAdmin"} +# Roles that confer "write" (may create their own private space + ingest). +WRITE_ROLES = { + "Admin", "SuperAdmin", "Curator", "Lab Member", "Submitter", + "Annotator", "Mapper", "Knowledge Contributor", +} + + +def _caps_for_role(role: str) -> Set[str]: + if role in ADMIN_ROLES: + return set(ALL_CAPS) # admins can do everything + if role in WRITE_ROLES: + return {CREATE_PRIVATE_SPACE, INGEST, RECOVER, READ_PRIVATE} + # any other active role (Reviewer, Validator, Moderator, ...) = read member content + return {READ_PRIVATE} + + +# --------------------------------------------------------------------------- +# lookups +# --------------------------------------------------------------------------- + +async def active_roles(email: Optional[str]) -> Set[str]: + """Active, non-expired role names for the JWT user's email (via profile).""" + if not email: + return set() + async with get_db_connection() as conn: + rows = await conn.fetch( + """ + SELECT r.role + FROM "Web_user_role" r + JOIN "Web_user_profile" p ON p.id = r.profile_id + WHERE p.email = $1 + AND r.is_active IS TRUE + AND (r.expires_at IS NULL OR r.expires_at > now()) + """, + email, + ) + return {row["role"] for row in rows} + + +async def granted_capabilities(email: Optional[str]) -> Set[str]: + """Extra capabilities granted directly to this user (delegated upgrades).""" + if not email: + return set() + async with get_db_connection() as conn: + rows = await conn.fetch( + "SELECT capability FROM user_capability_grants WHERE member = $1", + email, + ) + return {row["capability"] for row in rows if row["capability"] in ALL_CAPS} + + +async def role_granted_capabilities(roles: Set[str]) -> Set[str]: + """Capabilities attached to any of ``roles`` via role/group-level grants + (role_capability_grants). Lets an admin grant a whole custom group/role a + capability (e.g. give 'uk_collaborator' the ingest capability).""" + if not roles: + return set() + async with get_db_connection() as conn: + rows = await conn.fetch( + "SELECT capability FROM role_capability_grants WHERE role = ANY($1::text[])", + list(roles), + ) + return {row["capability"] for row in rows if row["capability"] in ALL_CAPS} + + +async def capabilities(email: Optional[str]) -> Set[str]: + """Effective capabilities = role-derived caps ∪ role/group grants ∪ per-user grants.""" + caps: Set[str] = set() + roles = await active_roles(email) + for r in roles: + caps |= _caps_for_role(r) + if roles: # only users with at least one role can be granted extras + caps |= await role_granted_capabilities(roles) + caps |= await granted_capabilities(email) + return caps + + +async def has_capability(email: Optional[str], cap: str) -> bool: + return cap in await capabilities(email) + + +async def is_admin(email: Optional[str]) -> bool: + return bool(await active_roles(email) & ADMIN_ROLES) + + +# --------------------------------------------------------------------------- +# delegated grants (admin only — enforced at the endpoint) +# --------------------------------------------------------------------------- + +async def grant_capability(member: str, capability: str, granted_by: str) -> None: + if capability not in GRANTABLE_CAPS: + raise ValueError(f"capability is not delegatable: {capability}") + async with get_db_connection() as conn: + await conn.execute( + """ + INSERT INTO user_capability_grants (member, capability, granted_by, created_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (member, capability) DO NOTHING + """, + member, capability, granted_by, time.time(), + ) + + +async def revoke_capability(member: str, capability: str) -> None: + async with get_db_connection() as conn: + await conn.execute( + "DELETE FROM user_capability_grants WHERE member = $1 AND capability = $2", + member, capability, + ) + + +async def list_grants(member: str) -> list: + async with get_db_connection() as conn: + rows = await conn.fetch( + "SELECT capability, granted_by, created_at FROM user_capability_grants WHERE member = $1", + member, + ) + return [dict(r) for r in rows] + + +# ---- role/group-level grants (admin only — enforced at the endpoint) -------- + +async def grant_role_capability(role: str, capability: str, granted_by: str) -> None: + if capability not in GRANTABLE_CAPS: + raise ValueError(f"capability is not delegatable: {capability}") + async with get_db_connection() as conn: + await conn.execute( + """ + INSERT INTO role_capability_grants (role, capability, granted_by, created_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (role, capability) DO NOTHING + """, + role, capability, granted_by, time.time(), + ) + + +async def revoke_role_capability(role: str, capability: str) -> None: + async with get_db_connection() as conn: + await conn.execute( + "DELETE FROM role_capability_grants WHERE role = $1 AND capability = $2", + role, capability, + ) + + +async def list_role_grants(role: Optional[str] = None) -> list: + async with get_db_connection() as conn: + if role: + rows = await conn.fetch( + "SELECT role, capability, granted_by, created_at FROM role_capability_grants WHERE role = $1 ORDER BY capability", + role, + ) + else: + rows = await conn.fetch( + "SELECT role, capability, granted_by, created_at FROM role_capability_grants ORDER BY role, capability" + ) + return [dict(r) for r in rows] diff --git a/query_service/core/routers/insert.py b/query_service/core/routers/insert.py index d765c52..a33bdf4 100644 --- a/query_service/core/routers/insert.py +++ b/query_service/core/routers/insert.py @@ -18,7 +18,7 @@ from fastapi import APIRouter, Request, HTTPException, status, UploadFile, File, Form, BackgroundTasks, Query, Body -from fastapi.responses import JSONResponse +from fastapi.responses import JSONResponse, Response from core.graph_database_connection_manager import insert_data_gdb, insert_data_gdb_async import logging from core.pydantic_schema import InputKGTripleSchema, NamedGraphSchema @@ -48,6 +48,23 @@ batch_insert_job_results, ) from core.configuration import load_environment +from core import spaces as _spaces +from core import rbac +from core.provenance import ( + build_ingestion_provenance, + build_recovery_provenance, + agent_ref, + write_provenance, + query_provenance_jsonld, + construct_for_job, + construct_for_named_graph, + delta_graph_for, + merge_delta_into_target, + count_graph_triples, + construct_delta_content, + delta_history_for_graph, + compare_deltas, +) import datetime import uuid import asyncio @@ -61,6 +78,14 @@ router = APIRouter() logger = logging.getLogger(__name__) + +def _agent_email(user): + """Caller's identity string (email), used for space membership checks.""" + try: + return user["email"] + except (KeyError, TypeError, IndexError): + return None + # Global dictionary to track running background tasks # Maps job_id -> asyncio.Task for checking if job is actually running _running_job_tasks: Dict[str, asyncio.Task] = {} @@ -77,6 +102,28 @@ # Supported file extensions SUPPORTED_EXTS = {"ttl", "turtle", "nt", "nq", "trig", "rdf", "owl", "jsonld", "json"} +# Triple-level change tracking. When enabled, each job stages its triples in a +# per-job delta graph (an exact, queryable record of what the job added) which is +# then merged into the target graph. Costs extra storage (delta graphs persist); +# set TRACK_TRIPLE_DELTAS=false to upload directly to the target instead. +TRACK_TRIPLE_DELTAS = os.getenv("TRACK_TRIPLE_DELTAS", "true").strip().lower() in ("1", "true", "yes", "on") + +# Resource-safety cap: maximum ingest jobs PROCESSING concurrently per worker +# process. Ingestion stays fire-and-forget (submit and forget) — this just bounds +# how many jobs run at once so a burst of submissions can't exhaust memory, the DB +# pool, or Oxigraph. Excess jobs return immediately and wait as 'pending' until a +# slot frees (backpressure without a queue). Effective global cap ≈ this × workers. +MAX_CONCURRENT_INGEST_JOBS = int(os.getenv("MAX_CONCURRENT_INGEST_JOBS", "3")) +_ingest_semaphore: Optional[asyncio.Semaphore] = None + + +def _get_ingest_semaphore() -> asyncio.Semaphore: + """Lazily create the per-worker ingest concurrency limiter (binds to the loop).""" + global _ingest_semaphore + if _ingest_semaphore is None: + _ingest_semaphore = asyncio.Semaphore(MAX_CONCURRENT_INGEST_JOBS) + return _ingest_semaphore + # Ensure job directory exists os.makedirs(JOB_BASE_DIR, exist_ok=True) @@ -279,29 +326,13 @@ async def upload_single_file_path( total_files=total_files, ) - # Process file (attach provenance if text-based RDF format) - if job_id and not skip_provenance: - from core.database import insert_processing_log - status_msg = f"Attaching provenance to {filename}" - await update_job_processing_state( - job_id=job_id, - current_stage="attaching_provenance", - status_message=status_msg - ) - # Log to history - await insert_processing_log( - job_id=job_id, - file_name=filename, - stage="attaching_provenance", - status_message=status_msg, - file_index=file_index, - total_files=total_files, - ) - - processed_filepath, provenance_success = await process_file_with_provenance( - filepath, user_id, ext, skip_provenance=skip_provenance - ) - + # Provenance is tracked natively as PROV-O in Oxigraph's dedicated provenance + # graph (see core/provenance.py and PROVENANCE_MODEL.md), NOT embedded into the + # domain data. The uploaded file is therefore sent to Oxigraph unmodified. + # `skip_provenance` is retained on the API for backward compatibility but no + # longer controls domain-data embedding (which has been removed). + processed_filepath = filepath + # Update processing state: Uploading file if job_id: from core.database import insert_processing_log @@ -440,23 +471,6 @@ async def upload_single_file_path( if resp_text and len(resp_text) > max_len: resp_text = resp_text[:max_len] + "... [truncated]" - # Surface provenance-attachment failures. process_file_with_provenance returns - # success=False when it was asked to attach provenance but rdflib parsing failed; - # in that case the ORIGINAL (un-provenanced) file was uploaded. Previously this - # was silently swallowed and the job still reported success, hiding a data-integrity - # gap. We now flag it explicitly so the job result records it. - provenance_requested = not skip_provenance and ext in [ - "ttl", "turtle", "nt", "nq", "jsonld", "json", "rdf", "owl" - ] - provenance_failed = provenance_requested and not provenance_success - if provenance_failed: - warning = ( - f"WARNING: provenance could not be attached to {filename} " - f"(RDF parsing failed); the original file was uploaded WITHOUT provenance metadata. " - ) - logger.warning(f"[upload_single_file_path] {warning.strip()}") - resp_text = warning + (resp_text or "") - return { "file": filename, "ext": ext, @@ -464,8 +478,6 @@ async def upload_single_file_path( "elapsed_s": elapsed, "http_status": resp.status_code, "success": success, - "provenance_attached": provenance_requested and provenance_success, - "provenance_requested": provenance_requested, "bps": bps, "response_body": resp_text, } @@ -476,6 +488,27 @@ async def run_ingest_job( max_concurrency: int, user_id: str, skip_provenance: bool = False, +): + """Background ingest runner, gated by a per-worker concurrency limiter so a + burst of concurrent submissions cannot exhaust resources and crash the process. + + While waiting for a slot the job stays 'pending' (accurate — it is queued). The + actual work runs in _run_ingest_job_body once a slot is acquired.""" + sem = _get_ingest_semaphore() + if sem.locked(): + logger.info( + f"[run_ingest_job] Job {job_id} is waiting for an ingest slot " + f"(max {MAX_CONCURRENT_INGEST_JOBS} concurrent/worker)" + ) + async with sem: + await _run_ingest_job_body(job_id, max_concurrency, user_id, skip_provenance) + + +async def _run_ingest_job_body( + job_id: str, + max_concurrency: int, + user_id: str, + skip_provenance: bool = False, ): """Background job runner for file ingestion. Processes files (attaches provenance) and uploads them to Oxigraph. @@ -488,7 +521,58 @@ async def run_ingest_job( # This prevents jobs from running indefinitely MAX_JOB_TIMEOUT = 2 * 60 * 60 # 2 hours job_start_time = time.time() - + + delta_graph = delta_graph_for(job_id) + + async def _write_ingestion_prov(status_label: str, results): + """Best-effort finalizer: merge the per-job delta graph into the target, + then record this job as a PROV-O IngestionActivity (+ IngestionDelta) in + Oxigraph. Never raises — a provenance failure must not fail the job.""" + try: + details = await get_job_details(job_id) + named_graph = details.get("graph") if details else None + if not named_graph: + return + results = results or [] + succ = sum(1 for r in results if r.get("success")) + fail = len(results) - succ + + added_count = None + effective_delta_graph = None + if TRACK_TRIPLE_DELTAS: + # Count what the job staged, then merge the delta into the target graph. + added_count = await count_graph_triples(delta_graph) + if added_count and added_count > 0: + await merge_delta_into_target(delta_graph, named_graph) + effective_delta_graph = delta_graph + + prov_graph = build_ingestion_provenance( + job_id=job_id, + agent_id=user_id, + named_graph_iri=named_graph, + started_at=datetime.datetime.fromtimestamp(job_start_time, datetime.timezone.utc).isoformat(), + ended_at=datetime.datetime.now(datetime.timezone.utc).isoformat(), + status=status_label, + total_files=len(results), + success_count=succ, + fail_count=fail, + results=results, + agent_type="user", + delta_graph=effective_delta_graph, + added_triple_count=added_count, + ) + await write_provenance(prov_graph) + + # Queue search indexing to run in the BACKGROUND (do not block the job + # on it — indexing a large graph can be slow). Best-effort enqueue. + try: + from core.indexing import enqueue_ingest + await enqueue_ingest(named_graph, effective_delta_graph) + except Exception as _se: + logger.warning(f"[run_ingest_job] Failed to queue search indexing for {job_id}: {_se}") + except Exception as _pe: + logger.warning(f"[run_ingest_job] Provenance write failed for {job_id}: {_pe}") + try: # Mark job as running (start_time was already set when job was created) from core.database import update_job_processing_state, insert_processing_log @@ -513,7 +597,10 @@ async def run_ingest_job( job_dir = job_details["job_dir"] graph = job_details["graph"] - + # When delta tracking is on, stage uploads in the per-job delta graph; + # _write_ingestion_prov merges it into the target graph at the end. + upload_graph = delta_graph if TRACK_TRIPLE_DELTAS else graph + # Collect files in job_dir (exclude .processed files) file_infos: List[Dict[str, Any]] = [] for name in os.listdir(job_dir): @@ -553,7 +640,7 @@ async def worker(fi: Dict[str, Any], index: int): client, filepath, size, - graph, + upload_graph, user_id, skip_provenance=skip_provenance, job_id=job_id, @@ -615,8 +702,13 @@ async def worker(fi: Dict[str, Any], index: int): status_message=status_msg, ) await update_job_status(job_id, "done", end_time=time.time()) + # Record PROV-O ingestion provenance in Oxigraph (source of truth) + _succ = sum(1 for r in all_results if r.get("success")) + _fail = len(all_results) - _succ + _status_label = "done" if _fail == 0 else ("failed" if _succ == 0 else "partial") + await _write_ingestion_prov(_status_label, all_results) logger.info(f"[run_ingest_job] Job {job_id} completed successfully") - + except asyncio.TimeoutError as e: # Mark job as errored due to timeout from core.database import update_job_processing_state, insert_processing_log @@ -632,6 +724,7 @@ async def worker(fi: Dict[str, Any], index: int): status_message=status_msg, ) await update_job_status(job_id, "error", end_time=time.time()) + await _write_ingestion_prov("error", []) logger.error(f"[run_ingest_job] Job {job_id} timed out: {e}", exc_info=True) except Exception as e: # Mark job as errored @@ -648,6 +741,7 @@ async def worker(fi: Dict[str, Any], index: int): status_message=status_msg, ) await update_job_status(job_id, "error", end_time=time.time()) + await _write_ingestion_prov("error", []) logger.error(f"[run_ingest_job] Job {job_id} failed: {e}", exc_info=True) finally: # Ensure job status is always updated, even if something goes wrong @@ -980,7 +1074,15 @@ async def recover_stuck_jobs( f"The job was marked as 'error' to prevent it from running indefinitely." ), ) - + + # Record the automated recovery as a PROV-O RecoveryActivity (system agent) + try: + await write_provenance( + build_recovery_provenance(job_id=job_id, cause=cause) + ) + except Exception as _pe: + logger.warning(f"[recover_stuck_jobs] Provenance write failed for {job_id}: {_pe}") + logger.info(f"[recover_stuck_jobs] Recovered {len(stuck_jobs)} stuck job(s) from server crash/restart") return len(stuck_jobs) else: @@ -1077,12 +1179,34 @@ async def insert_knowledge_graph_triples( }, status_code=400, ) - + + # Role-based authorization: ingesting requires a write-capable role. JWT + # scope is only API access — it does not by itself grant permission to ingest. + if not await rbac.has_capability(_agent_email(user), rbac.INGEST): + return JSONResponse( + {"error": "Not authorized to ingest: a write-capable role is required."}, + status_code=403, + ) + # If the graph belongs to a space, enforce space write authorization: owner/ + # editor membership, global admin, OR a space write access rule granting a + # group/role/member write (see spaces.can_write_space) — this is how an admin + # hands a whole group ingest access. Unmapped legacy graphs fall through to the + # endpoint scope check. + _graph_key = named_graph_iri if named_graph_iri.endswith("/") else named_graph_iri + "/" + _space_for_graph = await _spaces.get_space_for_graph(_graph_key) + if _space_for_graph is not None: + _allowed, _reason = await _spaces.can_write_space(_space_for_graph, _agent_email(user)) + if not _allowed: + return JSONResponse( + {"error": f"Not authorized to ingest into this graph: {_reason}", "named_graph_iri": named_graph_iri}, + status_code=403, + ) + job_id = uuid.uuid4().hex - + # Get Oxigraph endpoint from configuration endpoint = get_oxigraph_endpoint() - + raw_bytes = data.encode("utf-8") if len(raw_bytes) > MAX_RAW_SIZE_BYTES: return JSONResponse( @@ -1189,7 +1313,29 @@ async def insert_file_knowledge_graph_triples( }, status_code=400, ) - + + # Role-based authorization: ingesting requires a write-capable role. JWT + # scope is only API access — it does not by itself grant permission to ingest. + if not await rbac.has_capability(_agent_email(user), rbac.INGEST): + return JSONResponse( + {"error": "Not authorized to ingest: a write-capable role is required."}, + status_code=403, + ) + # If the graph belongs to a space, enforce space write authorization: owner/ + # editor membership, global admin, OR a space write access rule granting a + # group/role/member write (see spaces.can_write_space) — this is how an admin + # hands a whole group ingest access. Unmapped legacy graphs fall through to the + # endpoint scope check. + _graph_key = named_graph_iri if named_graph_iri.endswith("/") else named_graph_iri + "/" + _space_for_graph = await _spaces.get_space_for_graph(_graph_key) + if _space_for_graph is not None: + _allowed, _reason = await _spaces.can_write_space(_space_for_graph, _agent_email(user)) + if not _allowed: + return JSONResponse( + {"error": f"Not authorized to ingest into this graph: {_reason}", "named_graph_iri": named_graph_iri}, + status_code=403, + ) + job_id = uuid.uuid4().hex # generate for job tracking # Get Oxigraph endpoint from configuration @@ -1336,7 +1482,8 @@ async def run_and_track_job(): } @router.get("/insert/jobs", - include_in_schema=True + include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))], ) async def list_jobs( user: Annotated[LoginUserIn, Depends(get_current_user)], @@ -1362,7 +1509,8 @@ async def list_jobs( @router.get("/insert/user/jobs/detail", - include_in_schema=True + include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))], ) async def get_job_detail( user: Annotated[LoginUserIn, Depends(get_current_user)], @@ -1515,7 +1663,8 @@ async def get_job_detail( @router.get("/insert/jobs/check-recoverable", - include_in_schema=True + include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))], ) async def check_job_recoverable_endpoint( user: Annotated[LoginUserIn, Depends(get_current_user)], @@ -1554,7 +1703,8 @@ async def check_job_recoverable_endpoint( @router.post("/insert/jobs/recover", - include_in_schema=True + include_in_schema=True, + dependencies=[Depends(require_scopes(["write"]))], ) async def recover_stuck_jobs_endpoint( user: Annotated[LoginUserIn, Depends(get_current_user)], @@ -1590,6 +1740,11 @@ async def recover_stuck_jobs_endpoint( Returns the number of jobs recovered and details about recovered jobs. """ verify_user_access(user_id, user) + if not await rbac.has_capability(_agent_email(user), rbac.RECOVER): + return JSONResponse( + {"error": "Not authorized to recover jobs: a write-capable role is required."}, + status_code=403, + ) try: # For single job recovery, MUST check recoverability first (includes process check) if job_id: @@ -1714,7 +1869,9 @@ async def recover_stuck_jobs_endpoint( ) -@router.post("/register-named-graph") +@router.post("/register-named-graph", + dependencies=[Depends(require_scopes(["write"]))], + ) async def create_named_graph( user: Annotated[LoginUserIn, Depends(get_current_user)], request: NamedGraphSchema @@ -1743,10 +1900,20 @@ async def create_named_graph( """ named_graph_exists = await fetch_data_gdb_async(query) if not named_graph_exists.get("message", {}).get("boolean", False): + # Attribute the registration to the authenticated user (PROV-O). This is + # recorded on the registry entry itself (see named_graph_metadata) rather + # than duplicated as a separate activity in the provenance graph. + try: + agent_id = user["email"] or user["id"] + except (KeyError, TypeError, IndexError): + agent_id = "unknown" + agent_uri = str(agent_ref(str(agent_id))) + # Register the new named graph response = await insert_data_gdb_async(named_graph_metadata( named_graph_url=named_graph_url, description=description, + agent_uri=agent_uri, ) ) return response @@ -1765,3 +1932,143 @@ async def create_named_graph( detail=f"An error occurred processing the request {e}", ) + +@router.get( + "/provenance/job", + include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))], + summary="PROV-O provenance bundle for one ingestion job", + description=( + "Returns the **W3C PROV-O bundle** (JSON-LD) for a single ingestion job: " + "the `IngestionActivity` (with agent, start/end time, status, file counts), " + "the generated bundle entity, each per-file entity (upload status, HTTP " + "status, size), the `IngestionDelta` entity, and any recovery activity that " + "acted on the job. Access-controlled — the `user_id` must match the " + "authenticated caller." + ), +) +async def get_job_provenance( + user: Annotated[LoginUserIn, Depends(get_current_user)], + user_id: Annotated[str, Query(..., description="User identifier (must match the authenticated user)")], + job_id: Annotated[str, Query(..., description="Job identifier to fetch provenance for")], +): + """ + Return the full PROV-O provenance bundle (JSON-LD) for a single ingestion job, + reconstructed from the dedicated provenance graph in Oxigraph. + + Scope: everything about ONE job (activity + bundle + files + delta + recovery). + For a whole graph's history use /provenance/named-graph. + """ + verify_user_access(user_id, user) + jsonld = await query_provenance_jsonld(construct_for_job(job_id)) + if jsonld is None: + return JSONResponse( + {"error": "Failed to retrieve provenance from the graph database"}, + status_code=502, + ) + return Response(content=jsonld, media_type="application/ld+json") + + +@router.get( + "/provenance/named-graph", + include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))], + summary="PROV-O activity history for a named graph", + description=( + "Returns the **W3C PROV-O provenance** (JSON-LD) describing how a named " + "graph's data came to be: every ingestion activity that targeted it, with " + "the agent, start/end times, per-file entities, and job status.\n\n" + "Difference from `GET /api/query/registered-named-graphs`:\n" + "- **registered-named-graphs** = the *registry/catalog* — which graphs " + "exist and their registration metadata (one row per graph).\n" + "- **provenance/named-graph** = the *activity history* — what was ingested " + "into a given graph, when, and by whom (a PROV-O bundle, potentially many " + "activities). Reads the provenance graph `https://brainkb.org/provenance/`." + ), +) +async def get_named_graph_provenance( + user: Annotated[LoginUserIn, Depends(get_current_user)], + iri: Annotated[str, Query(..., description="Named graph IRI to fetch the ingestion/activity provenance for")], +): + """ + Return the PROV-O provenance (JSON-LD) for every ingestion activity that + targeted the given named graph. + + This is the *history of data mutations* on the graph, distinct from the + registry catalog returned by /api/query/registered-named-graphs. Registration + attribution lives on the registry entry (see that endpoint's `registered_by`). + """ + jsonld = await query_provenance_jsonld(construct_for_named_graph(iri)) + if jsonld is None: + return JSONResponse( + {"error": "Failed to retrieve provenance from the graph database"}, + status_code=502, + ) + return Response(content=jsonld, media_type="application/ld+json") + + +@router.get("/provenance/delta", include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))]) +async def get_job_delta( + user: Annotated[LoginUserIn, Depends(get_current_user)], + user_id: Annotated[str, Query(..., description="User identifier (must match the authenticated user)")], + job_id: Annotated[str, Query(..., description="Job identifier whose added triples to return")], +): + """ + GET /provenance/delta + Return the exact set of triples a job added (its delta graph) as JSON-LD. + This is the incremental change that job contributed to the target graph. + """ + verify_user_access(user_id, user) + jsonld = await query_provenance_jsonld(construct_delta_content(job_id)) + if jsonld is None: + return JSONResponse( + {"error": "Failed to retrieve delta from the graph database"}, + status_code=502, + ) + return Response(content=jsonld, media_type="application/ld+json") + + +@router.get("/provenance/delta/history", include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))]) +async def get_delta_history( + user: Annotated[LoginUserIn, Depends(get_current_user)], + iri: Annotated[str, Query(..., description="Named graph IRI to list the change history for")], +): + """ + GET /provenance/delta/history + Return the ordered change history of a named graph: one entry per ingestion + delta (job, added triple count, timestamp, status), newest first. + """ + history = await delta_history_for_graph(iri) + if history is None: + return JSONResponse( + {"error": "Failed to retrieve delta history from the graph database"}, + status_code=502, + ) + return {"named_graph_iri": iri, "changes": history, "total": len(history)} + + +@router.get("/provenance/delta/compare", include_in_schema=True, + dependencies=[Depends(require_scopes(["read"]))]) +async def compare_job_deltas( + user: Annotated[LoginUserIn, Depends(get_current_user)], + user_id: Annotated[str, Query(..., description="User identifier (must match the authenticated user)")], + job_id_a: Annotated[str, Query(..., description="First job identifier")], + job_id_b: Annotated[str, Query(..., description="Second job identifier")], +): + """ + GET /provenance/delta/compare + Compare the triples added by two jobs. Returns counts (A-only, B-only, + shared) and the differing triples as JSON-LD, so users can see exactly how + two ingestion changes differ. + """ + verify_user_access(user_id, user) + result = await compare_deltas(job_id_a, job_id_b) + if result is None: + return JSONResponse( + {"error": "Failed to compare deltas (one or both delta graphs unavailable)"}, + status_code=502, + ) + return result + diff --git a/query_service/core/routers/jwt_auth.py b/query_service/core/routers/jwt_auth.py index 61f8c31..6d416af 100644 --- a/query_service/core/routers/jwt_auth.py +++ b/query_service/core/routers/jwt_auth.py @@ -5,39 +5,47 @@ from core.database import get_db_connection, insert_data, get_scopes_by_user from core.models.user import UserIn, LoginUserIn from core.security import get_password_hash, authenticate_user, create_access_token +from core import rbac logger = logging.getLogger(__name__) router = APIRouter() -@router.post("/register", status_code=201) +@router.post("/register", include_in_schema=False) async def register(user: UserIn): - """ - Register a new user. Uses proper connection management to avoid connection leaks. - - Note: The unique constraint check is handled atomically by the database. - This prevents race conditions where two concurrent requests could both pass - a pre-check and then both attempt to insert the same email. - """ - async with get_db_connection() as conn: - hashed_password = await get_password_hash(user.password) - - # Let the database enforce uniqueness atomically - no pre-check needed - # insert_data will catch UniqueViolationError and return a user-friendly message - return await insert_data( - conn=conn, fullname=user.full_name, email=user.email, password=hashed_password - ) - - -@router.post("/token", include_in_schema=True) + """Self-registration is DISABLED — there is no separate register step. + Users are onboarded by signing in with Globus / ORCID / GitHub + (usermanagement `/api/auth/{provider}/login`, or `brainkb_globus_login` in the + skill), which auto-creates and links the profile on first login.""" + raise HTTPException( + status_code=status.HTTP_405_METHOD_NOT_ALLOWED, + detail=("Self-registration is disabled. Sign in with Globus / ORCID / GitHub " + "— your account is created automatically on first login."), + ) + + +@router.post("/login") +@router.post("/token", include_in_schema=False) # deprecated alias of /login async def login(user: LoginUserIn): """ - Authenticate user and return JWT token. Uses proper connection management to avoid connection leaks. - Reuses the same database connection for both authentication and scope retrieval to minimize connection pool usage. + Authenticate (password) and return a JWT. Primary path is `/api/login`; + `/api/token` is kept as a deprecated alias for backward compatibility. """ async with get_db_connection() as conn: authenticated_user = await authenticate_user(user.email, user.password, conn) scopes = await get_scopes_by_user(user_id=authenticated_user["id"], conn=conn) - access_token = create_access_token(authenticated_user["email"], scopes) + # Enrich the token with profile_id + roles so its shape matches + # usermanagement's v2 token. Signed with query_service's own secret + # (per-service isolation preserved); roles remain informational since + # authorization re-reads them from the DB (core.rbac). + email = authenticated_user["email"] + profile_id = await conn.fetchval( + 'SELECT id FROM "Web_user_profile" WHERE lower(email) = lower($1)', email + ) + roles = sorted(await rbac.active_roles(email)) + access_token = create_access_token( + email, scopes, + user_id=authenticated_user["id"], profile_id=profile_id, roles=roles, + ) return {"access_token": access_token, "token_type": "bearer"} diff --git a/query_service/core/routers/query.py b/query_service/core/routers/query.py index 88c521d..907d0f7 100644 --- a/query_service/core/routers/query.py +++ b/query_service/core/routers/query.py @@ -16,13 +16,14 @@ # @File : query.py # @Software: PyCharm -from fastapi import APIRouter +from fastapi import APIRouter, HTTPException from core.graph_database_connection_manager import fetch_data_gdb_async, check_named_graph_exists import logging from typing import Annotated from core.models.user import LoginUserIn from core.security import get_current_user, require_scopes from core.shared import taxonomy_postprocessing +from core import rbac from fastapi import Depends from pydantic import BaseModel, root_validator from typing import List @@ -30,17 +31,42 @@ logger = logging.getLogger(__name__) -@router.get("/query/registered-named-graphs") -async def get_named_graphs(): +@router.get( + "/query/registered-named-graphs", + dependencies=[Depends(require_scopes(["read"]))], + summary="List registered named graphs (the registry/catalog)", + description=( + "Returns the **catalog of named graphs** that have been registered in " + "BrainKB — i.e. *which* graphs exist and may be ingested into. Each entry " + "carries its `description`, `registered_at` timestamp, and `registered_by` " + "(the user who registered it). Data is read from the registry graph " + "`https://brainkb.org/metadata/named-graph`.\n\n" + "Visibility-filtered: graphs that belong to a **private space** the caller " + "is not a member of are omitted (so private graph existence is not leaked). " + "Public-space graphs and legacy (unmapped) graphs are always listed.\n\n" + "This answers *\"what graphs are available?\"*. It is NOT a history of what " + "was ingested — for the ingestion/activity history of a specific graph, use " + "`GET /api/provenance/named-graph?iri=…`." + ), +) +async def get_named_graphs(user: Annotated[LoginUserIn, Depends(get_current_user)]): + """List every registered named graph with its registration metadata, + filtered so private-space graphs the caller can't access are hidden. + + Registry (catalog) view: one row per graph with description, when it was + registered, and by whom. Contrast with /api/provenance/named-graph, which + returns the PROV-O activity history (ingestions) that targeted a graph. + """ query_named_graph = """ PREFIX prov: PREFIX dcterms: - Select distinct ?graph ?description ?registered_at + Select distinct ?graph ?description ?registered_at ?registered_by WHERE { GRAPH { ?graph dcterms:description ?description; prov:generatedAtTime ?registered_at. - } + OPTIONAL { ?graph prov:wasAttributedTo ?registered_by. } + } } """ response = await fetch_data_gdb_async(query_named_graph) @@ -54,17 +80,45 @@ async def get_named_graphs(): response_graph[graphs_info["graph"]["value"]] = { "graph": graphs_info["graph"]["value"], "description": graphs_info["description"]["value"], - "registered_at": graphs_info["registered_at"]["value"] + "registered_at": graphs_info["registered_at"]["value"], + "registered_by": graphs_info.get("registered_by", {}).get("value"), } + + # Hide graphs belonging to private spaces the caller is not a member of, so + # private graph existence is not leaked via the registry listing. + from core.spaces import hidden_graphs_for + try: + member = user["email"] + except (KeyError, TypeError, IndexError): + member = None + hidden = await hidden_graphs_for(member) + if hidden: + response_graph = {k: v for k, v in response_graph.items() if k not in hidden} return response_graph @router.get("/query/sparql/", - dependencies=[Depends(require_scopes(["write","admin"]))], + dependencies=[Depends(require_scopes(["admin"]))], + summary="Run an arbitrary SPARQL query (admin only)", + description=( + "Executes a caller-supplied SPARQL query against the graph database. " + "This is a powerful, unrestricted capability, so it is gated to the " + "**admin** scope only — deliberately NOT the default 'read' scope that " + "the fixed-shape read endpoints (e.g. /query/taxonomy, /query/" + "registered-named-graphs) use." + ), ) async def sparql_query( user: Annotated[LoginUserIn, Depends(get_current_user)], sparql_query: str ): + # Authorization is role-based: arbitrary SPARQL requires the sparql_admin + # capability (Admin/SuperAdmin). The JWT scope only gates API access. + try: + email = user["email"] + except (KeyError, TypeError, IndexError): + email = None + if not await rbac.has_capability(email, rbac.SPARQL_ADMIN): + raise HTTPException(status_code=403, detail="Arbitrary SPARQL requires an Admin/SuperAdmin role.") response = await fetch_data_gdb_async(sparql_query) return response diff --git a/query_service/core/routers/search.py b/query_service/core/routers/search.py new file mode 100644 index 0000000..4525044 --- /dev/null +++ b/query_service/core/routers/search.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# DISCLAIMER: This software is provided "as is" without any warranty, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose, and non-infringement. +# ----------------------------------------------------------------------------- + +# @Author : Tek Raj Chhetri +# @Email : tekraj@mit.edu +# @File : search.py (router) + +"""Search endpoint — hybrid Postgres-locator + Oxigraph-data, access-filtered.""" + +import logging +from typing import Annotated, Optional + +from fastapi import APIRouter, Depends, Query + +from core.models.user import LoginUserIn +from core.security import get_current_user, get_current_user_optional, require_scopes +from core import search as se +from core import indexing as ix + +router = APIRouter() +logger = logging.getLogger(__name__) + + +def _agent(user) -> Optional[str]: + if not user: + return None + try: + return user["email"] + except (KeyError, TypeError, IndexError): + return None + + +@router.get( + "/search", + summary="Search knowledge graphs (access-filtered)", + description=( + "Full-text search over the knowledge graphs, honoring space visibility. " + "Postgres locates matching subjects (fast, filtered by workspace/visibility); " + "the matched triples are then fetched from Oxigraph.\n\n" + "- **Anonymous** (no token): searches **public** spaces only.\n" + "- **Authenticated**: public spaces + the caller's own/member (private) spaces " + "(and legacy/unmapped graphs).\n" + "- Pass `space` to scope the search to a single space you can access; omit it " + "for a full search across everything you may read.\n\n" + "Private data is never returned to non-members — the filter is enforced in the " + "locator query itself." + ), +) +async def search( + user: Annotated[Optional[object], Depends(get_current_user_optional)], + q: Annotated[str, Query(..., min_length=1, description="Search terms")], + space: Annotated[Optional[str], Query(description="Restrict to a single space slug")] = None, + limit: Annotated[int, Query(ge=1, le=100)] = 25, + offset: Annotated[int, Query(ge=0)] = 0, +): + return await se.search(q=q, caller=_agent(user), space_slug=space, limit=limit, offset=offset) + + +@router.post( + "/search/reindex", + dependencies=[Depends(require_scopes(["admin"]))], + summary="Backfill/rebuild the search index (admin) — runs in background", + description="Queues a background task that reindexes every user graph into the " + "Postgres locator index. Returns immediately with a task_id; poll " + "GET /search/index-tasks for progress.", +) +async def reindex(user: Annotated[LoginUserIn, Depends(get_current_user)]): + task_id = await ix.enqueue_backfill() + return {"status": "queued", "task_id": task_id, + "message": "Backfill reindex queued; runs in the background."} + + +@router.get( + "/search/index-tasks", + dependencies=[Depends(require_scopes(["read"]))], + summary="List background indexing tasks and their status", +) +async def index_tasks( + user: Annotated[LoginUserIn, Depends(get_current_user)], + task_id: Annotated[Optional[str], Query(description="Fetch a single task by id")] = None, + limit: Annotated[int, Query(ge=1, le=200)] = 50, +): + if task_id: + t = await ix.get_task(task_id) + return t or {"error": "task not found"} + return {"tasks": await ix.list_tasks(limit)} diff --git a/query_service/core/routers/spaces.py b/query_service/core/routers/spaces.py new file mode 100644 index 0000000..5b3be17 --- /dev/null +++ b/query_service/core/routers/spaces.py @@ -0,0 +1,441 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# DISCLAIMER: This software is provided "as is" without any warranty, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose, and non-infringement. +# ----------------------------------------------------------------------------- + +# @Author : Tek Raj Chhetri +# @Email : tekraj@mit.edu +# @File : spaces.py (router) + +"""REST endpoints for spaces — private/public containers of named graphs.""" + +import logging +import re +from typing import Annotated, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, status +from fastapi.responses import JSONResponse, Response +from pydantic import BaseModel, HttpUrl + +from core.models.user import LoginUserIn +from core.security import get_current_user, get_current_user_optional, require_scopes +from core.shared import named_graph_metadata +from core.provenance import agent_ref, query_provenance_jsonld +from core.graph_database_connection_manager import insert_data_gdb_async, check_named_graph_exists +from core import spaces as sp +from core import rbac + +router = APIRouter() +logger = logging.getLogger(__name__) + +_SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}[a-z0-9]$") + + +def _agent(user) -> str: + """Caller's identity string (email preferred), matching the PROV agent id.""" + try: + return str(user["email"] or user["id"]) + except (KeyError, TypeError, IndexError): + return "unknown" + + +async def _can_manage(space: dict, email: str) -> bool: + """Who may manage a space (members/visibility/graphs/access-rules): + + * **Admin/SuperAdmin** — every space (platform-wide). + * **Owner** (the creator) — their own space. + * A non-admin with **manage_team_space** — ONLY team spaces they are + **assigned to** (a member of), not every team space. + * Anyone matched by a per-space **'manage'** access rule (explicit assignment). + + i.e. unless you're an Admin, you can manage only the team spaces you created or + were assigned to — never all of them.""" + if await rbac.is_admin(email): + return True + srole = await sp.member_role(space["space_id"], email) + if srole == "owner": + return True + if await sp.matches_access_rule(space["space_id"], "manage", email): + return True + if (space.get("space_type") == "team" and srole is not None + and await rbac.has_capability(email, rbac.MANAGE_TEAM_SPACE)): + return True + return False + + +class SpaceCreate(BaseModel): + slug: str + name: str + description: Optional[str] = None + visibility: str = "private" + space_type: str = "individual" # 'individual' | 'team' + + +class MemberIn(BaseModel): + member: str + role: str = "viewer" + + +class VisibilityIn(BaseModel): + visibility: str + + +class SpaceGraphIn(BaseModel): + named_graph_url: HttpUrl + description: str = "" + + +@router.post("/spaces", status_code=201, + dependencies=[Depends(require_scopes(["write"]))], + summary="Create a space", + description="Create an owner-controlled space (private by default). The " + "caller becomes its owner. Members can later be added and the " + "space flipped public for anonymous read access.") +async def create_space(body: SpaceCreate, user: Annotated[LoginUserIn, Depends(get_current_user)]): + if not _SLUG_RE.match(body.slug): + raise HTTPException(400, "slug must be lowercase alphanumeric/hyphen, 3-64 chars") + if body.visibility not in ("private", "public"): + raise HTTPException(400, "visibility must be 'private' or 'public'") + if body.space_type not in ("individual", "team"): + raise HTTPException(400, "space_type must be 'individual' or 'team'") + + # Role-based authorization: team spaces require create_team_space (Admin/ + # SuperAdmin, or a user an admin has granted it); individual/private spaces + # require create_private_space (any write-capable role). + email = _agent(user) + if body.space_type == "team": + if not await rbac.has_capability(email, rbac.CREATE_TEAM_SPACE): + raise HTTPException(403, "not authorized to create a team space (needs Admin/SuperAdmin " + "or a granted create_team_space capability)") + else: + if not await rbac.has_capability(email, rbac.CREATE_PRIVATE_SPACE): + raise HTTPException(403, "not authorized to create a space (needs a write-capable role)") + + if await sp.get_space(body.slug): + raise HTTPException(409, f"space '{body.slug}' already exists") + space = await sp.create_space(body.slug, body.name, body.description, email, + body.visibility, body.space_type) + await sp.mirror_space_to_rdf(space) + return space + + +@router.get("/spaces", + summary="List visible spaces", + description="Lists spaces the caller may see: all public spaces plus any " + "the caller is a member of. Anonymous callers see only public " + "spaces (no token required).") +async def list_spaces(user: Annotated[Optional[object], Depends(get_current_user_optional)]): + member = _agent(user) if user else None + return {"spaces": await sp.list_visible_spaces(member)} + + +@router.get("/spaces/{slug}", + summary="Get a space", + description="Returns a space's manifest (members, graphs, visibility) if the " + "caller may see it — public to anyone, private to members only.") +async def get_space(slug: str, user: Annotated[Optional[object], Depends(get_current_user_optional)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + member = _agent(user) if user else None + if space["visibility"] != "public": + # Private: needs membership AND a role that grants read_private + # (a JWT user with no role gets public content only). + role = await sp.member_role(space["space_id"], member) + if role is None or not await rbac.has_capability(member, rbac.READ_PRIVATE): + raise HTTPException(403, "private space — membership and a role are required") + # Fine-grained per-space read rules (if any) further restrict who can read. + if not await sp.space_action_permitted(space, "read", member): + raise HTTPException(403, "restricted by a space access rule (read)") + return space + + +@router.patch("/spaces/{slug}/visibility", + dependencies=[Depends(require_scopes(["write"]))], + summary="Set space visibility (owner only)", + description="Flip a space between 'private' and 'public'. Public spaces are " + "readable by anyone, including unauthenticated clients.") +async def set_visibility(slug: str, body: VisibilityIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + if not await _can_manage(space, _agent(user)): + raise HTTPException(403, "not authorized to change this space's visibility") + if body.visibility not in ("private", "public"): + raise HTTPException(400, "visibility must be 'private' or 'public'") + await sp.set_visibility(slug, body.visibility) + space = await sp.get_space(slug) + await sp.mirror_space_to_rdf(space) + return space + + +@router.post("/spaces/{slug}/members", + dependencies=[Depends(require_scopes(["write"]))], + summary="Add or update a member (owner only)") +async def add_member(slug: str, body: MemberIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + if not await _can_manage(space, _agent(user)): + raise HTTPException(403, "not authorized to manage members of this space") + if body.role not in sp.ROLES: + raise HTTPException(400, f"role must be one of {sp.ROLES}") + await sp.add_member(space["space_id"], body.member, body.role) + space = await sp.get_space(slug) + await sp.mirror_space_to_rdf(space) + return space + + +@router.delete("/spaces/{slug}/members/{member}", + dependencies=[Depends(require_scopes(["write"]))], + summary="Remove a member (owner only)") +async def remove_member(slug: str, member: str, user: Annotated[LoginUserIn, Depends(get_current_user)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + if not await _can_manage(space, _agent(user)): + raise HTTPException(403, "not authorized to manage members of this space") + await sp.remove_member(space["space_id"], member) + space = await sp.get_space(slug) + await sp.mirror_space_to_rdf(space) + return space + + +@router.post("/spaces/{slug}/graphs", + dependencies=[Depends(require_scopes(["write"]))], + summary="Register a named graph into a space (owner/editor)", + description="Registers a named graph and binds it to this space so that " + "ingestion and reads on that graph are governed by the space's " + "membership and visibility.") +async def add_graph(slug: str, body: SpaceGraphIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + _role = await sp.member_role(space["space_id"], _agent(user)) + if not (await _can_manage(space, _agent(user)) or _role in sp.WRITE_ROLES): + raise HTTPException(403, "only the space owner/editors (or a space manager) can add graphs") + + named_graph_url = str(body.named_graph_url) + if not named_graph_url.endswith("/"): + named_graph_url += "/" + + # Bind first: a graph is globally unique to one space, so if another space + # already holds it this must fail with a conflict rather than register registry + # metadata for a graph we cannot attach. + try: + await sp.attach_graph(space["space_id"], named_graph_url) + except sp.GraphAlreadyBound as e: + raise HTTPException(409, str(e)) + + # Register in the graph registry if not already there (idempotent-ish). + if not await check_named_graph_exists(named_graph_url): + await insert_data_gdb_async(named_graph_metadata( + named_graph_url=named_graph_url, + description=body.description, + agent_uri=str(agent_ref(_agent(user))), + )) + space = await sp.get_space(slug) + await sp.mirror_space_to_rdf(space) + return space + + +@router.get("/spaces/{slug}/data", + summary="Read a space's data (public = anonymous)", + description="Returns the RDF (JSON-LD) across all named graphs in the space. " + "Public spaces are readable by anyone (no token); private spaces " + "require membership.") +async def read_space_data(slug: str, user: Annotated[Optional[object], Depends(get_current_user_optional)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + member = _agent(user) if user else None + if space["visibility"] != "public": + # Private: membership AND a role that grants read_private. + if await sp.member_role(space["space_id"], member) is None \ + or not await rbac.has_capability(member, rbac.READ_PRIVATE): + raise HTTPException(403, "private space — membership and a role are required") + # Fine-grained per-space read rules (if any) further restrict who can read. + if not await sp.space_action_permitted(space, "read", member): + raise HTTPException(403, "restricted by a space access rule (read)") + if not space["graphs"]: + return Response(content='{"@graph": []}', media_type="application/ld+json") + jsonld = await query_provenance_jsonld(await sp.construct_space_graphs(space)) + if jsonld is None: + return JSONResponse({"error": "failed to read space data"}, status_code=502) + return Response(content=jsonld, media_type="application/ld+json") + + +# --------------------------------------------------------------------------- # +# Admin: delegated capability grants (Admin/SuperAdmin only) +# --------------------------------------------------------------------------- # + +class GrantIn(BaseModel): + member: str + capability: str + + +class RoleGrantIn(BaseModel): + role: str + capability: str + + +class AccessRuleIn(BaseModel): + action: str # read | write | manage + subject_type: str # global_role | member | space_role + subject_value: str # role name / email / viewer|editor|owner + + +@router.get("/spaces/{slug}/access-rules", + summary="List a space's fine-grained access rules") +async def list_access_rules(slug: str, user: Annotated[LoginUserIn, Depends(get_current_user)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + email = _agent(user) + # visible to managers or members of the space + if not await _can_manage(space, email) and await sp.member_role(space["space_id"], email) is None: + raise HTTPException(403, "must be a member or manager of the space") + return {"slug": slug, "rules": await sp.list_access_rules(space["space_id"])} + + +@router.post("/spaces/{slug}/access-rules", + dependencies=[Depends(require_scopes(["write"]))], + summary="Add a fine-grained access rule (space manager only)", + description="Restrict a space action to a subject. action: read|write|manage. " + "subject_type: global_role (e.g. 'Admin','Lab Member') | member " + "(an email) | space_role (viewer|editor|owner). When rules exist " + "for an action, only matching callers may perform it (owner and " + "Admin/SuperAdmin always bypass).") +async def add_access_rule(slug: str, body: AccessRuleIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + if not await _can_manage(space, _agent(user)): + raise HTTPException(403, "not authorized to manage this space's access rules") + try: + await sp.add_access_rule(space["space_id"], body.action, body.subject_type, + body.subject_value, _agent(user)) + except ValueError as e: + raise HTTPException(400, str(e)) + return {"slug": slug, "rules": await sp.list_access_rules(space["space_id"])} + + +@router.delete("/spaces/{slug}/access-rules/{rule_id}", + dependencies=[Depends(require_scopes(["write"]))], + summary="Delete a fine-grained access rule (space manager only)") +async def delete_access_rule(slug: str, rule_id: int, user: Annotated[LoginUserIn, Depends(get_current_user)]): + space = await sp.get_space(slug) + if not space: + return JSONResponse({"error": "space not found"}, status_code=404) + if not await _can_manage(space, _agent(user)): + raise HTTPException(403, "not authorized to manage this space's access rules") + await sp.remove_access_rule(space["space_id"], rule_id) + return {"slug": slug, "rules": await sp.list_access_rules(space["space_id"])} + + +@router.get("/admin/capabilities", + dependencies=[Depends(require_scopes(["admin"]))], + summary="List a user's effective capabilities (admin only)") +async def get_capabilities( + user: Annotated[LoginUserIn, Depends(get_current_user)], + member: Annotated[str, Query(..., description="User email to inspect")], +): + if not await rbac.is_admin(_agent(user)): + raise HTTPException(403, "Admin/SuperAdmin role required") + return { + "member": member, + "roles": sorted(await rbac.active_roles(member)), + "capabilities": sorted(await rbac.capabilities(member)), + "grants": await rbac.list_grants(member), + } + + +@router.post("/admin/capabilities/grant", + dependencies=[Depends(require_scopes(["admin"]))], + summary="Grant a capability to a user (admin only)", + description="Delegated upgrade: e.g. grant 'create_team_space' or " + "'manage_team_space' to a Curator/Lab Member so they can create " + "and manage team spaces. Requires the caller to hold an " + "Admin/SuperAdmin role.") +async def grant_capability(body: GrantIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + if not await rbac.is_admin(_agent(user)): + raise HTTPException(403, "Admin/SuperAdmin role required to grant capabilities") + if body.capability not in rbac.GRANTABLE_CAPS: + raise HTTPException(400, f"capability is not delegatable; valid: {sorted(rbac.GRANTABLE_CAPS)}") + await rbac.grant_capability(body.member, body.capability, _agent(user)) + return {"status": "granted", "member": body.member, "capability": body.capability} + + +@router.post("/admin/capabilities/revoke", + dependencies=[Depends(require_scopes(["admin"]))], + summary="Revoke a granted capability (admin only)") +async def revoke_capability(body: GrantIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + if not await rbac.is_admin(_agent(user)): + raise HTTPException(403, "Admin/SuperAdmin role required to revoke capabilities") + await rbac.revoke_capability(body.member, body.capability) + return {"status": "revoked", "member": body.member, "capability": body.capability} + + +@router.get("/admin/capabilities/available", + dependencies=[Depends(require_scopes(["admin"]))], + summary="List all KG capabilities and which are delegatable (admin only)", + description="Catalog of query_service capabilities. 'grantable' are the ones " + "an admin may delegate to a user or role/group; 'grant' and " + "'sparql_admin' are admin-intrinsic (not delegatable).") +async def available_capabilities(user: Annotated[LoginUserIn, Depends(get_current_user)]): + if not await rbac.is_admin(_agent(user)): + raise HTTPException(403, "Admin/SuperAdmin role required") + return { + "all": sorted(rbac.ALL_CAPS), + "grantable": sorted(rbac.GRANTABLE_CAPS), + "admin_only": sorted(rbac.ALL_CAPS - rbac.GRANTABLE_CAPS), + "descriptions": { + rbac.CREATE_PRIVATE_SPACE: "Create your own individual/private space", + rbac.CREATE_TEAM_SPACE: "Create a team (shared) space", + rbac.MANAGE_TEAM_SPACE: "Manage a team space's members, visibility, graphs, access rules", + rbac.INGEST: "Ingest data (also needs per-space write: membership or a space write access rule)", + rbac.RECOVER: "Recover stuck/errored ingest jobs", + rbac.READ_PRIVATE: "Read non-public content you're a member of", + rbac.SPARQL_ADMIN: "Run arbitrary SPARQL (admin-only, not delegatable)", + rbac.GRANT: "Grant/revoke capabilities to others (admin-only, not delegatable)", + }, + } + + +@router.get("/admin/capabilities/role", + dependencies=[Depends(require_scopes(["admin"]))], + summary="List capabilities granted to a role/group (admin only)") +async def role_capabilities( + user: Annotated[LoginUserIn, Depends(get_current_user)], + role: Annotated[str, Query(..., description="Role/group name, e.g. 'uk_collaborator'")], +): + if not await rbac.is_admin(_agent(user)): + raise HTTPException(403, "Admin/SuperAdmin role required") + return {"role": role, "grants": await rbac.list_role_grants(role)} + + +@router.post("/admin/capabilities/grant-role", + dependencies=[Depends(require_scopes(["admin"]))], + summary="Grant a capability to a whole role/group (admin only)", + description="Give every member of a role/group a delegatable capability — " + "e.g. grant 'ingest' or 'create_private_space' to a custom group " + "like 'uk_collaborator'. 'grant'/'sparql_admin' are not delegatable.") +async def grant_role_capability(body: RoleGrantIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + if not await rbac.is_admin(_agent(user)): + raise HTTPException(403, "Admin/SuperAdmin role required to grant capabilities") + if body.capability not in rbac.GRANTABLE_CAPS: + raise HTTPException(400, f"capability is not delegatable; valid: {sorted(rbac.GRANTABLE_CAPS)}") + await rbac.grant_role_capability(body.role, body.capability, _agent(user)) + return {"status": "granted", "role": body.role, "capability": body.capability} + + +@router.post("/admin/capabilities/revoke-role", + dependencies=[Depends(require_scopes(["admin"]))], + summary="Revoke a capability from a role/group (admin only)") +async def revoke_role_capability(body: RoleGrantIn, user: Annotated[LoginUserIn, Depends(get_current_user)]): + if not await rbac.is_admin(_agent(user)): + raise HTTPException(403, "Admin/SuperAdmin role required to revoke capabilities") + await rbac.revoke_role_capability(body.role, body.capability) + return {"status": "revoked", "role": body.role, "capability": body.capability} diff --git a/query_service/core/search.py b/query_service/core/search.py new file mode 100644 index 0000000..adb4eb1 --- /dev/null +++ b/query_service/core/search.py @@ -0,0 +1,195 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# DISCLAIMER: This software is provided "as is" without any warranty, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose, and non-infringement. +# ----------------------------------------------------------------------------- + +# @Author : Tek Raj Chhetri +# @Email : tekraj@mit.edu +# @File : search.py + +""" +Hybrid search: Postgres is a full-text **locator** index, Oxigraph is the source +of truth for the data. + +At ingest time each subject's literal text is indexed into `graph_search_index` +along with its named graph and owning space (workspace). A search query runs in +Postgres (fast, access-filtered by space visibility/membership) to LOCATE the +matching subjects, then the actual triples for that page of hits are fetched from +Oxigraph. + +Access model (identical to spaces): + * anonymous -> public-space subjects only + * logged-in -> public + the caller's member spaces (+ legacy/unmapped, authed) +""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional +from urllib.parse import quote + +import httpx + +from core.database import get_db_connection +from core.shared import get_oxigraph_auth +from core.graph_database_connection_manager import _get_endpoint +from core.provenance import query_provenance_jsonld +from core.spaces import get_space_for_graph + +logger = logging.getLogger(__name__) + + +async def _sparql_select(query: str) -> List[Dict[str, Any]]: + """Run a SPARQL SELECT and return its bindings (empty list on error).""" + try: + endpoint = _get_endpoint("get") + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(120.0, connect=15.0)) as client: + resp = await client.post( + endpoint, + data={"query": query}, + headers={"Accept": "application/sparql-results+json"}, + auth=auth, + ) + if resp.status_code != 200: + logger.warning("[search] SELECT failed (HTTP %s): %s", resp.status_code, (resp.text or "")[:400]) + return [] + return resp.json().get("results", {}).get("bindings", []) + except Exception as e: + logger.warning(f"[search] SELECT error: {e}", exc_info=True) + return [] + + +async def index_graph_subjects(named_graph_iri: str, delta_graph: Optional[str] = None) -> int: + """ + (Re)index the subjects of a named graph into the Postgres locator index. + + Each subject's searchable text is the concatenation of its literal objects, + read from Oxigraph. When `delta_graph` is given, only subjects touched by that + job are (re)indexed (their full text is still read from the target graph). + Best-effort — returns the number of subjects indexed (0 on failure). + """ + try: + if delta_graph: + q = f""" + SELECT ?s (GROUP_CONCAT(DISTINCT STR(?o); SEPARATOR=" ") AS ?text) + WHERE {{ + GRAPH <{delta_graph}> {{ ?s ?dp ?do }} + GRAPH <{named_graph_iri}> {{ ?s ?p ?o FILTER(isLiteral(?o)) }} + }} GROUP BY ?s + """ + else: + q = f""" + SELECT ?s (GROUP_CONCAT(DISTINCT STR(?o); SEPARATOR=" ") AS ?text) + WHERE {{ GRAPH <{named_graph_iri}> {{ ?s ?p ?o FILTER(isLiteral(?o)) }} }} + GROUP BY ?s + """ + rows = await _sparql_select(q) + if not rows: + return 0 + + space = await get_space_for_graph(named_graph_iri) + space_id = space["space_id"] if space else None + + import time + now = time.time() + records = [] + for b in rows: + subj = b.get("s", {}).get("value") + text = b.get("text", {}).get("value", "") + if subj and text.strip(): + records.append((named_graph_iri, space_id, subj, text, now)) + if not records: + return 0 + + async with get_db_connection() as conn: + await conn.executemany( + """ + INSERT INTO graph_search_index (named_graph_iri, space_id, subject, text, tsv, updated_at) + VALUES ($1, $2, $3, $4, to_tsvector('english', $4), $5) + ON CONFLICT (named_graph_iri, subject) DO UPDATE SET + space_id = EXCLUDED.space_id, + text = EXCLUDED.text, + tsv = EXCLUDED.tsv, + updated_at = EXCLUDED.updated_at + """, + records, + ) + logger.info(f"[search] Indexed {len(records)} subject(s) for {named_graph_iri}") + return len(records) + except Exception as e: + logger.warning(f"[search] Failed to index {named_graph_iri}: {e}", exc_info=True) + return 0 + + +async def reindex_graph_space(named_graph_iri: str, space_id: str) -> None: + """Point a graph's existing index rows at a (new) space — e.g. when a graph is + attached to a space after it was already indexed.""" + try: + async with get_db_connection() as conn: + await conn.execute( + "UPDATE graph_search_index SET space_id = $1 WHERE named_graph_iri = $2", + space_id, named_graph_iri, + ) + except Exception as e: + logger.warning(f"[search] Failed to reassign space for {named_graph_iri}: {e}", exc_info=True) + + +async def search(q: str, caller: Optional[str], space_slug: Optional[str] = None, + limit: int = 25, offset: int = 0) -> Dict[str, Any]: + """ + Locate matching subjects in Postgres (access-filtered), then fetch their triples + from Oxigraph. `caller` is the user email, or None for anonymous. + """ + # Build the access-filtered locator query. Access is authoritative from the + # spaces tables (joined live), so visibility flips need no reindex. + params: List[Any] = [q, caller] + where = [ + "i.tsv @@ plainto_tsquery('english', $1)", + "(s.visibility = 'public' OR m.member IS NOT NULL OR (i.space_id IS NULL AND $2 IS NOT NULL))", + ] + idx = 3 + if space_slug: + where.append(f"s.slug = ${idx}") + params.append(space_slug) + idx += 1 + limit_i, offset_i = idx, idx + 1 + params.extend([limit, offset]) + + sql = f""" + SELECT i.named_graph_iri, i.subject, i.text, + s.slug AS space_slug, s.visibility AS visibility, + ts_rank(i.tsv, plainto_tsquery('english', $1)) AS rank + FROM graph_search_index i + LEFT JOIN spaces s ON s.space_id = i.space_id + LEFT JOIN space_members m ON m.space_id = i.space_id AND m.member = $2 + WHERE {' AND '.join(where)} + ORDER BY rank DESC, i.subject + LIMIT ${limit_i} OFFSET ${offset_i} + """ + async with get_db_connection() as conn: + rows = await conn.fetch(sql, *params) + + hits = [] + for r in rows: + text = r["text"] or "" + hits.append({ + "subject": r["subject"], + "named_graph_iri": r["named_graph_iri"], + "space": r["space_slug"], + "visibility": r["visibility"] or ("legacy" if r["named_graph_iri"] else None), + "snippet": (text[:240] + "…") if len(text) > 240 else text, + }) + + # Fetch the located subjects' triples from Oxigraph (the source of truth). + data = None + if hits: + construct = "CONSTRUCT { ?s ?p ?o } WHERE { " + " UNION ".join( + f'{{ BIND(<{h["subject"]}> AS ?s) GRAPH <{h["named_graph_iri"]}> {{ ?s ?p ?o }} }}' + for h in hits + ) + " }" + data = await query_provenance_jsonld(construct) + + return {"query": q, "space": space_slug, "count": len(hits), "hits": hits, "data": data} diff --git a/query_service/core/security.py b/query_service/core/security.py index d6b7348..bcb8e15 100644 --- a/query_service/core/security.py +++ b/query_service/core/security.py @@ -21,7 +21,7 @@ import asyncio from typing import Annotated, List, Optional, Dict -from fastapi import Depends, HTTPException, status, WebSocket +from fastapi import Depends, HTTPException, status, WebSocket, Request from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from fastapi.security import OAuth2PasswordBearer from jose import ExpiredSignatureError, JWTError, jwt @@ -29,6 +29,7 @@ from core.configuration import load_environment from core.database import get_user +from core import jwks logger = logging.getLogger(__name__) @@ -48,11 +49,34 @@ def access_token_expire_minutes() -> int: return 30 -def create_access_token(email: str, scopes: List[str]) -> str: +def create_access_token( + email: str, + scopes: List[str], + *, + user_id: Optional[int] = None, + profile_id: Optional[int] = None, + roles: Optional[List[str]] = None, +) -> str: + """Mint a query_service access token. + + Claims are standardized to match usermanagement's v2 token shape + (``sub``/``scopes``/``profile_id``/``roles``/``auth_source``) so the token + is uniform across services. It is still signed with query_service's OWN + secret — per-service token isolation is preserved; a token minted here is + not accepted elsewhere. ``roles``/``profile_id`` are informational: query + authorization re-reads roles from the DB (see core.rbac), so a stale claim + cannot grant access. + """ expire = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta( minutes=access_token_expire_minutes() ) - jwt_data = {"sub": email, "exp": expire, "scopes": scopes} + jwt_data = {"sub": email, "exp": expire, "scopes": scopes, "auth_source": "password"} + if user_id is not None: + jwt_data["user_id"] = user_id + if profile_id is not None: + jwt_data["profile_id"] = profile_id + if roles is not None: + jwt_data["roles"] = roles encoded_jwt = jwt.encode(jwt_data, key=SECRET_KEY, algorithm=ALGORITHM) return encoded_jwt @@ -85,11 +109,30 @@ def decode_jwt(token: str): raise HTTPException(status_code=403, detail="Could not validate credentials") +def decode_token_any(token: str) -> dict: + """Decode a bearer token from either scheme, newest first: + + 1. Phase 2 SSO RS256 access token — verified against the issuer's JWKS + and required to carry ``aud == this service`` (see core.jwks). + 2. Legacy HS256 query_service token — verified with this service's own + secret (per-service isolation preserved). + + Returns the validated claims. Raises jose ``JWTError`` / + ``ExpiredSignatureError`` if neither scheme validates, so existing callers' + exception handling (401/403) keeps working unchanged. + """ + payload = jwks.verify_access_token(token) + if payload is not None: + return payload + # Fall back to the legacy HS256 token signed with our own secret. + return jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + + async def get_current_user( token: Annotated[str, Depends(oauth2_scheme)], ): try: - payload = decode_jwt(token) + payload = decode_token_any(token) email = payload.get("sub") if email is None: raise credentials_exception @@ -101,14 +144,59 @@ async def get_current_user( ) from e except JWTError as e: raise credentials_exception from e - user = await get_user(email=email) - if user is None: + # An OAuth caller's credential row is a SHELL with is_active=False (they have + # no usable password), so an active-only lookup 401s every Globus/ORCID/GitHub + # user despite a perfectly valid token. The token records how it was issued, so + # relax the filter only for those — on the password path is_active IS the + # deactivation switch and must keep being enforced. + user = await get_user(email=email, + include_inactive=payload.get("auth_source", "password") != "password") + # get_user returns False (not None) when there is no active user row, so check + # falsiness — otherwise `False` slips through as the "user" and downstream code + # (_agent, role lookup) breaks, yielding a misleading 403 instead of a 401. + if not user: raise credentials_exception return user +async def get_current_user_optional(request: Request): + """ + Return the authenticated user if a valid Bearer token is present, else None. + + Unlike get_current_user this NEVER raises on a missing/invalid token — it is + for endpoints that serve public resources anonymously but still want to know + the caller's identity when a token is supplied (e.g. public-space reads). + """ + auth = request.headers.get("authorization", "") + if not auth[:7].lower() == "bearer ": + return None + token = auth[7:].strip() + if not token: + return None + try: + payload = decode_token_any(token) + email = payload.get("sub") + if not email: + return None + # get_user returns False when no active user row; normalize to None so + # callers' truthiness/None checks behave (anonymous, not a bogus `False`). + # include_inactive for OAuth callers, as in get_current_user — otherwise a + # signed-in Globus user silently reads these endpoints as ANONYMOUS (e.g. + # list_spaces returning an empty list instead of their own spaces), which + # looks like "you have no data" rather than an auth failure. + return (await get_user( + email=email, + include_inactive=payload.get("auth_source", "password") != "password", + )) or None + except (ExpiredSignatureError, JWTError, Exception): + return None + + def verify_scopes(required_scopes: List[str], token: str) -> bool: - decoded_token = decode_jwt(token) + try: + decoded_token = decode_token_any(token) + except (ExpiredSignatureError, JWTError): + raise HTTPException(status_code=403, detail="Could not validate credentials") token_scopes = decoded_token.get("scopes", []) return all(scope in token_scopes for scope in required_scopes) @@ -186,9 +274,10 @@ async def authenticate_websocket(websocket: WebSocket, required_scopes: Optional logger.warning("No JWT token provided in WebSocket connection") return None - # Decode and validate JWT token (same logic as get_current_user) + # Decode and validate JWT token (same logic as get_current_user): + # RS256 SSO access token (aud-checked) first, then legacy HS256. try: - payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) + payload = decode_token_any(token) except ExpiredSignatureError: logger.warning("JWT token has expired") return None @@ -209,12 +298,15 @@ async def authenticate_websocket(websocket: WebSocket, required_scopes: Optional logger.warning(f"Insufficient scopes. Required: {required_scopes}, Token has: {token_scopes}") return None - # Get user from database (same as get_current_user) - user = await get_user(email=email) - if user is None: + # Get user from database (same as get_current_user, including the OAuth + # shell allowance — a websocket caller is the same identity as an HTTP one). + user = await get_user(email=email, + include_inactive=payload.get("auth_source", "password") != "password") + # get_user returns False (not None) when no active user row — check falsiness. + if not user: logger.warning(f"User not found for email: {email}") return None - + return user except Exception as e: diff --git a/query_service/core/shared.py b/query_service/core/shared.py index 2e8f8ac..200cf66 100644 --- a/query_service/core/shared.py +++ b/query_service/core/shared.py @@ -353,7 +353,7 @@ def chunk_ttl_to_named_graphs(ttl_str: str, named_graph_uri: str = "https://brai return chunks -def named_graph_metadata(named_graph_url, description): +def named_graph_metadata(named_graph_url, description, agent_uri=None): """ Generates metadata for a named graph using the PROV and DCTERMS ontologies. @@ -386,6 +386,13 @@ def named_graph_metadata(named_graph_url, description): g.add((prov_entity, RDF.type, PROV.Entity)) g.add((prov_entity,PROV.generatedAtTime, Literal(created_At, datatype=XSD.dateTime))) g.add((prov_entity,DCTERMS.description, Literal(description, datatype=XSD.string))) + # Record who registered the graph directly on the registry entry (PROV-O). + # This keeps registration provenance in one place (the registry graph) rather + # than duplicating it as a separate activity in the provenance graph. + if agent_uri: + agent = URIRef(agent_uri) + g.add((agent, RDF.type, PROV.Agent)) + g.add((prov_entity, PROV.wasAttributedTo, agent)) named_graph_metadata = convert_ttl_to_named_graph( ttl_str=g.serialize(format='turtle'), named_graph_uri="https://brainkb.org/metadata/named-graph" diff --git a/query_service/core/spaces.py b/query_service/core/spaces.py new file mode 100644 index 0000000..d5757ee --- /dev/null +++ b/query_service/core/spaces.py @@ -0,0 +1,523 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# DISCLAIMER: This software is provided "as is" without any warranty, +# express or implied, including but not limited to the warranties of +# merchantability, fitness for a particular purpose, and non-infringement. +# ----------------------------------------------------------------------------- + +# @Author : Tek Raj Chhetri +# @Email : tekraj@mit.edu +# @File : spaces.py + +""" +Spaces: owner-controlled containers of named graphs with private/public +visibility and team membership. + +A space is a sovereign, IRI-addressable container (think Solid-style pod) that a +user or team owns. Named graphs belong to a space; ingestion into a graph is +allowed only for the owning space's owner/editors, while reads are allowed to +members always and to ANYONE (even anonymous) when the space is public. + +Storage is hybrid (see SPACES_MODEL.md): + * Postgres (spaces / space_members / space_graphs) is the enforcement source of + truth — fast per-request authorization. + * A best-effort RDF mirror in the spaces metadata graph makes space manifests + portable/decentralizable and queryable via SPARQL. +""" + +from __future__ import annotations + +import logging +import time +import uuid +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import quote + +import httpx +from rdflib import Graph, Literal, Namespace, URIRef, RDF +from rdflib.namespace import DCTERMS + +from core.database import get_db_connection +from core.shared import get_oxigraph_auth +from core.graph_database_connection_manager import _get_endpoint +from core.provenance import agent_ref, BRAINKB, PROV + +logger = logging.getLogger(__name__) + +SPACES_METADATA_GRAPH = "https://brainkb.org/metadata/spaces/" +SPACE_BASE = "https://brainkb.org/space/" +SCHEMA = Namespace("https://schema.org/") + +ROLES = ("owner", "editor", "viewer") +READ_ROLES = ("owner", "editor", "viewer") +WRITE_ROLES = ("owner", "editor") + + +def space_iri(slug: str) -> str: + return f"{SPACE_BASE}{quote(str(slug), safe='')}" + + +# --------------------------------------------------------------------------- +# Postgres CRUD +# --------------------------------------------------------------------------- + +async def create_space(slug: str, name: str, description: Optional[str], owner: str, + visibility: str = "private", space_type: str = "individual") -> Dict[str, Any]: + """Create a space and register the owner as a member with role 'owner'.""" + if visibility not in ("private", "public"): + raise ValueError("visibility must be 'private' or 'public'") + if space_type not in ("individual", "team"): + raise ValueError("space_type must be 'individual' or 'team'") + space_id = uuid.uuid4().hex + now = time.time() + async with get_db_connection() as conn: + async with conn.transaction(): + await conn.execute( + """ + INSERT INTO spaces (space_id, slug, name, description, owner, visibility, space_type, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $8) + """, + space_id, slug, name, description, owner, visibility, space_type, now, + ) + await conn.execute( + """ + INSERT INTO space_members (space_id, member, role, added_at) + VALUES ($1, $2, 'owner', $3) + """, + space_id, owner, now, + ) + return await get_space(slug) + + +async def get_space(slug: str) -> Optional[Dict[str, Any]]: + async with get_db_connection() as conn: + row = await conn.fetchrow("SELECT * FROM spaces WHERE slug = $1", slug) + if not row: + return None + members = await conn.fetch( + "SELECT member, role FROM space_members WHERE space_id = $1 ORDER BY role, member", + row["space_id"], + ) + graphs = await conn.fetch( + "SELECT named_graph_iri FROM space_graphs WHERE space_id = $1 ORDER BY named_graph_iri", + row["space_id"], + ) + return { + "space_id": row["space_id"], + "slug": row["slug"], + "name": row["name"], + "description": row["description"], + "owner": row["owner"], + "visibility": row["visibility"], + "space_type": row.get("space_type", "individual"), + "iri": space_iri(row["slug"]), + "members": [{"member": m["member"], "role": m["role"]} for m in members], + "graphs": [g["named_graph_iri"] for g in graphs], + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + + +async def get_space_for_graph(named_graph_iri: str) -> Optional[Dict[str, Any]]: + """Return the space that owns a named graph, or None if the graph is unmapped.""" + async with get_db_connection() as conn: + row = await conn.fetchrow( + """ + SELECT s.* FROM spaces s + JOIN space_graphs g ON g.space_id = s.space_id + WHERE g.named_graph_iri = $1 + """, + named_graph_iri, + ) + if not row: + return None + return { + "space_id": row["space_id"], "slug": row["slug"], "name": row["name"], + "owner": row["owner"], "visibility": row["visibility"], + "space_type": row.get("space_type", "individual"), + } + + +async def hidden_graphs_for(member: Optional[str]) -> set: + """ + Return the set of named-graph IRIs the caller must NOT see in listings: graphs + belonging to a PRIVATE space the caller is not a member of. Public-space graphs + and legacy (unmapped) graphs are never hidden. Anonymous callers (member=None) + have every private-space graph hidden. + """ + async with get_db_connection() as conn: + rows = await conn.fetch( + """ + SELECT g.named_graph_iri + FROM space_graphs g + JOIN spaces s ON s.space_id = g.space_id + WHERE s.visibility = 'private' + AND NOT EXISTS ( + SELECT 1 FROM space_members m + WHERE m.space_id = s.space_id AND m.member = $1 + ) + """, + member, + ) + return {r["named_graph_iri"] for r in rows} + + +async def member_role(space_id: str, member: Optional[str]) -> Optional[str]: + if not member: + return None + async with get_db_connection() as conn: + return await conn.fetchval( + "SELECT role FROM space_members WHERE space_id = $1 AND member = $2", + space_id, member, + ) + + +async def list_visible_spaces(member: Optional[str]) -> List[Dict[str, Any]]: + """Spaces the caller may see, each annotated with THIS caller's permission: + - your_role: their space-membership role ('owner'|'editor'|'viewer') or None + - is_owner: whether they own the space + - access: how it is available to them — 'owner' | 'member' | 'public' + - can_write: whether their space role permits writing/ingest (owner/editor). + NOTE: an actual ingest ALSO requires the caller's global role to + grant the `ingest` capability and to pass any per-space access + rules — this flag reflects only the space-membership gate. + Anonymous callers (member=None) see only public spaces (your_role None).""" + async with get_db_connection() as conn: + if member: + rows = await conn.fetch( + """ + SELECT DISTINCT s.slug, s.name, s.description, s.owner, s.visibility, + s.space_type, s.created_at, m.role AS your_role + FROM spaces s + LEFT JOIN space_members m ON m.space_id = s.space_id AND m.member = $1 + WHERE s.visibility = 'public' OR m.member IS NOT NULL + ORDER BY s.created_at DESC + """, + member, + ) + else: + rows = await conn.fetch( + """ + SELECT s.slug, s.name, s.description, s.owner, s.visibility, + s.space_type, s.created_at, NULL::text AS your_role + FROM spaces s WHERE s.visibility = 'public' + ORDER BY s.created_at DESC + """, + ) + out: List[Dict[str, Any]] = [] + for r in rows: + your_role = r["your_role"] + is_owner = bool(member) and r["owner"] == member + access = "owner" if is_owner else ("member" if your_role else "public") + out.append({ + "slug": r["slug"], "name": r["name"], "description": r["description"], + "owner": r["owner"], "visibility": r["visibility"], + "space_type": r["space_type"], "iri": space_iri(r["slug"]), + "created_at": r["created_at"], + "your_role": your_role, + "is_owner": is_owner, + "access": access, + "can_write": your_role in ("owner", "editor"), + }) + return out + + +async def add_member(space_id: str, member: str, role: str) -> None: + if role not in ROLES: + raise ValueError(f"role must be one of {ROLES}") + async with get_db_connection() as conn: + await conn.execute( + """ + INSERT INTO space_members (space_id, member, role, added_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (space_id, member) DO UPDATE SET role = EXCLUDED.role + """, + space_id, member, role, time.time(), + ) + + +async def remove_member(space_id: str, member: str) -> None: + async with get_db_connection() as conn: + await conn.execute( + "DELETE FROM space_members WHERE space_id = $1 AND member = $2 AND role <> 'owner'", + space_id, member, + ) + + +async def set_visibility(slug: str, visibility: str) -> None: + if visibility not in ("private", "public"): + raise ValueError("visibility must be 'private' or 'public'") + async with get_db_connection() as conn: + await conn.execute( + "UPDATE spaces SET visibility = $1, updated_at = $2 WHERE slug = $3", + visibility, time.time(), slug, + ) + + +class GraphAlreadyBound(Exception): + """A named graph is already bound to a DIFFERENT space. + + named_graph_iri is globally UNIQUE in space_graphs, so a graph lives in exactly + one space. Attaching one that another space already holds cannot succeed, and + must not look like it did. + """ + + def __init__(self, named_graph_iri: str, slug: str): + self.named_graph_iri = named_graph_iri + self.slug = slug + super().__init__( + f"named graph '{named_graph_iri}' is already registered to space " + f"'{slug}'; a graph can belong to only one space" + ) + + +async def attach_graph(space_id: str, named_graph_iri: str) -> bool: + """Bind a named graph to a space. + + Returns True when newly attached, False when it was already attached to THIS + space (idempotent re-registration). Raises GraphAlreadyBound when another space + holds it — previously that case silently did nothing while the caller received + a success response. + """ + async with get_db_connection() as conn: + row = await conn.fetchrow( + """ + INSERT INTO space_graphs (space_id, named_graph_iri, added_at) + VALUES ($1, $2, $3) + ON CONFLICT (named_graph_iri) DO NOTHING + RETURNING id + """, + space_id, named_graph_iri, time.time(), + ) + if row is None: + # The insert was a no-op: the graph is already bound. Determine whether + # it is bound HERE (fine) or to another space (a conflict we must + # report). Crucially, do NOT touch the search index in the latter case: + # search access-filtering keys off graph_search_index.space_id, so + # repointing it would let this space's visibility/membership govern + # another space's graph while space_graphs still says otherwise. + owner = await conn.fetchrow( + """ + SELECT s.space_id, s.slug FROM space_graphs g + JOIN spaces s ON s.space_id = g.space_id + WHERE g.named_graph_iri = $1 + """, + named_graph_iri, + ) + if owner is not None and owner["space_id"] != space_id: + raise GraphAlreadyBound(named_graph_iri, owner["slug"]) + + # Point any already-indexed rows for this graph at the space so search + # access-filtering picks up the (new) workspace immediately. Done inline + # (not via core.search) to avoid an import cycle. Only reached when the + # graph genuinely belongs to this space. + await conn.execute( + "UPDATE graph_search_index SET space_id = $1 WHERE named_graph_iri = $2", + space_id, named_graph_iri, + ) + return row is not None + + +# --------------------------------------------------------------------------- +# Authorization (enforcement) +# --------------------------------------------------------------------------- + +ACTIONS = ("read", "write", "manage") +RULE_SUBJECTS = ("global_role", "member", "space_role") +_SPACE_ROLE_RANK = {"viewer": 1, "editor": 2, "owner": 3} + + +async def add_access_rule(space_id: str, action: str, subject_type: str, + subject_value: str, created_by: str) -> None: + if action not in ACTIONS: + raise ValueError(f"action must be one of {ACTIONS}") + if subject_type not in RULE_SUBJECTS: + raise ValueError(f"subject_type must be one of {RULE_SUBJECTS}") + if subject_type == "space_role" and subject_value not in _SPACE_ROLE_RANK: + raise ValueError("space_role subject_value must be viewer/editor/owner") + async with get_db_connection() as conn: + await conn.execute( + """ + INSERT INTO space_access_rules (space_id, action, subject_type, subject_value, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (space_id, action, subject_type, subject_value) DO NOTHING + """, + space_id, action, subject_type, subject_value, created_by, time.time(), + ) + + +async def remove_access_rule(space_id: str, rule_id: int) -> None: + async with get_db_connection() as conn: + await conn.execute( + "DELETE FROM space_access_rules WHERE id = $1 AND space_id = $2", rule_id, space_id + ) + + +async def list_access_rules(space_id: str, action: Optional[str] = None) -> List[Dict[str, Any]]: + async with get_db_connection() as conn: + if action: + rows = await conn.fetch( + "SELECT id, action, subject_type, subject_value FROM space_access_rules WHERE space_id=$1 AND action=$2 ORDER BY id", + space_id, action) + else: + rows = await conn.fetch( + "SELECT id, action, subject_type, subject_value FROM space_access_rules WHERE space_id=$1 ORDER BY action, id", + space_id) + return [dict(r) for r in rows] + + +async def matches_access_rule(space_id: str, action: str, email: Optional[str]) -> bool: + """True iff the caller matches at least one access rule for (space, action). + Pure rule match — no owner/admin bypass, no 'no rules' default.""" + from core import rbac + rules = await list_access_rules(space_id, action) + if not rules: + return False + roles = await rbac.active_roles(email) + srole = await member_role(space_id, email) + srank = _SPACE_ROLE_RANK.get(srole or "", 0) + for r in rules: + st, sv = r["subject_type"], r["subject_value"] + if st == "global_role" and sv in roles: + return True + if st == "member" and email and sv == email: + return True + if st == "space_role" and srank >= _SPACE_ROLE_RANK.get(sv, 99): + return True + return False + + +async def space_action_permitted(space: Dict[str, Any], action: str, email: Optional[str]) -> bool: + """ + Fine-grained per-space check for read/write. Returns True if the caller may do + `action` in this space under the space's access rules. + + - Owner and global Admin/SuperAdmin always pass (no lockout). + - No rules for the action -> True (the endpoint's normal capability / membership + / visibility gates still apply separately). + - Rules present -> caller must match at least one. + """ + from core import rbac + space_id = space["space_id"] + if email and await member_role(space_id, email) == "owner": + return True + if await rbac.is_admin(email): + return True + if not await list_access_rules(space_id, action): + return True + return await matches_access_rule(space_id, action, email) + + +async def can_write_space(space: Dict[str, Any], email: Optional[str]) -> Tuple[bool, str]: + """Whether ``email`` may WRITE (ingest) into ``space``. Write is GRANTED by any + of: + * global Admin/SuperAdmin, + * owner/editor membership of the space, + * a matching **write** access rule — by ``global_role`` (a group/role), + ``member`` (an email), or ``space_role``. + + This is what lets an admin hand a whole group ingest access to a team space + without adding every user as a member: add a write rule with + ``subject_type=global_role`` (e.g. ``Lab Member``). The caller still needs the + ``INGEST`` capability (a write-capable role) — enforced separately at the + endpoint — so a read-only group can't ingest even with a write rule.""" + from core import rbac + space_id = space["space_id"] + if await rbac.is_admin(email): + return True, "admin" + role = await member_role(space_id, email) + if role in WRITE_ROLES: + return True, f"member ({role})" + if await matches_access_rule(space_id, "write", email): + return True, "space write access rule (group/role/member)" + return False, ("requires owner/editor membership, or a space write access rule " + "granting your group/role write (ask an admin)") + + +async def authorize(named_graph_iri: str, member: Optional[str], need: str) -> Tuple[bool, str]: + """ + Decide whether ``member`` (a user email, or None if anonymous) may read/write + the given named graph, based on the owning space's visibility and membership. + + Backward-compat: a graph not mapped to any space is treated as legacy — access + falls through to whatever scope check the endpoint already applied (returns + allowed=True here). Only space-mapped graphs are governed by space ACL. + """ + space = await get_space_for_graph(named_graph_iri) + if space is None: + return True, "legacy (unmapped) graph — governed by endpoint scope only" + + if need == "read": + if space["visibility"] == "public": + return True, "public space" + role = await member_role(space["space_id"], member) + if role in READ_ROLES: + return True, f"member ({role})" + return False, "private space — membership required" + + if need == "write": + role = await member_role(space["space_id"], member) + if role in WRITE_ROLES: + return True, f"member ({role})" + return False, "write requires owner/editor membership of the space" + + return False, f"unknown access mode: {need}" + + +# --------------------------------------------------------------------------- +# RDF mirror (best-effort, for portability / decentralization) +# --------------------------------------------------------------------------- + +def _space_manifest_graph(space: Dict[str, Any]) -> Graph: + g = Graph() + g.bind("brainkb", BRAINKB) + g.bind("prov", PROV) + g.bind("dcterms", DCTERMS) + g.bind("schema", SCHEMA) + s = URIRef(space_iri(space["slug"])) + g.add((s, RDF.type, BRAINKB.Space)) + g.add((s, BRAINKB.slug, Literal(space["slug"]))) + g.add((s, SCHEMA.name, Literal(space["name"]))) + if space.get("description"): + g.add((s, DCTERMS.description, Literal(space["description"]))) + g.add((s, BRAINKB.visibility, Literal(space["visibility"]))) + g.add((s, BRAINKB.owner, agent_ref(space["owner"]))) + for m in space.get("members", []): + pred = {"owner": BRAINKB.owner, "editor": BRAINKB.editor, "viewer": BRAINKB.viewer}[m["role"]] + g.add((s, pred, agent_ref(m["member"]))) + for giri in space.get("graphs", []): + g.add((s, BRAINKB.containsGraph, URIRef(giri))) + return g + + +async def mirror_space_to_rdf(space: Dict[str, Any]) -> bool: + """Upsert a space's manifest into the spaces metadata graph via SPARQL Update. + Best-effort — a mirror failure never fails the enforcing Postgres operation.""" + try: + s = space_iri(space["slug"]) + triples = _space_manifest_graph(space).serialize(format="nt") + update = ( + f"DELETE {{ GRAPH <{SPACES_METADATA_GRAPH}> {{ <{s}> ?p ?o }} }} " + f"WHERE {{ GRAPH <{SPACES_METADATA_GRAPH}> {{ <{s}> ?p ?o }} }} ; " + f"INSERT DATA {{ GRAPH <{SPACES_METADATA_GRAPH}> {{ {triples} }} }}" + ) + endpoint = _get_endpoint("post") # .../update + auth = get_oxigraph_auth() + async with httpx.AsyncClient(timeout=httpx.Timeout(60.0, connect=15.0)) as client: + resp = await client.post(endpoint, data={"update": update}, auth=auth) + if resp.status_code in (200, 204): + return True + logger.warning("[spaces] RDF mirror failed (HTTP %s): %s", resp.status_code, (resp.text or "")[:400]) + return False + except Exception as e: + logger.warning(f"[spaces] Error mirroring space to RDF: {e}", exc_info=True) + return False + + +async def construct_space_graphs(space: Dict[str, Any]) -> str: + """SPARQL CONSTRUCT returning all triples across the space's named graphs.""" + graphs = space.get("graphs", []) + if not graphs: + return "CONSTRUCT {} WHERE {}" + unions = " UNION ".join(f"{{ GRAPH <{g}> {{ ?s ?p ?o }} }}" for g in graphs) + return f"CONSTRUCT {{ ?s ?p ?o }} WHERE {{ {unions} }}" diff --git a/query_service/docs/auth/flow-a.png b/query_service/docs/auth/flow-a.png new file mode 100644 index 0000000..ccb5d57 Binary files /dev/null and b/query_service/docs/auth/flow-a.png differ diff --git a/query_service/docs/auth/flow-b.png b/query_service/docs/auth/flow-b.png new file mode 100644 index 0000000..20c8411 Binary files /dev/null and b/query_service/docs/auth/flow-b.png differ diff --git a/query_service/docs/auth/flow-c.png b/query_service/docs/auth/flow-c.png new file mode 100644 index 0000000..412417e Binary files /dev/null and b/query_service/docs/auth/flow-c.png differ diff --git a/readme.md b/readme.md index 1ac77a9..a46b367 100644 --- a/readme.md +++ b/readme.md @@ -47,7 +47,19 @@ Once started, services are accessible at: - **API Token Manager (Django)**: `http://localhost:8000/` - Once you register JWT user you need to activate it using token manager. You can also assign permission. - **Query Service (FastAPI)**: `http://localhost:8010/` - - Now supports ingestion than just querying. + - Supports querying **and** ingestion of the knowledge graphs. + - Native W3C PROV-O provenance in the graph database, with triple-level delta + tracking (per-job delta graphs + query/compare endpoints). + - **Spaces**: team-owned, private/public containers of named graphs — keep data + private to members or publish it publicly (anonymous read). Per-endpoint JWT + scopes (`read`/`write`/`admin`); role-based authorization (see + `query_service/RBAC_MODEL.md`). Accepts both SSO (RS256/JWKS) and legacy + HS256 tokens. + - **Search**: hybrid full-text search — Postgres locator index (aware of + workspace + visibility) finds subjects, data is fetched from Oxigraph. Results + are access-filtered (anonymous sees public only). + - See `query_service/README.md`, `query_service/PROVENANCE_MODEL.md`, and + `query_service/SPACES_MODEL.md` for details. - **ML Service (FastAPI)**: `http://localhost:8007/` - Integrates StructSense (multi-agent NER + structured-resource extraction). - Hosts **SynthScholar** at `/api/synth-scholar/*` — PRISMA-guided literature @@ -57,7 +69,26 @@ Once started, services are accessible at: optional API keys (OpenRouter, NCBI, Semantic Scholar, CORE). - **Oxigraph SPARQL**: `http://localhost:7878/` (password protected) graph database - **pgAdmin**: `http://localhost:5051/` -- **User management service**: http://localhost:8004 +- **User management service (FastAPI)**: `http://localhost:8004` + - Canonical **identity** service: user profiles, roles/RBAC, and OAuth sign-in + (Globus / ORCID / GitHub). One canonical user; the credential row is linked to + the profile (identity unification). + - **Single sign-on** issuer (RS256 + JWKS): one login mints a refresh token, + exchanged for narrow per-service access tokens (`aud=`) that each + service verifies via `/.well-known/jwks.json`. A token for one service can't + be replayed against another. Legacy per-service HS256 tokens still work. + - Admins can activate users and assign roles/groups. See + `query_service/AUTH_UNIFICATION.md` and `usermanagement_service/README.md`. + +## Authentication + +BrainKB is moving to a single sign-on model — usermanagement is the sole token +issuer and each service verifies audience-scoped RS256 tokens against its JWKS, +while legacy per-service HS256 tokens remain accepted during migration. Web +sign-in returns both an access and a refresh token so the UI renews silently +(`USERMANAGEMENT_WEB_SESSION_TTL_MIN` / `USERMANAGEMENT_WEB_REFRESH_TTL_MIN`). +The full design, phases, and deployment env are in +[query_service/AUTH_UNIFICATION.md](query_service/AUTH_UNIFICATION.md). ## Documentation diff --git a/usermanagement_service/README.md b/usermanagement_service/README.md new file mode 100644 index 0000000..420fbbe --- /dev/null +++ b/usermanagement_service/README.md @@ -0,0 +1,313 @@ +# BrainKB User Management Service + +A modern, scalable user management service for neuroscience knowledge bases built with FastAPI, SQLAlchemy ORM, and PostgreSQL. + +## 🚀 Features + +- **🔐 JWT Authentication** - Secure token-based authentication +- **👥 User Profiles** - Comprehensive user profile management +- **📊 Activity Tracking** - Automatic logging of user activities +- **🎯 Contribution Management** - Track and manage user contributions +- **🏷️ Role-Based Access Control** - Multiple user roles and permissions +- **📈 Analytics** - User statistics and activity analytics +- **🔒 Security** - Input validation, SQL injection protection, XSS prevention + +## 🏗️ Architecture + +### Tech Stack +- **FastAPI** - Modern, fast web framework +- **SQLAlchemy ORM** - Type-safe database operations +- **PostgreSQL** - Robust relational database +- **Pydantic** - Data validation and serialization +- **JWT** - Secure authentication + +### Key Components +``` +core/ +├── models/ +│ ├── user.py # Pydantic models for API +│ └── database_models.py # SQLAlchemy ORM models +├── user_database.py # User-specific database operations +├── routers/ +│ ├── user_management.py # User management endpoints +│ └── jwt_auth.py # Authentication endpoints +├── security.py # JWT and password utilities +├── configuration.py # Environment configuration +└── main.py # FastAPI application +``` + +## 🚀 Quick Start + +### Prerequisites +- Python 3.8+ +- PostgreSQL 12+ + +### Installation + +1. **Clone and install** +```bash +git clone +cd usermanagement_service +pip install -r requirements.txt +``` + +2. **Configure environment** +```bash +cp .env.example .env +# Edit .env with your database and JWT settings +``` + +3. **Start the service** +```bash +uvicorn core.main:app --reload +``` + +### Docker Setup +```bash +# Start with Docker Compose +docker-compose -f docker-compose-postgres.yml up -d + +# Or build and run +docker build -t brainkb-user-service . +docker run -p 8000:8000 brainkb-user-service +``` + +## 📚 API Documentation + +Once running, visit: +- **Interactive API Docs**: http://localhost:8000/docs +- **ReDoc Documentation**: http://localhost:8000/redoc + +### Key Endpoints + +End-user sign-up happens automatically on first OAuth callback (see +*Configuring the Admin role* below) — there is **no self-registration**; users are +created on first OAuth login. Password login is at `/api/login` (`/api/token` is a +deprecated alias) for accounts that already exist. + +- `POST /api/login` - Legacy HS256 password login (mints a v2 JWT); `/api/token` = deprecated alias +- `GET /api/auth/providers` - List OAuth providers + which are configured +- `GET /api/auth/{provider}/login` - Start OAuth flow (returns `authorize_url`) +- `GET /api/auth/{provider}/callback` - OAuth callback; auto-creates profile + linked credential + default `Curator` role on first sign-in + +**Single sign-on (RS256 + JWKS)** — usermanagement is the single token issuer. +One login mints a short-lived **refresh** token; clients **exchange** it for narrow +per-service **access** tokens (`aud=`), which each service verifies via +the published JWKS. A token minted for one service can't be replayed against +another. See `../query_service/AUTH_UNIFICATION.md`. + +- `GET /.well-known/jwks.json` - Public keys for verifying SSO tokens +- `POST /api/auth/login` - `{email, password}` → refresh token (aud `brainkb-auth`) +- `POST /api/auth/exchange` - Bearer refresh + `{audience}` → per-service access token +- `POST /api/auth/cli/start` - `{provider}` → authorize URL for a CLI/skill OAuth + login (paste-code). The provider callback shows a short one-time code (minimal + page, no SPA) instead of redirecting to the frontend. +- `POST /api/auth/cli/exchange` - `{code}` → SSO refresh token (single-use). Lets + the MCP/skill complete a Globus/ORCID/GitHub login without the web UI. + +The signing key is auto-provisioned at container start (persisted on the +`./secrets` volume) unless `USERMANAGEMENT_JWT_PRIVATE_KEY_PEM`/`_FILE` is set. +This service also accepts SSO access tokens minted for `aud=usermanagement` on its +own protected routes (alongside the legacy v2 token). +- `GET /api/users/profile` - Get user profile +- `POST /api/users/profile` - Create/update profile +- `GET /api/users/activities` - Get user activities +- `POST /api/users/contributions` - Create contribution +- `GET /api/users/roles` - Get user roles +- `POST /api/users/roles` - Assign role + +### Admin: roles, permissions & moderation (`/api/admin`, Admin/SuperAdmin) + +- `GET/POST /api/admin/roles`, `PUT /api/admin/roles/{id}` — list/create custom + roles/groups (e.g. `uk_collaborator`). +- `GET/POST /api/admin/permissions` — list all permissions and **add new ones** + (`{name, resource, action, description}`); attach to roles via + `PUT /api/admin/roles/{id}/permissions`. +- `POST /api/admin/users/{profile_id}/roles`, `DELETE .../roles/{role}` — assign/ + remove a user's role. **Assigning/removing the `Admin` role is SuperAdmin-only** + (hierarchy: SuperAdmin > Admin); the `SuperAdmin` role is protected. +- `POST /api/admin/users/{profile_id}/ban` + `DELETE …/ban` — ban/unban. **Banning + an Admin is SuperAdmin-only.** Banning is the removal mechanism. +- `POST /api/admin/users/{activate,deactivate}` — toggle login access. + +**No hard deletion.** `DELETE /api/admin/users/{id}` is disabled (returns `405`) — +accounts are **banned** (reversible, preserves provenance/audit history), never +deleted. + +## 🔑 OAuth provider setup (Globus / ORCID / GitHub) + +BrainKB uses the **Authorization Code + PKCE** flow. The backend builds its +callback (redirect URI) as: + +``` +${USERMANAGEMENT_PUBLIC_BASE_URL}/api/auth/{provider}/callback +``` + +so for **Globus** the redirect URL to register is: + +| Environment | Redirect URL to register | +|---|---| +| Local | `http://localhost:8004/api/auth/globus/callback` | +| Deployment | `https:///api/auth/globus/callback` | + +It must **exactly** match `${USERMANAGEMENT_PUBLIC_BASE_URL}/api/auth/globus/callback`. +One app can register **both** URLs. This is the **usermanagement backend** callback +(`:8004`) — not the frontend and not query_service. The web login and the +CLI/skill paste-code login (`/api/auth/cli/start`) use the **same** callback, so +only this one redirect is needed. (`USERMANAGEMENT_FRONTEND_CALLBACK_URL` is where +the browser is sent *after* a web login; it is not registered with the provider.) + +> ⚠️ **App type matters.** A Globus **Service Account** (client-credentials) app +> has no redirect and **cannot** do user sign-in. Register a Globus app that +> **supports redirect URLs** (developers.globus.org → your Project → *Add an app* → +> a portal/web-app registration, not a service account) and add the redirect +> URL(s) above. Use that app's Client UUID + secret. + +Configure in `.env`: + +``` +GLOBUS_CLIENT_ID= +GLOBUS_CLIENT_SECRET= +USERMANAGEMENT_PUBLIC_BASE_URL=http://localhost:8004 # local +# USERMANAGEMENT_PUBLIC_BASE_URL=https://api.yourhost.org # deploy (public HTTPS) +# GITHUB_CLIENT_ID / GITHUB_CLIENT_SECRET, ORCID_CLIENT_ID / ORCID_CLIENT_SECRET similarly +``` + +- Scopes requested at authorize time: `openid email profile` (standard OIDC). If a + provider rejects the authorize, add those scopes to the app. +- Globus requires **HTTPS** redirects for non-localhost (localhost may be `http`). +- Verify: `GET /api/auth/providers` shows `globus: configured=true`, and + `POST /api/auth/cli/start {"provider":"globus"}` returns an `authorize_url` + pointing at `auth.globus.org` with your `redirect_uri`. + +## 🎯 User Roles + +### Content Contribution +- **Submitter** - Upload primary content +- **Annotator** - Add metadata and tags +- **Mapper** - Align concepts to ontologies +- **Curator** - Review and edit submissions + +### Quality Control +- **Reviewer** - Evaluate content quality +- **Validator** - Check schema compliance +- **Conflict Resolver** - Handle contradictions + +### Knowledge Management +- **Knowledge Contributor** - Add domain knowledge +- **Evidence Tracer** - Link supporting evidence +- **Provenance Tracker** - Track source metadata + +### Community Management +- **Moderator** - Oversee discussions +- **Ambassador** - Community outreach + +## 👑 Configuring the Admin role + +The `Admin` role is special — only admins can assign roles to other users via +`POST /api/admin/users/{profile_id}/roles`. That creates a chicken-and-egg +problem for the *first* admin, solved by env-var bootstrap. + +### 1. First admin (bootstrap, env-var) + +In the backend's `.env` (project root, e.g. `BrainKB/.env`): + +```env +USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS=you@example.com,colleague@example.com +``` + +Comma-separated; whitespace around emails is trimmed. Restart the service — +on every startup `core/bootstrap.py::promote_bootstrap_superadmins()` runs and: + +1. **Email already has a `UserProfile`** → assigns both `Admin` and + `SuperAdmin` (idempotent, safe to re-run). +2. **Email has not signed in yet** → no profile to update, but the + `require_admin` dependency honors the env allowlist as a fallback. The + user can sign in via OAuth, which creates their profile, and the next + restart persists the roles. + +Bootstrap also seeds baseline roles and grants both `Admin` and `SuperAdmin` +every permission in the registry, so a fresh admin has full access +immediately. The `SuperAdmin` role is protected — accounts holding it cannot +be banned, deleted, or have that role stripped via the admin UI/API. + +Verify: + +```bash +# After the user has signed in, with their JWT in $TOKEN: +curl -s -H "Authorization: Bearer $TOKEN" http://localhost:8004/api/users/me | jq .roles +# → ["Admin", ...] +``` + +### 2. Subsequent admins (UI) + +Once at least one admin exists, role management happens through the BrainKB +UI's admin surface: + +1. Sign in as an existing admin and visit `/admin/users`. +2. Search by name, email, or ORCID. +3. In the *Roles* column, pick **Admin** from the *+ add role* dropdown. + +Demote: click the `Admin ✕` chip on the user's row. + +Programmatic equivalent (any admin's JWT): + +```bash +curl -X POST http://localhost:8004/api/admin/users/{profile_id}/roles \ + -H "Authorization: Bearer $ADMIN_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{"role": "Admin", "is_active": true}' +``` + +See `UI_INTEGRATION.md` for the full set of admin endpoints. + +## 📊 Database Models + +### Core Models +- **User** - Authentication and basic info +- **UserProfile** - Detailed profile information +- **UserActivity** - Activity tracking and logging +- **UserContribution** - Content contribution management +- **UserRole** - Role assignments and permissions + +### Automatic Setup +The service automatically creates all database tables on startup using SQLAlchemy ORM models. No manual migration required! + +## 🔧 Development + +### Adding Features +1. **Add models** in `core/models/database_models.py` +2. **Add repository methods** in `core/user_database.py` +3. **Add API endpoints** in `core/routers/user_management.py` +4. **Add Pydantic models** in `core/models/user.py` + +### Testing +```bash +pytest +pytest --cov=core +``` + +## 📈 Performance + +- **Connection Pooling** - Efficient database connections +- **Optimized Indexes** - Fast query performance +- **Async Operations** - Non-blocking operations +- **Type Safety** - Catch errors at development time + +## 🔒 Security + +- **JWT Authentication** - Secure token-based auth +- **Password Hashing** - bcrypt for password security +- **Input Validation** - Pydantic model validation +- **SQL Injection Protection** - ORM prevents injection +- **Role-Based Access** - Granular permissions + +## 📞 Support + +- **Email**: tekraj@mit.edu +- **Issues**: GitHub Issues +- **Documentation**: `/docs` endpoint when running + +--- + +**Built for scalable neuroscience knowledge base user management!** 🧠🔬 \ No newline at end of file diff --git a/usermanagement_service/core/bootstrap.py b/usermanagement_service/core/bootstrap.py index 6f1ef89..01526b1 100644 --- a/usermanagement_service/core/bootstrap.py +++ b/usermanagement_service/core/bootstrap.py @@ -177,6 +177,19 @@ async def apply_inline_schema_migrations() -> None: 'ALTER TABLE "Web_user_profile" ADD COLUMN IF NOT EXISTS banned_at TIMESTAMP', 'ALTER TABLE "Web_user_profile" ADD COLUMN IF NOT EXISTS banned_by INTEGER REFERENCES "Web_user_profile"(id) ON DELETE SET NULL', 'ALTER TABLE "Web_user_profile" ADD COLUMN IF NOT EXISTS ban_reason TEXT', + # Identity unification (Phase 1): make the credential row (Web_jwtuser) + # an explicit 1:1 record for a profile instead of an email-only sibling. + # Web_user_profile is the canonical user; profile_id is the link. + 'ALTER TABLE "Web_jwtuser" ADD COLUMN IF NOT EXISTS profile_id INTEGER REFERENCES "Web_user_profile"(id) ON DELETE SET NULL', + 'CREATE INDEX IF NOT EXISTS ix_jwtuser_profile_id ON "Web_jwtuser"(profile_id)', + # OAuth CLI/skill paste-code flow: mark whether a state was started by the + # browser (web) or the MCP/skill (cli). New table Web_oauth_cli_result is + # created by create_all(). + 'ALTER TABLE "Web_oauth_state" ADD COLUMN IF NOT EXISTS mode VARCHAR(16) DEFAULT \'web\'', + # Backfill the link for pre-existing rows by matching email (the old + # implicit join key). Case-insensitive so mixed-case duplicates align. + 'UPDATE "Web_jwtuser" u SET profile_id = p.id FROM "Web_user_profile" p ' + 'WHERE u.profile_id IS NULL AND lower(u.email) = lower(p.email)', ] async with user_db_manager.get_async_session() as session: for stmt in statements: diff --git a/usermanagement_service/core/configuration.py b/usermanagement_service/core/configuration.py index 03b32ba..fbb5374 100644 --- a/usermanagement_service/core/configuration.py +++ b/usermanagement_service/core/configuration.py @@ -69,6 +69,25 @@ def load_environment(env_name="env"): "JWT_LOGIN_EMAIL": os.getenv("JWT_LOGIN_EMAIL"), "JWT_LOGIN_PASSWORD": os.getenv("JWT_LOGIN_PASSWORD"), + # Phase 2 SSO: RS256 single-issuer + JWKS. usermanagement is the sole + # issuer; a login mints a short-lived refresh token, exchanged for narrow + # per-service access tokens (aud=). See AUTH_UNIFICATION.md. + "USERMANAGEMENT_JWT_ISSUER": os.getenv("USERMANAGEMENT_JWT_ISSUER", "brainkb-usermanagement"), + # RS256 private key: PEM string, or a file path. If neither is set an + # EPHEMERAL key is generated at first use (dev only; not restart-safe). + "USERMANAGEMENT_JWT_PRIVATE_KEY_PEM": os.getenv("USERMANAGEMENT_JWT_PRIVATE_KEY_PEM"), + "USERMANAGEMENT_JWT_PRIVATE_KEY_FILE": os.getenv("USERMANAGEMENT_JWT_PRIVATE_KEY_FILE"), + "USERMANAGEMENT_ACCESS_TOKEN_TTL_MIN": os.getenv("USERMANAGEMENT_ACCESS_TOKEN_TTL_MIN", "15"), + "USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN": os.getenv("USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN", "720"), + # TTL of the web-session ACCESS JWT handed to the UI at OAuth login + # (?token=). Short-lived; the UI silently refreshes it. Default 12h. + "USERMANAGEMENT_WEB_SESSION_TTL_MIN": os.getenv("USERMANAGEMENT_WEB_SESSION_TTL_MIN", "720"), + # TTL of the web REFRESH token (?refresh=) the UI stores to mint new access + # tokens without re-login. Governs the overall web session length. 7d. + "USERMANAGEMENT_WEB_REFRESH_TTL_MIN": os.getenv("USERMANAGEMENT_WEB_REFRESH_TTL_MIN", "10080"), + # Services a refresh token may be exchanged for (valid aud values). + "USERMANAGEMENT_TOKEN_AUDIENCES": os.getenv("USERMANAGEMENT_TOKEN_AUDIENCES", "usermanagement,query_service,ml_service,chat_service"), + # OAuth / Admin Bootstrap "USERMANAGEMENT_PUBLIC_BASE_URL": os.getenv("USERMANAGEMENT_PUBLIC_BASE_URL", "http://localhost:8004"), "USERMANAGEMENT_FRONTEND_CALLBACK_URL": os.getenv("USERMANAGEMENT_FRONTEND_CALLBACK_URL", "http://localhost:3000/auth/callback"), @@ -153,6 +172,55 @@ def jwt_secret_key(self) -> Optional[str]: def jwt_bearer_token_url(self) -> str: """Get the JWT bearer token URL.""" return self._env_vars.get("JWT_BEARER_TOKEN_URL", "") + + # ---- Phase 2 SSO (RS256 single-issuer + JWKS) -------------------------- + @property + def jwt_issuer(self) -> str: + return self._env_vars.get("USERMANAGEMENT_JWT_ISSUER", "brainkb-usermanagement") + + @property + def jwt_private_key_pem(self) -> Optional[str]: + return self._env_vars.get("USERMANAGEMENT_JWT_PRIVATE_KEY_PEM") + + @property + def jwt_private_key_file(self) -> Optional[str]: + return self._env_vars.get("USERMANAGEMENT_JWT_PRIVATE_KEY_FILE") + + @property + def access_token_ttl_min(self) -> int: + try: + return int(self._env_vars.get("USERMANAGEMENT_ACCESS_TOKEN_TTL_MIN", "15")) + except (TypeError, ValueError): + return 15 + + @property + def refresh_token_ttl_min(self) -> int: + try: + return int(self._env_vars.get("USERMANAGEMENT_REFRESH_TOKEN_TTL_MIN", "720")) + except (TypeError, ValueError): + return 720 + + @property + def web_session_ttl_min(self) -> int: + """TTL (minutes) of the web-session ACCESS JWT issued to the UI at login.""" + try: + return int(self._env_vars.get("USERMANAGEMENT_WEB_SESSION_TTL_MIN", "720")) + except (TypeError, ValueError): + return 720 + + @property + def web_refresh_ttl_min(self) -> int: + """TTL (minutes) of the web REFRESH token the UI stores to renew access + tokens without re-login. Governs overall web session length.""" + try: + return int(self._env_vars.get("USERMANAGEMENT_WEB_REFRESH_TTL_MIN", "10080")) + except (TypeError, ValueError): + return 10080 + + @property + def token_audiences(self) -> list: + raw = self._env_vars.get("USERMANAGEMENT_TOKEN_AUDIENCES", "usermanagement,query_service,ml_service,chat_service") or "" + return [a.strip() for a in raw.split(",") if a.strip()] @property def jwt_login_username(self) -> str: diff --git a/usermanagement_service/core/database.py b/usermanagement_service/core/database.py index d64cd19..86d06ba 100644 --- a/usermanagement_service/core/database.py +++ b/usermanagement_service/core/database.py @@ -26,8 +26,8 @@ from core.models.database_models import ( Base, JWTUser, UserProfile, UserActivity, UserContribution, UserRole, UserCountry, UserOrganization, UserEducation, UserExpertise, AvailableRole, AvailableCountry, - OAuthIdentity, OAuthState, Permission, RolePermission, PageAccess, PageAccessRole, PageAccessUser, - AdminSetting, + OAuthIdentity, OAuthState, OAuthCliResult, Permission, RolePermission, PageAccess, PageAccessRole, PageAccessUser, + AdminSetting, PersonalAccessToken, ) from core.models.user import ActivityType, ContributionStatus @@ -1409,6 +1409,102 @@ async def purge_expired(self, session: AsyncSession) -> None: logger.error(f"Error purging expired oauth states: {str(e)}") +class OAuthCliResultRepository(UserBaseRepository): + """CLI/skill paste-code OAuth results: code -> refresh token, single-use.""" + + def __init__(self): + super().__init__(OAuthCliResult) + + async def store(self, session: AsyncSession, *, code: str, refresh_token: str, + email: Optional[str], expires_at: datetime) -> OAuthCliResult: + row = OAuthCliResult(code=code, refresh_token=refresh_token, email=email, + expires_at=expires_at, consumed=False) + session.add(row) + await session.flush() + return row + + async def consume(self, session: AsyncSession, code: str) -> Optional[OAuthCliResult]: + """Return the result for a code if valid (exists, unexpired, unconsumed), + marking it consumed. Returns None otherwise.""" + result = await session.execute(select(OAuthCliResult).where(OAuthCliResult.code == code)) + row = result.scalar_one_or_none() + if row is None or row.consumed or row.expires_at < datetime.utcnow(): + return None + row.consumed = True + await session.flush() + return row + + async def purge_expired(self, session: AsyncSession) -> None: + try: + await session.execute( + text('DELETE FROM "Web_oauth_cli_result" WHERE expires_at < :now OR consumed = true'), + {"now": datetime.utcnow()}, + ) + await session.flush() + except SQLAlchemyError as e: + logger.error(f"Error purging oauth cli results: {str(e)}") + + +class PersonalAccessTokenRepository(UserBaseRepository): + """Personal Access Tokens (opaque, hashed at rest). Used by the CLI/MCP to + authenticate without a browser after a one-time mint. Only the SHA-256 hash + is stored; the plaintext is returned to the user once at creation.""" + + def __init__(self): + super().__init__(PersonalAccessToken) + + async def create(self, session: AsyncSession, *, token_hash: str, prefix: str, + name: str, profile_id: Optional[int], jwt_user_id: Optional[int], + email: str, expires_at: datetime) -> PersonalAccessToken: + row = PersonalAccessToken( + token_hash=token_hash, prefix=prefix, name=name or "", profile_id=profile_id, + jwt_user_id=jwt_user_id, email=email, expires_at=expires_at, revoked=False, + ) + session.add(row) + await session.flush() + return row + + async def get_valid_by_hash(self, session: AsyncSession, token_hash: str) -> Optional[PersonalAccessToken]: + """Return the PAT for a hash if it is usable (exists, not revoked, not + expired), else None. Touches last_used_at as a side effect.""" + result = await session.execute( + select(PersonalAccessToken).where(PersonalAccessToken.token_hash == token_hash) + ) + row = result.scalar_one_or_none() + if row is None or row.revoked or row.expires_at < datetime.utcnow(): + return None + row.last_used_at = datetime.utcnow() + await session.flush() + return row + + async def list_for_profile(self, session: AsyncSession, profile_id: int) -> List[PersonalAccessToken]: + result = await session.execute( + select(PersonalAccessToken) + .where(PersonalAccessToken.profile_id == profile_id) + .order_by(PersonalAccessToken.created_at.desc()) + ) + return list(result.scalars().all()) + + async def get_owned(self, session: AsyncSession, pat_id: int, + profile_id: int) -> Optional[PersonalAccessToken]: + result = await session.execute( + select(PersonalAccessToken).where( + PersonalAccessToken.id == pat_id, + PersonalAccessToken.profile_id == profile_id, + ) + ) + return result.scalar_one_or_none() + + async def revoke(self, session: AsyncSession, pat_id: int, profile_id: int) -> bool: + """Revoke a PAT the caller owns. Returns True if a row was revoked.""" + row = await self.get_owned(session, pat_id, profile_id) + if row is None or row.revoked: + return False + row.revoked = True + await session.flush() + return True + + class PermissionRepository(UserBaseRepository): def __init__(self): super().__init__(Permission) @@ -1590,6 +1686,8 @@ async def check_access( available_country_repo = AvailableCountryRepository() oauth_identity_repo = OAuthIdentityRepository() oauth_state_repo = OAuthStateRepository() +oauth_cli_result_repo = OAuthCliResultRepository() +personal_access_token_repo = PersonalAccessTokenRepository() permission_repo = PermissionRepository() role_permission_repo = RolePermissionRepository() page_access_repo = PageAccessRepository() @@ -1652,3 +1750,85 @@ async def delete(self, session: AsyncSession, key: str) -> bool: admin_setting_repo = AdminSettingRepository() + + +# --------------------------------------------------------------------------- +# Identity provisioning (Phase 1 unification) +# --------------------------------------------------------------------------- + +async def provision_identity( + session: AsyncSession, + *, + email: str, + full_name: Optional[str] = None, + password_hash: Optional[str] = None, + default_role: str = "Curator", + existing_profile: Optional[UserProfile] = None, +) -> tuple: + """Single source of truth for creating/linking a BrainKB identity. + + Idempotently guarantees that, for ``email``, there is: + * a canonical ``Web_user_profile`` (the user of record), + * a ``Web_jwtuser`` credential row linked to it via ``profile_id``, + * at least one role (``default_role``) on the profile, + * ``Admin`` + ``SuperAdmin`` if the email is in the bootstrap allowlist. + + The caller owns the session/transaction (this does NOT commit). OAuth's rich + profile matching (by oauth-identity / orcid) stays in the router, which + passes the already-resolved profile via ``existing_profile``; password + onboarding passes just ``email`` + ``password_hash``. A pre-existing + credential's password is never overwritten here (login stays deterministic). + + Returns ``(profile, jwt_user, role_names)``. + """ + import secrets as _secrets + from core.security import get_password_hash # lazy: avoid import cycle + from core.models.user import UserRoleEnum + + email = (email or "").strip() + if not email: + raise HTTPException(status_code=400, detail="email is required to provision an identity") + display_name = full_name or email.split("@")[0] + + # 1) Canonical profile — reuse the resolved one, else find/create by email. + profile = existing_profile or await user_profile_repo.get_by_email(session, email) + if profile is None: + profile = await user_profile_repo.create_profile(session, name=display_name, email=email) + + # 2) Credential row (get-or-create). get_by_email_any_status avoids + # re-inserting (and colliding on the unique email) for inactive shells. + jwt_user = await jwt_user_repo.get_by_email_any_status(session, email) + if jwt_user is None: + # OAuth/no-password onboarding gets an unusable random secret; a real + # hash is used only when the caller set a password (registration). + secret = password_hash or get_password_hash(_secrets.token_urlsafe(48)) + jwt_user = await jwt_user_repo.create_user( + session=session, full_name=display_name, email=email, password=secret, + ) + + # 3) Link credential -> profile (the whole point of Phase 1). Set only when + # unset/stale so we never thrash an already-correct link. + if getattr(jwt_user, "profile_id", None) != profile.id: + jwt_user.profile_id = profile.id + jwt_user.updated_at = datetime.utcnow() + await session.flush() + + # 4) Default role if the profile has none yet. + role_names = await user_role_repo.get_user_role_names(session, profile.id) + if not role_names and default_role: + await user_role_repo.assign_role( + session, profile_id=profile.id, role=default_role, is_active=True, + ) + role_names = [default_role] + + # 5) Bootstrap-superadmin allowlist: elevate on first sight. Seed both Admin + # (permissions / page access) and SuperAdmin (the protected marker). + if (profile.email or "").lower() in config.bootstrap_superadmin_emails: + for role_name in (UserRoleEnum.ADMIN.value, UserRoleEnum.SUPERADMIN.value): + if role_name not in role_names: + await user_role_repo.assign_role( + session, profile_id=profile.id, role=role_name, is_active=True, + ) + role_names.append(role_name) + + return profile, jwt_user, role_names diff --git a/usermanagement_service/core/main.py b/usermanagement_service/core/main.py index 251e0cc..d62022e 100644 --- a/usermanagement_service/core/main.py +++ b/usermanagement_service/core/main.py @@ -1,4 +1,5 @@ import logging +import os from contextlib import asynccontextmanager from datetime import datetime @@ -16,6 +17,8 @@ from core.routers.oauth import router as oauth_router from core.routers.admin import router as admin_router from core.routers.access import router as access_router +from core.routers.sso import router as sso_router, wellknown_router +from core.routers.pat import router as pat_router from core.database import user_db_manager, user_activity_repo from core.models.user import ActivityType from core.security import verify_token @@ -121,13 +124,24 @@ async def lifespan(app: FastAPI): app = FastAPI(lifespan=lifespan) logger = logging.getLogger(__name__) -origins = [ +# Browser origins allowed to call this service. Kept identical across the four +# BrainKB services, which each hold their own copy and had drifted apart. +# CORS_ALLOWED_ORIGINS (comma-separated) adds to these without a code change. +# +# Dropped the schemeless "localhost:3000": the browser's Origin header always +# carries a scheme, so that entry could never match. +_DEFAULT_ORIGINS = [ + "https://brainkb.org", + "https://www.brainkb.org", "https://beta.brainkb.org", "https://sandbox.brainkb.org", - "localhost:3000", "http://localhost:3000", - "http://127.0.0.1:300", + "http://127.0.0.1:3000", ] +origins = sorted({ + *_DEFAULT_ORIGINS, + *(o.strip() for o in os.getenv("CORS_ALLOWED_ORIGINS", "").split(",") if o.strip()), +}) app.add_middleware( CORSMiddleware, @@ -147,6 +161,11 @@ async def lifespan(app: FastAPI): app.include_router(oauth_router, prefix="/api", tags=["OAuth"]) app.include_router(admin_router, prefix="/api/admin", tags=["Admin"]) app.include_router(access_router, prefix="/api", tags=["Access Control"]) +# Phase 2 SSO: JWKS at the root well-known path; auth endpoints under /api. +app.include_router(wellknown_router, tags=["SSO"]) +app.include_router(sso_router, prefix="/api", tags=["SSO"]) +# Personal Access Tokens (browser-free CLI/MCP auth). +app.include_router(pat_router, prefix="/api", tags=["PAT"]) # log all HTTP exception when raised diff --git a/usermanagement_service/core/models/database_models.py b/usermanagement_service/core/models/database_models.py index 1a89b06..60d1542 100644 --- a/usermanagement_service/core/models/database_models.py +++ b/usermanagement_service/core/models/database_models.py @@ -32,15 +32,22 @@ class JWTUser(Base): email: Mapped[str] = mapped_column(String(255), unique=True, nullable=False) password: Mapped[str] = mapped_column(String(255), nullable=False) is_active: Mapped[bool] = mapped_column(Boolean, default=False) + # Identity unification (Phase 1): the credential row is a 1:1 record for a + # canonical Web_user_profile. Nullable + SET NULL so a profile delete never + # orphans/blocks the credential; backfilled by email in bootstrap. + profile_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("Web_user_profile.id", ondelete="SET NULL"), nullable=True + ) created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) updated_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) - + # Relationships - JWT only, no profile relationships - + # Indexes __table_args__ = ( Index('idx_jwtuser_email', 'email'), Index('idx_jwtuser_active', 'is_active'), + Index('idx_jwtuser_profile_id', 'profile_id'), ) @@ -387,6 +394,9 @@ class OAuthState(Base): provider: Mapped[str] = mapped_column(String(32), nullable=False) code_verifier: Mapped[Optional[str]] = mapped_column(String(256)) redirect_after_login: Mapped[Optional[str]] = mapped_column(String(500)) + # 'web' (default, browser redirect to the SPA) or 'cli' (paste-code flow for + # the MCP/skill — the callback shows a short code instead of redirecting). + mode: Mapped[str] = mapped_column(String(16), default="web") created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) expires_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) @@ -396,6 +406,62 @@ class OAuthState(Base): ) +class OAuthCliResult(Base): + """Result bucket for the CLI/skill paste-code OAuth flow. After a CLI-initiated + OAuth callback provisions the user and mints an SSO refresh token, that token is + stored here keyed by a short human-typable code, shown in the browser. The + MCP/skill exchanges the code for the refresh token (single-use, short-lived).""" + __tablename__ = "Web_oauth_cli_result" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + code: Mapped[str] = mapped_column(String(32), unique=True, nullable=False) + refresh_token: Mapped[str] = mapped_column(Text, nullable=False) + email: Mapped[Optional[str]] = mapped_column(String(255)) + consumed: Mapped[bool] = mapped_column(Boolean, default=False) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + expires_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + __table_args__ = ( + Index('idx_oauth_cli_result_code', 'code'), + ) + + +class PersonalAccessToken(Base): + """A long-lived, opaque Personal Access Token (PAT) for CLI/MCP use. + + The plaintext token is shown to the user exactly ONCE at creation and is + never stored — only its SHA-256 hash is persisted, so a DB leak cannot + recover usable tokens. A PAT is presented by the MCP/skill and exchanged at + ``POST /api/auth/pat/exchange`` for short-lived per-service access tokens + (the same RS256 tokens the refresh-token flow issues). Unlike a signed JWT a + PAT is revocable instantly (``revoked``) and time-bounded (``expires_at``); + roles are re-read from the DB at exchange time, so authorization is never + stale. This is the "generate once, paste into the skill config, no browser + afterward" credential.""" + __tablename__ = "Web_personal_access_token" + + id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) + # SHA-256 hex of the full opaque secret — the ONLY copy we keep. + token_hash: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) + # Short, non-secret leading fragment shown in listings so a user can tell + # their tokens apart without exposing the secret (e.g. "brainkb_pat_9f3a"). + prefix: Mapped[str] = mapped_column(String(32), nullable=False) + name: Mapped[str] = mapped_column(String(120), nullable=False, default="") + profile_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey('Web_user_profile.id', ondelete='CASCADE')) + jwt_user_id: Mapped[Optional[int]] = mapped_column(Integer) + email: Mapped[str] = mapped_column(String(255), nullable=False) + revoked: Mapped[bool] = mapped_column(Boolean, default=False) + created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow) + last_used_at: Mapped[Optional[datetime]] = mapped_column(DateTime) + expires_at: Mapped[datetime] = mapped_column(DateTime, nullable=False) + + __table_args__ = ( + Index('idx_pat_token_hash', 'token_hash'), + Index('idx_pat_profile_id', 'profile_id'), + ) + + class Permission(Base): """Permission registry. A permission is a (resource, action) tuple, e.g. ('user', 'delete').""" __tablename__ = "Web_permission" diff --git a/usermanagement_service/core/routers/admin.py b/usermanagement_service/core/routers/admin.py index 9f81ddb..bafc9a4 100644 --- a/usermanagement_service/core/routers/admin.py +++ b/usermanagement_service/core/routers/admin.py @@ -17,10 +17,11 @@ from fastapi import APIRouter, Depends, HTTPException, Query from sqlalchemy import select, func +from pydantic import BaseModel from core.database import ( user_db_manager, user_profile_repo, user_role_repo, user_activity_repo, available_role_repo, permission_repo, role_permission_repo, page_access_repo, - oauth_identity_repo, + oauth_identity_repo, jwt_user_repo, ) from core.models.database_models import ( AvailableRole as AvailableRoleModel, @@ -36,6 +37,30 @@ AdminUserListItem, UserRoleInput, ActivityType, ) from core.security import require_admin +from core.configuration import config + +# Roles that make someone part of the admin tier. Managing these (assign/remove +# the role, or ban/delete such an account) is SuperAdmin-only — the hierarchy is +# SuperAdmin > Admin. The SuperAdmin role itself is additionally protected from +# removal/ban/delete everywhere below. +_ADMIN_TIER_ROLES = {"Admin", "SuperAdmin"} + + +async def _is_superadmin(admin: dict) -> bool: + """True if the acting caller currently holds SuperAdmin. Roles are re-read from + the DB (not trusted from the token) so a just-demoted SuperAdmin can't still act. + The bootstrap-superadmin allowlist is honored for first sign-in.""" + email = ((admin.get("sub") or admin.get("email")) if isinstance(admin, dict) else "") or "" + if email and email.lower() in config.bootstrap_superadmin_emails: + return True + from core.security import _current_roles_from_db + roles = await _current_roles_from_db(email.lower(), admin.get("profile_id") if isinstance(admin, dict) else None) + return "SuperAdmin" in roles + + +async def _require_superadmin(admin: dict, action: str) -> None: + if not await _is_superadmin(admin): + raise HTTPException(status_code=403, detail=f"Only a SuperAdmin can {action}.") logger = logging.getLogger(__name__) router = APIRouter() @@ -303,25 +328,16 @@ async def count_users(_admin: Annotated[dict, Depends(require_admin)]): return {"count": result.scalar_one()} -@router.delete("/users/{profile_id}", status_code=204) +@router.delete("/users/{profile_id}") async def delete_user(profile_id: int, _admin: Annotated[dict, Depends(require_admin)]): - async with user_db_manager.get_async_session() as session: - profile = await session.get(UserProfileModel, profile_id) - if not profile: - raise HTTPException(status_code=404, detail="User not found") - # SuperAdmin accounts are protected — they cannot be deleted via the - # admin endpoints. Drop them from the bootstrap allowlist + restart - # if this is genuinely needed. - target_roles = await user_role_repo.get_user_role_names(session, profile_id) - if "SuperAdmin" in (target_roles or []): - raise HTTPException( - status_code=403, - detail="SuperAdmin accounts cannot be deleted via the admin UI.", - ) - # Cascades delete activities, roles, contributions, countries, orgs, education, expertise, oauth_identity. - await session.delete(profile) - await session.commit() - return None + """User deletion is DISABLED by policy — ban the account instead + (`POST /users/{profile_id}/ban`). Banning is reversible and preserves the + user's provenance/audit history; hard deletion is not offered.""" + raise HTTPException( + status_code=405, + detail="User deletion is disabled. Ban the account instead " + "(POST /users/{profile_id}/ban) — reversible and preserves history.", + ) @router.post("/users/{profile_id}/roles", response_model=List[str]) @@ -334,6 +350,9 @@ async def assign_role_to_user( profile = await session.get(UserProfileModel, profile_id) if not profile: raise HTTPException(status_code=404, detail="User not found") + # Only a SuperAdmin may create/grant admin-tier roles (Admin/SuperAdmin). + if body.role in _ADMIN_TIER_ROLES: + await _require_superadmin(admin, f"assign the {body.role} role") await user_role_repo.assign_role( session=session, profile_id=profile_id, @@ -357,7 +376,7 @@ async def assign_role_to_user( async def remove_role_from_user( profile_id: int, role_name: str, - _admin: Annotated[dict, Depends(require_admin)], + admin: Annotated[dict, Depends(require_admin)], ): async with user_db_manager.get_async_session() as session: profile = await session.get(UserProfileModel, profile_id) @@ -369,6 +388,9 @@ async def remove_role_from_user( status_code=403, detail="The SuperAdmin role cannot be removed via the admin UI.", ) + # Demoting an Admin (removing the Admin role) is SuperAdmin-only. + if role_name == "Admin": + await _require_superadmin(admin, "remove the Admin role") await user_role_repo.remove_role(session, profile_id, role_name) roles = await user_role_repo.get_user_role_names(session, profile_id) await session.commit() @@ -495,6 +517,35 @@ async def admin_delete_openrouter_key(_admin: Annotated[dict, Depends(require_ad # which re-reads `is_banned` per request. To delete a user entirely use # DELETE /api/admin/users/{profile_id}. +class _EmailIn(BaseModel): + email: str + + +@router.post("/users/activate") +async def activate_user(body: _EmailIn, _admin: Annotated[dict, Depends(require_admin)]): + """Activate a user's account (set the JWT user `is_active=True`) by email. + Needed e.g. after a password self-registration, or to re-enable an account.""" + async with user_db_manager.get_async_session() as session: + u = await jwt_user_repo.get_by_email_any_status(session, body.email) + if not u: + raise HTTPException(status_code=404, detail="user not found") + changed = await jwt_user_repo.activate_user(session, u.id) + await session.commit() + return {"email": body.email, "is_active": True, "changed": bool(changed)} + + +@router.post("/users/deactivate") +async def deactivate_user(body: _EmailIn, _admin: Annotated[dict, Depends(require_admin)]): + """Deactivate a user's account (set the JWT user `is_active=False`) by email.""" + async with user_db_manager.get_async_session() as session: + u = await jwt_user_repo.get_by_email_any_status(session, body.email) + if not u: + raise HTTPException(status_code=404, detail="user not found") + changed = await jwt_user_repo.deactivate_user(session, u.id) + await session.commit() + return {"email": body.email, "is_active": False, "changed": bool(changed)} + + @router.post("/users/{profile_id}/ban") async def ban_user( profile_id: int, @@ -504,9 +555,9 @@ async def ban_user( """Suspend a user. Body: { "reason": str }. Idempotent — re-banning an already-banned user updates the reason and timestamp. - Refuses to ban yourself or to ban a SuperAdmin. Regular Admins are - bannable directly — multiple admins can coexist, and one admin moderating - another is part of the model. + Refuses to ban yourself or a SuperAdmin. Banning an Admin is SuperAdmin-only + (hierarchy: SuperAdmin > Admin); regular Admins can ban non-admin users. + Banning is how accounts are removed — deletion is disabled (we don't delete). """ reason = (payload.get("reason") or "").strip() if isinstance(payload, dict) else "" if not reason: @@ -529,9 +580,13 @@ async def ban_user( status_code=403, detail="SuperAdmin accounts cannot be banned via the admin UI.", ) + # Banning an Admin is SuperAdmin-only (SuperAdmin > Admin). + if "Admin" in (target_roles or []): + await _require_superadmin(admin, "ban an Admin account") + banned_at = datetime.utcnow() profile.is_banned = True - profile.banned_at = datetime.utcnow() + profile.banned_at = banned_at profile.banned_by = actor_id profile.ban_reason = reason await session.flush() @@ -545,12 +600,14 @@ async def ban_user( user_agent=None, ) await session.commit() + # Build the response from locals — after commit the ORM attributes are + # expired and touching them would trigger async lazy-load (MissingGreenlet). return { "profile_id": profile_id, "is_banned": True, - "banned_at": profile.banned_at.isoformat() if profile.banned_at else None, - "banned_by": profile.banned_by, - "ban_reason": profile.ban_reason, + "banned_at": banned_at.isoformat(), + "banned_by": actor_id, + "ban_reason": reason, } diff --git a/usermanagement_service/core/routers/jwt_auth.py b/usermanagement_service/core/routers/jwt_auth.py index f6ad117..b7e359e 100644 --- a/usermanagement_service/core/routers/jwt_auth.py +++ b/usermanagement_service/core/routers/jwt_auth.py @@ -11,7 +11,11 @@ router = APIRouter() -@router.post("/token") +# Legacy password login (issues a v2 HS256 token). The SSO login is +# /api/auth/login (refresh token); this stays as a deprecated compatibility path. +# `/api/login` is the preferred name; `/api/token` remains as a deprecated alias. +@router.post("/login") +@router.post("/token", include_in_schema=False) async def login(user: LoginUserIn, request: Request): async with user_db_manager.get_async_session() as session: # Authenticate user diff --git a/usermanagement_service/core/routers/oauth.py b/usermanagement_service/core/routers/oauth.py index 01da808..f9d0d4b 100644 --- a/usermanagement_service/core/routers/oauth.py +++ b/usermanagement_service/core/routers/oauth.py @@ -16,25 +16,28 @@ import base64 import hashlib import logging +import os import secrets from datetime import datetime, timedelta from typing import Optional from urllib.parse import urlencode from fastapi import APIRouter, HTTPException, Query, Request -from fastapi.responses import RedirectResponse +from fastapi.responses import RedirectResponse, HTMLResponse from core.configuration import config +from pydantic import BaseModel + +from core import tokens_rs256 from core.database import ( - user_db_manager, user_profile_repo, user_role_repo, jwt_user_repo, - oauth_identity_repo, oauth_state_repo, user_activity_repo, + user_db_manager, user_profile_repo, jwt_user_repo, + oauth_identity_repo, oauth_state_repo, oauth_cli_result_repo, + user_activity_repo, provision_identity, ) from core.models.user import ActivityType, OAuthLoginStart, UserRoleEnum -from core.models.database_models import UserProfile as UserProfileModel, JWTUser as JWTUserModel +from core.models.database_models import UserProfile as UserProfileModel from core.oauth import get_provider -from core.security import ( - create_access_token_v2, encrypt_token, get_password_hash, -) +from core.security import create_access_token_v2, encrypt_token logger = logging.getLogger(__name__) @@ -54,6 +57,51 @@ def _redirect_uri_for(provider_name: str) -> str: return f"{config.public_base_url.rstrip('/')}/api/auth/{provider_name}/callback" +# Unambiguous alphabet (no I/L/O/0/1) for the paste-code shown to users. +_CLI_CODE_ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789" # 30 symbols +# Paste-code length (raw chars, before dash grouping). The code is short-LIVED +# (~10 min) and single-use, but we still make it HIGH-ENTROPY for defense in +# depth: 20 chars over a 30-symbol alphabet ≈ 98 bits (~1e29 combinations), +# infeasible to brute-force within the 10-minute window even without rate limits. +# Clamped to 24 so the dash-grouped value still fits the +# Web_oauth_cli_result.code column (String(32)); configurable via env. +_CLI_CODE_LEN = min(24, max(8, int(os.getenv("USERMANAGEMENT_CLI_CODE_LEN", "20")))) + + +def _gen_cli_code() -> str: + """A long, single-use, ~10-min paste-code, grouped in 4s for readability, + e.g. ``A3KM-7QRS-9WXY-2BCD-EFGH``. High entropy so it can't be guessed in the + short window; it is only a handle exchanged once for the real refresh token.""" + raw = "".join(secrets.choice(_CLI_CODE_ALPHABET) for _ in range(_CLI_CODE_LEN)) + return "-".join(raw[i:i + 4] for i in range(0, len(raw), 4)) + + +def _cli_success_page(code: str) -> str: + """Minimal self-contained page shown after a CLI/skill OAuth login. Displays the + one-time code the user pastes back into the skill. No SPA/frontend needed.""" + safe = (code or "").replace("<", "").replace(">", "") + return ( + "" + "" + "BrainKB login" + "
" + "

✅ Signed in to BrainKB

" + "

Copy this one-time code and paste it back into your assistant " + "(brainkb_finish_login):

" + f"
{safe}
" + "

The code expires in ~10 minutes and can be used once. " + "You can close this tab afterward.

" + "
" + ) + + async def _upsert_profile_for_oauth(session, userinfo) -> UserProfileModel: """Find or create a UserProfile for an OAuth identity. Matching order: (1) existing OAuth identity → its linked profile, @@ -95,25 +143,6 @@ async def _upsert_profile_for_oauth(session, userinfo) -> UserProfileModel: return new_profile -async def _ensure_jwt_user_shell(session, email: str, full_name: str) -> JWTUserModel: - """Make sure a Web_jwtuser row exists for this email. OAuth users don't have - a usable password — we store a random high-entropy hash (can't be logged in - with, just exists so the JWT user_id claim is stable). The shell is created - with `is_active=False`, so the lookup must not filter by activation; using - `get_by_email` (active-only) here would re-INSERT on every sign-in and - collide with the unique-email constraint.""" - existing = await jwt_user_repo.get_by_email_any_status(session, email) - if existing: - return existing - random_password = secrets.token_urlsafe(48) - return await jwt_user_repo.create_user( - session=session, - full_name=full_name, - email=email, - password=get_password_hash(random_password), - ) - - # ---- routes ------------------------------------------------------------- @router.get("/auth/providers") @@ -129,17 +158,13 @@ async def list_providers(): } -@router.get("/auth/{provider_name}/login", response_model=OAuthLoginStart) -async def oauth_login( - provider_name: str, - redirect_after_login: Optional[str] = Query(None, description="Relative path to send the user to after login completes"), -): - """Start an OAuth flow. Returns the authorize URL; the UI redirects the browser there.""" +async def _begin_oauth(provider_name: str, redirect_after_login: Optional[str], mode: str): + """Mint + persist OAuth state (+PKCE) and return (authorize_url, state). + ``mode`` is 'web' (browser → SPA) or 'cli' (MCP/skill paste-code).""" try: provider = get_provider(provider_name) except KeyError: raise HTTPException(status_code=404, detail=f"Unknown provider: {provider_name}") - if not provider.is_configured(): raise HTTPException(status_code=503, detail=f"{provider_name} OAuth is not configured on the server") @@ -151,12 +176,8 @@ async def oauth_login( redirect_uri = _redirect_uri_for(provider.name) authorize_url = provider.authorize_url( - redirect_uri=redirect_uri, - state=state, - code_challenge=code_challenge, + redirect_uri=redirect_uri, state=state, code_challenge=code_challenge, ) - - # Persist state so the callback (on a different request) can validate it. async with user_db_manager.get_async_session() as session: await oauth_state_repo.create( session, @@ -164,13 +185,67 @@ async def oauth_login( provider=provider.name, code_verifier=code_verifier, redirect_after_login=redirect_after_login, + mode=mode, expires_at=datetime.utcnow() + timedelta(minutes=10), ) await session.commit() + return authorize_url, state + +@router.get("/auth/{provider_name}/login", response_model=OAuthLoginStart) +async def oauth_login( + provider_name: str, + redirect_after_login: Optional[str] = Query(None, description="Relative path to send the user to after login completes"), +): + """Start an OAuth flow (browser/SPA). Returns the authorize URL; the UI redirects the browser there.""" + authorize_url, state = await _begin_oauth(provider_name, redirect_after_login, "web") return OAuthLoginStart(authorize_url=authorize_url, state=state) +class _CliStartIn(BaseModel): + provider: str = "globus" + + +@router.post("/auth/cli/start", tags=["SSO"]) +async def oauth_cli_start(body: _CliStartIn): + """Start an OAuth flow for the MCP/skill (paste-code). Returns an authorize URL; + open it in a browser, sign in with the provider, then paste the short code the + browser shows into `brainkb_finish_login`. No web UI required.""" + authorize_url, state = await _begin_oauth(body.provider, None, "cli") + return { + "authorize_url": authorize_url, + "state": state, + "mode": "cli", + "instructions": ("Open authorize_url in a browser and sign in. When it " + "shows a code, paste it into brainkb_finish_login(code)."), + } + + +class _CliExchangeIn(BaseModel): + code: str + + +@router.post("/auth/cli/exchange", tags=["SSO"]) +async def oauth_cli_exchange(body: _CliExchangeIn): + """Exchange the paste-code (shown after a CLI OAuth login) for an SSO refresh + token. Single-use and short-lived.""" + code = (body.code or "").strip().upper() + async with user_db_manager.get_async_session() as session: + await oauth_cli_result_repo.purge_expired(session) + row = await oauth_cli_result_repo.consume(session, code) + # Read the token INSIDE the session — after commit the ORM attribute is + # expired and touching it would trigger async lazy-load (MissingGreenlet). + refresh_token = row.refresh_token if row is not None else None + await session.commit() + if not refresh_token: + raise HTTPException(status_code=400, detail="Invalid, expired, or already-used code.") + return { + "refresh_token": refresh_token, + "token_type": "refresh", + "expires_in": tokens_rs256.refresh_token_ttl_seconds(), + } + + @router.get("/auth/{provider_name}/callback") async def oauth_callback( provider_name: str, @@ -208,6 +283,7 @@ async def oauth_callback( raise HTTPException(status_code=400, detail="OAuth state expired") code_verifier = state_row.code_verifier redirect_after_login = state_row.redirect_after_login + login_mode = getattr(state_row, "mode", "web") or "web" await session.commit() redirect_uri = _redirect_uri_for(provider.name) @@ -221,6 +297,7 @@ async def oauth_callback( if not userinfo.provider_user_id: return RedirectResponse(_frontend_error_redirect("provider returned no user id"), status_code=302) + cli_code = None async with user_db_manager.get_async_session() as session: try: profile = await _upsert_profile_for_oauth(session, userinfo) @@ -237,23 +314,18 @@ async def oauth_callback( profile.updated_at = datetime.utcnow() await session.flush() - jwt_user = await _ensure_jwt_user_shell( + # Ensure a credential row linked to this profile, a default role, + # and bootstrap elevation — all via the single provisioning path. + # OAuth's own profile matching already ran above, so hand the + # resolved profile through as existing_profile. + profile, jwt_user, existing_roles = await provision_identity( session, email=profile.email, full_name=profile.name or userinfo.name or profile.email, + default_role=UserRoleEnum.CURATOR.value, + existing_profile=profile, ) - # Default role on first login = Curator. - existing_roles = await user_role_repo.get_user_role_names(session, profile.id) - if not existing_roles: - await user_role_repo.assign_role( - session, - profile_id=profile.id, - role=UserRoleEnum.CURATOR.value, - is_active=True, - ) - existing_roles = [UserRoleEnum.CURATOR.value] - # Upsert the oauth identity row (encrypt tokens at rest). token_expires_at = None if token_resp.expires_in: @@ -270,20 +342,6 @@ async def oauth_callback( raw_profile=userinfo.raw, ) - # Bootstrap-superadmin allowlist: if configured, elevate on first sight. - # Seed both Admin (for permissions / page-access checks) and - # SuperAdmin (the immutable marker that protects the account). - if (profile.email or "").lower() in config.bootstrap_superadmin_emails: - for role_name in (UserRoleEnum.ADMIN.value, UserRoleEnum.SUPERADMIN.value): - if role_name not in existing_roles: - await user_role_repo.assign_role( - session, - profile_id=profile.id, - role=role_name, - is_active=True, - ) - existing_roles.append(role_name) - # Log activity. await user_activity_repo.log_activity( session=session, @@ -302,7 +360,42 @@ async def oauth_callback( roles=existing_roles, scopes=scopes, auth_source=provider.name, + # Web-session TTL (default 12h), not the 30-min default: this token + # lives in the NextAuth session and isn't auto-refreshed, so a short + # TTL made the UI 401 (/api/users/me) mid-session. + expires_minutes=config.web_session_ttl_min, ) + # Web flow: also mint a longer-lived REFRESH token so the UI can renew + # its access token silently (no re-login) until this expires. Exchanged + # by the UI at /api/auth/exchange (audience=usermanagement). + web_refresh = None + if login_mode != "cli": + web_refresh = tokens_rs256.create_refresh_token( + email=profile.email, + profile_id=profile.id, + roles=existing_roles, + scopes=scopes, + auth_source=provider.name, + expires_minutes=config.web_refresh_ttl_min, + ) + # CLI/skill (paste-code) flow: mint an SSO refresh token and stash it + # behind a short code the browser will display for the user to paste. + if login_mode == "cli": + refresh = tokens_rs256.create_refresh_token( + email=profile.email, + profile_id=profile.id, + roles=existing_roles, + scopes=scopes, + auth_source=provider.name, + ) + cli_code = _gen_cli_code() + await oauth_cli_result_repo.store( + session, + code=cli_code, + refresh_token=refresh, + email=profile.email, + expires_at=datetime.utcnow() + timedelta(minutes=10), + ) await session.commit() except HTTPException: await session.rollback() @@ -312,7 +405,13 @@ async def oauth_callback( logger.exception("Error finalizing OAuth login") return RedirectResponse(_frontend_error_redirect(f"finalize_failed: {e}"), status_code=302) + # CLI/skill login: show the paste-code page instead of redirecting to the SPA. + if login_mode == "cli": + return HTMLResponse(_cli_success_page(cli_code)) + qs = {"token": token} + if web_refresh: + qs["refresh"] = web_refresh if redirect_after_login: qs["redirect"] = redirect_after_login return RedirectResponse(f"{config.frontend_callback_url}?{urlencode(qs)}", status_code=302) diff --git a/usermanagement_service/core/routers/pat.py b/usermanagement_service/core/routers/pat.py new file mode 100644 index 0000000..de90b7a --- /dev/null +++ b/usermanagement_service/core/routers/pat.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +"""Personal Access Tokens (PATs) — browser-free auth for the CLI / MCP skills. + +A PAT is an opaque, long-lived, revocable credential. The user mints one **once** +(needing a normal login only at that moment), pastes it into their MCP/skill +config (``BRAINKB_TOKEN``), and from then on every call authenticates with the +PAT — no browser, no password, no paste-code. + + POST /api/auth/tokens (authenticated) mint a PAT — plaintext shown ONCE + GET /api/auth/tokens (authenticated) list the caller's PATs (metadata only) + DELETE /api/auth/tokens/{id} (authenticated) revoke one of the caller's PATs + POST /api/auth/pat/exchange (PAT in body) PAT -> short-lived per-service access token + +Design notes +------------ +* The token is opaque (``brainkb_pat_``) — NOT a JWT and NOT RSA-signed — + so nothing key-related is exposed to the user. Only its SHA-256 hash is stored. +* Validation is a DB lookup (hash match, not revoked, not expired, user not + banned). That makes a PAT **instantly revocable**, unlike a signed token that + lives until it expires. +* On exchange we re-read the user's roles from the DB and derive scopes, then + mint the same RS256 per-service access token the refresh-token flow issues — + so downstream services need **no changes** and containment (per-``aud`` tokens) + is preserved. +""" +import hashlib +import logging +import os +import secrets +from datetime import datetime, timedelta + +from fastapi import APIRouter, Depends, HTTPException, status +from pydantic import BaseModel, Field + +from core import tokens_rs256 +from core.configuration import config +from core.database import ( + user_db_manager, personal_access_token_repo, user_profile_repo, + user_role_repo, jwt_user_repo, +) +from core.routers.sso import _scopes_for_roles +from core.security import get_current_user + +logger = logging.getLogger(__name__) + +router = APIRouter() + +_PAT_PREFIX = "brainkb_pat_" +# Default lifetime and hard cap for a PAT, in days (configurable via env). +_PAT_DEFAULT_DAYS = max(1, int(os.getenv("USERMANAGEMENT_PAT_DEFAULT_DAYS", "3"))) +_PAT_MAX_DAYS = max(1, int(os.getenv("USERMANAGEMENT_PAT_MAX_DAYS", "365"))) +# Sliding expiry: when true (default), each successful use pushes the PAT's expiry +# forward by _PAT_DEFAULT_DAYS (the idle window) — so an actively-used token keeps +# working and never re-prompts, while an unused one expires after that many idle +# days. The extension is capped at created_at + _PAT_MAX_DAYS (an absolute ceiling, +# so a token can't roll forever). Set false for fixed-lifetime tokens. +_PAT_SLIDING = os.getenv("USERMANAGEMENT_PAT_SLIDING", "true").strip().lower() not in ("0", "false", "no") +# Upper bound on how many active (unrevoked, unexpired) tokens a user may hold — +# a light guard against unbounded token sprawl. +_PAT_MAX_PER_USER = max(1, int(os.getenv("USERMANAGEMENT_PAT_MAX_PER_USER", "20"))) + + +def _hash(token: str) -> str: + return hashlib.sha256(token.encode()).hexdigest() + + +def _gen_token() -> str: + """Mint a fresh opaque PAT: ``brainkb_pat_`` + 43 urlsafe chars (~256 bits).""" + return _PAT_PREFIX + secrets.token_urlsafe(32) + + +class CreateTokenIn(BaseModel): + name: str = Field("", max_length=120, + description="Human label so you can tell tokens apart, e.g. 'laptop'.") + days: int = Field(_PAT_DEFAULT_DAYS, ge=1, le=_PAT_MAX_DAYS, + description=f"Lifetime in days (1..{_PAT_MAX_DAYS}).") + + +class PatExchangeIn(BaseModel): + token: str + audience: str + + +@router.post("/auth/tokens", tags=["PAT"]) +async def create_token(body: CreateTokenIn, current_user: dict = Depends(get_current_user)): + """Mint a Personal Access Token for the logged-in user. The plaintext token is + returned **once** — it is never stored (only a hash) and cannot be retrieved + again. Paste it into your MCP/skill config as ``BRAINKB_TOKEN``.""" + email = current_user.get("email") or current_user.get("sub") + if not email: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="no identity in token") + + days = min(max(1, int(body.days or _PAT_DEFAULT_DAYS)), _PAT_MAX_DAYS) + token = _gen_token() + token_hash = _hash(token) + # Non-secret display fragment: prefix + first 4 chars of the random part. + display_prefix = token[: len(_PAT_PREFIX) + 4] + expires_at = datetime.utcnow() + timedelta(days=days) + + async with user_db_manager.get_async_session() as session: + profile = await user_profile_repo.get_by_email(session, email) + profile_id = profile.id if profile else current_user.get("profile_id") + if profile and getattr(profile, "is_banned", False): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="account_suspended") + jwt_user = await jwt_user_repo.get_by_email_any_status(session, email) + jwt_user_id = jwt_user.id if jwt_user else current_user.get("user_id") + + if profile_id is not None: + existing = await personal_access_token_repo.list_for_profile(session, profile_id) + active = [t for t in existing if not t.revoked and t.expires_at >= datetime.utcnow()] + if len(active) >= _PAT_MAX_PER_USER: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=(f"You already have {len(active)} active tokens " + f"(max {_PAT_MAX_PER_USER}). Revoke one first."), + ) + + row = await personal_access_token_repo.create( + session, token_hash=token_hash, prefix=display_prefix, name=body.name, + profile_id=profile_id, jwt_user_id=jwt_user_id, email=email, expires_at=expires_at, + ) + # Capture values before commit (attributes expire afterward → MissingGreenlet). + pat_id = row.id + created_prefix = row.prefix + await session.commit() + + return { + "id": pat_id, + "token": token, # shown ONCE — never returned again + "prefix": created_prefix, + "name": body.name, + "expires_at": expires_at.isoformat() + "Z", + "expires_in_days": days, + "note": ("Copy this token now — it will not be shown again. Set it as " + "BRAINKB_TOKEN in your MCP/skill config."), + } + + +@router.get("/auth/tokens", tags=["PAT"]) +async def list_tokens(current_user: dict = Depends(get_current_user)): + """List the caller's PATs (metadata only — the secret is never returned).""" + email = current_user.get("email") or current_user.get("sub") + async with user_db_manager.get_async_session() as session: + profile = await user_profile_repo.get_by_email(session, email) + profile_id = profile.id if profile else current_user.get("profile_id") + if profile_id is None: + return {"tokens": []} + rows = await personal_access_token_repo.list_for_profile(session, profile_id) + now = datetime.utcnow() + tokens = [ + { + "id": r.id, + "name": r.name, + "prefix": r.prefix, + "created_at": r.created_at.isoformat() + "Z" if r.created_at else None, + "last_used_at": r.last_used_at.isoformat() + "Z" if r.last_used_at else None, + "expires_at": r.expires_at.isoformat() + "Z" if r.expires_at else None, + "revoked": r.revoked, + "expired": bool(r.expires_at and r.expires_at < now), + "active": (not r.revoked) and bool(r.expires_at and r.expires_at >= now), + } + for r in rows + ] + return {"tokens": tokens} + + +@router.delete("/auth/tokens/{pat_id}", tags=["PAT"]) +async def revoke_token(pat_id: int, current_user: dict = Depends(get_current_user)): + """Revoke one of the caller's PATs. Takes effect immediately — the next + exchange with that token fails.""" + email = current_user.get("email") or current_user.get("sub") + async with user_db_manager.get_async_session() as session: + profile = await user_profile_repo.get_by_email(session, email) + profile_id = profile.id if profile else current_user.get("profile_id") + if profile_id is None: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Token not found") + ok = await personal_access_token_repo.revoke(session, pat_id, profile_id) + await session.commit() + if not ok: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, + detail="Token not found or already revoked") + return {"revoked": True, "id": pat_id} + + +@router.post("/auth/pat/exchange", tags=["PAT"]) +async def pat_exchange(body: PatExchangeIn): + """Exchange a Personal Access Token for a short-lived per-service access token + (``aud=``). The PAT itself is the credential — no other auth needed. + + Roles are re-read fresh from the DB and scopes derived from them, so a + demoted/banned user (or a revoked token) cannot mint a privileged token.""" + token = (body.token or "").strip() + if not token.startswith(_PAT_PREFIX): + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not a BrainKB access token") + if body.audience not in config.token_audiences: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown audience '{body.audience}'. Allowed: {config.token_audiences}", + ) + + token_hash = _hash(token) + async with user_db_manager.get_async_session() as session: + row = await personal_access_token_repo.get_valid_by_hash(session, token_hash) + if row is None: + await session.commit() + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, + detail="Token invalid, expired, or revoked") + # Capture PAT-linked identity before touching related rows / committing. + email = row.email + profile_id = row.profile_id + jwt_user_id = row.jwt_user_id + + # Sliding expiry: recent use extends the window so an actively-used token + # never re-prompts; an idle one still expires after _PAT_DEFAULT_DAYS. Cap + # the roll at created_at + _PAT_MAX_DAYS so it can't live forever. Only ever + # push expiry forward, never shorten it. + if _PAT_SLIDING: + now = datetime.utcnow() + sliding = now + timedelta(days=_PAT_DEFAULT_DAYS) + cap = (row.created_at or now) + timedelta(days=_PAT_MAX_DAYS) + new_exp = min(sliding, cap) + if new_exp > row.expires_at: + row.expires_at = new_exp + + profile = await user_profile_repo.get_by_email(session, email) + if profile is not None: + profile_id = profile.id + if getattr(profile, "is_banned", False): + await session.commit() + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="account_suspended") + roles = await user_role_repo.get_user_role_names(session, profile_id) if profile_id else [] + await session.commit() + + access = tokens_rs256.create_access_token( + audience=body.audience, + email=email, + profile_id=profile_id, + roles=roles, + scopes=_scopes_for_roles(roles), + auth_source="pat", + jwt_user_id=jwt_user_id, + ) + return { + "access_token": access, + "token_type": "bearer", + "aud": body.audience, + "expires_in": tokens_rs256.access_token_ttl_seconds(), + } diff --git a/usermanagement_service/core/routers/sso.py b/usermanagement_service/core/routers/sso.py new file mode 100644 index 0000000..3994358 --- /dev/null +++ b/usermanagement_service/core/routers/sso.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- +"""Phase 2 SSO endpoints (RS256 single-issuer + JWKS). + + GET /.well-known/jwks.json public keys for token verification + POST /api/auth/login {email,password} -> refresh token (aud=brainkb-auth) + POST /api/auth/exchange Bearer , {audience} -> per-service access token + +usermanagement is the sole issuer. Clients log in once (refresh token), then +exchange for narrow, short-lived access tokens scoped to a single service via +the `aud` claim. Roles/scopes are re-read fresh from the DB at exchange time, so +a stale refresh token cannot carry stale authorization into a service. +""" +import logging + +from fastapi import APIRouter, Depends, HTTPException, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from pydantic import BaseModel + +from core import tokens_rs256 +from core.configuration import config +from core.database import ( + user_db_manager, jwt_user_repo, user_profile_repo, user_role_repo, +) +from core.models.user import LoginUserIn +from core.security import authenticate_user, get_current_user + +logger = logging.getLogger(__name__) + +# Auth endpoints (mounted at /api). JWKS is mounted separately at the root. +router = APIRouter() +wellknown_router = APIRouter() + +_bearer = HTTPBearer(auto_error=True) + +# Scopes are derived from roles (RBAC is the source of truth) so no separate +# scope-management (the old Django APItokenmanager) is needed. +_WRITE_ROLES = {"Admin", "SuperAdmin", "Curator", "Lab Member", "Submitter", + "Annotator", "Mapper", "Knowledge Contributor"} +_ADMIN_ROLES = {"Admin", "SuperAdmin"} + + +def _scopes_for_roles(roles) -> list: + rset = set(roles or []) + scopes = ["read"] + if rset & _WRITE_ROLES: + scopes.append("write") + if rset & _ADMIN_ROLES: + scopes.append("admin") + return scopes + + +def _merge_scopes(stored, roles) -> list: + """Scopes a token should carry: whatever is explicitly assigned, PLUS whatever + the user's roles imply. + + Roles are authoritative — that is what `/api/auth/session-exchange` and the PAT + exchange already assume ("RBAC is authoritative" in their docstrings). The + refresh-token paths did not: they read `Web_jwtuser_scopes` alone, a legacy + Django table that is only populated for accounts created through the old admin. + + An OAuth account has no rows there at all. Its `Web_jwtuser` row is the shell + created to supply a stable `user_id` claim, so a Globus SuperAdmin — including + one seeded by USERMANAGEMENT_BOOTSTRAP_SUPERADMIN_EMAILS, which assigns roles + and nothing else — got a token reading `roles: ["Admin", "SuperAdmin"]` and + `scopes: ["read"]`. query_service gates its admin routes on the scope claim + (`require_scopes(["admin"])`) and has no bootstrap allowlist of its own, so + every one of them answered 403 "Insufficient scopes" before reaching the role + check that would have passed. Meanwhile usermanagement's own admin routes + honour the env allowlist and worked — which is why the failure looked like a + missing audience rather than a scope derived from the wrong place. + + The union, rather than replacing: a legacy password account may hold an + explicitly granted scope that no role implies, and dropping it would be a + silent downgrade. + """ + return sorted(set(stored or []) | set(_scopes_for_roles(roles))) + + +class ExchangeIn(BaseModel): + audience: str + + +@wellknown_router.get("/.well-known/jwks.json", tags=["SSO"]) +async def jwks(): + """Public JWK Set. Services fetch and cache this to verify RS256 tokens.""" + return tokens_rs256.jwks() + + +@router.post("/auth/login", tags=["SSO"]) +async def sso_login(body: LoginUserIn): + """Authenticate with email/password and receive a refresh token. The refresh + token is not accepted by any service — exchange it at /api/auth/exchange.""" + async with user_db_manager.get_async_session() as session: + user_record = await authenticate_user(body.email, body.password, session) + if not user_record: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Incorrect email or password", + headers={"WWW-Authenticate": "Bearer"}, + ) + profile = await user_profile_repo.get_by_email(session, user_record.email) + profile_id = profile.id if profile else None + roles = await user_role_repo.get_user_role_names(session, profile.id) if profile else [] + # Roles first, then scopes derived from them — see _merge_scopes. A password + # SuperAdmin whose account predates the legacy scope table (or was created by + # the bootstrap allowlist) otherwise gets a read-only token too. + scopes = _merge_scopes( + await jwt_user_repo.get_user_scopes(session, user_record.id), roles) + + refresh = tokens_rs256.create_refresh_token( + email=user_record.email, + profile_id=profile_id, + roles=roles, + scopes=scopes, + auth_source="password", + ) + return { + "refresh_token": refresh, + "token_type": "refresh", + "expires_in": tokens_rs256.refresh_token_ttl_seconds(), + "audiences": config.token_audiences, + } + + +@router.post("/auth/exchange", tags=["SSO"]) +async def sso_exchange( + body: ExchangeIn, + creds: HTTPAuthorizationCredentials = Depends(_bearer), +): + """Exchange a refresh token for a short-lived access token scoped to one + service (`audience`). Roles/scopes are re-read fresh from the DB here.""" + try: + payload = tokens_rs256.verify_refresh_token(creds.credentials) + except Exception as e: + logger.info(f"Refresh token rejected: {e}") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid or expired refresh token", + headers={"WWW-Authenticate": "Bearer"}, + ) + + if body.audience not in config.token_audiences: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown audience '{body.audience}'. Allowed: {config.token_audiences}", + ) + + email = payload.get("sub") + auth_source = payload.get("auth_source", "password") + async with user_db_manager.get_async_session() as session: + # Look the credential row up regardless of is_active, because an inactive + # row means two different things here. OAuth onboarding deliberately + # creates a SHELL row with is_active=False (provision_identity / + # _ensure_jwt_user_shell) — an OAuth user has no usable password, and the + # shell exists only to supply a stable user_id claim. An active-only + # lookup therefore rejected every OAuth user: Globus/ORCID/GitHub logins + # could mint a refresh token and then never exchange it, which broke both + # the MCP/CLI flow and the UI's silent renew. + jwt_user = await jwt_user_repo.get_by_email_any_status(session, email) + if not jwt_user: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Unknown account") + # For PASSWORD credentials is_active is the deactivation switch that + # POST /api/admin/users/deactivate flips, so it must still be enforced — + # dropping the check outright would make deactivation a no-op here. + # OAuth accounts are removed by BANNING (the documented mechanism, since + # deletion is disabled), which the is_banned check below enforces. + if not jwt_user.is_active and auth_source == "password": + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Account inactive") + profile = await user_profile_repo.get_by_email(session, email) + if profile and getattr(profile, "is_banned", False): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="account_suspended") + profile_id = profile.id if profile else None + roles = await user_role_repo.get_user_role_names(session, profile.id) if profile else [] + # This is the MCP/CLI path: Globus login -> refresh token -> per-service + # access token. Reading the legacy scope table alone handed a Globus + # SuperAdmin `scopes: ["read"]` with `roles: ["Admin", "SuperAdmin"]`, and + # query_service's admin routes gate on the scope claim. See _merge_scopes. + scopes = _merge_scopes( + await jwt_user_repo.get_user_scopes(session, jwt_user.id), roles) + + access = tokens_rs256.create_access_token( + audience=body.audience, + email=email, + profile_id=profile_id, + roles=roles, + scopes=scopes, + auth_source=auth_source, + jwt_user_id=jwt_user.id, + ) + return { + "access_token": access, + "token_type": "bearer", + "aud": body.audience, + "expires_in": tokens_rs256.access_token_ttl_seconds(), + } + + +@router.post("/auth/session-exchange", tags=["SSO"]) +async def sso_session_exchange( + body: ExchangeIn, + current_user: dict = Depends(get_current_user), +): + """Exchange an authenticated **session token** (the web UI's usermanagement + JWT — v2 or SSO) for a short-lived per-service access token (`aud=`). + + This lets the web UI call query_service / ml_service with an audience-scoped + token derived from the logged-in user, instead of a shared service-account + password — so no password login is needed for those calls. Roles are re-read + fresh from the DB and scopes are derived from them (RBAC is authoritative).""" + if body.audience not in config.token_audiences: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unknown audience '{body.audience}'. Allowed: {config.token_audiences}", + ) + email = current_user.get("email") or current_user.get("sub") + if not email: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="no identity in token") + + async with user_db_manager.get_async_session() as session: + profile = await user_profile_repo.get_by_email(session, email) + profile_id = profile.id if profile else current_user.get("profile_id") + roles = await user_role_repo.get_user_role_names(session, profile.id) if profile else [] + jwt_user = await jwt_user_repo.get_by_email_any_status(session, email) + jwt_user_id = jwt_user.id if jwt_user else current_user.get("user_id") + + access = tokens_rs256.create_access_token( + audience=body.audience, + email=email, + profile_id=profile_id, + roles=roles, + scopes=_scopes_for_roles(roles), + auth_source=current_user.get("auth_source", "session"), + jwt_user_id=jwt_user_id, + ) + return { + "access_token": access, + "token_type": "bearer", + "aud": body.audience, + "expires_in": tokens_rs256.access_token_ttl_seconds(), + } diff --git a/usermanagement_service/core/security.py b/usermanagement_service/core/security.py index bfc4ebe..b3fe765 100644 --- a/usermanagement_service/core/security.py +++ b/usermanagement_service/core/security.py @@ -11,6 +11,7 @@ # ----------------------------------------------------------------------------- import logging +import os import base64 import hashlib from datetime import datetime, timedelta @@ -99,9 +100,14 @@ def create_access_token_v2( roles: List[str], scopes: List[str], auth_source: str = "password", + expires_minutes: Optional[int] = None, ) -> str: """Create a JWT carrying both JWT-level scopes and profile-level roles. - auth_source: 'password' | 'github' | 'orcid' | 'globus'.""" + auth_source: 'password' | 'github' | 'orcid' | 'globus'. + expires_minutes overrides the default 30-min TTL — the OAuth callback passes a + longer web-session TTL so the browser session token doesn't lapse after 30 min + and break the UI (it is stored in the NextAuth session and not auto-refreshed).""" + ttl = expires_minutes if (expires_minutes and expires_minutes > 0) else ACCESS_TOKEN_EXPIRE_MINUTES to_encode = { "sub": email, "scopes": scopes, @@ -109,7 +115,7 @@ def create_access_token_v2( "profile_id": profile_id, "roles": roles, "auth_source": auth_source, - "exp": datetime.utcnow() + timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES), + "exp": datetime.utcnow() + timedelta(minutes=ttl), } return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) @@ -159,11 +165,32 @@ def decrypt_token(ciphertext: Optional[str]) -> Optional[str]: return None +# The audience this service accepts in RS256 SSO access tokens (Phase 2). A token +# minted for another service (aud=query_service, ...) is NOT accepted here. +SERVICE_AUDIENCE = os.getenv("USERMANAGEMENT_SERVICE_AUDIENCE", "usermanagement") + + def verify_token(token: str) -> Union[dict, None]: - """Verify and decode a JWT token""" + """Verify and decode a bearer token, newest scheme first: + + 1. Phase 2 SSO RS256 access token minted for THIS service + (aud == usermanagement), verified with our own public key — we are the + issuer, so no network fetch. + 2. Legacy HS256 v2 token signed with this service's own secret. + + Returns the claims dict, or None if neither validates. Backward-compatible: + existing HS256 tokens keep working during migration.""" + # 1) RS256 SSO access token for this service. try: - payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) - return payload + from core import tokens_rs256 + payload = tokens_rs256.verify_access_token(token, audience=SERVICE_AUDIENCE) + if payload is not None: + return payload + except Exception as e: + logger.debug(f"RS256 verify skipped/failed, trying HS256: {e}") + # 2) Legacy HS256 token. + try: + return jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) except JWTError as e: logger.error(f"JWT token verification failed: {str(e)}") return None @@ -352,12 +379,28 @@ async def get_current_user_optional( return user_data -def require_admin(current_user: Annotated[dict, Depends(get_current_user)]) -> dict: - """Dependency: caller must have the Admin or SuperAdmin role (profile-level). - Checks JWT 'roles' claim, falling back to the bootstrap superadmin email - allowlist for the very first sign-in (before any role is assigned in the DB).""" +async def _current_roles_from_db(email: str, profile_id) -> list: + """Active role names for a user, read fresh from the DB (not the token).""" + from core.database import user_db_manager, user_role_repo, user_profile_repo + async with user_db_manager.get_async_session() as session: + pid = profile_id + if pid is None and email: + prof = await user_profile_repo.get_by_email(session, email) + pid = prof.id if prof else None + if pid is None: + return [] + return await user_role_repo.get_user_role_names(session, pid) or [] + + +async def require_admin(current_user: Annotated[dict, Depends(get_current_user)]) -> dict: + """Dependency: caller must currently hold Admin or SuperAdmin. + + Roles are re-read from the DB (not trusted from the token's `roles` claim) so a + revoked/demoted admin loses access immediately, without waiting for the token to + expire. Falls back to the bootstrap-superadmin allowlist for the very first + sign-in (before any role exists in the DB).""" email = (current_user.get("email") or "").lower() - roles = current_user.get("roles", []) or [] + roles = await _current_roles_from_db(email, current_user.get("profile_id")) if UserRoleEnum.ADMIN.value in roles or UserRoleEnum.SUPERADMIN.value in roles: return current_user diff --git a/usermanagement_service/core/tokens_rs256.py b/usermanagement_service/core/tokens_rs256.py new file mode 100644 index 0000000..bf768c6 --- /dev/null +++ b/usermanagement_service/core/tokens_rs256.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +"""RS256 single-issuer SSO tokens (auth Phase 2). + +usermanagement is the sole issuer. A single login mints a short-lived REFRESH +token (``aud=brainkb-auth``); clients EXCHANGE it for narrow, short-lived +per-service ACCESS tokens (``aud=``). Services verify tokens against +the published JWKS and require ``aud == ``, so a token minted for +one service cannot be replayed against another — containment is enforced by the +audience claim, not by separate shared secrets. + +Keys are RS256. The private key is read from ``USERMANAGEMENT_JWT_PRIVATE_KEY_PEM`` +(a PEM string) or ``USERMANAGEMENT_JWT_PRIVATE_KEY_FILE`` (a path). If neither is +set, an EPHEMERAL key is generated on first use (dev only) and a warning is +logged — such tokens do not survive a process restart. Provision a persistent +key for any real deployment so the JWKS ``kid`` is stable. +""" +import base64 +import hashlib +import logging +import os +import tempfile +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + +from jose import jwt +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from core.configuration import config + +logger = logging.getLogger(__name__) + +ALGORITHM = "RS256" +REFRESH_AUDIENCE = "brainkb-auth" # audience of the login/refresh token +REFRESH_TYP = "refresh" +ACCESS_TYP = "access" + +# Lazily initialized key material (module-level cache for the process lifetime). +_private_pem: Optional[str] = None +_public_pem: Optional[str] = None +_kid: Optional[str] = None + + +def _load_or_generate() -> None: + global _private_pem, _public_pem, _kid + if _private_pem is not None: + return + + pem = config.jwt_private_key_pem + if not pem and config.jwt_private_key_file: + try: + with open(config.jwt_private_key_file, "r") as fh: + pem = fh.read() + except OSError as e: + logger.warning(f"Could not read USERMANAGEMENT_JWT_PRIVATE_KEY_FILE: {e}") + + if not pem: + # No key configured. Fall back to a process-shared ephemeral key: persist + # it to a file so ALL uvicorn workers (and restarts) use the SAME key — + # otherwise each worker signs with its own key and cross-worker + # verification fails. Production should set an explicit key instead. + cache_path = os.getenv( + "USERMANAGEMENT_JWT_EPHEMERAL_KEY_PATH", + os.path.join(tempfile.gettempdir(), "brainkb_um_sso_key.pem"), + ) + if os.path.exists(cache_path): + try: + with open(cache_path, "r") as fh: + pem = fh.read() + logger.warning( + "Using cached EPHEMERAL RS256 key at %s. Set " + "USERMANAGEMENT_JWT_PRIVATE_KEY_PEM/_FILE for production.", + cache_path, + ) + except OSError: + pem = None + if not pem: + logger.warning( + "No USERMANAGEMENT_JWT_PRIVATE_KEY_PEM/_FILE configured — generating " + "an EPHEMERAL RS256 key at %s (shared across workers). Set a " + "persistent key for production.", + cache_path, + ) + new_priv = rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = new_priv.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode() + try: + # Atomic create: if another worker won the race, read theirs. + fd = os.open(cache_path, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + with os.fdopen(fd, "w") as fh: + fh.write(pem) + except FileExistsError: + with open(cache_path, "r") as fh: + pem = fh.read() + except OSError as e: + logger.warning(f"Could not persist ephemeral key to {cache_path}: {e}") + + priv = serialization.load_pem_private_key(pem.encode(), password=None) + pub = priv.public_key() + _private_pem = priv.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode() + _public_pem = pub.public_bytes( + serialization.Encoding.PEM, + serialization.PublicFormat.SubjectPublicKeyInfo, + ).decode() + # Stable kid derived from the public key so it changes only when the key does. + _kid = hashlib.sha256(_public_pem.encode()).hexdigest()[:16] + + +def _b64u_uint(n: int) -> str: + raw = n.to_bytes((n.bit_length() + 7) // 8 or 1, "big") + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def jwks() -> Dict[str, Any]: + """Public JWK Set for token verification (served at /.well-known/jwks.json).""" + _load_or_generate() + pub = serialization.load_pem_public_key(_public_pem.encode()) + nums = pub.public_numbers() + return { + "keys": [ + { + "kty": "RSA", + "use": "sig", + "alg": ALGORITHM, + "kid": _kid, + "n": _b64u_uint(nums.n), + "e": _b64u_uint(nums.e), + } + ] + } + + +def _now() -> datetime: + return datetime.now(timezone.utc) + + +def create_refresh_token( + *, + email: str, + profile_id: Optional[int], + roles: List[str], + scopes: List[str], + auth_source: str = "password", + expires_minutes: Optional[int] = None, +) -> str: + """Mint the login/refresh token (aud=brainkb-auth). Not accepted by services; + only exchangeable at /api/auth/exchange for a per-service access token. + expires_minutes overrides the default refresh TTL (the web flow passes a longer + one so the browser session can silently refresh over several days).""" + _load_or_generate() + now = _now() + ttl = expires_minutes if (expires_minutes and expires_minutes > 0) else config.refresh_token_ttl_min + claims = { + "iss": config.jwt_issuer, + "aud": REFRESH_AUDIENCE, + "sub": email, + "typ": REFRESH_TYP, + "profile_id": profile_id, + "roles": roles, + "scopes": scopes, + "auth_source": auth_source, + "iat": now, + "exp": now + timedelta(minutes=ttl), + } + return jwt.encode(claims, _private_pem, algorithm=ALGORITHM, headers={"kid": _kid}) + + +def create_access_token( + *, + audience: str, + email: str, + profile_id: Optional[int], + roles: List[str], + scopes: List[str], + auth_source: str = "password", + jwt_user_id: Optional[int] = None, +) -> str: + """Mint a narrow, short-lived access token for a single service (aud=).""" + _load_or_generate() + now = _now() + claims = { + "iss": config.jwt_issuer, + "aud": audience, + "sub": email, + "typ": ACCESS_TYP, + "user_id": jwt_user_id, + "profile_id": profile_id, + "roles": roles, + "scopes": scopes, + "auth_source": auth_source, + "iat": now, + "exp": now + timedelta(minutes=config.access_token_ttl_min), + } + return jwt.encode(claims, _private_pem, algorithm=ALGORITHM, headers={"kid": _kid}) + + +def verify_access_token(token: str, audience: str) -> Optional[Dict[str, Any]]: + """Verify an RS256 access token for ``audience`` using our OWN public key + (usermanagement is the issuer, so no network/JWKS fetch is needed). Returns + the claims on success, or None if it is not a valid RS256 access token for + this audience. Never raises — callers fall back to legacy HS256.""" + _load_or_generate() + try: + header = jwt.get_unverified_header(token) + except Exception: + return None + if header.get("alg") != ALGORITHM: + return None + try: + payload = jwt.decode( + token, _public_pem, algorithms=[ALGORITHM], + audience=audience, issuer=config.jwt_issuer, + ) + except Exception: + return None + if payload.get("typ") not in (None, ACCESS_TYP): + return None + return payload + + +def verify_refresh_token(token: str) -> Dict[str, Any]: + """Validate a refresh token (signature, iss, aud, exp). Raises jose errors on + failure. Verified with our own public key (no network).""" + _load_or_generate() + payload = jwt.decode( + token, + _public_pem, + algorithms=[ALGORITHM], + audience=REFRESH_AUDIENCE, + issuer=config.jwt_issuer, + ) + if payload.get("typ") != REFRESH_TYP: + raise ValueError("not a refresh token") + return payload + + +def access_token_ttl_seconds() -> int: + return config.access_token_ttl_min * 60 + + +def refresh_token_ttl_seconds() -> int: + return config.refresh_token_ttl_min * 60 diff --git a/usermanagement_service/requirements.txt b/usermanagement_service/requirements.txt index 787c7dd..2cb73d0 100644 --- a/usermanagement_service/requirements.txt +++ b/usermanagement_service/requirements.txt @@ -4,7 +4,12 @@ uvicorn==0.29.0 gunicorn==21.2.0 # HTTP requests -aiohttp==3.9.1 +# Shared site-packages: all four services install into ONE environment in +# Dockerfile.unified, so an exact pin here overrides what another service needs. +# ==3.9.1 downgraded the aiohttp ml_service requires (>=3.10, for +# aiohttp.SocketTimeoutError, which openai's vendored httpx_aiohttp imports) and +# silently disabled every /api/synth-scholar route. +aiohttp>=3.10,<4 # Logging rich==13.9.4 @@ -34,7 +39,8 @@ python-dotenv==1.0.0 asyncpg==0.29.0 # ORM Framework -sqlalchemy==2.0.23 +# Was ==2.0.23, which violated ml_service's >=2.0.30 in the shared environment. +sqlalchemy>=2.0.30 alembic==1.13.1 greenlet==3.0.3