diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a9155b3..16d18a1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,11 +15,14 @@ Secrets and privacy recognises far more: the built-in redaction catalog grows from 30 rules to 120 in 20 groups, and the API keys saved for the providers are recognised by value, whatever their format. The console shows the rules' names and descriptions as thinkwatch-core -writes them, in both languages. The thinkwatch-core crates move from -v0.83.1 to v0.85.0. +writes them, in both languages. A value recognised by mistake can be made +an exception without switching its rule off, and long conversations are +no longer scanned from the start every turn. The thinkwatch-core crates +move from v0.83.1 to v0.86.0. -No new setting, column or route; the redaction policy's shape is -unchanged. +No new setting or column. The redaction policy gains one optional field +(`exceptions`), with endpoints of its own under +`/api/admin/security/redact/exceptions`. ### Read before upgrading @@ -40,7 +43,7 @@ unchanged. ### Added -- **Expanded redaction rules** (thinkwatch-core v0.85.0): about 50 more +- **Expanded redaction rules** (thinkwatch-core v0.85.0 and later): about 50 more prefixed credentials (AI services, code hosting, cloud, payments and collaboration), credentials by position (request headers, command lines, URL parameters, bot webhooks, configuration assignments), @@ -67,8 +70,32 @@ unchanged. and no masked values; the security log shows it as the provider's API key. +- **Exception values** (`security.redact.exceptions`, the Exceptions + section under the Secrets and privacy rules). A value listed there is + neither replaced nor recorded, whichever rule finds it; the rules keep + working for every other value. A value is added by pasting it: the + server hashes it on arrival and keeps only its SHA-256, its masked form, + the rule that recognised it and when it was added, never the value. At + most 500; at the cap, adding is unavailable. Saved provider keys cannot + be exceptions. Listing them takes `settings:read`; adding and removing + take `settings:write` and `pii_redactor:write`, and each is written to + the audit log (`security.redact_exception_added`, + `security.redact_exception_removed`) with the rule and the masked value + only. The trial marks an excepted value as left as is. +- **Unchanged conversation history is not scanned again.** Each gateway + instance keeps, in memory, where the redaction rules matched in the + parts of recent requests it has scanned (positions and rule ids, never + values); the next turn of the same conversation takes those parts from + it and scans only what is new. What is found is the same as without it. + The cache is rebuilt whenever the rules, the providers' keys or the + exceptions change. + ### Changed +- **A redaction policy written through `PATCH /api/admin/settings` keeps + the stored exceptions**, whatever it carries: they are changed only + through their own endpoints, and the console writes the rest of the + policy back from the rule view, which does not carry them. - **Rule names and descriptions come from thinkwatch-core.** The security page shows each built-in rule's name, reason and description and each group's name as the rule view gives them, in English or Chinese; the diff --git a/Cargo.lock b/Cargo.lock index 1ca394cd..9c607902 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2944,7 +2944,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", + "socket2 0.6.5", "thiserror 2.0.21", "tokio", "tracing", @@ -2983,7 +2983,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.5", "tracing", "windows-sys 0.61.2", ] @@ -4418,6 +4418,7 @@ dependencies = [ "tower-http 0.7.1", "tracing", "tracing-subscriber", + "tw-guard", "url", "uuid", "wiremock", @@ -4837,8 +4838,8 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.85.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.85.0#7134a31380ca2b3784079e3f3e0d2f7c64ec967b" +version = "0.86.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.86.0#62aec779f28b8f2f087670cc5096af829bd5ad20" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -4852,16 +4853,16 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.85.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.85.0#7134a31380ca2b3784079e3f3e0d2f7c64ec967b" +version = "0.86.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.86.0#62aec779f28b8f2f087670cc5096af829bd5ad20" dependencies = [ "serde", ] [[package]] name = "tw-dialect" -version = "0.85.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.85.0#7134a31380ca2b3784079e3f3e0d2f7c64ec967b" +version = "0.86.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.86.0#62aec779f28b8f2f087670cc5096af829bd5ad20" dependencies = [ "memchr", "serde", @@ -4870,8 +4871,8 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.85.0" -source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.85.0#7134a31380ca2b3784079e3f3e0d2f7c64ec967b" +version = "0.86.0" +source = "git+https://github.com/ThinkWatchProject/ThinkWatch-Core.git?tag=v0.86.0#62aec779f28b8f2f087670cc5096af829bd5ad20" dependencies = [ "aho-corasick", "base64 0.22.1", diff --git a/Cargo.toml b/Cargo.toml index a1099855..79fa1aa8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,10 +70,10 @@ opt-level = 3 # never re-exported through a local shim. And the reverse: something only # this side uses (the at-rest crypto, IMDSv2 credentials, the gateway error) # lives here, not in core. -tw-bedrock = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.85.0" } -tw-breaker = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.85.0" } -tw-dialect = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.85.0" } -tw-guard = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.85.0" } +tw-bedrock = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.86.0" } +tw-breaker = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.86.0" } +tw-dialect = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.86.0" } +tw-guard = { git = "https://github.com/ThinkWatchProject/ThinkWatch-Core.git", tag = "v0.86.0" } # Web framework axum = { version = "0.8", features = ["macros", "ws"] } diff --git a/crates/common/src/guard_policy/exceptions.rs b/crates/common/src/guard_policy/exceptions.rs new file mode 100644 index 00000000..5094821c --- /dev/null +++ b/crates/common/src/guard_policy/exceptions.rs @@ -0,0 +1,102 @@ +//! Exception values of outbound redaction (`security.redact.exceptions`): +//! values that are never replaced or recorded, whichever rule finds them. +//! +//! They live in the redaction policy, as thinkwatch-core defines it +//! (`tw_guard::policy::RedactException`): **the SHA-256 of each value and +//! its masked form, never the value**. The value an admin pastes is hashed +//! as soon as it arrives and goes no further. +//! +//! **They are edited only through their own endpoints** +//! (`/api/admin/security/redact/exceptions`). The console writes the rest +//! of the policy back whole, rebuilt from the rule view, and the view does +//! not carry them: a policy written through the settings endpoint keeps the +//! stored exceptions ([`keep_stored`]) rather than losing them. + +use serde_json::Value; +use tw_guard::policy::RedactException; + +/// The exceptions a stored redaction policy holds. A policy that is +/// missing, or whose exceptions cannot be read, holds none. +pub fn stored(policy: Option<&Value>) -> Vec { + policy + .and_then(|p| p.get("exceptions")) + .and_then(|e| serde_json::from_value(e.clone()).ok()) + .unwrap_or_default() +} + +/// `policy` (a stored redaction policy, or none) with `list` as its +/// exceptions; the field is left out when the list is empty, as the +/// factory policy leaves it out. Everything else stays as stored. +pub fn with_list(policy: Option, list: &[RedactException]) -> Value { + let mut policy = match policy { + Some(Value::Object(map)) => Value::Object(map), + _ => Value::Object(Default::default()), + }; + let map = policy.as_object_mut().expect("an object, just made one"); + if list.is_empty() { + map.remove("exceptions"); + } else { + map.insert( + "exceptions".into(), + serde_json::to_value(list).expect("exceptions serialise"), + ); + } + policy +} + +/// A redaction policy about to be saved through the settings endpoint, +/// with the exceptions `stored` holds instead of whatever it carries. +pub fn keep_stored(incoming: Value, stored_policy: Option<&Value>) -> Value { + match incoming { + Value::Object(_) => with_list(Some(incoming), &stored(stored_policy)), + // Not an object: validation refuses it as it is. + other => other, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn one(sha: char) -> RedactException { + RedactException { + sha256: sha.to_string().repeat(64), + masked: "sk-an…7f9c".into(), + rule: "anthropic-api-key".into(), + added_at: 1, + } + } + + #[test] + fn a_settings_write_keeps_the_stored_exceptions() { + let stored_policy = with_list(Some(json!({"mode": "observe"})), &[one('a')]); + assert_eq!(stored(Some(&stored_policy)), [one('a')]); + // The console writes the policy without them, or with others. + for incoming in [ + json!({"mode": "enforce"}), + json!({"mode": "enforce", "exceptions": [one('b')]}), + json!({"mode": "enforce", "exceptions": []}), + ] { + let kept = keep_stored(incoming, Some(&stored_policy)); + assert_eq!(kept["mode"], "enforce"); + assert_eq!(stored(Some(&kept)), [one('a')]); + } + // Nothing stored: none come in either. + let kept = keep_stored(json!({"exceptions": [one('b')]}), None); + assert!(kept.get("exceptions").is_none(), "{kept}"); + // Not an object: left for validation to refuse. + assert_eq!(keep_stored(json!([]), Some(&stored_policy)), json!([])); + } + + #[test] + fn an_empty_list_leaves_the_field_out_and_the_rest_as_stored() { + let p = with_list( + Some(json!({"mode": "enforce", "disable": ["jwt"], "exceptions": [one('a')]})), + &[], + ); + assert_eq!(p, json!({"mode": "enforce", "disable": ["jwt"]})); + assert_eq!(with_list(None, &[]), json!({})); + assert!(stored(Some(&json!({"exceptions": "not a list"}))).is_empty()); + } +} diff --git a/crates/common/src/guard_policy/mod.rs b/crates/common/src/guard_policy/mod.rs index 193dddc0..876ebd7c 100644 --- a/crates/common/src/guard_policy/mod.rs +++ b/crates/common/src/guard_policy/mod.rs @@ -29,6 +29,7 @@ use tw_guard::redact::rules::RuleSet; use crate::dynamic_config::DynamicConfig; +pub mod exceptions; pub mod legacy; pub mod saved; diff --git a/crates/gateway/src/redaction.rs b/crates/gateway/src/redaction.rs index f7533eb7..ab939fd3 100644 --- a/crates/gateway/src/redaction.rs +++ b/crates/gateway/src/redaction.rs @@ -36,7 +36,8 @@ use std::sync::Arc; use think_watch_common::audit::AuditLogger; use think_watch_common::guard_policy::saved::{self, SavedSecrets}; use tw_guard::policy::{Mode, RedactPolicy}; -use tw_guard::redact::replace::Ledger; +use tw_guard::redact::flow::ScanCache; +use tw_guard::redact::replace::{Ledger, Scheme}; use tw_guard::redact::rules::{Finding, Kind, Rule, RuleSet, SavedSource}; pub use tw_guard::redact::flow::Shape; @@ -44,11 +45,22 @@ pub use tw_guard::redact::flow::Shape; use crate::guards::Caller; /// Outbound redaction as configured: a mode, and the rules — with the -/// providers' credentials, recognised by value. +/// providers' credentials, recognised by value, and the exception values +/// that are not. +/// +/// **One scan cache per compiled rule set** (thinkwatch-core's +/// [`ScanCache`]): a client sends the whole conversation every turn, and +/// the parts of it already scanned last turn are taken from the cache +/// instead of being scanned again. What is found is exactly what a scan +/// without it finds. The cache keeps hit positions and rule ids, never a +/// value, in this process's memory only; it is built with the rules and +/// dropped with them when a policy, the providers' keys or the exceptions +/// change. #[derive(Debug, Clone)] pub struct Redaction { pub mode: Mode, pub rules: Arc, + cache: Arc, } impl Redaction { @@ -59,6 +71,7 @@ impl Redaction { think_watch_common::guard_policy::redact_rules(policy), saved, )), + cache: Arc::new(ScanCache::new()), } } @@ -66,7 +79,15 @@ impl Redaction { /// in its written text (masked), and in enforce mode the ledger every /// hop is replaced with. Nothing when off. pub fn look(&self, body: &[u8], shape: Shape) -> (Vec, Ledger) { - tw_guard::redact::flow::look(self.mode, &self.rules, body, shape) + let l = tw_guard::redact::flow::look_hits_with( + self.mode, + &self.rules, + body, + shape, + Ledger::new(Scheme::SECRET), + Some(&self.cache), + ); + (l.found, l.ledger) } /// The bytes one hop sends, a request of `shape` — the format it goes @@ -77,12 +98,13 @@ impl Redaction { /// Byte for byte what came in, and the same ledger, unless in enforce /// mode and something is found. pub fn replace(&self, body: Vec, ledger: &Ledger, shape: Shape) -> (Vec, Ledger) { - let (out, ledger) = tw_guard::redact::flow::replace( + let (out, ledger) = tw_guard::redact::flow::replace_with( self.mode, &self.rules, bytes::Bytes::from(body), ledger, shape, + Some(&self.cache), ); (Vec::from(out), ledger) } @@ -552,6 +574,67 @@ mod tests { assert!(!writes_masked(&Rule::Custom("ssn".into()))); } + #[test] + fn an_exception_value_is_neither_replaced_nor_recorded() { + let other = "sk-ant-api03-BBBBBBBBBBBBBBBBBBBBBBBBBBBB"; + let r = Redaction::new( + &RedactPolicy { + mode: Mode::Enforce, + exceptions: vec![tw_guard::policy::RedactException { + sha256: tw_guard::redact::exceptions::sha256_hex(KEY), + masked: "sk-an…AAAA".into(), + rule: "anthropic-api-key".into(), + added_at: 1, + }], + ..Default::default() + }, + &Default::default(), + ); + let body = serde_json::json!({"messages": [ + {"role": "user", "content": format!("{KEY} and {other}")} + ]}) + .to_string(); + let (found, ledger) = r.look(body.as_bytes(), CHAT); + assert_eq!(found.len(), 1, "{found:?}"); + let (sent, _) = r.replace(body.into_bytes(), &ledger, CHAT); + let sent = String::from_utf8(sent).unwrap(); + assert!(sent.contains(KEY) && !sent.contains(other), "{sent}"); + } + + /// The scan cache takes the unchanged history from the turn before; the + /// findings, the numbering and what is sent are the same as without it. + #[test] + fn a_growing_conversation_redacts_the_same_turn_after_turn() { + let r = redaction(Mode::Enforce); + let filler = "lorem ipsum dolor sit amet ".repeat(200); + let mut messages = Vec::new(); + for turn in 0..30 { + let content = if turn % 7 == 0 { + format!( + "{filler} turn {turn} key sk-ant-api03-{}", + format!("{turn:02}").repeat(14) + ) + } else { + format!("{filler} turn {turn}") + }; + messages.push(serde_json::json!({"role": "user", "content": content})); + let body = serde_json::json!({"messages": messages}).to_string(); + let cached = r.look(body.as_bytes(), CHAT); + let plain = + tw_guard::redact::flow::look(Mode::Enforce, &r.rules, body.as_bytes(), CHAT); + assert_eq!(cached.0, plain.0, "turn {turn}"); + let (a, _) = r.replace(body.clone().into_bytes(), &cached.1, CHAT); + let (b, _) = tw_guard::redact::flow::replace( + Mode::Enforce, + &r.rules, + bytes::Bytes::from(body), + &plain.1, + CHAT, + ); + assert_eq!(a, b.to_vec(), "turn {turn}"); + } + } + #[test] fn findings_are_one_entry_per_rule_and_a_custom_rule_keeps_no_values() { let r = Redaction::new( diff --git a/crates/server/src/app.rs b/crates/server/src/app.rs index 2906396b..e4e551dc 100644 --- a/crates/server/src/app.rs +++ b/crates/server/src/app.rs @@ -1008,6 +1008,16 @@ pub fn create_console_app(config: &AppConfig, state: AppState) -> anyhow::Result "/api/admin/security/{guard}/test", post(handlers::admin::test_security), ) + // Redaction's exception values: their own endpoints, since the + // console writes the rest of the policy back whole. + .route( + "/api/admin/security/redact/exceptions", + get(handlers::admin::list_exceptions).post(handlers::admin::add_exception), + ) + .route( + "/api/admin/security/redact/exceptions/{sha256}", + delete(handlers::admin::remove_exception), + ) // Log forwarders CRUD .route( "/api/admin/log-forwarders", diff --git a/crates/server/src/handlers/admin.rs b/crates/server/src/handlers/admin.rs index f9ac1362..f2b259db 100644 --- a/crates/server/src/handlers/admin.rs +++ b/crates/server/src/handlers/admin.rs @@ -19,7 +19,7 @@ pub use oidc::{ pub use retention::{ check_body_retention_vs_lifecycle, reconcile_blob_lifecycle, reconcile_clickhouse_ttls, }; -pub use security::{get_security, test_security}; +pub use security::{add_exception, get_security, list_exceptions, remove_exception, test_security}; pub use settings::{ AuditConfigResponse, SystemInfo, UpdateSettingsRequest, get_all_settings, get_audit_settings, get_settings_by_category, get_system_settings, update_settings, @@ -42,7 +42,10 @@ pub use oidc::{ __path_update_oidc_draft, }; #[allow(unused_imports)] -pub use security::{__path_get_security, __path_test_security}; +pub use security::{ + __path_add_exception, __path_get_security, __path_list_exceptions, __path_remove_exception, + __path_test_security, +}; #[allow(unused_imports)] pub use settings::{ __path_get_all_settings, __path_get_audit_settings, __path_get_settings_by_category, diff --git a/crates/server/src/handlers/admin/security.rs b/crates/server/src/handlers/admin/security.rs index d96c8a05..eccb5db5 100644 --- a/crates/server/src/handlers/admin/security.rs +++ b/crates/server/src/handlers/admin/security.rs @@ -14,14 +14,25 @@ //! `content_filter:read` for the content filter and tool-call inspection; //! changing a policy takes `settings:write` and the guard's `:write`. //! Reading the policies is reading settings. +//! +//! Redaction's exception values have endpoints of their own +//! (`/api/admin/security/redact/exceptions`, see +//! `think_watch_common::guard_policy::exceptions`): listing them is reading +//! settings, adding and removing them is changing the redaction policy. +//! A value pasted to add is hashed on arrival; only its SHA-256 and masked +//! form are stored, answered or written to the audit log. use axum::Json; use axum::extract::{Path, State}; -use tw_guard::policy::Guard; +use axum::http::StatusCode; +use serde::{Deserialize, Serialize}; +use tw_guard::policy::{EXCEPTIONS_MAX, Guard, RedactException, RedactPolicy}; use tw_guard::trial::{TrialRequest, TrialResult}; use tw_guard::view::SecurityDetail; +use think_watch_common::dynamic_config; use think_watch_common::errors::AppError; +use think_watch_common::guard_policy; use crate::app::AppState; use crate::middleware::auth_guard::AuthUser; @@ -114,6 +125,205 @@ pub async fn test_security( .map_err(|e| AppError::BadRequest(e.to_string())) } +/// `GET /api/admin/security/redact/exceptions`. +#[derive(Debug, Serialize, utoipa::ToSchema)] +pub struct ExceptionList { + /// `{ sha256, masked, rule?, added_at }`, in the order added. + #[schema(value_type = Vec)] + pub items: Vec, + /// How many there may be. Adding is refused at this many. + pub max: usize, +} + +/// `POST /api/admin/security/redact/exceptions`: the value to except, as it +/// appears in requests. Surrounding whitespace is ignored. +#[derive(Deserialize, utoipa::ToSchema)] +pub struct AddException { + pub value: String, +} + +/// Never the value: this type ends up in error reports. +impl std::fmt::Debug for AddException { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AddException").finish_non_exhaustive() + } +} + +/// GET /api/admin/security/redact/exceptions — redaction's exception +/// values: each one's SHA-256, masked form, the rule that first recognised +/// it and when it was added. +#[utoipa::path( + get, + path = "/api/admin/security/redact/exceptions", + tag = "Settings", + responses( + (status = 200, description = "The exception values", body = ExceptionList), + (status = 403, description = "Forbidden"), + ), + security(("BearerAuth" = [])) +)] +pub async fn list_exceptions( + auth_user: AuthUser, + State(state): State, +) -> Result, AppError> { + auth_user + .require_global_permission(&state.db, "settings:read") + .await?; + let stored = state + .dynamic_config + .get(guard_policy::key(Guard::Redact)) + .await; + Ok(Json(ExceptionList { + items: guard_policy::exceptions::stored(stored.as_ref()), + max: EXCEPTIONS_MAX, + })) +} + +/// POST /api/admin/security/redact/exceptions — add a value. It is hashed +/// here and never stored, logged or answered; the answer is the exception +/// as stored. A saved provider key cannot be an exception. +#[utoipa::path( + post, + path = "/api/admin/security/redact/exceptions", + tag = "Settings", + request_body = AddException, + responses( + (status = 201, description = "Added: the exception as stored", body = Object), + (status = 400, description = "Empty, a saved provider key, or the list is full"), + (status = 403, description = "Forbidden"), + (status = 409, description = "Already an exception"), + ), + security(("BearerAuth" = [])) +)] +pub async fn add_exception( + auth_user: AuthUser, + State(state): State, + Json(req): Json, +) -> Result<(StatusCode, Json), AppError> { + require_write(&auth_user, &state).await?; + let saved = state.saved_secrets.load().compiled(); + let added = edit_exceptions(&state, &auth_user, |policy, list| { + let e = policy + .exception_of(&req.value, saved, chrono::Utc::now().timestamp_millis()) + .map_err(|e| AppError::BadRequest(e.to_string()))?; + if list.iter().any(|x| x.sha256 == e.sha256) { + return Err(AppError::Conflict("This value is already an exception.".into())); + } + if list.len() >= EXCEPTIONS_MAX { + return Err(AppError::BadRequest(format!( + "There are already {EXCEPTIONS_MAX} exceptions, the most there may be; remove one first." + ))); + } + list.push(e.clone()); + Ok(e) + }) + .await?; + state.audit.log( + auth_user + .audit("security.redact_exception_added") + .resource("system_settings") + .detail(serde_json::json!({"rule": added.rule, "masked": added.masked})), + ); + Ok((StatusCode::CREATED, Json(added))) +} + +/// DELETE /api/admin/security/redact/exceptions/{sha256} — remove one. +/// The answer is the exception removed. +#[utoipa::path( + delete, + path = "/api/admin/security/redact/exceptions/{sha256}", + tag = "Settings", + params(("sha256" = String, Path, description = "The exception's SHA-256, as listed")), + responses( + (status = 200, description = "Removed: the exception as it was stored", body = Object), + (status = 403, description = "Forbidden"), + (status = 404, description = "No such exception"), + ), + security(("BearerAuth" = [])) +)] +pub async fn remove_exception( + auth_user: AuthUser, + State(state): State, + Path(sha256): Path, +) -> Result, AppError> { + require_write(&auth_user, &state).await?; + let removed = edit_exceptions(&state, &auth_user, |_, list| { + let at = list + .iter() + .position(|e| e.sha256 == sha256) + .ok_or_else(|| AppError::NotFound("There is no such exception.".into()))?; + Ok(list.remove(at)) + }) + .await?; + state.audit.log( + auth_user + .audit("security.redact_exception_removed") + .resource("system_settings") + .detail(serde_json::json!({"rule": removed.rule, "masked": removed.masked})), + ); + Ok(Json(removed)) +} + +/// Changing the exceptions is changing the redaction policy. +async fn require_write(auth_user: &AuthUser, state: &AppState) -> Result<(), AppError> { + for p in ["settings:write", write_permission(Guard::Redact)] { + auth_user.require_global_permission(&state.db, p).await?; + } + Ok(()) +} + +/// Change the stored exceptions with `f`, which sees the policy as stored +/// and its exception list, and swap the guards in on every instance. The +/// row is locked from read to write, so two admins adding at once on two +/// instances both land. Nothing is written when `f` refuses. +async fn edit_exceptions( + state: &AppState, + auth_user: &AuthUser, + f: impl FnOnce(&RedactPolicy, &mut Vec) -> Result, +) -> Result { + let key = guard_policy::key(Guard::Redact); + let mut tx = state.db.begin().await?; + let stored: Option = + sqlx::query_scalar("SELECT value FROM system_settings WHERE key = $1 FOR UPDATE") + .bind(key) + .fetch_optional(&mut *tx) + .await?; + let policy: RedactPolicy = match &stored { + None => RedactPolicy::default(), + Some(v) => serde_json::from_value(v.clone()).map_err(|_| { + AppError::BadRequest(format!( + "The stored {key} cannot be read; save the redaction settings again first." + )) + })?, + }; + let mut list = policy.exceptions.clone(); + let out = f(&policy, &mut list)?; + let value = guard_policy::exceptions::with_list(stored, &list); + guard_policy::validate(Guard::Redact, &value).map_err(AppError::BadRequest)?; + let written = sqlx::query( + "UPDATE system_settings SET value = $1, updated_by = $2, updated_at = now() WHERE key = $3", + ) + .bind(&value) + .bind(auth_user.claims.sub) + .bind(key) + .execute(&mut *tx) + .await?; + if written.rows_affected() == 0 { + return Err(AppError::Internal(anyhow::anyhow!( + "system setting {key} is missing" + ))); + } + tx.commit().await?; + state + .dynamic_config + .reload() + .await + .map_err(AppError::Internal)?; + crate::app::reload_guards(state).await; + dynamic_config::notify_config_changed(&state.redis).await; + Ok(out) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/server/src/handlers/admin/settings.rs b/crates/server/src/handlers/admin/settings.rs index 7e7a3cc9..7a764958 100644 --- a/crates/server/src/handlers/admin/settings.rs +++ b/crates/server/src/handlers/admin/settings.rs @@ -265,7 +265,7 @@ pub struct UpdateSettingsRequest { pub async fn update_settings( auth_user: AuthUser, State(state): State, - Json(req): Json, + Json(mut req): Json, ) -> Result, AppError> { let mut permissions: Vec<&'static str> = std::iter::once("settings:write") .chain( @@ -282,6 +282,23 @@ pub async fn update_settings( .require_global_permission(&state.db, permission) .await?; } + // Redaction's exception values are edited through their own endpoints + // (`security::add_exception`): a policy written here keeps the stored + // ones, whatever it carries. The console rebuilds the policy from the + // rule view, which does not carry them. + let redact_key = guard_policy::key(tw_guard::policy::Guard::Redact); + if let Some(incoming) = req.settings.remove(redact_key) { + let stored: Option = + sqlx::query_scalar("SELECT value FROM system_settings WHERE key = $1") + .bind(redact_key) + .fetch_optional(&state.db) + .await?; + req.settings.insert( + redact_key.to_string(), + guard_policy::exceptions::keep_stored(incoming, stored.as_ref()), + ); + } + // Validate each setting for (key, value) in &req.settings { validate_setting(key, value)?; diff --git a/crates/server/src/openapi.rs b/crates/server/src/openapi.rs index 2994200a..ff483609 100644 --- a/crates/server/src/openapi.rs +++ b/crates/server/src/openapi.rs @@ -125,6 +125,9 @@ use crate::services::team_repository::Team; crate::handlers::admin::update_settings, crate::handlers::admin::get_security, crate::handlers::admin::test_security, + crate::handlers::admin::list_exceptions, + crate::handlers::admin::add_exception, + crate::handlers::admin::remove_exception, // Teams crate::handlers::teams::list_teams, crate::handlers::teams::get_team, diff --git a/crates/test-support/Cargo.toml b/crates/test-support/Cargo.toml index 3d0b1601..bd6683f7 100644 --- a/crates/test-support/Cargo.toml +++ b/crates/test-support/Cargo.toml @@ -56,3 +56,4 @@ rust_decimal = { workspace = true } [dev-dependencies] jsonwebtoken = { workspace = true } tokio = { workspace = true, features = ["test-util"] } +tw-guard = { workspace = true } diff --git a/crates/test-support/tests/content_filter_pii.rs b/crates/test-support/tests/content_filter_pii.rs index 0908e43e..0c682a38 100644 --- a/crates/test-support/tests/content_filter_pii.rs +++ b/crates/test-support/tests/content_filter_pii.rs @@ -1541,3 +1541,137 @@ async fn provider_credentials_reach_redaction_when_the_providers_load() { assert!(app.state.saved_secrets.load().is_empty()); assert!(app.state.guards.load().redaction.rules.saved().is_none()); } + +#[ignore = "integration test — run via `make test-it`"] +#[tokio::test] +async fn an_exception_value_is_added_by_hash_kept_across_settings_writes_and_removed() { + let app = TestApp::spawn_with_clickhouse().await; + let (con, admin) = admin_session_with_user(&app).await; + let admin_id = admin.user.id.to_string(); + app.set_setting("security.redact", json!({"mode": "enforce"})) + .await; + let upstream = echo_upstream().await; + let (key, _) = seed_route(&app, &upstream.uri(), "redact-exception").await; + let path = "/api/admin/security/redact/exceptions"; + + let r = con + .post(path, json!({"value": format!(" {KEY}\n")})) + .await + .unwrap(); + assert_eq!(r.status, 201, "{}", r.text()); + let added: Value = r.json().unwrap(); + assert_eq!(added["rule"], "anthropic-api-key"); + assert_eq!( + added["sha256"], + tw_guard::redact::exceptions::sha256_hex(KEY), + "hashed without the surrounding whitespace" + ); + assert!(!r.text().contains(KEY), "{}", r.text()); + // Once only. + let again = con.post(path, json!({"value": KEY})).await.unwrap(); + assert_eq!(again.status, 409, "{}", again.text()); + let empty = con.post(path, json!({"value": " "})).await.unwrap(); + assert_eq!(empty.status, 400, "{}", empty.text()); + + let list: Value = con.get(path).await.unwrap().json().unwrap(); + assert_eq!(list["max"], 500); + assert_eq!(list["items"].as_array().unwrap().len(), 1); + // Stored as the hash and the masked form, never the value. + let stored = app + .state + .dynamic_config + .get("security.redact") + .await + .unwrap(); + assert!(!stored.to_string().contains(KEY), "{stored}"); + + // The console writes the policy back without them: they stay. + let r = con + .patch( + "/api/admin/settings", + json!({"settings": {"security.redact": {"mode": "enforce", "disable": ["jwt"]}}}), + ) + .await + .unwrap(); + assert_eq!(r.status, 200, "{}", r.text()); + let list: Value = con.get(path).await.unwrap().json().unwrap(); + assert_eq!(list["items"].as_array().unwrap().len(), 1, "{list}"); + + // The gateway sends it as it is; the trial says it is an exception. + let (status, text) = post_as( + &app, + &key, + "/v1/chat/completions", + &json!({"model": "redact-exception", "messages": [ + {"role": "user", "content": format!("use {KEY}")} + ]}), + ) + .await; + assert_eq!(status, 200, "{text}"); + let body = String::from_utf8_lossy(&upstream.received_requests().await[0].body).into_owned(); + assert!(body.contains(KEY), "{body}"); + let trial: Value = con + .post( + "/api/admin/security/redact/test", + json!({"sample": format!("key {KEY}")}), + ) + .await + .unwrap() + .json() + .unwrap(); + assert_eq!(trial["hits"][0]["excepted"], true, "{trial}"); + assert_eq!(trial["output"], Value::Null, "{trial}"); + + // The audit log has the rule and the masked value only. + let events = audited(&app, &admin_id, "security.redact_exception_added").await; + assert_eq!(events[0]["rule"], "anthropic-api-key"); + assert!(!events[0].to_string().contains(KEY), "{}", events[0]); + + let sha = added["sha256"].as_str().unwrap(); + let r = con.delete(&format!("{path}/{sha}")).await.unwrap(); + assert_eq!(r.status, 200, "{}", r.text()); + let r = con.delete(&format!("{path}/{sha}")).await.unwrap(); + assert_eq!(r.status, 404); + let list: Value = con.get(path).await.unwrap().json().unwrap(); + assert!(list["items"].as_array().unwrap().is_empty()); + let stored = app + .state + .dynamic_config + .get("security.redact") + .await + .unwrap(); + assert_eq!(stored, json!({"mode": "enforce", "disable": ["jwt"]})); + let events = audited(&app, &admin_id, "security.redact_exception_removed").await; + assert_eq!(events[0]["masked"], added["masked"]); +} + +#[ignore = "integration test — run via `make test-it`"] +#[tokio::test] +async fn a_saved_provider_key_cannot_be_an_exception() { + use think_watch_common::json_secret::JsonSecret; + let app = TestApp::spawn().await; + let con = admin_session(&app).await; + let saved = "test-saved-upstream-key-0123456789"; + let enc = JsonSecret::encrypt(&format!("Bearer {saved}"), &app.state.config.encryption_key) + .unwrap() + .to_json(); + fixtures::create_provider( + &app.db, + &unique_name("saved-ex"), + "openai", + "https://api.example.com/v1", + Some(json!({"headers": [{"key": "Authorization", "value": enc}]})), + ) + .await + .unwrap(); + app.rebuild_gateway_router().await; + let r = con + .post( + "/api/admin/security/redact/exceptions", + json!({"value": saved}), + ) + .await + .unwrap(); + assert_eq!(r.status, 400, "{}", r.text()); + assert!(!r.text().contains(saved), "{}", r.text()); +} diff --git a/web/src/i18n/en.json b/web/src/i18n/en.json index 6af0e53d..c0c4fc92 100644 --- a/web/src/i18n/en.json +++ b/web/src/i18n/en.json @@ -344,6 +344,32 @@ "containsCaseSensitive": "Contains {{text}}, case-sensitive", "codepoints": "Code points {{ranges}}", "other": "Built-in check" + }, + "exceptions": { + "title": "Exceptions · {{count}}", + "lead": "These values are neither replaced nor recorded, whichever rule finds them. Only a hash and the masked value are kept.", + "empty": "No exceptions. A value that is recognised by mistake again and again can be made an exception, and the rule keeps working for every other value.", + "full": "There are {{max}} exceptions, the most there may be. Remove one to add another.", + "add": "Add exception", + "addTitle": "Add an exception", + "addDesc": "This value will no longer be replaced, nor recorded, by any rule. Saved provider API keys cannot be exceptions.", + "value": "Value", + "valueHint": "Paste the value as it appears in requests; spaces and line breaks around it are ignored. It is hashed on the server and not stored.", + "duplicate": "This value is already an exception.", + "added": "Added {{value}} as an exception", + "removed": "Removed {{value}} from the exceptions", + "remove": "Remove", + "removeTitle": "Remove {{value}} from the exceptions?", + "removeDesc": "The rules will replace or record this value again.", + "noRule": "No rule recognised it", + "loadFailed": "Could not load the exceptions", + "excepted": "Exception, left as is", + "inTrial": "Exceptions left as is: {{count}}", + "col": { + "value": "Value", + "rule": "Rule", + "added": "Added" + } } }, "mcpStore": { diff --git a/web/src/i18n/zh.json b/web/src/i18n/zh.json index 020631d0..1aec6bbb 100644 --- a/web/src/i18n/zh.json +++ b/web/src/i18n/zh.json @@ -344,6 +344,32 @@ "containsCaseSensitive": "包含 {{text}}，区分大小写", "codepoints": "码位 {{ranges}}", "other": "内置判据" + }, + "exceptions": { + "title": "例外 · {{count}} 个值", + "lead": "这些值在任何规则下都不替换、不记录。只保存哈希与打码后的值。", + "empty": "暂无例外。某个值反复被误认时，可只将该值列为例外，规则对其他值照常生效。", + "full": "例外已达上限 {{max}} 个，移出一个后才能添加。", + "add": "添加例外", + "addTitle": "添加例外", + "addDesc": "此值今后在任何规则下都不再替换，也不再记录。保存的供应商 API 密钥不能列为例外。", + "value": "值", + "valueHint": "按请求中出现的样子粘贴，首尾的空格与换行不计。值在服务器端计算哈希，不保存原值。", + "duplicate": "此值已在例外中。", + "added": "已将 {{value}} 列为例外", + "removed": "已将 {{value}} 移出例外", + "remove": "移出例外", + "removeTitle": "将 {{value}} 移出例外？", + "removeDesc": "移出后，规则将重新替换或记录此值。", + "noRule": "无规则认出", + "loadFailed": "无法加载例外", + "excepted": "例外，不处理", + "inTrial": "{{count}} 个值属于例外，不处理。", + "col": { + "value": "值", + "rule": "规则", + "added": "加入时间" + } } }, "mcpStore": { diff --git a/web/src/lib/security-types.ts b/web/src/lib/security-types.ts index 82ff6a50..7f397909 100644 --- a/web/src/lib/security-types.ts +++ b/web/src/lib/security-types.ts @@ -160,6 +160,26 @@ export interface GuardDetail { kinds: SecurityKindView[]; } +/** + * A redaction exception value: never replaced or recorded, whichever rule + * finds it. Only its SHA-256 and masked form are kept, never the value. + */ +export interface RedactException { + sha256: string; + masked: string; + /** The rule that first recognised it; absent when none did. */ + rule?: string; + /** Milliseconds since the epoch. */ + added_at: number; +} + +/** `GET /api/admin/security/redact/exceptions`. */ +export interface RedactExceptionList { + items: RedactException[]; + /** How many there may be. */ + max: number; +} + /** `GET /api/admin/security`. */ export interface SecurityDetail { redact: GuardDetail; @@ -207,8 +227,14 @@ export interface SecurityTestHit { * characters are drawn as `‹U+200B›`, a run as `‹U+E0049 ×12›`. */ excerpt: string; - /** What the rule does when the guard is on. */ + /** What the rule does when the guard is on. Absent on an exception value. */ action?: RuleAction | null; + /** + * Redaction: the value is an exception (`security.redact.exceptions`): + * recognised, but neither replaced nor recorded, and left as it is in + * `output`. + */ + excepted?: boolean; } export interface SecurityTestResult { diff --git a/web/src/routes/gateway/security/exceptions-panel.tsx b/web/src/routes/gateway/security/exceptions-panel.tsx new file mode 100644 index 00000000..b2d41844 --- /dev/null +++ b/web/src/routes/gateway/security/exceptions-panel.tsx @@ -0,0 +1,228 @@ +import { useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query'; +import { Plus } from 'lucide-react'; +import { toast } from 'sonner'; +import { Button } from '@/components/ui/button'; +import { Card, CardContent, CardHeader } from '@/components/ui/card'; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { Label } from '@/components/ui/label'; +import { Skeleton } from '@/components/ui/skeleton'; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table'; +import { Textarea } from '@/components/ui/textarea'; +import { ConfirmDialog } from '@/components/confirm-dialog'; +import { ApiError, api, apiDelete, apiPost } from '@/lib/api'; +import type { GuardDetail, RedactException, RedactExceptionList } from '@/lib/security-types'; +import { ruleName } from './names'; + +const PATH = '/api/admin/security/redact/exceptions'; +const EXCEPTIONS_KEY = ['security', 'redact', 'exceptions'] as const; + +/** + * Redaction's exception values, under the rule table: values that are + * never replaced or recorded, whichever rule finds them. Each row is the + * value masked, the rule that first recognised it, when it was added, and + * "Remove". A value is added by pasting it; the server hashes it on + * arrival and keeps only the hash and the masked form, so the console + * never sees it again. At the cap, adding is unavailable and says why. + */ +export function ExceptionsPanel({ detail, canWrite }: { detail: GuardDetail; canWrite: boolean }) { + const { t, i18n } = useTranslation(); + const queryClient = useQueryClient(); + const [adding, setAdding] = useState(false); + const [removing, setRemoving] = useState(null); + const query = useQuery({ + queryKey: EXCEPTIONS_KEY, + queryFn: ({ signal }) => api(PATH, { signal }), + }); + const remove = useMutation({ + mutationFn: (e: RedactException) => apiDelete(`${PATH}/${e.sha256}`), + onSuccess: (_, e) => { + setRemoving(null); + toast.success(t('contentSecurity.exceptions.removed', { value: e.masked })); + void queryClient.invalidateQueries({ queryKey: EXCEPTIONS_KEY }); + }, + onError: (err) => toast.error(err instanceof Error ? err.message : t('common.error')), + }); + + const items = query.data?.items ?? []; + const max = query.data?.max ?? 500; + const full = items.length >= max; + const rule = (e: RedactException) => + e.rule ? ruleName(i18n.language, detail, e.rule, false, e.rule) : t('contentSecurity.exceptions.noRule'); + + const addButton = ( + + ); + + return ( + + + {items.length === 0 ? ( + // Nothing excepted: one line and the button. +
+

{t('contentSecurity.exceptions.empty')}

+ {addButton} +
+ ) : ( + <> +
+

+ {t('contentSecurity.exceptions.title', { count: items.length })} +

+ {addButton} +
+

+ {full ? t('contentSecurity.exceptions.full', { max }) : t('contentSecurity.exceptions.lead')} +

+ + )} + + {(query.isPending || query.isError || items.length > 0) && ( + + {query.isPending ? ( + + ) : query.isError ? ( +

+ {t('contentSecurity.exceptions.loadFailed')}: {query.error.message} +

+ ) : ( + + + + + + + + + + {t('contentSecurity.exceptions.col.value')} + {t('contentSecurity.exceptions.col.rule')} + {t('contentSecurity.exceptions.col.added')} + + {t('common.actions')} + + + + + {items.map((e) => ( + + + {e.masked || '—'} + + + {rule(e)} + + + {e.added_at ? new Date(e.added_at).toLocaleString() : '—'} + + + + + + ))} + +
+ )} + + )} + {adding && setAdding(false)} />} + !o && !remove.isPending && setRemoving(null)} + title={t('contentSecurity.exceptions.removeTitle', { value: removing?.masked ?? '' })} + description={t('contentSecurity.exceptions.removeDesc')} + confirmLabel={t('contentSecurity.exceptions.remove')} + loading={remove.isPending} + onConfirm={() => removing && remove.mutate(removing)} + /> + + ); +} + +/** + * Paste a value to except it: for a test value that has not turned up in + * the log yet. The value goes to the server once and is forgotten here as + * soon as the dialog closes. + */ +function AddExceptionDialog({ onClose }: { onClose: () => void }) { + const { t } = useTranslation(); + const queryClient = useQueryClient(); + const [value, setValue] = useState(''); + const [error, setError] = useState(null); + const add = useMutation({ + mutationFn: (v: string) => apiPost(PATH, { value: v }), + onSuccess: (e) => { + toast.success(t('contentSecurity.exceptions.added', { value: e.masked })); + void queryClient.invalidateQueries({ queryKey: EXCEPTIONS_KEY }); + onClose(); + }, + onError: (err) => + setError( + err instanceof ApiError && err.status === 409 + ? t('contentSecurity.exceptions.duplicate') + : err instanceof Error + ? err.message + : t('common.error'), + ), + }); + const empty = value.trim() === ''; + return ( + !o && !add.isPending && onClose()}> + + + {t('contentSecurity.exceptions.addTitle')} + {t('contentSecurity.exceptions.addDesc')} + +
+ +