diff --git a/Cargo.lock b/Cargo.lock index 2eaec5ce..f2dc8534 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3670,7 +3670,7 @@ dependencies = [ [[package]] name = "tw-api" -version = "0.85.0" +version = "0.86.0" dependencies = [ "serde", "serde_json", @@ -3681,7 +3681,7 @@ dependencies = [ [[package]] name = "tw-bedrock" -version = "0.85.0" +version = "0.86.0" dependencies = [ "aws-credential-types", "aws-sigv4", @@ -3702,7 +3702,7 @@ dependencies = [ [[package]] name = "tw-breaker" -version = "0.85.0" +version = "0.86.0" dependencies = [ "serde", "serde_json", @@ -3710,7 +3710,7 @@ dependencies = [ [[package]] name = "tw-config" -version = "0.85.0" +version = "0.86.0" dependencies = [ "blake3", "libc", @@ -3736,7 +3736,7 @@ dependencies = [ [[package]] name = "tw-control" -version = "0.85.0" +version = "0.86.0" dependencies = [ "axum", "base64 0.22.1", @@ -3782,7 +3782,7 @@ dependencies = [ [[package]] name = "tw-dialect" -version = "0.85.0" +version = "0.86.0" dependencies = [ "memchr", "serde", @@ -3791,7 +3791,7 @@ dependencies = [ [[package]] name = "tw-engine" -version = "0.85.0" +version = "0.86.0" dependencies = [ "chrono", "serde", @@ -3805,7 +3805,7 @@ dependencies = [ [[package]] name = "tw-gateway" -version = "0.85.0" +version = "0.86.0" dependencies = [ "arc-swap", "async-stream", @@ -3856,7 +3856,7 @@ dependencies = [ [[package]] name = "tw-guard" -version = "0.85.0" +version = "0.86.0" dependencies = [ "aho-corasick", "base64 0.22.1", @@ -3875,7 +3875,7 @@ dependencies = [ [[package]] name = "tw-handover" -version = "0.85.0" +version = "0.86.0" dependencies = [ "libc", "serde", @@ -3888,7 +3888,7 @@ dependencies = [ [[package]] name = "tw-link" -version = "0.85.0" +version = "0.86.0" dependencies = [ "serde", "serde_json", @@ -3902,7 +3902,7 @@ dependencies = [ [[package]] name = "tw-observe" -version = "0.85.0" +version = "0.86.0" dependencies = [ "tokio", "tracing", @@ -3911,7 +3911,7 @@ dependencies = [ [[package]] name = "tw-pack" -version = "0.85.0" +version = "0.86.0" dependencies = [ "age", "blake3", @@ -3935,7 +3935,7 @@ dependencies = [ [[package]] name = "tw-plugin" -version = "0.85.0" +version = "0.86.0" dependencies = [ "libc", "rand 0.10.2", @@ -3951,7 +3951,7 @@ dependencies = [ [[package]] name = "tw-pricing" -version = "0.85.0" +version = "0.86.0" dependencies = [ "arc-swap", "flate2", @@ -3964,14 +3964,14 @@ dependencies = [ [[package]] name = "tw-secret" -version = "0.85.0" +version = "0.86.0" dependencies = [ "thiserror 2.0.20", ] [[package]] name = "tw-store" -version = "0.85.0" +version = "0.86.0" dependencies = [ "blake3", "bytes", @@ -3991,7 +3991,7 @@ dependencies = [ [[package]] name = "tw-sync" -version = "0.85.0" +version = "0.86.0" dependencies = [ "age", "axum", @@ -4019,7 +4019,7 @@ dependencies = [ [[package]] name = "tw-types" -version = "0.85.0" +version = "0.86.0" dependencies = [ "serde", "serde_json", @@ -4028,7 +4028,7 @@ dependencies = [ [[package]] name = "tw-watch" -version = "0.85.0" +version = "0.86.0" dependencies = [ "libc", "notify", @@ -4039,7 +4039,7 @@ dependencies = [ [[package]] name = "tw-yaml" -version = "0.85.0" +version = "0.86.0" dependencies = [ "saphyr-parser", "serde_yaml_ng", @@ -4049,7 +4049,7 @@ dependencies = [ [[package]] name = "twcore" -version = "0.85.0" +version = "0.86.0" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 73ab79d2..fb376381 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,7 +61,7 @@ categories = ["network-programming", "web-programming::http-server"] # 二进制走 CalVer,crate 走 SemVer —— 两者是两回事,卖给不同的人。 # 这里是 crate 的版本。 -version = "0.85.0" +version = "0.86.0" [workspace.dependencies] # ── 内部 crate ─────────────────────────────────────────── diff --git a/crates/tw-api/msg-codes.txt b/crates/tw-api/msg-codes.txt index 05055290..31de383a 100644 --- a/crates/tw-api/msg-codes.txt +++ b/crates/tw-api/msg-codes.txt @@ -101,6 +101,9 @@ config.plugin.duplicate config.plugin.file config.plugin.sha256 config.provider_concurrency_range +config.redact_exception_duplicate +config.redact_exception_hash_bad +config.redact_exceptions_too_many config.rejected config.rejected_at config.remote_port_is_gateway @@ -568,6 +571,11 @@ security.bad_content_pattern security.bad_label security.bad_pattern security.content_action_bad +security.exception_forgotten +security.exception_not_exceptable +security.exception_saved_secret +security.exception_value_empty +security.exceptions_full security.pattern_empty security.redact_action_unknown security.redact_match_bad @@ -577,6 +585,7 @@ security.rule_scope_codepoints security.rule_scope_empty security.unknown_action security.unknown_custom_rule +security.unknown_event security.unknown_guard security.unknown_rule sync.already_on diff --git a/crates/tw-api/src/ep.rs b/crates/tw-api/src/ep.rs index 4f6fc761..dee382fa 100644 --- a/crates/tw-api/src/ep.rs +++ b/crates/tw-api/src/ep.rs @@ -198,6 +198,8 @@ endpoints! { // ─────────────────────────────────────────────── 安全 Security: GET "/security", () => api::SecurityDetail; SecurityEvents: GET "/security/events", api::SecurityEventsQuery => api::SecurityEventsPage; + /// 一项防护从某个时刻起每条规则命中了几次(记录的 `count` 加起来),按规则排 + SecurityRuleCounts: GET "/security/counts", api::SecurityRuleHitsQuery => Vec; SetSecurityMode: PUT "/security/{guard}/mode" [guard], api::ModeSave => api::ConfigWritten; ToggleBuiltinRule: PUT "/security/{guard}/builtin/{id}" [guard, id], api::RuleToggle => api::ConfigWritten; SetBuiltinRuleAction: PUT "/security/{guard}/builtin/{id}/action" [guard, id], api::ActionSave => api::ConfigWritten; @@ -207,6 +209,12 @@ endpoints! { UpdateCustomRule: PUT "/security/{guard}/custom/{name}" [guard, name], api::CustomRuleSave => api::ConfigWritten; DeleteCustomRule: DELETE "/security/{guard}/custom/{name}" [guard, name], api::BaseVersion => api::ConfigWritten; TestSecurity: POST "/security/{guard}/test" [guard], api::SecurityTestRequest => api::SecurityTestResult; + /// 出站脱敏的例外值:列出、手动加入(交原值,core 当场算哈希)、把安全日志的一条列为例外 + /// (按那一行的号)、移出(按哈希)。加入时已经在表里的不再写一次 + RedactExceptions: GET "/security/exceptions", () => api::RedactExceptions; + AddRedactException: POST "/security/exceptions", api::ExceptionAdd => api::ExceptionAdded; + ExceptSecurityEvent: POST "/security/events/{id}/exception" [id], api::BaseVersion => api::ExceptionAdded; + RemoveRedactException: DELETE "/security/exceptions/{sha256}" [sha256], api::BaseVersion => api::ConfigWritten; // ─────────────────────────────────────────────── 脚本插件 // diff --git a/crates/tw-api/src/lib.rs b/crates/tw-api/src/lib.rs index 9783382d..efbb33ec 100644 --- a/crates/tw-api/src/lib.rs +++ b/crates/tw-api/src/lib.rs @@ -26,7 +26,7 @@ pub use endpoint::{Endpoint, ErrorBody, Format, Info, Method, fill}; // `SecurityTestRequest`……) pub use tw_guard::content::Outcome as ContentOutcome; pub use tw_guard::locate::{Direction as SecurityDirection, Part as HitPart}; -pub use tw_guard::policy::{ContentMatch, ContentScope, Guard, Mode as GuardMode}; +pub use tw_guard::policy::{ContentMatch, ContentScope, Guard, Mode as GuardMode, RedactException}; /// 保存的凭据是哪一项(出站脱敏的 `thinkwatch-saved-secret`)。定义在 tw-guard:企业版只依赖 /// 第一层,也要用它 pub use tw_guard::redact::rules::{SavedSource, SavedSourceKind}; @@ -1116,7 +1116,22 @@ pub const MSG_CODES: &str = include_str!("../msg-codes.txt"); /// 的 `upstream-key` / `upstream-header` / `access-key` / `proxy-password` / `control-key` / /// `sync-credential`,`name` 是上游名、密钥名、代理名,控制面密钥和同步凭据是空的),这时打码 /// 后的值是空串。按位置认的(`in-context` 一类)打码只留最后四位 -pub const CONTROL_API_VERSION: u32 = 56; +/// +/// **57 起出站脱敏有例外值**(配置的 `security.redact.exceptions`,存值的 SHA-256 和打码后的样子): +/// 例外的值在任何规则下都不换、不记。新端点 `GET /security/exceptions`(→ +/// [`RedactExceptions`])、`POST /security/exceptions`([`ExceptionAdd`]:交原值,core 当场 +/// 算哈希、按认出它的规则打码 → [`ExceptionAdded`])、`POST /security/events/{id}/exception` +/// (把安全日志的那一行列为例外,[`BaseVersion`] → [`ExceptionAdded`];值的哈希只在 core 的内存里 +/// 留最近两万行,更早的、重启之前的是 409 `security.exception_forgotten`)、 +/// `DELETE /security/exceptions/{sha256}`(→ [`ConfigWritten`])。[`SecurityEventView`] 多了 +/// `excepted`(这个值此刻是例外)和 `exceptable`(这一条能列为例外),「测试…」的命中 +/// (`SecurityTestHit`)多了 `excepted`(例外的值照样标出来,`action` 是空的)。新码: +/// `config.redact_exceptions_too_many`、`config.redact_exception_hash_bad`、 +/// `config.redact_exception_duplicate`、`security.exception_*`、`security.exceptions_full`、 +/// `security.unknown_event`。新端点 `GET /security/counts`([`SecurityRuleHitsQuery`] → +/// [`SecurityRuleHits`] 的列表):一项防护从某个时刻起每条规则命中了几次,各条记录的 `count` +/// 加起来,规则表的组头用 +pub const CONTROL_API_VERSION: u32 = 57; /// `/live` 的速度看最近多久(见 [`LiveView::tokens_per_sec`]),毫秒 pub const LIVE_RATE_WINDOW_MS: u64 = 3_000; @@ -4396,6 +4411,28 @@ pub struct SecurityEventsQuery { pub limit: Option, } +/// `GET /security/counts`:一项防护从某个时刻起每条规则命中了几次(规则表的组头「最近 7 天 +/// 命中 N 次」)。 +#[derive(Debug, Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SecurityRuleHitsQuery { + pub guard: Guard, + /// 从这个时刻起(毫秒,含)。不给是全部 + #[serde(default, skip_serializing_if = "Option::is_none")] + pub since_ms: Option, +} + +/// 一条规则命中了几次([`SecurityRuleHitsQuery`])。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct SecurityRuleHits { + /// 内置规则的 id,或者自定义规则的名字 + pub rule: String, + pub custom: bool, + /// 这段时间里各条记录的 `count` 加起来([`SecurityEventView::count`]),不论做了什么 + pub count: i64, +} + /// `GET /config/at`:配置原文里的第几个字节落在哪个字段上。 #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] @@ -6234,6 +6271,13 @@ pub struct SecretItem { /// 是哪一项保存的凭据(规则 `thinkwatch-saved-secret`:这时 `masked` 是空的)。别的规则没有 #[serde(default, skip_serializing_if = "Option::is_none")] pub source: Option, + /// 值的 SHA-256([`tw_guard::redact::rules::Finding::sha256`]),给记录器留在内存里,日后 + /// 「列为例外」照它加;不能列为例外的(保存的凭据、几处重叠合成的一段)没有。**只在进程里 + /// 走**:事件流、在途快照都不带它,请求库也不存 —— 界面要列为例外时交回的是日志那一行的号, + /// 用不着哈希;一个不需要它的地方拿到它,对短的值(手机号)就是一张可以穷举的答案 + #[serde(skip)] + #[cfg_attr(feature = "ts", ts(skip))] + pub value_sha256: Option, } /// 各项防护在一段时间里各留下了几条记录。 @@ -6334,6 +6378,18 @@ pub struct SecurityEventView { /// 出站脱敏认出的是哪一项保存的凭据(规则 `thinkwatch-saved-secret`)。别的没有 #[serde(default, skip_serializing_if = "Option::is_none")] pub source: Option, + /// 出站脱敏:这个值此刻在例外表里(`security.redact.exceptions`),界面上标「例外」。 + /// 读的时候按现在的例外表算,不是记下时的 + #[serde(default)] + pub excepted: bool, + /// 出站脱敏:这一条能「列为例外」(`POST /security/events/{id}/exception`)。保存的凭据、 + /// 几处重叠合成的一段、别的防护的记录不能;已经是例外的也是 false。 + /// + /// **这两样只对最近的记录算得出**:值的哈希不进请求库,只在 core 的内存里留最近两万行 + /// (重启就没了)。更早的记录两样都是 false —— 界面上不给按钮也不标「例外」,那个值可以 + /// 粘贴进「添加例外」手动加入 + #[serde(default)] + pub exceptable: bool, } /// 一条安全记录最多列几处命中([`SecurityEventView::locations`])。再多的只数个数 @@ -6532,6 +6588,47 @@ pub struct RulesToggle { pub base_version: Option, } +/// 出站脱敏的例外表(`GET /security/exceptions`):按配置里的先后(先加的在前)。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct RedactExceptions { + /// 每一项是配置里的那一项原样:哈希、打码后的样子、最先认出它的规则、加入的时刻。移出例外 + /// 按 `sha256` + pub items: Vec, + /// 最多几个([`tw_guard::policy::EXCEPTIONS_MAX`])。满了「添加例外」置灰 + pub max: u32, +} + +/// 手动加入一个例外值(`POST /security/exceptions`):还没出现在日志里的测试值。 +/// +/// **原值只在这个请求里**:core 收到就算哈希,不写进配置、日志、事件和回答。两头的空白去掉 +#[derive(Clone, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ExceptionAdd { + pub value: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub base_version: Option, +} + +/// 不把值打出来:这个类型带着用户粘贴的原值,一条调试日志就能把它留在盘上 +impl std::fmt::Debug for ExceptionAdd { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ExceptionAdd") + .field("value", &format_args!("<{} bytes>", self.value.len())) + .field("base_version", &self.base_version) + .finish() + } +} + +/// 加入了例外(或者它本来就在:这时 `version` 是没改动的那一版)。 +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +pub struct ExceptionAdded { + pub version: String, + /// 例外表里的那一项 + pub exception: RedactException, +} + /// 改一条内置规则在第三档下做什么。 #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "ts", derive(ts_rs::TS))] @@ -7030,6 +7127,17 @@ pub struct PluginRunView { #[cfg(test)] mod tests { + /// 手动加入例外的请求带着原值:调试输出里不能有它 + #[test] + fn an_exception_request_does_not_print_its_value() { + let req = ExceptionAdd { + value: "sk-ant-api03-SECRETSECRETSECRET".into(), + base_version: None, + }; + let d = format!("{req:?}"); + assert!(!d.contains("SECRET"), "{d}"); + } + use super::*; fn attempt(outcome: AttemptOutcome, status: Option, code: Option<&str>) -> AttemptView { diff --git a/crates/tw-api/src/ts.rs b/crates/tw-api/src/ts.rs index b86ce804..fd518ab0 100644 --- a/crates/tw-api/src/ts.rs +++ b/crates/tw-api/src/ts.rs @@ -579,8 +579,34 @@ mod tests { for field in ["output: string | null", "refused: boolean"] { assert!(res.contains(field), "{field}: {res}"); } + // 例外值:哈希不出进程,视图和事件里都没有 + let item = decl_of(&ts, "SecretItem"); + assert!(!item.contains("value_sha256"), "{item}"); + let hit = decl_of(&ts, "SecurityTestHit"); + assert!(hit.contains("excepted: boolean"), "{hit}"); + let ex = decl_of(&ts, "RedactException"); + for field in [ + "sha256: string", + "masked: string", + "rule?: string", + "added_at: number", + ] { + assert!(ex.contains(field), "{field}: {ex}"); + } + assert!(decl_of(&ts, "RedactExceptions").contains("items: Array")); + assert!(decl_of(&ts, "ExceptionAdded").contains("exception: RedactException")); + assert!(ts.contains( + " RemoveRedactException: { method: \"DELETE\", path: \"/security/exceptions/{sha256}\", params: [\"sha256\"], format: \"json\" }," + )); + let counts = decl_of(&ts, "SecurityRuleHits"); + for field in ["rule: string", "custom: boolean", "count: number"] { + assert!(counts.contains(field), "{field}: {counts}"); + } let log = decl_of(&ts, "SecurityEventView"); + assert!(!log.contains("value_sha256"), "{log}"); for field in [ + "excepted: boolean", + "exceptable: boolean", "match?: ContentMatch", "revealed?: string", "session?: string", diff --git a/crates/tw-config/src/lib.rs b/crates/tw-config/src/lib.rs index fe1126cc..f6a0b02f 100644 --- a/crates/tw-config/src/lib.rs +++ b/crates/tw-config/src/lib.rs @@ -1317,8 +1317,8 @@ pub use reload::{Rejected, Stage, stand_in, try_parse}; pub use retention::Retention; pub use security::{ ContentAction, ContentMatch, ContentPolicy, ContentScope, CustomContentRule, CustomRedactRule, - CustomToolRule, Mode as SecurityMode, PolicyError, RedactAction, RedactPolicy, Security, - ToolAction, ToolPolicy, policy_msg, + CustomToolRule, Mode as SecurityMode, PolicyError, RedactAction, RedactException, RedactPolicy, + Security, ToolAction, ToolPolicy, policy_msg, }; // Billing 在本文件里定义,这里不必再导出 pub use store::{Fingerprint, Loaded, StoreError, version_of}; diff --git a/crates/tw-config/src/security.rs b/crates/tw-config/src/security.rs index 05668b5d..49b941a1 100644 --- a/crates/tw-config/src/security.rs +++ b/crates/tw-config/src/security.rs @@ -13,8 +13,8 @@ pub use tw_guard::policy::{ ContentAction, ContentMatch, ContentPolicy, ContentScope, CustomContentRule, CustomRedactRule, - CustomToolRule, Mode, PolicyError, RedactAction, RedactPolicy, Security, ToolAction, - ToolPolicy, + CustomToolRule, Mode, PolicyError, RedactAction, RedactException, RedactPolicy, Security, + ToolAction, ToolPolicy, }; use tw_types::{Msg, msg}; @@ -68,6 +68,21 @@ pub fn policy_msg(e: &PolicyError) -> Msg { "custom content rule `{name}` checks answers and matches by codepoints; code points are \ checked in requests only" ), + PolicyError::TooManyExceptions { count } => msg!( + "config.redact_exceptions_too_many", + count = count.to_string(), + max = tw_guard::policy::EXCEPTIONS_MAX.to_string() => + "security.redact.exceptions has {count} values; at most {max} are allowed" + ), + PolicyError::BadExceptionHash { sha256 } => msg!( + "config.redact_exception_hash_bad", sha256 = sha256.clone() => + "the exception `{sha256}` in security.redact.exceptions is not a SHA-256 written as 64 \ + lowercase hexadecimal characters" + ), + PolicyError::DuplicateException { sha256 } => msg!( + "config.redact_exception_duplicate", sha256 = sha256.clone() => + "the exception `{sha256}` appears twice in security.redact.exceptions" + ), PolicyError::UnknownRule { guard, id } => msg!( "config.unknown_rule", guard = guard.slug(), rule = id.clone() => "security.{guard} names `{rule}`, which is not a built-in rule" @@ -137,6 +152,14 @@ mod tests { "content:\n custom:\n - { name: a, pattern: 'U+200B', match: codepoints, scope: [response] }", "config.rule_scope_codepoints", ), + ( + "redact:\n exceptions:\n - { sha256: abc, masked: x, added_at: 1 }", + "config.redact_exception_hash_bad", + ), + ( + "redact:\n exceptions:\n - { sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 }\n - { sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 }", + "config.redact_exception_duplicate", + ), ]; for (yaml, code) in errors { let e = serde_yaml_ng::from_str::(yaml) diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index 5c491b0d..164e2dba 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -1283,6 +1283,60 @@ pub fn sections() -> Vec
{ "自定义规则:匹配到的内容和凭据一样替换,或只记录。", ), ), + row( + "exceptions", + Kind::Objs("security.redact.exceptions[]"), + Def::Is("[]"), + t( + "Values that are never replaced or recorded, under any rule. Each is kept as a hash, never as the value itself; at most 500. A saved secret cannot be an exception: an entry whose rule is `thinkwatch-saved-secret` is read but has no effect.", + "例外值:在任何规则下都不替换、不记录。只保存哈希,不保存值本身;最多 500 个。保存的凭据不能列为例外:规则为 `thinkwatch-saved-secret` 的条目可以读入,但不生效。", + ), + ), + ], + }, + Section { + path: "security.redact.exceptions[]", + ty: checked!( + RedactException, + "{sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08}" + ), + rows: vec![ + row( + "sha256", + Kind::Str, + Def::Required, + t( + "SHA-256 of the value, as 64 lowercase hexadecimal characters. Escapes in the request body are decoded first: the hash is of the value as written.", + "值的 SHA-256,64 个小写十六进制字符。请求体中的转义先解开:哈希按值本身计算。", + ), + ), + row( + "masked", + Kind::Str, + Def::Is("''"), + t( + "The value as shown when it was added, masked the same way as in the security log.", + "加入时显示的值,打码方式与安全日志相同。", + ), + ), + row( + "rule", + Kind::Str, + Def::Is("''"), + t( + "The rule that first recognised the value: a built-in rule id or a custom rule name. Empty for a value added by hand that no rule recognises.", + "最先认出该值的规则:内置规则的 id 或自定义规则的名称。手动添加且没有规则认出的值为空。", + ), + ), + row( + "added_at", + Kind::Int, + Def::Is("0"), + t( + "When the value was added, in milliseconds since the Unix epoch.", + "加入的时间,Unix 纪元以来的毫秒数。", + ), + ), ], }, Section { diff --git a/crates/tw-control/src/lib.rs b/crates/tw-control/src/lib.rs index d8b15e35..b5a50a71 100644 --- a/crates/tw-control/src/lib.rs +++ b/crates/tw-control/src/lib.rs @@ -823,6 +823,7 @@ async fn history( (Some(from), Some(to)) => g.db().security_of_requests(from, to).map_err(records)?, _ => Default::default(), }; + security::annotate(&s.config(), &g, security.values_mut().flatten()); // 插件改过的那些,同样一次取完 let changed = match ( rows.iter().map(|r| r.id).min(), @@ -873,10 +874,9 @@ async fn history_search( let ids: Vec = found.rows.iter().map(|r| r.id).collect(); let (mut security, changed) = { let g = store.lock().await; - ( - g.db().security_of(&ids).map_err(records)?, - g.db().changed_by_plugins(&ids).map_err(records)?, - ) + let mut security = g.db().security_of(&ids).map_err(records)?; + security::annotate(&s.config(), &g, security.values_mut().flatten()); + (security, g.db().changed_by_plugins(&ids).map_err(records)?) }; Ok(Json(tw_api::HistorySearchPage { rows: found @@ -1125,12 +1125,13 @@ async fn request_detail( truncated: original_len > stored, }) }; - let security = g + let mut security = g .db() .security_of_requests(id, id) .map_err(records)? .remove(&id) .unwrap_or_default(); + security::annotate(&s.config(), &g, &mut security); // 插件的每一次运行。**还在跑的请求也有**:请求钩子在发往上游之前就记下了 let plugins: Vec = g .db() diff --git a/crates/tw-control/src/security.rs b/crates/tw-control/src/security.rs index f9f2909e..30e09552 100644 --- a/crates/tw-control/src/security.rs +++ b/crates/tw-control/src/security.rs @@ -22,6 +22,25 @@ //! 写,几十条就是几十次热重载、几十个版本; //! - **自定义规则写成配置里那个类型序列化出来的样子**,默认值不写 —— 写进去的就是 //! 读回来的那一份。 +//! +//! # 例外值 +//! +//! 出站脱敏的例外值([`tw_guard::redact::exceptions`])写在 `security.redact.exceptions`,和规则 +//! 一样走配置的写入路径,随配置导出、同步、备份。**原值不落在任何地方**: +//! +//! - 手动加入时界面交原值,这里当场算哈希、按认出它的规则打码,写进配置的只有这两样; +//! 原值不进日志([`tw_api::ExceptionAdd`] 的 `Debug` 不打它)、不进回答。 +//! - 从安全日志加入时界面交的是**那一行的号**,不是哈希。值的哈希**不进请求库**,只在记录器 +//! 的内存里留最近两万行([`tw_store::Recorder::value_hash`]),重启就没了:对手机号这种短的 +//! 值,一个不加盐的 SHA-256 配上打码后留下的几位是能穷举回去的,存进库等于把号码存了一份, +//! 也没理由交给不需要它的界面(事件流、日志视图都不带它)。更早的记录没法从日志加入(409), +//! 粘贴那个值手动加入。界面要的「这一条已经是例外」「这一条能列为例外」读的时候在这里 +//! 照内存里的哈希和现在的例外表算好([`annotate`]),不在内存里的两样都是 false。 +//! - 几处重叠的命中合成的一段、只记录的规则让开之后剩下的几截没有哈希 +//! ([`tw_guard::redact::rules::Finding::sha256`]):没有哪条规则认过那一段,列为例外挡不住 +//! 什么。 +//! - 移出按哈希:例外表本身(`GET /security/exceptions`)就是配置里的那一份,界面 +//! 拿得到配置原文,哈希在这里不多泄露什么。 use axum::Json; use axum::extract::{Path, Query, State}; @@ -45,6 +64,7 @@ pub fn router() -> axum::Router { axum::Router::new() .at(ep::Security, detail) .at(ep::SecurityEvents, events) + .at(ep::SecurityRuleCounts, counts) .at(ep::SetSecurityMode, set_mode) .at(ep::ToggleBuiltinRule, toggle_builtin) .at(ep::SetBuiltinRuleAction, set_builtin_action) @@ -53,6 +73,10 @@ pub fn router() -> axum::Router { .at(ep::UpdateCustomRule, update_custom) .at(ep::DeleteCustomRule, delete_custom) .at(ep::TestSecurity, test) + .at(ep::RedactExceptions, exceptions) + .at(ep::AddRedactException, add_exception) + .at(ep::ExceptSecurityEvent, add_exception_from_event) + .at(ep::RemoveRedactException, remove_exception) } /// 这一边对每项防护要知道的事。路径里写的就是配置里的那个键。 @@ -207,9 +231,25 @@ async fn events( q.limit.unwrap_or(100).clamp(1, PAGE_MAX), ) .map_err(crate::records)?; + let mut page = page; + annotate(&s.config(), &g, &mut page.events); Ok(Json(page)) } +/// 每条规则命中了几次。只读索引(`security_events_rule_hits`),不翻日志 +async fn counts( + State(s): State, + Query(q): Query, +) -> Result>, Fail> { + let store = crate::need_store(&s)?; + let g = store.lock().await; + let hits = g + .db() + .security_rule_hits(q.guard.slug(), q.since_ms.unwrap_or(0)) + .map_err(crate::records)?; + Ok(Json(hits)) +} + // ---------------------------------------------------------------- 写 async fn set_mode( @@ -560,10 +600,12 @@ fn rule_fail(e: PolicyError) -> Fail { PolicyError::EmptyScope { .. } => scope_empty(), PolicyError::BlockInAnswer { .. } => block_in_answer(), PolicyError::CodepointsInAnswer { .. } => codepoints_in_answer(), - // 只有一条规则的策略里不会有:名字重复由写文件那一步说,内置规则的 id 不在这里写 - e @ (PolicyError::DuplicateName { .. } | PolicyError::UnknownRule { .. }) => { - tw_config::policy_msg(&e) - } + // 只有一条规则的策略里不会有:名字重复由写文件那一步说,内置规则的 id、例外表不在这里写 + e @ (PolicyError::DuplicateName { .. } + | PolicyError::UnknownRule { .. } + | PolicyError::TooManyExceptions { .. } + | PolicyError::BadExceptionHash { .. } + | PolicyError::DuplicateException { .. }) => tw_config::policy_msg(&e), }; fail(StatusCode::BAD_REQUEST, m) } @@ -687,6 +729,218 @@ async fn delete_custom( Ok(Json(tw_api::ConfigWritten { version })) } +// ---------------------------------------------------------------- 例外值 + +/// 例外表那一节:按 `sha256` 认 +const EXCEPTIONS: edit::Section = edit::Section { + path: &["security", "redact", "exceptions"], + what: "redaction exception", + key: "sha256", +}; + +/// 安全日志的记录按现在的例外表标上 `excepted`、`exceptable`(见文件头)。**每个返回日志 +/// 记录的端点都要过一遍**:哈希不在库里,在记录器的内存里,只有最近的那些行有 +/// ([`tw_store::Recorder::value_hash`]);不在内存里的两样都是 false +pub(crate) fn annotate<'a>( + cfg: &tw_config::Config, + rec: &tw_store::Recorder, + views: impl IntoIterator, +) { + let set = cfg.security.redact.exception_set(); + for v in views { + if v.guard != Guard::Redact { + continue; + } + let Some(Some(hash)) = rec.value_hash(v.id) else { + continue; + }; + v.excepted = set.as_ref().is_some_and(|ex| ex.contains_hex(hash)); + v.exceptable = !v.excepted; + } +} + +async fn exceptions(State(s): State) -> Json { + Json(tw_api::RedactExceptions { + items: s.config().security.redact.exceptions.clone(), + max: tw_guard::policy::EXCEPTIONS_MAX as u32, + }) +} + +/// 手动加入:交来的是原值。**算完哈希它就没用了**,不进任何一条日志、不进回答 +async fn add_exception( + State(s): State, + Json(req): Json, +) -> Result, Fail> { + // 网关手里那一份保存的凭据:贴进来一把上游的密钥,认得出它、不收 + let saved = s.gateway.runtime().redact.saved().cloned(); + let e = s + .config() + .security + .redact + .exception_of(&req.value, saved, crate::config::now_ms() as i64) + .map_err(exception_fail)?; + drop(req.value); + put_exception(&s, e, req.base_version.as_deref()).await +} + +/// 从安全日志的一条加入:哈希、打码后的样子、规则都取自那一行 +async fn add_exception_from_event( + State(s): State, + Path(id): Path, + Json(req): Json, +) -> Result, Fail> { + let store = crate::need_store(&s)?; + let (src, hash) = { + let g = store.lock().await; + let src = g + .db() + .exception_source(id) + .map_err(crate::records)? + .ok_or_else(|| { + fail( + StatusCode::NOT_FOUND, + msg!( + "security.unknown_event", id = id.to_string() => + "There is no security log entry {id}; it may have passed the retention \ + period." + ), + ) + })?; + let hash = g.value_hash(id).map(|h| h.map(str::to_string)); + (src, hash) + }; + let not_exceptable = || { + fail( + StatusCode::BAD_REQUEST, + msg!( + "security.exception_not_exceptable" => + "Only a value found by outbound redaction can be an exception, and a saved \ + secret or a span joined from overlapping matches cannot." + ), + ) + }; + let sha256 = match (src.guard, hash) { + (Guard::Redact, Some(Some(h))) => h, + // 工具调用审查、内容过滤的记录没有可以例外的「值」;保存的凭据、几处重叠合成的一段 + // 列为例外也挡不住什么 + (Guard::Redact, Some(None)) | (Guard::InspectTools | Guard::Content, _) => { + return Err(not_exceptable()); + } + // 记下时的哈希只在内存里留最近的一批,重启就没了(见 `tw_store::Recorder::value_hash`) + (Guard::Redact, None) => { + return Err(fail( + StatusCode::CONFLICT, + msg!( + "security.exception_forgotten" => + "This entry is too old to be listed as an exception from the log; paste the \ + value into Add exception instead." + ), + )); + } + }; + let e = tw_config::RedactException { + sha256, + masked: src.masked, + rule: src.rule, + added_at: crate::config::now_ms() as i64, + }; + put_exception(&s, e, req.base_version.as_deref()).await +} + +/// 写进例外表。**已经在表里的不再写一次**(不论是哪条规则加的):回那一项和现在的版本。 +/// 满了是 409。 +/// +/// 两样都在写入那一步里、对着那一刻的配置查:两个「列为例外」同时到,后写的那个读到的是先写 +/// 完的那一份。写的时候磁盘上的配置被别人改了([`tw_config::StoreError::Conflict`])、而调用方 +/// 没带版本时重来一次,重来时就看见了先写进去的那一项 +async fn put_exception( + s: &ControlState, + e: tw_config::RedactException, + base_version: Option<&str>, +) -> Result, Fail> { + let m = item(&e)?; + let mut tries = 0; + loop { + tries += 1; + // 已经在表里:那一项和没改动的那一版 + let mut have: Option<(tw_config::RedactException, String)> = None; + let r = s + .cfg + .transform(base_version, Origin::Ui, |text, cfg| { + let list = &cfg.security.redact.exceptions; + if let Some(x) = list.iter().find(|x| x.sha256 == e.sha256) { + have = Some((x.clone(), tw_config::version_of(text))); + return Err(ApplyError::InUse(already_listed())); + } + if list.len() >= tw_guard::policy::EXCEPTIONS_MAX { + return Err(ApplyError::InUse(msg!( + "security.exceptions_full", max = tw_guard::policy::EXCEPTIONS_MAX.to_string() => + "There are already {max} exceptions; remove one before adding another." + ))); + } + Ok(edit::upsert(text, EXCEPTIONS, None, &m)?) + }) + .await; + if let Some((exception, version)) = have { + return Ok(Json(tw_api::ExceptionAdded { version, exception })); + } + match r { + Ok(version) => { + return Ok(Json(tw_api::ExceptionAdded { + version, + exception: e, + })); + } + Err(ApplyError::Store(tw_config::StoreError::Conflict { .. })) + if base_version.is_none() && tries < 3 => {} + Err(err) => return Err(apply_fail(err)), + } + } +} + +/// 只在 [`put_exception`] 里头用来停下那次写入,不会到界面上(已经在表里的回 200),所以 +/// 没有码 +fn already_listed() -> tw_types::Msg { + tw_types::Msg { + code: String::new(), + args: Default::default(), + text: String::new(), + } +} + +async fn remove_exception( + State(s): State, + Path(sha256): Path, + Query(q): Query, +) -> Result, Fail> { + let version = s + .cfg + .transform(q.base_version.as_deref(), Origin::Ui, |text, _| { + Ok(edit::remove(text, EXCEPTIONS, &sha256)?) + }) + .await + .map_err(apply_fail)?; + Ok(Json(tw_api::ConfigWritten { version })) +} + +/// 一个值加不进例外表 +fn exception_fail(e: tw_guard::policy::ExceptionError) -> Fail { + use tw_guard::policy::ExceptionError; + let m = match e { + ExceptionError::Empty => { + msg!("security.exception_value_empty" => "The value is empty.") + } + ExceptionError::Saved => msg!( + "security.exception_saved_secret" => + "This value is a secret saved in ThinkWatch; saved secrets are always replaced and \ + cannot be exceptions." + ), + // 配置读进来时校验过,到这里还编不起来是绕过校验写进去的:那是这边的问题 + ExceptionError::Policy(e) => return internal(e), + }; + fail(StatusCode::BAD_REQUEST, m) +} + // ---------------------------------------------------------------- 测试 fn unknown_rule(id: &str) -> Fail { diff --git a/crates/tw-control/tests/handover.rs b/crates/tw-control/tests/handover.rs index ad9c8219..73501c2a 100644 --- a/crates/tw-control/tests/handover.rs +++ b/crates/tw-control/tests/handover.rs @@ -146,7 +146,7 @@ async fn status_reports_the_handover_phase_and_the_draining_count() { let st = status(&s).await; assert_eq!(st.handover, tw_api::HandoverState::Draining); assert_eq!(st.draining, 3); - assert_eq!(st.api_version, 56); + assert_eq!(st.api_version, 57); } /// arm 之后(以及接替者正在接管时)不开始新的账号登录:回调端口不交接。(平常、排空中的 diff --git a/crates/tw-control/tests/security.rs b/crates/tw-control/tests/security.rs index a8405e8a..f116f83e 100644 --- a/crates/tw-control/tests/security.rs +++ b/crates/tw-control/tests/security.rs @@ -113,16 +113,21 @@ fn event( } fn bed_with(yaml: &str, seed: impl FnOnce(&tw_store::Db)) -> Bed { + bed_rec(yaml, |rec| seed(rec.db())) +} + +/// [`bed_with`],种子交给记录器:经事件记下的安全日志,值的哈希留在记录器的内存里 +fn bed_rec(yaml: &str, seed: impl FnOnce(&mut tw_store::Recorder)) -> Bed { let d = tempfile::tempdir().unwrap(); let p = d.path().join("config.yaml"); std::fs::write(&p, yaml).unwrap(); let db = tw_store::Db::open(&d.path().join("data.db")).unwrap(); - seed(&db); - let rec = tw_store::Recorder::new( + let mut rec = tw_store::Recorder::new( db, tw_store::Blobs::new(d.path().join("blobs")), tw_pricing::shared(tw_pricing::PriceBook::builtin().unwrap()), ); + seed(&mut rec); let cfg = tw_config::try_parse(yaml).unwrap(); let gw = tw_gateway::AppState::new(cfg).unwrap(); let bus = gw.bus.clone(); @@ -1572,3 +1577,427 @@ async fn an_unknown_guard_names_all_three() { let text = v["text"].as_str().unwrap(); assert!(text.contains("redact, inspect_tools or content"), "{text}"); } + +// ─────────────────────────────────────────────────────────── 例外值 + +fn sha256(v: &str) -> String { + tw_guard::redact::exceptions::sha256_hex(v) +} + +/// R5:手动加入。原值只在请求里:配置里只有哈希和打码后的样子;加入之后「测试…」把它标成 +/// 例外;再加一次不写第二份;移出之后又是命中 +#[tokio::test] +async fn a_value_added_by_hand_is_kept_as_a_hash_and_can_be_removed() { + let b = bed(BASE); + let (st, v) = call( + &b.app, + "GET", + "/security/exceptions", + serde_json::Value::Null, + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(v, json!({ "items": [], "max": 500 })); + + let (st, v) = call( + &b.app, + "POST", + "/security/exceptions", + json!({ "value": format!("{KEY}\n") }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + let h = sha256(KEY); + assert_eq!(v["exception"]["sha256"], h); + assert_eq!(v["exception"]["rule"], "anthropic-api-key"); + assert_eq!(v["exception"]["masked"], "sk-an…AAAA"); + let version = v["version"].clone(); + // 原值哪儿都没有:配置文件、回答 + let file = b.file(); + assert!(file.contains(&h) && !file.contains(KEY), "{file}"); + assert!(!v.to_string().contains(KEY), "{v}"); + + // 再加一次:不写第二份,版本不变 + let (st, v) = call( + &b.app, + "POST", + "/security/exceptions", + json!({ "value": KEY }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(v["version"], version); + assert_eq!(b.parsed().security.redact.exceptions.len(), 1); + + // 「测试…」:认得出,标成例外,不换 + let (_, v) = call( + &b.app, + "POST", + "/security/redact/test", + json!({ "sample": format!("key {KEY}") }), + ) + .await; + assert_eq!(v["hits"].as_array().unwrap().len(), 1, "{v}"); + assert_eq!(v["hits"][0]["excepted"], true); + assert_eq!(v["hits"][0]["action"], serde_json::Value::Null); + assert_eq!(v["output"], serde_json::Value::Null); + + // 列出来就是配置里那一份 + let (_, v) = call( + &b.app, + "GET", + "/security/exceptions", + serde_json::Value::Null, + ) + .await; + assert_eq!(v["items"][0]["sha256"], h); + + // 认不出的值:规则是空的,按最小心的打码 + let (_, v) = call( + &b.app, + "POST", + "/security/exceptions", + json!({ "value": "fixture-value-for-tests-9876" }), + ) + .await; + assert_eq!(v["exception"]["masked"], "…9876", "{v}"); + assert!(v["exception"].get("rule").is_none(), "{v}"); + let (st, v) = call( + &b.app, + "POST", + "/security/exceptions", + json!({ "value": " " }), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST); + assert_eq!(v["code"], "security.exception_value_empty"); + + // 移出:又是命中 + let (st, v) = call( + &b.app, + "DELETE", + &format!("/security/exceptions/{h}"), + serde_json::Value::Null, + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert!(!b.file().contains(&h), "{}", b.file()); + let (_, v) = call( + &b.app, + "POST", + "/security/redact/test", + json!({ "sample": format!("key {KEY}") }), + ) + .await; + assert_eq!(v["hits"][0]["excepted"], false, "{v}"); + let (st, _) = call( + &b.app, + "DELETE", + &format!("/security/exceptions/{h}"), + serde_json::Value::Null, + ) + .await; + assert_eq!(st, StatusCode::NOT_FOUND); + // 别的规则接口照常走得到(路径和 `/security/{guard}/…` 不打架) + let (st, v) = call( + &b.app, + "PUT", + "/security/redact/mode", + json!({ "mode": "enforce" }), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); +} + +/// 保存的凭据不能列为例外:上游的密钥贴进来是 400,一个字都不写 +#[tokio::test] +async fn a_saved_secret_cannot_be_added() { + let saved = "sk-relay-0123456789abcdefghij"; + let b = bed(&BASE.replace("key: sk-relay", &format!("key: {saved}"))); + let before = b.file(); + let (st, v) = call( + &b.app, + "POST", + "/security/exceptions", + json!({ "value": saved }), + ) + .await; + assert_eq!(st, StatusCode::BAD_REQUEST, "{v}"); + assert_eq!(v["code"], "security.exception_saved_secret"); + assert_eq!(b.file(), before); +} + +/// 满了 500 个再加是 409;读配置时超出是配置错误 +#[tokio::test] +async fn the_list_holds_at_most_500() { + let mut yaml = format!("{BASE}security:\n redact:\n exceptions:\n"); + for i in 0..500 { + yaml.push_str(&format!( + " - {{ sha256: {}, masked: x }}\n", + sha256(&i.to_string()) + )); + } + let b = bed(&yaml); + let (st, v) = call( + &b.app, + "POST", + "/security/exceptions", + json!({ "value": KEY }), + ) + .await; + assert_eq!(st, StatusCode::CONFLICT, "{v}"); + assert_eq!(v["code"], "security.exceptions_full"); + yaml.push_str(&format!(" - {{ sha256: {} }}\n", sha256("one more"))); + let e = tw_config::try_parse(&yaml).unwrap_err(); + assert_eq!(e.message.code, "config.redact_exceptions_too_many", "{e}"); +} + +/// 从安全日志加入:界面交那一行的号,哈希从记录器的内存里取,日志的视图里没有哈希。加入之后 +/// 同一个值的每一条都标「例外」,不能再加。不在内存里的(重启之前记的)两样都不标,加入是 409 +#[tokio::test] +async fn a_logged_value_is_added_by_its_row_and_the_log_marks_it() { + let h = sha256(KEY); + let found = + |at_ms: u64, rule: &str, value_sha256: Option| tw_api::Event::SecretsFound { + id: 1, + provider: "中转".into(), + replaced: false, + items: vec![tw_api::SecretItem { + rule: rule.into(), + custom: false, + kind: tw_api::SecretKind::AiServices, + masked: "sk-an…AAAA".into(), + count: 1, + replaced: false, + detail: Default::default(), + source: None, + value_sha256, + }], + at_ms, + }; + let b = bed_rec(BASE, |rec| { + rec.db().insert(&request(1, 1_000, "中转")).unwrap(); + // 1:重启之前记的,内存里没有 + rec.db() + .insert_security_event(&event(1, 1_000, "redact", "anthropic-api-key", "recorded")) + .unwrap(); + // 2、3:同一个值 + rec.on_event(&found(1_001, "anthropic-api-key", Some(h.clone()))); + rec.on_event(&found(1_002, "anthropic-api-key", Some(h.clone()))); + // 4:保存的凭据(或者几处重叠合成的一段):没有哈希 + rec.on_event(&found(1_003, "thinkwatch-saved-secret", None)); + // 5 + rec.db() + .insert_security_event(&event(1, 1_004, "inspect_tools", "curl-pipe-sh", "cut")) + .unwrap(); + }); + async fn log(b: &Bed, h: &str) -> Vec { + let (st, v) = call(&b.app, "GET", "/security/events", serde_json::Value::Null).await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert!(!v.to_string().contains(h), "日志的视图带了哈希:{v}"); + v["events"].as_array().unwrap().clone() + } + let rows = log(&b, &h).await; + // 倒序:5 4 3 2 1 + let flags: Vec<(bool, bool)> = rows + .iter() + .map(|e| { + ( + e["excepted"].as_bool().unwrap(), + e["exceptable"].as_bool().unwrap(), + ) + }) + .collect(); + assert_eq!( + flags, + [ + (false, false), + (false, false), + (false, true), + (false, true), + (false, false) + ] + ); + let id = rows[3]["id"].as_i64().unwrap(); + + let (st, v) = call( + &b.app, + "POST", + &format!("/security/events/{id}/exception"), + json!({}), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(v["exception"]["sha256"], h); + assert_eq!(v["exception"]["masked"], "sk-an…AAAA"); + assert_eq!(v["exception"]["rule"], "anthropic-api-key"); + assert_eq!(b.parsed().security.redact.exceptions.len(), 1); + + let rows = log(&b, &h).await; + let flags: Vec<(bool, bool)> = rows + .iter() + .map(|e| { + ( + e["excepted"].as_bool().unwrap(), + e["exceptable"].as_bool().unwrap(), + ) + }) + .collect(); + assert_eq!( + flags, + [ + (false, false), + (false, false), + (true, false), + (true, false), + (false, false) + ] + ); + // 请求详情里的记录也标着(按号正序:1 不在内存里,2 是例外) + let (_, d) = call(&b.app, "GET", "/request/1", serde_json::Value::Null).await; + assert_eq!(d["row"]["security"][0]["excepted"], false, "{d}"); + assert_eq!(d["row"]["security"][1]["excepted"], true, "{d}"); + // 再加一次:已经在表里,200,不写第二份 + let (st, v) = call( + &b.app, + "POST", + &format!("/security/events/{}/exception", rows[2]["id"]), + json!({}), + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!(b.parsed().security.redact.exceptions.len(), 1); + + for (id, status, code) in [ + ( + rows[0]["id"].as_i64().unwrap(), + StatusCode::BAD_REQUEST, + "security.exception_not_exceptable", + ), + ( + rows[1]["id"].as_i64().unwrap(), + StatusCode::BAD_REQUEST, + "security.exception_not_exceptable", + ), + ( + rows[4]["id"].as_i64().unwrap(), + StatusCode::CONFLICT, + "security.exception_forgotten", + ), + (999, StatusCode::NOT_FOUND, "security.unknown_event"), + ] { + let (st, v) = call( + &b.app, + "POST", + &format!("/security/events/{id}/exception"), + json!({}), + ) + .await; + assert_eq!((st, v["code"].as_str()), (status, Some(code)), "{v}"); + } +} + +/// 规则表的组头:一项防护从某个时刻起每条规则命中了几次,按记录的 `count` 加 +#[tokio::test] +async fn hits_per_rule_add_up_the_counts_since_a_time() { + let b = bed_with(BASE, |db| { + db.insert(&request(1, 1_000, "中转")).unwrap(); + for (at, rule, action, count) in [ + (500, "anthropic-api-key", "recorded", 9), + (1_001, "anthropic-api-key", "recorded", 2), + (1_002, "anthropic-api-key", "replaced", 3), + (1_003, "jwt", "recorded", 1), + ] { + db.insert_security_event(&tw_store::SecurityEvent { + count, + ..event(1, at, "redact", rule, action) + }) + .unwrap(); + } + db.insert_security_event(&tw_store::SecurityEvent { + custom: true, + count: 4, + ..event(1, 1_004, "redact", "项目号", "recorded") + }) + .unwrap(); + db.insert_security_event(&event(1, 1_005, "inspect_tools", "curl-pipe-sh", "cut")) + .unwrap(); + }); + let (st, v) = call( + &b.app, + "GET", + "/security/counts?guard=redact&since_ms=1000", + serde_json::Value::Null, + ) + .await; + assert_eq!(st, StatusCode::OK, "{v}"); + assert_eq!( + v, + json!([ + { "rule": "anthropic-api-key", "custom": false, "count": 5 }, + { "rule": "jwt", "custom": false, "count": 1 }, + { "rule": "项目号", "custom": true, "count": 4 }, + ]) + ); + let (_, v) = call( + &b.app, + "GET", + "/security/counts?guard=redact", + serde_json::Value::Null, + ) + .await; + assert_eq!(v[0]["count"], 14, "{v}"); + let (_, v) = call( + &b.app, + "GET", + "/security/counts?guard=inspect_tools", + serde_json::Value::Null, + ) + .await; + assert_eq!( + v, + json!([{ "rule": "curl-pipe-sh", "custom": false, "count": 1 }]) + ); +} + +/// 同时加:同一个值两次都是 200、只写一份;只剩一个空位时两个不同的值一个进去、一个 409 +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn adding_at_the_same_time_is_idempotent_and_honours_the_cap() { + let b = bed(BASE); + let add = |v: &str| { + let app = b.app.clone(); + let v = v.to_string(); + async move { call(&app, "POST", "/security/exceptions", json!({ "value": v })).await } + }; + let (x, y) = tokio::join!(add(KEY), add(KEY)); + assert_eq!((x.0, y.0), (StatusCode::OK, StatusCode::OK), "{x:?} {y:?}"); + assert_eq!(b.parsed().security.redact.exceptions.len(), 1); + + let mut yaml = format!("{BASE}security:\n redact:\n exceptions:\n"); + for i in 0..499 { + yaml.push_str(&format!( + " - {{ sha256: {}, masked: x }}\n", + sha256(&i.to_string()) + )); + } + let b = bed(&yaml); + let add = |v: &str| { + let app = b.app.clone(); + let v = v.to_string(); + async move { call(&app, "POST", "/security/exceptions", json!({ "value": v })).await } + }; + let (x, y) = tokio::join!( + add("first-fixture-value-0001"), + add("second-fixture-value-0002") + ); + let mut codes = [x.0, y.0]; + codes.sort(); + assert_eq!(codes, [StatusCode::OK, StatusCode::CONFLICT], "{x:?} {y:?}"); + let full = if x.0 == StatusCode::CONFLICT { + &x.1 + } else { + &y.1 + }; + assert_eq!(full["code"], "security.exceptions_full"); + assert_eq!(b.parsed().security.redact.exceptions.len(), 500); +} diff --git a/crates/tw-gateway/src/guard.rs b/crates/tw-gateway/src/guard.rs index 0546183f..e2debe9f 100644 --- a/crates/tw-gateway/src/guard.rs +++ b/crates/tw-gateway/src/guard.rs @@ -39,7 +39,7 @@ use std::collections::HashSet; use tw_config::SecurityMode as Mode; use tw_guard::content::Screening; use tw_guard::redact::flow; -pub use tw_guard::redact::flow::Shape; +pub use tw_guard::redact::flow::{ScanCache, Shape}; use tw_guard::redact::replace::Ledger; use tw_guard::redact::rules::{Finding, Hit, Rule, RuleSet}; @@ -89,6 +89,7 @@ pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8], shape: Option) body, shape, Ledger::new(tw_guard::redact::replace::Scheme::SECRET), + None, ) } @@ -100,12 +101,13 @@ pub fn look_from( shape: Option, seed: Ledger, ) -> (Vec, Ledger) { - let l = look_hits_from(mode, rules, body, shape, seed); + let l = look_hits_from(mode, rules, body, shape, seed, None); (l.found, l.ledger) } /// [`look_from`],连同找到的命中(见 [`flow::Look::hits`]):报出去的每一项要说得出在哪儿 -/// ([`detail::secrets`])。 +/// ([`detail::secrets`])。`cache` 是运行时的扫描缓存([`crate::state::Runtime::scan_cache`]): +/// 对话历史里没变的消息取上一轮扫的,找出来的一模一样。 /// 认不出形状的不找:和关着一样,什么都没有 pub fn look_hits_from( mode: Mode, @@ -113,9 +115,10 @@ pub fn look_hits_from( body: &[u8], shape: Option, seed: Ledger, + cache: Option<&ScanCache>, ) -> flow::Look { match shape { - Some(shape) => flow::look_hits(mode, rules, body, shape, seed), + Some(shape) => flow::look_hits_with(mode, rules, body, shape, seed, cache), None => flow::Look { found: Vec::new(), ledger: Ledger::new(tw_guard::redact::replace::Scheme::SECRET), @@ -140,7 +143,8 @@ pub fn replace( } /// [`replace`],`found` 是按这套规则在 `body` 上已经找过的命中([`look_hits`] 在同一份字节上 -/// 找的,同一个形状):照它换,不再找一遍。没找过的是 `None`,和 [`replace`] 一样。 +/// 找的,同一个形状):照它换,不再找一遍。没找过的是 `None`,和 [`replace`] 一样,没变的段 +/// 取 `cache` 里扫过的(见 [`look_hits_from`])。 pub fn replace_found( mode: Mode, rules: &RuleSet, @@ -148,9 +152,14 @@ pub fn replace_found( ledger: &Ledger, shape: Option, found: Option<&[Hit]>, + cache: Option<&ScanCache>, ) -> (bytes::Bytes, Ledger) { + let replace = |body| match shape { + Some(shape) => flow::replace_with(mode, rules, body, ledger, shape, cache), + None => (body, ledger.clone()), + }; let Some(found) = found.filter(|_| mode.acts() && !rules.is_empty()) else { - return replace(mode, rules, body, ledger, shape); + return replace(body); }; // 只记录的规则命中的不换 let hits = rules.to_replace(found); @@ -160,7 +169,7 @@ pub fn replace_found( } // 找得到命中的一定是 UTF-8(见 `flow::look_hits`) let Ok(text) = std::str::from_utf8(&body) else { - return replace(mode, rules, body, ledger, shape); + return replace(body); }; let r = tw_guard::redact::replace::apply(text, &hits, ledger.clone()); (bytes::Bytes::from(r.text), r.ledger) @@ -252,6 +261,8 @@ pub fn items( replaced: seen.replaced && !seen.redaction.rules.records(&f.rule), detail, source: f.source.clone(), + // 只给安全日志:事件流不带它(见 `tw_api::SecretItem::value_sha256`) + value_sha256: f.sha256.clone(), }) .collect() } @@ -824,6 +835,7 @@ mod tests { &seen.ledger, crate::guard::TEST_SHAPE, Some(&hits), + None, ); assert_eq!(known.0, again.0); assert_eq!(known.1.table(), again.1.table()); @@ -836,6 +848,7 @@ mod tests { &fresh(), crate::guard::TEST_SHAPE, Some(&hits), + None, ); assert_eq!(out, body); // 什么都没找到:原样,不拷贝 @@ -847,6 +860,7 @@ mod tests { &fresh(), crate::guard::TEST_SHAPE, Some(&[]), + None, ); assert_eq!(out.as_ptr(), plain.as_ptr()); assert!(l.is_empty()); @@ -902,6 +916,7 @@ mod tests { rule: Rule::Builtin("thinkwatch-saved-secret"), masked: String::new(), count: 1, + sha256: None, source: Some(SavedSource { kind: SavedSourceKind::UpstreamKey, name: name.into(), diff --git a/crates/tw-gateway/src/server/pipeline.rs b/crates/tw-gateway/src/server/pipeline.rs index c21f633e..639c14fd 100644 --- a/crates/tw-gateway/src/server/pipeline.rs +++ b/crates/tw-gateway/src/server/pipeline.rs @@ -1114,11 +1114,13 @@ fn start( /// 找到的命中跟着请求体交去留档:落盘前打码不再把同一份正文找一遍(见 /// [`crate::bodies::BodyRecord::found`])。 fn look(rt: &Runtime, req: &Inbound) -> tw_guard::redact::flow::Look { - crate::guard::look_hits( + crate::guard::look_hits_from( rt.config.security.redact.mode, &rt.redact, &req.body, crate::client_api::ClientApi::written(req.uri.path()), + tw_guard::redact::replace::Ledger::new(tw_guard::redact::replace::Scheme::SECRET), + Some(&rt.scan_cache), ) } diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index 361c1b81..d672cd2a 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -679,8 +679,15 @@ pub(super) async fn try_upstreams<'a>( Some(d) => Some(crate::guard::Shape::Generate(d)), None => crate::client_api::ClientApi::written(&out.path), }; - let (body, ledger) = - crate::guard::replace_found(mode, &rt.redact, unsealed, seed, shape, known); + let (body, ledger) = crate::guard::replace_found( + mode, + &rt.redact, + unsealed, + seed, + shape, + known, + Some(&rt.scan_cache), + ); (body, ledger, differs) }); diff --git a/crates/tw-gateway/src/server/pipeline/plug.rs b/crates/tw-gateway/src/server/pipeline/plug.rs index 02071375..4ad3fe9d 100644 --- a/crates/tw-gateway/src/server/pipeline/plug.rs +++ b/crates/tw-gateway/src/server/pipeline/plug.rs @@ -131,7 +131,8 @@ pub(super) async fn attempt( .map_or_else(|| started.ledger.clone(), |b| b.ledger().clone()); // 插件改过的这一份按它要发往的路径认形状 let shape = crate::client_api::ClientApi::written(&c.path); - let look = crate::guard::look_hits_from(mode, &rt.redact, &body, shape, seed); + let look = + crate::guard::look_hits_from(mode, &rt.redact, &body, shape, seed, Some(&rt.scan_cache)); let (found, ledger) = (look.found, look.ledger); // 先前几跳报过的不再报(重试、故障转移时插件写进来的还是那几个,见 `crate::guard::Reported`) let already = started.found.len() + reported.secrets(); diff --git a/crates/tw-gateway/src/server/upgrade.rs b/crates/tw-gateway/src/server/upgrade.rs index 2514dad4..1783b03a 100644 --- a/crates/tw-gateway/src/server/upgrade.rs +++ b/crates/tw-gateway/src/server/upgrade.rs @@ -348,6 +348,7 @@ pub(super) async fn ws_upgrade( let rules = crate::ws::Rules { redact_mode: rt.config.security.redact.mode, redact: rt.redact.clone(), + scan_cache: rt.scan_cache.clone(), inspect_mode: rt.config.security.inspect_tools.mode, tools: rt.tools.clone(), screen: crate::guard::Screen::of(&rt), diff --git a/crates/tw-gateway/src/state.rs b/crates/tw-gateway/src/state.rs index 8d023d64..63b2c61e 100644 --- a/crates/tw-gateway/src/state.rs +++ b/crates/tw-gateway/src/state.rs @@ -38,6 +38,9 @@ pub struct Runtime { /// **和配置一起建、一起换**,而不是每个请求现编一次 —— 自定义规则是 /// 正则,摆在数据面上现编就是每个请求白付一次编译。 pub redact: Arc, + /// 出站脱敏的扫描缓存:对话历史里没变的消息不再扫(见 [`tw_guard::redact::cache`])。 + /// **跟着规则一起换**:重载配置换上一份空的,旧规则下扫出来的结果不会留到新规则下 + pub scan_cache: Arc, /// 工具调用审查的规则。同上。 pub tools: Arc, /// 内容过滤的规则(隐藏字符那一组也在里面)。同上 @@ -117,6 +120,7 @@ impl Runtime { clients, allow, redact: Arc::new(redact), + scan_cache: Arc::new(tw_guard::redact::cache::ScanCache::new()), tools: Arc::new(tools), content: Arc::new(content), answer, @@ -132,6 +136,8 @@ impl Runtime { clients: self.clients.clone(), allow: self.allow.clone(), redact: self.redact.clone(), + // 规则没变,扫过的照样作数 + scan_cache: self.scan_cache.clone(), tools: self.tools.clone(), content: self.content.clone(), answer: self.answer.clone(), @@ -787,6 +793,19 @@ mod tests { assert!(rt.clients.contains_key("a") && rt.clients.contains_key("b")); } + /// 扫描缓存跟着规则走:重载配置换上一份新的,旧规则下扫出来的不会留到新规则下;只换插件时 + /// 规则没变,接着用同一份 + #[test] + fn a_reload_brings_a_fresh_scan_cache() { + let s = AppState::new(cfg()).unwrap(); + let before = s.runtime().scan_cache.clone(); + s.reload(cfg()).unwrap(); + assert!(!Arc::ptr_eq(&before, &s.runtime().scan_cache)); + let rt = s.runtime(); + let same = rt.with_plugins(crate::plugin::PluginSet::default()); + assert!(Arc::ptr_eq(&rt.scan_cache, &same.scan_cache)); + } + #[test] fn a_known_key_resolves_to_its_client_name() { let s = AppState::new(cfg()).unwrap(); diff --git a/crates/tw-gateway/src/ws.rs b/crates/tw-gateway/src/ws.rs index 07f32d75..74b2086c 100644 --- a/crates/tw-gateway/src/ws.rs +++ b/crates/tw-gateway/src/ws.rs @@ -218,6 +218,8 @@ pub struct Upstream { pub struct Rules { pub redact_mode: tw_config::SecurityMode, pub redact: Arc, + /// 运行时的扫描缓存,和 `redact` 是同一份运行时的(见 [`crate::state::Runtime::scan_cache`]) + pub scan_cache: Arc, pub inspect_mode: tw_config::SecurityMode, pub tools: Arc, /// 内容过滤 @@ -1566,11 +1568,14 @@ fn outbound(state: &AppState, p: &mut Pipes, id: u64, text: String) -> UpMsg { // 换的和报出去的是同一批:只在写出来的文字里找,我们自己的占位符、base64 载荷不算(见 // `crate::guard::find`) let scope = tw_guard::redact::flow::Scope::Written(shape); - let hits = tw_guard::redact::flow::hits_in(&text, &p.rules.redact, scope); + let cache = Some(&*p.rules.scan_cache); + let (hits, exact) = + tw_guard::redact::flow::hits_exact_with(&text, &p.rules.redact, scope, cache); if hits.is_empty() { return UpMsg::Text(text.into()); } - let found = tw_guard::redact::flow::reported(&text, &p.rules.redact, &hits, scope); + let found = + tw_guard::redact::flow::reported_exact(&text, &p.rules.redact, &hits, exact, scope, cache); let replaced = mode.acts().then(|| { // 只记录的规则命中的不换 let r = tw_guard::redact::replace::apply( diff --git a/crates/tw-guard/src/policy.rs b/crates/tw-guard/src/policy.rs index 46bdfca5..d6671da3 100644 --- a/crates/tw-guard/src/policy.rs +++ b/crates/tw-guard/src/policy.rs @@ -19,6 +19,11 @@ //! label: PROJECT # 可选;占位符 <>;不写 = SECRET //! action: record # 可选;placeholder | record;不写 = placeholder //! disabled: true # 可选 +//! exceptions: # 例外值:这几个值在任何规则下都不换、不记(最多 500 个) +//! - sha256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 +//! masked: sk-an…7f9c # 加入时打码后的样子,只给人看 +//! rule: openai-api-key # 最先认出它的规则;手动加入、没有规则认得时不写 +//! added_at: 1767225600000 # 加入的时刻,毫秒 //! inspect_tools: //! mode: observe //! actions: { rm-rf-root: record } # 只写和出厂不同的;cut | record @@ -378,6 +383,57 @@ pub struct RedactPolicy { pub actions: BTreeMap, #[serde(default, skip_serializing_if = "Vec::is_empty")] pub custom: Vec, + /// 例外值(见 [`crate::redact::exceptions`]):值的 SHA-256 在这里的命中,哪条规则的都不换、 + /// 不记。最多 [`EXCEPTIONS_MAX`] 个 + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub exceptions: Vec, +} + +pub use crate::redact::exceptions::EXCEPTIONS_MAX; + +/// 一个例外值。**只存哈希和打码后的样子,不存原值**:例外表随配置导出、同步、备份。 +/// +/// 规则是保存的凭据(`thinkwatch-saved-secret`)的读得进来、**但不生效**:保存的凭据本就该换, +/// 界面上也不给它「列为例外」。不当成配置错误 —— 那样的话一份手改过的配置会整份读不进来, +/// 而它要的效果(不生效)本来就是对的。 +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "ts", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct RedactException { + /// 值(解开转义之后的样子)的 SHA-256,64 个小写十六进制字符 + pub sha256: String, + /// 加入时打码后的样子(和安全日志里的一样),只给人看是哪一个 + #[serde(default)] + pub masked: String, + /// 最先认出它的规则:内置规则的 id 或者自定义规则的名字。手动加入、没有规则认得它时是空的 + #[serde(default, skip_serializing_if = "String::is_empty")] + pub rule: String, + /// 加入的时刻,毫秒 + #[serde(default)] + pub added_at: i64, +} + +/// 一个值加不进例外表。`code()` 是稳定的码。 +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ExceptionError { + #[error("the value is empty")] + Empty, + /// 是一项保存的凭据:它本就该换 + #[error("a saved secret cannot be an exception")] + Saved, + /// 现在的策略本身编不起来(绕过了校验写进去的) + #[error(transparent)] + Policy(#[from] PolicyError), +} + +impl ExceptionError { + pub fn code(&self) -> &'static str { + match self { + ExceptionError::Empty => "exception_value_empty", + ExceptionError::Saved => "exception_saved_secret", + ExceptionError::Policy(e) => e.code(), + } + } } /// 用户自己写的一条工具调用规则:按工具调用的参数匹配。 @@ -548,6 +604,17 @@ pub enum PolicyError { checked in requests only" )] CodepointsInAnswer { name: String }, + #[error( + "security.redact.exceptions has {count} values; at most {} are allowed", + EXCEPTIONS_MAX + )] + TooManyExceptions { count: usize }, + #[error( + "the exception `{sha256}` is not a SHA-256 written as 64 lowercase hexadecimal characters" + )] + BadExceptionHash { sha256: String }, + #[error("the exception `{sha256}` appears twice")] + DuplicateException { sha256: String }, } impl PolicyError { @@ -565,6 +632,9 @@ impl PolicyError { PolicyError::EmptyScope { .. } => "rule_scope_empty", PolicyError::BlockInAnswer { .. } => "rule_scope_block", PolicyError::CodepointsInAnswer { .. } => "rule_scope_codepoints", + PolicyError::TooManyExceptions { .. } => "redact_exceptions_too_many", + PolicyError::BadExceptionHash { .. } => "redact_exception_hash_bad", + PolicyError::DuplicateException { .. } => "redact_exception_duplicate", } } @@ -580,7 +650,11 @@ impl PolicyError { | PolicyError::EmptyScope { .. } | PolicyError::BlockInAnswer { .. } | PolicyError::CodepointsInAnswer { .. } => Guard::Content, - PolicyError::BadLabel { .. } | PolicyError::BadMatch { .. } => Guard::Redact, + PolicyError::BadLabel { .. } + | PolicyError::BadMatch { .. } + | PolicyError::TooManyExceptions { .. } + | PolicyError::BadExceptionHash { .. } + | PolicyError::DuplicateException { .. } => Guard::Redact, } } } @@ -645,7 +719,99 @@ impl RedactPolicy { .chain(&self.disable) .chain(self.actions.keys()), |id| crate::redact::rules::builtin(id).is_some(), + )?; + self.check_exceptions() + } + + /// 例外表:不超过 [`EXCEPTIONS_MAX`] 个,每个都是我们写出去的那种哈希,不重复。 + /// **规则名不查**:它只是给人看的记录,那条自定义规则之后删了、改了名,例外照样是那个值 + pub fn check_exceptions(&self) -> Result<(), PolicyError> { + if self.exceptions.len() > EXCEPTIONS_MAX { + return Err(PolicyError::TooManyExceptions { + count: self.exceptions.len(), + }); + } + let mut seen = std::collections::HashSet::new(); + for e in &self.exceptions { + let Some(d) = crate::redact::exceptions::parse_hex(&e.sha256) else { + return Err(PolicyError::BadExceptionHash { + sha256: e.sha256.clone(), + }); + }; + if !seen.insert(d) { + return Err(PolicyError::DuplicateException { + sha256: e.sha256.clone(), + }); + } + } + Ok(()) + } + + /// 编好的例外表,给规则集用([`RuleSet::with_exceptions`])。**保存的凭据那条规则的不算** + /// (见 [`RedactException`]),写错的哈希跳过(校验时就拒绝了,到不了这里)。一个都不剩 + /// 是 `None` + pub fn exception_set(&self) -> Option> { + crate::redact::exceptions::Exceptions::new( + self.exceptions + .iter() + .filter(|e| e.rule != crate::redact::exceptions::SAVED_RULE) + .filter_map(|e| crate::redact::exceptions::parse_hex(&e.sha256)), ) + .map(std::sync::Arc::new) + } + + /// 这个哈希已经在例外表里了(不论是哪条规则加的) + pub fn has_exception(&self, sha256: &str) -> bool { + self.exceptions.iter().any(|e| e.sha256 == sha256) + } + + /// 手动加入的一个值写成例外的样子(R5:用于还没出现在日志里的测试值)。**只返回哈希和 + /// 打码后的样子,原值不出这个函数**。 + /// + /// 两头的空白去掉(粘贴时常带一个换行)。按现在的规则(不带例外)找一遍,有规则认得出 + /// 整个值的,按它打码、记下它;现在开着的规则都认不出时再按全部内置规则找一遍;都认不出 + /// 的 `rule` 是空的,打码按 [`crate::redact::rules::masked_unknown`]。 + /// + /// `saved` 是网关手里那一份保存的凭据:是其中一项的不收([`ExceptionError::Saved`])。 + pub fn exception_of( + &self, + value: &str, + saved: Option>, + added_at: i64, + ) -> Result { + use crate::redact::rules::{BUILTINS, masked, masked_unknown}; + let value = value.trim(); + if value.is_empty() { + return Err(ExceptionError::Empty); + } + if saved.as_ref().is_some_and(|s| s.source_of(value).is_some()) { + return Err(ExceptionError::Saved); + } + let whole = |set: &RuleSet| { + crate::redact::flow::hits_plain(value, set) + .into_iter() + .find(|h| h.bytes == (0..value.len())) + .map(|h| h.rule) + }; + let current = self.rules()?.without_exceptions().with_saved_set(saved); + let rule = whole(¤t).or_else(|| { + let ids: Vec<&str> = BUILTINS.iter().map(|b| b.id).collect(); + whole(&RuleSet::only(&ids)) + }); + if rule + .as_ref() + .is_some_and(|r| !crate::redact::exceptions::exceptable(r)) + { + return Err(ExceptionError::Saved); + } + Ok(RedactException { + sha256: crate::redact::exceptions::sha256_hex(value), + masked: rule + .as_ref() + .map_or_else(|| masked_unknown(value), |r| masked(r, value)), + rule: rule.map(|r| r.id().to_string()).unwrap_or_default(), + added_at, + }) } /// 这条内置规则在第三档下做什么(改过的按改过的)。出厂一律换成占位符 @@ -695,14 +861,15 @@ impl RedactPolicy { set = set.recording(crate::redact::rules::Rule::Builtin(b.id)); } } - Ok(set) + Ok(set.with_exceptions(self.exception_set())) } /// 只有一条内置规则,**不管它开没开**,动作按这份策略走。「试一条」用它。不是内置规则的 /// id 是 `None` pub fn one_builtin(&self, id: &str) -> Option { let b = crate::redact::rules::builtin(id)?; - let set = RuleSet::only(&[b.id]); + // 例外跨规则:试一条规则时也照样不换 + let set = RuleSet::only(&[b.id]).with_exceptions(self.exception_set()); Some(match self.builtin_action(b) { RedactAction::Record => set.recording(crate::redact::rules::Rule::Builtin(b.id)), RedactAction::Placeholder => set, @@ -1458,4 +1625,229 @@ content: .count() ); } + + // ---------------------------------------------------------------- 例外值 + + const KEY: &str = "sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAA"; + const OTHER: &str = "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; + + fn excepting(values: &[&str], extra: &str) -> RedactPolicy { + let mut p: RedactPolicy = serde_yaml_ng::from_str(extra).unwrap(); + p.exceptions = values + .iter() + .map(|v| RedactException { + sha256: crate::redact::exceptions::sha256_hex(v), + masked: crate::redact::rules::masked_unknown(v), + rule: String::new(), + added_at: 1, + }) + .collect(); + p.check().unwrap(); + p + } + + fn body(text: &str) -> String { + serde_json::json!({"messages": [{"role": "user", "content": text}]}).to_string() + } + + const SHAPE: crate::redact::flow::Shape = + crate::redact::flow::Shape::Generate(tw_dialect::ir::Dialect::Anthropic); + + /// R4:例外的是这个值,不是这条规则下的这个值 —— 另一条规则(这里是一条自定义规则)认出 + /// 同一个值也不换、不报;别的值照常 + #[test] + fn an_exception_holds_across_every_rule_in_both_modes() { + let p = excepting( + &[KEY], + "custom:\n - { name: 同一个值, pattern: 'sk-ant-api03-A+', match: regex }\n", + ); + let rs = p.rules().unwrap(); + assert!(rs.exceptions().is_some_and(|e| e.len() == 1)); + let b = body(&format!("{KEY} 和 {OTHER}")); + for mode in [Mode::Observe, Mode::Enforce] { + let found = crate::redact::flow::find(mode, &rs, b.as_bytes(), SHAPE); + assert_eq!(found.len(), 1, "{mode}: {found:?}"); + assert_eq!(found[0].rule.id(), "github-personal-token"); + } + let l = crate::redact::flow::look_hits( + Mode::Enforce, + &rs, + b.as_bytes(), + SHAPE, + crate::redact::replace::Ledger::new(crate::redact::replace::Scheme::SECRET), + ); + let (out, ledger) = + crate::redact::flow::replace(Mode::Enforce, &rs, b.clone().into(), &l.ledger, SHAPE); + let out = String::from_utf8(out.to_vec()).unwrap(); + assert!(out.contains(KEY) && !out.contains(OTHER), "{out}"); + // 还原照旧:换掉的那一个换回来,例外的那个原样 + let echo = format!("{KEY} <>"); + assert_eq!( + crate::redact::replace::restore_text(&echo, &ledger), + format!("{KEY} {OTHER}") + ); + // 没有例外时认得出它(两条规则认到同一处,合成一处报) + let mut none = p.clone(); + none.exceptions.clear(); + let found = + crate::redact::flow::find(Mode::Observe, &none.rules().unwrap(), b.as_bytes(), SHAPE); + assert_eq!(found.len(), 2, "{found:?}"); + // 只剩那条自定义规则时:它认得出,例外照样挡住它 + let mut custom_only = p.clone(); + custom_only.disable = vec!["anthropic-api-key".into()]; + let rs2 = custom_only.rules().unwrap(); + assert!( + crate::redact::flow::find(Mode::Enforce, &rs2, body(KEY).as_bytes(), SHAPE).is_empty() + ); + custom_only.exceptions.clear(); + let found = crate::redact::flow::find( + Mode::Enforce, + &custom_only.rules().unwrap(), + body(KEY).as_bytes(), + SHAPE, + ); + assert_eq!(found.len(), 1); + assert_eq!(found[0].rule.id(), "同一个值"); + // 试一条规则时也一样 + assert!( + crate::redact::rules::scan_plain(KEY, &p.one_builtin("anthropic-api-key").unwrap()) + .is_empty() + ); + // 解码过的正文(插件那一层、企业版)同样不认 + assert!(crate::redact::rules::scan_text(KEY, &rs).is_empty()); + } + + /// 请求体里写着转义的值,按解开之后的样子比 + #[test] + fn an_exception_matches_the_value_as_written_not_as_escaped() { + let value = "line1\"quoted\"-PRJ-123456"; + let p = excepting( + &[value], + "custom:\n - { name: 项目号, pattern: 'line1\"quoted\"-PRJ-\\d{6}', match: regex }\n", + ); + let b = body(value); + assert!(b.contains("\\\""), "{b}"); + assert!( + crate::redact::flow::find(Mode::Observe, &p.rules().unwrap(), b.as_bytes(), SHAPE) + .is_empty() + ); + } + + /// 保存的凭据不能列为例外:配置里写着也不生效 + #[test] + fn a_saved_secret_is_never_an_exception() { + use crate::redact::rules::{SavedSecret, SavedSource, SavedSourceKind}; + let saved = "0123456789abcdef0123456789abcdef"; + let mut p = excepting(&[saved], "{}"); + p.exceptions[0].rule = crate::redact::exceptions::SAVED_RULE.into(); + p.check().unwrap(); + assert!(p.exception_set().is_none(), "这一条不算"); + let secret = SavedSecret { + value: saved.into(), + source: SavedSource { + kind: SavedSourceKind::UpstreamKey, + name: "A".into(), + }, + hosts: vec![], + }; + let rs = p.rules().unwrap().with_saved(vec![secret.clone()]); + let found = crate::redact::flow::find(Mode::Observe, &rs, body(saved).as_bytes(), SHAPE); + assert_eq!(found.len(), 1); + assert_eq!(found[0].sha256, None, "保存的凭据连哈希都不留"); + // 规则写的不是它、值却是一项保存的凭据:哈希在表里也照样认 + let p = excepting(&[saved], "{}"); + let rs = p.rules().unwrap().with_saved(vec![secret.clone()]); + let found = crate::redact::flow::find(Mode::Observe, &rs, body(saved).as_bytes(), SHAPE); + assert_eq!(found.len(), 1, "{found:?}"); + // 也加不进去 + let set = crate::redact::rules::Saved::new(vec![secret]).map(std::sync::Arc::new); + assert_eq!( + RedactPolicy::default().exception_of(saved, set, 0), + Err(ExceptionError::Saved) + ); + } + + #[test] + fn the_exception_list_is_checked() { + let one = |sha256: &str| RedactException { + sha256: sha256.into(), + masked: String::new(), + rule: String::new(), + added_at: 0, + }; + let mut p = RedactPolicy { + exceptions: (0..EXCEPTIONS_MAX) + .map(|i| one(&crate::redact::exceptions::sha256_hex(&i.to_string()))) + .collect(), + ..Default::default() + }; + p.check().unwrap(); + p.exceptions + .push(one(&crate::redact::exceptions::sha256_hex("x"))); + let e = p.check().unwrap_err(); + assert_eq!(e.code(), "redact_exceptions_too_many"); + assert_eq!(e.guard(), Guard::Redact); + let h = crate::redact::exceptions::sha256_hex("a"); + for (list, code) in [ + (vec![one("abc")], "redact_exception_hash_bad"), + (vec![one(&h.to_uppercase())], "redact_exception_hash_bad"), + (vec![one(&h), one(&h)], "redact_exception_duplicate"), + ] { + let p = RedactPolicy { + exceptions: list, + ..Default::default() + }; + assert_eq!(p.check().unwrap_err().code(), code); + } + // 写成配置里的样子:规则是空的不写 + let p = RedactPolicy { + exceptions: vec![RedactException { + rule: String::new(), + ..one(&h) + }], + ..Default::default() + }; + let y = serde_yaml_ng::to_string(&p).unwrap(); + assert_eq!( + y, + format!("exceptions:\n- sha256: {h}\n masked: ''\n added_at: 0\n") + ); + assert_eq!(serde_yaml_ng::from_str::(&y).unwrap(), p); + assert!( + serde_yaml_ng::from_str::("exceptions:\n- { sha256: x, value: y }") + .is_err(), + "不认的字段是错" + ); + } + + /// R5:手动加入。认得出的按那条规则打码、记下规则;认不出的规则是空的、按最小心的打码。 + /// 返回的东西里没有原值 + #[test] + fn a_value_added_by_hand_is_hashed_and_masked() { + let p = RedactPolicy::default(); + let e = p.exception_of(&format!(" {KEY}\n"), None, 42).unwrap(); + assert_eq!(e.sha256, crate::redact::exceptions::sha256_hex(KEY)); + assert_eq!(e.rule, "anthropic-api-key"); + assert_eq!( + e.masked, + crate::redact::rules::masked( + &crate::redact::rules::Rule::Builtin("anthropic-api-key"), + KEY + ) + ); + assert_eq!(e.added_at, 42); + assert!(!format!("{e:?}").contains(&KEY[8..]), "{e:?}"); + // 出厂关着的规则也认:邮箱 + let e = p.exception_of("jane@example.com", None, 0).unwrap(); + assert_eq!( + (e.rule.as_str(), e.masked.as_str()), + ("email", "j…@example.com") + ); + // 哪条都认不出 + let e = p.exception_of("my-fixture-value-123456", None, 0).unwrap(); + assert_eq!((e.rule.as_str(), e.masked.as_str()), ("", "…3456")); + let e = p.exception_of("short", None, 0).unwrap(); + assert_eq!(e.masked, "…"); + assert_eq!(p.exception_of(" \n", None, 0), Err(ExceptionError::Empty)); + } } diff --git a/crates/tw-guard/src/redact/cache.rs b/crates/tw-guard/src/redact/cache.rs new file mode 100644 index 00000000..738eb687 --- /dev/null +++ b/crates/tw-guard/src/redact/cache.rs @@ -0,0 +1,1056 @@ +//! 历史消息的扫描结果复用。 +//! +//! 客户端每一轮都把整段对话历史发上来:一段 200 轮的会话,第 200 轮的请求里 99% 的文字前 +//! 199 轮已经扫过。这里按「一段原文 + 规则集」记下扫出来的命中,下一轮没变的那几段直接取, +//! 只扫新增的。 +//! +//! # 以什么为单位 +//! +//! **不是一个个正文字符串**:好几种判据要看字符串外面 —— 配置赋值、按字段名认的值看的是 +//! 键(`"password": "…"` 的键和值是两个字符串),保存的凭据、口令、助记词的边界判断要往前 +//! 看几个字节。一个字符串单拿出来扫,和它在请求体里时扫出来的不一样。 +//! +//! 单位是请求体顶层的**一项**([`split`]): +//! +//! - 值是一个全是对象的数组(`messages`、`input`、`contents`、`tools`、数组写的 `system`) +//! 的,挨着的元素攒够 [`RUN_BYTES`] 算一段(一条长消息自己就够)—— 原文那一截外面套一对 +//! 方括号就是一个合规的 JSON,单拿出来照样按 JSON 扫。从数组开头往后攒:历史只在末尾加长 +//! 时,前面攒好的几段一字不变。不一条条拆:几万条短消息(Codex 的长会话)一条条单独解析、 +//! 扫、查表,比整段扫一遍慢十几倍,也远远超出能存的条数; +//! - 别的顶层项(`model`、字符串写的 `system`、`instructions`……)挨着的几项合成一段,扫的 +//! 时候外面套一对花括号(`{"model":"…","system":"…"}`),键还在。顶层相邻的两项有的判据是 +//! 连起来看的(`"lat":"…","lng":"…"` 是一对坐标),不拆开。 +//! +//! 一条消息里的键和值、字段和它旁边的字段都在同一段里。各段之间隔着的是 JSON 的结构 +//! (`},{`、`],"tools":[`),**判据在这里都断开**:跨 token 接起来认的(分组写的卡号只认 +//! 单个空格、助记词只认空白和逗号、IBAN、MAC、坐标)遇到引号、括号就断;私钥块、自定义规则 +//! 只在一个字符串里找;往前看几个字节的(转义的尾巴、`$` `{` 开头的变量名)看到的都是结构上 +//! 的字;往后找值的(请求头、配置赋值)过不了 `}`。所以一段单拿出来扫的结果,平移到它在原文 +//! 里的位置,就是整段请求体扫出来落在它里面的那些。 +//! +//! **只有一种跨得过去**:命令行(`curl -u`、`mysql -p`)一直认到命令分隔或者换行为止,中间 +//! 隔着字符串也接着认(`["curl","-u","用户:口令"]` 是数组里的几项)。一段末尾一条没结束的命令, +//! 在整段请求体里会把下一段开头、头一个换行之前的选项也认成它的。所以每段记着自己末尾有没有 +//! 这样一条命令;有的话看一眼它在后面认到哪儿,那一截里有选项的开头就整段扫。 +//! +//! # 不缓存的情形 +//! +//! 有几条判据**看得到隔壁的消息**,开着其中任何一条就整个请求照旧整段扫([`CONTEXTUAL`]): +//! +//! - 要前文写明是什么的号码(护照、通行证、澳门身份证、日本个人番号、IMEI,和公网 IP +//! 的「版本号」排除)往前看 20 个字符:一条消息开头的号码,看得到上一条消息末尾的「护照」; +//! - 按字段名认的姓名、住址……:裸的 `name` 要同一个对象里、或者前后两行以内有别的个人信息 +//! 字段才算,不是 JSON 的那一种跨得出一段。 +//! +//! 这几条出厂都关着。另外,保存的凭据里有一项**本身带引号**的话也不缓存:按值找的时候它能 +//! 跨过一个字符串的结尾,从一段连到另一段。 +//! +//! 请求体不是顶层为对象的合规 JSON、嵌套太深、有不认得的写法的,一律整段扫([`split`])。 +//! 拿不准就整段扫:结果和不缓存时**一模一样**,是这一层唯一的要求(对照测试拿多轮对话两条路 +//! 各走一遍比)。 +//! +//! # 存的是去掉不算的之后的命中 +//! +//! 找完之后那几步 —— 去掉压在占位符上的、只留正文里的、去掉键和 base64 载荷上的 +//! ([`super::flow::kept`])—— 每一步只看命中落在哪儿,一段一段地做和整段一起做留下的一样。 +//! 它们也按段做、跟着存下来:不然有命中的请求(历史里贴过一把密钥,之后每一轮都有)每轮还得 +//! 把整段请求体解析一遍认正文。哪些字符串算正文只看这一段自己和它在哪个数组里 +//! ([`tw_dialect::written`] 按协议的结构一项一项、一个元素一个元素地认),所以数组里的元素 +//! 装进一个只有它的同名数组里认(`{"messages":[元素]}`),键里带上是哪个数组、按哪种形状认。 +//! 合并重叠的命中([`super::rules::resolve`])在整个请求上做。 +//! +//! # 头一回见的对话 +//! +//! 按段找要给每一段算哈希、单独校验、单独扫,比整段扫一遍慢;第一轮存下来的又要到下一轮才用得 +//! 上。所以一段没存过的对话(第一轮、重启之后、改过前面的历史)**照旧整段做,做完交给后台线程 +//! 按段存**([`ScanCache::fill`]):存的就是整段做出来的命中按段分开,后台只切段、校验、算哈希。 +//! 这个请求只多付一次抄原文的功夫。 +//! +//! 没存过的不用切段就认得出:每段存过的对话留一个记号([`Mark`],请求体开头的几 KB 和对话 +//! 历史里靠后的一截),两样都对上才切段、按段取。按段取时没存着的占了一大半(前面的历史改过) +//! 的,也改回整段做、后台重存。段数比能存的还多的请求不走缓存。 +//! +//! 请求体短于 [`BODY_MIN`] 的不走缓存:整段扫一遍也就几十微秒,固定的功夫占得出一两成。 +//! +//! 没有命中的段不认正文(解析一遍不便宜),请求里别处有命中时才补认、记下来([`Found::readable`]) +//! +//! # 键、存什么、存多少 +//! +//! 键是 SHA-256(规则集的指纹 + 今天的日期 + 在哪儿找(整段,还是哪种形状的正文)+ 是哪个 +//! 数组里的元素 + 这一段的原文)。指纹见 [`RuleSet::fingerprint`]:改规则、改保存的密钥即整体 +//! 失效;之后往规则集里加的、会改变命中的东西(例外值……)也要喂进指纹,那个函数把字段一个个 +//! 拆开写,漏了编译不过。日期也算进去:身份证号、卡号的出生日期、有效期按「今天」核对,跨了 +//! 一天同一段文字的结论可能不同。 +//! +//! 存的只有命中的区间和规则 id,**不存值**:要换的时候从原文现取。 +//! +//! 最近用过的 [`CAPACITY`] 段,LRU,只在内存里,跟着网关的运行时一起建、一起换(配置一重载 +//! 就是一份新的)。太短的段([`MIN_BYTES`] 以下)扫一遍比算哈希还快,不存;太长的 +//! ([`MAX_BYTES`] 以上)照扫不存,免得一段就挤掉几百条。命中数也有总量上限([`HITS_BUDGET`]): +//! 一份贴进来的凭据清单可能一段就是几万处。 +//! +//! 命中了多少、存了多少不进日志、不进界面:这是内部机制,用户不需要知道。 + +use std::collections::{BTreeMap, HashMap}; +use std::ops::Range; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::mpsc::SyncSender; +use std::sync::{Arc, Mutex}; + +use sha2::{Digest, Sha256}; + +use super::flow::{Scope, Shape}; +use super::rules::{Hit, RuleSet}; + +/// 最多记多少段 +pub const CAPACITY: usize = 2_000; +/// 短于这么多字节的段不存:扫一遍不比算一次哈希、查一次表慢 +pub const MIN_BYTES: usize = 64; +/// 长于这么多字节的段照扫、不存 +pub const MAX_BYTES: usize = 1 << 20; +/// 一段的命中多于这么多处的不存 +const ENTRY_HITS_MAX: usize = 4_096; +/// 所有段加起来最多存多少处命中。一处几十个字节,合起来几 MB +pub const HITS_BUDGET: usize = 65_536; +/// 请求体短于这么多字节的不走缓存:整段扫一遍也就几十微秒,切段、算哈希、查表这些固定的 +/// 功夫占得出一两成,省下来的却只有几十微秒 +const BODY_MIN: usize = 64 * 1024; +/// 数组里挨着的元素攒到这么长才算一段(见 [`split`])。一条条单拿出来的话,几万条短消息的 +/// 请求(Codex 的长会话)每条都要单独解析、单独扫、单独查表,比整段扫一遍慢十几倍,也远远 +/// 超出 [`CAPACITY`] +const RUN_BYTES: usize = 4_096; +/// 嵌套深于这么多层的不切。serde_json 解析有层数上限(128):一段单拿出来比在请求体里 +/// 浅两层,深到边上的话,请求体解不开、单拿出来却解得开,两边就不再都按 JSON 扫了 +const DEPTH_MAX: usize = 100; + +/// 看得到隔壁那一段的规则(见模块说明)。开着任何一条,整个请求照旧整段扫 +pub const CONTEXTUAL: &[&str] = &[ + "cn-passport", + "mainland-travel-permit", + "hk-macau-taiwan-id", + "jp-my-number", + "imei", + "public-ip", + "labeled-name", + "labeled-address", + "labeled-birth-date", + "labeled-account", + "labeled-health", +]; + +/// 扫描结果的缓存。一个网关运行时一份,并发的请求共用(内部加锁,锁里只查表、不扫)。 +pub struct ScanCache { + shared: Arc, + /// 请求体短于这么多字节的不走缓存([`BODY_MIN`];测试里放低,小请求体也走一遍) + body_min: usize, + /// 后台补存的那个线程的收件口(见 [`ScanCache::fill`])。缓存建的时候就起好:起一个线程要 + /// 几十微秒,不该算在哪个请求头上。缓存扔掉时收件口跟着扔,线程收工。起不来是 `None`,那就 + /// 不补存(每轮照旧整段做) + worker: Option>, +} + +/// 请求和后台补存共用的那部分 +struct Shared { + lru: Mutex, + /// 交给后台、还没存完的请求有几个 + pending: AtomicUsize, + /// 该存的段没存着、扫了一遍的次数(测试看缓存是不是真的用上了) + #[cfg(test)] + misses: AtomicUsize, + /// 按段取来、找来的请求有几个(测试看缓存是不是真的用上了) + #[cfg(test)] + warm: AtomicUsize, +} + +/// 记着最近多少段对话的记号([`Mark`]) +const MARKS: usize = 256; +/// 请求的开头取多长([`head`]) +const HEAD_BYTES: usize = 4_096; + +/// 一段存过的对话的记号:请求的开头([`head`]),和对话历史里靠后的一截在原文里的位置、它的 +/// SHA-256。 +/// +/// 同一段对话一轮轮只在末尾加长,开头不变,靠后的那一截也还在原来的位置上;换了一段对话的, +/// 开头可以一样(几 KB 的系统提示、Codex 的 instructions 在最前面),那一截却对不上。两样都对上 +/// 才去切段、按段取:头一回见的对话这样只多算两次几 KB 的哈希,不用切段 +#[derive(Clone, PartialEq, Eq)] +struct Mark { + head: Key, + at: Range, + sum: Key, +} + +/// 后台最多压着几个没存完的请求。再多就不补了:下一轮照样整段做、再补一次 +const PENDING_MAX: usize = 2; + +/// [`ScanCache::kept`] 的结论 +pub(crate) enum Kept { + /// 按段取来、找来的(同 [`super::flow::kept`],区间是原文里的) + Hits(Vec), + /// 这个请求不按段走:调用方照旧整段做 + Whole, + /// 这段对话还没存过(第一轮、重启之后、改过前面的历史):调用方照旧整段做,做完交给 + /// [`ScanCache::fill`] 在后台按段存下来 + Cold(Stamp), +} + +/// 一次整段扫描是按什么做的:这套规则的指纹(不用再算一遍),和找的时候的日期 +#[derive(Debug, Clone, Copy)] +pub(crate) struct Stamp { + fp: [u8; 32], + day: u32, +} + +/// 交给后台补存的一个请求:原文、整段做出来的命中,和存的时候要的键的那几样 +struct Job { + text: String, + hits: Vec, + fp: [u8; 32], + day: u32, + shape: Option, + cmdline: bool, + /// 整段做的时候认过正文、认得出([`Found::readable`]) + readable: bool, +} + +impl Default for ScanCache { + fn default() -> Self { + Self::new() + } +} + +/// 里面存的是区间和规则 id,打出来也没什么可看的 +impl std::fmt::Debug for ScanCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ScanCache").finish_non_exhaustive() + } +} + +type Key = [u8; 32]; + +/// 一段找出来的([`Found`])和它最近一次用到的时刻 +struct Entry { + found: Found, + tick: u64, +} + +/// 一段单拿出来找的结果 +#[derive(Clone)] +struct Found { + /// 去掉不算的之后、合并之前的命中([`super::flow::kept`]),区间是这一段单拿出来时的位置 + /// (套了花括号的从那个 `{` 算起) + hits: Arc<[Hit]>, + /// 这一段里有一条命令到结尾都没结束([`super::secrets::place::command_open_at_end`]): + /// 整段请求体里它会接着认后面的选项,要看一眼后面 + open: bool, + /// 这一段的正文认过、认得出([`tw_dialect::written::ranges`])。整段找的不用认,是 + /// `true`;没有命中的段先不认(解析一遍不便宜),等请求里别处有命中时再补(见 + /// [`ScanCache::kept`]) + readable: bool, +} + +#[derive(Default)] +struct Lru { + map: HashMap, + /// 最近用到的先后:tick → 键。最小的那个最久没用 + order: BTreeMap, + tick: u64, + /// 存着的命中一共多少处 + hits: usize, + /// 存过的对话各留一个记号([`Mark`]),新的在后面。对不上的请求是一段没存过的对话,不用 + /// 切段就知道 + marks: std::collections::VecDeque, +} + +impl Lru { + fn get(&mut self, k: &Key) -> Option { + let e = self.map.get_mut(k)?; + self.order.remove(&e.tick); + self.tick += 1; + e.tick = self.tick; + self.order.insert(e.tick, *k); + Some(e.found.clone()) + } + + /// 一个请求按段存过了:记下它的记号 + fn saw(&mut self, mark: Mark) { + if !self.marks.contains(&mark) { + if self.marks.len() >= MARKS { + self.marks.pop_front(); + } + self.marks.push_back(mark); + } + } + + /// 这一段的正文认过了、认得出(见 [`Found::readable`]) + fn mark_readable(&mut self, k: &Key) { + if let Some(e) = self.map.get_mut(k) { + e.found.readable = true; + } + } + + fn put(&mut self, k: Key, found: Found) { + if found.hits.len() > ENTRY_HITS_MAX { + return; + } + if let Some(old) = self.map.remove(&k) { + self.order.remove(&old.tick); + self.hits -= old.found.hits.len(); + } + self.tick += 1; + self.hits += found.hits.len(); + self.order.insert(self.tick, k); + self.map.insert( + k, + Entry { + found, + tick: self.tick, + }, + ); + while self.map.len() > CAPACITY || self.hits > HITS_BUDGET { + let Some((_, old)) = self.order.pop_first() else { + break; + }; + if let Some(e) = self.map.remove(&old) { + self.hits -= e.found.hits.len(); + } + } + } +} + +impl Shared { + /// 锁坏了(另一个线程拿着锁 panic 了)照用:表里只有完整写进去的条目 + fn lock(&self) -> std::sync::MutexGuard<'_, Lru> { + self.lru.lock().unwrap_or_else(|e| e.into_inner()) + } +} + +impl ScanCache { + pub fn new() -> Self { + let shared = Arc::new(Shared { + lru: Mutex::new(Lru::default()), + pending: AtomicUsize::new(0), + #[cfg(test)] + misses: AtomicUsize::new(0), + #[cfg(test)] + warm: AtomicUsize::new(0), + }); + let (tx, rx) = std::sync::mpsc::sync_channel::(PENDING_MAX); + let worker = { + let shared = shared.clone(); + std::thread::Builder::new() + .name("redact-cache".into()) + .spawn(move || { + for job in rx { + // 一个请求体在这里出了错,不能让整个后台停掉:停了之后这份缓存再也 + // 存不进东西,每个没存过的请求还白抄一份原文 + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + store(&shared, job) + })); + shared.pending.fetch_sub(1, Ordering::Release); + } + }) + .ok() + .map(|_| tx) + }; + Self { + shared, + worker, + body_min: BODY_MIN, + } + } + + /// 测试用:请求体短到 1 KB 也走缓存,对照测试不用造几十 KB 的请求体 + #[cfg(test)] + pub(crate) fn for_tests() -> Self { + Self { + body_min: 1_024, + ..Self::new() + } + } + + fn lock(&self) -> std::sync::MutexGuard<'_, Lru> { + self.shared.lock() + } + + /// 存着几段。先等后台把交过去的存完([`ScanCache::settle`]):慢的机器上后台还没轮到 + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.settle(); + self.lock().map.len() + } + + /// 到现在为止扫了几段该存的 + #[cfg(test)] + pub(crate) fn misses(&self) -> usize { + self.shared.misses.load(Ordering::Relaxed) + } + + /// 到现在为止按段取来、找来的请求有几个 + #[cfg(test)] + pub(crate) fn warm(&self) -> usize { + self.shared.warm.load(Ordering::Relaxed) + } + + /// 存着的命中一共几处 + #[cfg(test)] + pub(crate) fn hits_stored(&self) -> usize { + self.lock().hits + } + + /// 等后台把交过去的都存完 + #[cfg(test)] + pub(crate) fn settle(&self) { + while self.shared.pending.load(Ordering::Acquire) > 0 { + std::thread::sleep(std::time::Duration::from_millis(1)); + } + } + + /// [`super::flow::hits_in`] 合并之前的样子([`super::flow::kept`]:按规则找到、去掉了占位符 + /// 上的、正文外面的、键和载荷上的),**结果一模一样**,只是没变的段取上次的。顺序可以 + /// 不同:之后要排序、合并。 + /// + /// **这段对话没存过的,不在这儿按段找**([`Kept::Cold`]):按段找要给每一段算哈希、单独 + /// 校验、单独扫,比整段扫一遍慢一半以上,而第一轮存下来的要到下一轮才用得上。所以先对一下 + /// 记号([`Mark`]),对不上就让调用方整段做,做完在后台存。 + pub(crate) fn kept(&self, text: &str, rules: &RuleSet, scope: Scope) -> Kept { + if rules.is_empty() || text.len() < self.body_min || !cacheable(rules) { + return Kept::Whole; + } + let fp = rules.fingerprint(); + let day = super::rules::today_ymd(); + let shape = match scope { + Scope::Whole => None, + Scope::Written(shape) => Some(shape), + }; + let tag = scope_tag(shape); + // 这段对话没存过:不用切段(几万条短消息的请求,切一遍就要扫一遍的四分之一) + let h = head(&fp, day, &tag, text); + let marks: Vec = self + .lock() + .marks + .iter() + .filter(|m| m.head == h) + .cloned() + .collect(); + let seen = marks.iter().any(|m| { + text.as_bytes() + .get(m.at.clone()) + .is_some_and(|w| <[u8; 32]>::from(Sha256::digest(w)) == m.sum) + }); + if !seen { + return Kept::Cold(Stamp { fp, day }); + } + let Some(pieces) = split(text) else { + return Kept::Whole; + }; + // 段数比能存的还多:一轮存进去的,下一轮还没用到就挤掉了,算哈希、查表白费功夫 + if pieces.len() > CAPACITY { + return Kept::Whole; + } + let keyed = |p: &Piece| (MIN_BYTES..=MAX_BYTES).contains(&p.range.len()); + let keys: Vec> = pieces + .iter() + .map(|p| keyed(p).then(|| key(&fp, day, &tag, p, text))) + .collect(); + // 锁里只查表;没存着的在锁外面找,找完再一起存进去 + let mut got: Vec> = { + let mut lru = self.lock(); + keys.iter() + .map(|k| k.as_ref().and_then(|k| lru.get(k))) + .collect() + }; + // 没存着的占了一大半(前面的历史改过):一段段找不如整段做、在后台重存 + let missing: usize = pieces + .iter() + .zip(&got) + .filter(|(p, g)| g.is_none() && keyed(p)) + .map(|(p, _)| p.range.len()) + .sum(); + if missing * 2 > text.len() { + return Kept::Cold(Stamp { fp, day }); + } + let cmdline = rules.is_on("command-line-password"); + let mut fresh: Vec<(Key, Found)> = Vec::new(); + for (i, p) in pieces.iter().enumerate() { + if got[i].is_some() { + continue; + } + let Some(found) = find(text, p, rules, shape, cmdline) else { + return Kept::Whole; + }; + if let Some(k) = keys[i] { + #[cfg(test)] + self.shared.misses.fetch_add(1, Ordering::Relaxed); + fresh.push((k, found.clone())); + } + got[i] = Some(found); + } + if !fresh.is_empty() { + let mut lru = self.lock(); + for (k, found) in fresh { + lru.put(k, found); + } + } + let mut out = Vec::new(); + let found: Vec<&Found> = got.iter().flatten().collect(); + if found.len() != pieces.len() || leaks(text, &pieces, |i| found[i].open) { + return Kept::Whole; + } + for (p, f) in pieces.iter().zip(&found) { + out.extend(f.hits.iter().map(|h| Hit { + bytes: p.to_body(h.bytes.start)..p.to_body(h.bytes.end), + rule: h.rule.clone(), + label: h.label.clone(), + })); + } + // 有命中时,整段做的那一边要把整个请求体认一遍正文;认不出(一个超出范围的数、一个 + // 落单的代理对,只看结构的 JSON 校验放过了它们)就一处都不算。没有命中的段这时补认, + // 认过的记下来,下一轮不再认 + if let Some(shape) = shape + && !out.is_empty() + { + let mut known = Vec::new(); + for (i, p) in pieces.iter().enumerate() { + if found[i].readable { + continue; + } + if written(text, p, shape).is_none() { + return Kept::Whole; + } + known.extend(keys[i]); + } + if !known.is_empty() { + let mut lru = self.lock(); + for k in &known { + lru.mark_readable(k); + } + } + } + #[cfg(test)] + self.shared.warm.fetch_add(1, Ordering::Relaxed); + Kept::Hits(out) + } + + /// [`Kept::Cold`] 的请求整段做完了:在后台按段存下来,下一轮就用得上。`hits` 是整段做出来、 + /// 合并之前的命中([`super::flow::kept`]),`written`:做的时候认没认正文、认不认得出 + /// (没有命中时不认,是 `None`)。 + /// + /// **存的就是整段做出来的、按段分开**:结果和一段段单拿出来找的一模一样(见模块说明), + /// 不用再找一遍;后台只切段、校验、算哈希。原文抄一份交过去,存完就扔,不进缓存。后台压着 + /// 的太多就不补了([`PENDING_MAX`]) + pub(crate) fn fill( + &self, + text: &str, + rules: &RuleSet, + stamp: Stamp, + scope: Scope, + hits: &[Hit], + written: Option, + ) { + // 身份证号、卡号按「今天」核对:找的时候和现在隔了零点,结果不能记在今天的键下 + if written == Some(false) + || stamp.day != super::rules::today_ymd() + || self.worker.is_none() + || self.shared.pending.load(Ordering::Acquire) >= PENDING_MAX + { + return; + } + let job = Job { + text: text.to_string(), + hits: hits.to_vec(), + fp: stamp.fp, + day: stamp.day, + shape: match scope { + Scope::Whole => None, + Scope::Written(shape) => Some(shape), + }, + cmdline: rules.is_on("command-line-password"), + readable: scope == Scope::Whole || written == Some(true), + }; + let Some(tx) = &self.worker else { + return; + }; + self.shared.pending.fetch_add(1, Ordering::AcqRel); + if tx.try_send(job).is_err() { + self.shared.pending.fetch_sub(1, Ordering::AcqRel); + } + } +} + +/// 记号里的那一截(见 [`Mark`]):最大的那个数组(对话历史)倒数第三段,不到四段的取第一段。 +/// 下一轮只在末尾加长的话,它还在原来的位置上、一字不差 +fn window(pieces: &[Piece]) -> Option> { + let mut sizes: Vec<(&Range, usize)> = Vec::new(); + for p in pieces { + if let Some(k) = &p.parent { + match sizes.iter_mut().find(|(x, _)| *x == k) { + Some((_, n)) => *n += p.range.len(), + None => sizes.push((k, p.range.len())), + } + } + } + let (k, _) = sizes.into_iter().max_by_key(|(_, n)| *n)?; + let of: Vec<&Piece> = pieces + .iter() + .filter(|p| p.parent.as_ref() == Some(k)) + .collect(); + let p = if of.len() >= 4 { + of[of.len() - 3] + } else { + of[0] + }; + Some(p.range.clone()) +} + +/// 有一段末尾一条没结束的命令伸进了后面、认得到后面的选项(见模块说明):这时一段段单拿出来 +/// 找的和整段做的不一样。`open(i)`:第 `i` 段末尾有这样一条命令 +fn leaks(text: &str, pieces: &[Piece], open: impl Fn(usize) -> bool) -> bool { + // 没结束的命令往后看到了哪儿:后一段的命令结束在这之前的话,那一截已经看过了 + let mut looked = 0; + for (i, p) in pieces.iter().enumerate() { + if open(i) && p.range.end >= looked { + let (option, end) = + super::secrets::place::option_before_command_end(text.as_bytes(), p.range.end); + if option { + return true; + } + looked = end; + } + } + false +} + +/// 后台:把整段做出来的命中按段分开存下来(见 [`ScanCache::fill`])。哪一样对不上(切不开、 +/// 不是 JSON、有伸进后面的命令、一处命中跨了段)就一段都不存 +fn store(shared: &Shared, job: Job) { + let text = job.text.as_str(); + let Some(pieces) = split(text) else { + return; + }; + if pieces.len() > CAPACITY || !super::rules::json_like(text) { + return; + } + let open: Vec = pieces + .iter() + .map(|p| job.cmdline && super::secrets::place::command_open_at_end(&text[p.range.clone()])) + .collect(); + if leaks(text, &pieces, |i| open[i]) { + return; + } + let mut each: Vec> = vec![Vec::new(); pieces.len()]; + for h in job.hits { + let at = pieces.partition_point(|p| p.range.end <= h.bytes.start); + let Some(p) = pieces.get(at) else { + return; + }; + if h.bytes.start < p.range.start || h.bytes.end > p.range.end { + return; + } + // 单拿出来时前面多一个括号 + let shift = p.range.start - 1; + each[at].push(Hit { + bytes: h.bytes.start - shift..h.bytes.end - shift, + ..h + }); + } + let tag = scope_tag(job.shape); + let mark = window(&pieces).map(|at| Mark { + head: head(&job.fp, job.day, &tag, text), + sum: Sha256::digest(&text.as_bytes()[at.clone()]).into(), + at, + }); + let mut lru = shared.lock(); + if let Some(m) = mark { + lru.saw(m); + } + for ((p, hits), open) in pieces.iter().zip(each).zip(open) { + if !(MIN_BYTES..=MAX_BYTES).contains(&p.range.len()) { + continue; + } + let k = key(&job.fp, job.day, &tag, p, text); + lru.put( + k, + Found { + hits: hits.into(), + open, + readable: job.readable, + }, + ); + } +} + +/// 这套规则下能不能按段缓存(见模块说明的「不缓存的情形」) +pub(crate) fn cacheable(rules: &RuleSet) -> bool { + !CONTEXTUAL.iter().any(|id| rules.is_on(id)) + && !rules + .saved() + .is_some_and(|s| s.values().any(|v| v.contains('"'))) +} + +/// 一个请求的开头:键的那几样加上请求体的头 [`HEAD_BYTES`] 个字节(见 [`Mark`]) +fn head(fp: &[u8; 32], day: u32, scope: &str, text: &str) -> Key { + let b = text.as_bytes(); + let mut h = Sha256::new(); + h.update(b"head"); + h.update(fp); + h.update(day.to_le_bytes()); + h.update((scope.len() as u64).to_le_bytes()); + h.update(scope.as_bytes()); + h.update(&b[..b.len().min(HEAD_BYTES)]); + h.finalize().into() +} + +/// 键里的「在哪儿找」:整段是空的,正文是哪种形状 +fn scope_tag(shape: Option) -> String { + shape.map(|s| format!("{s:?}")).unwrap_or_default() +} + +/// 一段的键:指纹、日期、在哪儿找(整段,还是哪种形状的正文)、是哪个数组里的元素(还是 +/// 套了花括号的几项)、原文 +fn key(fp: &[u8; 32], day: u32, scope: &str, p: &Piece, text: &str) -> Key { + let b = text.as_bytes(); + let mut h = Sha256::new(); + h.update(fp); + h.update(day.to_le_bytes()); + h.update((scope.len() as u64).to_le_bytes()); + h.update(scope.as_bytes()); + match &p.parent { + Some(k) => { + h.update([p.open]); + h.update((k.len() as u64).to_le_bytes()); + h.update(&b[k.clone()]); + } + None => h.update([0]), + } + h.update(&b[p.range.clone()]); + h.finalize().into() +} + +/// 这一段里哪些字符串是正文,区间是单拿出来时的位置。数组里的一串元素装进一个只有它们的同名 +/// 数组里认(`{"messages":[元素, …]}`)—— 一个元素里哪些字符串算正文只看它自己和它在哪个数组 +/// 里,不看它是第几个、旁边是谁。认不出是 `None` +fn written(text: &str, p: &Piece, shape: Shape) -> Option>> { + let piece = p.text(text); + match &p.parent { + None => tw_dialect::written::ranges(shape, &piece), + Some(k) => { + let head = format!("{{\"{}\":", &text[k.clone()]); + let alone = format!("{head}{piece}}}"); + Some( + tw_dialect::written::ranges(shape, &alone)? + .into_iter() + .map(|r| r.start - head.len()..r.end - head.len()) + .collect(), + ) + } + } +} + +/// 单拿出来找一段:扫、去掉不算的([`super::flow::kept`])。`shape`:只找这种形状的正文 +/// ([`written`]);`None` 是整段都找。`None`:这一段单拿出来不是 JSON、正文认不出、或者命中 +/// 落在套上去的括号上,整个请求照旧整段做 +fn find( + text: &str, + p: &Piece, + rules: &RuleSet, + shape: Option, + cmdline: bool, +) -> Option { + let piece = p.text(text); + // 整段是 JSON、单拿出来却不是的话(这一段本身就坏了),两边扫法不一样 + if !super::rules::json_like(&piece) { + return None; + } + let raw = super::rules::scan_unresolved(&piece, rules); + // 套上去的那对括号不是原文:落在它上面的命中(不该有)对不回原文 + let len = piece.len(); + if raw.iter().any(|h| h.bytes.start == 0 || h.bytes.end >= len) { + return None; + } + let open = cmdline && super::secrets::place::command_open_at_end(&text[p.range.clone()]); + let (hits, readable) = match shape { + None => (super::flow::kept(&piece, raw, None:: _>), true), + Some(shape) => { + // 没有命中就不认正文(见 `Found::readable`) + let mut readable = None; + let hits = super::flow::kept( + &piece, + raw, + Some(|| { + let w = written(text, p, shape); + readable = Some(w.is_some()); + w + }), + ); + match readable { + Some(false) => return None, + Some(true) => (hits, true), + None => (hits, false), + } + } + }; + Some(Found { + hits: hits.into(), + open, + readable, + }) +} + +/// 请求体里的一段(见模块说明)。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct Piece { + /// 在原文里的字节区间 + pub range: Range, + /// 单拿出来扫的时候外面套的括号:挨着的几个顶层项(`"key": value, …`)套 `{`,数组里 + /// 挨着的几个元素(`{…}, {…}`)套 `[` + pub open: u8, + /// 数组里的元素:那个数组是哪个顶层键的值(键在原文里引号之间的那一段) + pub parent: Option>, +} + +impl Piece { + /// 单拿出来扫的那段文字 + fn text(&self, body: &str) -> String { + let raw = &body[self.range.clone()]; + let mut s = String::with_capacity(raw.len() + 2); + s.push(char::from(self.open)); + s.push_str(raw); + s.push(if self.open == b'[' { ']' } else { '}' }); + s + } + + /// 单拿出来扫时的位置 → 原文里的位置(前面多了一个括号) + fn to_body(&self, at: usize) -> usize { + self.range.start + at - 1 + } +} + +/// JSON 的空白(只有这四个:别的空白字符在 JSON 里不合规) +fn is_ws(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\r') +} + +fn skip_ws(b: &[u8], mut i: usize) -> usize { + while i < b.len() && is_ws(b[i]) { + i += 1; + } + i +} + +/// `b[i]` 是一个字符串开头的引号:返回结尾那个引号后面的位置。没结尾是 `None` +fn string_end(b: &[u8], i: usize) -> Option { + let mut j = i + 1; + loop { + // 短的(键、角色、类型)逐字节看完就到头了,比起一次 SIMD 查找还省;长的再交给它 + let near = (j + 24).min(b.len()); + let k = match b[j.min(near)..near] + .iter() + .position(|&c| c == b'"' || c == b'\\') + { + Some(k) => k, + None if near == b.len() => return None, + None => near - j + memchr::memchr2(b'"', b'\\', &b[near..])?, + }; + if b[j + k] == b'\\' { + j += k + 2; + } else { + return Some(j + k + 1); + } + } +} + +/// 引号和括号 +const STRUCTURE: [bool; 256] = { + let mut t = [false; 256]; + t[b'"' as usize] = true; + t[b'{' as usize] = true; + t[b'}' as usize] = true; + t[b'[' as usize] = true; + t[b']' as usize] = true; + t +}; + +/// 从 `i` 起的一个值到哪儿为止,和它里面最深有几层(数和字面量是 0 层)。只认结构:括号 +/// 配不配对、数写得对不对由单拿出来那一段的 JSON 校验管([`super::rules::json_like`]) +fn value_end(b: &[u8], i: usize) -> Option<(usize, usize)> { + match *b.get(i)? { + b'"' => Some((string_end(b, i)?, 0)), + b'{' | b'[' => { + let (mut depth, mut deepest) = (0usize, 0usize); + let mut j = i; + while j < b.len() { + // 结构以外的字节(数、字面量、空白、逗号、冒号)一张表跳过 + if !STRUCTURE[usize::from(b[j])] { + j += 1; + continue; + } + match b[j] { + b'"' => { + j = string_end(b, j)?; + continue; + } + b'{' | b'[' => { + depth += 1; + deepest = deepest.max(depth); + } + b'}' | b']' => { + depth -= 1; + if depth == 0 { + return Some((j + 1, deepest)); + } + } + _ => {} + } + j += 1; + } + None + } + _ => { + let mut j = i; + while j < b.len() && !is_ws(b[j]) && !matches!(b[j], b',' | b'}' | b']') { + j += 1; + } + (j > i).then_some((j, 0)) + } + } +} + +/// 一个键能不能不经校验就算合规:没有转义、没有控制字符。按段扫时被拆开的数组的键不在 +/// 任何一段里,没人替它校验 +fn plain_key(k: &[u8]) -> bool { + !k.iter().any(|&c| c == b'\\' || c < 0x20) +} + +/// 把请求体切成段(见模块说明)。**整个请求体是一个顶层为对象的 JSON**,段与段之间只有 +/// 结构上的字和空白,键都不重复;任何一样对不上就是 `None`,整段扫。 +/// +/// 各段单拿出来都是 JSON([`super::rules::json_like`],扫的时候核对),请求体就是 JSON: +/// 段外面的结构这里核对过了。反过来也是:请求体是 JSON 的话,它的每一项、数组的每个元素都是 +pub(crate) fn split(text: &str) -> Option> { + let b = text.as_bytes(); + let mut i = skip_ws(b, 0); + if b.get(i) != Some(&b'{') { + return None; + } + i = skip_ws(b, i + 1); + let mut out = Vec::new(); + let mut keys: Vec> = Vec::new(); + // 正在接的几个挨着的、不拆开的顶层项 + let mut group: Option> = None; + if b.get(i) == Some(&b'}') { + return None; + } + loop { + // "key" + if b.get(i) != Some(&b'"') { + return None; + } + let member = i; + let key_end = string_end(b, i)?; + let key = &b[i + 1..key_end - 1]; + // 重复的键:解析时后一个盖住前一个,哪些字符串算正文跟着变,按段扫说不清。只差在 + // 大小写和下划线上的也算重复:Gemini 的 `systemInstruction` 和 `system_instruction` + // 是同一项,两个都写着时只认一个 + let norm: Vec = key + .iter() + .filter(|&&c| c != b'_') + .map(u8::to_ascii_lowercase) + .collect(); + if keys.contains(&norm) { + return None; + } + keys.push(norm); + i = skip_ws(b, key_end); + if b.get(i) != Some(&b':') { + return None; + } + i = skip_ws(b, i + 1); + let value = i; + // 请求体可能就断在冒号后面(坏掉的客户端、插件的输出):值不一定有 + let items = (b.get(value) == Some(&b'[') && plain_key(key)) + .then(|| elements(b, value)) + .flatten(); + let (end, depth) = match &items { + Some((_, end, depth)) => (*end, *depth), + None => value_end(b, value)?, + }; + if depth + 1 > DEPTH_MAX { + return None; + } + match items { + Some((items, _, _)) => { + out.extend(group.take().map(|range| Piece { + range, + open: b'{', + parent: None, + })); + // 挨着的元素攒够 `RUN_BYTES` 算一段。从数组开头往后攒:历史只在末尾加长时, + // 前面攒好的那几段一字不变 + let mut run: Option> = None; + for r in items { + let r = run.take().map_or(r.clone(), |x| x.start..r.end); + if r.len() >= RUN_BYTES { + out.push(Piece { + range: r, + open: b'[', + parent: Some(member + 1..key_end - 1), + }); + } else { + run = Some(r); + } + } + out.extend(run.map(|range| Piece { + range, + open: b'[', + parent: Some(member + 1..key_end - 1), + })); + } + // 挨着的几项合成一段:顶层相邻的两项(`"lat":"…","lng":"…"`、`"a":["…Authorization"], + // "x-api-key":"…"`)有的判据是连起来看的 + _ => { + let start = group.take().map_or(member, |g| g.start); + group = Some(start..end); + } + } + i = skip_ws(b, end); + match b.get(i) { + Some(b',') => i = skip_ws(b, i + 1), + Some(b'}') => break, + _ => return None, + } + } + out.extend(group.map(|range| Piece { + range, + open: b'{', + parent: None, + })); + (skip_ws(b, i + 1) == b.len()).then_some(out) +} + +/// `b[from..]` 是一个**全是对象**、至少一个元素的数组的话:每个元素的区间、数组在哪儿结束、 +/// 里面最深有几层(同 [`value_end`]) +fn elements(b: &[u8], from: usize) -> Option<(Vec>, usize, usize)> { + let mut i = skip_ws(b, from + 1); + let mut out = Vec::new(); + let mut deepest = 0; + loop { + if b.get(i) != Some(&b'{') { + return None; + } + let (end, depth) = value_end(b, i)?; + deepest = deepest.max(depth + 1); + out.push(i..end); + i = skip_ws(b, end); + match b.get(i) { + Some(b',') => i = skip_ws(b, i + 1), + Some(b']') => return Some((out, i + 1, deepest)), + _ => return None, + } + } +} + +#[cfg(test)] +mod bench; +#[cfg(test)] +mod tests; diff --git a/crates/tw-guard/src/redact/cache/bench.rs b/crates/tw-guard/src/redact/cache/bench.rs new file mode 100644 index 00000000..feaa367a --- /dev/null +++ b/crates/tw-guard/src/redact/cache/bench.rs @@ -0,0 +1,231 @@ +//! 带缓存和不带缓存各要多久(`cargo test --release -p tw-guard --lib redact::cache::bench -- --ignored --nocapture`)。 +//! +//! 每种请求体量三样,各取几遍的中位数:不带缓存、头一回(一份空的缓存)、存好之后(后台存完再量)。 +//! 请求体有四种:几万条很短的消息(短于存的下限)、几万条七十来字节的、三千条三百字节的、两千条 +//! 两千字节的;外加一段 200 轮的对话,量第 10、100、200 轮(存好的是前一轮的,这一轮新加的两句 +//! 要扫)。量的是「看一遍」([`flow::look_hits_with`]:找、报、编号)。 +//! +//! 再加一段新的对话、开头的系统提示和存过的另一段一样(开头对得上,要切开看了才知道没存过)。 +//! +//! 头一回不能比不带缓存慢两成以上,存好之后不能比不带缓存慢(留 5% 的量的误差)。**不在 CI 上跑**(机器快慢不一、 +//! 测试并行跑),改了缓存在本机跑一遍 + +use std::time::{Duration, Instant}; + +use serde_json::{Value, json}; +use tw_dialect::ir::Dialect; + +use super::ScanCache; +use crate::policy::Mode; +use crate::redact::flow::{self, Shape}; +use crate::redact::replace::{Ledger, Scheme}; +use crate::redact::rules::RuleSet; + +const SHAPE: Shape = Shape::Generate(Dialect::Anthropic); + +const CODE: &str = r#"```rust +fn parse(input: &str) -> Result { + let mut out = Config::default(); + for (i, line) in input.lines().enumerate() { + let Some((k, v)) = line.split_once('=') else { continue }; + match k.trim() { + "port" => out.port = v.trim().parse().map_err(|_| Error::Line(i))?, + "host" => out.host = v.trim().to_string(), + _ => {} + } + } + Ok(out) +} +```"#; + +const PROSE: &str = "The function reads the file line by line and splits each line at the first \ +equals sign. Lines without one are skipped, unknown keys are ignored, and a port that does not \ +parse is reported with its line number so the caller can point at it. "; + +/// 第 `i` 轮:用户问一句(有时贴一段代码),模型答一段(带代码,有时调一次工具) +fn turn(i: usize, key_at: Option) -> [Value; 2] { + let mut ask = format!( + "Turn {i}: why does the parser skip line {}? {PROSE}", + i % 17 + ); + if i % 3 == 0 { + ask.push_str(CODE); + } + if key_at == Some(i) { + ask.push_str(" my key is sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"); + } + let user = json!({"role": "user", "content": ask}); + let mut blocks = vec![json!({"type": "text", + "text": format!("{PROSE}{PROSE}\n{CODE}\n{PROSE}")})]; + if i % 4 == 0 { + blocks.push( + json!({"type": "tool_use", "id": format!("toolu_{i}"), "name": "bash", + "input": {"command": format!("cargo test -p parser -- line_{i}")}}), + ); + } + let assistant = json!({"role": "assistant", "content": blocks}); + [user, assistant] +} + +fn body(history: &[Value]) -> String { + let system = PROSE.repeat(40); + let tools: Vec = (0..8) + .map(|t| { + json!({"name": format!("tool_{t}"), "description": PROSE.repeat(6), + "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}}}) + }) + .collect(); + json!({"model": "claude", "max_tokens": 4096, "system": system, + "messages": history, "tools": tools, "stream": true}) + .to_string() +} + +fn look(rules: &RuleSet, text: &str, cache: Option<&ScanCache>) -> flow::Look { + flow::look_hits_with( + Mode::Enforce, + rules, + text.as_bytes(), + SHAPE, + Ledger::new(Scheme::SECRET), + cache, + ) +} + +fn median(mut v: Vec) -> Duration { + v.sort(); + v[v.len() / 2] +} + +const ROUNDS: usize = 21; + +/// 不带缓存、头一回、存好之后。`warm_with`:存好的是哪一份(多轮对话是前一轮的)。 +/// +/// 三样在每一遍里各量一次、轮换先后:单独一次的耗时比连着跑几遍的长(处理器的缓存是凉的), +/// 不带缓存的连着跑、带缓存的单独跑的话,比出来的不是一回事 +fn measure(rules: &RuleSet, text: &str, warm_with: &str) -> (Duration, Duration, Duration) { + let (mut plain, mut cold, mut warm) = (Vec::new(), Vec::new(), Vec::new()); + for round in 0..ROUNDS { + // 缓存跟着运行时建,请求来的时候后台线程早就起好、闲着 + let fresh = ScanCache::new(); + let filled = ScanCache::new(); + for _ in 0..2 { + look(rules, warm_with, Some(&filled)); + filled.settle(); + } + std::thread::sleep(Duration::from_millis(2)); + for k in 0..3 { + match (round + k) % 3 { + 0 => { + let t = Instant::now(); + std::hint::black_box(look(rules, text, None)); + plain.push(t.elapsed()); + } + 1 => { + let t = Instant::now(); + std::hint::black_box(look(rules, text, Some(&fresh))); + cold.push(t.elapsed()); + fresh.settle(); + } + _ => { + let t = Instant::now(); + let l = std::hint::black_box(look(rules, text, Some(&filled))); + warm.push(t.elapsed()); + let p = look(rules, text, None); + assert_eq!((l.found, l.hits), (p.found, p.hits)); + filled.settle(); + } + } + } + } + (median(plain), median(cold), median(warm)) +} + +fn row(name: &str, text: &str, (plain, cold, warm): (Duration, Duration, Duration)) -> (f64, f64) { + let (c, w) = ( + cold.as_secs_f64() / plain.as_secs_f64(), + warm.as_secs_f64() / plain.as_secs_f64(), + ); + println!( + "{name:<22} | {:>7.0} | {:>9.2?} | {:>9.2?} {c:>5.2}x | {:>9.2?} {w:>5.2}x", + text.len() as f64 / 1024.0, + plain, + cold, + warm + ); + (c, w) +} + +fn messages(n: usize, content: impl Fn(usize) -> String) -> String { + let msgs: Vec = (0..n) + .map(|i| json!({"role": if i % 2 == 0 {"user"} else {"assistant"}, "content": content(i)})) + .collect(); + json!({"model": "m", "messages": msgs}).to_string() +} + +#[test] +#[ignore] +fn cold_and_warm_against_no_cache() { + let rules = RuleSet::defaults(); + println!( + "\nbody | KB | no cache | cold (ratio) | warm (ratio)" + ); + let mut worst = (0f64, 0f64); + let mut note = |r: (f64, f64)| worst = (worst.0.max(r.0), worst.1.max(r.1)); + for (name, text) in [ + ("20k x <64 B", messages(20_000, |i| format!("ok {i}"))), + ( + "20k x 70 B", + messages(20_000, |i| { + format!("{i} {}", "lorem ipsum dolor ".repeat(3)) + }), + ), + ( + "3k x 300 B", + messages(3_000, |i| { + format!("{i} {}", "lorem ipsum dolor ".repeat(16)) + }), + ), + ( + "2k x 2000 B", + messages(2_000, |i| { + format!("{i} {}", "lorem ipsum dolor ".repeat(111)) + }), + ), + ] { + note(row(name, &text, measure(&rules, &text, &text))); + } + for key_at in [None, Some(3)] { + let mut history = Vec::new(); + let mut bodies = Vec::new(); + for i in 1..=200 { + history.extend(turn(i, key_at)); + bodies.push(body(&history)); + } + for n in [10, 100, 200] { + let name = format!( + "chat turn {n}{}", + if key_at.is_some() { " +key" } else { "" } + ); + note(row( + &name, + &bodies[n - 1], + measure(&rules, &bodies[n - 1], &bodies[n - 2]), + )); + } + } + // 一段新的对话,开头(几 KB 的系统提示)和存过的另一段一模一样:开头对得上,要切开再看 + let chat = |seed: usize| { + let mut all = vec![json!({"role": "system", "content": PROSE.repeat(40)})]; + for i in 1..=100 { + all.extend(turn(i + seed, None)); + } + json!({"messages": all, "model": "m"}).to_string() + }; + // 存好的是另一段对话:这一列量的也是头一回,按头一回的标准算 + let (other, new) = (chat(1_000), chat(0)); + let (c, w) = row("new chat, same system", &new, measure(&rules, &new, &other)); + note((c.max(w), 0.0)); + assert!(worst.0 <= 1.2, "头一回比不带缓存慢了 {:.2} 倍", worst.0); + // 不走缓存的(太短的请求体)两边做的是同一件事,差的是量的误差 + assert!(worst.1 <= 1.05, "存好之后比不带缓存慢了 {:.2} 倍", worst.1); +} diff --git a/crates/tw-guard/src/redact/cache/tests.rs b/crates/tw-guard/src/redact/cache/tests.rs new file mode 100644 index 00000000..81718231 --- /dev/null +++ b/crates/tw-guard/src/redact/cache/tests.rs @@ -0,0 +1,1291 @@ +use std::sync::Arc; + +use serde_json::{Value, json}; +use tw_dialect::ir::Dialect; + +use super::*; +use crate::policy::Mode; +use crate::redact::flow::{self, Scope, Shape}; +use crate::redact::replace::{Ledger, Scheme}; +use crate::redact::rules::{BUILTINS, Pattern, Rule, SavedSecret, SavedSource, SavedSourceKind}; + +// ---------------------------------------------------------------- 造数据 + +/// 可复现的伪随机数 +struct Rng(u64); + +impl Rng { + fn next(&mut self, n: usize) -> usize { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + (self.0 % n.max(1) as u64) as usize + } + + fn pick<'a, T>(&mut self, xs: &'a [T]) -> &'a T { + &xs[self.next(xs.len())] + } + + fn chance(&mut self, one_in: usize) -> bool { + self.next(one_in) == 0 + } +} + +/// 补上 Luhn 校验位 +fn luhn_done(body: &str) -> String { + let sum: u32 = body + .bytes() + .rev() + .enumerate() + .map(|(i, c)| { + let x = u32::from(c - b'0'); + if i % 2 == 0 { + (x * 2) / 10 + (x * 2) % 10 + } else { + x + } + }) + .sum(); + format!("{body}{}", (10 - sum % 10) % 10) +} + +/// 补上身份证号的校验位 +fn id_done(first17: &str) -> String { + let sum: u32 = first17 + .bytes() + .enumerate() + .map(|(i, c)| u32::from(c - b'0') * (2u32.pow(17 - i as u32) % 11)) + .sum(); + let c = (12 - sum % 11) % 11; + let check = if c == 10 { + 'X' + } else { + char::from(b'0' + c as u8) + }; + format!("{first17}{check}") +} + +const SAVED: &str = "q7Rk2LmN8vXc4TpW9zYb"; +const MNEMONIC: &str = + "legal winner thank year wave sausage worth useful legal winner thank yellow"; + +/// 一小段:有的是各条规则认得的值,有的专挑段与段的接缝(消息开头、结尾)放 —— 半串 +/// 助记词、一组卡号、只有前文才说得清的号码、结尾的转义 +fn fragment(r: &mut Rng) -> String { + let card = luhn_done(&format!("62220212345{:04}", r.next(10_000))); + let id = id_done(&format!("11010119900307{:03}", r.next(1000))); + let pool: Vec = vec![ + "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789".into(), + "ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789".into(), + format!("AKIA{}", ["QRSTUVWXYZ234567", "ABCDEFGHIJKLMNOP"][r.next(2)]), + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk".into(), + "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7\n-----END PRIVATE KEY-----".into(), + "postgres://app:hunter2xyz9@db.internal:5432/orders".into(), + "mail alice@corp.example.com now".into(), + card.clone(), + format!("{} {} {} {}", &card[..4], &card[4..8], &card[8..12], &card[12..]), + format!("{}-{}-{}-{}", &card[..4], &card[4..8], &card[8..12], &card[12..]), + card[..4].to_string(), + card[12..].to_string(), + id, + "call 13800138000".into(), + MNEMONIC.into(), + "legal winner thank year wave sausage".into(), + "worth useful legal winner thank yellow".into(), + "API_TOKEN=abc123XYZ789qq".into(), + "DB_PASSWORD=\"s3cr3t-Pa55w0rd\"".into(), + "password: hunter2hunter2".into(), + "export SECRET_KEY='Zx9-qq81_ab77'".into(), + "TOKEN=".into(), + "api_key".into(), + "Authorization: Bearer abcDEF123456ghiJKL789mno".into(), + "Authorization".into(), + "https://hooks.example.com/cb?token=abcDEF123456ghi&x=1".into(), + "mysql -u root -pS3cretPass99 db".into(), + "护照号码 E12345678".into(), + "护照".into(), + "curl -s https://api.example.com".into(), + "-u admin:S3cretPass99 x".into(), + "--password=S3cretPass99".into(), + "-pS3cretPass99".into(), + "E12345678".into(), + "通行证 C12345678".into(), + "39.9042, 116.4074".into(), + "39.9042".into(), + "116.4074".into(), + "IBAN DE89370400440532013000".into(), + "DE89".into(), + "10.0.0.5 and 8.8.8.8 v1.2.3.4".into(), + "MAC 00:1A:2B:3C:4D:5E".into(), + "00:1A:2B".into(), + "<>".into(), + "<>".into(), + SAVED.into(), + format!("x{SAVED}"), + "E-123456 and MYCORP-222222".into(), + "mycorp-333333".into(), + "今天天气晴,适合出门。这是一段中文。".into(), + "姓名:张三,电话 13800138000".into(), + "{\"real_name\":\"张三\",\"phone\":\"13800138000\"}".into(), + "{\"name\":\"李四\"}".into(), + "name".into(), + "address: 北京市朝阳区建国路 88 号".into(), + "C:\\Users\\dev\\".into(), + "he said \"hi\"".into(), + "\\".into(), + "\\n".into(), + "1.".into(), + "12".into(), + "the quick brown fox jumps over the lazy dog".into(), + "fn main() { let x = vec![1, 2, 3]; }".into(), + "abandon".into(), + "zoo".into(), + format!("data:image/png;base64,{}", "iVBORw0KGgo".repeat(30)), + "VIN 1HGCM82633A004352".into(), + "IMEI 490154203237518".into(), + "SSN 123-45-6789".into(), + "+1 415 555 2671".into(), + "京A12345".into(), + ]; + r.pick(&pool).clone() +} + +const SEPS: &[&str] = &[" ", " ", "\n", ", ", "", "\t", "。", ": ", "\"", "\\"]; + +/// 一段正文:几个片段,用各种分隔连起来 +fn prose(r: &mut Rng) -> String { + let n = 1 + r.next(6); + let mut s = String::new(); + for i in 0..n { + if i > 0 { + s.push_str(r.pick(SEPS)); + } + s.push_str(&fragment(r)); + } + // 有的长一点,过 [`MIN_BYTES`] + if r.chance(2) { + s.push(' '); + s.push_str(&"lorem ipsum dolor sit amet ".repeat(1 + r.next(4))); + } + s +} + +/// 第 `turn` 轮用户和模型各说一句,写成 `dialect` 的样子 +fn messages(r: &mut Rng, dialect: Dialect) -> [Value; 2] { + let user = prose(r); + let reply = prose(r); + let args = json!({"q": prose(r), "token": fragment(r)}).to_string(); + let blob = "iVBORw0KGgo".repeat(40); + match dialect { + Dialect::Anthropic => { + let assistant = if r.chance(2) { + json!({"role": "assistant", "content": reply}) + } else { + json!({"role": "assistant", "content": [ + {"type": "thinking", "thinking": prose(r), "signature": blob}, + {"type": "text", "text": reply}, + {"type": "tool_use", "id": "toolu_1", "name": "run", + "input": {"cmd": prose(r), "api_key": fragment(r)}}, + ]}) + }; + let user = if r.chance(2) { + json!({"content": user, "role": "user"}) + } else { + json!({"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": prose(r)}, + {"type": "text", "text": user}, + ]}) + }; + [user, assistant] + } + Dialect::Chat => [ + json!({"role": "user", "content": user}), + json!({"role": "assistant", "content": reply, + "tool_calls": [{"id": "call_1", "type": "function", + "function": {"name": "f", "arguments": args}}]}), + ], + Dialect::Responses => [ + json!({"type": "message", "role": "user", + "content": [{"type": "input_text", "text": user}]}), + if r.chance(2) { + json!({"type": "function_call", "call_id": "c1", "name": "f", "arguments": args}) + } else { + json!({"type": "reasoning", "id": "rs_0a1b13800138000", + "summary": [{"type": "summary_text", "text": reply}], + "encrypted_content": blob}) + }, + ], + _ => [ + json!({"role": "user", "parts": [{"text": user}]}), + json!({"role": "model", "parts": [{"text": reply}, + {"functionCall": {"name": "f", "args": {"q": prose(r)}}}]}), + ], + } +} + +/// 一轮的请求体:顶层的几项加上到这一轮为止的全部历史 +fn body(dialect: Dialect, system: &str, history: &[Value], extra: &Value) -> String { + let tools = json!([{"name": "run", "description": format!("runs things {system}"), + "input_schema": {"type": "object"}}]); + let mut b = match dialect { + Dialect::Anthropic => json!({"model": "m", "system": system, "messages": history, + "tools": tools, "max_tokens": 1024}), + Dialect::Chat => { + let mut all = vec![json!({"role": "system", "content": system})]; + all.extend(history.iter().cloned()); + json!({"model": "m", "messages": all, "stream": true}) + } + Dialect::Responses => json!({"model": "m", "instructions": system, "input": history, + "tools": [{"type": "function", "name": "f", + "description": system}]}), + _ => json!({"systemInstruction": {"parts": [{"text": system}]}, "contents": history}), + }; + if let (Some(o), Some(x)) = (b.as_object_mut(), extra.as_object()) { + for (k, v) in x { + o.insert(k.clone(), v.clone()); + } + } + b.to_string() +} + +/// 非 ASCII 的字都写成 `\uXXXX`(Python 写的客户端默认这样) +fn ascii_escaped(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for c in s.chars() { + if c.is_ascii() { + out.push(c); + } else { + let mut buf = [0u16; 2]; + for u in c.encode_utf16(&mut buf) { + out.push_str(&format!("\\u{u:04x}")); + } + } + } + out +} + +/// 内置的全开,除了 `skip` 里的 +fn all_but(skip: &[&str]) -> RuleSet { + let ids: Vec<&str> = BUILTINS + .iter() + .map(|b| b.id) + .filter(|id| !skip.contains(id)) + .collect(); + RuleSet::only(&ids) +} + +fn saved(list: &[&str]) -> Vec { + list.iter() + .map(|v| SavedSecret { + value: v.to_string(), + source: SavedSource { + kind: SavedSourceKind::UpstreamKey, + name: "u".into(), + }, + hosts: Vec::new(), + }) + .collect() +} + +fn with_customs(set: RuleSet) -> RuleSet { + let rule = |text: &str, regex: bool, case_sensitive: bool| Pattern { + text: text.into(), + regex, + case_sensitive, + }; + set.with_rule("corp", rule("MYCORP-", false, false), None) + .unwrap() + .with_rule("emp", rule(r"E-\d{6}", true, true), Some("EMP")) + .unwrap() + .with_rule("lead", rule(r"^\w+", true, true), None) + .unwrap() + .with_rule("word", rule(r"\bzoo\b", true, false), None) + .unwrap() +} + +/// 对照用的几套规则:出厂、内置全开(除了看得到隔壁的那几条)、内置全开(缓存整个不用)、 +/// 加自定义的、加保存的凭据的、有只记录的 +fn rule_sets() -> Vec<(&'static str, RuleSet)> { + vec![ + ("defaults", RuleSet::defaults()), + ("all-local", all_but(CONTEXTUAL)), + ("all", all_but(&[])), + ("custom", with_customs(RuleSet::defaults())), + ( + "saved", + all_but(CONTEXTUAL).with_saved(saved(&[SAVED, "another-saved-value-123"])), + ), + ( + "record", + with_customs(all_but(CONTEXTUAL)) + .recording(Rule::Builtin("email")) + .recording(Rule::Builtin("bank-card")) + .recording(Rule::Custom("lead".into())), + ), + ] +} + +fn shape(d: Dialect) -> Shape { + Shape::Generate(d) +} + +/// 走一遍带缓存的,等后台把这一轮存完 +fn fill(cache: &ScanCache, text: &str, rules: &RuleSet, scope: Scope) -> Vec { + let hits = flow::hits_in_with(text, rules, scope, Some(cache)); + cache.settle(); + hits +} + +/// 同一份请求体两条路各走一遍:找到的、报出去的、账本、换过的那一份都一样 +fn agree(text: &str, rules: &RuleSet, d: Dialect, cache: &ScanCache, what: &str) { + // 上一回交给后台的先存完:取不取得到缓存不看时机 + cache.settle(); + let s = shape(d); + for scope in [Scope::Written(s), Scope::Whole] { + let plain = flow::hits_in(text, rules, scope); + let cached = flow::hits_in_with(text, rules, scope, Some(cache)); + assert_eq!(plain, cached, "{what} {scope:?}\n{text}"); + assert_eq!( + flow::reported(text, rules, &plain, scope), + flow::reported_with(text, rules, &plain, scope, Some(cache)), + "{what} reported {scope:?}" + ); + } + let fresh = || Ledger::new(Scheme::SECRET).avoiding(text); + let a = flow::look_hits(Mode::Enforce, rules, text.as_bytes(), s, fresh()); + let b = flow::look_hits_with( + Mode::Enforce, + rules, + text.as_bytes(), + s, + fresh(), + Some(cache), + ); + assert_eq!(a.found, b.found, "{what} found"); + assert_eq!(a.hits, b.hits, "{what} hits"); + assert_eq!(a.ledger.table(), b.ledger.table(), "{what} ledger"); + let (x, _) = flow::replace(Mode::Enforce, rules, text.to_string().into(), &a.ledger, s); + let (y, _) = flow::replace_with( + Mode::Enforce, + rules, + text.to_string().into(), + &a.ledger, + s, + Some(cache), + ); + assert_eq!(x, y, "{what} replaced"); +} + +/// 按段扫,**不管规则看不看得到隔壁**:用来说明为什么那几条规则开着时不缓存 +fn piecewise(text: &str, rules: &RuleSet) -> Vec { + let mut out = Vec::new(); + for p in split(text).expect("切得开") { + let piece = p.text(text); + for h in crate::redact::rules::scan_unresolved(&piece, rules) { + out.push(Hit { + bytes: p.to_body(h.bytes.start)..p.to_body(h.bytes.end), + ..h + }); + } + } + crate::redact::rules::resolve(out, rules) +} + +// ---------------------------------------------------------------- 对照 + +/// 多轮对话(每轮加一句用户、一句模型)两条路各走一遍,每一轮都一样。缓存跨轮留着: +/// 第二轮起历史全是取的,新的那两句是扫的;同一轮再走一遍,全是取的 +#[test] +fn multi_turn_conversations_find_exactly_what_a_full_scan_finds() { + let mut r = Rng(0x5eed_cafe_f00d_1234); + let sets = rule_sets(); + let (mut used, mut cacheable_convs) = (0, 0); + for conv in 0..36 { + let d = [ + Dialect::Anthropic, + Dialect::Chat, + Dialect::Responses, + Dialect::Gemini, + ][conv % 4]; + let (name, rules) = &sets[r.next(sets.len())]; + let cache = ScanCache::for_tests(); + let system = prose(&mut r); + // 顶层另外几项:不是正文的 metadata、名字像口令的一个数组(拆开的数组的键不在任何 + // 一段里) + let extra = match r.next(5) { + 0 => json!({"metadata": {"user_id": fragment(&mut r)}}), + 1 => json!({"api_token": [{"x": fragment(&mut r)}], "temperature": 0.5}), + 2 => json!({"passport": [{"no": "E12345678"}]}), + // 挨着正文那几项(按键排序:`instructions`、`system` 后面,最前面)的几项 + 3 => json!({"j1": fragment(&mut r), "j2": fragment(&mut r), + "system_a": fragment(&mut r), "a0": fragment(&mut r), + "contents_z": [{"t": fragment(&mut r)}]}), + _ => json!({}), + }; + let escaped = r.chance(4); + let mut history = Vec::new(); + // 上一轮的请求够不够长、走没走缓存 + let mut warm = false; + for turn in 0..7 { + history.extend(messages(&mut r, d)); + let mut text = body(d, &system, &history, &extra); + if escaped { + text = ascii_escaped(&text); + } + let what = format!("conv {conv} ({name}, {d:?}) turn {turn}"); + let before = cache.misses(); + agree(&text, rules, d, &cache, &what); + // 头一轮之后每一轮只扫新的那两句(只记录的规则另是一套、另存一份) + let per_set = if rules.recording_only().is_some() { + 2 + } else { + 1 + }; + if warm { + assert!( + cache.misses() - before <= 2 * per_set * 6, + "{what}: 扫了 {} 段", + cache.misses() - before + ); + } + warm = text.len() >= BODY_MIN; + let before = cache.misses(); + agree(&text, rules, d, &cache, &format!("{what} again")); + assert_eq!(cache.misses(), before, "{what}: 同一份再来一遍全是取的"); + } + // 改了一条历史(客户端压缩过上下文、用户编辑了上一句):改了的那条重扫,别的照取 + if history.len() > 2 { + let at = r.next(history.len()); + history[at] = messages(&mut r, d)[0].clone(); + let text = body(d, &system, &history, &extra); + agree(&text, rules, d, &cache, &format!("conv {conv} edited")); + } + // 历史里有一条伸进下一段的命令(片段里有 `curl`、`-u …`)的对话整段做、不存 + if cacheable(rules) { + (used, cacheable_convs) = (used + usize::from(cache.warm() > 0), cacheable_convs + 1); + } + } + assert!( + used * 2 > cacheable_convs, + "只有 {used}/{cacheable_convs} 段对话用上了缓存" + ); +} + +/// 换了规则、同一份缓存接着用:取出来的是新规则扫的,不是旧的 +#[test] +fn changing_the_rules_or_the_saved_keys_never_reuses_old_results() { + let mut r = Rng(0xfeed_beef_0000_0042); + let cache = ScanCache::for_tests(); + let mut history = Vec::new(); + for _ in 0..6 { + history.extend(messages(&mut r, Dialect::Anthropic)); + } + history.push( + json!({"role": "user", "content": format!("{SAVED} and MYCORP-1 {}", + "padding text ".repeat(10))}), + ); + let text = body(Dialect::Anthropic, "sys", &history, &json!({})); + let base = all_but(CONTEXTUAL); + let variants = [ + base.clone(), + base.clone().with_saved(saved(&[SAVED])), + with_customs(base.clone()), + base.clone().recording(Rule::Builtin("email")), + RuleSet::defaults(), + base.clone().with_saved(saved(&[SAVED])), + base, + ]; + for (i, rules) in variants.iter().enumerate() { + agree( + &text, + rules, + Dialect::Anthropic, + &cache, + &format!("variant {i}"), + ); + } + let saved_hits = |rules: &RuleSet| { + flow::hits_in_with( + &text, + rules, + Scope::Written(shape(Dialect::Anthropic)), + Some(&cache), + ) + .iter() + .filter(|h| h.rule.id() == "thinkwatch-saved-secret") + .count() + }; + assert_eq!(saved_hits(&variants[0]), 0); + assert!(saved_hits(&variants[1]) > 0, "加了保存的密钥,同一段要重扫"); +} + +/// 几个线程共用一份缓存、各扫各的对话:和不带缓存的一样 +#[test] +fn concurrent_requests_share_one_cache_safely() { + let cache = Arc::new(ScanCache::for_tests()); + let rules = Arc::new(all_but(CONTEXTUAL).with_saved(saved(&[SAVED]))); + let workers: Vec<_> = (0..4u64) + .map(|t| { + let (cache, rules) = (cache.clone(), rules.clone()); + std::thread::spawn(move || { + let mut r = Rng(0x1234_5678 + t); + let mut history = Vec::new(); + for turn in 0..8 { + history.extend(messages(&mut r, Dialect::Chat)); + let text = body(Dialect::Chat, "sys prompt here", &history, &json!({})); + agree( + &text, + &rules, + Dialect::Chat, + &cache, + &format!("thread {t} turn {turn}"), + ); + } + }) + }) + .collect(); + for w in workers { + w.join().unwrap(); + } + assert!(cache.len() > 0); +} + +// ---------------------------------------------------------------- 为什么那几条不缓存 + +/// 只有前文写明了才算的号码:一条消息开头的护照号,前文是上一条消息末尾的「护照」。整段扫 +/// 认得出,按段扫认不出 —— 所以这条开着时不缓存,结果照样和整段扫一样 +#[test] +fn a_keyword_at_the_end_of_the_previous_message_is_why_some_rules_are_not_cached() { + let rules = RuleSet::only(&["cn-passport"]); + // 够长,各成一段(短的几条攒成一段,见 `RUN_BYTES`) + let pad = "padding ".repeat(RUN_BYTES / 8); + let text = json!({"messages": [ + {"content": format!("{pad} 护照")}, + {"content": "E12345678"}, + ]}) + .to_string(); + let whole = crate::redact::rules::scan(&text, &rules); + assert_eq!(whole.len(), 1, "{text}"); + assert!(piecewise(&text, &rules).is_empty(), "按段扫看不到上一条"); + assert!(!cacheable(&rules)); + agree( + &text, + &rules, + Dialect::Anthropic, + &ScanCache::for_tests(), + "passport", + ); +} + +/// 坐标的纬度、经度隔着引号里的东西照样配对(`"lat":"…","lng":"…"`):顶层挨着的几项合成 +/// 一段,不拆开 +#[test] +fn neighbouring_top_level_fields_stay_in_one_piece() { + let rules = RuleSet::only(&["geo-coordinates"]); + let pad = "padding ".repeat(20); + let text = json!({ + "instructions": format!("{pad} 39.90421"), + "lng": format!("116.40742 {pad}"), + "input": [{"type": "message", "content": pad}], + }) + .to_string(); + let whole = crate::redact::rules::scan(&text, &rules); + assert_eq!(whole.len(), 2, "{text}"); + assert_eq!(piecewise(&text, &rules), whole); + assert!(cacheable(&rules)); + agree( + &text, + &rules, + Dialect::Responses, + &ScanCache::for_tests(), + "geo", + ); +} + +/// 一条消息末尾一条没结束的命令,整段扫的时候接着认下一条消息开头的选项:这时候整段扫; +/// 后面头一个换行之前没有选项的话,照样按段取 +#[test] +fn a_command_left_open_at_the_end_of_a_message_reaches_into_the_next() { + let rules = RuleSet::defaults(); + let pad = "padding ".repeat(RUN_BYTES / 8); + let msgs = |second: &str| { + json!({"messages": [ + {"content": format!("{pad} run curl -s https://api.example.com")}, + {"content": second}, + ]}) + .to_string() + }; + let text = msgs(&format!("-u admin:S3cretPass99 {pad}")); + let whole = crate::redact::rules::scan(&text, &rules); + assert!( + whole + .iter() + .any(|h| &text[h.bytes.clone()] == "S3cretPass99"), + "{whole:?}" + ); + assert_ne!(piecewise(&text, &rules), whole, "单拿出来扫认不出"); + let cache = ScanCache::for_tests(); + agree(&text, &rules, Dialect::Anthropic, &cache, "open command"); + assert!(text.len() > 1_024); + // 后面那一条头一个换行之前没有选项:按段取 + let text = msgs(&format!("ok\n-u admin:S3cretPass99 {pad}")); + let cache = ScanCache::for_tests(); + agree( + &text, + &rules, + Dialect::Anthropic, + &cache, + "closed by a newline", + ); + let before = cache.misses(); + agree( + &text, + &rules, + Dialect::Anthropic, + &cache, + "closed by a newline, again", + ); + assert_eq!(cache.misses(), before); + assert!(cache.len() >= 2); +} + +/// 每一条看得到隔壁的规则开着都不缓存;出厂的规则里一条都没有 +#[test] +fn contextual_rules_exist_and_are_off_by_default() { + for id in CONTEXTUAL { + let b = crate::redact::rules::builtin(id).unwrap_or_else(|| panic!("{id} 不是内置规则")); + assert!(!b.on_by_default, "{id}"); + assert!(!cacheable(&RuleSet::only(&[id])), "{id}"); + } + assert!(cacheable(&RuleSet::defaults())); + assert!(cacheable(&all_but(CONTEXTUAL))); + // 保存的凭据本身带引号:按值找时能跨过一个字符串的结尾 + assert!(!cacheable( + &RuleSet::defaults().with_saved(saved(&["abc\"def123456789"])) + )); + assert!(cacheable(&RuleSet::defaults().with_saved(saved(&[SAVED])))); +} + +// ---------------------------------------------------------------- 切段 + +#[test] +fn a_body_splits_into_members_and_array_elements() { + let text = r#" {"model":"m", "messages" : [ {"role":"user","content":"a"} ,{"x":[1,{"y":"]"}]}],"system":"s\"}","tools":[],"n":[1,2]} "#; + let got: Vec<(&str, char)> = split(text) + .unwrap() + .iter() + .map(|p| (&text[p.range.clone()], char::from(p.open))) + .collect(); + assert_eq!( + got, + [ + (r#""model":"m""#, '{'), + // 短的元素攒成一段 + (r#"{"role":"user","content":"a"} ,{"x":[1,{"y":"]"}]}"#, '['), + (r#""system":"s\"}","tools":[],"n":[1,2]"#, '{'), + ] + ); +} + +#[test] +fn anything_unusual_is_scanned_whole() { + for bad in [ + "", + "[]", + "{}", + r#""text""#, + r#"{"a":1,"a":2}"#, + r#"{"a":1} x"#, + r#"{"a":1,}"#, + r#"{"a" 1}"#, + r#"{"a":[{"b":1}"#, + r#"{"a":"unterminated}"#, + "\u{feff}{\"a\":1}", + ] { + assert!(split(bad).is_none(), "{bad:?}"); + } + // 拆开的数组的键带转义:没有哪一段替它校验,整项算一段 + let p = split(r#"{"me\u0073sages":[{"a":1}]}"#).unwrap(); + assert_eq!(p.len(), 1); + assert_eq!(p[0].open, b'{'); + // 太深 + let deep = format!(r#"{{"a":{}{}}}"#, "[".repeat(120), "]".repeat(120)); + assert!(split(&deep).is_none()); +} + +/// 坏掉的请求体(一段里面不是合规的 JSON)不按段扫,照旧整段扫 +#[test] +fn a_broken_piece_falls_back_to_the_whole_scan() { + let rules = RuleSet::defaults(); + let pad = "padding ".repeat(200); + let text = + format!(r#"{{"messages":[{{"content":"{pad} AKIAQRSTUVWXYZ234567"}},{{"content":01}}]}}"#); + let cache = ScanCache::for_tests(); + fill(&cache, &text, &rules, Scope::Whole); + fill(&cache, &text, &rules, Scope::Whole); + assert_eq!(cache.len(), 0, "{text}"); + agree(&text, &rules, Dialect::Anthropic, &cache, "broken"); +} + +/// 只看结构的 JSON 校验放过、认正文时却解不开的请求体(超出范围的数、落单的代理对):整段做的 +/// 时候一处都不算。那一段没有命中也一样 —— 别的段的命中不能照算 +#[test] +fn a_body_whose_written_text_cannot_be_read_counts_nothing_either_way() { + let rules = RuleSet::defaults(); + let pad = "padding ".repeat(150); + for odd in [r#"{"n":1e400}"#, r#"{"s":"\ud800"}"#] { + let text = format!( + r#"{{"messages":[{{"role":"user","content":"{pad} AKIAQRSTUVWXYZ234567"}},{odd},{{"role":"user","content":"{pad}"}}]}}"# + ); + let written = Scope::Written(shape(Dialect::Anthropic)); + assert!(flow::hits_in(&text, &rules, written).is_empty(), "{odd}"); + let cache = ScanCache::for_tests(); + agree(&text, &rules, Dialect::Anthropic, &cache, odd); + agree(&text, &rules, Dialect::Anthropic, &cache, odd); + } +} + +// ---------------------------------------------------------------- 容量 + +fn conversation(n: usize, each: usize) -> String { + let history: Vec = (0..n) + .map(|i| json!({"role": "user", "content": format!("{i:06} {}", "x".repeat(each))})) + .collect(); + json!({"messages": history}).to_string() +} + +/// 数组里的元素攒够 `RUN_BYTES` 才算一段:几万条短消息不是几万段;够长的一条自己一段 +#[test] +fn short_elements_are_scanned_together() { + let text = conversation(20_000, 10); + let pieces = split(&text).unwrap(); + assert!(pieces.len() < 20_000 * 50 / RUN_BYTES, "{}", pieces.len()); + assert!( + pieces[..pieces.len() - 1] + .iter() + .all(|p| p.range.len() >= RUN_BYTES) + ); + let long = conversation(5, RUN_BYTES); + assert_eq!(split(&long).unwrap().len(), 5); + // 末尾加长:前面攒好的几段一字不变 + let more = conversation(20_100, 10); + let after = split(&more).unwrap(); + for (a, b) in pieces[..pieces.len() - 1].iter().zip(&after) { + assert_eq!(text[a.range.clone()], more[b.range.clone()]); + } + let rules = RuleSet::defaults(); + agree( + &text, + &rules, + Dialect::Anthropic, + &ScanCache::for_tests(), + "short", + ); +} + +#[test] +fn short_and_huge_pieces_are_scanned_but_not_kept() { + let rules = RuleSet::defaults(); + // 整段请求体太短不走缓存;一段不到 64 字节的不存 + let text = json!({"model": "m", "messages": [{"role": "user", "content": "y".repeat(2000)}]}) + .to_string(); + let cache = ScanCache::for_tests(); + fill(&cache, &text, &rules, Scope::Whole); + assert_eq!(cache.len(), 1, "`\"model\":\"m\"` 不存"); + // 一条超过 1 MiB 的不存,旁边的照存 + let text = json!({"messages": [ + {"role": "user", "content": "y".repeat(MAX_BYTES + 1)}, + {"role": "user", "content": "z".repeat(100)}, + ]}) + .to_string(); + let cache = ScanCache::for_tests(); + fill(&cache, &text, &rules, Scope::Whole); + assert_eq!(cache.len(), 1); +} + +/// 存满了挤掉最久没用的 +#[test] +fn the_cache_keeps_the_most_recent_entries() { + let found = || Found { + hits: Arc::from(Vec::new()), + open: false, + readable: true, + }; + let key = |i: usize| -> Key { Sha256::digest(i.to_le_bytes()).into() }; + let mut lru = Lru::default(); + for i in 0..CAPACITY + 300 { + lru.put(key(i), found()); + } + assert_eq!(lru.map.len(), CAPACITY); + assert!(lru.get(&key(299)).is_none()); + assert!(lru.get(&key(300)).is_some()); + // 刚用过的这一条不会是下一个被挤掉的 + lru.put(key(CAPACITY + 300), found()); + assert!(lru.get(&key(300)).is_some()); + assert!(lru.get(&key(301)).is_none()); +} + +/// 段数比能存的还多:不走缓存(一轮存进去的下一轮用不上) +#[test] +fn a_body_with_more_pieces_than_the_cache_holds_is_scanned_whole() { + let text = conversation(CAPACITY + 1, RUN_BYTES); + assert_eq!(split(&text).unwrap().len(), CAPACITY + 1); + let cache = ScanCache::for_tests(); + for _ in 0..2 { + fill(&cache, &text, &RuleSet::defaults(), Scope::Whole); + } + assert_eq!(cache.len(), 0); +} + +#[test] +fn the_cache_keeps_a_bounded_number_of_hits() { + let rules = RuleSet::defaults(); + // 一段的命中太多的不存;存着的命中加起来有上限 + let key = "AKIAQRSTUVWXYZ234567"; + let many = |n: usize| { + let history: Vec = (0..n) + .map(|i| json!({"role": "user", "content": format!("{i} {}", vec![key; 3000].join(" "))})) + .collect(); + json!({"messages": history}).to_string() + }; + let cache = ScanCache::for_tests(); + let text = many(30); + // 头一轮整段做、后台存;第二轮按段取 + fill(&cache, &text, &rules, Scope::Whole); + let found = fill(&cache, &text, &rules, Scope::Whole); + assert_eq!(found.len(), 90_000); + assert!( + cache.hits_stored() <= HITS_BUDGET, + "{}", + cache.hits_stored() + ); + assert!(cache.len() < 30); + let huge = json!({"messages": [{"role": "user", + "content": vec![key; ENTRY_HITS_MAX + 1].join(" ")}]}) + .to_string(); + let cache = ScanCache::for_tests(); + fill(&cache, &huge, &rules, Scope::Whole); + assert_eq!(cache.len(), 0); +} + +// ---------------------------------------------------------------- 指纹 + +#[test] +fn the_fingerprint_changes_with_everything_that_changes_hits() { + let base = RuleSet::defaults(); + let fp = base.fingerprint(); + // 同样的规则另建一份:HashSet 的顺序不同,指纹一样 + assert_eq!(fp, RuleSet::defaults().fingerprint()); + let ids: Vec<&str> = BUILTINS + .iter() + .filter(|b| b.on_by_default) + .map(|b| b.id) + .collect(); + let mut rev = ids.clone(); + rev.reverse(); + assert_eq!( + RuleSet::only(&ids).fingerprint(), + RuleSet::only(&rev).fingerprint() + ); + let pattern = |text: &str| Pattern { + text: text.into(), + regex: false, + case_sensitive: true, + }; + let changed = [ + RuleSet::only(&ids[1..]), + base.clone().with_saved(saved(&[SAVED])), + base.clone().with_saved(saved(&["another-saved-value-123"])), + base.clone().with_custom("a", "x+").unwrap(), + base.clone().with_custom("b", "x+").unwrap(), + base.clone().with_custom("a", "x*").unwrap(), + base.clone().with_rule("a", pattern("x+"), None).unwrap(), + base.clone() + .with_rule( + "a", + Pattern { + case_sensitive: false, + ..pattern("x+") + }, + None, + ) + .unwrap(), + base.clone() + .with_rule("a", pattern("x+"), Some("L")) + .unwrap(), + base.clone() + .with_rule("a", pattern("x+"), Some("")) + .unwrap(), + base.clone().recording(Rule::Builtin("jwt")), + base.clone().recording(Rule::Custom("jwt".into())), + base.clone() + .with_custom("a", "1") + .unwrap() + .with_custom("b", "2") + .unwrap(), + base.clone() + .with_custom("b", "2") + .unwrap() + .with_custom("a", "1") + .unwrap(), + ]; + let mut seen = vec![fp]; + for (i, s) in changed.iter().enumerate() { + let f = s.fingerprint(); + assert!(!seen.contains(&f), "第 {i} 个和前面的撞了"); + seen.push(f); + } +} + +const KEYS: &[&str] = &[ + "content", + "text", + "c", + "password", + "api_key", + "name", + "phone", + "lat", + "lng", + "token", + "passport", + "Authorization", + "x", + "real_name", + "address", + "imei", + "护照", + "url", + "DB_PASSWORD", + "role", +]; + +fn adv_string(r: &mut Rng) -> String { + let mut s = String::new(); + let n = 1 + r.next(3); + for i in 0..n { + if i > 0 { + s.push_str(r.pick(SEPS)); + } + s.push_str(&fragment(r)); + } + s +} + +fn adv_object(r: &mut Rng, depth: usize) -> Value { + let mut m = serde_json::Map::new(); + for _ in 0..1 + r.next(3) { + let k = r.pick(KEYS).to_string(); + let v = if depth < 2 && r.chance(4) { + adv_object(r, depth + 1) + } else if r.chance(8) { + json!(r.next(100000)) + } else { + json!(adv_string(r)) + }; + m.insert(k, v); + } + Value::Object(m) +} + +/// 每一条规则单独开着,在随手拼出来的请求体上(键专挑口令、字段名、坐标、护照这些会连起来 +/// 看的,值专挑各条规则认得的片段)两条路各走一遍:一样。按段扫、不管看不看得到隔壁的话, +/// 有几条会不一样(命令行、护照、按字段名认的)—— 前者靠每段记着的「没结束的命令」,后几条 +/// 靠 [`CONTEXTUAL`] +#[test] +fn every_rule_alone_agrees_on_adversarial_bodies() { + let mut r = Rng(0x0dd_ba11_7777_1234); + let mut piecewise_differs = std::collections::BTreeSet::new(); + let mut cached = 0; + for _ in 0..120 { + let mut top = serde_json::Map::new(); + for _ in 0..1 + r.next(4) { + let k = r.pick(KEYS).to_string(); + let v = match r.next(3) { + 0 => json!(adv_string(&mut r)), + 1 => Value::Array((0..1 + r.next(4)).map(|_| adv_object(&mut r, 0)).collect()), + _ => adv_object(&mut r, 0), + }; + top.insert(k, v); + } + // 过了不走缓存的长度下限 + top.insert("zz_pad".into(), json!([{"p": "lorem ipsum ".repeat(100)}])); + let mut text = Value::Object(top).to_string(); + if r.chance(4) { + text = ascii_escaped(&text); + } + for b in BUILTINS { + let rules = RuleSet::only(&[b.id]).with_saved(saved(&[SAVED])); + let whole = crate::redact::rules::scan(&text, &rules); + let cache = ScanCache::for_tests(); + for scope in [Scope::Whole, Scope::Written(shape(Dialect::Anthropic))] { + let got = flow::hits_in_with(&text, &rules, scope, Some(&cache)); + assert_eq!(flow::hits_in(&text, &rules, scope), got, "{}\n{text}", b.id); + } + cached += usize::from(cache.len() > 0); + if piecewise(&text, &rules) != whole { + piecewise_differs.insert(b.id); + } + } + } + assert!(cached > 0); + for id in piecewise_differs { + assert!( + CONTEXTUAL.contains(&id) || id == "command-line-password", + "{id} 单拿出来扫不一样,却没有防着" + ); + } +} + +// ---------------------------------------------------------------- review 0.86 + +/// Review: every (end of one element, start of the next) pair from the seams most likely to +/// carry a judgment across `},{` — open commands, header names, grouped numbers, mnemonics, +/// coordinates, private-key blocks — across several array boundaries and with exceptions in +/// the rule set. Cached and uncached must agree. +#[test] +fn review_seams_between_elements_agree() { + let ends = [ + "curl", + "run curl", + "mysql", + "curl -s", + "mysql -h db", + "PGPASSWORD", + "Authorization", + "Authorization:", + "x-api-key", + "api_key", + "?access_token", + "https://x.example/?sig", + "legal winner thank year wave sausage", + "6222 0212", + "39.90421", + "DE89 3704", + "00:1A:2B", + "-----BEGIN PRIVATE KEY-----", + "sk-ant-api03-", + "\\", + "x", + "curl \\", + "curl\t", + "mysql\\", + ]; + let starts = [ + "-u admin:S3cretPass99", + "-pS3cretPass99", + "--password=S3cretPass99", + "=S3cretPass99", + ": Bearer abcDEF123456ghiJKL789mno", + "Bearer abcDEF123456ghiJKL789mno", + "=abcDEF123456ghi", + "worth useful legal winner thank yellow", + "3456 7890", + "116.40741", + "0440 5320 1300 0", + "3C:4D:5E", + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASC\n-----END PRIVATE KEY-----", + "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", + "n", + " -u admin:S3cretPass99", + "\t-pS3cretPass99", + "\\n-u admin:S3cretPass99", + "'-u' 'admin:S3cretPass99'", + ]; + let filler = "lorem ipsum dolor sit amet ".repeat(50); + let ex = crate::redact::exceptions::Exceptions::new( + ["S3cretPass99", "abcDEF123456ghiJKL789mno"] + .iter() + .map(|v| { + crate::redact::exceptions::parse_hex(&crate::redact::exceptions::sha256_hex(v)) + .unwrap() + }), + ) + .map(Arc::new); + let sets = [ + ("all-local", all_but(CONTEXTUAL)), + ( + "custom+ex", + with_customs(all_but(CONTEXTUAL)).with_exceptions(ex.clone()), + ), + ( + "saved+ex", + all_but(CONTEXTUAL) + .with_saved(saved(&[SAVED])) + .with_exceptions(ex), + ), + ]; + for (name, rules) in &sets { + let cache = ScanCache::for_tests(); + for a in ends { + for b in starts { + let bodies = [ + // two messages + json!({"model": "m", "messages": [ + {"role": "user", "content": filler}, + {"role": "user", "content": a}, + {"role": "assistant", "content": b}]}), + // system array → messages + json!({"model": "m", "system": [{"type": "text", "text": filler}, + {"type": "text", "text": a}], + "messages": [{"role": "user", "content": b}]}), + // top-level group → messages + json!({"system": a, "messages": [{"role": "user", "content": b}, + {"role": "user", "content": filler}]}), + // messages → tools + json!({"messages": [{"role": "user", "content": filler}, + {"role": "user", "content": a}], + "tools": [{"name": "t", "description": b, "input_schema": {}}]}), + // tool_use input → next block in another message + json!({"messages": [ + {"role": "assistant", "content": [{"type": "tool_use", "id": "1", + "name": "sh", "input": {"argv": ["x", a]}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "1", + "content": [{"type": "text", "text": b}]}]}, + {"role": "user", "content": filler}]}), + ]; + for body in bodies { + let text = body.to_string(); + agree( + &text, + rules, + Dialect::Anthropic, + &cache, + &format!("{name} {a:?} | {b:?}"), + ); + let esc = ascii_escaped(&text); + agree( + &esc, + rules, + Dialect::Anthropic, + &cache, + &format!("{name} esc {a:?} | {b:?}"), + ); + } + } + } + } +} + +/// Review: Bedrock and Realtime shapes are not in the multi-turn comparison. +#[test] +fn review_bedrock_and_other_shapes_agree() { + let mut r = Rng(0x0bad_5eed_0086_0001); + let rules = with_customs(all_but(CONTEXTUAL)).with_saved(saved(&[SAVED])); + let cache = ScanCache::for_tests(); + for round in 0..40 { + let mut msgs = Vec::new(); + for _ in 0..6 { + msgs.push(json!({"role": "user", "content": [{"text": prose(&mut r)}, + {"toolResult": {"toolUseId": "t", "content": [{"text": prose(&mut r)}, + {"json": {"k": fragment(&mut r)}}]}}]})); + msgs.push(json!({"role": "assistant", "content": [ + {"reasoningContent": {"reasoningText": {"text": prose(&mut r), "signature": "s"}}}, + {"toolUse": {"toolUseId": "t", "name": "n", "input": {"cmd": prose(&mut r)}}}]})); + } + let text = json!({"system": [{"text": prose(&mut r)}, {"text": fragment(&mut r)}], + "messages": msgs, + "toolConfig": {"tools": [{"toolSpec": {"name": "n", "description": prose(&mut r)}}]}}) + .to_string(); + let s = Shape::Generate(Dialect::Bedrock); + for scope in [Scope::Written(s), Scope::Whole] { + assert_eq!( + flow::hits_in(&text, &rules, scope), + flow::hits_in_with(&text, &rules, scope, Some(&cache)), + "bedrock round {round}\n{text}" + ); + } + let emb = json!({"model": "m", "input": [prose(&mut r), prose(&mut r)], + "encoding_format": "float", "user": fragment(&mut r), + "x": [{"y": fragment(&mut r)}, {"y": "lorem ipsum ".repeat(20)}], + "pad": "lorem ipsum ".repeat(100)}) + .to_string(); + for s in [ + Shape::Embeddings, + Shape::Completion, + Shape::GeminiEmbed, + Shape::RealtimeEvent, + ] { + assert_eq!( + flow::hits_in(&emb, &rules, Scope::Written(s)), + flow::hits_in_with(&emb, &rules, Scope::Written(s), Some(&cache)), + "{s:?} round {round}\n{emb}" + ); + } + } +} + +// ---------------------------------------------------------------- review 86b + +/// 顶层一项的冒号后面就到头了(客户端截断、插件写坏):切段要说「切不开」,不能越界 +#[test] +fn review86b_split_survives_a_body_cut_after_a_colon() { + for text in ["{\"a\":", "{\"a\":1,\"b\":", "{\"a\":1,\"b\": "] { + let got = std::panic::catch_unwind(|| split(text)); + assert!(matches!(got, Ok(None)), "{text:?}"); + } +} + +/// Chat 请求体:几条各自成段的长消息,后面跟着 `"stream":` 和 `tail` +fn review86b_chat(tail: &str) -> String { + let msgs: Vec = (0..6) + .map(|i| { + json!({"role": "user", "content": format!("turn {i} {}", "lorem ipsum ".repeat(400))}) + .to_string() + }) + .collect(); + format!( + "{{\"model\":\"m\",\"messages\":[{}],\"stream\":{tail}", + msgs.join(",") + ) +} + +/// 等后台存完,最多等两秒(后台线程死了的话 `settle` 会一直等下去) +fn review86b_settle(cache: &ScanCache) -> bool { + for _ in 0..2_000 { + if cache.shared.pending.load(Ordering::Acquire) == 0 { + return true; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + false +} + +/// 一个截断的请求体(没有命中、头一回见)交给后台补存:后台线程切段时越界 panic,之后这份缓存 +/// 再也存不进任何东西 +#[test] +fn review86b_a_cut_body_does_not_kill_the_background_store() { + let cache = ScanCache::for_tests(); + let rules = RuleSet::defaults(); + let scope = Scope::Written(Shape::Generate(Dialect::Chat)); + let cut = review86b_chat(""); + assert_eq!( + flow::hits_in(&cut, &rules, scope), + flow::hits_in_with(&cut, &rules, scope, Some(&cache)) + ); + let _ = review86b_settle(&cache); + let good = review86b_chat("true}"); + let _ = flow::hits_in_with(&good, &rules, scope, Some(&cache)); + assert!( + review86b_settle(&cache), + "后台线程没有收工:pending 一直不归零" + ); + assert!(cache.len() > 0, "截断的请求体之后一段都没存"); +} + +/// 存过的对话、同一个开头,这一轮截断在顶层的冒号后面:记号对得上,前台切段越界 panic +#[test] +fn review86b_a_cut_turn_of_a_stored_conversation_does_not_panic() { + let cache = ScanCache::for_tests(); + let rules = RuleSet::defaults(); + let scope = Scope::Written(Shape::Generate(Dialect::Chat)); + let good = review86b_chat("true}"); + let _ = flow::hits_in_with(&good, &rules, scope, Some(&cache)); + assert!(review86b_settle(&cache)); + assert!(cache.len() > 0); + let cut = review86b_chat(" "); + let got = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + flow::hits_in_with(&cut, &rules, scope, Some(&cache)) + })); + assert_eq!(got.ok(), Some(flow::hits_in(&cut, &rules, scope))); +} diff --git a/crates/tw-guard/src/redact/exception_tests.rs b/crates/tw-guard/src/redact/exception_tests.rs new file mode 100644 index 00000000..eb51bfde --- /dev/null +++ b/crates/tw-guard/src/redact/exception_tests.rs @@ -0,0 +1,175 @@ +//! 0.86 的审查(扫描缓存 + 例外值):报出去的哈希列为例外之后一定管用。 +//! +//! 「哈希能不能从打码后的样子穷举回去」那一条不在这里:安全日志不再存哈希,只在内存里留最近 +//! 的一批(见 `tw_store::Recorder`),配置里的例外表照旧是不加盐的 SHA-256(见配置手册的 +//! 本机安全边界一节)。 + +use crate::policy::{Mode, RedactException, RedactPolicy}; +use crate::redact::flow::{self, Shape}; +use crate::redact::rules::{BUILTINS, RuleSet}; + +const SHAPE: Shape = Shape::Generate(tw_dialect::ir::Dialect::Anthropic); + +fn body(text: &str) -> String { + serde_json::json!({"messages": [{"role": "user", "content": text}]}).to_string() +} + +fn except(mut p: RedactPolicy, hashes: &[String]) -> RedactPolicy { + p.exceptions = hashes + .iter() + .map(|h| RedactException { + sha256: h.clone(), + masked: String::new(), + rule: String::new(), + added_at: 0, + }) + .collect(); + p.check().unwrap(); + p +} + +/// Every hash the security log records for a finding must be one that, once listed as an +/// exception, actually stops that value from being reported. Otherwise "List as exception" +/// writes an exception that never matches, and the log then shows the row as `excepted`. +fn assert_logged_hashes_take_effect(p: &RedactPolicy, text: &str) { + let b = body(text); + let found = flow::find(Mode::Enforce, &p.rules().unwrap(), b.as_bytes(), SHAPE); + assert!(!found.is_empty(), "nothing found in {text:?}"); + for f in &found { + let Some(h) = &f.sha256 else { continue }; + let ex = except(p.clone(), std::slice::from_ref(h)); + let again = flow::find(Mode::Enforce, &ex.rules().unwrap(), b.as_bytes(), SHAPE); + assert!( + !again.iter().any(|g| g.sha256.as_ref() == Some(h)), + "excepting the logged hash of {:?} ({}) has no effect: {again:?}", + f.masked, + f.rule.id() + ); + } +} + +/// Two rules that overlap partially are merged into one hit (`united`); the finding's hash is +/// the hash of the merged span, which no single rule ever matched. +#[test] +fn excepting_a_merged_finding_takes_effect() { + let p: RedactPolicy = serde_yaml_ng::from_str( + "custom:\n - { name: a, pattern: 'PRJ-[0-9]+', match: regex }\n - { name: b, pattern: '[0-9]+-INTERNAL', match: regex }\n", + ) + .unwrap(); + assert_logged_hashes_take_effect(&p, "ticket PRJ-123456-INTERNAL today"); + // 合成的那一处没有哪条规则认过:不给哈希,界面上也就没有「列为例外」 + let b = body("ticket PRJ-123456-INTERNAL today"); + let found = flow::find(Mode::Enforce, &p.rules().unwrap(), b.as_bytes(), SHAPE); + assert_eq!(found.len(), 1, "{found:?}"); + assert_eq!(found[0].sha256, None); + // 单独一条规则认出来的照样有 + let b = body("ticket PRJ-123456 today"); + let found = flow::find(Mode::Enforce, &p.rules().unwrap(), b.as_bytes(), SHAPE); + assert!(found[0].sha256.is_some(), "{found:?}"); +} + +/// A record-only rule's hit that overlaps a placeholder rule is cut into fragments +/// (`outside_of`); the finding's hash is the hash of a fragment. +#[test] +fn excepting_a_record_only_fragment_takes_effect() { + let p: RedactPolicy = serde_yaml_ng::from_str( + "enable: [email]\nactions: { email: record }\ncustom:\n - { name: who, pattern: 'alice.ops' }\n", + ) + .unwrap(); + assert_logged_hashes_take_effect(&p, "mail alice.ops@corp.example.com now"); +} + +/// Exploratory: built-in rules alone, realistic texts. +#[test] +fn excepting_builtin_findings_takes_effect() { + let ids: Vec<&str> = BUILTINS.iter().map(|b| b.id).collect(); + let _ = RuleSet::only(&ids); + let p: RedactPolicy = serde_yaml_ng::from_str(&format!( + "enable: [{}]\n", + ids.iter() + .filter(|i| !crate::redact::cache::CONTEXTUAL.contains(i)) + .copied() + .collect::>() + .join(", ") + )) + .unwrap(); + let mut bad = Vec::new(); + for t in [ + "postgres://app:hunter2xyz9@db.internal:5432/orders", + "https://u:p4ssw0rdXYZ@example.com/x", + "mysql -u root -pS3cretPass99@corp.example.com db", + "Authorization: Bearer sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", + "API_KEY=sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", + "password: alice@corp.example.com", + "https://hooks.example.com/cb?token=alice@corp.example.com&x=1", + ] { + let r = std::panic::catch_unwind(|| assert_logged_hashes_take_effect(&p, t)); + if r.is_err() { + bad.push(t); + } + } + assert!(bad.is_empty(), "dead exceptions for: {bad:?}"); +} + +/// Built-in rule pairs side by side, with every separator: no logged hash is a dead exception. +#[test] +fn builtin_overlaps_leave_no_dead_hash() { + let ids: Vec<&str> = BUILTINS + .iter() + .map(|b| b.id) + .filter(|i| !crate::redact::cache::CONTEXTUAL.contains(i)) + .collect(); + let p: RedactPolicy = + serde_yaml_ng::from_str(&format!("enable: [{}]\n", ids.join(", "))).unwrap(); + let pool = [ + "sk-ant-api03-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789", + "ghp_aBcDeFgHiJkLmNoPqRsTuVwXyZ0123456789", + "AKIAQRSTUVWXYZ234567", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dBjftJeZ4CVPmB92K27uhbUJU1p1r_wW1gFWFOEjXk", + "postgres://app:hunter2xyz9@db.internal:5432/orders", + "alice@corp.example.com", + "6222021234567890", + "13800138000", + "API_TOKEN=abc123XYZ789qq", + "password: hunter2hunter2", + "Authorization: Bearer abcDEF123456ghiJKL789mno", + "https://hooks.example.com/cb?token=abcDEF123456ghi&x=1", + "mysql -u root -pS3cretPass99 db", + "39.9042, 116.4074", + "DE89370400440532013000", + "10.0.0.5", + "00:1A:2B:3C:4D:5E", + "110101199003071234", + "SSN 123-45-6789", + "+1 415 555 2671", + "1HGCM82633A004352", + ]; + let mut bad = Vec::new(); + for a in pool { + for b in pool { + for sep in ["", ":", "@", "/", "=", "-", "."] { + let t = format!("x {a}{sep}{b} y"); + let body = body(&t); + let rs = p.rules().unwrap(); + let found = flow::find(Mode::Enforce, &rs, body.as_bytes(), SHAPE); + for f in &found { + let Some(h) = &f.sha256 else { continue }; + let ex = except(p.clone(), std::slice::from_ref(h)); + let again = + flow::find(Mode::Enforce, &ex.rules().unwrap(), body.as_bytes(), SHAPE); + if again.iter().any(|g| g.sha256.as_ref() == Some(h)) { + bad.push(format!("{t:?} -> {} {}", f.rule.id(), f.masked)); + } + } + } + } + } + bad.sort(); + bad.dedup(); + assert!( + bad.is_empty(), + "{} dead:\n{}", + bad.len(), + bad.iter().take(40).cloned().collect::>().join("\n") + ); +} diff --git a/crates/tw-guard/src/redact/exceptions.rs b/crates/tw-guard/src/redact/exceptions.rs new file mode 100644 index 00000000..c47657f2 --- /dev/null +++ b/crates/tw-guard/src/redact/exceptions.rs @@ -0,0 +1,179 @@ +//! 例外值:某个值反复被误认时,只把这个值列为例外,规则本身照常工作。 +//! +//! **例外的是「这个值」,不是「这条规则下的这个值」**:列入之后,它在哪条规则下都不换、 +//! 不记。存的是值的 SHA-256(小写十六进制),不存原值 —— 例外表随配置导出、同步、备份, +//! 原值写进去就是把要防的东西抄了一份带走。 +//! +//! 值按**解开转义之后的样子**算哈希(和账本记的原值是同一个样子,见 +//! [`crate::redact::replace::Ledger`]):请求体里写着 `\n` 的、插件那一层在正文里看到的、 +//! 用户粘贴进来的,是同一个值、同一个哈希。 +//! +//! **保存的凭据不能列为例外**(规则 `thinkwatch-saved-secret`):那是用户自己存进 ThinkWatch +//! 的密钥,本就该换。它的命中不查例外表;配置里写着这条规则的例外读得进来、但不生效 +//! ([`crate::policy::RedactPolicy::exception_set`])。 + +use std::collections::HashSet; + +use sha2::{Digest, Sha256}; + +use super::rules::Rule; + +/// 例外最多几个。界面上满了「添加例外」置灰;配置里写多了是配置错误 +pub const EXCEPTIONS_MAX: usize = 500; + +/// 保存的凭据那条规则的 id:它的命中不能列为例外 +pub const SAVED_RULE: &str = "thinkwatch-saved-secret"; + +/// 一个值的 SHA-256,32 个字节 +pub type Digest32 = [u8; 32]; + +/// 一个值(解开转义之后的样子)的 SHA-256,小写十六进制 +pub fn sha256_hex(value: &str) -> String { + hex(&Sha256::digest(value.as_bytes()).into()) +} + +/// 32 个字节写成 64 个小写十六进制字符 +pub fn hex(d: &Digest32) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut s = String::with_capacity(64); + for b in d { + s.push(HEX[(b >> 4) as usize] as char); + s.push(HEX[(b & 15) as usize] as char); + } + s +} + +/// 反过来。**只认 64 个小写十六进制字符**:大写的、短一截的都不是我们写出去的样子, +/// 认下来的话两种写法的同一个哈希在去重时就成了两个 +pub fn parse_hex(s: &str) -> Option { + let b = s.as_bytes(); + if b.len() != 64 { + return None; + } + let nib = |c: u8| match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + _ => None, + }; + let mut out = [0u8; 32]; + for (i, o) in out.iter_mut().enumerate() { + *o = (nib(b[2 * i])? << 4) | nib(b[2 * i + 1])?; + } + Some(out) +} + +/// 这条规则的命中能不能列为例外:保存的凭据不能 +pub fn exceptable(rule: &Rule) -> bool { + !matches!(rule, Rule::Builtin(id) if *id == SAVED_RULE) +} + +/// 编好的例外表。**跟着配置一起换**:和规则集一起建一次([`crate::redact::rules::RuleSet::with_exceptions`]) +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Exceptions { + set: HashSet, + /// 排好序的全部哈希再算一次 SHA-256:例外表变了它就变(见 [`Self::fingerprint`]) + fingerprint: Digest32, +} + +impl Exceptions { + /// 一个都没有是 `None`:调用方据此整个跳过,**例外表为空时一个哈希都不算** + pub fn new(hashes: impl IntoIterator) -> Option { + let set: HashSet = hashes.into_iter().collect(); + if set.is_empty() { + return None; + } + let mut sorted: Vec<&Digest32> = set.iter().collect(); + sorted.sort_unstable(); + let mut h = Sha256::new(); + for d in sorted { + h.update(d); + } + Some(Self { + set, + fingerprint: h.finalize().into(), + }) + } + + pub fn len(&self) -> usize { + self.set.len() + } + + pub fn is_empty(&self) -> bool { + self.set.is_empty() + } + + /// 例外表的指纹:全部哈希排好序之后的 SHA-256。**同一组例外,不论写的先后,指纹一样**。 + /// 按「规则集版本」缓存扫描结果的地方把它算进版本里:改了例外表,缓存整体作废 + pub fn fingerprint(&self) -> Digest32 { + self.fingerprint + } + + /// 这个哈希在不在表里 + pub fn contains(&self, d: &Digest32) -> bool { + self.set.contains(d) + } + + /// 十六进制写的哈希在不在表里。写得不对的不在 + pub fn contains_hex(&self, s: &str) -> bool { + parse_hex(s).is_some_and(|d| self.contains(&d)) + } + + /// 这个值(解开转义之后的样子)是不是例外 + pub fn excepts(&self, value: &str) -> bool { + self.contains(&Sha256::digest(value.as_bytes()).into()) + } + + /// 这个值是不是例外,`raw` 是从 JSON 原文里截下来的一段:先解开转义(和账本一样,见 + /// [`crate::redact::replace::Ledger`]) + pub(crate) fn excepts_json(&self, raw: &str) -> bool { + if raw.contains('\\') { + self.excepts(&crate::locate::unescape(raw)) + } else { + self.excepts(raw) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_hash_is_lowercase_hex_and_reads_back() { + let h = sha256_hex("test"); + assert_eq!( + h, + "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + ); + assert_eq!(parse_hex(&h).map(|d| hex(&d)), Some(h.clone())); + assert_eq!( + parse_hex(&h.to_uppercase()), + None, + "大写的不是我们写出去的样子" + ); + assert_eq!(parse_hex(&h[..63]), None); + assert_eq!(parse_hex(""), None); + } + + #[test] + fn the_fingerprint_follows_the_set_not_the_order() { + let a = parse_hex(&sha256_hex("a")).unwrap(); + let b = parse_hex(&sha256_hex("b")).unwrap(); + let ab = Exceptions::new([a, b]).unwrap(); + let ba = Exceptions::new([b, a]).unwrap(); + assert_eq!(ab.fingerprint(), ba.fingerprint()); + assert_ne!( + ab.fingerprint(), + Exceptions::new([a]).unwrap().fingerprint() + ); + assert!(Exceptions::new([]).is_none(), "空表就是没有"); + } + + #[test] + fn a_value_from_json_is_matched_unescaped() { + let ex = Exceptions::new([parse_hex(&sha256_hex("a\nb")).unwrap()]).unwrap(); + assert!(ex.excepts("a\nb")); + assert!(ex.excepts_json("a\\nb")); + assert!(!ex.excepts("a\\nb"), "正文里的反斜杠不当成转义"); + } +} diff --git a/crates/tw-guard/src/redact/flow.rs b/crates/tw-guard/src/redact/flow.rs index 4cb8ad80..504cd5b7 100644 --- a/crates/tw-guard/src/redact/flow.rs +++ b/crates/tw-guard/src/redact/flow.rs @@ -36,14 +36,22 @@ //! 替换档下 [`look`] 按客户端原文里出现的先后给找到的值编好号,每一跳都接着这本账换。 //! 各跳各起一本账的话,转换过格式、字段换了顺序的那一跳,同一把密钥可能是 2 号,而 //! 上一跳、存下来的那份请求里它是 1 号 —— 请求和回答对不上号。 +//! +//! # 没变的历史不再扫 +//! +//! 带 `_with` 的几个入口多收一个扫描缓存([`ScanCache`],网关运行时一份):客户端每一轮都把 +//! 整段历史发上来,没变的消息取上一轮找出来的命中,只找新增的。**找出来的和不带缓存时一模 +//! 一样**(缓存替掉的是「按规则扫一遍、去掉不算的」那几步,见 [`kept`];合并照旧在整个请求 +//! 上做),所以编号、报法、换法都不变。不带缓存的入口(`None`)就是原来的样子。 use std::ops::Range; pub use tw_dialect::written::Shape; use crate::policy::Mode; +pub use crate::redact::cache::ScanCache; use crate::redact::replace::{Ledger, Scheme}; -use crate::redact::rules::{Finding, Hit, RuleSet}; +use crate::redact::rules::{Exact, Finding, Hit, RuleSet}; /// 在一份文字的哪些地方找。 #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -74,15 +82,73 @@ pub fn hits(text: &str, rules: &RuleSet) -> Vec { /// **先在整段上找、再看落在哪儿**:大多数请求一处都不命中,正文和载荷在哪儿等有了命中 /// 再找 —— 认正文要把请求体解析一遍。 pub fn hits_in(text: &str, rules: &RuleSet, scope: Scope) -> Vec { - let mut hits = crate::redact::rules::scan_unresolved(text, rules); + hits_in_with(text, rules, scope, None) +} + +/// [`hits_in`],没变的段取 `cache` 里上次找的(见模块说明)。找出来的一模一样 +pub fn hits_in_with( + text: &str, + rules: &RuleSet, + scope: Scope, + cache: Option<&ScanCache>, +) -> Vec { + hits_exact_with(text, rules, scope, cache).0 +} + +/// [`hits_in_with`],连同合并之前每条规则原样认出来的那些段([`Exact`]):报记录时据此判断 +/// 一个值能不能列为例外(见 [`crate::redact::rules::Finding::sha256`],[`reported_exact`]) +pub fn hits_exact_with( + text: &str, + rules: &RuleSet, + scope: Scope, + cache: Option<&ScanCache>, +) -> (Vec, Exact) { + use crate::redact::cache::Kept; + // 整段做:认没认正文、认不认得出记下来,交给缓存在后台存的时候要 + let whole = |written: &mut Option| { + kept( + text, + crate::redact::rules::scan_unresolved(text, rules), + match scope { + Scope::Written(shape) => Some(move || { + let w = tw_dialect::written::ranges(shape, text); + *written = Some(w.is_some()); + w + }), + Scope::Whole => None, + }, + ) + }; + let hits = match cache.map(|c| (c, c.kept(text, rules, scope))) { + Some((_, Kept::Hits(hits))) => hits, + Some((c, Kept::Cold(stamp))) => { + let mut written = None; + let hits = whole(&mut written); + c.fill(text, rules, stamp, scope, &hits, written); + hits + } + Some((_, Kept::Whole)) | None => whole(&mut None), + }; + let exact = Exact::of(&hits); + (crate::redact::rules::resolve(hits, rules), exact) +} + +/// [`hits_in`] 里合并之前的那几步:在 `text` 上按规则找到的 `hits` 去掉不算的 —— 压在占位符上 +/// 的、`written` 交出的正文区间外面的(`None` 是整段都算,交出 `None` 是请求体不是 JSON,一处都 +/// 不算)、键和 base64 载荷上的。每一步只留不删、各看各的,所以一段一段地做(扫描缓存, +/// [`ScanCache`])和整段一起做,留下的是同一批 +pub(crate) fn kept(text: &str, mut hits: Vec, written: Option) -> Vec +where + W: FnOnce() -> Option>>, +{ if hits.is_empty() { return hits; } off_placeholders(text, &mut hits); if !hits.is_empty() - && let Scope::Written(shape) = scope + && let Some(written) = written { - match tw_dialect::written::ranges(shape, text) { + match written() { Some(written) => within(&mut hits, &written), None => hits.clear(), } @@ -90,7 +156,39 @@ pub fn hits_in(text: &str, rules: &RuleSet, scope: Scope) -> Vec { if !hits.is_empty() { outside(&mut hits, untouched(text)); } - crate::redact::rules::resolve(hits, rules) + hits +} + +/// 值是例外([`crate::redact::exceptions`])、因此 [`hits_in`] 不报的那些命中:没有例外表时 +/// 它们是会报的。「测试…」用它把例外的值标出来(「例外,不处理」),而不是让它像没认出来 +/// 一样消失。 +/// +/// 按起点排好;同一处几条规则都认得的只留一条(规则照最先找到的)。**不合并重叠的**: +/// 它们不换,合起来也说不出是哪个值。不走扫描缓存:只有「测试…」用它 +pub fn excepted_in(text: &str, rules: &RuleSet, scope: Scope) -> Vec { + let Some(ex) = rules.exceptions() else { + return Vec::new(); + }; + let mut hits = crate::redact::rules::scan_unresolved(text, &rules.without_exceptions()); + hits.retain(|h| { + crate::redact::exceptions::exceptable(&h.rule) && ex.excepts_json(&text[h.bytes.clone()]) + }); + let mut hits = kept( + text, + hits, + match scope { + Scope::Written(shape) => Some(move || tw_dialect::written::ranges(shape, text)), + Scope::Whole => None, + }, + ); + hits.sort_by_key(|h| (h.bytes.start, h.bytes.end)); + hits.dedup_by(|a, b| a.bytes == b.bytes); + hits +} + +/// [`excepted_in`] 一段纯文本,和 [`hits_plain`] 同一个换算。 +pub fn excepted_plain(text: &str, rules: &RuleSet) -> Vec { + crate::redact::rules::plain_with(text, |encoded| excepted_in(encoded, rules, Scope::Whole)) } /// 去掉压在我们自己的占位符上的命中(见 [`hits`])。在解码过的正文上找的调用方(桌面版的 @@ -152,7 +250,8 @@ pub fn find(mode: Mode, rules: &RuleSet, body: &[u8], shape: Shape) -> Vec Vec Vec { + reported_with(text, rules, found, scope, None) +} + +/// [`reported`],另找的那一遍也走 `cache`(只记录的那几条规则是另一套,指纹不同,各存各的)。 +/// 不知道 `found` 里哪些段是原样认出来的:记录不带哈希([`crate::redact::rules::Finding::sha256`]), +/// 要带的用 [`reported_exact`] +pub fn reported_with( + text: &str, + rules: &RuleSet, + found: &[Hit], + scope: Scope, + cache: Option<&ScanCache>, +) -> Vec { + reported_as(text, rules, found, None, scope, cache) +} + +/// [`reported_with`],`exact` 是 [`hits_exact_with`] 和 `found` 一起交回来的:原样认出来的值 +/// 带上哈希,「列为例外」照它加 +pub fn reported_exact( + text: &str, + rules: &RuleSet, + found: &[Hit], + exact: Exact, + scope: Scope, + cache: Option<&ScanCache>, +) -> Vec { + reported_as(text, rules, found, Some(exact), scope, cache) +} + +fn reported_as( + text: &str, + rules: &RuleSet, + found: &[Hit], + exact: Option, + scope: Scope, + cache: Option<&ScanCache>, +) -> Vec { + let findings = |hits: &[Hit], exact: &Option| match exact { + Some(e) => crate::redact::rules::findings_exact(text, hits, rules, e), + None => crate::redact::rules::findings_in(text, hits, rules), + }; let Some(rec) = rules.recording_only() else { - return crate::redact::rules::findings_in(text, found, rules); + return findings(found, &exact); }; // 只记录的规则的命中在 `found` 里可能被截成了几截(让开要换的那几处,见 // `crate::redact::rules::scan`):报的是它整个认出来的那一处,另找一遍 @@ -173,9 +313,14 @@ pub fn reported(text: &str, rules: &RuleSet, found: &[Hit], scope: Scope) -> Vec .filter(|h| !rules.records(&h.rule)) .cloned() .collect(); - all.extend(hits_in(text, &rec, scope)); + let (more, more_exact) = hits_exact_with(text, &rec, scope, cache); + all.extend(more); all.sort_by_key(|h| h.bytes.start); - crate::redact::rules::findings_in(text, &all, rules) + let exact = exact.map(|mut e| { + e.extend(more_exact); + e + }); + findings(&all, &exact) } /// 一本新账,让开 `body` 里已经写着的占位符(见 [`Ledger::avoiding`])。 @@ -233,6 +378,18 @@ pub struct Look { /// [`look_from`],把找到的命中一起交回去(见 [`Look::hits`])。 pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8], shape: Shape, seed: Ledger) -> Look { + look_hits_with(mode, rules, body, shape, seed, None) +} + +/// [`look_hits`],没变的段取 `cache` 里上次扫的(见模块说明) +pub fn look_hits_with( + mode: Mode, + rules: &RuleSet, + body: &[u8], + shape: Shape, + seed: Ledger, + cache: Option<&ScanCache>, +) -> Look { let empty = || Look { found: Vec::new(), ledger: Ledger::new(Scheme::SECRET), @@ -245,8 +402,8 @@ pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8], shape: Shape, seed: L return empty(); }; let scope = Scope::Written(shape); - let hits = hits_in(text, rules, scope); - let found = reported(text, rules, &hits, scope); + let (hits, exact) = hits_exact_with(text, rules, scope, cache); + let found = reported_exact(text, rules, &hits, exact, scope, cache); let ledger = if !mode.acts() { Ledger::new(Scheme::SECRET) } else { @@ -269,6 +426,18 @@ pub fn replace( body: bytes::Bytes, ledger: &Ledger, shape: Shape, +) -> (bytes::Bytes, Ledger) { + replace_with(mode, rules, body, ledger, shape, None) +} + +/// [`replace`],没变的段取 `cache` 里上次扫的(见模块说明) +pub fn replace_with( + mode: Mode, + rules: &RuleSet, + body: bytes::Bytes, + ledger: &Ledger, + shape: Shape, + cache: Option<&ScanCache>, ) -> (bytes::Bytes, Ledger) { if !mode.acts() || rules.is_empty() { return (body, ledger.clone()); @@ -277,7 +446,7 @@ pub fn replace( let Ok(text) = std::str::from_utf8(&body) else { return (body, ledger.clone()); }; - let found = hits_in(text, rules, Scope::Written(shape)); + let found = hits_in_with(text, rules, Scope::Written(shape), cache); // 只记录的规则命中的不换 let hits = rules.to_replace(&found); if hits.is_empty() { diff --git a/crates/tw-guard/src/redact/mod.rs b/crates/tw-guard/src/redact/mod.rs index bfe1639c..02db0b0a 100644 --- a/crates/tw-guard/src/redact/mod.rs +++ b/crates/tw-guard/src/redact/mod.rs @@ -3,8 +3,12 @@ //! 一个请求体怎么走(先看一遍、编号、每一跳换)在 [`flow`],两个网关共用。 mod anchor; +pub mod cache; mod common; mod engine; +#[cfg(test)] +mod exception_tests; +pub mod exceptions; pub mod flow; mod nest; mod personal; diff --git a/crates/tw-guard/src/redact/rules.rs b/crates/tw-guard/src/redact/rules.rs index 6f8981aa..58f2bffb 100644 --- a/crates/tw-guard/src/redact/rules.rs +++ b/crates/tw-guard/src/redact/rules.rs @@ -37,6 +37,7 @@ use serde::{Deserialize, Serialize}; use super::anchor::{ ALNUM, ANY_LEN, Anchor, Anchors, Bytes, C_DASH, C_DIGIT, C_DOT, C_LETTER, C_UNDER, Chars, DIGIT, }; +use super::exceptions::Exceptions; use super::secrets::ASSIGNMENT_SUFFIXES; pub use super::secrets::saved::{ SAVED_MIN_CHARS, Saved, SavedSecret, SavedSource, SavedSourceKind, @@ -624,6 +625,9 @@ pub struct RuleSet { /// 用户在 ThinkWatch 里保存的凭据(见 [`RuleSet::with_saved`])。规则 `thinkwatch-saved-secret` /// 开着才按它找 saved: Option>, + /// 例外值(见 [`super::exceptions`]):命中的值在表里的,哪条规则的都不换、不报。`None` 是 + /// 一个都没有,一个哈希都不算 + exceptions: Option>, /// 自定义规则编成的预筛([`CustomIndex`]),头一次扫的时候编。改了自定义规则就作废 index: OnceLock>, } @@ -646,6 +650,7 @@ impl RuleSet { custom: Vec::new(), record: HashSet::new(), saved: None, + exceptions: None, index: OnceLock::new(), } } @@ -657,6 +662,7 @@ impl RuleSet { custom: Vec::new(), record: HashSet::new(), saved: None, + exceptions: None, index: OnceLock::new(), } } @@ -672,6 +678,7 @@ impl RuleSet { custom: Vec::new(), record: HashSet::new(), saved: None, + exceptions: None, index: OnceLock::new(), } } @@ -759,6 +766,42 @@ impl RuleSet { self.saved.as_ref()?.source_of(value) } + /// 带上例外值(见 [`super::exceptions`])。`None` 是不带。**跟着配置一起换** + pub fn with_exceptions(mut self, exceptions: Option>) -> Self { + self.exceptions = exceptions; + self + } + + /// 例外值。没有是 `None` + pub fn exceptions(&self) -> Option<&Arc> { + self.exceptions.as_ref() + } + + /// 去掉例外值的一份:「测试…」要看见例外的值被哪条规则认出来过 + pub fn without_exceptions(&self) -> RuleSet { + let mut set = self.clone(); + set.exceptions = None; + set + } + + /// 例外表的指纹([`Exceptions::fingerprint`]),没有例外是 `None`。按规则集缓存扫描结果的 + /// 地方要把它算进规则集的版本:例外表一变,同一段正文找出来的命中就变了 + pub fn exceptions_fingerprint(&self) -> Option { + self.exceptions.as_ref().map(|e| e.fingerprint()) + } + + /// `out` 里去掉值是例外的命中(保存的凭据的不去,见 [`super::exceptions`])。`text` 是 + /// JSON 原文:值先解开转义再算哈希 + fn drop_excepted(&self, text: &str, out: &mut Vec) { + if let Some(ex) = &self.exceptions + && !out.is_empty() + { + out.retain(|h| { + !super::exceptions::exceptable(&h.rule) || !ex.excepts_json(&text[h.bytes.clone()]) + }); + } + } + /// 这条规则只记录:动作是「仅记录」 pub fn recording(mut self, rule: Rule) -> Self { self.record.insert(rule); @@ -790,6 +833,7 @@ impl RuleSet { .collect(), record: self.record.clone(), saved: self.saved.clone(), + exceptions: self.exceptions.clone(), index: OnceLock::new(), }) } @@ -826,6 +870,69 @@ impl RuleSet { pub fn is_empty(&self) -> bool { self.on.is_empty() && self.custom.is_empty() } + + /// 这套规则的指纹(SHA-256):**找出来的命中可能因之而变的每一样**都在里面 —— 开着哪些 + /// 内置规则、每条自定义规则(顺序、写法、怎么认、标签)、哪些只记录、保存的凭据的值。 + /// 两套规则指纹一样,在同一段文字上找出来的就一样。扫描缓存([`super::cache`])拿它做键的 + /// 一部分:改了规则、改了保存的密钥,以前存下的结果整个不作数。 + /// + /// **字段一个一个拆开写、不留 `..`**:往 [`RuleSet`] 里加一样会影响命中的东西(例外值……) + /// 时,这里编译不过,逼着把它也喂进指纹 —— 漏了的话,缓存会把改之前的结果当成改之后的。 + /// 只影响报法、不影响命中的(编好的预筛)写成 `_` + pub fn fingerprint(&self) -> [u8; 32] { + use sha2::{Digest, Sha256}; + let RuleSet { + on, + custom, + record, + saved, + exceptions, + index: _, + } = self; + let mut h = Sha256::new(); + // 每一段先写长度:`ab` + `c` 和 `a` + `bc` 不是同一个指纹 + let mut put = |tag: &[u8], b: &[u8]| { + h.update(tag); + h.update((b.len() as u64).to_le_bytes()); + h.update(b); + }; + // HashSet 的顺序每个实例都不一样,排好再写 + let mut ids: Vec<&str> = on.iter().copied().collect(); + ids.sort_unstable(); + for id in ids { + put(b"on", id.as_bytes()); + } + // 自定义规则的顺序算数:同一处两条都认得时,合成一处后报的是先记下的那条 + for c in custom { + put(b"custom", c.name.as_bytes()); + put(b"pattern", c.pattern.text.as_bytes()); + put( + b"how", + &[ + u8::from(c.pattern.regex), + u8::from(c.pattern.case_sensitive), + ], + ); + put(b"label", c.label.as_deref().unwrap_or("").as_bytes()); + put(b"labeled", &[u8::from(c.label.is_some())]); + } + let mut rec: Vec<(bool, &str)> = record.iter().map(|r| (r.custom(), r.id())).collect(); + rec.sort_unstable(); + for (custom, id) in rec { + put(b"record", &[u8::from(custom)]); + put(b"record-id", id.as_bytes()); + } + if let Some(saved) = saved { + for v in saved.values() { + put(b"saved", v.as_bytes()); + } + } + // 例外值改了,缓存里按旧例外表去掉(或留下)的命中就不对了 + if let Some(ex) = exceptions { + put(b"exceptions", &ex.fingerprint()); + } + h.finalize().into() + } } /// 一个字节是不是 token 的一部分。**token 只由 ASCII 组成**,所以按字节切和按字符切切出来的 @@ -2112,6 +2219,9 @@ pub(crate) fn scan_unresolved(text: &str, set: &RuleSet) -> Vec { // 接着用 let spans = super::engine::scan_tokens(text, set, json, &mut out); custom_hits(text, set, json, spans, &mut out); + // **例外值在合并重叠之前去掉**:先合并的话,一个例外值和旁边另一处命中合成一处之后, + // 那一处就跟着例外一起不换了 + set.drop_excepted(text, &mut out); out } @@ -2244,11 +2354,21 @@ pub fn scan_text(text: &str, set: &RuleSet) -> Vec { custom: Vec::new(), record: set.record.clone(), saved: set.saved.clone(), + exceptions: set.exceptions.clone(), index: OnceLock::new(), }; let mut out = scan_plain(text, &builtins); for c in &set.custom { - for m in c.re.find_iter(text).filter(|m| !m.is_empty()) { + for m in + c.re.find_iter(text) + .filter(|m| !m.is_empty()) + // 解码过的正文:值原样算哈希,不再解一次转义 + .filter(|m| { + !set.exceptions + .as_ref() + .is_some_and(|ex| ex.excepts(m.as_str())) + }) + { out.push(Hit { bytes: m.range(), rule: Rule::Custom(c.name.clone()), @@ -2284,21 +2404,26 @@ pub fn masked(rule: &Rule, value: &str) -> String { // 保存的凭据一个字都不留:报的是它是哪一项([`Finding::source`]) Kind::Saved => String::new(), // 按位置认的多半是口令,没有前缀:留头五个就是留了大半个口令。只留最后四位,短的全打掉 - Kind::InContext => { - let n = value.chars().count(); - if n >= 16 { - let tail: String = value.chars().skip(n - 4).collect(); - format!("…{tail}") - } else { - "…".to_string() - } - } + Kind::InContext => masked_unknown(value), // 助记词的头几个字母就是头一个词 Kind::Crypto if rule.id() == "crypto-mnemonic" => "…".to_string(), _ => mask(value), } } +/// 不知道是什么的一个值打码后的样子:不足 16 个字符的全打掉,长的只留最后四位(`…7f9c`)。 +/// 按位置认的口令、手动加入例外时哪条规则都认不出的值用它 —— 说不清它是什么,就按最该 +/// 小心的那一种(口令)打 +pub fn masked_unknown(value: &str) -> String { + let n = value.chars().count(); + if n >= 16 { + let tail: String = value.chars().skip(n - 4).collect(); + format!("…{tail}") + } else { + "…".to_string() + } +} + /// 邮箱只留本地部分的第一个字和域名:`j…@example.com`。认出是哪一个地址,看这些 /// 就够了;本地部分只有一个字的,那个字也不留 fn mask_email(s: &str) -> String { @@ -2349,6 +2474,34 @@ pub struct Finding { pub count: u64, /// 是哪一项保存的凭据(规则 `thinkwatch-saved-secret`);别的规则是 `None` pub source: Option, + /// 值(解开转义之后的样子)的 SHA-256,小写十六进制:「列为例外」照它加(见 + /// [`super::exceptions`])。**只在把它列为例外真的管用时才有**: + /// + /// - 保存的凭据不能列为例外,是 `None`; + /// - 这个值的每一处都得是**某条规则原原本本认出来的那一段**。两条规则认到互相重叠的两段时 + /// 合成了一处([`resolve`]),只记录的规则让开要换的那几处、截成了几截,这些段没有哪条 + /// 规则认过:例外表里写上它的哈希,什么都挡不住。这时也是 `None`; + /// - 不知道哪些段是原样认出来的([`findings`]、[`findings_in`] 拿到的只是合并之后的命中) + /// 也是 `None`:宁可不给,也不给一个挡不住的。知道的是 [`crate::redact::flow`] 那几个 + /// 报记录的入口。 + pub sha256: Option, +} + +/// 合并重叠之前、每条规则原原本本认出来的那些段(见 [`Finding::sha256`])。 +#[derive(Debug, Clone, Default)] +pub struct Exact(HashSet<(usize, usize)>); + +impl Exact { + /// `hits` 是合并之前的命中([`scan_unresolved`] 之后、[`resolve`] 之前) + pub(crate) fn of(hits: &[Hit]) -> Self { + Self(hits.iter().map(|h| (h.bytes.start, h.bytes.end)).collect()) + } + pub(crate) fn extend(&mut self, other: Exact) { + self.0.extend(other.0); + } + fn has(&self, r: &Range) -> bool { + self.0.contains(&(r.start, r.end)) + } } /// 把命中合并成「哪条规则 × 哪个值 × 几次」。**同一个值只报一次**: @@ -2361,30 +2514,52 @@ pub struct Finding { /// **合并按「规则 × 值」查表**,不在已经合过的里面一条条找:一个请求里贴进来几万个不同 /// 的值时(一份导出的凭据清单),后者是平方级的,几十万个值要几分钟。 pub fn findings(text: &str, hits: &[Hit]) -> Vec { - findings_with(text, hits, None) + findings_with(text, hits, None, None) } /// 同 [`findings`],保存的凭据的命中带上是哪一项([`Finding::source`])。**按「规则 × 值」合并**, /// 所以两项不同的保存的凭据是两条,即使它们打码之后一样(都是空串) pub fn findings_in(text: &str, hits: &[Hit], set: &RuleSet) -> Vec { - findings_with(text, hits, Some(set)) + findings_with(text, hits, Some(set), None) +} + +/// 同 [`findings_in`],`exact` 是合并之前每条规则原样认出来的段:每一处都在里面的值带上哈希 +/// ([`Finding::sha256`]) +pub(crate) fn findings_exact( + text: &str, + hits: &[Hit], + set: &RuleSet, + exact: &Exact, +) -> Vec { + findings_with(text, hits, Some(set), Some(exact)) } -fn findings_with(text: &str, hits: &[Hit], set: Option<&RuleSet>) -> Vec { - let mut out: Vec<(&Rule, &str, u64)> = Vec::new(); +fn findings_with( + text: &str, + hits: &[Hit], + set: Option<&RuleSet>, + exact: Option<&Exact>, +) -> Vec { + // 规则、值、几处、每一处都是原样认出来的 + let mut out: Vec<(&Rule, &str, u64, bool)> = Vec::new(); let mut seen: HashMap<(&Rule, &str), usize> = HashMap::new(); for h in hits { let value = &text[h.bytes.clone()]; + let whole = exact.is_some_and(|e| e.has(&h.bytes)); match seen.entry((&h.rule, value)) { - Entry::Occupied(at) => out[*at.get()].2 += 1, + Entry::Occupied(at) => { + let o = &mut out[*at.get()]; + o.2 += 1; + o.3 &= whole; + } Entry::Vacant(slot) => { slot.insert(out.len()); - out.push((&h.rule, value, 1)); + out.push((&h.rule, value, 1, whole)); } } } out.into_iter() - .map(|(rule, value, count)| Finding { + .map(|(rule, value, count, whole)| Finding { masked: masked(rule, value), source: match (rule, set) { (Rule::Builtin("thinkwatch-saved-secret"), Some(set)) => { @@ -2392,6 +2567,14 @@ fn findings_with(text: &str, hits: &[Hit], set: Option<&RuleSet>) -> Vec None, }, + // 和比对例外表时一样,先解开转义再算(`text` 是 JSON 原文) + sha256: (whole && super::exceptions::exceptable(rule)).then(|| { + if value.contains('\\') { + super::exceptions::sha256_hex(&crate::locate::unescape(value)) + } else { + super::exceptions::sha256_hex(value) + } + }), rule: rule.clone(), count, }) @@ -2817,6 +3000,7 @@ mod tests { .map(|(rule, value, count)| Finding { source: None, masked: masked(&rule, value), + sha256: None, rule, count, }) diff --git a/crates/tw-guard/src/redact/secrets/place.rs b/crates/tw-guard/src/redact/secrets/place.rs index a0494ae7..6257b5ba 100644 --- a/crates/tw-guard/src/redact/secrets/place.rs +++ b/crates/tw-guard/src/redact/secrets/place.rs @@ -472,6 +472,43 @@ fn option_at(b: &[u8], k: usize) -> bool { b[k] == b'-' && k > 0 && (b[k - 1].is_ascii_whitespace() || matches!(b[k - 1], b'"' | b'\'')) } +/// 扫描缓存([`crate::redact::cache`])用:`text` 里有没有一条命令(`curl`、`mysql`……)到结尾 +/// 都没结束。**命令跨得过字符串**(`["curl","-u","用户:口令"]` 是数组里的几项,见 +/// [`command_end`]),在整段请求体里它会接着往后,把后面一段开头的选项也认成它的 —— 这一段 +/// 单拿出来扫就看不到了。 +/// +/// 宁可多报:以命令名**结尾**的一串 token 字符就算(前面粘着转义的尾巴,切 token 时会切掉的, +/// 也算上)。命令名先用一个自动机找(正文里难得出现,几乎整段一跳而过),找到的最后一个往后 +/// 一直没有命令分隔才算 —— 前面的那些后面也隔着同一个分隔 +pub(in crate::redact) fn command_open_at_end(text: &str) -> bool { + static AC: std::sync::OnceLock = std::sync::OnceLock::new(); + let ac = AC.get_or_init(|| { + let words = WORDS + .iter() + .filter(|(_, m)| matches!(m, Mark::Curl | Mark::Mysql)) + .map(|(w, _)| *w); + aho_corasick::AhoCorasick::builder() + .ascii_case_insensitive(true) + .build(words) + .expect("命令名编得成自动机") + }); + let b = text.as_bytes(); + // 一串 token 字符以命令名结尾、后面跟着空白或者引号(`command_word`) + let last = ac + .find_overlapping_iter(text) + .map(|m| m.end()) + .filter(|&e| b.get(e).is_none_or(|&c| !tok_byte(c)) && command_word(b, e)) + .last(); + last.is_some_and(|e| command_end(b, e) == b.len()) +} + +/// 扫描缓存用:从 `from` 起到这条命令结束([`command_end`])之前,有没有一个选项的开头。一条 +/// 没结束的命令([`command_open_at_end`])在后面这一截里认得出的口令,都从一个选项开头 +pub(in crate::redact) fn option_before_command_end(b: &[u8], from: usize) -> (bool, usize) { + let end = command_end(b, from); + ((from.max(1)..end).any(|k| option_at(b, k)), end) +} + /// `curl -u 用户:口令`、`--user 用户:口令`、`-u用户:口令` 里的口令。`e` 是 `curl` 的终点。 /// 只有用户名的(`-u user`、`-u sk_…:`,口令为空)不算 —— 后者的密钥在用户名那一段,归 /// 前缀规则。返回这条命令在哪儿结束 diff --git a/crates/tw-guard/src/redact/secrets/saved.rs b/crates/tw-guard/src/redact/secrets/saved.rs index 483d8d7a..6781c289 100644 --- a/crates/tw-guard/src/redact/secrets/saved.rs +++ b/crates/tw-guard/src/redact/secrets/saved.rs @@ -181,6 +181,12 @@ impl Saved { } } + /// 每一项的真值,按收下的先后。规则集的指纹([`super::super::rules::RuleSet::fingerprint`]) + /// 和扫描缓存用:换了一项保存的凭据,以前扫的结果就不作数了 + pub(crate) fn values(&self) -> impl Iterator { + self.secrets.iter().map(|s| s.value.as_str()) + } + fn item(&self, value: &str) -> Option<&SavedSecret> { self.by_value.get(value).map(|&k| &self.secrets[k]) } diff --git a/crates/tw-guard/src/trial.rs b/crates/tw-guard/src/trial.rs index 91922abc..fe7b0fd8 100644 --- a/crates/tw-guard/src/trial.rs +++ b/crates/tw-guard/src/trial.rs @@ -86,6 +86,10 @@ pub struct TrialHit { /// 出站脱敏认出的是一项保存的凭据(`thinkwatch-saved-secret`):是哪一项。这时 `excerpt` 是空的 #[serde(default, skip_serializing_if = "Option::is_none")] pub source: Option, + /// 出站脱敏:这个值是例外(`security.redact.exceptions`),规则认得出、但不换也不记, + /// 界面上写「例外,不处理」。这时 `action` 是空的,`output` 里也原样留着它 + #[serde(default)] + pub excepted: bool, } /// 试的结果。 @@ -204,9 +208,11 @@ fn redact( let matching = req.matching.unwrap_or_default(); let pattern = redact_match(pattern, matching, req.case_sensitive.unwrap_or(false)) .ok_or(TrialError::BadMatch { matching })?; + // 例外跨规则:一条还没存的规则也照样不换例外的值 let set = RuleSet::none() .with_rule(TRIAL, pattern, label.as_deref()) - .map_err(|e| TrialError::BadPattern { detail: e.detail })?; + .map_err(|e| TrialError::BadPattern { detail: e.detail })? + .with_exceptions(p.exception_set()); match action.unwrap_or_default() { RedactAction::Record => { set.recording(crate::redact::rules::Rule::Custom(TRIAL.into())) @@ -242,23 +248,32 @@ fn redact( ) .text }); - Ok(TrialResult { - hits: hits + let hit = |h: &crate::redact::rules::Hit, excepted: bool| TrialHit { + rule: h.rule.id().to_string(), + custom: h.rule.custom(), + start: utf16_at(sample, h.bytes.start), + end: utf16_at(sample, h.bytes.end), + excerpt: crate::redact::rules::masked(&h.rule, &sample[h.bytes.clone()]), + action: (!excepted).then(|| { + if rules.records(&h.rule) { + RuleAction::Record + } else { + RuleAction::Placeholder + } + }), + source: rules.saved_source(&sample[h.bytes.clone()]).cloned(), + excepted, + }; + // 例外的值照样标出来:用户要看得见「这个值认出来了,只是例外」,而不是像没认出来一样 + let mut out: Vec = hits.iter().map(|h| hit(h, false)).collect(); + out.extend( + crate::redact::flow::excepted_plain(sample, &rules) .iter() - .map(|h| TrialHit { - rule: h.rule.id().to_string(), - custom: h.rule.custom(), - start: utf16_at(sample, h.bytes.start), - end: utf16_at(sample, h.bytes.end), - excerpt: crate::redact::rules::masked(&h.rule, &sample[h.bytes.clone()]), - action: Some(if rules.records(&h.rule) { - RuleAction::Record - } else { - RuleAction::Placeholder - }), - source: rules.saved_source(&sample[h.bytes.clone()]).cloned(), - }) - .collect(), + .map(|h| hit(h, true)), + ); + out.sort_by_key(|h| (h.start, h.end)); + Ok(TrialResult { + hits: out, output, refused: false, }) @@ -314,6 +329,7 @@ fn tools( RuleAction::Record }), source: None, + excepted: false, }) }) .collect(); @@ -430,6 +446,7 @@ fn content(p: &ContentPolicy, req: &TrialRequest) -> Result TrialResult { .collect(), action: Some(ContentAction::of(rule.action()).into()), source: None, + excepted: false, }) }) .collect(); @@ -933,4 +951,57 @@ mod tests { .unwrap(); assert!(r.hits.is_empty()); } + + /// 例外的值标成「例外」,不算命中:不给动作,发出去的样子里原样留着 + #[test] + fn an_excepted_value_is_flagged_not_hit() { + let key = "sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAA"; + let other = "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; + let mut s = Security::default(); + s.redact.exceptions = vec![crate::policy::RedactException { + sha256: crate::redact::exceptions::sha256_hex(key), + masked: String::new(), + rule: "anthropic-api-key".into(), + added_at: 0, + }]; + let sample = format!("{key} {other}"); + let r = run(Guard::Redact, &s, &req(&sample)).unwrap(); + assert_eq!(r.hits.len(), 2, "{r:?}"); + assert_eq!(r.hits[0].rule, "anthropic-api-key"); + assert!(r.hits[0].excepted && r.hits[0].action.is_none()); + assert!(!r.hits[1].excepted && r.hits[1].action == Some(RuleAction::Placeholder)); + assert_eq!(r.output.unwrap(), format!("{key} <>")); + // 试一条正在写的规则也一样:例外跨规则 + let r = run( + Guard::Redact, + &s, + &TrialRequest { + pattern: Some("sk-ant-api03-".into()), + ..req(key) + }, + ) + .unwrap(); + assert!(r.hits.iter().all(|h| !h.excepted), "认的不是整个值:{r:?}"); + let r = run( + Guard::Redact, + &s, + &TrialRequest { + pattern: Some(key.into()), + ..req(key) + }, + ) + .unwrap(); + assert_eq!(r.hits.len(), 1); + assert!(r.hits[0].excepted && r.output.is_none(), "{r:?}"); + let r = run( + Guard::Redact, + &s, + &TrialRequest { + rule: Some("anthropic-api-key".into()), + ..req(key) + }, + ) + .unwrap(); + assert!(r.hits.len() == 1 && r.hits[0].excepted, "{r:?}"); + } } diff --git a/crates/tw-guard/src/view.rs b/crates/tw-guard/src/view.rs index b32b9f89..6bc0908c 100644 --- a/crates/tw-guard/src/view.rs +++ b/crates/tw-guard/src/view.rs @@ -931,6 +931,7 @@ mod tests { disabled: true, }, ], + exceptions: Vec::new(), }, inspect_tools: ToolPolicy { mode: Mode::Off, @@ -1066,6 +1067,8 @@ mod tests { disabled: !r.enabled, }) .collect(), + // 例外值不在规则视图里,有自己的接口 + exceptions: Vec::new(), }; let (enable, disable) = lists(&view.inspect_tools.rules); let inspect_tools = ToolPolicy { diff --git a/crates/tw-store/src/db.rs b/crates/tw-store/src/db.rs index b30df281..cc8eb1da 100644 --- a/crates/tw-store/src/db.rs +++ b/crates/tw-store/src/db.rs @@ -411,6 +411,8 @@ impl Db { // - `requests_routed`:各条路由、规则命中了多少要的那几项,落库时从路由那一列里取出来 // 存在索引里。数的时候只读索引,不再把一周几 MB 的 JSON 读出来解一遍(见 // `route_hits`) + // - `security_events_rule_hits`:一项防护每条规则最近几天命中几次,只读索引(见 + // `security_rule_hits`)。规则表的组头每次打开都要它 let sql = format!( "CREATE INDEX IF NOT EXISTS requests_client ON requests (client, at_ms) WHERE client > ''; @@ -418,7 +420,9 @@ impl Db { WHERE session IS NOT NULL AND local = 0; CREATE INDEX IF NOT EXISTS requests_routed ON requests (at_ms, {ROUTE}, {RULE}, {REWRITTEN_BY}, {DENIED_BY}, error IS NOT NULL) - WHERE {ROUTED};" + WHERE {ROUTED}; + CREATE INDEX IF NOT EXISTS security_events_rule_hits + ON security_events (guard, at_ms, rule, custom, count);" ); if let Err(e) = self.conn.execute_batch(&sql) { tracing::warn!( @@ -1128,8 +1132,8 @@ impl Db { .collect()) } - /// 记一条安全日志。 - pub fn insert_security_event(&self, e: &SecurityEvent) -> Result<(), DbError> { + /// 记一条安全日志,返回它的号。 + pub fn insert_security_event(&self, e: &SecurityEvent) -> Result { self.conn.execute( "INSERT INTO security_events (at_ms, request_id, guard, rule, custom, action, provider, client, tool, excerpt, count, @@ -1157,7 +1161,53 @@ impl Db { .and_then(|s| serde_json::to_string(s).ok()), ], )?; - Ok(()) + Ok(self.conn.last_insert_rowid()) + } + + /// 一项防护从 `since_ms` 起每条规则命中了几次:各条记录的 `count` 加起来,不论做了什么 + /// (只记录的、换掉的、切断的都算)。按规则排,没命中的规则不在里面。 + /// + /// **按记录的 `count` 加,不是数行**:出站脱敏一行是「一个请求里的一个值」,同一个值在一个 + /// 请求里出现三次是一行、`count` 3;内容过滤一行是一条规则在一个请求里的几处 + pub fn security_rule_hits( + &self, + guard: &str, + since_ms: i64, + ) -> Result, DbError> { + let mut st = self.conn.prepare( + "SELECT rule, custom, SUM(count) FROM security_events + WHERE guard = ?1 AND at_ms >= ?2 + GROUP BY rule, custom + ORDER BY rule, custom", + )?; + let rows = st.query_map(params![guard, since_ms], |r| { + Ok(tw_api::SecurityRuleHits { + rule: r.get(0)?, + custom: r.get::<_, i64>(1)? != 0, + count: r.get(2)?, + }) + })?; + Ok(rows.collect::>()?) + } + + /// 安全日志的第 `id` 条拿去「列为例外」时要的那几样(哈希不在库里,在记录器的内存里,见 + /// [`crate::Recorder::value_hash`])。没有这一条(过了保留期、库重建过)是 `None` + pub fn exception_source(&self, id: i64) -> Result, DbError> { + let row = self + .conn + .query_row( + "SELECT guard, rule, excerpt FROM security_events WHERE id = ?1", + [id], + |r| { + Ok(ExceptionSource { + guard: slug_col(r, 0, tw_api::Guard::from_slug)?, + rule: r.get(1)?, + masked: r.get(2)?, + }) + }, + ) + .optional()?; + Ok(row) } /// 安全日志的一页,按时间倒序,连同这一段一共几条、各做了什么。 @@ -2063,6 +2113,15 @@ pub struct SecurityEvent { pub source: Option, } +/// 安全日志里的一条拿去「列为例外」时要的那几样([`Db::exception_source`])。 +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExceptionSource { + pub guard: tw_api::Guard, + pub rule: String, + /// 打码后的值,原样进例外表 + pub masked: String, +} + /// 读安全日志时的那段 SELECT。**上游、密钥、模型、会话优先取请求那一行的。** /// /// 发给上游的模型名也取请求那一行的,**只在它可能发到了上游时**(`tw_reached`,和密钥的 @@ -2149,6 +2208,10 @@ fn security_view(r: &rusqlite::Row) -> rusqlite::Result, + /// 最近几行出站脱敏的安全日志,各自命中的值的哈希(见 [`Hashes`]) + hashes: Hashes, +} + +/// 最近 [`Hashes::CAP`] 行出站脱敏的安全日志:日志的号 → 那个值的 SHA-256(不能列为例外的是 +/// `None`)。「列为例外」和日志上的「例外」标签靠它。 +/// +/// **只在内存里,不落盘,重启就没了。**库里不存任何值的哈希:一个不加盐的 SHA-256 对手机号、 +/// 身份证号这种短的值,配上打码后留下的那几位,是能穷举回去的 —— 存进请求库等于把号码本身 +/// 存了一份,而请求库会被导出、拷走、附在问题报告里。只留最近的一批,是因为要列为例外的 +/// 多半就是刚看到的那几条;更早的没有按钮,用户照样可以粘贴那个值手动加入。 +#[derive(Debug, Default)] +struct Hashes { + by_id: HashMap>, + order: std::collections::VecDeque, +} + +impl Hashes { + /// 留多少行。一行几十个字节,两万行一两 MB + const CAP: usize = 20_000; + + fn put(&mut self, id: i64, hash: Option) { + if self.by_id.insert(id, hash).is_none() { + self.order.push_back(id); + } + while self.order.len() > Self::CAP { + if let Some(old) = self.order.pop_front() { + self.by_id.remove(&old); + } + } + } + + fn get(&self, id: i64) -> Option> { + self.by_id.get(&id).map(Option::as_deref) + } } /// 记下的一行里,密钥和上游的用量上限要的那几样(见 `tw_gateway::key_limits`、 @@ -165,6 +200,7 @@ impl Recorder { bus: None, transcripts: Default::default(), settled: None, + hashes: Hashes::default(), } } @@ -180,13 +216,26 @@ impl Recorder { self } - /// 记一条安全日志。**写不进去只记一行日志** —— 观测挂了,代理照跑。 - fn record_security(&self, e: crate::db::SecurityEvent) { - if let Err(err) = self.db.insert_security_event(&e) { - tracing::debug!("the security record could not be written: {err}"); + /// 记一条安全日志,返回它的号。**写不进去只记一行日志** —— 观测挂了,代理照跑。 + fn record_security(&self, e: crate::db::SecurityEvent) -> Option { + match self.db.insert_security_event(&e) { + Ok(id) => Some(id), + Err(err) => { + tracing::debug!("the security record could not be written: {err}"); + None + } } } + /// 安全日志第 `id` 条(出站脱敏的)命中的值的 SHA-256,「列为例外」照它加(见 [`Hashes`])。 + /// + /// - `None`:这一条不在内存里 —— 不是出站脱敏的、core 重启之前记的、或者早已被后来的挤掉了; + /// - `Some(None)`:在内存里,但不能列为例外(保存的凭据、几处重叠合成的一段); + /// - `Some(Some(哈希))`:能列为例外。 + pub fn value_hash(&self, id: i64) -> Option> { + self.hashes.get(id) + } + pub fn db(&self) -> &Db { &self.db } @@ -413,7 +462,7 @@ impl Recorder { let client = self.inflight.get(id).map(|p| p.client.clone()); let session = self.inflight.get(id).and_then(|p| p.session.clone()); for it in items { - self.record_security(crate::db::SecurityEvent { + let row = self.record_security(crate::db::SecurityEvent { at_ms: *at_ms as i64, request_id: *id as i64, guard: tw_api::Guard::Redact, @@ -434,6 +483,9 @@ impl Recorder { detail: it.detail.clone(), source: it.source.clone(), }); + if let Some(row) = row { + self.hashes.put(row, it.value_sha256.clone()); + } } } // 内容过滤:一条规则一条,`count` 是几处(码位规则是几个字符)。回答里的在回答结束时报 @@ -2092,6 +2144,23 @@ mod pricing_report_tests { mod security_tests { use super::tests::{finished, rec, started}; + /// 一个值的哈希(随便哪个,这里测的是它记下、读回、不出进程) + const HASH: &str = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"; + + /// 内存里只留最近的那些:多出来的从最早的挤掉 + #[test] + fn only_the_latest_value_hashes_are_kept() { + let mut h = super::Hashes::default(); + let cap = super::Hashes::CAP as i64; + for id in 1..=cap + 5 { + h.put(id, Some(format!("{id}"))); + } + assert_eq!(h.get(5), None); + assert_eq!(h.get(6), Some(Some("6"))); + assert_eq!(h.get(cap + 5), Some(Some(&*format!("{}", cap + 5)))); + assert_eq!(h.by_id.len(), super::Hashes::CAP); + } + fn secrets(replaced: bool) -> tw_api::Event { tw_api::Event::SecretsFound { id: 1, @@ -2107,6 +2176,7 @@ mod security_tests { replaced, detail: Default::default(), source: None, + value_sha256: Some(HASH.into()), }, tw_api::SecretItem { rule: "公司令牌".into(), @@ -2117,6 +2187,7 @@ mod security_tests { replaced, detail: Default::default(), source: None, + value_sha256: None, }, ], at_ms: 10, @@ -2150,6 +2221,33 @@ mod security_tests { // 密钥和模型取自请求那一行 assert_eq!(got[0].client, "claude-code"); assert_eq!(got[0].model, "claude-sonnet-4-5"); + // 值的哈希只在记录器的内存里:库里没有,事件写成 JSON 也不带它 + assert_eq!(r.value_hash(got[1].id), Some(Some(HASH))); + assert_eq!(r.value_hash(got[0].id), Some(None), "不能列为例外的也记着"); + assert_eq!(r.value_hash(999), None); + assert!(!serde_json::to_string(&got).unwrap().contains(HASH)); + assert!( + !serde_json::to_string(&secrets(replaced)) + .unwrap() + .contains(HASH) + ); + let mut st = r + .db() + .conn + .prepare("SELECT * FROM security_events") + .unwrap(); + let n = st.column_count(); + let dump: Vec = st + .query_map([], |row| { + Ok((0..n) + .map(|i| format!("{:?}", row.get_ref(i).unwrap())) + .collect::>() + .join("|")) + }) + .unwrap() + .map(Result::unwrap) + .collect(); + assert!(!dump.join("\n").contains(HASH), "库里有哈希"); } } @@ -2333,6 +2431,7 @@ mod security_detail_tests { replaced: true, detail, source: None, + value_sha256: None, }], at_ms: 10, } diff --git a/docs/config.md b/docs/config.md index 10dc0edf..20d2c4ca 100644 --- a/docs/config.md +++ b/docs/config.md @@ -37,6 +37,17 @@ through the control plane cannot change it ([`listen.control`](#cfg-listen-control)), not to keep it from local programs; upstream and gateway keys are shown as written. +Redaction exceptions +([`security.redact.exceptions`](#cfg-security-redact-exceptions)) are stored +as a plain, unsalted SHA-256 of each value. That is no secret from anyone +who can read the file: a short value such as a phone number can be recovered +from its hash by trying every number. The file already holds upstream keys in +plain text, so the hash adds nothing to what it gives away; but it travels +with exports, sync and backups, so list only values that are not sensitive. +The request database holds no hash of any value found: the hashes behind the +security log's "list as exception" are kept in memory for the most recent +20,000 entries and dropped when core restarts. + Tool-call inspection has a built-in rule for this directory, `thinkwatch-data`. A tool call whose path or command points into one of the default locations above, or into `/var/lib/thinkwatch` or `/etc/thinkwatch` @@ -851,6 +862,7 @@ written text. | `disable` | list of strings | `[]` | Built-in rules to switch off, by id. | | `actions` | map of built-in rule id → `placeholder` \| `record` | `{}` | What a built-in rule does under `enforce`, written only where it differs from the factory setting (`email: record`). `placeholder` sends a placeholder in place of the value and puts the value back where the answer repeats the placeholder; `record` sends the value as it is and records the match. | | `custom` | list of [`security.redact.custom[]`](#cfg-security-redact-custom) | `[]` | Rules of your own: whatever a rule matches is replaced like a credential, or only recorded. | +| `exceptions` | list of [`security.redact.exceptions[]`](#cfg-security-redact-exceptions) | `[]` | Values that are never replaced or recorded, under any rule. Each is kept as a hash, never as the value itself; at most 500. A saved secret cannot be an exception: an entry whose rule is `thinkwatch-saved-secret` is read but has no effect. | @@ -1030,6 +1042,32 @@ of a number, and the first character and the domain of an email address. A custom rule replaces with `<>` unless it names its own placeholder (`label`). +Exceptions: a value that a rule keeps matching but that is not sensitive (a +test fixture, a sample key in documentation) can be listed under +`exceptions`. The rules keep working; only that value is left alone, under +every rule, in both `observe` and `enforce`. Each entry holds the SHA-256 of +the value, never the value itself, so the list can travel with exports, +sync and backups (see the note on the plain SHA-256 under +[where the file is](#where-the-file-is)). The app adds entries from the +security log or from a pasted value; written by hand, `sha256` is the hash of +the value as it appears in the message, with no surrounding quotes or JSON +escapes. Only recent log entries can be listed from the log: the hash of each +value is kept in memory for the last 20,000 entries and not across a restart, +and a value made of overlapping matches joined into one cannot be listed at +all. Older entries show neither the button nor the exception tag; paste the +value instead. + + + + +| Field | Type | Default | Description | +|---|---|---|---| +| `sha256` | string | **required** | SHA-256 of the value, as 64 lowercase hexadecimal characters. Escapes in the request body are decoded first: the hash is of the value as written. | +| `masked` | string | `''` | The value as shown when it was added, masked the same way as in the security log. | +| `rule` | string | `''` | The rule that first recognised the value: a built-in rule id or a custom rule name. Empty for a value added by hand that no rule recognises. | +| `added_at` | integer | `0` | When the value was added, in milliseconds since the Unix epoch. | + + #### `security.inspect_tools` Tool calls a model returns are checked against the rules. Under `enforce`, diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index 1fa18336..4d137298 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -17,6 +17,8 @@ ThinkWatch Core 只读一个文件:`config.yaml`。本文逐项说明其中每 这两项权限就是这份文件在本机上的全部保护:挡得住其他用户,挡不住以同一用户身份运行的程序。这样的程序能读到文件里的每一把密钥,也能用其中的控制通道密钥经控制通道修改配置。出站脱敏不改变这一点:它保护的是请求带出本机的内容,不是磁盘上的文件。显示配置时给控制通道密钥打码,是为了让经控制通道的写入改不了它(见 [`listen.control`](#cfg-listen-control)),而不是对本机程序隐藏它;上游密钥和网关密钥按原样显示。 +脱敏的例外值([`security.redact.exceptions`](#cfg-security-redact-exceptions))以每个值不加盐的 SHA-256 保存。能读到这份文件的人不难还原它:手机号这样的短值,逐个试遍所有号码即可从哈希得出原值。文件里本就以明文保存上游密钥,哈希并没有多泄露什么;但它会随导出、同步和备份带走,因此只把不敏感的值列为例外。请求数据库不保存任何命中值的哈希:安全日志「列为例外」所用的哈希只在内存中保留最近 20,000 条,core 重启后即清空。 + 工具调用审查为这个目录内置了一条规则 `thinkwatch-data`:工具调用的路径或命令指向上述默认位置之一,或服务器上的 `/var/lib/thinkwatch`、`/etc/thinkwatch` 时记录下来,`enforce` 下切断响应。这样模型无法被诱导去读取这些密钥、改写约束它自己的防护。只是提到这个路径(例如在正在编辑的文档里)不算。`THINKWATCH_HOME` 指向别处时,可以在 [`security.inspect_tools`](#cfg-security-inspect_tools) 里为那个路径加一条自定义规则。 没有配置文件时,`twcore serve` 会写入一份初始配置;`twcore init` 也可以按需生成。两者生成的内容如下: @@ -622,6 +624,7 @@ pricing: | `disable` | 字符串列表 | `[]` | 关掉内置规则,按 id。 | | `actions` | 映射: 内置规则 id → `placeholder` \| `record` | `{}` | 内置规则在 `enforce` 下的动作,只写与出厂不同的(`email: record`)。`placeholder` 将值替换为占位符后发送,回答中出现该占位符时还原为原值;`record` 原样发送,记录命中。 | | `custom` | 对象列表,见 [`security.redact.custom[]`](#cfg-security-redact-custom) | `[]` | 自定义规则:匹配到的内容和凭据一样替换,或只记录。 | +| `exceptions` | 对象列表,见 [`security.redact.exceptions[]`](#cfg-security-redact-exceptions) | `[]` | 例外值:在任何规则下都不替换、不记录。只保存哈希,不保存值本身;最多 500 个。保存的凭据不能列为例外:规则为 `thinkwatch-saved-secret` 的条目可以读入,但不生效。 | @@ -777,6 +780,19 @@ pricing: 夹在更长的一串字母或数字中间的号码不认;请求体中以 JSON 数值写出的号码(例如工具调用的参数)也不认,替换它会使请求体不再是合法的 JSON。占位符写明原来是什么(`<>`、`<>`、`<>`、`<>`),安全日志中号码只显示最后四位,邮箱只显示第一个字和域名。自定义规则替换为 `<>`,写了占位符名称(`label`)时用它。 +例外值:某个值反复被规则认出、但并不敏感(测试用的固定值、文档里的示例密钥)时,可以写进 `exceptions`。规则照常工作,只有这个值在任何规则下、`observe` 和 `enforce` 两档都不处理。每一项只保存值的 SHA-256,不保存值本身,因此可以随导出、同步和备份一起带走。(不加盐的 SHA-256,见[文件位置](#文件位置))。应用从安全日志或粘贴的值加入;手写时 `sha256` 是值在消息中原本的样子的哈希,不含两边的引号和 JSON 转义。只有最近的日志记录可以从日志列为例外:每个值的哈希只在内存中保留最近 20,000 条、不跨重启;由几处重叠的命中合成的一段不能列为例外。更早的记录既没有按钮也不显示「例外」标签,可以粘贴该值手动加入。 + + + + +| 字段 | 类型 | 默认值 | 说明 | +|---|---|---|---| +| `sha256` | 字符串 | **必填** | 值的 SHA-256,64 个小写十六进制字符。请求体中的转义先解开:哈希按值本身计算。 | +| `masked` | 字符串 | `''` | 加入时显示的值,打码方式与安全日志相同。 | +| `rule` | 字符串 | `''` | 最先认出该值的规则:内置规则的 id 或自定义规则的名称。手动添加且没有规则认出的值为空。 | +| `added_at` | 整数 | `0` | 加入的时间,Unix 纪元以来的毫秒数。 | + + #### `security.inspect_tools` 按规则检查模型返回的工具调用。`enforce` 下命中处置为 `cut` 的规则时切断响应,客户端拿不到可执行的完整调用。 diff --git a/release-notes/0.86.0.md b/release-notes/0.86.0.md new file mode 100644 index 00000000..c291aafe --- /dev/null +++ b/release-notes/0.86.0.md @@ -0,0 +1,20 @@ +This release lets a falsely matched value be excepted from outbound redaction without turning its rule off, adds per-rule hit counts for the security log, and stops rescanning conversation history that has not changed. + +**Upgrade notes** + +- The control-plane protocol version goes from 56 to 57. +- The request store's schema is unchanged (28); nothing is cleared. An index for the hit counts is added on first start. +- New configuration key `security.redact.exceptions`; existing configurations are valid as they are. +- New message codes: `config.redact_exception_duplicate`, `config.redact_exception_hash_bad`, `config.redact_exceptions_too_many`, `security.exception_forgotten`, `security.exception_not_exceptable`, `security.exception_saved_secret`, `security.exception_value_empty`, `security.exceptions_full`, `security.unknown_event`. + +**Exceptions.** A value listed as an exception is neither replaced nor recorded by any redaction rule, in observe and replace mode alike. The rule keeps working for every other value. + +- `security.redact.exceptions` holds up to 500 entries `{ sha256, masked, rule, added_at }`: the SHA-256 of the value (lowercase hex), the masked form shown when it was added, the rule that first recognised it, and when. The value itself is never written anywhere. The configuration manual notes that a hash of a short value such as a phone number can be guessed, the same way the file already holds upstream keys in plain text. +- Credentials saved in ThinkWatch (`thinkwatch-saved-secret`) cannot be excepted; such entries are ignored. +- `GET /security/exceptions`, `POST /security/exceptions` (a value, hashed at once), `POST /security/events/{id}/exception` (from a security-log row) and `DELETE /security/exceptions/{sha256}`. Adding a value that is already listed returns the existing entry; a full list answers 409. +- Adding from the log works for the latest 20,000 redaction rows while the core is running: their value hashes are kept in memory only, never in the request store, and are gone after a restart. Log views carry `exceptable` and `excepted` for those rows. A finding made of two overlapping matches is not exceptable, since no single value stands behind it. +- The trial marks an excepted value with `excepted` instead of reporting a hit. + +**Hit counts per rule.** `GET /security/counts?guard=&since_ms=` returns `[{ rule, custom, count }]`, the sum of the log rows' counts for one guard since a time, from an index rather than by reading the log. + +**History is not rescanned.** Clients send the whole conversation every turn. The gateway now keeps the redaction scan results of unchanged history (rule ids and positions only, never values) in memory, keyed by the rules in force, and scans only what is new. Results are identical to a full scan; rules that look at neighbouring text (field names, context words) bypass the cache. On a 200-turn chat, scanning a turn takes 0.25 to 0.46 of the time it took before, and a request is never slower than without the cache.