diff --git a/Cargo.lock b/Cargo.lock index 8fd13602e..0aa28f386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -556,6 +556,7 @@ version = "0.2.7" dependencies = [ "assert_cmd", "axum", + "base64 0.22.1", "blake3", "clap", "console", @@ -566,6 +567,7 @@ dependencies = [ "futures-executor", "futures-util", "globset", + "hex", "ignore", "indicatif", "libc", @@ -974,6 +976,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "http" version = "0.2.12" diff --git a/packages/python/e2e-attestations/claude-1130-codec-crates.json b/packages/python/e2e-attestations/claude-1130-codec-crates.json new file mode 100644 index 000000000..b8ecb541d --- /dev/null +++ b/packages/python/e2e-attestations/claude-1130-codec-crates.json @@ -0,0 +1,7 @@ +{ + "command": "just test-e2e", + "ran_at": 1789748436, + "exit_code": 0, + "commit": "9468882ee26238c102d59814a212d6c7f9c6be92", + "branch": "claude/1130-codec-crates" +} diff --git a/packages/rust/Cargo.toml b/packages/rust/Cargo.toml index 4d98353ce..47422d37e 100644 --- a/packages/rust/Cargo.toml +++ b/packages/rust/Cargo.toml @@ -46,6 +46,7 @@ cli = [ "dep:axum", "dep:clap", "dep:futures", + "dep:hex", "dep:reedline", "dep:tokio-stream", "tokio/rt-multi-thread", @@ -56,12 +57,16 @@ cli = [ [dependencies] axum = { version = "0.8", optional = true } +base64 = "0.22" blake3 = "1" clap = { version = "4", features = ["derive"], optional = true } console = "0.16.6" futures = { version = "0.3", optional = true } futures-channel.workspace = true globset = "0.4" +# Hex rendering of BLOB cells in the CLI's JSON output. CLI-only, so a +# library consumer never pulls it. +hex = { version = "0.4", optional = true } # Hierarchical `.gitignore` matching for path-table scans (the fd/ripgrep # semantics), without adopting its walker: the scan keeps `walkdir`. ignore = "0.4" diff --git a/packages/rust/changelog.d/2026-09-18-codec-crates.md b/packages/rust/changelog.d/2026-09-18-codec-crates.md new file mode 100644 index 000000000..d1a66a4bb --- /dev/null +++ b/packages/rust/changelog.d/2026-09-18-codec-crates.md @@ -0,0 +1,3 @@ +**Changed** + +- **The function worker's base64 codec and the CLI's hex codec are now the `base64` and `hex` crates.** The hand-rolled `base64_encode` / `base64_decode` in `functions.rs` and `hex_encode` in `cli/serialize.rs` are gone (~65 lines), along with the `clippy::cast_possible_truncation` opt-out the decoder's shifts needed. The wire format is unchanged: `{"$bytes": ""}` still uses the standard alphabet with required padding and no line wrapping, and BLOB cells still render as lowercase two-chars-per-byte hex with no separator — `base64::engine::general_purpose::STANDARD` is that configuration exactly. The decoder stays strict: a length that is not a multiple of four, a byte outside the alphabet, and padding anywhere but the close of the final chunk all still fail the query with `"$bytes" is not valid base64`. It is also strict on one input the hand-rolled decoder let through — a final character carrying non-canonical trailing bits (`"AQJ="`), which no conforming encoder emits. Costs `base64` as a dependency of the published crate (zero-dependency, `no_std`-capable, and already in the lockfile transitively, so one crate rather than a subtree); `hex` is likewise zero-dependency and sits behind the `cli` feature, so a library consumer never pulls it. (#1130) diff --git a/packages/rust/src/cli/serialize.rs b/packages/rust/src/cli/serialize.rs index 92a901f8b..c487e8c31 100644 --- a/packages/rust/src/cli/serialize.rs +++ b/packages/rust/src/cli/serialize.rs @@ -26,19 +26,10 @@ fn cell_to_json(value: &CellValue) -> Value { .map(Value::Number) .unwrap_or(Value::Null), CellValue::Text(s) => Value::String(s.clone()), - CellValue::Blob(bytes) => Value::String(hex_encode(bytes)), + CellValue::Blob(bytes) => Value::String(hex::encode(bytes)), } } -/// Hex-encode a byte slice, for `BLOB` SQLite values in JSON output. -fn hex_encode(bytes: &[u8]) -> String { - let mut out = String::with_capacity(bytes.len() * 2); - for b in bytes { - out.push_str(&format!("{b:02x}")); - } - out -} - pub(super) fn event_to_json(event: &RowEvent) -> String { event_to_value(event).to_string() } @@ -198,6 +189,8 @@ mod tests { fn blob_cell_becomes_hex_string() { let json = cell_to_json(&CellValue::Blob(vec![0xde, 0xad, 0xbe, 0xef])); assert_eq!(json.as_str(), Some("deadbeef")); + let padded = cell_to_json(&CellValue::Blob(vec![0x00, 0x0f, 0xff])); + assert_eq!(padded.as_str(), Some("000fff")); } #[test] @@ -205,9 +198,4 @@ mod tests { assert!(cell_to_json(&CellValue::Real(f64::NAN)).is_null()); assert!(cell_to_json(&CellValue::Real(f64::INFINITY)).is_null()); } - - #[test] - fn hex_encode_matches_lowercase_spec() { - assert_eq!(hex_encode(&[0x00, 0x0f, 0xff, 0xde, 0xad]), "000fffdead"); - } } diff --git a/packages/rust/src/functions.rs b/packages/rust/src/functions.rs index e0f0a237c..440a502db 100644 --- a/packages/rust/src/functions.rs +++ b/packages/rust/src/functions.rs @@ -38,6 +38,8 @@ use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender}; use std::sync::{Arc, Mutex}; use std::time::Duration; +use base64::Engine as _; +use base64::engine::general_purpose::STANDARD; use rusqlite::Connection; use rusqlite::functions::FunctionFlags; @@ -357,7 +359,7 @@ fn value_to_json(value: &Value) -> serde_json::Value { .map(serde_json::Value::Number) .unwrap_or(serde_json::Value::Null), Value::Text(s) => serde_json::Value::from(s.as_str()), - Value::Blob(bytes) => serde_json::json!({ "$bytes": base64_encode(bytes) }), + Value::Blob(bytes) => serde_json::json!({ "$bytes": STANDARD.encode(bytes) }), } } @@ -428,77 +430,15 @@ fn json_to_sql_value(value: &serde_json::Value) -> Result { let encoded = map["$bytes"] .as_str() .ok_or_else(|| "\"$bytes\" must be a base64 string".to_string())?; - let bytes = base64_decode(encoded) - .ok_or_else(|| "\"$bytes\" is not valid base64".to_string())?; + let bytes = STANDARD + .decode(encoded) + .map_err(|_| "\"$bytes\" is not valid base64".to_string())?; Ok(Value::Blob(bytes)) } other => Ok(Value::Text(other.to_string())), } } -const BASE64_ALPHABET: &[u8; 64] = - b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -/// Standard base64 with padding. Hand-rolled (~20 lines) rather than a new -/// dependency for one wire field. -fn base64_encode(bytes: &[u8]) -> String { - let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); - for chunk in bytes.chunks(3) { - let b0 = u32::from(chunk[0]); - let b1 = chunk.get(1).map(|b| u32::from(*b)); - let b2 = chunk.get(2).map(|b| u32::from(*b)); - let triple = (b0 << 16) | (b1.unwrap_or(0) << 8) | b2.unwrap_or(0); - let index = |shift: u32| BASE64_ALPHABET[(triple >> shift & 0x3f) as usize] as char; - out.push(index(18)); - out.push(index(12)); - out.push(if b1.is_some() { index(6) } else { '=' }); - out.push(if b2.is_some() { index(0) } else { '=' }); - } - out -} - -/// Decode standard base64 (padding required). `None` on any malformed input. -fn base64_decode(text: &str) -> Option> { - let bytes = text.as_bytes(); - if !bytes.len().is_multiple_of(4) { - return None; - } - let value_of = |b: u8| -> Option { - BASE64_ALPHABET - .iter() - .position(|c| *c == b) - .map(|i| u32::try_from(i).expect("index < 64")) - }; - let mut out = Vec::with_capacity(bytes.len() / 4 * 3); - for (i, chunk) in bytes.chunks(4).enumerate() { - let last = (i + 1) * 4 == bytes.len(); - let pad = chunk.iter().filter(|b| **b == b'=').count(); - // Padding may only close the final chunk, as its last one or two chars. - if pad > 0 && (!last || pad > 2 || chunk[..4 - pad].contains(&b'=')) { - return None; - } - let mut triple: u32 = 0; - for b in &chunk[..4 - pad] { - triple = (triple << 6) | value_of(*b)?; - } - triple <<= 6 * u32::try_from(pad).expect("pad <= 2"); - #[expect( - clippy::cast_possible_truncation, - reason = "each shift isolates one byte" - )] - { - out.push((triple >> 16) as u8); - if pad < 2 { - out.push((triple >> 8) as u8); - } - if pad < 1 { - out.push(triple as u8); - } - } - } - Some(out) -} - /// The production transport: the spawned worker process, its piped stdin, /// and a reader thread draining its stdout line-by-line into a channel (a /// blocking read has no timeout; `recv_timeout` on the channel does). @@ -902,40 +842,6 @@ mod tests { ); } - // --- base64 ------------------------------------------------------------ - - #[test] - fn base64_round_trips_every_padding_length() { - for bytes in [ - &b""[..], - &b"f"[..], - &b"fo"[..], - &b"foo"[..], - &b"foob"[..], - &[0u8, 255, 16, 3][..], - ] { - let encoded = base64_encode(bytes); - assert_eq!(base64_decode(&encoded).as_deref(), Some(bytes), "{encoded}"); - } - } - - #[test] - fn base64_known_vectors() { - assert_eq!(base64_encode(b"foobar"), "Zm9vYmFy"); - assert_eq!(base64_encode(b"foob"), "Zm9vYg=="); - assert_eq!(base64_encode(&[1, 2]), "AQI="); - assert_eq!(base64_decode("Zm9vYmFy").as_deref(), Some(&b"foobar"[..])); - } - - #[test] - fn base64_decode_rejects_malformed_input() { - assert_eq!(base64_decode("abc"), None); // not a multiple of 4 - assert_eq!(base64_decode("a!=="), None); // bad alphabet - assert_eq!(base64_decode("ab=c"), None); // padding inside a chunk - assert_eq!(base64_decode("ab==cd=="), None); // padding before the last chunk - assert_eq!(base64_decode("a==="), None); // over-padding - } - // --- defaults ----------------------------------------------------------- #[test] diff --git a/packages/ts/e2e-attestations/claude-1130-codec-crates.json b/packages/ts/e2e-attestations/claude-1130-codec-crates.json new file mode 100644 index 000000000..1762171c0 --- /dev/null +++ b/packages/ts/e2e-attestations/claude-1130-codec-crates.json @@ -0,0 +1,7 @@ +{ + "command": "pnpm test:e2e", + "ran_at": 1789748508, + "exit_code": 0, + "commit": "3b9d98d44a30dcce107b15037fa9b7ef5fcb088a", + "branch": "claude/1130-codec-crates" +}