diff --git a/.claude/plans/utf8-validation.md b/.claude/plans/utf8-validation.md new file mode 100644 index 0000000..826707a --- /dev/null +++ b/.claude/plans/utf8-validation.md @@ -0,0 +1,829 @@ +# UTF-8 validation in the JSON lexers (L & VL) + +> Branch `fix/utf8-validation-prototype`. Supersedes the prototype commit `fb066a1`. +> Companion to [fredbi/core `.claude/plans/default-lexer-roadmap.md`] §0.4/§3.1 and +> `string-unescape-optimization.md` (which listed UTF-8 validation as *explicitly out +> of scope*: "we deliberately don't validate on the alias path" — this plan closes +> that hole). +> Status legend: ✅ done · ⏳ in progress · ⬜ todo · 🔬 measure first. + +## OUTCOME — round 3 shipped (2026-07-30): the writers + +The writers had the mirror-image bug and one inconsistency of their own. Probed, not assumed: + +| input to `String*` | before | `Raw` before | +|---|---|---| +| invalid byte mid-string | silently → U+FFFD | passed through | +| overlong / encoded surrogate | silently → 2–3× U+FFFD | passed through | +| **truncated rune at end** | **errored** | passed through | + +So a trailing incomplete rune errored while every other fault was silently rewritten. Now one knob decides, and it is +the *same type* the lexers use: `UTF8Policy` moved to `json/internal/utf8x` and both packages alias it, so a document +refused on the way out and one rejected on the way in mean the same thing. + +**Default is `UTF8Strict`** (Fred's call). `WithUTF8Policy` is a `BufferedOption` (so `Indented`/`YAML` get it through +their existing buffered-option wrappers); `Unbuffered` takes `WithUnbufferedUTF8Policy` — two names only because Go +options are typed per writer. + +**What is checked** (Fred: *"not needed to check again what comes from token, only from raw []byte and io.Reader"*): + +- **Validated** — `String`, `StringBytes`, `StringRunes`, `StringCopy`, `Raw`, `RawCopy`, and `VerbatimValue` (which + renders through the ordinary string path). +- **Trusted** — `Token` and `VerbatimToken`. Documented loophole: a document lexed with `UTF8Passthrough` can carry + ill-formed bytes in, and then `Token` *silently substitutes* U+FFFD (the escaper decodes as it goes, so output stays + valid but differs with no error) while `VerbatimToken` emits the bytes byte-for-byte, as verbatim requires. +- **Unchecked** — `NumberBytes`/`NumberCopy`, which already document themselves as unchecked; non-ASCII in a number is + a grammar problem, not a UTF-8 one. + +**Two real bugs the new tests found, both fixed:** + +1. `RawCopy` and `StringCopy` validated each chunk independently, so a lead byte at the end of one read followed by a + non-continuation at the start of the next was tolerated by both halves and emitted. `RawCopy` now holds the + incomplete tail back and re-joins it with the next read (reads land at an offset in the buffer that leaves room, so + no second buffer and no allocation); `StringCopy`'s stitched sequence is now validated too — it was assembled by + the stitcher and had never passed through the chunk check. +2. Under `UTF8Replace` a truncated tail still *errored* instead of substituting, on every escaped path. The remainder + is now policy-driven like everything else. + +**Cost** (`BenchmarkUTF8PolicyCallerPath`, strict vs passthrough): + +- escaped paths (`StringBytes`): within noise — the escaper dominates and the AVX2 validator is a rounding error + (ascii 1767 vs 1841 MB/s, cjk 279 vs 291); +- `Raw`: **5.9 vs 8.8 GB/s (−33%)**. Raw is otherwise a memcpy, so a full validation pass is proportionally large. + That is the honest price of "Strict means the output really is UTF-8"; callers who had already validated can say + `UTF8Passthrough` and get the old speed back; +- the token-driven path (what `BenchmarkWriters` measures) is untouched by construction. + +**Not done:** deduplicating the two escapers (`escape.go:escapedBytes` and `buffered.writeEscaped` still carry the +same logic twice). Plan §10 item 3.1 hoped `utf8x` would unify them, but they differ in how they emit (append to a +buffer vs write-with-flush), so the honest sharing is the *rule*, not the code — pinned by +`TestWriterSubstitutionMatchesLexer`, which cross-checks writer output against `utf8x.Sanitize`. + +## OUTCOME — round 2 shipped (2026-07-30) + +The simdutf/Keiser–Lemire `lookup4` validator is ported to avo AVX2 (`json/internal/utf8x/_asm` → +`validate_amd64.s`), 32 B/iter, and the BOM and VL-contract items are done. **The gate is now met.** + +`UTF8Strict` vs no validation, benchstat 10×300 ms: + +| workload | round 1 (L) | round 2 (L) | round 1 (VL) | round 2 (VL) | +|---|---|---|---|---| +| canada_geometry | ~ | ~ | ~ | ~ | +| citm_catalog | ~ | ~ | ~ | ~ | +| citm_catalog_min | ~ | ~ | +5.0% | ~ | +| golang_source | ~ | ~ | ~ | ~ | +| twitter_status | **−10.8%** | **−6.3%** | **−8.2%** | **−1.7%** | + +Geomean −1.4% → **−0.36%**. Raw validator throughput (`BenchmarkValid`): 5–12× the stdlib once the kernel engages — +`cjk/4096` 248 ns vs 2420 ns (~16 GB/s), `cjk/128` 12.5 ns vs 80 ns. + +`avx2Min` settled at **32** (one block, the kernel's hard minimum). The micro-benchmark shows a clear win from 32 +bytes up (`latin/32`: 6.7 ns vs 18.1 ns); end-to-end the 32-vs-64 choice is not statistically distinguishable, so the +micro-benchmark decided it. + +**How the kernel is trusted.** Hand-written vector code that accepts ill-formed input fails silently, so the +differential suite is the deliverable, not a formality: + +- exhaustive over **all 65 536 byte pairs** (which is exactly what the three VPSHUFB tables encode) at 10 alignments; +- exhaustive over **every lead byte C0–FF × all 2nd × all 3rd bytes** (4.2 M) at 3 alignments — this is what exercises + the prev2/prev3 continuation logic the pairwise sweep cannot reach; +- every 4-byte lead F0–FF × every 2nd byte × 5 tails × 8 alignments (TOO_LARGE / OVERLONG_4 live here); +- every length 0…256, every truncation shape pinned at the end, seam straddles across the last block boundary; +- 40 k random/structured-noise cases; 9.9 M fuzz executions, each re-checked at 32 shifted alignments (~325 M + validations) — all against `utf8.Valid`; +- `TestKernelIsActuallyExercised` fails the suite from becoming vacuous if AVX2 is absent or the length gate keeps + everything scalar. + +**Mutation-tested:** three single-bit corruptions of the lookup tables (dropping SURROGATE from the `1110____` lead, +dropping it from the `____1101` low nibble, flipping a bit in `byte_2_high`) were each caught. The suite has teeth. + +**Design note — the tail.** The kernel validates whole 32-byte blocks only and says nothing about a sequence crossing +the end; `Valid` rewinds to the last sequence boundary and finishes with `utf8.Valid`. That is simpler than an in-asm +zero-padded tail and mirrors what simdutf does to locate errors. It does mean a value just past a block boundary pays +a scalar tail of up to 31 bytes, which the micro-benchmark shows costing more than the vector part for such lengths. +Closing that would need the kernel to accept a carried-in `prev` block so the final partial block can be revalidated +from an overlapping offset — a signature change plus its own differential pass. Not done; the gate is met without it. + +**BOM (§2.8) — now shipped**, the way it was originally scoped minus one thing: the mark is consumed at offset 0 and +is **not** re-emitted by `VL`. Carrying it in `LeadingSpace()` would need `blankStart` changes across 6 core sites (4 +generated) plus ~20 streaming resets; consuming it costs one comparison per document and touches no scan loop. The +`FuzzLexer` round-trip invariant is correspondingly relaxed to "byte-exact except a leading BOM", which is the +direction RFC 8259 §8.1 asks for. `i_structure_UTF-8_BOM_empty_object` flips to accept; the two `n_structure_*BOM*` +cases stay rejected, and the `i_` golden diff showed exactly that one line changing. + +## OUTCOME — round 1 shipped (2026-07-30) + +All ten ill-formed-UTF-8 conformance cases now reject, in every mode; the L/VL surrogate-escape divergence is gone; +`y_ 95/95, n_ 188/188, xfail 0` still holds. 2052 new subtests, 2.5M fuzz executions clean, lint clean. + +**Cost of the default policy vs no validation** (benchstat, 10×300 ms, MB/s): + +| workload | L | VL | +|---|---|---| +| canada_geometry | ~ (p=0.68) | ~ (p=0.44) | +| citm_catalog | ~ (p=0.25) | ~ (p=0.74) | +| citm_catalog_min | ~ (p=0.53) | +5.0% (p=0.009) | +| golang_source | ~ (p=0.35) | ~ (p=0.58) | +| twitter_status | **−10.8%** (p=0.000) | **−8.2%** (p=0.000) | + +Geomean −1.4%. The four ASCII-dominated workloads are statistically indistinguishable from no validation — the fused +detection works as designed, and the prototype's AVX2 hole (§4.2) is closed: `golang_source` went from −6.6% to ~0 and +`citm_catalog_min` from −10.0% to ~0. + +**Gate result: ASCII target met (≤3%), `twitter_status` missed (≤8% wanted, −10.8% measured).** The arithmetic says +this is exactly the validator pass and nothing else: 109 KB of non-ASCII string bytes at `utf8.Valid`'s ~1.2 GB/s is +~90 µs against a 651 µs baseline ≈ 14%, observed 10.8%. So **round 2 (§5.4) is justified** — an AVX2 `lookup4` +validator at ~20 GB/s takes that to ~5 µs, i.e. under 1%. Awaiting the go-ahead. + +**Deviations from the plan as written:** + +1. **BOM trimming (§2.8) is NOT shipped** — only the UTF-16 diagnostic (`ErrNotUTF8`) is. Accepting a leading UTF-8 + BOM breaks VL's byte-exact round-trip, which `FuzzLexer` enforces (seed #62, `"{}"`), and carrying the mark + in `LeadingSpace()` means `blankStart` must start at 0 on the first token: 6 sites, 4 of them in generated + `scan_gen.go`, plus ~20 per-token `l.blanks[:0]` resets in the streaming cores. That is its own commit with a full + codegen verification, not a rider on this one. +2. **Replacement granularity (§2.4) was corrected during review** to one U+FFFD per *invalid byte* — `utf8.DecodeRune` + always advances 1 on error, so the original "maximal subpart, matching DecodeRune's advance" was self-contradictory. + The deciding argument is that `writer.escapedBytes` already does per-byte, so lexer, writer and caller-side + `range value` now agree. +3. **`ensureWindow` was added to `refill.go`** (not in the plan): the surrogate lookahead must *peek* the following + `\uXXXX` rather than consume it, so that `\uD800\uD800` yields two U+FFFD instead of swallowing the second escape. + `consumeN` cannot serve that — a read that straddles a refill cannot be given back — so the cold path compacts the + window instead. Safe there because the scanners that reach it hold no live buffer index. + +## 0. TL;DR + +**The bug.** Neither `L` nor `VL` validates UTF-8 in string bodies. The string +scanners look for exactly three stop bytes (`"`, `\`, `< 0x20`); every byte +`>= 0x80` is waved through and aliased straight into the token value. So +`["\xff"]`, `["\xc0\xaf"]` (overlong), `["\xed\xa0\x80"]` (encoded surrogate) and +`["\xf4\xbf\xbf\xbf"]` (> U+10FFFF) are all **accepted**, and the invalid bytes +land in the caller's `[]byte`. RFC 8259 §8.1 requires JSON text to be UTF-8, so +this is a genuine conformance bug — 10 `i_string_*` conformance cases, plus a +secondary escape-level variant (§2.3). + +**The fix.** One user-visible knob, `WithUTF8Policy`, with three values: +`UTF8Strict` (default — error), `UTF8Replace` (mute the error, substitute U+FFFD), +`UTF8Passthrough` (today's unvalidated behavior, documented as unsafe). +The prototype's `fused`/`naive` are **implementation strategies**, not policies: +they disappear from the API and are chosen internally from CPU/build. + +**The performance answer.** The concern that this "wipes out our vast string-scan +improvements" is real for a naive `utf8.Valid` at the funnel (**−10% to −32%** +measured, §5.1) but *not* for the fused design, provided one hole in the prototype +is plugged: the AVX2-scanned region is not accumulated, so it conservatively forces +validation on **every long string** — including the 10905 pure-ASCII long strings of +`golang_source`. Plugging it costs ~4 instructions per 32-byte iteration +(`VPMOVMSKB` of the raw data, already loaded, masked below the stop lane). Real-world +JSON is 96–100% ASCII strings by count (§5.2), so after the fix the overwhelming +majority of values are proven valid UTF-8 **inside the existing pass, with no second +read and no call**. + +**Also in scope** (settled on review): a leading UTF-8 BOM is trimmed (§2.8, carried in +`VL.LeadingSpace()` so round-trip stays exact), and the shared primitives land in a new +`json/internal/utf8x` (§7.1) — the one package location reachable from **both** the +lexers and the writers, because the writers have the mirror-image bug and will reuse +them (§10 round 3). + +**Sequencing.** Round 1 = correctness + fused detection + the AVX2 kernel flag + +`utf8.Valid` as the validator. Round 2 = port the simdutf/Keiser–Lemire `lookup4` +validator to avo AVX2 — **only if** round 1's numbers on `twitter_status` justify it. +Round 3 = the writers. + +--- + +## 1. What is actually broken + +### 1.1 The alias path never looks at high bytes + +`swar.StringStopMask` (`internal/swar/swar.go:47`) and the AVX2 kernel +(`internal/strscan/_asm/asm.go`) both flag exactly `< 0x20`, `0x22`, `0x5c`. Both +are documented as "multibyte-safe", which is true in the sense that they never +*mis*-flag a continuation byte — but that is precisely why nothing ever inspects +those bytes. `consumeStringWhole` then does `value := data[start:i:i]` and hands the +raw window to the caller. + +### 1.2 Everything downstream inherits it + +`L` decodes escapes into `CurrentValue`; raw non-ASCII bytes are `append`ed +verbatim. `VL` aliases the source. Both therefore emit `token.T` / `token.VT` values +that a caller will `string(...)`-convert, at which point Go silently produces U+FFFD +runes on iteration — the corruption becomes invisible rather than absent. + +### 1.3 Evidence (JSONTestSuite, current `master` behavior) + +All ten of these are **accepted** by `L/bytes`, `L/reader` and `VL/bytes` today: + +| file | bytes | why it is invalid | +|---|---|---| +| `i_string_invalid_utf-8` | `5b 22 ff 22 5d` | `0xff` is never a valid lead | +| `i_string_lone_utf8_continuation_byte` | `81` | continuation with no lead | +| `i_string_iso_latin_1` | `e9` | latin-1 `é`, truncated lead | +| `i_string_truncated-utf-8` | `e0 ff` | 3-byte lead, bad continuation | +| `i_string_UTF-8_invalid_sequence` | `e6 97 a5 d1 88 fa` | valid prefix then `0xfa` | +| `i_string_UTF8_surrogate_U+D800` | `ed a0 80` | encoded UTF-16 surrogate | +| `i_string_not_in_unicode_range` | `f4 bf bf bf` | U+13FFFF > U+10FFFF | +| `i_string_overlong_sequence_2_bytes` | `c0 af` | overlong `/` | +| `i_string_overlong_sequence_6_bytes` | `fc 83 bf bf bf bf` | obsolete 6-byte form | +| `i_string_overlong_sequence_6_bytes_null` | `fc 80 80 80 80 80` | overlong NUL | + +They are `i_` (implementation-defined) in the suite, so the conformance test stays +green — that is why this went unnoticed. Under `UTF8Strict` all ten become +**rejected**; under `UTF8Replace` they are accepted with the offending bytes +substituted. + +--- + +## 2. Semantics (settled) + +### 2.1 The policy + +```go +// UTF8Policy governs what the lexer does with input that cannot be represented +// as a Unicode scalar value. +type UTF8Policy int + +const ( + UTF8Strict UTF8Policy = iota // default: error out (codes.ErrInvalidUTF8) + UTF8Replace // substitute U+FFFD, no error + UTF8Passthrough // no validation at all — UNSAFE, trusted input only +) + +func WithUTF8Policy(p UTF8Policy) Option +``` + +- **Default is `UTF8Strict` for both `L` and `VL`.** This is a behavior change: + documents previously accepted are now rejected. It is the whole point of the fix + and belongs in the changelog as a breaking-ish conformance change. +- `UTF8Passthrough` exists as an escape hatch for callers who have already validated + their input (and for A/B benchmarking of the validation cost). Its godoc must say + plainly that invalid bytes will reach the caller's `[]byte`. +- `WithUTF8ValidationMode` / `WithUTF8Validation` from the prototype are **deleted**. + Nothing about `fused` / `naive` / SWAR / AVX2 is user-visible; `WithoutAVX2` stays + the single vector-path escape hatch and now also disables the vector validator. + +### 2.2 What counts as invalid (raw bytes) + +Exactly Go's `utf8.Valid` semantics, which is also RFC 3629 + Unicode: +truncated sequences, over-long encodings, lead/continuation mismatches, encoded +surrogates `U+D800..U+DFFF`, and anything above `U+10FFFF`. No exceptions: we do +**not** accept WTF-8 or CESU-8. + +### 2.3 What counts as invalid (`\u` escapes) — L/VL divergence to fix + +The same policy governs `\u` escapes that do not form a Unicode scalar value. This +is a second, independent way an invalid character enters a value, and today the two +lexers disagree about it: + +| input | `L` today | `VL` today | strict (new) | replace (new) | +|---|---|---|---|---| +| `["\uD888ሴ"]` | **accept** → U+FFFD | reject | reject | `"�ሴ"` | +| `["\uD800\uD800\n"]` | **accept** → U+FFFD + swallows the 2nd escape | reject | reject | U+FFFD, U+FFFD, `\n` | +| `["\uDd1e\uD834"]` | **accept** → U+FFFD | reject | reject | U+FFFD, U+FFFD | +| `["\uDADA"]` (no 2nd) | reject | reject | reject | U+FFFD | + +Root cause in `L`: `input.unescapeUnicodeSequence` (`string.go:659`) calls +`utf16.DecodeRune`, which *returns* `utf8.RuneError` for a broken pair, and the +subsequent `utf8.ValidRune(0xFFFD)` is **true** — so the failure is laundered into a +silent replacement. `VL`'s `validateUnicodeWhole` (`string_raw.go`) checks +`DecodeRune(...) == utf8.RuneError` explicitly and errors. + +> Historical note: `default-lexer-roadmap.md` §0.1 recorded this divergence in the +> *opposite* direction ("L rejects, VL accepts"). Conformance fix group E made `VL` +> validate its escapes, which inverted the asymmetry rather than removing it. + +Fixes: +- **strict** — `unescapeUnicodeSequence` must treat `DecodeRune == RuneError` as + `ErrSurrogateEscape`, matching `validateUnicodeWhole`. +- **replace** — `validateUnicodeWhole` must *stop* erroring on a broken pair; + `unescapeUnicodeSequence` must emit U+FFFD and **not consume the second escape** + (today it does, so `\uD800\uD800` yields one U+FFFD instead of two). + `token.decodeUnicode` (`token/unescape.go:108`) already has the correct + maximal-subpart behavior — copy its shape. + +### 2.4 Replacement granularity + +**One U+FFFD per invalid byte** — i.e. `utf8.DecodeRune`'s error advance, which is +always width 1. `\xe0\xa0\xff` → three replacements. + +The alternatives were considered and rejected: + +| rule | `\xe0\xa0\xff` | who does it | +|---|---|---| +| **per invalid byte** ✅ | 3 × U+FFFD | Go: `range` over a string, `utf8.DecodeRune`, **and our own `writer.escapedBytes`** | +| maximal subpart (Unicode TR §3.9) | 2 × U+FFFD | browsers / WHATWG | +| per run | 1 × U+FFFD | `bytes.ToValidUTF8` | + +Decider: `writers/default-writer/escape.go:escapedBytes` **already** substitutes +U+FFFD per invalid byte (`utf8.DecodeRune` → `utf8.AppendRune`, the comment says +"invalid runes are represented as �"). Picking the same rule means the lexer and +the writer agree, and `string(value)` / `range value` in caller code agrees too. It is +also the cheapest. + +Independent of the byte rule: each **broken `\u` escape** is one U+FFFD, so +`\uD800\uD800` → two. That is already `token.decodeUnicode`'s behavior. + +### 2.5 The verbatim pledge (TODO §4 — resolved) + +`VL` **does** substitute U+FFFD in the value under `UTF8Replace`. Restated pledge: + +> `VL` reproduces its input byte-for-byte **for input that is valid UTF-8**. Under +> `UTF8Replace` an ill-formed sequence is replaced in the emitted value; the caller +> explicitly asked for that trade. Offsets, `Line()`, `Column()` and `LeadingSpace()` +> are **unaffected** — they are derived from the scan cursor over the source, never +> from the value — so positions stay exact even for a mangled token. + +Justification: a document containing invalid UTF-8 is not valid JSON text, so a +byte-faithful round-trip of it re-emits invalid JSON. A caller who enables +replacement wants clean bytes out. Callers who need absolute byte fidelity keep the +default `UTF8Strict` (which never copies and never mangles — it only errors) or opt +into `UTF8Passthrough`. + +**Escapes are the exception**, and this falls out naturally: `VL` keeps string bodies +raw, so a broken `\uD800\uD800` stays *as source text* in the value; the U+FFFD +substitution happens at decode time in `token.Unescape`, which already does exactly +that. So under `UTF8Replace`, `VL` mangles *bytes* but never rewrites *escape text* — +the value stays a faithful raw string, and `Unescape` yields the sanitized form. + +### 2.6 Zero-copy invariants preserved + +| policy | value | copy? | +|---|---|---| +| `UTF8Strict`, valid input | aliased as today | no | +| `UTF8Strict`, invalid input | n/a — error | no | +| `UTF8Replace`, valid input | aliased as today | no | +| `UTF8Replace`, invalid input | rewritten into `CurrentValue` | only the offending value | +| `UTF8Passthrough` | aliased as today | no | + +Validation is read-only. **No policy adds a copy to a valid document.** + +### 2.7 `UTF8Passthrough` and escapes (settled) + +> **FRED** agree (should be documented): explicitly escaped UTF-8 MUST produce some +> rune, so U+FFFD when broken and policy is lenient. + +`UTF8Passthrough` skips **raw-byte** validation only. A `\uXXXX` escape must always +decode to *some* rune — there is no "pass the bytes through" option for an escape, +because the escape text is not the value. So: + +| | raw bytes | broken `\u` pair | +|---|---|---| +| `UTF8Strict` | error | error | +| `UTF8Replace` | U+FFFD per invalid byte | U+FFFD per broken escape | +| `UTF8Passthrough` | untouched, uninspected | U+FFFD per broken escape | + +Passthrough and Replace therefore share the whole escape path; only Strict differs. +This must be stated explicitly in the `UTF8Passthrough` godoc, not left implied. + +### 2.8 Leading byte-order mark (settled — now in scope) + +> **FRED** we'll add support for leading BOM trimming + +A UTF-8 BOM (`EF BB BF`) at **offset 0 only** is consumed as insignificant lead-in +before the top-level value. RFC 8259 §8.1 forbids *emitting* one but explicitly +permits ignoring one on input. `i_structure_UTF-8_BOM_empty_object` (`EF BB BF 7B 7D`) +flips from reject to **accept**. + +Rules, with the traps: + +- **Exactly one, exactly at offset 0.** A second BOM, or a BOM after any token, is + not special: it is U+FEFF, valid UTF-8, and therefore an invalid *token* between + values (rejected) or an ordinary character inside a string (kept). No change there. +- **`n_structure_UTF8_BOM_no_data`** (`EF BB BF`, nothing else) must **stay rejected**: + trimming leaves an empty document → `ErrNoData`. Falls out for free. +- **`n_structure_incomplete_UTF8_BOM`** (`EF BB 7B 7D` — a *truncated* BOM) must + **stay rejected**: only the full 3-byte sequence is trimmed, and `EF BB` is then an + invalid token. Falls out for free, and is now also caught as invalid UTF-8. +- **`VL` must not lose it.** Trimming the BOM would silently break round-trip. It is + accumulated into `l.blanks`, exactly like leading whitespace, so `LeadingSpace()` + on the first token replays it and a re-serialized document is byte-identical. + This is the one place the implementation differs between L and VL. +- **Streaming.** The check runs on the first fill, before the first token scan, and + must handle a window that starts with a *partial* BOM (buffer sizes are ≥ 32 bytes + and aligned, so a 3-byte prefix cannot be split — assert it rather than assume it). +- **Bonus (cheap, do it):** `FF FE` / `FE FF` at offset 0 is a UTF-16 BOM. Today it + fails as a generic invalid token. Report a dedicated `ErrNotUTF8` ("input appears to + be UTF-16, only UTF-8 is supported") — it turns a baffling parse error into an + actionable one, and costs one comparison on a cold path. + (`i_string_UTF-16LE_with_BOM`, `i_string_utf16BE_no_BOM`, `i_string_utf16LE_no_BOM` + stay rejected either way.) + +Not an option knob: trimming only ever *accepts* input that is rejected today, so +there is nothing to opt out of. + +--- + +## 3. Where validation hooks in + +`finishStringValue` (`string.go:461`) is a true funnel: all eight string paths reach +it. That is where the decision is taken; the *detection* is what varies per path. + +| # | path | file:func | round-1 detection | +|---|---|---|---| +| 1 | L whole, clean | `string.go:consumeStringWhole` | fused SWAR + AVX2 flag | +| 2 | L whole, escaped | `string.go:consumeStringEscaped` | 🔬 assume non-ASCII (validate) | +| 3 | L stream, fast | `string.go:consumeStringStreamFast` | fused SWAR + AVX2 flag | +| 4 | L stream, byte-wise | `string.go:consumeStringStreaming` | 🔬 assume non-ASCII | +| 5 | VL whole, clean | `string_raw.go:consumeStringRawWhole` | fused SWAR + AVX2 flag | +| 6 | VL whole, escaped | `string_raw.go:consumeStringRawEscaped` | 🔬 assume non-ASCII | +| 7 | VL stream, fast | `string_raw.go:consumeStringRawStreamFast` | fused SWAR + AVX2 flag | +| 8 | VL stream, byte-wise | `string_raw.go:consumeStringRawStreaming` | 🔬 assume non-ASCII | + +"assume non-ASCII" = correct but pessimistic: those values always pay a validation +pass. Escaped/streaming strings are the minority; **measure before optimizing** +(`golang_source` is escape-heavy and is the workload to watch — if it shows, the same +`hi |= w` accumulation goes into the clean-run scans of paths 2/4/6/8). + +The prototype forked `consumeStringWhole`/`consumeStringRawWhole` into `…V` twins +selected at dispatch. **Do not keep the fork.** The accumulator is one `OR` on a word +that is already in a register; a duplicated scanner doubles the surface where the +inliner/register-allocator can regress (the roadmap's §4.2 lesson: a fast-path change +regressed the slow path by 12% when they shared a frame). Land the accumulator in the +single existing function and verify with the existing devirt/inline tests. + +Non-string tokens need nothing: invalid bytes in numbers/literals/structure already +fail the grammar (`n_number_invalid-utf-8-in-int` etc. all pass today). + +--- + +## 4. Detection: fusing the non-ASCII test into the existing scan + +### 4.1 SWAR region — free + +Already right in the prototype: + +```go +w := binary.LittleEndian.Uint64(data[i:]) +hi |= w // <- the whole cost +if m := swar.StringStopMask(w); m != 0 { ... } +``` + +`hi & swar.HighBits != 0` ⇒ some byte in the scanned words was `>= 0x80`. +Over-reporting (bytes past the stop lane in the matching word) is safe: it can only +trigger a redundant validation, never skip a needed one. The scalar tail ORs each +byte it inspects. + +### 4.2 AVX2 region — the prototype's hole (the important fix) + +Today `strscan.ScanStop` returns only an index, so the prototype sets +`hi = ^uint64(0)` for anything it delegated — i.e. **every string longer than +`guessLong` (16) clean bytes is force-validated**. `golang_source` has 10905 such +strings and zero non-ASCII bytes; it pays a full `utf8.Valid` over 858 KB for +nothing. This, not UTF-8 itself, is most of the measured regression. + +Fix — the kernel already has the data in a YMM register: + +``` +VPMOVMSKB acc, mask // existing: stop-byte lanes +VPMOVMSKB data, hmask // NEW: sign bit of every byte == "byte >= 0x80" +ORL hmask, nonascii // NEW: accumulate +``` + +and on the `found` branch, zero the lanes at/above the stop before OR-ing +(`MOVL $1,cl-shift; SHLL; DECL` — plain shifts, no BMI2 needed, so the existing +AVX2-only CPUID gate stays valid). The scalar tail loop ORs each byte. + +Signature becomes: + +```go +func stringStopIndexAVX2(data []byte) (idx int, nonASCII bool) +func ScanStop(data []byte) (idx int, nonASCII bool) // SWAR fallback accumulates too +``` + +`ScanStop` has four other call sites (the clean-run scans in paths 2/4/6/8); they can +ignore the second result in round 1. + +**Cost:** one `VPMOVMSKB` + one `ORL` per 32 bytes, off the critical dependency chain +(the loop's exit test is unchanged). Expected ≈ 0 measurable — must be confirmed by +re-running the `strscan` micro-benchmarks and the corpus sweep. + +### 4.3 Result + +A pure-ASCII string is proven valid UTF-8 **without a second read of its bytes and +without a call**. Since 96–100% of corpus strings are pure ASCII (§5.2), the common +case is genuinely free, which is the whole basis for claiming this does not undo the +string-scan work. + +--- + +## 5. Validation: the second pass, and how expensive it may stay + +### 5.1 Measured cost of the prototype (baseline for the work) + +`BenchmarkUTF8Validate`, MB/s, median of 6 × 200 ms, vs `off`: + +| workload | L naive | L fused | VL naive | VL fused | +|---|---|---|---|---| +| canada_geometry | −1.0% | −0.3% | +5.0% | +3.8% | +| citm_catalog | −19.9% | −1.0% | −16.4% | −1.0% | +| citm_catalog_min | −23.2% | −10.0% | −9.8% | +1.0% | +| golang_source | −26.0% | −6.6% | −25.5% | −13.3% | +| twitter_status | −31.9% | −23.8% | −25.2% | −19.6% | + +Reading: `naive` (validate every value) is unacceptable. `fused` is already close to +free where strings are short (`citm_catalog`), and loses where strings are *long* +(`golang_source`, `citm_catalog_min`, `twitter_status`) — the §4.2 hole — plus a +genuine non-ASCII cost on `twitter_status`. + +### 5.2 Why the fused design can work — corpus profile + +`TestCorpusStringProfile`: + +| workload | strings | ≥16 B | string bytes | non-ASCII strings | non-ASCII bytes | +|---|---|---|---|---|---| +| canada_geometry | 12 | 1 | 90 (0.0%) | 0 (0.00%) | 0 | +| citm_catalog | 26604 | 1360 | 221 KB (12.8%) | 108 (0.41%) | 2.4 KB | +| citm_catalog_min | 26604 | 1360 | 221 KB (44.2%) | 108 (0.41%) | 2.4 KB | +| golang_source | 102451 | 10905 | 858 KB (44.2%) | 0 (0.00%) | 0 | +| twitter_status | 18099 | 6405 | 368 KB (58.3%) | 755 (4.17%) | 109 KB | + +So: detection must be free (it is, §4), and the validator only ever sees 0–4% of +values — but on `twitter_status` those carry 109 KB, 30% of all string bytes. + +### 5.3 Round-1 validator: `utf8.Valid` + +`utf8.Valid` has an 8-byte ASCII fast path then a scalar `acceptRanges` loop +(~1–1.5 GB/s on genuinely multibyte data). For four of five workloads it will be +invoked on ~0 bytes. Only `twitter_status` pays. + +Error position: `utf8.Valid` gives none. For strict mode we want the offset of the +offending byte in the error. Use a small `validateIndex(b []byte) int` (returns −1 or +the first bad index) built on `utf8.DecodeRune`, called **only on the failing value** — +the fast answer stays `utf8.Valid`. This mirrors simdutf, which likewise SIMD-detects +block-wise and then `rewind_and_validate_with_errors` scalar to locate the byte. + +### 5.4 Round-2 (🔬 gated on round-1 numbers): the simdutf port + +What `simdutf::validate_utf8_with_errors` actually does +(`src/generic/utf8_validation/utf8_lookup4_algorithm.h`) — the Keiser–Lemire +`lookup4` algorithm, per 32-byte block: + +1. `prev1 = concat_shift(prev_block, block, 1)` — `VPALIGNR` + `VPERM2I128`. +2. Three `VPSHUFB` table lookups, `AND`-ed together (`check_special_cases`): + `prev1 >> 4` (16-entry table), `prev1 & 0x0F`, `block >> 4`. The AND of the three + is a per-byte bitset of violated rules: TOO_SHORT / TOO_LONG / OVERLONG_2 / + OVERLONG_3 / OVERLONG_4 / SURROGATE / TOO_LARGE / TWO_CONTS. This single step + catches every 2-byte-sequence error class, including surrogates and overlongs. +3. `check_multibyte_lengths`: `prev2`/`prev3` saturating-subtract to assert that + positions requiring a 2nd/3rd continuation actually have one; `XOR` with (2). +4. `error |= …`; an all-ASCII block short-circuits to just `error |= prev_incomplete`. +5. `is_incomplete` on the last block catches a sequence truncated at the end. +6. Errors are **block-granular**; the exact offset comes from a scalar rewind. + +Port scope for us: AVX2 only (no AVX-512/ICL paths, no NEON), 32 bytes/iter, one +`YMM` of carried state (`prev_block`) plus an error accumulator. ~120 lines of avo. +`VPSHUFB` is per-128-bit-lane on AVX2, which the algorithm already accounts for +(the tables are duplicated across lanes). Expected 15–25 GB/s, i.e. the +`twitter_status` cost collapses from ~100 µs to ~5 µs. + +Portable/SWAR fallback: **do not** attempt a SWAR `lookup4`. Follow the classic +branch-per-sequence range table (the Trifunović article's approach) or simply keep +`utf8.Valid`, which already is that, well-tuned, in the stdlib. Recommendation: keep +`utf8.Valid` as the non-amd64 / `WithoutAVX2` validator and add only the AVX2 kernel. + +**Decision gate:** if round 1 lands every workload within ~3% and `twitter_status` +within ~8%, round 2 is optional polish and can be deferred behind the roadmap's +Phase 4. Re-measure before writing any assembly. + +--- + +## 6. Replacement (`UTF8Replace`) implementation + +Only reached when detection says non-ASCII **and** validation says invalid — i.e. +never on the hot path. + +```go +// in the shared package (§7.1): append src to dst, U+FFFD per invalid byte. +func utf8x.Sanitize(dst, src []byte) []byte +``` + +- Copies the valid prefix in bulk (up to `utf8x.FirstInvalid`), then per-rune from + there. +- The lexer wraps it: writes into `in.CurrentValue` — the existing scratch — so no new allocation and the + reuse contract is unchanged. Careful: on the L escaped path the value *is* + `in.CurrentValue`, so sanitize in place / via the second scratch, not aliasing + itself. (`PreviousBuffer` is not free for this; add a small `sanitizeScratch` or do + a right-to-left in-place rewrite — replacement is 3 bytes and the shortest + ill-formed subpart is 1, so the value can *grow*: in-place needs care, prefer the + scratch.) +- `MaxValueBytes` is re-checked **after** rewriting (a value can grow by up to 3×). +- `VL`: applies to raw bytes only; escape text is never rewritten (§2.5). + +--- + +## 7. Plumbing + +### 7.1 Where the primitives live (revised — writers will reuse them) + +> **FRED** we'll add a check on writers too. We might reuse the same primitive +> functions for fast-checking this so we might reconsider a refactor to higher level +> of "internal" package for those. + +Import reachability decides this, so it is worth getting right on the first commit +rather than moving code later: + +| package | reachable from | +|---|---| +| `json/lexers/default-lexer/internal/{swar,strscan}` | `default-lexer` only | +| `json/lexers/internal/scan` | anything under `json/lexers/…` | +| **`json/internal/…`** | **anything under `json/…` — lexers *and* writers** | +| — | `json/lexers/yaml-lexer` is a **separate module**: cannot import any of these | + +So the new, shareable code goes in a new **`json/internal/utf8x`**: + +```go +package utf8x + +func Valid(b []byte) bool // fast yes/no; AVX2 kernel in round 2, utf8.Valid now +func FirstInvalid(b []byte) int // -1, or the index of the first bad byte (cold, error path) +func Sanitize(dst, src []byte) []byte // append src to dst, U+FFFD per invalid byte (§2.4) +``` + +`writers/default-writer/escape.go` then adopts `utf8x.Sanitize`'s rule instead of its +open-coded `DecodeRune`/`AppendRune` loop, and gains the strict variant it lacks today +(see §10 round 3). + +What does **not** move: the *fused detection* is inline SWAR in the lexer's scanners +and stays there — it is not a callable primitive, it is an `OR` in an existing loop. + +**Deliberately deferred:** hoisting `swar` and `strscan` themselves to `json/internal/` +so the writers can use the SWAR masks too. It is a pure move, but it touches the +hottest, most codegen-sensitive files in the tree. Do it as its **own commit**, before +or after this work but never inside it, gated on `TestInlinable` + the devirt tests + +a full corpus sweep showing byte-identical performance. + +### 7.2 Lexer-side plumbing + +- `options.utf8Policy UTF8Policy` replaces `validateMode int`; `L.reset()` mirrors it + into `in.UTF8Policy` next to `NoAVX2` (keep it in the cold tail of the struct so hot + cursor field offsets are untouched — the prototype already respects this). +- `in.SawNonASCII` stays, but must be **reset per value** (the prototype leaves it + set once any pessimistic path runs; harmless today because it only over-triggers, + but it becomes a real correctness trap the moment anything else reads it). + Prefer passing it as a local/return value rather than as `Input` state, if that + survives the inliner. +- New error code `ErrInvalidUTF8 = "invalid UTF-8 sequence in string"` in + `error-codes/errors.go`. Do **not** reuse `ErrInvalidRune`, whose text says + "in unicode escape sequence" and whose meaning is the escape-level failure. +- Pooled lexers (`pools.go`): confirm the policy survives `Reset*` the way + `elideSeparator` does (`VL.reset()` has a comment about exactly this trap). + +--- + +## 8. Test plan + +- **Conformance** — the 10 files of §1.3 flip to reject under strict; add a + `UTF8Replace` mode to `conformanceModes()` where they flip back to accept. + Add the missing **`VL/reader`** mode (today only `L/bytes`, `L/reader`, `VL/bytes` + run — VL's streaming paths 7/8 are unexercised by conformance). +- **Golden table** — per path (§3, all 8) × per policy (3): a table test with + hand-built inputs that force each path (whole/stream × clean/escaped × L/VL), + asserting accept/reject, the exact value bytes, **and the error offset**. +- **Escape semantics** — the §2.3 table verbatim, both lexers, all three policies, + including `\uD800\uD800\n` yielding *two* replacements. +- **Boundary** — an ill-formed sequence straddling a refill boundary in streaming + mode (`WithBufferSize(64)` and sizes that split a 3/4-byte sequence). +- **BOM** (§2.8) — accepted at offset 0; `VL` replays it via `LeadingSpace()` and a + round-trip of `EF BB BF {}` is byte-identical; BOM-only and truncated-BOM still + rejected; a BOM *after* the first token is still an error; UTF-16 BOM yields + `ErrNotUTF8`. +- **`utf8x` unit + fuzz** — the primitives tested on their own, before the lexer: + `Valid` must agree with `utf8.Valid` on random input, `Sanitize` must be idempotent + and always produce `utf8.Valid` output. This is what makes the round-2 AVX2 kernel + swap-in safe. +- **Differential** — corpus + fuzz corpus: for every string token, our verdict must + agree with `utf8.Valid`, and under replace our output must equal a reference + maximal-subpart sanitizer. +- **Fuzz** — extend `FuzzLexer`/`FuzzYL` seeds with invalid-UTF-8 bodies; assert the + invariant "a token value emitted under Strict or Replace is always `utf8.Valid`". + This is the single strongest guard and directly encodes TODO §1. +- **Inline/devirt guards** — `TestInlinable` (swar) and the existing devirt tests must + stay green; re-check `strscan`'s micro-benchmarks after the kernel change. +- **Bench** — `BenchmarkUTF8Validate` (currently in + `json/benchmarks/lexers/lanes/utf8scratch_test.go`, a throwaway compass instrument) + gets rewritten against the policy API and kept until the numbers are settled. + +--- + +## 9. Conformance audit (TODO §6) + +Current run: **y_ 95/95, n_ 188/188, xfail 0, 35 `i_` cases.** Full `i_` inventory +with current verdicts and the expected post-change verdict: + +| group | files | now | strict | note | +|---|---|---|---|---| +| **raw invalid UTF-8** | the 10 of §1.3 | accept | **reject** | the bug being fixed | +| **broken surrogate escapes, L≠VL** | `1st_valid_surrogate_2nd_invalid`, `incomplete_surrogates_escape_valid`, `inverted_surrogates_U+1D11E` | L accept / VL reject | **reject** (both) | §2.3 — real divergence | +| **broken surrogate escapes, agreed** | `1st_surrogate_but_2nd_missing`, `incomplete_surrogate_pair`, `incomplete_surrogate_and_escape_valid`, `invalid_lonely_surrogate`, `invalid_surrogate`, `lone_second_surrogate`, `object_key_lone_2nd_surrogate` | reject | reject | already strict | +| **UTF-16 input** | `utf16BE_no_BOM`, `utf16LE_no_BOM`, `UTF-16LE_with_BOM` | reject | reject | documented: UTF-8 only | +| **BOM** | `structure_UTF-8_BOM_empty_object` | reject | **accept** | §2.8 — leading BOM is now consumed; NOT carried in `VL.LeadingSpace()`, see the round-2 outcome | +| **number magnitude** | 10 `i_number_*` (huge exp, overflow, underflow) | accept | accept | correct by design — we don't evaluate numbers | +| **depth** | `structure_500_nested_arrays` | accept | accept | correct since the stack rewrite | + +**Assessment of "how many similar bugs remain hidden".** The conformance harness +itself has two blind spots, and both are worth closing as part of this work: + +1. `i_` cases are recorded but never asserted, so a *change* in implementation-defined + behavior is invisible. Fix: snapshot the `i_` verdict table as a golden file and + fail on drift. That would have caught this bug the day the alias fast path landed. +2. `VL/reader` is not a conformance mode, so `consumeStringRawStreamFast` / + `consumeStringRawStreaming` are conformance-untested. Two of the eight string + paths have no coverage here. + +Beyond UTF-8 and the BOM (§2.8, now handled), the surviving `i_` divergences are all +*deliberate*: number magnitude (we don't evaluate numbers, so nothing overflows), +UTF-16 input (documented UTF-8-only), and deep nesting (correct since the stack +rewrite). So after this work the honest statement is: **the lexers are conformant on +`y_`/`n_`, and every implementation-defined behavior is deliberate, documented and +pinned against drift.** No further hidden bugs of this class are visible in the +parsing suite — which is exactly why blind spot #1 above matters: it is the only +thing that would catch the *next* one. + +--- + +## 10. Work breakdown + +**Round 1 — correctness + free detection** + +- ✅ 1.0 New package `json/internal/utf8x` (§7.1): `Valid` / `FirstInvalid` / + `Sanitize` + its own unit + fuzz tests, independent of the lexer. +- ✅ 1.1 `ErrInvalidUTF8` error code (+ `ErrNotUTF8` for the UTF-16 BOM, §2.8). +- ✅ 1.2 `UTF8Policy` + `WithUTF8Policy`; delete `WithUTF8ValidationMode` / + `WithUTF8Validation`; plumb through `options` → `L.reset()` → `Input`. +- ✅ 1.3 Revert the prototype's `…WholeV` forks; fold `hi |= w` into + `consumeStringWhole` / `consumeStringRawWhole` / the two `…StreamFast` paths. +- ✅ 1.4 `ScanStop` + AVX2 kernel return `nonASCII` (avo regen, `go generate`). + Verify `TestInlinable` + `strscan` micro-benchmarks. +- ✅ 1.5 Validation at `finishStringValue`: `utf8x.Valid` gated on detection, plus + `utf8x.FirstInvalid` for the error offset. +- ✅ 1.6 Escape-level unification (§2.3): strict rejects broken pairs in `L`; stop + swallowing the second escape; passthrough/replace share the escape path (§2.7). +- ✅ 1.7 `utf8x.Sanitize` + `UTF8Replace` wiring (L values, VL raw bytes). +- ✅ 1.8 Leading BOM (§2.8): trim at offset 0; `VL` carries it in `blanks`; UTF-16 BOM + diagnostic. Guard `n_structure_UTF8_BOM_no_data` / `n_structure_incomplete_UTF8_BOM`. +- ✅ 1.9 Tests §8; conformance updates + `VL/reader` mode + `i_` golden snapshot. +- ✅ 1.10 Re-measure the §5.1 matrix. **Gate:** ≤3% on ASCII workloads, ≤8% on + `twitter_status`. +- ✅ 1.11 Docs: `DESIGN.md`, `README.md`, package godoc, and the verbatim-pledge + restatement (§2.5). + +**Round 2 — ✅ shipped; gate now met (twitter_status −6.3% L / −1.7% VL)** + +- ✅ 2.1 Port `check_special_cases` + `check_multibyte_lengths` + `is_incomplete` to + avo AVX2 (`validateUTF8AVX2`), 32 B/iter, carried `prev_block`. +- ✅ 2.2 Scalar rewind for the error offset (simdutf's `rewind_and_validate_with_errors`). +- ✅ 2.3 Differential fuzz kernel-vs-`utf8.Valid` over random byte strings — mandatory + before trusting hand-written vector validation. +- ✅ 2.4 Re-measure. + +**Round 3 — ✅ shipped; default is UTF8Strict** + +The writers have the mirror-image bug, already half-handled: +`writers/default-writer/escape.go:escapedBytes` silently substitutes U+FFFD for +invalid input bytes ("invalid runes are represented as �"), i.e. it hard-codes +`UTF8Replace` with **no way to get an error**. RFC 8259 says we must not emit invalid +UTF-8, and silently mangling a caller's payload without telling them is the same class +of complaint as the lexer's silent acceptance. + +- ⏸️ 3.1 (NOT done — the two escapers differ in how they emit; the *rule* is shared and pinned by a test instead, see the round-3 outcome) Re-express `escapedBytes`' default branch on `utf8x` so lexer and writer share + one rule (§2.4) and one implementation. +- ✅ 3.2 Give the writers the same `UTF8Policy` knob. **Default: Strict.** Open: what default? Symmetry with + the lexer says Strict; today's behavior is Replace. **Strict** is the recommendation + — a writer that silently corrupts is worse than one that refuses — but it is a + louder breaking change than the lexer's, so it wants its own decision. +- ✅ 3.3 The `\u` escape emission path needs the same audit (surrogate pairs on output). + +**Follow-ups (out of scope, filed)** + +- ⬜ The YAML lexer (`json/lexers/yaml-lexer`, `YL`) has its own scanner and is not + covered by this plan — same audit needed. Note it is a **separate module**, so it + cannot import `json/internal/utf8x`; sharing would need the primitives published or + vendored. + +> **FRED** yes. We have another plan to run a full YAML conformance suite on that one, and possibly +> contribute fixes to goccy/go-yaml. Another story, another time. + +- ⬜ Hoist `swar` / `strscan` to `json/internal/` so writers can use the SWAR masks + (§7.1) — pure move, own commit, codegen-verified. + +--- + +## 11. Open questions + +*(Both original questions resolved 2026-07-30 — kept for the record.)* + +1. ~~**Should `UTF8Passthrough` also skip the escape-level checks?**~~ **Resolved:** + no — an escape must always produce a rune. Settled and specified in §2.7. + +> **FRED** agree (should be documented): explicitly escaped UTF-8 MUST produce some rune, so U+FFFD when broken and +> policy is lenient. + +2. ~~**Do we want `UTF8Replace` to be observable?**~~ **Resolved: no.** No counter, no + token flag; replacement is silent by design. Do not add hot-path state for it. + +> **FRED** no. + +**Still open (needs a decision before it is implemented, not before round 1 starts):** + +3. The writers' default policy (3.2 above): Strict — matching the lexer and refusing to + emit corrupted output — or Replace, preserving today's behavior? diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..8b97c9c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,13 @@ +# Conformance fixtures and golden files are DATA, not source: their exact bytes ARE the test. +# +# Without this, a checkout with core.autocrlf=true (the Windows default) rewrites every LF to +# CRLF and silently changes what is being tested — a YAML fixture that hinges on a trailing +# space or a tab becomes a different document, a JSON fixture that must be rejected may start +# being accepted, and every golden file mismatches. A harness may normalise line endings +# defensively as well, but the fixtures should never have been translated in the first place. +testdata/** -text +**/testdata/** -text +*.golden -text + +# Generated assembly is checked in and must match what `go generate` produces byte for byte. +*.s -text diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..8b92632 --- /dev/null +++ b/NOTICE @@ -0,0 +1,44 @@ +Copyright 2015-2025 go-swagger maintainers + +// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers +// SPDX-License-Identifier: Apache-2.0 + +This software library, github.com/go-openapi/core, includes software developed +by the go-swagger and go-openapi maintainers ("go-swagger maintainers"). + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this software except in compliance with the License. + +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0. + +It ships with copies of other software which license terms are recalled below. + +https://github.com/nst/JSONTestSuite +=========================== + +// SPDX-FileCopyrightText: Copyright (c) 2016 Nicolas Seriot +// SPDX-License-Identifier: MIT + +MIT License + +Copyright (c) 2016 Nicolas Seriot + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/go.work b/go.work index 30bffa0..4f2825e 100644 --- a/go.work +++ b/go.work @@ -4,6 +4,7 @@ use ( . ./json ./json/benchmarks + ./json/internal/utf8x/_asm ./json/lexers/default-lexer/internal/strscan/_asm ./json/lexers/yaml-lexer ) diff --git a/json/benchmarks/lexers/lanes/utf8scratch_test.go b/json/benchmarks/lexers/lanes/utf8scratch_test.go new file mode 100644 index 0000000..7afef93 --- /dev/null +++ b/json/benchmarks/lexers/lanes/utf8scratch_test.go @@ -0,0 +1,132 @@ +package lane + +// SCRATCH (utf-8 validation prototype): measures the cost of the three prototype +// validation modes across the corpus. Delete when the design is settled. + +import ( + "fmt" + "io" + "testing" + "unicode/utf8" + + lexer "github.com/go-openapi/core/json/lexers/default-lexer" + "github.com/go-openapi/core/json/lexers/token" + "github.com/go-openapi/core/json/testdata/workloads" +) + +func BenchmarkUTF8Validate(b *testing.B) { + suite, err := workloads.Corpus() + if err != nil { + b.Fatal(err) + } + + modes := []struct { + name string + policy lexer.UTF8Policy + }{ + {"passthrough", lexer.UTF8Passthrough}, // the pre-validation baseline + {"strict", lexer.UTF8Strict}, // the new default + {"replace", lexer.UTF8Replace}, + } + + for _, wl := range suite { + data := wl.Data + b.Run(wl.Name, func(b *testing.B) { + for _, m := range modes { + b.Run("L/"+m.name, func(b *testing.B) { + var sink int + lx := lexer.NewWithBytes(data, lexer.WithUTF8Policy(m.policy)) + b.SetBytes(int64(len(data))) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + lx.ResetWithBytes(data) + for t := range lx.Tokens() { + sink += int(t.Kind()) + } + } + fmt.Fprint(io.Discard, sink) + }) + b.Run("VL/"+m.name, func(b *testing.B) { + var sink int + lx := lexer.NewVerbatimWithBytes(data, lexer.WithUTF8Policy(m.policy)) + b.SetBytes(int64(len(data))) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + lx.ResetWithBytes(data) + for t := range lx.Tokens() { + sink += int(t.Kind()) + } + } + fmt.Fprint(io.Discard, sink) + }) + } + }) + } +} + +// TestCorpusStringProfile reports, per workload, how much of the payload is string +// bytes and what share of those strings carry a non-ASCII byte -- i.e. how many +// values a fused detector would hand to the validator. +func TestCorpusStringProfile(t *testing.T) { + suite, err := workloads.Corpus() + if err != nil { + t.Fatal(err) + } + + for _, wl := range suite { + var ( + strings, nonASCII, strBytes, nonASCIIBytes int + longStrings, longNonASCII int + ) + lx := lexer.NewWithBytes(wl.Data) + for tok := range lx.Tokens() { + if tok.Kind() != token.String && tok.Kind() != token.Key { + continue + } + v := tok.Value() + strings++ + strBytes += len(v) + if len(v) >= 16 { + longStrings++ + } + if !isASCII(v) { + nonASCII++ + nonASCIIBytes += len(v) + if len(v) >= 16 { + longNonASCII++ + } + } + if !utf8.Valid(v) { + t.Errorf("%s: corpus carries invalid utf8", wl.Name) + } + } + if !lx.Ok() { + t.Fatalf("%s: %v", wl.Name, lx.Err()) + } + t.Logf( + "%-20s bytes=%d strings=%d (%d long) strBytes=%d (%.1f%% of payload) nonASCII=%d (%.2f%% of strings, %d long) nonASCIIBytes=%d", + wl.Name, + len(wl.Data), + strings, + longStrings, + strBytes, + 100*float64(strBytes)/float64(len(wl.Data)), + nonASCII, + 100*float64(nonASCII)/float64(max(strings, 1)), + longNonASCII, + nonASCIIBytes, + ) + } +} + +func isASCII(b []byte) bool { + for _, c := range b { + if c >= 0x80 { + return false + } + } + + return true +} diff --git a/json/internal/utf8x/_asm/asm.go b/json/internal/utf8x/_asm/asm.go new file mode 100644 index 0000000..8ab6259 --- /dev/null +++ b/json/internal/utf8x/_asm/asm.go @@ -0,0 +1,195 @@ +//nolint:revive,mnd +package main + +import ( + "github.com/mmcloughlin/avo/attr" + . "github.com/mmcloughlin/avo/build" + . "github.com/mmcloughlin/avo/operand" +) + +func main() { + validateUTF8() + + Generate() +} + +// table16 emits a 16-byte read-only lookup table for VPSHUFB. +// +// VPSHUFB indexes within each 128-bit lane, so the table is broadcast into both lanes of a YMM at load time +// (VBROADCASTI128) and one 16-byte copy is all that needs to live in rodata. +func table16(name string, b ...byte) Mem { + if len(b) != 16 { + panic(name + ": a VPSHUFB table must be exactly 16 bytes") + } + m := GLOBL(name, attr.RODATA|attr.NOPTR) + for i, v := range b { + DATA(i, U8(v)) + } + + return m +} + +// UTF-8 error classes of the Keiser-Lemire "lookup4" algorithm (simdutf +// src/generic/utf8_validation/utf8_lookup4_algorithm.h). Each is one bit of a per-byte bitset; a byte is ill-formed +// iff its three table lookups agree on at least one violated rule. +const ( + tooShort = 1 << 0 // 11______ followed by 0_______ or 11______ + tooLong = 1 << 1 // 0_______ followed by 10______ + overlong3 = 1 << 2 // 11100000 100_____ + tooLarge = 1 << 3 // above U+10FFFF + surrogate = 1 << 4 // 11101101 101_____ : an encoded UTF-16 surrogate + overlong2 = 1 << 5 // 1100000_ 10______ + tooLarge1000 = 1 << 6 + overlong4 = 1 << 6 // 11110000 1000____ (shares the bit with tooLarge1000) + twoConts = 1 << 7 // 10______ 10______ + + // carry marks the classes that depend only on the high nibble of the leading byte, so the low-nibble table must + // let them through. + carry = tooShort | tooLong | twoConts +) + +// validateUTF8 emits the AVX2 UTF-8 validator: three VPSHUFB table lookups classify every (previous byte, byte) pair, +// and a saturating-subtract pass asserts that positions requiring a 2nd or 3rd continuation byte have one. +// +// It deliberately does NOT handle a partial trailing block or the end-of-input "sequence left incomplete" check: the +// Go caller re-validates the last few bytes scalar-ly from a sequence boundary (see ScanUTF8), which is both simpler +// and how simdutf itself locates errors. +func validateUTF8() { + TEXT("validateUTF8BlocksAVX2", NOSPLIT, "func(data []byte) bool") + Doc( + "validateUTF8BlocksAVX2 reports whether the whole 32-byte blocks of data are valid UTF-8, ignoring any sequence that continues past the end. len(data) must be a non-zero multiple of 32. AVX2, 32 bytes/iter.", + ) + ptr := Load(Param("data").Base(), GP64()) + n := Load(Param("data").Len(), GP64()) + + // byte_1_high: indexed by the HIGH nibble of the previous byte. + tableHigh := YMM() + VBROADCASTI128(table16( + "utf8t1h", + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, + tooLong, // 0_______ : ASCII lead + twoConts, + twoConts, + twoConts, + twoConts, // 10______ : continuation + tooShort|overlong2, // 1100____ + tooShort, // 1101____ + tooShort|overlong3|surrogate, // 1110____ + tooShort|tooLarge|tooLarge1000|overlong4, // 1111____ + ), tableHigh) + + // byte_1_low: indexed by the LOW nibble of the previous byte. + tableLow := YMM() + VBROADCASTI128(table16("utf8t1l", + carry|overlong3|overlong2|overlong4, // ____0000 + carry|overlong2, // ____0001 + carry, carry, // ____001_ + carry|tooLarge, // ____0100 + carry|tooLarge|tooLarge1000, // ____0101 + carry|tooLarge|tooLarge1000, // ____011_ + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, // ____1___ + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000|surrogate, // ____1101 + carry|tooLarge|tooLarge1000, + carry|tooLarge|tooLarge1000, + ), tableLow) + + // byte_2_high: indexed by the HIGH nibble of the current byte. + tableCur := YMM() + VBROADCASTI128(table16("utf8t2h", + tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, // 0_______ + tooLong|overlong2|twoConts|overlong3|tooLarge1000|overlong4, // 1000____ + tooLong|overlong2|twoConts|overlong3|tooLarge, // 1001____ + tooLong|overlong2|twoConts|surrogate|tooLarge, // 101_____ + tooLong|overlong2|twoConts|surrogate|tooLarge, + tooShort, tooShort, tooShort, tooShort, // 11______ + ), tableCur) + + nibble := YMM() + VPBROADCASTB(ConstData("utf8n", U8(0x0f)), nibble) + high := YMM() + VPBROADCASTB(ConstData("utf8h", U8(0x80)), high) + // 0xe0-0x80 and 0xf0-0x80: a saturating subtract leaves the high bit set exactly where a 3rd (resp. 4th) byte of a + // sequence is required. + third := YMM() + VPBROADCASTB(ConstData("utf8c3", U8(0xe0-0x80)), third) + fourth := YMM() + VPBROADCASTB(ConstData("utf8c4", U8(0xf0-0x80)), fourth) + + // prev holds the previous block; it starts as zeros, which is the correct "nothing precedes the input" state + // (NUL is ASCII, so it imposes no continuation). + prev := YMM() + VPXOR(prev, prev, prev) + errAcc := YMM() + VPXOR(errAcc, errAcc, errAcc) + + i := GP64() + XORQ(i, i) + + Label("utf8loop") + input := YMM() + VMOVDQU(Mem{Base: ptr, Index: i, Scale: 1}, input) + + // The three "previous byte" views. All three shift the same concatenation of the previous block's high lane with + // this block's low lane, so the permute is computed once. + perm := YMM() + VPERM2I128(Imm(0x21), input, prev, perm) + prev1, prev2, prev3 := YMM(), YMM(), YMM() + VPALIGNR(Imm(15), perm, input, prev1) + VPALIGNR(Imm(14), perm, input, prev2) + VPALIGNR(Imm(13), perm, input, prev3) + + // check_special_cases: AND of the three lookups leaves a bit set only where every view agrees the pair is illegal. + idx := YMM() + VPSRLW(Imm(4), prev1, idx) + VPAND(nibble, idx, idx) + b1h := YMM() + VPSHUFB(idx, tableHigh, b1h) + + VPAND(nibble, prev1, idx) + b1l := YMM() + VPSHUFB(idx, tableLow, b1l) + + VPSRLW(Imm(4), input, idx) + VPAND(nibble, idx, idx) + b2h := YMM() + VPSHUFB(idx, tableCur, b2h) + + sc := YMM() + VPAND(b1h, b1l, sc) + VPAND(b2h, sc, sc) + + // check_multibyte_lengths: a position two bytes after a 3-byte lead, or three after a 4-byte lead, MUST be a + // continuation. XOR-ing that requirement against the special-case bits leaves a set bit exactly where the two + // disagree — i.e. a missing or unexpected continuation. + must := YMM() + VPSUBUSB(third, prev2, must) + tmp := YMM() + VPSUBUSB(fourth, prev3, tmp) + VPOR(tmp, must, must) + VPAND(high, must, must) + VPXOR(sc, must, must) + VPOR(must, errAcc, errAcc) + + VMOVDQA(input, prev) + ADDQ(Imm(32), i) + CMPQ(i, n) + JB(LabelRef("utf8loop")) + + ok := GP8() + VPTEST(errAcc, errAcc) + SETEQ(ok) // ZF is set when the accumulator is all zero: no error was ever flagged + Store(ok, ReturnIndex(0)) + VZEROUPPER() + RET() +} diff --git a/json/internal/utf8x/_asm/doc.go b/json/internal/utf8x/_asm/doc.go new file mode 100644 index 0000000..6fac4a5 --- /dev/null +++ b/json/internal/utf8x/_asm/doc.go @@ -0,0 +1,7 @@ +// Package _asm is a generator-only module for the AVX2 UTF-8 validation kernel. +// +// It isolates the avo build-time dependency: only `go generate` (see ../validate_amd64.go) runs it, and the generated +// ../validate_amd64.s has no avo import, so the parent json module never pulls avo. +// +// Lives under a `_`-prefixed directory so the go tool ignores it when listing/building the parent module. +package main diff --git a/json/internal/utf8x/_asm/go.mod b/json/internal/utf8x/_asm/go.mod new file mode 100644 index 0000000..345f343 --- /dev/null +++ b/json/internal/utf8x/_asm/go.mod @@ -0,0 +1,11 @@ +module github.com/go-openapi/core/json/internal/utf8x/_asm + +go 1.25.8 + +require github.com/mmcloughlin/avo v0.6.1-0.20260701065415-16419356370f + +require ( + golang.org/x/mod v0.38.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/tools v0.48.0 // indirect +) diff --git a/json/internal/utf8x/_asm/go.sum b/json/internal/utf8x/_asm/go.sum new file mode 100644 index 0000000..397ad9f --- /dev/null +++ b/json/internal/utf8x/_asm/go.sum @@ -0,0 +1,10 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/mmcloughlin/avo v0.6.1-0.20260701065415-16419356370f h1:eLpJhGSyRTsY+l0MFH5zGdcm15rwu7Cm48S257NMHRQ= +github.com/mmcloughlin/avo v0.6.1-0.20260701065415-16419356370f/go.mod h1:+Pk+j3KceMwAIGb32HZSCvkMCzJvxy6xCRk+urz7iPw= +golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= +golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= +golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= diff --git a/json/internal/utf8x/bench_test.go b/json/internal/utf8x/bench_test.go new file mode 100644 index 0000000..27a888d --- /dev/null +++ b/json/internal/utf8x/bench_test.go @@ -0,0 +1,55 @@ +package utf8x + +import ( + "bytes" + "fmt" + "io" + "testing" + "unicode/utf8" +) + +// BenchmarkValid compares this package's validator against the stdlib across the value sizes the lexer actually hands +// it, so the AVX2 length gate (avx2Min) can be set from measurement rather than guesswork. +// +// The lexer only calls in here for values that already contain a byte >= 0x80 (detection is fused into the string +// scan), so the interesting inputs are non-ASCII text, not ASCII. +func BenchmarkValid(b *testing.B) { + corpora := []struct { + name string + unit string + }{ + // a Latin word with accents: mostly ASCII with a sprinkle of 2-byte sequences, the common European shape + {name: "latin", unit: "les donn\u00e9es d'entr\u00e9e sont valid\u00e9es "}, + // CJK: dense 3-byte sequences, the twitter_status shape + {name: "cjk", unit: "\u65e5\u672c\u8a9e\u306e\u30c6\u30ad\u30b9\u30c8 "}, + // emoji: 4-byte sequences + {name: "emoji", unit: "\U0001F600\U0001F601\U0001F602 "}, + } + sizes := []int{16, 32, 48, 64, 96, 128, 256, 512, 1024, 4096} + + var sink bool + for _, c := range corpora { + for _, size := range sizes { + data := bytes.Repeat([]byte(c.unit), size/len(c.unit)+1)[:size] + // never cut a multi-byte sequence: the benchmark must measure valid input + for len(data) > 0 && !utf8.Valid(data) { + data = data[:len(data)-1] + } + + b.Run(fmt.Sprintf("%s/%d/utf8x", c.name, size), func(b *testing.B) { + b.SetBytes(int64(len(data))) + for b.Loop() { + sink = Valid(data) + } + }) + b.Run(fmt.Sprintf("%s/%d/stdlib", c.name, size), func(b *testing.B) { + b.SetBytes(int64(len(data))) + for b.Loop() { + sink = utf8.Valid(data) + } + }) + } + } + + fmt.Fprint(io.Discard, sink) +} diff --git a/json/internal/utf8x/cpuid_amd64.s b/json/internal/utf8x/cpuid_amd64.s new file mode 100644 index 0000000..1da73f9 --- /dev/null +++ b/json/internal/utf8x/cpuid_amd64.s @@ -0,0 +1,20 @@ +#include "textflag.h" + +// func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) +TEXT ·cpuid(SB), NOSPLIT, $0-24 + MOVL eaxArg+0(FP), AX + MOVL ecxArg+4(FP), CX + CPUID + MOVL AX, eax+8(FP) + MOVL BX, ebx+12(FP) + MOVL CX, ecx+16(FP) + MOVL DX, edx+20(FP) + RET + +// func xgetbv() (eax, edx uint32) +TEXT ·xgetbv(SB), NOSPLIT, $0-8 + MOVL $0, CX + XGETBV + MOVL AX, eax+0(FP) + MOVL DX, edx+4(FP) + RET diff --git a/json/internal/utf8x/detect_amd64.go b/json/internal/utf8x/detect_amd64.go new file mode 100644 index 0000000..f5020c1 --- /dev/null +++ b/json/internal/utf8x/detect_amd64.go @@ -0,0 +1,43 @@ +//go:build amd64 + +package utf8x + +// cpuid executes CPUID with the given leaf (EAX) and subleaf (ECX) and returns the four result registers. +// +// Implemented in cpuid_amd64.s. +func cpuid(eaxArg, ecxArg uint32) (eax, ebx, ecx, edx uint32) + +// xgetbv reads the extended control register XCR0 (ECX=0) and returns its low and high halves. +// +// Implemented in cpuid_amd64.s. Only call after CPUID reports OSXSAVE, or XGETBV itself faults. +func xgetbv() (eax, edx uint32) + +// useAVX2 records, once at package init, whether this CPU supports AVX2 with the OS having enabled YMM state. +// +// This is the precondition for calling validateUTF8BlocksAVX2. +// On any machine that fails the check, [Valid] stays on the stdlib scalar path and the AVX2 instructions are never +// executed. +var useAVX2 = detectAVX2() //nolint:gochecknoglobals // private immutable runtime environment detection + +//nolint:mnd // avx offsets +func detectAVX2() bool { + const ( + osxsave = uint32(1) << 27 // CPUID.1:ECX.OSXSAVE — OS enabled XSAVE/XGETBV + avx = uint32(1) << 28 // CPUID.1:ECX.AVX + avx2bit = uint32(1) << 5 // CPUID.(7,0):EBX.AVX2 + xcr0YMM = uint32(0x6) // XCR0 bits 1 (XMM) + 2 (YMM) must both be set + ) + + if maxLeaf, _, _, _ := cpuid(0, 0); maxLeaf < 7 { + return false + } + if _, _, ecx1, _ := cpuid(1, 0); ecx1&osxsave == 0 || ecx1&avx == 0 { + return false + } + if xcr0, _ := xgetbv(); xcr0&xcr0YMM != xcr0YMM { + return false + } + _, ebx7, _, _ := cpuid(7, 0) //nolint:dogsled // we need 4 registers + + return ebx7&avx2bit != 0 +} diff --git a/json/internal/utf8x/utf8x.go b/json/internal/utf8x/utf8x.go new file mode 100644 index 0000000..3c3f61e --- /dev/null +++ b/json/internal/utf8x/utf8x.go @@ -0,0 +1,145 @@ +// Package utf8x holds the UTF-8 validation and sanitization primitives shared by the JSON lexers and writers. +// +// It lives under json/internal (rather than under a lexer- or writer-private internal package) because it is the only +// location both subtrees can import: the lexers validate what they read, the writers validate what they emit, and both +// must agree byte-for-byte on what "invalid" means and on what an invalid sequence is replaced with. +// +// The functions here are deliberately whole-buffer: the lexers detect the *presence* of non-ASCII inline, fused into +// their existing stop-byte scan (see the swar package), and only call in here for the values that actually carry a byte +// >= 0x80. So [Valid] is not on the hot path for the overwhelmingly common all-ASCII value. +package utf8x + +import "unicode/utf8" + +const ( + // replacement is the UTF-8 encoding of U+FFFD, the substitute for an invalid byte. + replacement = string(utf8.RuneError) + + // replacementWidth is its encoded width (3). + replacementWidth = len(replacement) +) + +// What "valid" means here is exactly [utf8.Valid]'s definition, which is RFC 3629 plus Unicode: truncated sequences, +// overlong encodings, lead/continuation mismatches, encoded surrogates (U+D800..U+DFFF) and anything above U+10FFFF +// are all invalid. WTF-8 and CESU-8 are not accepted. [Valid] itself is platform-specific (see validate_amd64.go). + +// FirstInvalid returns the index of the first byte of the first ill-formed sequence in b, or -1 if b is valid UTF-8. +// +// It is the cold, error-path counterpart of [Valid]: callers use [Valid] for the verdict and only come here to report +// *where* the input went wrong. Keeping the position search separate is what lets [Valid] stay a bulk scan (the same +// split simdutf makes between its SIMD checker and rewind_and_validate_with_errors). +func FirstInvalid(b []byte) int { + for i := 0; i < len(b); { + if b[i] < utf8.RuneSelf { + i++ + + continue + } + + r, size := utf8.DecodeRune(b[i:]) + if r == utf8.RuneError && size <= 1 { + return i + } + i += size + } + + return -1 +} + +// Sanitize appends src to dst, substituting U+FFFD for every invalid byte, and returns the extended slice. +// +// Granularity is **one replacement per invalid byte** — [utf8.DecodeRune]'s error advance — so "\xe0\xa0\xff" yields +// three replacement runes. That is what ranging over a Go string produces, and what the JSON writer's escaper already +// did before this package existed, so the lexer, the writer and caller-side string conversion all agree. +// +// A valid src is appended in one bulk copy with no per-rune work. +func Sanitize(dst, src []byte) []byte { + i := FirstInvalid(src) + if i < 0 { + return append(dst, src...) + } + + dst = append(dst, src[:i]...) + + for i < len(src) { + if src[i] < utf8.RuneSelf { + dst = append(dst, src[i]) + i++ + + continue + } + + r, size := utf8.DecodeRune(src[i:]) + if r == utf8.RuneError && size <= 1 { + dst = append(dst, replacement...) + i++ + + continue + } + dst = append(dst, src[i:i+size]...) + i += size + } + + return dst +} + +// SanitizedLen returns the length Sanitize would produce for src, or -1 if src is valid UTF-8 (nothing to do). +// +// It lets a caller enforce a size cap before committing to the rewrite, since replacing a 1-byte invalid sequence with +// U+FFFD grows the value. +func SanitizedLen(src []byte) int { + i := FirstInvalid(src) + if i < 0 { + return -1 + } + + n := i + for i < len(src) { + if src[i] < utf8.RuneSelf { + n++ + i++ + + continue + } + + r, size := utf8.DecodeRune(src[i:]) + if r == utf8.RuneError && size <= 1 { + n += replacementWidth + i++ + + continue + } + n += size + i += size + } + + return n +} + +// Policy selects what a lexer or writer does with data that cannot be represented as a sequence of Unicode scalar +// values: an ill-formed UTF-8 byte sequence, or a \u escape that does not denote one. +// +// It lives here, alongside the validator, so the lexers and the writers share one definition and cannot drift apart — +// a document rejected on the way in and one refused on the way out must mean the same thing. Both packages alias it +// (lexer.UTF8Policy, writer.UTF8Policy), so callers never name this internal package. +// +// The zero value is [PolicyStrict]: validation is on unless a caller opts out. +type Policy uint8 + +const ( + // PolicyStrict rejects the data. This is the default on both sides. + PolicyStrict Policy = iota + + // PolicyReplace substitutes U+FFFD — one per invalid byte, one per broken escape — so what is emitted is always + // valid UTF-8. See [Sanitize] for the exact granularity. + PolicyReplace + + // PolicyPassthrough skips raw-byte validation entirely, letting ill-formed bytes through untouched. + // + // UNSAFE: only for data already known to be valid UTF-8. A \u escape still yields U+FFFD when broken, because an + // escape must always produce some rune. + PolicyPassthrough +) + +// Validates reports whether the policy inspects raw bytes at all. +func (p Policy) Validates() bool { return p != PolicyPassthrough } diff --git a/json/internal/utf8x/utf8x_test.go b/json/internal/utf8x/utf8x_test.go new file mode 100644 index 0000000..330244b --- /dev/null +++ b/json/internal/utf8x/utf8x_test.go @@ -0,0 +1,196 @@ +package utf8x + +import ( + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" +) + +// invalidSamples are the ill-formed byte sequences the JSON lexers used to accept silently — the ten string bodies of +// JSONTestSuite's i_string_* UTF-8 cases, plus a few shapes those do not cover. +// +// firstBad is the index of the first byte of the first ill-formed sequence. +var invalidSamples = []struct { //nolint:gochecknoglobals // shared by several tests + name string + in []byte + firstBad int + // want is the sanitized form: one U+FFFD per invalid byte (see [Sanitize]). + want string +}{ + {name: "lone ff", in: []byte{0xff}, firstBad: 0, want: "�"}, + {name: "lone continuation", in: []byte{0x81}, firstBad: 0, want: "�"}, + {name: "iso latin-1", in: []byte{0xe9}, firstBad: 0, want: "�"}, + {name: "truncated 3-byte", in: []byte{0xe0, 0xff}, firstBad: 0, want: "��"}, + { + name: "valid prefix then invalid lead", + in: []byte{0xe6, 0x97, 0xa5, 0xd1, 0x88, 0xfa}, + firstBad: 5, + want: "\u65e5\u0448\ufffd", + }, + {name: "encoded surrogate U+D800", in: []byte{0xed, 0xa0, 0x80}, firstBad: 0, want: "���"}, + {name: "beyond U+10FFFF", in: []byte{0xf4, 0xbf, 0xbf, 0xbf}, firstBad: 0, want: "����"}, + {name: "overlong 2 bytes", in: []byte{0xc0, 0xaf}, firstBad: 0, want: "��"}, + { + name: "obsolete 6-byte form", + in: []byte{0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf}, + firstBad: 0, + want: "������", + }, + { + name: "overlong NUL, 6 bytes", + in: []byte{0xfc, 0x80, 0x80, 0x80, 0x80, 0x80}, + firstBad: 0, + want: "������", + }, + // shapes the conformance corpus does not carry + {name: "truncated at end", in: []byte("ok\xe2\x82"), firstBad: 2, want: "ok��"}, + { + name: "invalid between valid", + in: []byte("a\xffbéc"), + firstBad: 1, + want: "a�béc", + }, + { + // the maximal-subpart rule would give 2 replacements here; we deliberately give 3 (one per byte). + name: "partial 3-byte then invalid", + in: []byte{0xe0, 0xa0, 0xff}, + firstBad: 0, + want: "���", + }, +} + +var validSamples = []struct { //nolint:gochecknoglobals // shared by several tests + name string + in []byte +}{ + {name: "empty", in: []byte{}}, + {name: "ascii", in: []byte("the quick brown fox")}, + {name: "2-byte", in: []byte("café naïve")}, + {name: "3-byte", in: []byte("\u65e5\u672c\u8a9e")}, + {name: "4-byte", in: []byte("\U0001D11E\U0001F600")}, + {name: "encoded U+FFFD", in: []byte("�")}, // a legitimate replacement char is valid input + {name: "max scalar", in: []byte("\U0010FFFF")}, + {name: "NUL and DEL", in: []byte{0x00, 0x7f}}, + {name: "BOM as content", in: []byte("\uFEFF")}, +} + +func TestValid(t *testing.T) { + t.Parallel() + + for _, tc := range validSamples { + t.Run("valid/"+tc.name, func(t *testing.T) { + t.Parallel() + assert.True(t, Valid(tc.in)) + assert.Equal(t, -1, FirstInvalid(tc.in)) + }) + } + + for _, tc := range invalidSamples { + t.Run("invalid/"+tc.name, func(t *testing.T) { + t.Parallel() + assert.False(t, Valid(tc.in)) + assert.Equal(t, tc.firstBad, FirstInvalid(tc.in)) + }) + } +} + +func TestSanitize(t *testing.T) { + t.Parallel() + + t.Run("valid input is appended verbatim", func(t *testing.T) { + t.Parallel() + for _, tc := range validSamples { + assert.Equal(t, string(tc.in), string(Sanitize(nil, tc.in))) + assert.Equal(t, -1, SanitizedLen(tc.in)) + } + }) + + t.Run("invalid input is replaced per byte", func(t *testing.T) { + t.Parallel() + for _, tc := range invalidSamples { + got := Sanitize(nil, tc.in) + assert.Equalf(t, tc.want, string(got), "sanitizing %q", tc.in) + assert.Truef(t, utf8.Valid(got), "sanitized output must be valid: %q", got) + assert.Equalf( + t, + len(got), + SanitizedLen(tc.in), + "SanitizedLen must predict the rewrite of %q", + tc.in, + ) + } + }) + + t.Run("appends to an existing buffer", func(t *testing.T) { + t.Parallel() + dst := []byte("prefix:") + assert.Equal(t, "prefix:a�b", string(Sanitize(dst, []byte("a\xffb")))) + }) + + t.Run("is idempotent", func(t *testing.T) { + t.Parallel() + for _, tc := range invalidSamples { + once := Sanitize(nil, tc.in) + assert.Equal(t, string(once), string(Sanitize(nil, once))) + } + }) +} + +// TestSanitizeMatchesWriterRule pins the granularity choice against the rule Go itself applies when a caller ranges +// over a string — the reason we replace per byte rather than per maximal subpart. +func TestSanitizeMatchesWriterRule(t *testing.T) { + t.Parallel() + + for _, tc := range invalidSamples { + var want []byte + for _, r := range string(tc.in) { // range over a string yields U+FFFD per invalid byte + want = utf8.AppendRune(want, r) + } + assert.Equalf(t, string(want), string(Sanitize(nil, tc.in)), + "Sanitize must agree with range-over-string for %q", tc.in) + } +} + +// FuzzValid pins the two invariants the whole design rests on: our verdict is the stdlib's verdict, and a sanitized +// value is always valid UTF-8. When the AVX2 kernel replaces the scalar Valid, this is the test that keeps it honest. +func FuzzValid(f *testing.F) { + for _, tc := range validSamples { + f.Add(tc.in) + } + for _, tc := range invalidSamples { + f.Add(tc.in) + } + f.Add([]byte("\xf0\x9f\x98\x80 mixed \xc3 tail")) + + f.Fuzz(func(t *testing.T, in []byte) { + require.Equal(t, utf8.Valid(in), Valid(in)) + + idx := FirstInvalid(in) + require.Equal(t, utf8.Valid(in), idx < 0) + if idx >= 0 { + require.True( + t, + utf8.Valid(in[:idx]), + "the prefix before the first fault must itself be valid", + ) + } + + out := Sanitize(nil, in) + require.True(t, utf8.Valid(out), "sanitized output must always be valid UTF-8") + require.Equal(t, string(out), string(Sanitize(nil, out)), "sanitizing must be idempotent") + + if idx < 0 { + require.Equal( + t, + string(in), + string(out), + "valid input must be passed through unchanged", + ) + require.Equal(t, -1, SanitizedLen(in)) + } else { + require.Equal(t, len(out), SanitizedLen(in)) + } + }) +} diff --git a/json/internal/utf8x/validate_amd64.go b/json/internal/utf8x/validate_amd64.go new file mode 100644 index 0000000..8f4e355 --- /dev/null +++ b/json/internal/utf8x/validate_amd64.go @@ -0,0 +1,85 @@ +//go:build amd64 + +package utf8x + +import "unicode/utf8" + +//go:generate sh -c "cd _asm && go run . -out ../validate_amd64.s" + +// validateUTF8BlocksAVX2 is the avo-generated AVX2 kernel (validate_amd64.s): a port of the Keiser-Lemire "lookup4" +// algorithm simdutf uses (src/generic/utf8_validation/utf8_lookup4_algorithm.h), 32 bytes per YMM iteration. +// +// Three VPSHUFB table lookups classify every (previous byte, byte) pair — the AND of the three leaves a bit set only +// where all three agree the pair violates the same rule (too short, too long, overlong, surrogate, too large, two +// continuations) — and a saturating-subtract pass asserts that a position which must carry the 2nd or 3rd +// continuation of a sequence actually does. +// +// It validates only whole 32-byte blocks and deliberately says nothing about a sequence that continues past the end: +// errors are detected block-wise, exactly as in simdutf, and the seam is resolved scalar-ly by [Valid]. len(data) must +// be a non-zero multiple of 32, and the CPU must support AVX2 ([useAVX2]). +func validateUTF8BlocksAVX2(data []byte) bool + +// avx2Min is the length below which the vector kernel cannot pay for its constant setup (7 broadcasts) plus the +// non-inlinable call. It is also the kernel's hard minimum: it processes whole 32-byte blocks and needs at least one. +// +// Set from BenchmarkValid: at 32 bytes and up the kernel already wins (latin/32: 6.7ns vs the stdlib's 18.1ns), and +// the win grows to 5-12x by 128 bytes. End-to-end on the corpus the 32-vs-64 choice is not distinguishable, so the +// micro-benchmark decides. +// +// The validator only ever runs on values already known to contain a byte >= 0x80 — detection is fused into the string +// scan — so the inputs reaching it are non-ASCII text, which is what BenchmarkValid measures. +const avx2Min = blockBytes + +// blockBytes is the kernel's stride: one YMM register. +const blockBytes = 32 + +// blockMask rounds a length down to a whole number of kernel blocks. +const blockMask = ^(blockBytes - 1) + +// maxContinuations is the longest run of continuation bytes a well-formed sequence can have (a 4-byte sequence has +// three). It bounds how far [Valid] rewinds to find the seam. +const maxContinuations = 3 + +// Valid reports whether b is entirely valid UTF-8. +// +// On amd64 with AVX2 it runs the vector kernel over the whole 32-byte blocks and finishes with the stdlib scalar +// validator over the trailing bytes, rewound to the start of the last sequence so a sequence straddling that boundary +// — or truncated at the very end, which the block kernel does not check — is fully validated exactly once. +func Valid(b []byte) bool { + if !useAVX2 || len(b) < avx2Min { + return utf8.Valid(b) + } + + blocks := len(b) & blockMask + if !validateUTF8BlocksAVX2(b[:blocks]) { + return false + } + + return utf8.Valid(b[seam(b, blocks):]) +} + +// seam returns the index at which scalar validation must resume so that everything the block kernel could not check is +// covered, and everything before it was checked. +// +// The kernel validates a pair only when both its bytes are inside the block region, so the unchecked remainder is: the +// trailing bytes past the last whole block, plus any sequence that begins before the boundary and reaches it. Such a +// sequence starts at most three bytes back, so walking back over continuation bytes (bounded) and then taking the +// non-continuation byte that precedes them lands exactly on its first byte. +// +// The bound is not a heuristic: more than three continuation bytes in a row is itself ill-formed, and returning an +// index that lands mid-sequence makes the scalar pass reject — which is the correct answer for that input. +func seam(b []byte, blocks int) int { + start := blocks + for range maxContinuations { + if start == 0 || b[start-1]&0xC0 != 0x80 { + break + } + start-- + } + + if start > 0 { + start-- // include the byte that leads the sequence (or is an ASCII byte of its own) + } + + return start +} diff --git a/json/internal/utf8x/validate_amd64.s b/json/internal/utf8x/validate_amd64.s new file mode 100644 index 0000000..031ff80 --- /dev/null +++ b/json/internal/utf8x/validate_amd64.s @@ -0,0 +1,117 @@ +// Code generated by command: go run asm.go -out ../validate_amd64.s. DO NOT EDIT. + +#include "textflag.h" + +// func validateUTF8BlocksAVX2(data []byte) bool +// Requires: AVX, AVX2 +TEXT ·validateUTF8BlocksAVX2(SB), NOSPLIT, $0-25 + MOVQ data_base+0(FP), AX + MOVQ data_len+8(FP), CX + VBROADCASTI128 utf8t1h<>+0(SB), Y0 + VBROADCASTI128 utf8t1l<>+0(SB), Y1 + VBROADCASTI128 utf8t2h<>+0(SB), Y2 + VPBROADCASTB utf8n<>+0(SB), Y3 + VPBROADCASTB utf8h<>+0(SB), Y4 + VPBROADCASTB utf8c3<>+0(SB), Y5 + VPBROADCASTB utf8c4<>+0(SB), Y6 + VPXOR Y7, Y7, Y7 + VPXOR Y8, Y8, Y8 + XORQ DX, DX + +utf8loop: + VMOVDQU (AX)(DX*1), Y9 + VPERM2I128 $0x21, Y9, Y7, Y7 + VPALIGNR $0x0f, Y7, Y9, Y10 + VPALIGNR $0x0e, Y7, Y9, Y11 + VPALIGNR $0x0d, Y7, Y9, Y7 + VPSRLW $0x04, Y10, Y12 + VPAND Y3, Y12, Y12 + VPSHUFB Y12, Y0, Y13 + VPAND Y3, Y10, Y12 + VPSHUFB Y12, Y1, Y10 + VPSRLW $0x04, Y9, Y12 + VPAND Y3, Y12, Y12 + VPSHUFB Y12, Y2, Y12 + VPAND Y13, Y10, Y10 + VPAND Y12, Y10, Y10 + VPSUBUSB Y5, Y11, Y11 + VPSUBUSB Y6, Y7, Y7 + VPOR Y7, Y11, Y11 + VPAND Y4, Y11, Y11 + VPXOR Y10, Y11, Y11 + VPOR Y11, Y8, Y8 + VMOVDQA Y9, Y7 + ADDQ $0x20, DX + CMPQ DX, CX + JB utf8loop + VPTEST Y8, Y8 + SETEQ AL + MOVB AL, ret+24(FP) + VZEROUPPER + RET + +DATA utf8t1h<>+0(SB)/1, $0x02 +DATA utf8t1h<>+1(SB)/1, $0x02 +DATA utf8t1h<>+2(SB)/1, $0x02 +DATA utf8t1h<>+3(SB)/1, $0x02 +DATA utf8t1h<>+4(SB)/1, $0x02 +DATA utf8t1h<>+5(SB)/1, $0x02 +DATA utf8t1h<>+6(SB)/1, $0x02 +DATA utf8t1h<>+7(SB)/1, $0x02 +DATA utf8t1h<>+8(SB)/1, $0x80 +DATA utf8t1h<>+9(SB)/1, $0x80 +DATA utf8t1h<>+10(SB)/1, $0x80 +DATA utf8t1h<>+11(SB)/1, $0x80 +DATA utf8t1h<>+12(SB)/1, $0x21 +DATA utf8t1h<>+13(SB)/1, $0x01 +DATA utf8t1h<>+14(SB)/1, $0x15 +DATA utf8t1h<>+15(SB)/1, $0x49 +GLOBL utf8t1h<>(SB), RODATA|NOPTR, $16 + +DATA utf8t1l<>+0(SB)/1, $0xe7 +DATA utf8t1l<>+1(SB)/1, $0xa3 +DATA utf8t1l<>+2(SB)/1, $0x83 +DATA utf8t1l<>+3(SB)/1, $0x83 +DATA utf8t1l<>+4(SB)/1, $0x8b +DATA utf8t1l<>+5(SB)/1, $0xcb +DATA utf8t1l<>+6(SB)/1, $0xcb +DATA utf8t1l<>+7(SB)/1, $0xcb +DATA utf8t1l<>+8(SB)/1, $0xcb +DATA utf8t1l<>+9(SB)/1, $0xcb +DATA utf8t1l<>+10(SB)/1, $0xcb +DATA utf8t1l<>+11(SB)/1, $0xcb +DATA utf8t1l<>+12(SB)/1, $0xcb +DATA utf8t1l<>+13(SB)/1, $0xdb +DATA utf8t1l<>+14(SB)/1, $0xcb +DATA utf8t1l<>+15(SB)/1, $0xcb +GLOBL utf8t1l<>(SB), RODATA|NOPTR, $16 + +DATA utf8t2h<>+0(SB)/1, $0x01 +DATA utf8t2h<>+1(SB)/1, $0x01 +DATA utf8t2h<>+2(SB)/1, $0x01 +DATA utf8t2h<>+3(SB)/1, $0x01 +DATA utf8t2h<>+4(SB)/1, $0x01 +DATA utf8t2h<>+5(SB)/1, $0x01 +DATA utf8t2h<>+6(SB)/1, $0x01 +DATA utf8t2h<>+7(SB)/1, $0x01 +DATA utf8t2h<>+8(SB)/1, $0xe6 +DATA utf8t2h<>+9(SB)/1, $0xae +DATA utf8t2h<>+10(SB)/1, $0xba +DATA utf8t2h<>+11(SB)/1, $0xba +DATA utf8t2h<>+12(SB)/1, $0x01 +DATA utf8t2h<>+13(SB)/1, $0x01 +DATA utf8t2h<>+14(SB)/1, $0x01 +DATA utf8t2h<>+15(SB)/1, $0x01 +GLOBL utf8t2h<>(SB), RODATA|NOPTR, $16 + +DATA utf8n<>+0(SB)/1, $0x0f +GLOBL utf8n<>(SB), RODATA|NOPTR, $1 + +DATA utf8h<>+0(SB)/1, $0x80 +GLOBL utf8h<>(SB), RODATA|NOPTR, $1 + +DATA utf8c3<>+0(SB)/1, $0x60 +GLOBL utf8c3<>(SB), RODATA|NOPTR, $1 + +DATA utf8c4<>+0(SB)/1, $0x70 +GLOBL utf8c4<>(SB), RODATA|NOPTR, $1 diff --git a/json/internal/utf8x/validate_amd64_test.go b/json/internal/utf8x/validate_amd64_test.go new file mode 100644 index 0000000..483973c --- /dev/null +++ b/json/internal/utf8x/validate_amd64_test.go @@ -0,0 +1,285 @@ +//go:build amd64 + +package utf8x + +import ( + "bytes" + "math/rand" + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/require" +) + +// The AVX2 kernel is hand-written vector code whose failure mode is silence: a wrong table entry accepts an +// ill-formed sequence, and nothing else in the system would notice. So it is tested against the stdlib exhaustively +// where exhaustive is possible, and by construction everywhere else. +// +// Two properties are hammered specifically because they are where a lookup4 port goes wrong: +// +// - Block seams. State is carried between 32-byte blocks through prev1/prev2/prev3, so a sequence must be judged +// identically no matter which byte offset it starts at. Every case is therefore swept across all 32 alignments. +// - The tail. The kernel deliberately validates whole blocks only; [Valid] resolves the rest scalar-ly. An error +// that falls in the seam between the two is exactly what a naive split would miss. + +// requireAgrees asserts our verdict matches the stdlib's for b, and reports the input on failure. +func requireAgrees(t *testing.T, b []byte) { + t.Helper() + + if got, want := Valid(b), utf8.Valid(b); got != want { + t.Fatalf("Valid=%v, utf8.Valid=%v for % x (len=%d)", got, want, b, len(b)) + } +} + +// TestValidAgreesExhaustive2And3Bytes sweeps EVERY 2-byte sequence and every 3-byte sequence, embedded at every +// alignment within a buffer long enough to reach the vector path. +// +// This covers the whole classification table: all 256 lead bytes against all 256 second bytes is precisely what the +// three VPSHUFB lookups encode, so a single wrong table entry cannot survive it. +func TestValidAgreesExhaustive2And3Bytes(t *testing.T) { + const pad = 96 // > avx2Min, so the vector path runs + + t.Run("2 bytes", func(t *testing.T) { + buf := make([]byte, pad) + for hi := range 256 { + for lo := range 256 { + for _, at := range []int{0, 1, 30, 31, 32, 33, 62, 63, 64, pad - 2} { + for i := range buf { + buf[i] = 'a' + } + buf[at], buf[at+1] = byte(hi), byte(lo) + if got, want := Valid(buf), utf8.Valid(buf); got != want { + t.Fatalf( + "at=%d seq=%02x %02x: Valid=%v, utf8.Valid=%v", + at, + hi, + lo, + got, + want, + ) + } + } + } + } + }) + + // Every lead byte (C0..FF) against every possible 2nd and 3rd byte: 4.2M sequences. This is what exercises the + // continuation-count logic (prev2/prev3 saturating subtract) that the pairwise sweep above cannot reach. Leads + // below C0 are ASCII or stray continuations, both fully decided by the pairwise table. + t.Run("3 bytes after every lead", func(t *testing.T) { + if testing.Short() { + t.Skip("4.2M sequences x alignments") + } + buf := make([]byte, pad) + for i := range buf { + buf[i] = 'a' + } + for b0 := 0xC0; b0 <= 0xFF; b0++ { + for b1 := range 256 { + for b2 := range 256 { + for _, at := range []int{0, 31, 62} { + buf[at], buf[at+1], buf[at+2] = byte(b0), byte(b1), byte(b2) + if got, want := Valid(buf), utf8.Valid(buf); got != want { + t.Fatalf("at=%d seq=%02x %02x %02x: Valid=%v, utf8.Valid=%v", + at, b0, b1, b2, got, want) + } + buf[at], buf[at+1], buf[at+2] = 'a', 'a', 'a' + } + } + } + } + }) +} + +// TestKernelIsActuallyExercised guards the whole differential suite from becoming vacuous: if the CPU lacks AVX2, or +// the length gate keeps every fixture on the scalar path, every "agrees with the stdlib" test above would pass by +// comparing the stdlib against itself. +func TestKernelIsActuallyExercised(t *testing.T) { + if !useAVX2 { + t.Skip( + "no AVX2 on this CPU: Valid is the stdlib scalar validator and the kernel tests are vacuous", + ) + } + + t.Logf("AVX2 kernel active (avx2Min=%d)", avx2Min) + + // drive the kernel directly, so the result cannot come from the scalar fallback + valid := bytes.Repeat([]byte("a\u00e9\u65e5"), 16) // 96 bytes, a whole number of blocks + require.Zero(t, len(valid)%32) + require.True(t, validateUTF8BlocksAVX2(valid)) + + for _, bad := range [][]byte{ + {0xFF}, {0x80}, {0xC0, 0xAF}, {0xED, 0xA0, 0x80}, {0xF4, 0xBF, 0xBF, 0xBF}, {0xE0, 0x80, 0x80}, + } { + b := bytes.Repeat([]byte("a"), 96) + copy(b[40:], bad) + require.Falsef(t, validateUTF8BlocksAVX2(b), "the kernel must reject % x", bad) + } +} + +// TestValidAgreesFourByteLeads sweeps the 4-byte space where it actually discriminates: every lead in F0..F7 against +// every second byte, with the last two continuations both valid and invalid. This is where TOO_LARGE / TOO_LARGE_1000 +// / OVERLONG_4 live, and they are all decided by (lead, second byte). +func TestValidAgreesFourByteLeads(t *testing.T) { + const pad = 96 + tails := [][2]byte{{0x80, 0x80}, {0xBF, 0xBF}, {0x80, 0x41}, {0x41, 0x80}, {0xBF, 0xC0}} + + buf := make([]byte, pad) + for lead := 0xF0; lead <= 0xFF; lead++ { + for second := range 256 { + for _, tail := range tails { + for _, at := range []int{0, 29, 30, 31, 32, 60, 61, 62} { + for i := range buf { + buf[i] = 'a' + } + buf[at], buf[at+1] = byte(lead), byte(second) + buf[at+2], buf[at+3] = tail[0], tail[1] + if got, want := Valid(buf), utf8.Valid(buf); got != want { + t.Fatalf("at=%d seq=%02x %02x % x: Valid=%v, utf8.Valid=%v", + at, lead, second, tail, got, want) + } + } + } + } + } +} + +// TestValidAgreesAtEveryLength checks every length from 0 through several blocks, for a valid buffer and for one +// truncated mid-sequence at the very end — the case the block kernel cannot see and the scalar seam must catch. +func TestValidAgreesAtEveryLength(t *testing.T) { + base := bytes.Repeat( + []byte("a\u00e9\u65e5\U0001D11E"), + 64, + ) // 1-, 2-, 3- and 4-byte sequences, so every length cuts differently + + for n := range base { + requireAgrees(t, base[:n]) + } + + // a truncated sequence pinned at the end of buffers of every length + for _, trunc := range [][]byte{{0xC3}, {0xE6}, {0xE6, 0x97}, {0xF0}, {0xF0, 0x9D}, {0xF0, 0x9D, 0x84}} { + for n := range 200 { + b := append(bytes.Repeat([]byte("a"), n), trunc...) + requireAgrees(t, b) + require.Falsef( + t, + Valid(b), + "a sequence truncated at the end must be rejected (n=%d, % x)", + n, + trunc, + ) + } + } +} + +// TestValidAgreesOnSeamStraddle walks a valid multi-byte sequence one byte at a time across the last block boundary, +// where the vector pass hands over to the scalar one. +func TestValidAgreesOnSeamStraddle(t *testing.T) { + for _, seq := range []string{"\u00e9", "\u65e5", "\U0001D11E"} { + for total := 64; total < 160; total++ { + for at := total - len(seq) - 4; at <= total-len(seq); at++ { + if at < 0 { + continue + } + b := bytes.Repeat([]byte("a"), total) + copy(b[at:], seq) + requireAgrees(t, b) + require.Truef( + t, + Valid(b), + "a valid sequence at %d of %d must be accepted", + at, + total, + ) + } + } + } +} + +// TestValidAgreesRandom throws structured noise at it: mostly-valid text with sporadic corruption, which is what real +// ill-formed input looks like, plus pure random bytes. +func TestValidAgreesRandom(t *testing.T) { + //nolint:gosec // a deterministic PRNG is the point: reproducible test input, not cryptography + rng := rand.New(rand.NewSource(42)) + + alphabet := []string{"a", "\u00e9", "\u65e5", "\U0001D11E", "\x00", "\x7f"} + for range 20000 { + n := rng.Intn(300) + var b []byte + for len(b) < n { + b = append(b, alphabet[rng.Intn(len(alphabet))]...) + } + for range rng.Intn(4) { // corrupt a few bytes + if len(b) > 0 { + corrupt := rng.Intn( + 256, + ) //nolint:gosec // bounded to 0..255, so the conversion cannot overflow + b[rng.Intn(len(b))] = byte(corrupt) + } + } + requireAgrees(t, b) + } + + for range 20000 { + b := make([]byte, rng.Intn(300)) + _, _ = rng.Read(b) + requireAgrees(t, b) + } +} + +// TestSeamCoversEverythingTheKernelSkips pins the hand-off contract directly: the index [Valid] resumes scalar +// validation from must never be past the start of a sequence that the block kernel could not fully check. +func TestSeamCoversEverythingTheKernelSkips(t *testing.T) { + cases := []struct { + name string + b []byte + blocks int + want int + }{ + {name: "ascii before the boundary", b: bytes.Repeat([]byte("a"), 64), blocks: 64, want: 63}, + { + name: "3-byte sequence ending at the boundary", + b: append(bytes.Repeat([]byte("a"), 61), 0xE6, 0x97, 0xA5), + blocks: 64, want: 61, + }, + { + name: "4-byte sequence straddling the boundary", + b: append(bytes.Repeat([]byte("a"), 62), 0xF0, 0x9D, 0x84, 0x9E), + blocks: 64, want: 62, + }, + { + name: "more continuations than any sequence may have", + b: append(bytes.Repeat([]byte("a"), 60), 0x80, 0x80, 0x80, 0x80), + blocks: 64, want: 60, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := seam(tc.b, tc.blocks) + require.Equal(t, tc.want, got) + require.LessOrEqual(t, got, tc.blocks) + // resuming there must reproduce the stdlib verdict for the whole buffer + require.Equal(t, utf8.Valid(tc.b), utf8.Valid(tc.b[got:]) && utf8.Valid(tc.b[:got])) + }) + } +} + +// FuzzValidAVX2 is the standing guard on the kernel: whatever the fuzzer finds, our verdict is the stdlib's. +func FuzzValidAVX2(f *testing.F) { + f.Add(bytes.Repeat([]byte("a\u00e9\u65e5\U0001D11E"), 8)) + f.Add(append(bytes.Repeat([]byte("a"), 64), 0xE6, 0x97)) + f.Add(append(bytes.Repeat([]byte("\u00e9"), 40), 0xFF)) + f.Add(bytes.Repeat([]byte{0x80}, 70)) + f.Add(bytes.Repeat([]byte{0xF4, 0xBF, 0xBF, 0xBF}, 20)) + + f.Fuzz(func(t *testing.T, b []byte) { + require.Equalf(t, utf8.Valid(b), Valid(b), "% x", b) + + // and again at every offset within a block, since the kernel carries state across blocks + for shift := 1; shift <= 32; shift++ { + shifted := append(bytes.Repeat([]byte("a"), shift), b...) + require.Equalf(t, utf8.Valid(shifted), Valid(shifted), "shift=%d % x", shift, b) + } + }) +} diff --git a/json/internal/utf8x/validate_noasm.go b/json/internal/utf8x/validate_noasm.go new file mode 100644 index 0000000..725e0f0 --- /dev/null +++ b/json/internal/utf8x/validate_noasm.go @@ -0,0 +1,11 @@ +//go:build !amd64 + +package utf8x + +import "unicode/utf8" + +// Valid reports whether b is entirely valid UTF-8. +// +// Off amd64 there is no vector kernel, so this is the stdlib scalar validator (which carries its own 8-byte ASCII +// fast path). The answer is identical on every platform; only the speed differs. +func Valid(b []byte) bool { return utf8.Valid(b) } diff --git a/json/lexers/README.md b/json/lexers/README.md index 39b9449..4dfd409 100644 --- a/json/lexers/README.md +++ b/json/lexers/README.md @@ -2,12 +2,25 @@ This package exposes an interface `Lexer` and a token type `token.T` to parse JSON input. -Notice that the same interface may be used to parse `YAML`, or any other serialization format +Notice that the same interface may be used to parse `YAML` or any other serialization format that converts to JSON (e.g. a hierarchy of scalars, arrays and dictionaries). Several lexers implementations are proposed: -* a semantic JSON lexer -* a verbatim JSON lexer -* a ND-JSON lexer -* a YAML lexer +* [x] a semantic JSON lexer +* [x] a verbatim JSON lexer, that preserves blank space and unicode escaped sequences +* [x] a YAML lexer +* [ ] a ND-JSON lexer (TODO) + +Acknowlegements and credits +=========================== + +Our JSON lexer is an original research and development: it is not a copy of another software. + +However, we are greatly grateful to the following projects that have provided inspiration, +astute methods and algorithms that we could study, reproduce or try to improve further. + +* https://github.com/mailru/easyjson (MIT) - The first pionner for a "fast" low-level JSON encoder/decoder +* https://github.com/go-json-experiment/json (BSD 3-Clause) - go standard library encoding/json/v2 +* https://github.com/simdutf/simdutf (MIT+Apache) - extremely fast C++ UTF-8 validator +* https://github.com/minio/simdjson-go (Apache) - pionnered go & assembly with a go port of simdjson diff --git a/json/lexers/default-lexer/DESIGN.md b/json/lexers/default-lexer/DESIGN.md index 82cc70d..803fe8e 100644 --- a/json/lexers/default-lexer/DESIGN.md +++ b/json/lexers/default-lexer/DESIGN.md @@ -426,6 +426,14 @@ The kernel is `stringstop_amd64.s`, **generated by `avo`**, and its generator li 5. The `_asm` sub-module has its own `go.mod`/`go.sum`; update them there, not in the parent, when bumping `avo`. +There is a **second** `avo` kernel outside this package: the UTF-8 validator in +`json/internal/utf8x` (`validate_amd64.s`, generator in `json/internal/utf8x/_asm`), a port +of simdutf's `lookup4` algorithm. It follows the same rules, and carries its own CPUID +detection because it is a different module path. Its differential suite +(`validate_amd64_test.go`) is exhaustive over all byte pairs and over every lead byte against +all 2nd/3rd bytes — a hand-written validator that wrongly ACCEPTS is silent, so that suite is +the safety net and must stay green (it is mutation-tested: corrupting one table byte fails it). + ### 10.3 Adding an option 1. Add the field to `options` (`options.go`) and a `With…` constructor following the diff --git a/json/lexers/default-lexer/README.md b/json/lexers/default-lexer/README.md index e11ff89..132237f 100644 --- a/json/lexers/default-lexer/README.md +++ b/json/lexers/default-lexer/README.md @@ -52,6 +52,7 @@ Since we want it to be flexible, there are a few available options: (interned string keys, raw integer array indices). Opt-in and off by default — enabling it forfeits the zero-allocation guarantee (the path stack and retained keys allocate). * tunable context window for reporting errors + * strict/lax UTF8 validation Additional objectives: @@ -101,6 +102,8 @@ Differences with `encoding/json/v2` | pull iterator | ✅ || ⏸️ | | limit stack | ✅ || ✅ | | limit tok size | ✅ || ❌ | +| valid UTF8 | ✅ || ✅ | +| unchecked UTF8 | ⬜ || ❌ | Trade-offs when comparing to `github.com/go-json-experiment/json/jsontext` (stdlib `json/v2`). @@ -121,10 +124,68 @@ Streaming mode degrades speed by 15-20%. Pull iterator (`NextToken`) mode degrades speed by 10-15%. +## UTF-8 + +RFC 8259 §8.1 requires JSON text to be UTF-8, and both lexers enforce that on string values. This covers both ways a +value can end up holding a non-character: + +* an ill-formed UTF-8 byte sequence in the source (truncated, overlong, an encoded surrogate, beyond U+10FFFF); +* a `\uXXXX` escape that does not denote a Unicode scalar value — an unpaired, lone or inverted surrogate. + +`WithUTF8Policy` picks what happens: + +| policy | ill-formed bytes | broken `\u` escape | +|---|---|---| +| `UTF8Strict` (default) | `ErrInvalidUTF8` | `ErrSurrogateEscape` | +| `UTF8Replace` | U+FFFD per invalid byte | U+FFFD per broken escape | +| `UTF8Passthrough` | passed through untouched | U+FFFD per broken escape | + +An escape always produces *some* rune — the escape text is not the value — which is why `UTF8Passthrough` still +substitutes there. + +`VL` keeps escapes as source text, so under `UTF8Replace` it rewrites ill-formed *bytes* but never rewrites escape +text; the substitution for a broken escape appears when the value is decoded with `token.Unescape`. + +### What mangling costs `VL` + +U+FFFD encodes as **three** bytes and replaces **one**, so a mangled value is 2 bytes longer per ill-formed byte than +the text it came from — and a multi-byte fault is several ill-formed bytes, not one (a truncated 3-byte sequence +becomes three replacements: 9 bytes where the source had 3). For such a value: + +* `len(token.Value())` is **not** the width of its source span; +* an index into the value **cannot** be added to the token's start to get a source offset; +* the original bytes are **not** recoverable from the token. + +Token *positions* are unaffected: `Line()`, `Column()`, `Offset()` and `LeadingSpace()` derive from the scan cursor +walking the source, never from the value, so they stay exact under every policy. A formatter or linter can still point +at the right place — it just cannot assume the value it holds is the text that was there. Callers who need the source +bytes should use `UTF8Strict` (and handle the error) or `UTF8Passthrough` (and validate downstream themselves). + +A leading UTF-8 BOM is the other documented exception to byte-exact round-tripping, for a different reason: it is +consumed before any token exists, so it belongs to no token's leading space and is not re-emitted. + +**Cost.** Detection is fused into the string scan already being performed — every SWAR word and every AVX2 block is +OR-accumulated, so "did this value contain a byte >= 0x80" is answered for free — and only values that actually carry +one reach the validator — where an AVX2 port of simdutf's `lookup4` algorithm handles them at 5-12x the stdlib's rate. +On the reference corpus, `UTF8Strict` versus no validation is statistically indistinguishable on the four +ASCII-dominated workloads and costs ~6% (`L`) / ~2% (`VL`) on `twitter_status`, which is 30% non-ASCII by string bytes. + +The input must be UTF-8: a UTF-16 document is rejected with `ErrNotUTF8`. A leading UTF-8 BOM is accepted and +consumed before the first token — RFC 8259 §8.1 permits ignoring one on input and asks implementations not to emit +one, so it is not reproduced on output (see above). + ## Conformance tests Our implementation of the JSON lexers pass the full JSON conformance suite. No compromise on strictness. +Beyond the suite's `y_`/`n_` contract, the implementation-defined (`i_`) cases are snapshotted in +`testdata/conformance_i_behavior.golden`, so a change in *implementation-defined* behavior shows up as a reviewable +diff instead of silence. Regenerate it with: + +``` +go test ./lexers/default-lexer -run TestConformanceParsing -update-golden +``` + ## Benchmarks See [a comparison](../benchmark/benchviz/README.md) @@ -159,6 +220,7 @@ to avoid. PGO is the supported lever. ## Roadmap +* ND-JSON lexer * AVX2 support is currently provided as assembly kernels for amd64 only * AVX512 is likely overkill for our usage and I don't have the hardware to test it thoroughly -* AVX support this will be eventually replaced by go native support for AV2 & AVX512 (currently experimental) +* AVX support will be eventually replaced by go native support for AVX2 & AVX512 (currently experimental) diff --git a/json/lexers/default-lexer/conformance_lineendings_test.go b/json/lexers/default-lexer/conformance_lineendings_test.go new file mode 100644 index 0000000..b102a40 --- /dev/null +++ b/json/lexers/default-lexer/conformance_lineendings_test.go @@ -0,0 +1,129 @@ +package lexer + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" +) + +// Line-ending independence for the JSON conformance fixtures. +// +// A conformance fixture's exact bytes ARE the test. A checkout with core.autocrlf=true — the Windows default — +// rewrites every LF to CRLF, and then a document that must be REJECTED can start being accepted (or the reverse), +// with the failure reading as a lexer bug rather than a checkout artefact. +// +// .gitattributes marks testdata/** as -text so git never translates them, and that is the real fix. These tests +// cover the other half: a suite that arrived some other way — an unmarked clone, a zip, a copy through a Windows +// tool — must still measure the same documents. + +// normalizeLineEndings turns CRLF into LF. +// +// A lone CR is deliberately left alone: it is a meaningful byte inside a JSON document (an unescaped control +// character, which a conforming parser must reject), so collapsing it would change the very thing several n_ cases +// test. TestConformanceFixturesAreStoredWithLF pins that no vendored fixture carries a raw CR, which is what makes +// this safe: every CR that could appear has been introduced by translation. +func normalizeLineEndings(data []byte) []byte { + if !bytes.Contains(data, []byte("\r")) { + return data + } + + return bytes.ReplaceAll(data, []byte("\r\n"), []byte("\n")) +} + +// TestConformanceFixturesAreStoredWithLF guards the fixtures as committed. It is also what licenses the +// normalisation above: if upstream ever ships a fixture containing a genuine CR, this fails and the normalisation +// has to be revisited rather than silently eating it. +func TestConformanceFixturesAreStoredWithLF(t *testing.T) { + root := filepath.Join(currentDir(), "..", "testdata", "JSONTestSuite") + + var withCR []string + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + + //nolint:gosec // path comes from walking our own vendored fixtures + data, rerr := os.ReadFile(path) + if rerr != nil { + return rerr + } + if bytes.Contains(data, []byte("\r")) { + rel, _ := filepath.Rel(root, path) + withCR = append(withCR, rel) + } + + return nil + }) + require.NoError(t, err) + + assert.Emptyf( + t, + withCR, + "fixtures must be stored without carriage returns (see .gitattributes); found CR in %v", + withCR, + ) +} + +func TestNormalizeLineEndings(t *testing.T) { + cases := []struct { + name string + in string + want string + }{ + {name: "no CR at all", in: "{\n}\n", want: "{\n}\n"}, + {name: "CRLF", in: "{\r\n}\r\n", want: "{\n}\n"}, + {name: "mixed", in: "[\r\n1,\n2\r\n]", want: "[\n1,\n2\n]"}, + {name: "lone CR is preserved", in: "[\"a\rb\"]", want: "[\"a\rb\"]"}, + {name: "empty", in: "", want: ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, string(normalizeLineEndings([]byte(tc.in)))) + }) + } +} + +// TestConformanceVerdictsSurviveCRLF is the property that matters: converting every fixture to CRLF must not change +// a single accept/reject verdict, in any lexer mode. +// +// Without the normalisation this fails — a document whose only fault is an unescaped newline inside a string, for +// instance, is a different document once that newline becomes CRLF. +func TestConformanceVerdictsSurviveCRLF(t *testing.T) { + dir := filepath.Join(currentDir(), "..", "testdata", "JSONTestSuite", "test_parsing") + entries, err := os.ReadDir(dir) + require.NoError(t, err) + require.NotEmpty(t, entries) + + modes := conformanceModes() + + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, ".json") { + continue + } + + raw, rerr := os.ReadFile(filepath.Join(dir, name)) + require.NoError(t, rerr) + + lf := normalizeLineEndings(raw) + crlf := normalizeLineEndings(bytes.ReplaceAll(lf, []byte("\n"), []byte("\r\n"))) + + require.Equalf(t, string(lf), string(crlf), + "%s: normalising a CRLF copy must reproduce the LF original", name) + + maximum := len(lf) + 16 + for _, m := range modes { + assert.Equalf(t, m.run(lf, maximum).accepted, m.run(crlf, maximum).accepted, + "%s/%s: the verdict changed for a CRLF checkout", name, m.name) + } + } +} diff --git a/json/lexers/default-lexer/conformance_test.go b/json/lexers/default-lexer/conformance_test.go index d87b552..6f2024b 100644 --- a/json/lexers/default-lexer/conformance_test.go +++ b/json/lexers/default-lexer/conformance_test.go @@ -2,6 +2,7 @@ package lexer import ( "bytes" + "flag" "os" "path/filepath" "sort" @@ -45,10 +46,13 @@ func TestConformanceParsing(t *testing.T) { continue } - data, rerr := os.ReadFile(filepath.Join(dir, name)) + raw, rerr := os.ReadFile(filepath.Join(dir, name)) if rerr != nil { t.Fatalf("cannot read %s: %v", name, rerr) } + // A fixture's exact bytes ARE the test, so a checkout that translated them must not change the verdict: + // see normalizeLineEndings and TestConformanceFixturesAreStoredWithLF. + data := normalizeLineEndings(raw) maximum := len(data) + 16 // generous upper bound on token count var want byte @@ -65,9 +69,11 @@ func TestConformanceParsing(t *testing.T) { // Aggregate verdicts across modes; a file "conforms" only if every mode agrees with the expectation. conforms := true detail := make([]string, 0, len(modes)) + byMode := make(map[string]bool, len(modes)) for _, m := range modes { v := m.run(data, maximum) detail = append(detail, m.name+"="+verdictStr(v)) + byMode[m.name] = v.accepted switch want { case 'y': @@ -81,6 +87,27 @@ func TestConformanceParsing(t *testing.T) { } } + // UTF8Replace may only turn a rejection into an acceptance, and only for the ill-formed-UTF-8 / + // broken-escape cases it is designed to sanitize. Anything else means replacement relaxed the grammar. + if byMode["L/replace"] != byMode["L/bytes"] { + switch { + case !byMode["L/replace"]: + t.Errorf("UTF8Replace REJECTED a document the default policy accepts: %s\n %s", + name, strings.Join(detail, " ")) + case !conformanceUTF8Relaxed()[name]: + t.Errorf("UTF8Replace accepted %s, which the default policy rejects, "+ + "but it is not a known ill-formed-UTF-8 case: add it to conformanceUTF8Relaxed "+ + "or fix the relaxation\n %s", name, strings.Join(detail, " ")) + } + } else if conformanceUTF8Relaxed()[name] { + t.Errorf( + "conformanceUTF8Relaxed lists %q but UTF8Replace no longer changes its verdict: "+ + "remove it\n %s", + name, + strings.Join(detail, " "), + ) + } + switch want { case 'i': iCount++ @@ -125,7 +152,7 @@ func TestConformanceParsing(t *testing.T) { } } - // Emit the implementation-defined behavior table for the record. + // Emit the implementation-defined behavior table for the record, and pin it against drift. sort.Strings(iReport) t.Logf("=== implementation-defined (i_) behavior: %d cases ===", iCount) for _, line := range iReport { @@ -133,6 +160,90 @@ func TestConformanceParsing(t *testing.T) { } t.Logf("=== summary: y_ pass=%d, n_ pass=%d, xfail=%d, unexpected-pass=%d ===", passYes, passNo, xfailCount, unexpectedOK) + + checkIBehaviorGolden(t, iReport) +} + +//nolint:gochecknoglobals // the standard golden-file switch must be a package-level flag +var updateGolden = flag.Bool( + "update-golden", + false, + "rewrite the conformance i_ behavior snapshot from the current run", +) + +// checkIBehaviorGolden pins how the lexers treat every implementation-DEFINED (i_) case. +// +// These cases are outside the suite's y_/n_ contract, so nothing asserted them before — which is exactly how the +// UTF-8 bug survived: ten i_string_* documents carrying ill-formed UTF-8 were silently ACCEPTED, and the suite stayed +// green from the day the zero-copy string fast path landed. Snapshotting the table turns any future change in +// implementation-defined behavior into a reviewable diff instead of silence. +// +// Run `go test ./lexers/default-lexer -run TestConformanceParsing -update-golden` after an intentional change. +func checkIBehaviorGolden(t *testing.T, iReport []string) { + t.Helper() + + golden := filepath.Join(currentDir(), "testdata", "conformance_i_behavior.golden") + got := strings.Join(iReport, "\n") + "\n" + + if *updateGolden { + if err := os.WriteFile(golden, []byte(got), 0o600); err != nil { + t.Fatalf("cannot write %s: %v", golden, err) + } + t.Logf("updated %s", golden) + + return + } + + raw, err := os.ReadFile(golden) + if err != nil { + t.Fatalf("cannot read %s (run with -update-golden to create it): %v", golden, err) + } + want := string(normalizeLineEndings(raw)) // the golden records verdicts, not bytes on a line + + if want == got { + return + } + + wantLines := strings.Split(strings.TrimRight(want, "\n"), "\n") + gotLines := strings.Split(strings.TrimRight(got, "\n"), "\n") + t.Errorf("implementation-defined behavior changed (%d recorded cases, %d now).\n"+ + "If the change is intended, re-run with -update-golden and review the diff.", len(wantLines), len(gotLines)) + for _, line := range diffLines(wantLines, gotLines) { + t.Errorf(" %s", line) + } +} + +// diffLines reports the entries that differ between two sorted "name: verdicts" snapshots. +func diffLines(want, got []string) []string { + index := func(lines []string) map[string]string { + m := make(map[string]string, len(lines)) + for _, line := range lines { + name, verdicts, _ := strings.Cut(line, ": ") + m[name] = verdicts + } + + return m + } + + wantByName, gotByName := index(want), index(got) + + var out []string + for name, w := range wantByName { + switch g, ok := gotByName[name]; { + case !ok: + out = append(out, "gone: "+name+" (was "+w+")") + case g != w: + out = append(out, "changed: "+name+"\n was: "+w+"\n now: "+g) + } + } + for name, g := range gotByName { + if _, ok := wantByName[name]; !ok { + out = append(out, "new: "+name+" ("+g+")") + } + } + sort.Strings(out) + + return out } // conformanceXFail lists test_parsing files whose current behavior diverges from the JSONTestSuite expectation (y_/n_). @@ -217,6 +328,53 @@ func conformanceModes() []conformanceMode { return drainVL(NewVerbatimWithBytes(data), maximum) }, }, + { + // VL's streaming string scanners (consumeStringRawStreamFast / consumeStringRawStreaming) had no + // conformance coverage at all until this mode was added. + name: "VL/reader", + run: func(data []byte, maximum int) conformanceVerdict { + return drainVL(NewVerbatim(bytes.NewReader(data), WithBufferSize(64)), maximum) + }, + }, + { + // UTF8Replace must not change what is ACCEPTED except for ill-formed UTF-8, which it sanitizes instead of + // rejecting. Running it over the whole suite pins that it relaxes nothing else. + name: "L/replace", + run: func(data []byte, maximum int) conformanceVerdict { + return drainL(NewWithBytes(data, WithUTF8Policy(UTF8Replace)), maximum) + }, + }, + } +} + +// conformanceUTF8Relaxed lists the test_parsing files that UTF8Replace accepts although the default policy rejects +// them: exactly the ill-formed-UTF-8 and broken-surrogate-escape cases, which replacement sanitizes to U+FFFD. +// +// Any OTHER file whose verdict differs between the two policies is a bug — replacement must not relax the grammar. +func conformanceUTF8Relaxed() map[string]bool { + return map[string]bool{ + // ill-formed UTF-8 bytes + "i_string_UTF-8_invalid_sequence.json": true, + "i_string_UTF8_surrogate_U+D800.json": true, + "i_string_invalid_utf-8.json": true, + "i_string_iso_latin_1.json": true, + "i_string_lone_utf8_continuation_byte.json": true, + "i_string_not_in_unicode_range.json": true, + "i_string_overlong_sequence_2_bytes.json": true, + "i_string_overlong_sequence_6_bytes.json": true, + "i_string_overlong_sequence_6_bytes_null.json": true, + "i_string_truncated-utf-8.json": true, + // \u escapes that do not denote a scalar value + "i_object_key_lone_2nd_surrogate.json": true, + "i_string_1st_surrogate_but_2nd_missing.json": true, + "i_string_1st_valid_surrogate_2nd_invalid.json": true, + "i_string_incomplete_surrogate_and_escape_valid.json": true, + "i_string_incomplete_surrogate_pair.json": true, + "i_string_incomplete_surrogates_escape_valid.json": true, + "i_string_invalid_lonely_surrogate.json": true, + "i_string_invalid_surrogate.json": true, + "i_string_inverted_surrogates_U+1D11E.json": true, + "i_string_lone_second_surrogate.json": true, } } diff --git a/json/lexers/default-lexer/doc.go b/json/lexers/default-lexer/doc.go index 0fa9e26..0709689 100644 --- a/json/lexers/default-lexer/doc.go +++ b/json/lexers/default-lexer/doc.go @@ -42,4 +42,23 @@ // lexer.WithMaxContainerStack(512), // nesting depth ceiling // lexer.WithMaxValueBytes(1<<20), // per-value / whitespace ceiling // ) +// +// # UTF-8 +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, and both lexers enforce it on string values: an ill-formed byte +// sequence, or a \u escape that does not denote a Unicode scalar value (an unpaired or inverted surrogate), is +// rejected by default with [codes.ErrInvalidUTF8] / [codes.ErrSurrogateEscape]. +// +// Callers who would rather sanitize than fail can select [UTF8Replace], which substitutes U+FFFD (one per invalid +// byte, one per broken escape); [UTF8Passthrough] restores the unvalidated behavior for input already known to be +// well-formed. See [WithUTF8Policy]. +// +// Validation is not a second pass over the value: detection of non-ASCII bytes is fused into the string scan the +// lexer already performs, so a pure-ASCII value — the overwhelmingly common case — is proven valid with no extra read +// and no call, and only values that actually carry a byte >= 0x80 reach the validator, where an AVX2 kernel handles +// them. Measured on the reference corpus, enabling it is free on ASCII-dominated documents and costs ~6% (L) / ~2% +// (VL) on heavily non-ASCII ones (twitter_status). +// +// The input must be UTF-8: a UTF-16 document is rejected with [codes.ErrNotUTF8]. A leading UTF-8 BOM is consumed +// before the first token and is not re-emitted by [VL] (RFC 8259 §8.1 asks implementations not to emit one). package lexer diff --git a/json/lexers/default-lexer/fuzz_test.go b/json/lexers/default-lexer/fuzz_test.go index 080b857..31bc898 100644 --- a/json/lexers/default-lexer/fuzz_test.go +++ b/json/lexers/default-lexer/fuzz_test.go @@ -23,7 +23,9 @@ package lexer // the same grammar; only their token *shape* differs). // 5. Verbatim round-trip: for any ACCEPTED input, reassembling the raw VL token // stream (leading blanks + verbatim token text) reproduces the input byte for -// byte — the defining property of the verbatim lexer. +// byte — the defining property of the verbatim lexer. The single exception is a +// leading UTF-8 BOM, consumed before any token exists and therefore not +// re-emitted (input.CheckBOM; RFC 8259 §8.1 asks implementations not to emit one). // 6. Well-formed on accept: an ACCEPTED token stream obeys the JSON grammar // (balanced and correctly typed containers, keys only in object key slots, // key/value alternation, exactly one root value). The lexers are "almost a @@ -41,6 +43,7 @@ import ( "iter" "strings" "testing" + "unicode/utf8" "github.com/go-openapi/core/json/lexers/token" ) @@ -403,6 +406,17 @@ func FuzzLexer(f *testing.F) { `123456789012345678901234567890`, `1e999999`, `-`, + // ill-formed UTF-8 in string bodies (rejected by the default policy; see utf8_test.go) + "[\"\xff\"]", + "[\"\xc0\xaf\"]", + "[\"\xed\xa0\x80\"]", + "[\"\xe0\xff\"]", + "[\"ok\xe2\x82\"]", + "{\"k\xff\":1}", + "[\"caf\xc3\xa9\"]", + "[\"\xe6\x97\xa5\xe6\x9c\xac\"]", + `["\uD800\uD800"]`, + `["𝄞"]`, } for _, s := range seeds { f.Add([]byte(s)) @@ -453,10 +467,16 @@ func FuzzLexer(f *testing.F) { } // --- verbatim round-trip on accepted input --- + // + // A leading UTF-8 BOM is the ONE documented exception: it is consumed before any token exists (see + // input.CheckBOM), so it belongs to no token's leading space and is not re-emitted. RFC 8259 §8.1 asks + // implementations not to emit a BOM, so dropping it is the conformant direction — but it does mean the + // round-trip is byte-exact only for the document body. if vPullErr == nil { + want := bytes.TrimPrefix(data, []byte("\uFEFF")) rt := fzVerbatimRoundtrip(data, n) - if !bytes.Equal(rt, data) { - t.Fatalf("VL round-trip mismatch:\n in =%q\n out=%q", data, rt) + if !bytes.Equal(rt, want) { + t.Fatalf("VL round-trip mismatch:\n in =%q\n out=%q\n want=%q", data, rt, want) } } @@ -470,5 +490,112 @@ func FuzzLexer(f *testing.F) { ) } } + + // --- UTF-8: no ill-formed sequence may reach the caller --- + fzUTF8Invariants(t, data, n) }) } + +// fzUTF8Invariants pins the promise the UTF-8 work exists to make: under the default policy an accepted document +// yields only well-formed values, and under UTF8Replace EVERY document that is otherwise grammatical is accepted and +// still yields only well-formed values. +// +// It is the broadest guard we have, because it holds for arbitrary bytes rather than for a fixture table: the fused +// non-ASCII detection is duplicated across eight scanners, and a miss in any one of them shows up here. +func fzUTF8Invariants(t *testing.T, data []byte, n int) { + t.Helper() + + check := func(lane string, policy UTF8Policy, values [][]byte, err error) { + if err != nil { + return + } + for _, v := range values { + if !utf8.Valid(v) { + t.Fatalf( + "%s emitted an ill-formed value under policy %d: %q\ninput=%q", + lane, + policy, + v, + data, + ) + } + } + } + + for _, policy := range []UTF8Policy{UTF8Strict, UTF8Replace} { + lVals, lErr := fzStringValues( + NewWithBytes(append([]byte(nil), data...), WithUTF8Policy(policy)), + n, + ) + check("L/bytes", policy, lVals, lErr) + + sVals, sErr := fzStringValues( + New(bytes.NewReader(data), WithBufferSize(32), WithUTF8Policy(policy)), n) + check("L/reader", policy, sVals, sErr) + + // VL keeps escapes as source text, so its values are checked through the decoder — the form a caller + // actually consumes. + vVals, vErr := fzVerbatimDecodedValues( + NewVerbatimWithBytes(append([]byte(nil), data...), WithUTF8Policy(policy)), n) + check("VL/bytes", policy, vVals, vErr) + + vsVals, vsErr := fzVerbatimDecodedValues( + NewVerbatim(bytes.NewReader(data), WithBufferSize(32), WithUTF8Policy(policy)), n) + check("VL/reader", policy, vsVals, vsErr) + + // Replacement must never be the reason a document is rejected: it only ever turns a UTF-8 error into a + // substitution, so anything the default policy accepts it must accept too. + if policy == UTF8Replace && lErr != nil { + if _, strictErr := fzStringValues( + NewWithBytes(append([]byte(nil), data...)), n); strictErr == nil { + t.Fatalf( + "UTF8Replace rejected a document the default policy accepts: %v\ninput=%q", + lErr, + data, + ) + } + } + } +} + +// fzStringValues drains a semantic lexer and returns the value of every String/Key token. +func fzStringValues(lex *L, n int) ([][]byte, error) { + var out [][]byte + limit := fzCap(n) + for i := 0; ; i++ { + if i > limit { + panic("lexer did not terminate (utf8 invariants)") + } + tok := lex.NextToken() + if !lex.Ok() { + return out, lex.Err() + } + if k := tok.Kind(); k == token.String || k == token.Key { + out = append(out, bytes.Clone(tok.Value())) + } + if tok.IsEOF() { + return out, nil + } + } +} + +// fzVerbatimDecodedValues drains a verbatim lexer and returns every String/Key value in DECODED form. +func fzVerbatimDecodedValues(lex *VL, n int) ([][]byte, error) { + var out [][]byte + limit := fzCap(n) + for i := 0; ; i++ { + if i > limit { + panic("lexer did not terminate (utf8 invariants, verbatim)") + } + tok := lex.NextToken() + if !lex.Ok() { + return out, lex.Err() + } + if k := tok.Kind(); k == token.String || k == token.Key { + out = append(out, token.Unescape(bytes.Clone(tok.Value()))) + } + if tok.IsEOF() { + return out, nil + } + } +} diff --git a/json/lexers/default-lexer/internal/input/input.go b/json/lexers/default-lexer/internal/input/input.go index b48e33c..90e1e8c 100644 --- a/json/lexers/default-lexer/internal/input/input.go +++ b/json/lexers/default-lexer/internal/input/input.go @@ -1,6 +1,10 @@ package input -import "io" +import ( + "io" + + "github.com/go-openapi/core/json/internal/utf8x" +) // Input holds the lexer's buffered-input and scan-cursor state: the buffer window and read position, the // value-accumulation scratch, the reader and refill buffers, the error state, and the two grammar flags the string @@ -41,4 +45,25 @@ type Input struct { NoAVX2 bool MaxValueBytes int KeepPreviousBuffer int + + // UTF8Policy governs what happens to a string value that is not valid UTF-8. Mirrored from the like-named option in + // L.reset(); the lexer package aliases this type so it is the single definition. + UTF8Policy UTF8Policy + + // sanitized holds the U+FFFD-substituted rewrite of an ill-formed value under UTF8Replace. + // + // It is a buffer of its own rather than CurrentValue because on the unescaping paths the value to rewrite IS + // CurrentValue. It is allocated lazily on the first ill-formed value, so a well-formed document never pays for it. + sanitized []byte } + +// UTF8Policy is [utf8x.Policy]: the lexers and the writers share one definition so they cannot drift apart. The +// lexer package aliases this in turn, so callers name neither internal package. +type UTF8Policy = utf8x.Policy + +// The policy values, re-exported so the scanners read UTF8Strict rather than utf8x.PolicyStrict. +const ( + UTF8Strict = utf8x.PolicyStrict + UTF8Replace = utf8x.PolicyReplace + UTF8Passthrough = utf8x.PolicyPassthrough +) diff --git a/json/lexers/default-lexer/internal/input/refill.go b/json/lexers/default-lexer/internal/input/refill.go index b2ee525..69e9ed0 100644 --- a/json/lexers/default-lexer/internal/input/refill.go +++ b/json/lexers/default-lexer/internal/input/refill.go @@ -37,6 +37,7 @@ func (in *Input) FirstFill() { } else { in.Err = err } + in.CheckBOM() return } @@ -45,6 +46,7 @@ func (in *Input) FirstFill() { } } in.Bufferized = n + in.CheckBOM() } // ReadMore provides more input from the internal buffer or consumes from the input stream. @@ -99,6 +101,41 @@ func (in *Input) ConsumeNull(_ byte) token.T { return token.NullToken } +// ensureWindow makes at least n unconsumed bytes contiguously available in the window, compacting the consumed prefix +// away and reading more from the source. It reports whether that many could be made available. +// +// It exists for the surrogate-escape lookahead, which must inspect the following `\uXXXX` BEFORE deciding whether it +// belongs to the current escape (see [Input.peekLowSurrogate]) — consumeN cannot serve that, since it consumes what it +// reads and a straddling read cannot be given back. +// +// Compaction is safe on that path: the only scanners that reach it hold no live buffer index (the streaming string +// paths accumulate into CurrentValue), and in whole-buffer mode it never compacts at all — it just answers whether the +// bytes are there. Being a cold path (surrogate escapes only), it deliberately does not maintain PreviousBuffer. +func (in *Input) ensureWindow(n int) bool { + if in.Bufferized-in.Consumed >= n { + return true + } + + if in.WholeBuffer { + return false // the whole input is already here: what is missing does not exist + } + + if in.Consumed > 0 { + in.Bufferized = copy(in.Buffer, in.Buffer[in.Consumed:in.Bufferized]) + in.Consumed = 0 + } + + for in.Bufferized < n { + m, err := in.R.Read(in.Buffer[in.Bufferized:]) + in.Bufferized += m + if err != nil || m == 0 { + break + } + } + + return in.Bufferized >= n +} + // consumeN consumes a small buffer of n bytes to decide tokens such as "true", "false" or "null". func (in *Input) consumeN(buffer []byte) error { minReadSize := len(buffer) diff --git a/json/lexers/default-lexer/internal/input/string.go b/json/lexers/default-lexer/internal/input/string.go index b02da06..27fef47 100644 --- a/json/lexers/default-lexer/internal/input/string.go +++ b/json/lexers/default-lexer/internal/input/string.go @@ -7,10 +7,10 @@ import ( "unicode/utf16" "unicode/utf8" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/default-lexer/internal/strscan" "github.com/go-openapi/core/json/lexers/default-lexer/internal/swar" codes "github.com/go-openapi/core/json/lexers/error-codes" - scan "github.com/go-openapi/core/json/lexers/internal/scan" "github.com/go-openapi/core/json/lexers/token" ) @@ -89,16 +89,21 @@ func (in *Input) consumeStringStreamFast() token.T { // run stays clean past guessLong. // Identical probe to consumeStringWhole, but bounded by the window end n = in.Bufferized. i := start + var hi uint64 // OR of the content bytes scanned; see consumeStringWhole guard := start + guessLong if in.NoAVX2 { guard = n + 1 } for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { break @@ -106,13 +111,19 @@ func (in *Input) consumeStringStreamFast() token.T { } if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if i >= n { @@ -135,7 +146,7 @@ func (in *Input) consumeStringStreamFast() token.T { in.Offset += uint64(end - start) in.Consumed = end - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Offset += uint64(i - start) @@ -168,6 +179,11 @@ func (in *Input) consumeStringWhole() token.T { // // The overwhelmingly common case (no escapes, no control chars) aliases the input with zero copy. i := start + // hi accumulates every content byte the scan passes over, so (hi & swar.HighBits) != 0 answers "did this value + // contain a byte >= 0x80" for free — the word is already in a register. A value that is pure ASCII is thereby + // proven valid UTF-8 with no second pass and no call; only the rest reaches the validator (see finishStringValue). + // Lanes at or after the stop are trimmed off so the answer is exact, matching strscan.ScanStop's. + var hi uint64 // guard is where the inline probe stops and delegates to the AVX2 scan. // With WithoutAVX2 it is pushed past the buffer so the loop never breaks to delegate — the string is scanned // entirely by the inline SWAR word loop (the pre-AVX2 baseline), no vector call at alin. @@ -176,11 +192,15 @@ func (in *Input) consumeStringWhole() token.T { guard = n + 1 } for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) // exact stop lane; skips the scalar re-scan + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) // exact stop lane; skips the scalar re-scan + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { break // guessLong clean bytes in — leave the loop to delegate below @@ -194,13 +214,19 @@ func (in *Input) consumeStringWhole() token.T { // above. i lands on the stop byte or on n. if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if i >= n { in.Consumed, in.Offset = i, uint64(i) @@ -221,7 +247,7 @@ func (in *Input) consumeStringWhole() token.T { i++ // past the closing quote in.Consumed, in.Offset = i, uint64(i) - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Consumed, in.Offset = i, uint64(i) @@ -234,15 +260,18 @@ func (in *Input) consumeStringWhole() token.T { // It is a separate function on purpose — keeping the byte-by-byte escape machinery out of this frame insulates the // fast path's codegen from it (and vice versa); they were previously one function, where a fast-path change could // regress the slow path by ~12% and vice versa (plan §4.2). - return in.consumeStringEscaped(start, i) + return in.consumeStringEscaped(start, i, hi&swar.HighBits != 0) } // consumeStringEscaped is the unescape slow path, split out of consumeStringWhole. // -// It is entered with data[i] == escape and start..i the clean prefix already scanned. -// It copies that prefix then unescapes the rest; the loop invariant is that data[i] is the next "stop" byte (quote, -// escape, or control) — clean runs between stops are copied in bulk rather than byte-by-byte. -func (in *Input) consumeStringEscaped(start, i int) token.T { +// It is entered with data[i] == escape and start..i the clean prefix already scanned; nonASCII reports whether that +// prefix carried a byte >= 0x80, and is extended here over every further raw run copied into the value. +// +// Runes produced by a \u escape are deliberately NOT accumulated: they are valid by construction (the escape decoder +// rejects or replaces anything that is not a scalar value), so a value whose only non-ASCII comes from escapes still +// skips the validator. +func (in *Input) consumeStringEscaped(start, i int, nonASCII bool) token.T { data := in.Buffer n := in.Bufferized @@ -254,7 +283,7 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { i++ in.Consumed, in.Offset = i, uint64(i) - return in.finishStringValue(in.CurrentValue) + return in.finishStringValue(in.CurrentValue, nonASCII) case c == escape: i++ @@ -324,32 +353,45 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { // fast. // The bound is checked against len + the run width *before* the append so an over-long value is rejected without // copying a huge clean run, and escape-only expansion (zero-width run) is still caught. + var hi uint64 stop := i probe := min(i+swarProbe, n) for ; stop < probe; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if stop == probe && stop < n { // run outran the scalar probe → SWAR, guess long past guessLong for stop+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[stop:])); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-i >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } } if in.MaxValueBytes > 0 && len(in.CurrentValue)+(stop-i) > in.MaxValueBytes { @@ -358,6 +400,7 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { return token.None } + nonASCII = nonASCII || hi&swar.HighBits != 0 in.CurrentValue = append(in.CurrentValue, data[i:stop]...) i = stop } @@ -370,7 +413,18 @@ func (in *Input) consumeStringEscaped(start, i int) token.T { // finishStringValue turns a scanned string body into a Key (in object key position) or String token, handling the // trailing colon for keys. -func (in *Input) finishStringValue(value []byte) token.T { +// +// It is the single funnel every string path reaches, so it is where the UTF-8 policy is applied. nonASCII is the +// verdict the scan already produced: false means the value is pure ASCII and therefore valid UTF-8 by construction, so +// the common case costs one predictable branch and never touches the bytes again. +func (in *Input) finishStringValue(value []byte, nonASCII bool) token.T { + if nonASCII && in.UTF8Policy.Validates() && !utf8x.Valid(value) { + var ok bool + if value, ok = in.handleInvalidUTF8(value); !ok { + return token.None + } + } + if in.ExpectKey { // the following colon is validated on the next scan (see in.AfterKey) in.ExpectKey = false @@ -382,8 +436,19 @@ func (in *Input) finishStringValue(value []byte) token.T { return token.MakeWithValue(token.String, value) } +// consumeStringStreaming is the byte-by-byte scan over a refilling buffer: it decodes escapes and copies content into +// in.CurrentValue so the value survives buffer turnover. +// +// The clean-run bulk scan in its default branch is deliberately inline rather than shared with the three sibling +// scanners: extracting it costs a call per run and, more importantly, the escape-analysis and inlining properties of +// this frame are load-bearing (see the fast/slow split note on consumeStringEscaped). +// +//nolint:maintidx // one long switch by design; splitting it perturbs the scan's codegen (see core_pull_buffer.go) func (in *Input) consumeStringStreaming() token.T { - var escapeSequence bool + var ( + escapeSequence bool + nonASCII bool // did any raw source byte carry the high bit (see consumeStringEscaped) + ) in.CurrentValue = in.CurrentValue[:0] for { @@ -430,7 +495,7 @@ func (in *Input) consumeStringStreaming() token.T { continue } - return in.finishStringValue(in.CurrentValue) + return in.finishStringValue(in.CurrentValue, nonASCII) case slash: if escapeSequence { @@ -493,6 +558,7 @@ func (in *Input) consumeStringStreaming() token.T { return token.None } + nonASCII = nonASCII || b >= utf8.RuneSelf in.CurrentValue = append(in.CurrentValue, b) // bulk-scan the rest of this clean run within the current window (§10.3 Phase 1c): a long clean stretch (e.g. @@ -506,36 +572,48 @@ func (in *Input) consumeStringStreaming() token.T { n := in.Bufferized runStart := in.Consumed stop := runStart + var hi uint64 probe := min(stop+swarProbe, n) for ; stop < probe; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if stop == probe && stop < n { // run outran the scalar probe → SWAR for stop+8 <= n { - if m := swar.StringStopMask( - binary.LittleEndian.Uint64(data[stop:]), - ); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-runStart >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } } + nonASCII = nonASCII || hi&swar.HighBits != 0 if stop > runStart { if in.MaxValueBytes > 0 && len(in.CurrentValue)+(stop-runStart) > in.MaxValueBytes { @@ -552,51 +630,31 @@ func (in *Input) consumeStringStreaming() token.T { } } +// unescapeUnicodeSequence decodes a \uXXXX escape (the leading "\u" is already consumed), combining a surrogate pair +// when one follows. +// +// A code unit that does not denote a Unicode scalar value is governed by the UTF-8 policy (see +// [Input.brokenSurrogate]): rejected under UTF8Strict, decoded to U+FFFD otherwise. Crucially the trailing escape is +// only consumed when it actually completes the pair, so `\uD800\uD800` yields TWO replacement runes rather than +// swallowing the second escape — the behavior [token.Unescape] already has for verbatim values. func (in *Input) unescapeUnicodeSequence() (rune, error) { - var buf [4]byte - if err := in.consumeN(buf[:]); err != nil { - return utf8.RuneError, codes.ErrUnicodeEscape + _, r, err := in.readHex4() + if err != nil { + return utf8.RuneError, err } - high1, highOK1 := scan.Unhex(buf[0]) - low1, lowOK1 := scan.Unhex(buf[1]) - high2, highOK2 := scan.Unhex(buf[2]) - low2, lowOK2 := scan.Unhex(buf[3]) - if !lowOK1 || !highOK1 || !lowOK2 || !highOK2 { - return utf8.RuneError, codes.ErrUnicodeEscape + if !utf16.IsSurrogate(r) { + // four hex digits that are not a surrogate always denote a scalar value: nothing further to check + return r, nil } - unicodeEscape := uint32(high1)<<12 + uint32(low1)<<8 + uint32(high2)<<4 + uint32(low2) - r := rune(unicodeEscape) - if utf16.IsSurrogate(r) { - // this is a surrogate pair to encode a UTF-16 codepoint in 2 pairs expect this to follow: \uXXXX. - var nextBuf [6]byte - if err := in.consumeN(nextBuf[:]); err != nil { - return utf8.RuneError, codes.ErrSurrogateEscape - } - - if nextBuf[0] != escape || nextBuf[1] != 'u' { - return utf8.RuneError, codes.ErrSurrogateEscape - } + if low, ok := in.peekLowSurrogate(); ok { + if decoded := utf16.DecodeRune(r, low); decoded != utf8.RuneError { + in.takePeeked(surrogateEscapeLen) - high1, highOK1 = scan.Unhex(nextBuf[2]) - low1, lowOK1 = scan.Unhex(nextBuf[3]) - high2, highOK2 = scan.Unhex(nextBuf[4]) - low2, lowOK2 = scan.Unhex(nextBuf[5]) - if !lowOK1 || !highOK1 || !lowOK2 || !highOK2 { - return utf8.RuneError, codes.ErrUnicodeEscape + return decoded, nil } - - unicodeEscape2 := uint32(high1)<<12 + uint32(low1)<<8 + uint32(high2)<<4 + uint32(low2) - r1 := r - r2 := rune(unicodeEscape2) - - r = utf16.DecodeRune(r1, r2) - } - - if !utf8.ValidRune(r) { - return utf8.RuneError, codes.ErrInvalidRune } - return r, nil + return utf8.RuneError, in.brokenSurrogate() } diff --git a/json/lexers/default-lexer/internal/input/string_raw.go b/json/lexers/default-lexer/internal/input/string_raw.go index 2d7e7f2..b0e988b 100644 --- a/json/lexers/default-lexer/internal/input/string_raw.go +++ b/json/lexers/default-lexer/internal/input/string_raw.go @@ -10,7 +10,6 @@ import ( "github.com/go-openapi/core/json/lexers/default-lexer/internal/strscan" "github.com/go-openapi/core/json/lexers/default-lexer/internal/swar" codes "github.com/go-openapi/core/json/lexers/error-codes" - scan "github.com/go-openapi/core/json/lexers/internal/scan" "github.com/go-openapi/core/json/lexers/token" ) @@ -34,16 +33,21 @@ func (in *Input) consumeStringRawWhole() token.T { start := in.Consumed i := start + var hi uint64 // fused non-ASCII accumulator, see consumeStringWhole guard := start + guessLong if in.NoAVX2 { guard = n + 1 // WithoutAVX2: never delegate, pure inline SWAR (see consumeStringWhole) } for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { break // guessLong clean bytes in — delegate below (call kept out of the loop) @@ -53,13 +57,19 @@ func (in *Input) consumeStringRawWhole() token.T { // consumeStringWhole). if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if i >= n { in.Consumed, in.Offset = i, uint64(i) @@ -81,7 +91,7 @@ func (in *Input) consumeStringRawWhole() token.T { i++ in.Consumed, in.Offset = i, uint64(i) - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Consumed, in.Offset = i, uint64(i) @@ -91,7 +101,7 @@ func (in *Input) consumeStringRawWhole() token.T { } // an escape was found at i: validate the rest but keep the raw bytes. - return in.consumeStringRawEscaped(start, i) + return in.consumeStringRawEscaped(start, i, hi&swar.HighBits != 0) } // consumeStringRawEscaped validates a string that contains at least one escape (data[i] == escape) without decoding it, @@ -99,7 +109,10 @@ func (in *Input) consumeStringRawWhole() token.T { // // Clean runs between escapes are skipped with the same adaptive scalar-probe-then-SWAR scan the decoder uses // (consumeStringEscaped), but with no copying — so a sparse-escape string with a long clean tail stays fast. -func (in *Input) consumeStringRawEscaped(start, i int) token.T { +// nonASCII carries the verdict for the clean prefix already scanned and is extended over every further raw run; see +// consumeStringEscaped. Unlike the decoding path, the value here keeps its escapes as source text, so a \u escape +// contributes only its (ASCII) escape bytes. +func (in *Input) consumeStringRawEscaped(start, i int, nonASCII bool) token.T { data := in.Buffer n := in.Bufferized @@ -116,7 +129,7 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { i++ in.Consumed, in.Offset = i, uint64(i) - return in.finishStringValue(value) + return in.finishStringValue(value, nonASCII) case c == escape: i++ @@ -130,7 +143,7 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { case doubleQuote, escape, slash, 'b', 'f', 'n', 'r', 't': i++ case 'u': - next, err := validateUnicodeWhole(data, i+1, n) + next, err := validateUnicodeWhole(data, i+1, n, in.UTF8Policy) if err != nil { in.Consumed, in.Offset = i, uint64(i) in.Err = err @@ -156,34 +169,48 @@ func (in *Input) consumeStringRawEscaped(start, i int) token.T { // (see consumeStringEscaped). run := i stop := i + 1 + hi := uint64(c) // data[i], the byte that entered this run probe := min(stop+swarProbe, n) for ; stop < probe; stop++ { - if b := data[stop]; b == doubleQuote || b == escape || b < controlCharsUpperBound { + b := data[stop] + if b == doubleQuote || b == escape || b < controlCharsUpperBound { break } + hi |= uint64(b) } if stop == probe && stop < n { // outran the scalar probe → SWAR, guess long past guessLong for stop+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[stop:])); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-run >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if b := data[stop]; b == doubleQuote || b == escape || + b := data[stop] + if b == doubleQuote || b == escape || b < controlCharsUpperBound { break } + hi |= uint64(b) } } + nonASCII = nonASCII || hi&swar.HighBits != 0 i = stop } } @@ -215,16 +242,21 @@ func (in *Input) consumeStringRawStreamFast() token.T { start := in.Consumed // first content byte (opening quote already consumed) i := start + var hi uint64 // fused non-ASCII accumulator, see consumeStringWhole guard := start + guessLong if in.NoAVX2 { guard = n + 1 } for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - i += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + i += k break } + hi |= w i += 8 if i >= guard { break @@ -232,13 +264,19 @@ func (in *Input) consumeStringRawStreamFast() token.T { } if i >= guard && i+8 <= n { if c := data[i]; c != doubleQuote && c != escape && c >= controlCharsUpperBound { - i += strscan.ScanStop(data[i:n]) + delta, nonASCII := strscan.ScanStop(data[i:n]) + i += delta + if nonASCII { + hi |= swar.HighBits + } } } for ; i < n; i++ { - if c := data[i]; c == doubleQuote || c == escape || c < controlCharsUpperBound { + c := data[i] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if i >= n { @@ -261,7 +299,7 @@ func (in *Input) consumeStringRawStreamFast() token.T { in.Offset += uint64(end - start) in.Consumed = end - return in.finishStringValue(value) + return in.finishStringValue(value, hi&swar.HighBits != 0) case c < controlCharsUpperBound: in.Offset += uint64(i - start) @@ -279,6 +317,7 @@ func (in *Input) consumeStringRawStreamFast() token.T { // consumeStringRawStreaming is the raw scan over a refilling buffer: it copies the source bytes verbatim (escapes // intact) into l.in.CurrentValue while validating them, so the value survives buffer turnover. func (in *Input) consumeStringRawStreaming() token.T { + var nonASCII bool // did any raw source byte carry the high bit (see consumeStringEscaped) in.CurrentValue = in.CurrentValue[:0] for { @@ -305,7 +344,7 @@ func (in *Input) consumeStringRawStreaming() token.T { switch { case b == doubleQuote: - return in.finishStringValue(in.CurrentValue) + return in.finishStringValue(in.CurrentValue, nonASCII) case b == escape: in.CurrentValue = append(in.CurrentValue, escape) @@ -321,6 +360,7 @@ func (in *Input) consumeStringRawStreaming() token.T { return token.None default: + nonASCII = nonASCII || b >= utf8.RuneSelf in.CurrentValue = append(in.CurrentValue, b) // bulk-scan the rest of this clean run within the current window (§10.5c, the raw analogue of @@ -335,36 +375,48 @@ func (in *Input) consumeStringRawStreaming() token.T { n := in.Bufferized runStart := in.Consumed stop := runStart + var hi uint64 probe := min(stop+swarProbe, n) for ; stop < probe; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } if stop == probe && stop < n { // run outran the scalar probe → SWAR/AVX2 for stop+8 <= n { - if m := swar.StringStopMask( - binary.LittleEndian.Uint64(data[stop:]), - ); m != 0 { - stop += swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[stop:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) + stop += k break } + hi |= w stop += 8 if stop-runStart >= guessLong && !in.NoAVX2 { - stop += strscan.ScanStop(data[stop:n]) + delta, runNonASCII := strscan.ScanStop(data[stop:n]) + stop += delta + if runNonASCII { + hi |= swar.HighBits + } break } } for ; stop < n; stop++ { - if c := data[stop]; c == doubleQuote || c == escape || + c := data[stop] + if c == doubleQuote || c == escape || c < controlCharsUpperBound { break } + hi |= uint64(c) } } + nonASCII = nonASCII || hi&swar.HighBits != 0 if stop > runStart { if in.MaxValueBytes > 0 && len(in.CurrentValue)+(stop-runStart) > in.MaxValueBytes { @@ -406,70 +458,37 @@ func (in *Input) rawEscapeStreaming() error { // // The leading "\u" has already been appended. func (in *Input) rawUnicodeStreaming() error { - var buf [4]byte - if err := in.consumeN(buf[:]); err != nil { - return codes.ErrUnicodeEscape + start := len(in.CurrentValue) + digits, r, err := in.readHex4() + if err != nil { + return err } - code, ok := scan.Hex4(buf[0], buf[1], buf[2], buf[3]) - if !ok { - return codes.ErrUnicodeEscape - } - in.CurrentValue = append(in.CurrentValue, buf[:]...) + in.CurrentValue = append(in.CurrentValue, digits[:]...) - r := rune(code) - if utf16.IsSurrogate(r) { - var nb [6]byte // "\uYYYY" - if err := in.consumeN(nb[:]); err != nil { - return codes.ErrSurrogateEscape - } - if nb[0] != escape || nb[1] != 'u' { - return codes.ErrSurrogateEscape - } - code2, ok2 := scan.Hex4(nb[2], nb[3], nb[4], nb[5]) - if !ok2 { - return codes.ErrUnicodeEscape - } - if utf16.DecodeRune(r, rune(code2)) == utf8.RuneError { - return codes.ErrSurrogateEscape - } - in.CurrentValue = append(in.CurrentValue, nb[:]...) - } else if !utf8.ValidRune(r) { - return codes.ErrInvalidRune + if !utf16.IsSurrogate(r) { + return nil } - return nil -} + // a high surrogate must be completed by \uXXXX; peek so an escape that does not complete it is left to be + // validated on its own (see [Input.peekLowSurrogate]). + if low, ok := in.peekLowSurrogate(); ok { + if utf16.DecodeRune(r, low) != utf8.RuneError { + in.CurrentValue = append( + in.CurrentValue, + in.Buffer[in.Consumed:in.Consumed+surrogateEscapeLen]...) + in.takePeeked(surrogateEscapeLen) -// validateUnicodeWhole validates a \uXXXX sequence at pos (first hex digit, past the 'u') in a whole buffer, following -// one surrogate pair when present. -// -// It returns the index just past the validated sequence, or an error. -func validateUnicodeWhole(data []byte, pos, n int) (int, error) { - if pos+4 > n { - return pos, codes.ErrUnicodeEscape - } - code, ok := scan.Hex4(data[pos], data[pos+1], data[pos+2], data[pos+3]) - if !ok { - return pos, codes.ErrUnicodeEscape + return nil + } } - pos += 4 - r := rune(code) - if utf16.IsSurrogate(r) { - if pos+6 > n || data[pos] != escape || data[pos+1] != 'u' { - return pos, codes.ErrSurrogateEscape - } - code2, ok2 := scan.Hex4(data[pos+2], data[pos+3], data[pos+4], data[pos+5]) - if !ok2 { - return pos, codes.ErrUnicodeEscape - } - if utf16.DecodeRune(r, rune(code2)) == utf8.RuneError { - return pos, codes.ErrSurrogateEscape - } - pos += 6 - } else if !utf8.ValidRune(r) { - return pos, codes.ErrInvalidRune + if berr := in.brokenSurrogate(); berr != nil { + in.CurrentValue = in.CurrentValue[:start] // the value is discarded with the error; keep it consistent anyway + + return berr } - return pos, nil + // UTF8Replace / UTF8Passthrough: VL keeps the escape as SOURCE TEXT, so nothing is rewritten here. The U+FFFD + // substitution happens on decode, in token.Unescape, which already yields it for a broken pair (plan §2.5). + return nil } diff --git a/json/lexers/default-lexer/internal/input/utf8.go b/json/lexers/default-lexer/internal/input/utf8.go new file mode 100644 index 0000000..37d7fbb --- /dev/null +++ b/json/lexers/default-lexer/internal/input/utf8.go @@ -0,0 +1,209 @@ +package input + +import ( + "unicode/utf16" + "unicode/utf8" + + "github.com/go-openapi/core/json/internal/utf8x" + codes "github.com/go-openapi/core/json/lexers/error-codes" + scan "github.com/go-openapi/core/json/lexers/internal/scan" +) + +// surrogateEscapeLen is the width of the `\uXXXX` that must follow a high surrogate to complete a pair. +const surrogateEscapeLen = 6 + +// The UTF-8 byte order mark, and the two UTF-16 ones — the latter recognized only to turn an unhelpful error into an +// accurate one. +const ( + bomUTF8Len = 3 + bom0, bom1, bom2 = 0xEF, 0xBB, 0xBF + + bomUTF16Len = 2 + bomUTF16LE0 = 0xFF + bomUTF16LE1 = 0xFE + bomUTF16BE0 = 0xFE + bomUTF16BE1 = 0xFF +) + +// CheckBOM consumes a UTF-8 byte order mark opening the input, and rejects a UTF-16 one. +// +// RFC 8259 §8.1 forbids *emitting* a BOM but explicitly allows an implementation to ignore one on input, and enough +// producers emit one that rejecting the document outright is unhelpful. +// +// It runs once, before the first token, on the buffer's opening bytes only — so it costs one comparison per document +// and touches no scan loop. Only the complete three-byte sequence at offset 0 is consumed: a truncated BOM, or a +// U+FEFF anywhere else, stays what it is (an invalid token between values, or an ordinary character inside a string), +// which is what keeps n_structure_incomplete_UTF8_BOM and n_structure_UTF8_BOM_no_data rejected. +// +// The mark is NOT re-emitted by the verbatim lexer: it is consumed before any token exists, so it is not part of any +// token's leading space (see [VL.LeadingSpace]). A round-trip therefore drops it — the one documented exception to +// VL's byte-exactness on valid input, and the direction RFC 8259 §8.1 prefers. +// +// A UTF-16 document cannot be lexed at all (we are UTF-8 only) and would otherwise fail with a baffling "invalid JSON +// token" on its first byte; codes.ErrNotUTF8 says what is actually wrong. Nothing is consumed there — the document is +// rejected either way — and 0xFF/0xFE can never legitimately open a JSON value, so there is no input this misjudges. +func (in *Input) CheckBOM() { + if in.Consumed != 0 || in.Offset != 0 || in.Bufferized < bomUTF16Len { + return + } + + data := in.Buffer + switch { + case (data[0] == bomUTF16LE0 && data[1] == bomUTF16LE1) || + (data[0] == bomUTF16BE0 && data[1] == bomUTF16BE1): + in.Err = codes.ErrNotUTF8 + + case in.Bufferized >= bomUTF8Len && + data[0] == bom0 && data[1] == bom1 && data[2] == bom2: + in.Consumed, in.Offset = bomUTF8Len, bomUTF8Len + } +} + +// brokenSurrogate applies the UTF-8 policy to a \u escape that does not denote a Unicode scalar value: an unpaired +// high surrogate, an inverted pair, or a lone low surrogate. +// +// A \u escape must always produce SOME rune — the escape text is not the value, so there is nothing to pass through — +// which is why UTF8Passthrough behaves like UTF8Replace here and only UTF8Strict errors. +// +// Both lexers route through this so they cannot drift apart again: L used to launder the failure into a silent U+FFFD +// (utf16.DecodeRune returns utf8.RuneError, and utf8.ValidRune(U+FFFD) is true) while VL rejected it. +func (in *Input) brokenSurrogate() error { + if in.UTF8Policy == UTF8Strict { + return codes.ErrSurrogateEscape + } + + return nil +} + +// peekLowSurrogate inspects — WITHOUT consuming — the next `\uXXXX` and reports the code unit it carries. +// +// It does not consume, because a sequence that turns out not to complete a pair must be left for the caller to process +// as an escape in its own right: `\uD800\uD800` is two ill-formed units and must yield two U+FFFD, not one. +func (in *Input) peekLowSurrogate() (rune, bool) { + if !in.ensureWindow(surrogateEscapeLen) { + return 0, false + } + + w := in.Buffer[in.Consumed : in.Consumed+surrogateEscapeLen] + if w[0] != escape || w[1] != 'u' { + return 0, false + } + + code, ok := scan.Hex4(w[2], w[3], w[4], w[5]) + if !ok { + return 0, false + } + + return rune(code), true +} + +// takePeeked consumes n bytes previously inspected by [Input.peekLowSurrogate]. +func (in *Input) takePeeked(n int) { + in.Consumed += n + in.Offset += uint64(n) +} + +// readHex4 consumes the four hex digits of a \uXXXX escape (the leading "\u" is already consumed) and returns them +// verbatim along with the code unit they encode. The digits are unambiguously part of this escape, so consuming them +// needs no lookahead. +// +// The raw digits are returned because the verbatim scanner appends them to its value and cannot read them back off the +// window: consumeN may have crossed a refill. +func (in *Input) readHex4() ([4]byte, rune, error) { + var buf [4]byte + if err := in.consumeN(buf[:]); err != nil { + return buf, 0, codes.ErrUnicodeEscape + } + + code, ok := scan.Hex4(buf[0], buf[1], buf[2], buf[3]) + if !ok { + return buf, 0, codes.ErrUnicodeEscape + } + + return buf, rune(code), nil +} + +// validateUnicodeWhole validates a \uXXXX sequence at pos (first hex digit, past the 'u') in a whole buffer, following +// one surrogate pair when present. +// +// It returns the index just past the validated sequence, or an error. A `\uXXXX` that does not complete a pair is NOT +// consumed here: it is left to be validated as its own escape on the next turn (see [Input.peekLowSurrogate]). +func validateUnicodeWhole(data []byte, pos, n int, policy UTF8Policy) (int, error) { + if pos+4 > n { + return pos, codes.ErrUnicodeEscape + } + + code, ok := scan.Hex4(data[pos], data[pos+1], data[pos+2], data[pos+3]) + if !ok { + return pos, codes.ErrUnicodeEscape + } + pos += 4 + + r := rune(code) + if !utf16.IsSurrogate(r) { + return pos, nil + } + + if pos+surrogateEscapeLen <= n && data[pos] == escape && data[pos+1] == 'u' { + code2, ok2 := scan.Hex4(data[pos+2], data[pos+3], data[pos+4], data[pos+5]) + if ok2 && utf16.DecodeRune(r, rune(code2)) != utf8.RuneError { + return pos + surrogateEscapeLen, nil + } + } + + if policy == UTF8Strict { + return pos, codes.ErrSurrogateEscape + } + + return pos, nil +} + +// handleInvalidUTF8 applies the configured [UTF8Policy] to a string value that has just been found to carry an +// ill-formed UTF-8 sequence. +// +// It is deliberately out of line: it is unreachable for valid input, so keeping it out of finishStringValue's frame +// leaves the funnel's codegen to the fast path (the same split the string fast/slow paths use). +// +// It returns the value to emit and whether lexing may continue; under [UTF8Strict] it sets in.Err and returns false. +func (in *Input) handleInvalidUTF8(value []byte) ([]byte, bool) { + if in.UTF8Policy == UTF8Strict { + in.Err = codes.ErrInvalidUTF8 + in.Offset = in.faultOffset(value) + + return nil, false + } + + // UTF8Replace: rewrite into a dedicated scratch. It cannot be in.CurrentValue, which IS the value on the + // unescaping paths. + if in.MaxValueBytes > 0 && utf8x.SanitizedLen(value) > in.MaxValueBytes { + // substituting U+FFFD can grow the value (3 bytes for every invalid byte), so the cap is re-checked against the + // rewritten length, not the scanned one. + in.Err = codes.ErrMaxValueBytes + + return nil, false + } + + in.sanitized = utf8x.Sanitize(in.sanitized[:0], value) + + return in.sanitized, true +} + +// faultOffset maps the first ill-formed byte of value back to a stream offset. +// +// On entry the cursor sits just past the closing quote, so the value body starts at Offset-len(value)-1. That is exact +// for a value aliased from the input (every non-escaped path, and so every case a caller is likely to debug); when +// escapes were decoded the value is shorter than its source span, and the reported offset is correspondingly an +// under-estimate pointing inside the same string. +func (in *Input) faultOffset(value []byte) uint64 { + idx := utf8x.FirstInvalid(value) + if idx < 0 { + return in.Offset + } + + back := uint64(len(value)-idx) + 1 + if back > in.Offset { + return 0 + } + + return in.Offset - back +} diff --git a/json/lexers/default-lexer/internal/strscan/_asm/asm.go b/json/lexers/default-lexer/internal/strscan/_asm/asm.go index efaff32..5de7b83 100644 --- a/json/lexers/default-lexer/internal/strscan/_asm/asm.go +++ b/json/lexers/default-lexer/internal/strscan/_asm/asm.go @@ -10,16 +10,29 @@ package main import ( . "github.com/mmcloughlin/avo/build" . "github.com/mmcloughlin/avo/operand" + "github.com/mmcloughlin/avo/reg" ) func main() { - TEXT("stringStopIndexAVX2", NOSPLIT, "func(data []byte) int") + stringStop() + + Generate() +} + +func stringStop() { + TEXT("stringStopIndexAVX2", NOSPLIT, "func(data []byte) (int, bool)") Doc( - "stringStopIndexAVX2 returns the index of the first byte that is < 0x20, '\"' (0x22) or '\\' (0x5c), or len(data) if none. AVX2, 32 bytes/iter.", + "stringStopIndexAVX2 returns the index of the first byte that is < 0x20, '\"' (0x22) or '\\' (0x5c), or len(data) if none, and whether any byte BEFORE that index has its high bit set (i.e. the scanned run is not pure ASCII). AVX2, 32 bytes/iter.", ) ptr := Load(Param("data").Base(), GP64()) n := Load(Param("data").Len(), GP64()) + // nonascii accumulates "some scanned byte was >= 0x80". In the vector loop it collects VPMOVMSKB sign-bit masks; + // in the scalar tail it collects the byte's 0x80 bit. Only its zero/non-zero state is ever read, so mixing the two + // encodings is safe. + nonascii := GP32() + XORL(nonascii, nonascii) + c1f := YMM() VPBROADCASTB(ConstData("c1f", U8(0x1f)), c1f) c22 := YMM() @@ -49,18 +62,37 @@ func main() { acc := YMM() VPOR(ctrl, q, acc) VPOR(acc, bs, acc) + // the sign bit of every lane IS "byte >= 0x80", so the non-ASCII test is one extra VPMOVMSKB on the block already + // loaded — off the loop's exit-test dependency chain. + hmask := GP32() + VPMOVMSKB(data, hmask) mask := GP32() VPMOVMSKB(acc, mask) TESTL(mask, mask) JNZ(LabelRef("found")) + ORL(hmask, nonascii) // whole block is string content: account for all 32 lanes ADDQ(Imm(32), i) JMP(LabelRef("loop32")) Label("found") off := GP64() TZCNTL(mask, off.As32()) // bit index 0..31, zero-extends to 64 + // Only the lanes BEFORE the stop belong to the value, so keep the bits of hmask strictly below mask's lowest set + // bit: (mask-1) &^ mask. Four plain ALU ops — deliberately not BMI's BLSMSK/ANDN, since the CPUID gate here checks + // AVX2 only (TZCNT above is safe because it decodes as BSF, which agrees for a non-zero operand; ANDN has no such + // fallback). + below := GP32() + MOVL(mask, below) + DECL(below) + notMask := GP32() + MOVL(mask, notMask) + NOTL(notMask) + ANDL(notMask, below) + ANDL(below, hmask) + ORL(hmask, nonascii) ADDQ(off, i) Store(i, ReturnIndex(0)) + storeNonASCII(nonascii) VZEROUPPER() RET() @@ -76,18 +108,28 @@ func main() { JE(LabelRef("foundtail")) CMPL(b, Imm(0x5c)) JE(LabelRef("foundtail")) + ANDL(Imm(0x80), b) // keep only "is this byte >= 0x80"; b is dead after the comparisons + ORL(b, nonascii) INCQ(i) JMP(LabelRef("tailloop")) Label("foundtail") Store(i, ReturnIndex(0)) + storeNonASCII(nonascii) VZEROUPPER() RET() Label("notfound") Store(n, ReturnIndex(0)) + storeNonASCII(nonascii) VZEROUPPER() RET() +} - Generate() +// storeNonASCII writes (nonascii != 0) to the bool result. +func storeNonASCII(nonascii reg.GPVirtual) { + TESTL(nonascii, nonascii) + flag := GP8() + SETNE(flag) + Store(flag, ReturnIndex(1)) } diff --git a/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go b/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go index 9b8f837..e2ef789 100644 --- a/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go +++ b/json/lexers/default-lexer/internal/strscan/bench_amd64_test.go @@ -25,13 +25,15 @@ func BenchmarkStringStop(b *testing.B) { b.Run(fmt.Sprintf("len=%04d/SWAR", n), func(b *testing.B) { b.SetBytes(int64(n)) for b.Loop() { - sink += scanStopSWAR(data) + idx, _ := scanStopSWAR(data) + sink += idx } }) b.Run(fmt.Sprintf("len=%04d/AVX2", n), func(b *testing.B) { b.SetBytes(int64(n)) for b.Loop() { - sink += stringStopIndexAVX2(data) + idx, _ := stringStopIndexAVX2(data) + sink += idx } }) } diff --git a/json/lexers/default-lexer/internal/strscan/scan_amd64.go b/json/lexers/default-lexer/internal/strscan/scan_amd64.go index a607db9..783dac9 100644 --- a/json/lexers/default-lexer/internal/strscan/scan_amd64.go +++ b/json/lexers/default-lexer/internal/strscan/scan_amd64.go @@ -7,8 +7,12 @@ package strscan // stringStopIndexAVX2 is the avo-generated AVX2 kernel (stringstop_amd64.s): it returns the index of the first byte < // 0x20, '"' (0x22) or '\' (0x5c), or len(data) if none, scanning 32 bytes per YMM iteration. // +// The second result reports whether any byte BEFORE that index has its high bit set. It is free: the sign bit of every +// lane already means "byte >= 0x80", so one extra VPMOVMSKB on the block already in the register, off the loop's +// exit-test dependency chain, tells the caller whether the run it just scanned needs UTF-8 validation at all. +// // It uses AVX/AVX2/BMI instructions and must only be called when [useAVX2] is true. -func stringStopIndexAVX2(data []byte) int +func stringStopIndexAVX2(data []byte) (int, bool) // avx2Min is the minimum remaining-byte count at which the AVX2 kernel beats the 8-byte SWAR loop, once its 3-constant // broadcast setup and the (non-inlinable) call are accounted for. @@ -18,12 +22,15 @@ func stringStopIndexAVX2(data []byte) int const avx2Min = 32 // ScanStop returns the index of the first JSON string-stop byte (a control char < 0x20, '"', or '\') in data, or -// len(data) if none. +// len(data) if none, together with whether any byte before that index is >= 0x80. // // It is called only for the long-string case (the caller found a clean first word), so it dispatches to the AVX2 kernel // when the CPU supports it and enough bytes remain, and to SWAR otherwise. // The WithoutAVX2 knob is handled by the caller (it simply never delegates here), so ScanStop always tries the kernel. -func ScanStop(data []byte) int { +// +// The non-ASCII result is what lets the string scanners fuse UTF-8 detection into the scan they already perform: a run +// this reports as pure ASCII is proven valid UTF-8 with no second pass. Both implementations answer identically. +func ScanStop(data []byte) (int, bool) { if useAVX2 && len(data) >= avx2Min { return stringStopIndexAVX2(data) } diff --git a/json/lexers/default-lexer/internal/strscan/scan_noasm.go b/json/lexers/default-lexer/internal/strscan/scan_noasm.go index be22352..104633f 100644 --- a/json/lexers/default-lexer/internal/strscan/scan_noasm.go +++ b/json/lexers/default-lexer/internal/strscan/scan_noasm.go @@ -3,10 +3,10 @@ package strscan // ScanStop returns the index of the first JSON string-stop byte (a control char < 0x20, '"', or '\') in data, or -// len(data) if none. +// len(data) if none, together with whether any byte before that index is >= 0x80. // // On non-amd64 there is no vector kernel, so the long-string path is the same 8-byte SWAR scan the inline fast path // uses. -func ScanStop(data []byte) int { +func ScanStop(data []byte) (int, bool) { return scanStopSWAR(data) } diff --git a/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s b/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s index ecf8efc..0f45e6f 100644 --- a/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s +++ b/json/lexers/default-lexer/internal/strscan/stringstop_amd64.s @@ -2,61 +2,81 @@ #include "textflag.h" -// func stringStopIndexAVX2(data []byte) int +// func stringStopIndexAVX2(data []byte) (int, bool) // Requires: AVX, AVX2, BMI -TEXT ·stringStopIndexAVX2(SB), NOSPLIT, $0-32 +TEXT ·stringStopIndexAVX2(SB), NOSPLIT, $0-33 MOVQ data_base+0(FP), AX MOVQ data_len+8(FP), CX + XORL DX, DX VPBROADCASTB c1f<>+0(SB), Y0 VPBROADCASTB c22<>+0(SB), Y1 VPBROADCASTB c5c<>+0(SB), Y2 - XORQ DX, DX + XORQ BX, BX loop32: - LEAQ 32(DX), BX - CMPQ BX, CX + LEAQ 32(BX), SI + CMPQ SI, CX JG tail - VMOVDQU (AX)(DX*1), Y3 + VMOVDQU (AX)(BX*1), Y3 VPMINUB Y3, Y0, Y4 VPCMPEQB Y4, Y3, Y4 VPCMPEQB Y3, Y1, Y5 - VPCMPEQB Y3, Y2, Y3 + VPCMPEQB Y3, Y2, Y6 VPOR Y4, Y5, Y4 - VPOR Y4, Y3, Y4 - VPMOVMSKB Y4, BX - TESTL BX, BX + VPOR Y4, Y6, Y4 + VPMOVMSKB Y3, SI + VPMOVMSKB Y4, DI + TESTL DI, DI JNZ found - ADDQ $0x20, DX + ORL SI, DX + ADDQ $0x20, BX JMP loop32 found: - TZCNTL BX, AX - ADDQ AX, DX - MOVQ DX, ret+24(FP) + TZCNTL DI, AX + MOVL DI, CX + DECL CX + NOTL DI + ANDL DI, CX + ANDL CX, SI + ORL SI, DX + ADDQ AX, BX + MOVQ BX, ret+24(FP) + TESTL DX, DX + SETNE AL + MOVB AL, ret1+32(FP) VZEROUPPER RET tail: tailloop: - CMPQ DX, CX + CMPQ BX, CX JGE notfound - MOVBLZX (AX)(DX*1), BX - CMPL BX, $0x20 + MOVBLZX (AX)(BX*1), SI + CMPL SI, $0x20 JL foundtail - CMPL BX, $0x22 + CMPL SI, $0x22 JE foundtail - CMPL BX, $0x5c + CMPL SI, $0x5c JE foundtail - INCQ DX + ANDL $0x80, SI + ORL SI, DX + INCQ BX JMP tailloop foundtail: - MOVQ DX, ret+24(FP) + MOVQ BX, ret+24(FP) + TESTL DX, DX + SETNE AL + MOVB AL, ret1+32(FP) VZEROUPPER RET notfound: - MOVQ CX, ret+24(FP) + MOVQ CX, ret+24(FP) + TESTL DX, DX + SETNE AL + MOVB AL, ret1+32(FP) VZEROUPPER RET diff --git a/json/lexers/default-lexer/internal/strscan/strscan.go b/json/lexers/default-lexer/internal/strscan/strscan.go index 6226a83..24b418f 100644 --- a/json/lexers/default-lexer/internal/strscan/strscan.go +++ b/json/lexers/default-lexer/internal/strscan/strscan.go @@ -23,19 +23,31 @@ import ( // [swar.StringStopMask]/[swar.FirstByte] so a stop is found at exactly the same byte here as on the inline fast path. // // It is the non-amd64 implementation of [ScanStop] and the amd64 short-remainder fallback. -func scanStopSWAR(data []byte) int { +// +// The second result reports whether any byte strictly before the stop is >= 0x80, accumulated from the words the scan +// already loaded (the exact analogue of the AVX2 kernel's VPMOVMSKB accumulator). Lanes at or after the stop are +// masked out so the answer is exact, not merely conservative. +func scanStopSWAR(data []byte) (int, bool) { n, i := len(data), 0 + var hi uint64 for i+8 <= n { - if m := swar.StringStopMask(binary.LittleEndian.Uint64(data[i:])); m != 0 { - return i + swar.FirstByte(m) + w := binary.LittleEndian.Uint64(data[i:]) + if m := swar.StringStopMask(w); m != 0 { + k := swar.FirstByte(m) + hi |= swar.LanesBelow(w, k) // only the lanes before the stop belong to the value + + return i + k, hi&swar.HighBits != 0 } + hi |= w i += 8 } for ; i < n; i++ { - if b := data[i]; b < 0x20 || b == '"' || b == '\\' { - return i + b := data[i] + if b < 0x20 || b == '"' || b == '\\' { + return i, hi&swar.HighBits != 0 } + hi |= uint64(b) } - return n + return n, hi&swar.HighBits != 0 } diff --git a/json/lexers/default-lexer/internal/strscan/strscan_test.go b/json/lexers/default-lexer/internal/strscan/strscan_test.go index 93c89e8..162df27 100644 --- a/json/lexers/default-lexer/internal/strscan/strscan_test.go +++ b/json/lexers/default-lexer/internal/strscan/strscan_test.go @@ -1,19 +1,23 @@ package strscan import ( + "bytes" "math/rand" "testing" ) -// scalarStop is the reference oracle: first index of a byte < 0x20 || '"' || '\'. -func scalarStop(data []byte) int { +// scalarStop is the reference oracle: first index of a byte < 0x20 || '"' || '\', and whether any byte STRICTLY +// BEFORE that index is >= 0x80. +func scalarStop(data []byte) (int, bool) { + var nonASCII bool for i, b := range data { if b < 0x20 || b == '"' || b == '\\' { - return i + return i, nonASCII } + nonASCII = nonASCII || b >= 0x80 } - return len(data) + return len(data), nonASCII } // TestScanStopOracle sweeps ScanStop (the shipped gate, AVX2 or SWAR depending on the host) and the portable @@ -47,14 +51,17 @@ func TestScanStopOracle(t *testing.T) { } } - want := scalarStop(data) + want, wantNonASCII := scalarStop(data) // the AVX2-gated path and the portable SWAR path (which WithoutAVX2 forces the lexer onto) must both match the - // scalar oracle. - if got := ScanStop(data); got != want { - t.Fatalf("ScanStop n=%d trial=%d: got %d want %d", n, trial, got, want) + // scalar oracle — for the stop index AND for the fused non-ASCII verdict the string scanners rely on to skip + // UTF-8 validation. + if got, gotNonASCII := ScanStop(data); got != want || gotNonASCII != wantNonASCII { + t.Fatalf("ScanStop n=%d trial=%d: got (%d,%v) want (%d,%v)", + n, trial, got, gotNonASCII, want, wantNonASCII) } - if got := scanStopSWAR(data); got != want { - t.Fatalf("scanStopSWAR n=%d trial=%d: got %d want %d", n, trial, got, want) + if got, gotNonASCII := scanStopSWAR(data); got != want || gotNonASCII != wantNonASCII { + t.Fatalf("scanStopSWAR n=%d trial=%d: got (%d,%v) want (%d,%v)", + n, trial, got, gotNonASCII, want, wantNonASCII) } } } @@ -68,12 +75,89 @@ func TestScanStopHighBytesSafe(t *testing.T) { for i := range data { data[i] = 0x80 | byte(i&0x3f) } - if got := ScanStop(data); got != len(data) { - t.Fatalf("high bytes flagged as stop: got %d, want %d", got, len(data)) + if got, nonASCII := ScanStop(data); got != len(data) || !nonASCII { + t.Fatalf( + "high bytes flagged as stop: got (%d,%v), want (%d,true)", + got, + nonASCII, + len(data), + ) } data[70] = 0x1f - if got := ScanStop(data); got != 70 { - t.Fatalf("got %d, want 70", got) + if got, nonASCII := ScanStop(data); got != 70 || !nonASCII { + t.Fatalf("got (%d,%v), want (70,true)", got, nonASCII) + } +} + +// TestScanStopNonASCIIExactness pins the boundary the fused UTF-8 detection depends on: bytes at or after the stop +// must NOT count. A conservative over-report would only cost a redundant validation, but a *false negative* would let +// invalid UTF-8 through, so the mask has to be exact in the direction that matters and is checked both ways here. +func TestScanStopNonASCIIExactness(t *testing.T) { + cases := []struct { + name string + data []byte + wantStop int + wantNonASCII bool + }{ + {name: "empty", data: []byte{}, wantStop: 0}, + {name: "ascii only, no stop", data: []byte("plain ascii run"), wantStop: 15}, + {name: "stop first, high byte after", data: append([]byte{'"'}, 0xc3, 0xa9), wantStop: 0}, + { + name: "high byte right after a late stop", + data: append(append([]byte("0123456789abcdefghijklmnopqrstuv"), '"'), 0xff, 0xff), + wantStop: 32, + }, + { + name: "high byte right before a late stop", + data: append(append([]byte("0123456789abcdefghijklmnopqrstu"), 0xc3), '"'), + wantStop: 32, + wantNonASCII: true, + }, + { + name: "high byte in the scalar tail", + data: append(bytes.Repeat([]byte{'a'}, 40), 0xe2, 0x82, 0xac, '"'), + wantStop: 43, + wantNonASCII: true, + }, + { + name: "tail high byte after the stop", + data: append(bytes.Repeat([]byte{'a'}, 40), '"', 0xe2, 0x82, 0xac), + wantStop: 40, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + wantStop, wantNonASCII := scalarStop(tc.data) + if wantStop != tc.wantStop || wantNonASCII != tc.wantNonASCII { + t.Fatalf("the case table disagrees with the oracle: table (%d,%v), oracle (%d,%v)", + tc.wantStop, tc.wantNonASCII, wantStop, wantNonASCII) + } + if got, nonASCII := ScanStop( + tc.data, + ); got != tc.wantStop || + nonASCII != tc.wantNonASCII { + t.Errorf( + "ScanStop: got (%d,%v), want (%d,%v)", + got, + nonASCII, + tc.wantStop, + tc.wantNonASCII, + ) + } + if got, nonASCII := scanStopSWAR( + tc.data, + ); got != tc.wantStop || + nonASCII != tc.wantNonASCII { + t.Errorf( + "scanStopSWAR: got (%d,%v), want (%d,%v)", + got, + nonASCII, + tc.wantStop, + tc.wantNonASCII, + ) + } + }) } } diff --git a/json/lexers/default-lexer/internal/swar/swar.go b/json/lexers/default-lexer/internal/swar/swar.go index e9b8fc3..4da0742 100644 --- a/json/lexers/default-lexer/internal/swar/swar.go +++ b/json/lexers/default-lexer/internal/swar/swar.go @@ -32,6 +32,20 @@ const ( // lowest set bit belongs to the lowest-address lane. func FirstByte(mask uint64) int { return bits.TrailingZeros64(mask) >> 3 } +// HighBits is the high bit of every lane: a word OR-accumulated over a byte run is non-ASCII iff (acc & HighBits) != 0. +// +// This is what makes UTF-8 detection free in the string scanners: every word the stop-scan loads is OR-ed into an +// accumulator, so a value proven to be pure ASCII — the overwhelmingly common case — is proven valid UTF-8 with no +// second pass over its bytes. +const HighBits = high + +// LanesBelow returns w with every lane from index k upward cleared, for k in 0..8. +// +// It is used to trim the word in which a scan stopped: only the lanes BEFORE the stop belong to the value, so only +// they may contribute to a non-ASCII verdict. Go defines an over-wide shift as zero, so k == 0 (stop in the first +// lane, nothing precedes it) correctly yields 0 with no branch. +func LanesBelow(w uint64, k int) uint64 { return w & (^uint64(0) >> (64 - 8*k)) } + // Broadcast replicates b into all eight lanes of a word. func Broadcast(b byte) uint64 { return lo * uint64(b) } diff --git a/json/lexers/default-lexer/options.go b/json/lexers/default-lexer/options.go index 6811b49..ca3fbf0 100644 --- a/json/lexers/default-lexer/options.go +++ b/json/lexers/default-lexer/options.go @@ -1,5 +1,7 @@ package lexer +import "github.com/go-openapi/core/json/lexers/default-lexer/internal/input" + type ( // Option for the lexer. Option func(options) options @@ -11,10 +13,67 @@ type ( keepPreviousBuffer int elideSeparator bool noAVX2 bool + utf8Policy UTF8Policy jsonPointer bool } ) +// UTF8Policy selects what the lexer does with a string that is not valid UTF-8 — an ill-formed byte sequence in the +// source, or a \u escape that does not denote a Unicode scalar value. +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, so the default ([UTF8Strict]) rejects such input. See +// [WithUTF8Policy]. +type UTF8Policy = input.UTF8Policy + +const ( + // UTF8Strict rejects the document: [L.Err] reports [codes.ErrInvalidUTF8] for ill-formed bytes, or + // [codes.ErrSurrogateEscape] for a broken \u surrogate pair. + // + // This is the default for both [L] and [VL], and it never copies: validation only reads the value the scan already + // produced, so the zero-copy aliasing of string values is unaffected. + UTF8Strict = input.UTF8Strict + + // UTF8Replace accepts the document and substitutes U+FFFD (the replacement character) — one per invalid byte, one + // per broken escape — so every emitted value is valid UTF-8. + // + // Only an offending value is rewritten (and therefore copied out of the input); valid values are still aliased with + // zero copy. + // + // A mangled value no longer maps onto its source text: U+FFFD is three bytes replacing one, so its length differs + // from the source span and an index into it cannot be turned into a source offset. Token POSITIONS are unaffected + // — [L.Offset], [VL.Line], [VL.Column] and [VL.LeadingSpace] come from the scan cursor, never from the value. See + // the [VL] documentation for the full contract. + UTF8Replace = input.UTF8Replace + + // UTF8Passthrough disables raw-byte validation: ill-formed sequences are passed to the caller untouched, exactly as + // the lexer behaved before UTF-8 validation existed. + // + // A \u escape still yields U+FFFD when it does not denote a scalar value, because an escape must always decode to + // some rune — there is nothing to "pass through" for an escape, the escape text is not the value. + // + // UNSAFE: use it only for input already known to be valid UTF-8 (or when a downstream stage validates). Invalid + // bytes will reach your []byte and silently become U+FFFD the moment the value is iterated as a string. + UTF8Passthrough = input.UTF8Passthrough +) + +// WithUTF8Policy selects how string values that are not valid UTF-8 are handled: rejected ([UTF8Strict], the default), +// sanitized to U+FFFD ([UTF8Replace]), or waved through unchecked ([UTF8Passthrough]). +// +// It applies to both the semantic lexer [L] and the verbatim lexer [VL], and it covers both ways a value can carry a +// non-character: ill-formed UTF-8 in the source bytes, and a \u escape that does not denote a Unicode scalar value +// (an unpaired or inverted surrogate). +// +// The strategy used to enforce the policy is not a knob: detection of non-ASCII bytes is fused into the string scan +// the lexer already performs, so an all-ASCII value — the overwhelmingly common case — is proven valid with no second +// pass. Only values that actually carry a byte >= 0x80 are validated. +func WithUTF8Policy(policy UTF8Policy) Option { + return func(o options) options { + o.utf8Policy = policy + + return o + } +} + const defaultBufferBytes = 4096 // bufferSizeAlignment is the granularity to which a caller-supplied buffer size is rounded up. diff --git a/json/lexers/default-lexer/semantic_lexer.go b/json/lexers/default-lexer/semantic_lexer.go index 3096163..72667d7 100644 --- a/json/lexers/default-lexer/semantic_lexer.go +++ b/json/lexers/default-lexer/semantic_lexer.go @@ -226,6 +226,7 @@ func (l *L) reset() { l.in.MaxValueBytes = l.maxValueBytes l.in.KeepPreviousBuffer = l.keepPreviousBuffer l.in.NoAVX2 = l.noAVX2 + l.in.UTF8Policy = l.utf8Policy l.current = token.None l.in.Offset = 0 l.in.Consumed = 0 @@ -239,6 +240,12 @@ func (l *L) reset() { l.tokCol = 0 l.in.CurrentValue = l.in.CurrentValue[:0] // TODO: possibly preallocate value buffer to some configurable size + if l.in.WholeBuffer { + // a bound buffer can be inspected right away; a streaming lexer is checked after its first fill instead (see + // input.FirstFill), which is also where a stream promoted to whole-buffer mode is handled. + l.in.CheckBOM() + } + if l.nestingLevel != nil { l.nestingLevel = l.nestingLevel[:1] } else { diff --git a/json/lexers/default-lexer/testdata/conformance_i_behavior.golden b/json/lexers/default-lexer/testdata/conformance_i_behavior.golden new file mode 100644 index 0000000..40c420a --- /dev/null +++ b/json/lexers/default-lexer/testdata/conformance_i_behavior.golden @@ -0,0 +1,35 @@ +i_number_double_huge_neg_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_huge_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_neg_int_huge_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_pos_double_huge_exp.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_real_neg_overflow.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_real_pos_overflow.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_real_underflow.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_too_big_neg_int.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_too_big_pos_int.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_number_very_big_negative_int.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_object_key_lone_2nd_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_1st_surrogate_but_2nd_missing.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_1st_valid_surrogate_2nd_invalid.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_UTF-16LE_with_BOM.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=reject +i_string_UTF-8_invalid_sequence.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_UTF8_surrogate_U+D800.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_incomplete_surrogate_and_escape_valid.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_incomplete_surrogate_pair.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_incomplete_surrogates_escape_valid.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_invalid_lonely_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_invalid_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_invalid_utf-8.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_inverted_surrogates_U+1D11E.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_iso_latin_1.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_lone_second_surrogate.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_lone_utf8_continuation_byte.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_not_in_unicode_range.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_overlong_sequence_2_bytes.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_overlong_sequence_6_bytes.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_overlong_sequence_6_bytes_null.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_truncated-utf-8.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=accept +i_string_utf16BE_no_BOM.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=reject +i_string_utf16LE_no_BOM.json: L/bytes=reject L/reader=reject VL/bytes=reject VL/reader=reject L/replace=reject +i_structure_500_nested_arrays.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept +i_structure_UTF-8_BOM_empty_object.json: L/bytes=accept L/reader=accept VL/bytes=accept VL/reader=accept L/replace=accept diff --git a/json/lexers/default-lexer/utf8_test.go b/json/lexers/default-lexer/utf8_test.go new file mode 100644 index 0000000..bace211 --- /dev/null +++ b/json/lexers/default-lexer/utf8_test.go @@ -0,0 +1,586 @@ +package lexer + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" + + codes "github.com/go-openapi/core/json/lexers/error-codes" + "github.com/go-openapi/core/json/lexers/token" +) + +// UTF-8 validation of string values (see .claude/plans/utf8-validation.md). +// +// The lexers used to wave every byte >= 0x80 straight into the token value: the string scanners stop only on '"', +// '\\' and control chars, so ill-formed UTF-8 was never looked at. These tests pin the three policies across every +// string path — whole-buffer vs streaming, clean vs escaped, L vs VL — because the detection that gates validation is +// fused into each scanner separately and can only be wrong per-path. + +// utf8Case is one input and what each policy must make of it. +type utf8Case struct { + name string + // body is the string BODY (no quotes); the case is exercised as a value, as an object key, and with a long + // pure-ASCII prefix that forces the AVX2/long-string path. + body string + // wantStrict is the error the default policy must report (nil means the input is valid). + wantStrict error + // wantReplaced is the decoded value under UTF8Replace. + wantReplaced string + // wantRawKept is true when VL keeps the body verbatim under UTF8Replace: it rewrites invalid BYTES but never + // rewrites escape text (the U+FFFD for a broken escape appears on decode, in token.Unescape). + wantRawKept bool +} + +func utf8Cases() []utf8Case { + return []utf8Case{ + // --- valid: must be untouched by every policy --- + {name: "ascii", body: "plain", wantReplaced: "plain"}, + {name: "2-byte", body: "caf\xc3\xa9", wantReplaced: "caf\u00e9"}, + {name: "3-byte", body: "\xe6\x97\xa5\xe6\x9c\xac", wantReplaced: "\u65e5\u672c"}, + {name: "4-byte", body: "\xf0\x9d\x84\x9e", wantReplaced: "\U0001D11E"}, + {name: "max scalar", body: "\xf4\x8f\xbf\xbf", wantReplaced: "\U0010FFFF"}, + {name: "encoded U+FFFD", body: "\xef\xbf\xbd", wantReplaced: "\ufffd"}, + { + name: "valid surrogate pair escape", + body: `\uD834\uDD1E`, + wantReplaced: "\U0001D11E", + wantRawKept: true, + }, + + // --- ill-formed bytes: one U+FFFD per invalid byte --- + {name: "lone ff", body: "\xff", wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd"}, + { + name: "lone continuation", + body: "\x81", + wantStrict: codes.ErrInvalidUTF8, + wantReplaced: "\ufffd", + }, + {name: "latin-1", body: "\xe9", wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd"}, + { + name: "truncated 3-byte", body: "\xe0\xff", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd", + }, + { + name: "valid then invalid lead", body: "\xe6\x97\xa5\xd1\x88\xfa", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\u65e5\u0448\ufffd", + }, + { + name: "encoded surrogate", body: "\xed\xa0\x80", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd\ufffd", + }, + { + name: "beyond U+10FFFF", body: "\xf4\xbf\xbf\xbf", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd\ufffd\ufffd", + }, + { + name: "overlong", body: "\xc0\xaf", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "\ufffd\ufffd", + }, + { + name: "invalid between valid", body: "a\xffb", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "a\ufffdb", + }, + { + name: "truncated at end", body: "ok\xe2\x82", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "ok\ufffd\ufffd", + }, + + // --- ill-formed \u escapes: one U+FFFD per broken escape, and the NEXT escape is not swallowed --- + { + name: "unpaired high surrogate then escape", body: `\uD800\uD800\n`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd\ufffd\n", wantRawKept: true, + }, + { + name: "high surrogate then non-surrogate", body: `\uD888\u1234`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd\u1234", wantRawKept: true, + }, + { + name: "inverted pair", body: `\uDd1e\uD834`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd\ufffd", wantRawKept: true, + }, + { + name: "lone high surrogate at end", body: `\uDADA`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd", wantRawKept: true, + }, + { + name: "lone low surrogate", body: `\uDFAA`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffd", wantRawKept: true, + }, + { + name: "surrogate then literal text", body: `\uD800abc`, + wantStrict: codes.ErrSurrogateEscape, wantReplaced: "\ufffdabc", wantRawKept: true, + }, + + // --- mixed: an escape and an ill-formed byte in the same value --- + { + name: "escape then bad byte", body: "a\\nb\xffc", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "a\nb\ufffdc", + }, + { + name: "bad byte then escape", body: "a\xffb\\tc", + wantStrict: codes.ErrInvalidUTF8, wantReplaced: "a\ufffdb\tc", + }, + } +} + +// longPrefix is longer than guessLong (16) so the scan delegates to the AVX2/long-string kernel, whose fused +// non-ASCII accumulator is a separate implementation from the inline SWAR one. +const longPrefix = "0123456789abcdefghijklmnopqrstuvwxyz0123456789" + +// utf8Driver is one way of running a document through a lexer: the eight string paths are reached by crossing +// {L, VL} x {whole-buffer, streaming} x {clean, escaped}, and the escaped axis comes from the fixtures themselves. +type utf8Driver struct { + name string + // run drains the document and returns the decoded value of the first String/Key token, the raw token value, and + // the lexer error. + run func(doc []byte, policy UTF8Policy) (decoded, raw []byte, err error) +} + +func utf8Drivers() []utf8Driver { + drainL := func(lx *L) (decoded, raw []byte, err error) { + for tok := range lx.Tokens() { + if k := tok.Kind(); (k == token.String || k == token.Key) && decoded == nil { + decoded = bytes.Clone(tok.Value()) + raw = decoded + } + } + + return decoded, raw, lx.Err() + } + drainVL := func(lx *VL) (decoded, raw []byte, err error) { + for tok := range lx.Tokens() { + if k := tok.Kind(); (k == token.String || k == token.Key) && raw == nil { + raw = bytes.Clone(tok.Value()) + decoded = token.Unescape(bytes.Clone(raw)) + } + } + + return decoded, raw, lx.Err() + } + + return []utf8Driver{ + {name: "L/bytes", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainL(NewWithBytes(doc, WithUTF8Policy(p))) + }}, + {name: "L/bytes/noavx2", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainL(NewWithBytes(doc, WithUTF8Policy(p), WithoutAVX2(true))) + }}, + {name: "L/reader", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainL(New(newChunkReader(doc, 7), WithUTF8Policy(p), WithBufferSize(32))) + }}, + {name: "VL/bytes", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainVL(NewVerbatimWithBytes(doc, WithUTF8Policy(p))) + }}, + {name: "VL/bytes/noavx2", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainVL(NewVerbatimWithBytes(doc, WithUTF8Policy(p), WithoutAVX2(true))) + }}, + {name: "VL/reader", run: func(doc []byte, p UTF8Policy) ([]byte, []byte, error) { + return drainVL( + NewVerbatim(newChunkReader(doc, 7), WithUTF8Policy(p), WithBufferSize(32)), + ) + }}, + } +} + +// newChunkReader returns a reader that hands out at most size bytes per Read, so a value — and an ill-formed sequence +// or a \u escape inside it — is forced across refill boundaries. +func newChunkReader(data []byte, size int) *chunkReader { + return &chunkReader{data: data, size: size} +} + +type chunkReader struct { + data []byte + size int + pos int +} + +func (r *chunkReader) Read(p []byte) (int, error) { + if r.pos >= len(r.data) { + return 0, io.EOF + } + n := copy(p, r.data[r.pos:min(r.pos+r.size, len(r.data))]) + r.pos += n + + return n, nil +} + +func TestUTF8Policy(t *testing.T) { + // each fixture is exercised in three shapes, which route to different scanners + shapes := []struct { + name string + doc func(body string) string + }{ + {name: "value", doc: func(b string) string { return `["` + b + `"]` }}, + {name: "key", doc: func(b string) string { return `{"` + b + `":1}` }}, + {name: "long-value", doc: func(b string) string { return `["` + longPrefix + b + `"]` }}, + } + + for _, tc := range utf8Cases() { + for _, shape := range shapes { + doc := []byte(shape.doc(tc.body)) + prefix := "" + if shape.name == "long-value" { + prefix = longPrefix + } + + for _, drv := range utf8Drivers() { + t.Run(tc.name+"/"+shape.name+"/"+drv.name, func(t *testing.T) { + isVerbatim := strings.HasPrefix(drv.name, "VL") + + t.Run("strict", func(t *testing.T) { + decoded, _, err := drv.run(doc, UTF8Strict) + if tc.wantStrict == nil { + require.NoError(t, err) + assert.Equal(t, prefix+tc.wantReplaced, string(decoded)) + + return + } + require.Error(t, err) + assert.ErrorIs(t, err, tc.wantStrict) + }) + + t.Run("replace", func(t *testing.T) { + decoded, raw, err := drv.run(doc, UTF8Replace) + require.NoError(t, err) + assert.Equal(t, prefix+tc.wantReplaced, string(decoded)) + assert.Truef(t, utf8.Valid(decoded), + "an emitted value must always be valid UTF-8, got %q", decoded) + + // VL keeps escape text verbatim; only ill-formed BYTES are rewritten in the raw value. + if isVerbatim && tc.wantRawKept { + assert.Equal(t, prefix+tc.body, string(raw)) + } + }) + + t.Run("passthrough", func(t *testing.T) { + _, raw, err := drv.run(doc, UTF8Passthrough) + require.NoError(t, err) + // ill-formed bytes reach the caller untouched. Only checked for bodies without escapes: L + // decodes them, so its "raw" is the decoded value and would not match the source text. + if tc.wantStrict == codes.ErrInvalidUTF8 && + !strings.Contains(tc.body, `\`) { + assert.Equal(t, prefix+tc.body, string(raw)) + } + }) + }) + } + } + } +} + +// TestUTF8StrictErrorOffset pins that the reported offset points at the first ill-formed byte, not at the end of the +// value — the reason FirstInvalid exists next to Valid. +func TestUTF8StrictErrorOffset(t *testing.T) { + cases := []struct { + name string + doc string + want uint64 + wantByte byte + }{ + {name: "first byte of the value", doc: "[\"\xffabc\"]", want: 2, wantByte: 0xff}, + // the truncated 3-byte lead is the start of the ill-formed sequence, not the 0xff that reveals it + {name: "after a valid prefix", doc: "[\"ok\xe0\xff\"]", want: 4, wantByte: 0xe0}, + { + name: "in a long value", doc: "[\"" + longPrefix + "\xff\"]", + want: uint64(2 + len(longPrefix)), wantByte: 0xff, + }, + {name: "in a key", doc: "{\"k\xff\":1}", want: 3, wantByte: 0xff}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + lx := NewWithBytes([]byte(tc.doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.ErrorIs(t, lx.Err(), codes.ErrInvalidUTF8) + require.Equal(t, tc.want, lx.Offset()) + assert.Equal(t, tc.wantByte, tc.doc[lx.Offset()], + "the offset must land on the first byte of the ill-formed sequence") + }) + } +} + +// TestUTF8ReplaceKeepsValidValuesAliased pins the zero-copy invariant: only an offending value is copied out of the +// input, so enabling replacement costs nothing on a well-formed document. +func TestUTF8ReplaceKeepsValidValuesAliased(t *testing.T) { + doc := []byte(`["` + longPrefix + `", "caf\u00e9"]`) + + for _, p := range []UTF8Policy{UTF8Strict, UTF8Replace, UTF8Passthrough} { + lx := NewWithBytes(doc, WithUTF8Policy(p)) + lx.NextToken() // '[' + require.True(t, lx.Ok()) + tok := lx.NextToken() // the long clean string + require.True(t, lx.Ok()) + require.Equal(t, token.String, tok.Kind()) + + v := tok.Value() + require.NotEmpty(t, v) + assert.Samef(t, &doc[2], &v[0], + "a valid value must still alias the caller's buffer under policy %d", p) + } +} + +// TestUTF8MaxValueBytesAfterReplacement pins that the size cap is re-checked against the REWRITTEN length: replacing a +// 1-byte fault with U+FFFD grows the value threefold, so a value that fit before sanitizing may not fit after. +func TestUTF8MaxValueBytesAfterReplacement(t *testing.T) { + doc := []byte("[\"ab\xff\"]") // 3 bytes scanned, 5 bytes once sanitized + + lx := NewWithBytes(doc, WithUTF8Policy(UTF8Replace), WithMaxValueBytes(4)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.ErrorIs(t, lx.Err(), codes.ErrMaxValueBytes) + + lx = NewWithBytes(doc, WithUTF8Policy(UTF8Replace), WithMaxValueBytes(8)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.NoError(t, lx.Err()) +} + +// TestUTF8SplitSequence forces an ill-formed sequence and a \u escape to straddle a refill boundary at every possible +// split point: the streaming scanners assemble the value across windows, so a fault can be split between the window +// that detected non-ASCII and the one that carries the rest. +func TestUTF8SplitSequence(t *testing.T) { + docs := []struct { + name string + doc string + want error + }{ + { + name: "ill-formed bytes", + doc: "[\"" + longPrefix + "\xe0\xff" + longPrefix + "\"]", + want: codes.ErrInvalidUTF8, + }, + { + name: "broken surrogate pair", + doc: `["` + longPrefix + `\uD800\uD800` + longPrefix + `"]`, + want: codes.ErrSurrogateEscape, + }, + {name: "valid surrogate pair", doc: `["` + longPrefix + `\uD834\uDD1E` + longPrefix + `"]`}, + {name: "valid multibyte", doc: "[\"" + longPrefix + "caf\xc3\xa9" + longPrefix + "\"]"}, + } + + for _, d := range docs { + for chunk := 1; chunk <= 24; chunk++ { + t.Run(fmt.Sprintf("%s/chunk=%d", d.name, chunk), func(t *testing.T) { + for _, verbatim := range []bool{false, true} { + var err error + var decoded []byte + if verbatim { + lx := NewVerbatim(newChunkReader([]byte(d.doc), chunk), + WithBufferSize(32), WithUTF8Policy(UTF8Replace)) + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + decoded = token.Unescape(bytes.Clone(tok.Value())) + } + } + err = lx.Err() + } else { + lx := New(newChunkReader([]byte(d.doc), chunk), + WithBufferSize(32), WithUTF8Policy(UTF8Replace)) + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + decoded = bytes.Clone(tok.Value()) + } + } + err = lx.Err() + } + + require.NoErrorf(t, err, "verbatim=%v: replacement never errors", verbatim) + assert.Truef(t, utf8.Valid(decoded), + "verbatim=%v: emitted value must be valid UTF-8, got %q", verbatim, decoded) + } + + // the same document under the default policy must agree with itself at every chunk size + lx := New(newChunkReader([]byte(d.doc), chunk), WithBufferSize(32)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + if d.want == nil { + require.NoError(t, lx.Err()) + } else { + require.ErrorIs(t, lx.Err(), d.want) + } + }) + } + } +} + +// TestUTF8LexersAgree is the anti-divergence guard: L and VL must accept and reject exactly the same documents under +// every policy, and must decode to the same string. L accepting what VL rejected is precisely the bug class this work +// closed (L laundered a broken surrogate pair into a silent U+FFFD while VL errored). +func TestUTF8LexersAgree(t *testing.T) { + for _, tc := range utf8Cases() { + for _, doc := range []string{`["` + tc.body + `"]`, `{"` + tc.body + `":1}`} { + for _, p := range []UTF8Policy{UTF8Strict, UTF8Replace, UTF8Passthrough} { + t.Run(fmt.Sprintf("%s/%d", tc.name, p), func(t *testing.T) { + got := make([]string, 0, len(utf8Drivers())) + for _, drv := range utf8Drivers() { + decoded, _, err := drv.run([]byte(doc), p) + got = append( + got, + fmt.Sprintf("%s: err=%v decoded=%q", drv.name, err != nil, decoded), + ) + } + for _, line := range got[1:] { + assert.Equal(t, + strings.SplitN(got[0], ": ", 2)[1], + strings.SplitN(line, ": ", 2)[1], + "lexer drivers disagree on %q:\n %s", doc, strings.Join(got, "\n ")) + } + }) + } + } + } +} + +// TestUTF8ReplaceMangledValueContract pins what mangling does and does not preserve, because the two halves are easy +// to conflate: a rewritten VALUE no longer maps onto its source span (U+FFFD is 3 bytes replacing 1), while token +// POSITIONS stay exact because they come from the scan cursor rather than the value. +func TestUTF8ReplaceMangledValueContract(t *testing.T) { + // one ill-formed byte, then a truncated 3-byte sequence: 1 + 3 faults => 4 replacements, 12 bytes for 4 source + const body = "a\xffb\xe0\xa0\xffc" + doc := []byte(" [\"" + body + "\", 7]") + + for _, verbatim := range []bool{false, true} { + t.Run(fmt.Sprintf("verbatim=%v", verbatim), func(t *testing.T) { + var ( + value []byte + tokenOffset uint64 + line, column int + ) + if verbatim { + lx := NewVerbatimWithBytes(doc, WithUTF8Policy(UTF8Replace)) + require.Equal(t, token.Delimiter, lx.NextToken().Kind()) // '[' + tok := lx.NextToken() + require.True(t, lx.Ok()) + require.Equal(t, token.String, tok.Kind()) + value, line, column = bytes.Clone(tok.Value()), lx.Line(), lx.Column() + tokenOffset = lx.Offset() + } else { + lx := NewWithBytes(doc, WithUTF8Policy(UTF8Replace)) + require.Equal(t, token.Delimiter, lx.NextToken().Kind()) // '[' + tok := lx.NextToken() + require.True(t, lx.Ok()) + require.Equal(t, token.String, tok.Kind()) + value = bytes.Clone(tok.Value()) + tokenOffset = lx.Offset() + } + + // the value is well-formed, and LONGER than the text it came from + assert.True(t, utf8.Valid(value)) + assert.Equal(t, "a�b���c", string(value)) + assert.Greaterf(t, len(value), len(body), + "U+FFFD is 3 bytes replacing 1: a mangled value cannot have its source width") + + // positions are still exact: the cursor sits just past the closing quote of the string + wantOffset := uint64(len(" [\"") + len(body) + 1) + assert.Equalf(t, wantOffset, tokenOffset, + "the scan cursor must track the SOURCE, not the rewritten value") + if verbatim { + assert.Equal(t, 1, line) + assert.Equal(t, len(" [")+1, column) + } + }) + } +} + +// TestUTF8BOM pins the leading-BOM rules: the complete mark at offset 0 is consumed, and nothing else is. +func TestUTF8BOM(t *testing.T) { + const bom = "\uFEFF" + + t.Run("consumed at offset 0", func(t *testing.T) { + for _, doc := range []string{bom + "{}", bom + `{"a":1}`, bom + " [1,2]", bom + "null"} { + lx := NewWithBytes([]byte(doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + require.NoErrorf(t, lx.Err(), "input %q", doc) + + // streaming, with the mark split across the very first reads + for chunk := 1; chunk <= 4; chunk++ { + sl := New(newChunkReader([]byte(doc), chunk), WithBufferSize(32)) + for range sl.Tokens() { //nolint:revive // draining is the point + } + assert.NoErrorf(t, sl.Err(), "input %q, chunk=%d", doc, chunk) + } + } + }) + + t.Run("offsets account for the consumed mark", func(t *testing.T) { + vl := NewVerbatimWithBytes([]byte(bom + "{}")) + tok := vl.NextToken() + require.True(t, vl.Ok()) + assert.Equal(t, token.Delimiter, tok.Kind()) + // the '{' is the fourth byte of the document + assert.Equal(t, uint64(len(bom)+1), vl.Offset()) + }) + + t.Run("not re-emitted by the verbatim lexer", func(t *testing.T) { + // the documented exception to VL's byte-exact round-trip: the mark belongs to no token's leading space + vl := NewVerbatimWithBytes([]byte(bom + " {}")) + tok := vl.NextToken() + require.True(t, vl.Ok()) + require.Equal(t, token.Delimiter, tok.Kind()) + assert.Equal(t, " ", string(vl.LeadingSpace())) + }) + + t.Run("only the complete mark, only at offset 0", func(t *testing.T) { + for _, tc := range []struct { + name string + doc string + }{ + {name: "truncated mark", doc: "\xef\xbb{}"}, + {name: "mark alone", doc: bom}, + {name: "mark after a value", doc: "{}" + bom}, + {name: "mark inside an array", doc: "[" + bom + "1]"}, + {name: "doubled mark", doc: bom + bom + "{}"}, + } { + t.Run(tc.name, func(t *testing.T) { + lx := NewWithBytes([]byte(tc.doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + assert.Error(t, lx.Err()) + }) + } + }) + + t.Run("a mark inside a string is ordinary content", func(t *testing.T) { + lx := NewWithBytes([]byte(`["` + bom + `"]`)) + var got []byte + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + got = bytes.Clone(tok.Value()) + } + } + require.NoError(t, lx.Err()) + assert.Equal(t, bom, string(got)) + }) +} + +// TestUTF16BOMDiagnostic pins the accurate error for a UTF-16 document, which would otherwise fail as a generic +// invalid token on its first byte. +func TestUTF16BOMDiagnostic(t *testing.T) { + for _, tc := range []struct { + name string + doc string + }{ + {name: "LE", doc: "\xff\xfe[\x00\"\x00\xe9\x00\"\x00]\x00"}, + {name: "BE", doc: "\xfe\xff\x00[\x00\"\x00\xe9\x00\"\x00]"}, + } { + t.Run(tc.name, func(t *testing.T) { + lx := NewWithBytes([]byte(tc.doc)) + for range lx.Tokens() { //nolint:revive // draining is the point + } + assert.ErrorIs(t, lx.Err(), codes.ErrNotUTF8) + + vl := NewVerbatim(newChunkReader([]byte(tc.doc), 3), WithBufferSize(32)) + for range vl.Tokens() { //nolint:revive // draining is the point + } + assert.ErrorIs(t, vl.Err(), codes.ErrNotUTF8) + }) + } +} diff --git a/json/lexers/default-lexer/verbatim_lexer.go b/json/lexers/default-lexer/verbatim_lexer.go index 3e9030e..f8dda08 100644 --- a/json/lexers/default-lexer/verbatim_lexer.go +++ b/json/lexers/default-lexer/verbatim_lexer.go @@ -16,6 +16,40 @@ var _ lexers.Lexer = &VL{} // Whenever consuming from an io.Reader, the stream is buffered so it is not useful to use a buffered reader. // // The lexer performs JSON grammar check such as opening/closing brackets, expected sequence of delimiter/values etc. +// +// # The verbatim pledge +// +// VL reproduces its input byte-for-byte: the token stream plus [VL.LeadingSpace] reassembles the source exactly, and +// every token value is a slice of the source with its escapes intact. Under the default [UTF8Strict] policy this is +// unconditional, because input that is not valid UTF-8 is rejected rather than emitted. +// +// Two documented exceptions: +// +// - A leading UTF-8 BOM is consumed before any token exists, so it belongs to no token's leading space and is not +// re-emitted. RFC 8259 §8.1 asks implementations not to emit a BOM, so this is the conformant direction. +// - Under [UTF8Replace], an ill-formed byte sequence inside a string value is rewritten (see "Mangling" below). +// +// # Mangling under UTF8Replace, and what it costs +// +// [UTF8Replace] substitutes U+FFFD for each ill-formed byte in a string value. Only the offending value is affected — +// every other token still aliases the source — but for that value the byte-for-byte pledge is off, and so is the +// ability to map value bytes back onto source bytes: +// +// - U+FFFD encodes as THREE bytes and replaces ONE. A value carrying k ill-formed bytes is 2k bytes longer than its +// source span, so len(token.Value()) no longer equals the width of the text it came from. Multi-byte faults +// compound this: a truncated three-byte sequence is three separate ill-formed bytes, hence three replacements and +// nine bytes where the source had three. +// - Consequently an index INTO a mangled value cannot be turned into a source offset by adding it to the token's +// start, and the source text of such a value cannot be recovered from the token. If you need the original bytes, +// use [UTF8Strict] (and handle the error) or [UTF8Passthrough] (and validate downstream yourself). +// +// What does NOT change: [VL.Line], [VL.Column], [L.Offset] and [VL.LeadingSpace] stay exact under every policy. They +// are derived from the scan cursor walking the source, never from the value, so token POSITIONS remain trustworthy +// even when a token's CONTENT has been rewritten. A linter or formatter can still point at the right place; it just +// cannot assume the value it holds is the text that was there. +// +// Escape text is never rewritten: VL keeps `\uXXXX` as source text, so a broken surrogate pair stays verbatim in the +// value and its U+FFFD appears only when the value is decoded with [token.Unescape]. type VL struct { *L } diff --git a/json/lexers/error-codes/errors.go b/json/lexers/error-codes/errors.go index 07c3a5b..769dded 100644 --- a/json/lexers/error-codes/errors.go +++ b/json/lexers/error-codes/errors.go @@ -28,6 +28,8 @@ const ( ErrUnicodeEscape LexerError = "invalid unicode escape sequence" ErrUnknownEscape LexerError = "unknown escape sequence" ErrInvalidRune LexerError = "invalid rune in unicode escape sequence" + ErrInvalidUTF8 LexerError = "invalid UTF-8 sequence in string" + ErrNotUTF8 LexerError = "input is not UTF-8 encoded (UTF-16 byte order mark found)" ErrSurrogateEscape LexerError = "expected an escaped UTF-16 code point to come as a surrogate pair" ErrDelimitedValue LexerError = "value should follow a delimiter" ErrCommaInContainer LexerError = "comma should appear within an object or an array" diff --git a/json/lexers/yaml-lexer/fuzz_test.go b/json/lexers/yaml-lexer/fuzz_test.go index 5bfbe3d..0e9ed21 100644 --- a/json/lexers/yaml-lexer/fuzz_test.go +++ b/json/lexers/yaml-lexer/fuzz_test.go @@ -173,6 +173,14 @@ func FuzzYL(f *testing.F) { "123: ok", "? [a, b]\n: c", "[", "{", "a: [", ": :", "\x00", "\xff\xfe", "\xef\xbb\xbfa: 1", "a:\t1", "a: !!str 7\nb: !!bool yes", + // merge-key cycles: a mapping whose own "<<" aliases itself used to recurse forever in + // resolveMapping and kill the process with a fatal stack overflow (CI: "fuzzing process + // hung or terminated unexpectedly"). + "e: &b\n <<: *b\n", + "e: &b\n <<: *b\n c ", + "e: &b\n a: 1\n <<: *b\n c: 3\n", + "e: &b\n <<: [*b]\n", + "a: &x\n - *x\n", } for _, s := range seeds { f.Add([]byte(s)) diff --git a/json/lexers/yaml-lexer/lexer.go b/json/lexers/yaml-lexer/lexer.go index ee8f09a..0050e8a 100644 --- a/json/lexers/yaml-lexer/lexer.go +++ b/json/lexers/yaml-lexer/lexer.go @@ -45,10 +45,12 @@ type YL struct { ptr expressions.Pointer ptrFrames []ptrFrame - // build-time transient state (nil outside build): anchor table and the set of anchors - // currently being expanded (alias cycle guard). See walk.go. + // build-time transient state (nil outside build): anchor table and the two alias-cycle + // guards — the set of anchors currently being expanded, and the set of "<<" merge sources + // whose entries are currently being flattened. See walk.go. anchors map[string]ast.Node expanding map[string]bool + merging map[ast.Node]bool } // emit is one entry in the materialised token stream: the token plus the source position diff --git a/json/lexers/yaml-lexer/merge_cycle_test.go b/json/lexers/yaml-lexer/merge_cycle_test.go new file mode 100644 index 0000000..02a333f --- /dev/null +++ b/json/lexers/yaml-lexer/merge_cycle_test.go @@ -0,0 +1,181 @@ +package lexer_test + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" + + yamllexer "github.com/go-openapi/core/json/lexers/yaml-lexer" +) + +// Merge-key ("<<") cycles. +// +// A mapping whose own merge key aliases itself resolves to its own entries, which contain that same merge key. The +// flattening in resolveMapping therefore recursed forever and the process died with a fatal stack overflow — not a +// Go panic, so nothing could recover it and no circuit breaker (WithMaxTokens, WithMaxContainerStack) applied. CI +// found it as "fuzzing process hung or terminated unexpectedly". +// +// mergeEntries did have an anchor guard, but it only spanned resolving the alias to a node list — which does not +// recurse — and was released before those entries were expanded, which is where the cycle closes. + +// runBounded lexes data on a watchdog so a regression shows up as a failure rather than taking the suite down with +// it. A stack overflow cannot be recovered, so this cannot catch that directly; it does catch a plain runaway. +func runBounded(t *testing.T, data string, opts ...yamllexer.Option) (int, error) { + t.Helper() + + type result struct { + n int + err error + } + done := make(chan result, 1) + + go func() { + l := yamllexer.NewWithBytes([]byte(data), opts...) + n := 0 + for range l.Tokens() { + n++ + if n > 100000 { + break + } + } + done <- result{n: n, err: l.Err()} + }() + + select { + case r := <-done: + return r.n, r.err + case <-time.After(10 * time.Second): + t.Fatalf("lexing did not terminate for %q", data) + + return 0, nil + } +} + +func TestMergeKeyCycleIsRejected(t *testing.T) { + cases := []struct { + name string + in string + }{ + { + name: "mapping merges itself", + in: "e: &b\n <<: *b\n", + }, + { + name: "mapping merges itself among other keys", + in: "e: &b\n a: 1\n <<: *b\n c: 3\n", + }, + { + name: "self-merge inside a merge sequence", + in: "e: &b\n <<: [*b]\n", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := runBounded(t, tc.in) + require.Error(t, err, "a merge-key cycle must be reported, not expanded forever") + assert.ErrorIs(t, err, yamllexer.ErrAliasCycle) + }) + } +} + +// TestMergeCycleNeedsSelfReference records WHY the self-merge is the only shape that reaches the cycle guard: an +// anchor must be defined before it is aliased, so a multi-node merge cycle cannot be written at all — closing it +// needs a forward reference, which fails earlier as an unknown anchor. That is not a second bug; it is the reason +// the fix only has to handle self-reference. +func TestMergeCycleNeedsSelfReference(t *testing.T) { + for _, tc := range []struct { + name string + in string + }{ + {name: "two-step", in: "x: &x\n <<: *y\ny: &y\n <<: *x\n"}, + {name: "three-step", in: "a: &a\n <<: *b\nb: &b\n <<: *c\nc: &c\n <<: *a\n"}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := runBounded(t, tc.in) + require.Error(t, err) + assert.ErrorIs( + t, + err, + yamllexer.ErrUnknownAnchor, + "the cycle cannot form: the second anchor is not defined yet at the point it is aliased", + ) + }) + } +} + +// TestMergeKeyLegitimateUsesStillWork guards the other side of the fix: the guard keys on the merge SOURCE node, so +// merging one anchor several times — redundant but perfectly legal YAML — must not be mistaken for a cycle. +// +// (Repeating "<<" twice as two separate keys in the same mapping is not in this table: goccy rejects that at parse +// time as a duplicate key, before the merge resolution ever runs.) +func TestMergeKeyLegitimateUsesStillWork(t *testing.T) { + cases := []struct { + name string + in string + wantKey string // a key that must survive the merge + }{ + { + name: "simple merge", + in: "base: &b {a: 1, b: 2}\nderived:\n <<: *b\n c: 3\n", + wantKey: "a", + }, + { + name: "same anchor merged into two siblings", + in: "base: &b {a: 1}\nx:\n <<: *b\ny:\n <<: *b\n", + wantKey: "a", + }, + { + name: "same anchor twice in a merge sequence", + in: "base: &b {a: 1}\nx:\n <<: [*b, *b]\n", + wantKey: "a", + }, + { + name: "chained merges, no cycle", + in: "a: &a {x: 1}\nb: &b\n <<: *a\n y: 2\nc:\n <<: *b\n z: 3\n", + wantKey: "x", + }, + { + name: "merge sequence of distinct anchors", + in: "p: &p {a: 1}\nq: &q {b: 2}\nr:\n <<: [*p, *q]\n", + wantKey: "b", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + n, err := runBounded(t, tc.in) + require.NoError(t, err, "a legal merge must not be mistaken for a cycle") + assert.Positive(t, n) + }) + } +} + +// TestAliasCycleWithoutMergeStillRejected pins the pre-existing guard in walkAlias, which was already correct: only +// the merge path was broken, and the fix must not have disturbed it. +func TestAliasCycleWithoutMergeStillRejected(t *testing.T) { + // a sequence that contains an alias to itself + _, err := runBounded(t, "a: &x\n - *x\n") + require.Error(t, err) + assert.ErrorIs(t, err, yamllexer.ErrAliasCycle) +} + +// TestDeepMergeChainTerminates checks that a long but ACYCLIC merge chain is still resolved rather than tripping the +// cycle guard — the guard is a stack, so it must unmark on the way out. +func TestDeepMergeChainTerminates(t *testing.T) { + const depth = 200 + + var b strings.Builder + b.WriteString("k0: &a0 {v: 1}\n") + for i := 1; i <= depth; i++ { + fmt.Fprintf(&b, "k%d: &a%d\n <<: *a%d\n", i, i, i-1) + } + + n, err := runBounded(t, b.String()) + require.NoError(t, err) + assert.Positive(t, n) +} diff --git a/json/lexers/yaml-lexer/testdata/fuzz/FuzzYL/97af9da7c4afa4dc b/json/lexers/yaml-lexer/testdata/fuzz/FuzzYL/97af9da7c4afa4dc new file mode 100644 index 0000000..4117223 --- /dev/null +++ b/json/lexers/yaml-lexer/testdata/fuzz/FuzzYL/97af9da7c4afa4dc @@ -0,0 +1,2 @@ +go test fuzz v1 +[]byte("e: &b\n <<: *b\n c ") diff --git a/json/lexers/yaml-lexer/walk.go b/json/lexers/yaml-lexer/walk.go index 277de2e..cef4b84 100644 --- a/json/lexers/yaml-lexer/walk.go +++ b/json/lexers/yaml-lexer/walk.go @@ -43,9 +43,11 @@ func (l *YL) build() { l.anchors = map[string]ast.Node{} l.expanding = map[string]bool{} + l.merging = map[ast.Node]bool{} l.walkValue(f.Docs[0].Body, 0) l.anchors = nil l.expanding = nil + l.merging = nil } // safeParse runs the goccy parser, converting a recoverable panic into an error so a @@ -209,7 +211,23 @@ func (l *YL) resolveMapping(values []*ast.MappingValueNode) []entryKV { return } if _, isMerge := mv.Key.(*ast.MergeKeyNode); isMerge { + // The cycle guard has to span the EXPANSION of the merged entries, not just the + // resolution of the alias to a node list: a mapping whose own merge key aliases + // itself (`e: &b\n <<: *b`) resolves to its own entries, which contain that same + // merge key, so add() would recurse forever. mergeEntries' anchor guard is already + // released by then — it only covers the lookup, which does not recurse. + // + // Keying on the merge source node rather than the anchor name also catches cycles + // formed through a chain of aliases or a merge sequence, and lets the same anchor be + // merged twice in sibling positions, which is redundant but legal. + if l.merging[mv.Value] { + l.err = ErrAliasCycle + + return + } + l.merging[mv.Value] = true add(l.mergeEntries(mv.Value), true) + delete(l.merging, mv.Value) continue } diff --git a/json/writers/default-writer/README.md b/json/writers/default-writer/README.md index b28093a..3f4664c 100644 --- a/json/writers/default-writer/README.md +++ b/json/writers/default-writer/README.md @@ -16,6 +16,37 @@ TODOs: * [] YAML output is not great, because at this moment, strings remain JSON strings, without accounting for YAML escaping rules. +## UTF-8 + +RFC 8259 §8.1 requires JSON text to be UTF-8. `WithUTF8Policy` selects what happens to caller-supplied data that +is not: + +| policy | behavior | +|---|---| +| `UTF8Strict` (default) | refuse it: the writer errors with `ErrInvalidUTF8` and short-circuits | +| `UTF8Replace` | substitute U+FFFD, one per invalid byte — the writers' behavior before the policy existed | +| `UTF8Passthrough` | write the bytes through unchecked (produces invalid JSON; for pre-validated data) | + +`Unbuffered` takes `WithUnbufferedUTF8Policy`; `Indented` and `YAML` take it through +`WithIndentBufferedOptions` / `WithYAMLBufferedOptions`. The substitution rule is shared with the lexers +(`json/internal/utf8x`), so a value sanitized on the way in and one sanitized on the way out are the same bytes. + +Go strings may legally hold ill-formed UTF-8, so `String()` and `StringBytes()` can trip the default on data from a +non-Unicode source. That is the intended change: silently corrupting a payload is worse than refusing it. + +### What is checked + +The policy applies to what the **caller** supplies: `String`, `StringBytes`, `StringRunes`, `StringCopy`, `Raw`, +`RawCopy`. Streaming entry points validate the stream as a whole, not chunk by chunk, so a sequence split across two +reads is not mistaken for a fault and a fault spanning the boundary is not missed. + +It does **not** apply to values arriving in a lexer token (`Token`, `VerbatimToken`): the lexers already validate +string values, and re-checking them would cost the token-copy path its reason for existing. The loophole that leaves +— and it is deliberate — is that a document lexed with `UTF8Passthrough` can carry ill-formed bytes into the writer, +where `Token` silently substitutes U+FFFD (the escaper decodes as it goes) and `VerbatimToken` writes the bytes out +byte-for-byte, as verbatim requires. `VerbatimValue` is not in that group: it renders through the ordinary string +path and is checked. Numbers (`NumberBytes`, `NumberCopy`) are unchecked, as their documentation already says. + ## Performance * Allocations diff --git a/json/writers/default-writer/base_writer.go b/json/writers/default-writer/base_writer.go index 9349757..646d83b 100644 --- a/json/writers/default-writer/base_writer.go +++ b/json/writers/default-writer/base_writer.go @@ -10,6 +10,7 @@ import ( "unicode/utf8" "unsafe" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/token" "github.com/go-openapi/core/json/stores/values" "github.com/go-openapi/core/json/types" @@ -20,6 +21,11 @@ type wrt interface { writeSingleByte(byte) writeBinary([]byte) writeEscaped([]byte) []byte + // escapePolicy reports how ill-formed UTF-8 in caller-supplied data must be handled. + // + // Named for its role rather than after the option: `buffered` embeds both baseWriter and bufferedOptions, so a + // method here called utf8Policy would collide with the promoted option field of that name. + escapePolicy() UTF8Policy // Ok is the cheap, inlinable hot-path status check (w.err == nil), used for the // per-operation guards instead of the value-receiver Err which copies and wraps. Ok() bool @@ -30,6 +36,7 @@ type baseWriter struct { w io.Writer written int64 err error + utf8 UTF8Policy } // Ok tells the status of the writer. @@ -64,6 +71,9 @@ func (w *baseWriter) Size() int64 { return w.written } +// escapePolicy reports how ill-formed UTF-8 in caller-supplied data must be handled. See [WithUTF8Policy]. +func (w *baseWriter) escapePolicy() UTF8Policy { return w.utf8 } + func (w *baseWriter) inc(n int) { w.written += int64(n) } @@ -122,12 +132,25 @@ func (w *commonWriter[T]) Bool(v bool) { } // Raw appends raw bytes to the buffer, without quotes and without escaping. +// +// The bytes are caller-supplied, so the UTF-8 policy applies: under [UTF8Strict] ill-formed input is refused. Nothing +// is escaped or substituted — "raw" means raw — so [UTF8Replace] does not rewrite anything here either; it only +// declines to complain. func (w *commonWriter[T]) Raw(data []byte) { if !w.jw.Ok() || len(data) == 0 { return } - w.jw.writeBinary(data) + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + + scratch := holder.Slice() + out, ok := w.rawUTF8(data, &scratch) + if !ok { + return + } + + w.jw.writeBinary(out) } // String writes a string as a JSON string value enclosed by double quotes, with escaping. @@ -166,10 +189,18 @@ func (w *commonWriter[T]) StringRunes(data []rune) { buf := holder.Slice() for _, r := range data { + if !utf8.ValidRune(r) && w.jw.escapePolicy() == UTF8Strict { + // utf8.AppendRune would silently turn a surrogate or an out-of-range value into U+FFFD + w.jw.SetErr(fmt.Errorf("%w: rune %U is not a Unicode scalar value: %w", + ErrInvalidUTF8, r, ErrDefaultWriter)) + + return + } buf = utf8.AppendRune(buf, r) } - w.writeText(buf) + // the runes were just encoded, so the bytes are well-formed by construction + w.writeTextTrusted(buf) } // NumberBytes writes a slice of bytes as a JSON number. @@ -197,8 +228,15 @@ func (w *commonWriter[T]) RawCopy(r io.Reader) { } bufHolder, redeemReadBuffer := poolOfReadBuffers.BorrowWithRedeem() + defer redeemReadBuffer() buf := bufHolder.Slice() + if w.jw.escapePolicy() != UTF8Passthrough { + w.rawCopyValidated(r, buf) + + return + } + for { n, err := r.Read(buf) if err != nil && !errors.Is(err, io.EOF) { @@ -218,8 +256,6 @@ func (w *commonWriter[T]) RawCopy(r io.Reader) { break } } - - redeemReadBuffer() } func (w *commonWriter[T]) StringCopy(r io.Reader) { @@ -245,6 +281,10 @@ func (w *commonWriter[T]) StringCopy(r io.Reader) { } if n > 0 { + if !w.checkUTF8Chunk(buf[:n]) { + return + } + remainder = w.jw.writeEscaped(buf[:n]) if !w.jw.Ok() { return @@ -287,13 +327,11 @@ func (w *commonWriter[T]) StringCopy(r io.Reader) { // Note: this must not clobber the outer loop's n/err, which drive termination. if _, rerr := io.ReadFull(r, single[notWritten:runeSize]); rerr != nil { if errors.Is(rerr, io.EOF) || errors.Is(rerr, io.ErrUnexpectedEOF) { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) + // the input ended inside the sequence: it is truncated, not merely split + if !w.finishRemainder(remainder) { + return + } + w.jw.writeSingleByte(quote) return } @@ -303,19 +341,18 @@ func (w *commonWriter[T]) StringCopy(r io.Reader) { return } + // the stitched sequence never passed through checkUTF8Chunk (it was assembled here, not read as a + // chunk), so it must be checked on its own or a fault spanning the read boundary slips through + if !w.checkUTF8(single[:runeSize]) { + return + } + remainder = w.jw.writeEscaped(single[:runeSize]) if !w.jw.Ok() { return } - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // a full rune's worth of bytes that still does not complete: truncated return } } @@ -528,7 +565,8 @@ func (w *commonWriter[T]) Token(tok token.T) { // ignore } case token.String, token.Key: - w.writeText(tok.Value()) + // a token value comes from a lexer, which already validated it: see writeTextTrusted + w.writeTextTrusted(tok.Value()) case token.Number: w.NumberBytes(tok.Value()) case token.Boolean: @@ -578,6 +616,83 @@ func (w *commonWriter[T]) VerbatimValue(value values.VerbatimValue) { w.Value(value.Value) } +// rawCopyValidated is [commonWriter.RawCopy] under [UTF8Strict] and [UTF8Replace]: it streams the reader through +// while validating it as a single UTF-8 stream rather than as independent chunks. +// +// A read boundary may fall inside a multi-byte sequence, which is not an error — the sequence is completed by the +// next read. So the trailing incomplete bytes of a chunk are HELD BACK rather than written, and re-joined with the +// bytes that follow; reads land at an offset in buf that leaves room for them, so the join needs no second buffer and +// no allocation. An incomplete sequence still pending when the reader ends is a genuine truncation. +// +// Validating chunk-locally instead would leave a hole exactly here: a lead byte at the end of one chunk followed by a +// non-continuation at the start of the next would be tolerated by both halves and emitted. +// +// On a genuine fault [UTF8Strict] stops with an error and [UTF8Replace] rewrites that chunk with U+FFFD substituted, +// so "the output is always valid UTF-8" holds for the streaming path exactly as it does for [commonWriter.Raw]. +func (w *commonWriter[T]) rawCopyValidated(r io.Reader, buf []byte) { + // maxCarry bytes of headroom at the front of buf hold the incomplete tail of the previous chunk. + const maxCarry = utf8.UTFMax - 1 + if len(buf) <= maxCarry { + w.jw.SetErr(fmt.Errorf("read buffer too small for UTF-8 validation: %w", ErrDefaultWriter)) + + return + } + + strict := w.jw.escapePolicy() == UTF8Strict + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + scratch := holder.Slice() + + carry := 0 + for { + n, err := r.Read(buf[maxCarry:]) + if err != nil && !errors.Is(err, io.EOF) { + w.jw.SetErr(err) + + return + } + + atEOF := n == 0 || (err != nil && errors.Is(err, io.EOF)) + chunk := buf[maxCarry-carry : maxCarry+n] + + idx := utf8x.FirstInvalid(chunk) + switch { + case idx < 0: + carry = 0 + case !utf8.FullRune(chunk[idx:]) && !atEOF: + // the chunk ends inside a sequence the next read will complete: hold those bytes back + carry = len(chunk) - idx + chunk = chunk[:idx] + case strict: + w.jw.SetErr(fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, idx, ErrDefaultWriter)) + + return + default: + // UTF8Replace: rewrite this chunk, held-back bytes included. A sequence truncated by the end of the + // input lands here too, and becomes one U+FFFD per byte like any other ill-formed sequence. + scratch = utf8x.Sanitize(scratch[:0], chunk) + chunk = scratch + carry = 0 + } + + if len(chunk) > 0 { + w.jw.writeBinary(chunk) + if !w.jw.Ok() { + return + } + } + + if carry > 0 { + // move the held-back tail into the headroom, where the next read joins onto it + copy(buf[maxCarry-carry:maxCarry], buf[maxCarry+n-carry:maxCarry+n]) + } + + if atEOF { + return + } + } +} + // append writes down the result of AppendText. // // This borrows a temporary buffer to decode the result of AppendText() @@ -613,18 +728,28 @@ func (w *commonWriter[T]) writeTextString(input string) { w.writeText(b) } +// writeText writes CALLER-supplied bytes as a quoted, escaped JSON string, subject to the UTF-8 policy. +// +// Values that came from a lexer token go through [commonWriter.writeTextTrusted] instead: the lexers already +// guarantee valid UTF-8, so re-checking them would be pure overhead on the hot token-copy path. func (w *commonWriter[T]) writeText(data []byte) { + if !w.checkUTF8(data) { + return + } + + w.writeTextTrusted(data) +} + +// writeTextTrusted writes bytes that are already known to be valid UTF-8 as a quoted, escaped JSON string. +// +// "Trusted" means sourced from a lexer token: [L] and [VL] validate string values as they scan, so the writer does +// not pay to check them again. That trust is only as good as the lexer's policy — see the package documentation for +// the loophole this leaves. +func (w *commonWriter[T]) writeTextTrusted(data []byte) { w.jw.writeSingleByte(quote) remainder := w.jw.writeEscaped(data) - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune (invalid first byte): %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // nothing follows this value, so the trailing bytes are a truncated sequence, not a split one return } w.jw.writeSingleByte(quote) diff --git a/json/writers/default-writer/buffered.go b/json/writers/default-writer/buffered.go index 68fb89c..06aa403 100644 --- a/json/writers/default-writer/buffered.go +++ b/json/writers/default-writer/buffered.go @@ -12,6 +12,7 @@ import ( "unicode/utf8" "unsafe" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/token" "github.com/go-openapi/core/json/stores/values" "github.com/go-openapi/core/json/types" @@ -43,12 +44,14 @@ type buffered struct { } func NewBuffered(w io.Writer, opts ...BufferedOption) *Buffered { + o := bufferedOptionsWithDefaults(opts) writer := &Buffered{ buffered: buffered{ baseWriter: baseWriter{ - w: w, + w: w, + utf8: o.utf8Policy, }, - bufferedOptions: bufferedOptionsWithDefaults(opts), + bufferedOptions: o, }, } writer.borrowBuffer() @@ -310,6 +313,11 @@ func (w *buffered) writeEscaped(data []byte) (remainder []byte) { return nil } } + if r == utf8.RuneError && runeWidth == 1 && w.utf8 == UTF8Passthrough { + w.buf = append(w.buf, c) // ill-formed, but the caller asked for its bytes untouched + + continue + } w.buf = utf8.AppendRune(w.buf, r) // invalid runes are represented as \uFFFD i += runeWidth - 1 } @@ -369,7 +377,16 @@ func (w *Buffered) Raw(data []byte) { return } - w.jw.writeBinary(data) + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + + scratch := holder.Slice() + out, ok := w.rawUTF8(data, &scratch) + if !ok { + return + } + + w.jw.writeBinary(out) } // String is generated by writegen from commonWriter.String; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -402,10 +419,18 @@ func (w *Buffered) StringRunes(data []rune) { buf := holder.Slice() for _, r := range data { + if !utf8.ValidRune(r) && w.jw.escapePolicy() == UTF8Strict { + // utf8.AppendRune would silently turn a surrogate or an out-of-range value into U+FFFD + w.jw.SetErr(fmt.Errorf("%w: rune %U is not a Unicode scalar value: %w", + ErrInvalidUTF8, r, ErrDefaultWriter)) + + return + } buf = utf8.AppendRune(buf, r) } - w.writeText(buf) + // the runes were just encoded, so the bytes are well-formed by construction + w.writeTextTrusted(buf) } // NumberBytes is generated by writegen from commonWriter.NumberBytes; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -429,8 +454,15 @@ func (w *Buffered) RawCopy(r io.Reader) { } bufHolder, redeemReadBuffer := poolOfReadBuffers.BorrowWithRedeem() + defer redeemReadBuffer() buf := bufHolder.Slice() + if w.jw.escapePolicy() != UTF8Passthrough { + w.rawCopyValidated(r, buf) + + return + } + for { n, err := r.Read(buf) if err != nil && !errors.Is(err, io.EOF) { @@ -450,8 +482,6 @@ func (w *Buffered) RawCopy(r io.Reader) { break } } - - redeemReadBuffer() } // StringCopy is generated by writegen from commonWriter.StringCopy; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -478,6 +508,10 @@ func (w *Buffered) StringCopy(r io.Reader) { } if n > 0 { + if !w.checkUTF8Chunk(buf[:n]) { + return + } + remainder = w.jw.writeEscaped(buf[:n]) if !w.jw.Ok() { return @@ -520,13 +554,11 @@ func (w *Buffered) StringCopy(r io.Reader) { // Note: this must not clobber the outer loop's n/err, which drive termination. if _, rerr := io.ReadFull(r, single[notWritten:runeSize]); rerr != nil { if errors.Is(rerr, io.EOF) || errors.Is(rerr, io.ErrUnexpectedEOF) { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) + // the input ended inside the sequence: it is truncated, not merely split + if !w.finishRemainder(remainder) { + return + } + w.jw.writeSingleByte(quote) return } @@ -536,19 +568,18 @@ func (w *Buffered) StringCopy(r io.Reader) { return } + // the stitched sequence never passed through checkUTF8Chunk (it was assembled here, not read as a + // chunk), so it must be checked on its own or a fault spanning the read boundary slips through + if !w.checkUTF8(single[:runeSize]) { + return + } + remainder = w.jw.writeEscaped(single[:runeSize]) if !w.jw.Ok() { return } - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // a full rune's worth of bytes that still does not complete: truncated return } } @@ -726,7 +757,8 @@ func (w *Buffered) Token(tok token.T) { // ignore } case token.String, token.Key: - w.writeText(tok.Value()) + // a token value comes from a lexer, which already validated it: see writeTextTrusted + w.writeTextTrusted(tok.Value()) case token.Number: w.NumberBytes(tok.Value()) case token.Boolean: @@ -769,6 +801,71 @@ func (w *Buffered) VerbatimValue(value values.VerbatimValue) { w.Value(value.Value) } +// rawCopyValidated is generated by writegen from commonWriter.rawCopyValidated; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) rawCopyValidated(r io.Reader, buf []byte) { + // maxCarry bytes of headroom at the front of buf hold the incomplete tail of the previous chunk. + const maxCarry = utf8.UTFMax - 1 + if len(buf) <= maxCarry { + w.jw.SetErr(fmt.Errorf("read buffer too small for UTF-8 validation: %w", ErrDefaultWriter)) + + return + } + + strict := w.jw.escapePolicy() == UTF8Strict + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + scratch := holder.Slice() + + carry := 0 + for { + n, err := r.Read(buf[maxCarry:]) + if err != nil && !errors.Is(err, io.EOF) { + w.jw.SetErr(err) + + return + } + + atEOF := n == 0 || (err != nil && errors.Is(err, io.EOF)) + chunk := buf[maxCarry-carry : maxCarry+n] + + idx := utf8x.FirstInvalid(chunk) + switch { + case idx < 0: + carry = 0 + case !utf8.FullRune(chunk[idx:]) && !atEOF: + // the chunk ends inside a sequence the next read will complete: hold those bytes back + carry = len(chunk) - idx + chunk = chunk[:idx] + case strict: + w.jw.SetErr(fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, idx, ErrDefaultWriter)) + + return + default: + // UTF8Replace: rewrite this chunk, held-back bytes included. A sequence truncated by the end of the + // input lands here too, and becomes one U+FFFD per byte like any other ill-formed sequence. + scratch = utf8x.Sanitize(scratch[:0], chunk) + chunk = scratch + carry = 0 + } + + if len(chunk) > 0 { + w.jw.writeBinary(chunk) + if !w.jw.Ok() { + return + } + } + + if carry > 0 { + // move the held-back tail into the headroom, where the next read joins onto it + copy(buf[maxCarry-carry:maxCarry], buf[maxCarry+n-carry:maxCarry+n]) + } + + if atEOF { + return + } + } +} + // append is generated by writegen from commonWriter.append; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Buffered) append(n encoding.TextAppender) { buf, redeem := poolOfNumberBuffers.BorrowWithRedeem() @@ -806,18 +903,96 @@ func (w *Buffered) writeTextString(input string) { // writeText is generated by writegen from commonWriter.writeText; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Buffered) writeText(data []byte) { + if !w.checkUTF8(data) { + return + } + + w.writeTextTrusted(data) +} + +// writeTextTrusted is generated by writegen from commonWriter.writeTextTrusted; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) writeTextTrusted(data []byte) { w.jw.writeSingleByte(quote) remainder := w.jw.writeEscaped(data) - if len(remainder) > 0 { + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // nothing follows this value, so the trailing bytes are a truncated sequence, not a split one + return + } + w.jw.writeSingleByte(quote) +} + +// checkUTF8 is generated by writegen from commonWriter.checkUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) checkUTF8(data []byte) bool { + if w.jw.escapePolicy() != UTF8Strict || utf8x.Valid(data) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, utf8x.FirstInvalid(data), ErrDefaultWriter), + ) + + return false +} + +// rawUTF8 is generated by writegen from commonWriter.rawUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) rawUTF8(data []byte, scratch *[]byte) ([]byte, bool) { + policy := w.jw.escapePolicy() + if policy == UTF8Passthrough || utf8x.Valid(data) { + return data, true + } + + if policy == UTF8Strict { w.jw.SetErr( fmt.Errorf( - "unexpected incomplete rune (invalid first byte): %c: %w", - remainder, + "%w at byte %d: %w", + ErrInvalidUTF8, + utf8x.FirstInvalid(data), ErrDefaultWriter, ), ) - return + return nil, false } - w.jw.writeSingleByte(quote) + + *scratch = utf8x.Sanitize((*scratch)[:0], data) + + return *scratch, true +} + +// finishRemainder is generated by writegen from commonWriter.finishRemainder; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) finishRemainder(remainder []byte) bool { + switch w.jw.escapePolicy() { + case UTF8Strict: + w.jw.SetErr(fmt.Errorf("%w: input ends inside a multi-byte sequence (% x): %w", + ErrInvalidUTF8, remainder, ErrDefaultWriter)) + + return false + + case UTF8Passthrough: + w.jw.writeBinary(remainder) + + default: // UTF8Replace + var buf [utf8.UTFMax * utf8.UTFMax]byte + w.jw.writeBinary(utf8x.Sanitize(buf[:0], remainder)) + } + + return w.jw.Ok() +} + +// checkUTF8Chunk is generated by writegen from commonWriter.checkUTF8Chunk; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Buffered) checkUTF8Chunk(chunk []byte) bool { + if w.jw.escapePolicy() != UTF8Strict { + return true + } + + idx := utf8x.FirstInvalid(chunk) + if idx < 0 || !utf8.FullRune(chunk[idx:]) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d of this chunk: %w", ErrInvalidUTF8, idx, ErrDefaultWriter), + ) + + return false } diff --git a/json/writers/default-writer/buffered_options.go b/json/writers/default-writer/buffered_options.go index f4d081e..5309560 100644 --- a/json/writers/default-writer/buffered_options.go +++ b/json/writers/default-writer/buffered_options.go @@ -25,6 +25,25 @@ func WithBufferSize(size int) BufferedOption { } } +// WithUTF8Policy selects what the writer does with caller-supplied data that is not valid UTF-8: refuse it +// ([UTF8Strict], the default), substitute U+FFFD ([UTF8Replace]), or write it through unchecked +// ([UTF8Passthrough]). +// +// It applies to every entry point that takes bytes, a string, runes or an [io.Reader] from the caller. Values that +// come from a lexer token ([commonWriter.Token], [commonWriter.VerbatimToken], [commonWriter.VerbatimValue]) are NOT +// re-checked — the lexers already guarantee valid UTF-8 unless they were themselves relaxed. See the package +// documentation for that loophole. +// +// [Indented] and [YAML] take this through WithIndentBufferedOptions / WithYAMLBufferedOptions; +// [Unbuffered] has its own [WithUnbufferedUTF8Policy]. +func WithUTF8Policy(policy UTF8Policy) BufferedOption { + return func(o bufferedOptions) bufferedOptions { + o.utf8Policy = policy + + return o + } +} + // bufferedOptions carries the (immutable, unexported) [Buffered] configuration. // // It holds configuration only. Runtime state such as the working-buffer redeem handle lives on @@ -32,6 +51,7 @@ func WithBufferSize(size int) BufferedOption { // no finalizer for the options themselves). type bufferedOptions struct { bufferSize int + utf8Policy UTF8Policy } func bufferedOptionsWithDefaults(opts []BufferedOption) bufferedOptions { diff --git a/json/writers/default-writer/doc.go b/json/writers/default-writer/doc.go index e5c0e17..698f212 100644 --- a/json/writers/default-writer/doc.go +++ b/json/writers/default-writer/doc.go @@ -11,4 +11,38 @@ // // Similarly, it handles "null" values as actual values. Absent data (i.e. a nil slice of bytes) is not considered // a "null" value and is therefore not rendered at all. +// +// # UTF-8 +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, so a writer that emits an ill-formed sequence emits a document no +// conforming parser has to accept. By default ([UTF8Strict]) the writers refuse such data with [ErrInvalidUTF8] +// instead of writing it; [UTF8Replace] substitutes U+FFFD (one per invalid byte) and [UTF8Passthrough] writes the +// bytes through unchecked. See [WithUTF8Policy]. +// +// Note that a Go string may legally hold ill-formed UTF-8, so [Buffered.String] and [Buffered.StringBytes] can trip +// the default on data that arrived from a non-Unicode source. Before this policy existed the writers substituted +// U+FFFD silently and unconditionally — [UTF8Replace] is that old behavior, now explicit. +// +// The substitution rule is shared with the lexers, so a value sanitized on the way in and one sanitized on the way +// out are the same bytes. +// +// # What is checked, and the token loophole +// +// The policy applies to data the CALLER supplies: String, StringBytes, StringRunes, StringCopy, Raw and RawCopy. +// +// It does NOT apply to values that arrive in a lexer token — Token and VerbatimToken. The lexers validate string +// values as they scan, so re-checking them here would be pure overhead on the token-copy path that exists to be fast. +// That trust is only as good as the lexer's own policy, which leaves a deliberate loophole: if a document is lexed +// with lexer.UTF8Passthrough, its tokens may carry ill-formed bytes, and then +// +// - Token does not report them, but the escaper decodes as it goes, so they are silently substituted with U+FFFD: +// the output stays valid UTF-8 while differing from the input with no error; +// - VerbatimToken writes the value byte-for-byte with no escaping at all — that is what verbatim means — so there +// the ill-formed bytes DO reach the output. +// +// VerbatimValue is NOT in that group: it renders through the ordinary string path, so it is checked like any other +// caller data. If you lex with a relaxed policy and write the tokens out, validate at the boundary you care about. +// +// Numbers are also unchecked ([Buffered.NumberBytes], [Buffered.NumberCopy]): they are documented as taking whatever +// the caller provides, and a number containing non-ASCII is already invalid JSON for grammatical reasons. package writer diff --git a/json/writers/default-writer/errors.go b/json/writers/default-writer/errors.go index 698a917..7bc347b 100644 --- a/json/writers/default-writer/errors.go +++ b/json/writers/default-writer/errors.go @@ -13,4 +13,9 @@ const ( // ErrUnsupportedInterface means that a method is called but the underlying buffer does not support this interface. ErrUnsupportedInterface Error = "the underlying buffer does not support this interface" + + // ErrInvalidUTF8 means that caller-supplied data is not valid UTF-8 and the writer runs under [UTF8Strict]. + // RFC 8259 §8.1 requires JSON text to be UTF-8, so writing it would emit a document no conforming parser has to + // accept. Select [UTF8Replace] to substitute U+FFFD instead. + ErrInvalidUTF8 Error = "data to write is not valid UTF-8" ) diff --git a/json/writers/default-writer/escape.go b/json/writers/default-writer/escape.go index 9e1eb2c..dfe2bf0 100644 --- a/json/writers/default-writer/escape.go +++ b/json/writers/default-writer/escape.go @@ -4,7 +4,14 @@ import ( "unicode/utf8" ) -func escapedBytes(input, output []byte) ([]byte, []byte) { +// escapedBytes escapes input into output per the JSON string rules, returning the escaped bytes and any trailing +// bytes that form an incomplete UTF-8 sequence (which the caller stitches with the next read, or reports). +// +// policy decides what happens to a byte that is not part of a well-formed sequence: [UTF8Passthrough] copies it +// verbatim, anything else substitutes U+FFFD — one per invalid byte, the same granularity as utf8x.Sanitize, so the +// lexer and the writer agree on what a sanitized value looks like. Under [UTF8Strict] the caller has already +// rejected such input, so the substitution is unreachable there. +func escapedBytes(input, output []byte, policy UTF8Policy) ([]byte, []byte) { var ( p int escaped bool @@ -70,6 +77,14 @@ func escapedBytes(input, output []byte) ([]byte, []byte) { return output, input[i:] } r, runeWidth := utf8.DecodeRune(input[i:]) + if r == utf8.RuneError && runeWidth == 1 && policy == UTF8Passthrough { + output = append( + output, + c, + ) // ill-formed, but the caller asked for its bytes untouched + + continue + } output = utf8.AppendRune(output, r) // invalid runes are represented as \uFFFD i += runeWidth - 1 } diff --git a/json/writers/default-writer/escape_regression_test.go b/json/writers/default-writer/escape_regression_test.go index 0946688..bd31393 100644 --- a/json/writers/default-writer/escape_regression_test.go +++ b/json/writers/default-writer/escape_regression_test.go @@ -167,7 +167,7 @@ func TestStringCopyTruncatedRuneErrors(t *testing.T) { func expectedJSONString(s string) string { out := make([]byte, 0, len(s)+2) out = append(out, '"') - escaped, remainder := escapedBytes([]byte(s), make([]byte, 0, len(s))) + escaped, remainder := escapedBytes([]byte(s), make([]byte, 0, len(s)), UTF8Replace) out = append(out, escaped...) out = append(out, remainder...) // none expected for complete inputs out = append(out, '"') diff --git a/json/writers/default-writer/unbuffered.go b/json/writers/default-writer/unbuffered.go index c059e84..3a98ef1 100644 --- a/json/writers/default-writer/unbuffered.go +++ b/json/writers/default-writer/unbuffered.go @@ -10,6 +10,7 @@ import ( "unicode/utf8" "unsafe" + "github.com/go-openapi/core/json/internal/utf8x" "github.com/go-openapi/core/json/lexers/token" "github.com/go-openapi/core/json/stores/values" "github.com/go-openapi/core/json/types" @@ -58,13 +59,14 @@ type unbuffered struct { // NewUnbuffered JSON writer that copies JSON to [io.Writer] w, without buffering. // -// [Unbuffered] has no configurable options and holds no pooled resources, so there is nothing to -// relinquish: no options finalizer is registered. -func NewUnbuffered(w io.Writer, _ ...UnbufferedOption) *Unbuffered { +// [Unbuffered] holds no pooled resources, so there is nothing to relinquish: no options finalizer is registered. +func NewUnbuffered(w io.Writer, opts ...UnbufferedOption) *Unbuffered { + o := unbufferedOptionsWithDefaults(opts) writer := &Unbuffered{ unbuffered: unbuffered{ baseWriter: baseWriter{ - w: w, + w: w, + utf8: o.utf8Policy, }, }, } @@ -122,7 +124,7 @@ func (w *unbuffered) writeEscaped(data []byte) []byte { escapedHolder, redeemEscaped := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(len(data)) escapedBuffer := escapedHolder.Slice() - escapedBuffer, remainder := escapedBytes(data, escapedBuffer) + escapedBuffer, remainder := escapedBytes(data, escapedBuffer, w.utf8) w.writeBinary(escapedBuffer) redeemEscaped() @@ -181,7 +183,16 @@ func (w *Unbuffered) Raw(data []byte) { return } - w.jw.writeBinary(data) + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + + scratch := holder.Slice() + out, ok := w.rawUTF8(data, &scratch) + if !ok { + return + } + + w.jw.writeBinary(out) } // String is generated by writegen from commonWriter.String; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -214,10 +225,18 @@ func (w *Unbuffered) StringRunes(data []rune) { buf := holder.Slice() for _, r := range data { + if !utf8.ValidRune(r) && w.jw.escapePolicy() == UTF8Strict { + // utf8.AppendRune would silently turn a surrogate or an out-of-range value into U+FFFD + w.jw.SetErr(fmt.Errorf("%w: rune %U is not a Unicode scalar value: %w", + ErrInvalidUTF8, r, ErrDefaultWriter)) + + return + } buf = utf8.AppendRune(buf, r) } - w.writeText(buf) + // the runes were just encoded, so the bytes are well-formed by construction + w.writeTextTrusted(buf) } // NumberBytes is generated by writegen from commonWriter.NumberBytes; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -241,8 +260,15 @@ func (w *Unbuffered) RawCopy(r io.Reader) { } bufHolder, redeemReadBuffer := poolOfReadBuffers.BorrowWithRedeem() + defer redeemReadBuffer() buf := bufHolder.Slice() + if w.jw.escapePolicy() != UTF8Passthrough { + w.rawCopyValidated(r, buf) + + return + } + for { n, err := r.Read(buf) if err != nil && !errors.Is(err, io.EOF) { @@ -262,8 +288,6 @@ func (w *Unbuffered) RawCopy(r io.Reader) { break } } - - redeemReadBuffer() } // StringCopy is generated by writegen from commonWriter.StringCopy; DO NOT EDIT (edit commonWriter, re-run go generate). @@ -290,6 +314,10 @@ func (w *Unbuffered) StringCopy(r io.Reader) { } if n > 0 { + if !w.checkUTF8Chunk(buf[:n]) { + return + } + remainder = w.jw.writeEscaped(buf[:n]) if !w.jw.Ok() { return @@ -332,13 +360,11 @@ func (w *Unbuffered) StringCopy(r io.Reader) { // Note: this must not clobber the outer loop's n/err, which drive termination. if _, rerr := io.ReadFull(r, single[notWritten:runeSize]); rerr != nil { if errors.Is(rerr, io.EOF) || errors.Is(rerr, io.ErrUnexpectedEOF) { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) + // the input ended inside the sequence: it is truncated, not merely split + if !w.finishRemainder(remainder) { + return + } + w.jw.writeSingleByte(quote) return } @@ -348,19 +374,18 @@ func (w *Unbuffered) StringCopy(r io.Reader) { return } + // the stitched sequence never passed through checkUTF8Chunk (it was assembled here, not read as a + // chunk), so it must be checked on its own or a fault spanning the read boundary slips through + if !w.checkUTF8(single[:runeSize]) { + return + } + remainder = w.jw.writeEscaped(single[:runeSize]) if !w.jw.Ok() { return } - if len(remainder) > 0 { - w.jw.SetErr( - fmt.Errorf( - "unexpected incomplete rune at end of input: %c: %w", - remainder, - ErrDefaultWriter, - ), - ) - + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // a full rune's worth of bytes that still does not complete: truncated return } } @@ -538,7 +563,8 @@ func (w *Unbuffered) Token(tok token.T) { // ignore } case token.String, token.Key: - w.writeText(tok.Value()) + // a token value comes from a lexer, which already validated it: see writeTextTrusted + w.writeTextTrusted(tok.Value()) case token.Number: w.NumberBytes(tok.Value()) case token.Boolean: @@ -581,6 +607,71 @@ func (w *Unbuffered) VerbatimValue(value values.VerbatimValue) { w.Value(value.Value) } +// rawCopyValidated is generated by writegen from commonWriter.rawCopyValidated; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) rawCopyValidated(r io.Reader, buf []byte) { + // maxCarry bytes of headroom at the front of buf hold the incomplete tail of the previous chunk. + const maxCarry = utf8.UTFMax - 1 + if len(buf) <= maxCarry { + w.jw.SetErr(fmt.Errorf("read buffer too small for UTF-8 validation: %w", ErrDefaultWriter)) + + return + } + + strict := w.jw.escapePolicy() == UTF8Strict + holder, redeem := poolOfEscapedBuffers.BorrowWithSizeAndRedeem(0) + defer redeem() + scratch := holder.Slice() + + carry := 0 + for { + n, err := r.Read(buf[maxCarry:]) + if err != nil && !errors.Is(err, io.EOF) { + w.jw.SetErr(err) + + return + } + + atEOF := n == 0 || (err != nil && errors.Is(err, io.EOF)) + chunk := buf[maxCarry-carry : maxCarry+n] + + idx := utf8x.FirstInvalid(chunk) + switch { + case idx < 0: + carry = 0 + case !utf8.FullRune(chunk[idx:]) && !atEOF: + // the chunk ends inside a sequence the next read will complete: hold those bytes back + carry = len(chunk) - idx + chunk = chunk[:idx] + case strict: + w.jw.SetErr(fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, idx, ErrDefaultWriter)) + + return + default: + // UTF8Replace: rewrite this chunk, held-back bytes included. A sequence truncated by the end of the + // input lands here too, and becomes one U+FFFD per byte like any other ill-formed sequence. + scratch = utf8x.Sanitize(scratch[:0], chunk) + chunk = scratch + carry = 0 + } + + if len(chunk) > 0 { + w.jw.writeBinary(chunk) + if !w.jw.Ok() { + return + } + } + + if carry > 0 { + // move the held-back tail into the headroom, where the next read joins onto it + copy(buf[maxCarry-carry:maxCarry], buf[maxCarry+n-carry:maxCarry+n]) + } + + if atEOF { + return + } + } +} + // append is generated by writegen from commonWriter.append; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Unbuffered) append(n encoding.TextAppender) { buf, redeem := poolOfNumberBuffers.BorrowWithRedeem() @@ -618,18 +709,96 @@ func (w *Unbuffered) writeTextString(input string) { // writeText is generated by writegen from commonWriter.writeText; DO NOT EDIT (edit commonWriter, re-run go generate). func (w *Unbuffered) writeText(data []byte) { + if !w.checkUTF8(data) { + return + } + + w.writeTextTrusted(data) +} + +// writeTextTrusted is generated by writegen from commonWriter.writeTextTrusted; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) writeTextTrusted(data []byte) { w.jw.writeSingleByte(quote) remainder := w.jw.writeEscaped(data) - if len(remainder) > 0 { + if len(remainder) > 0 && !w.finishRemainder(remainder) { + // nothing follows this value, so the trailing bytes are a truncated sequence, not a split one + return + } + w.jw.writeSingleByte(quote) +} + +// checkUTF8 is generated by writegen from commonWriter.checkUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) checkUTF8(data []byte) bool { + if w.jw.escapePolicy() != UTF8Strict || utf8x.Valid(data) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, utf8x.FirstInvalid(data), ErrDefaultWriter), + ) + + return false +} + +// rawUTF8 is generated by writegen from commonWriter.rawUTF8; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) rawUTF8(data []byte, scratch *[]byte) ([]byte, bool) { + policy := w.jw.escapePolicy() + if policy == UTF8Passthrough || utf8x.Valid(data) { + return data, true + } + + if policy == UTF8Strict { w.jw.SetErr( fmt.Errorf( - "unexpected incomplete rune (invalid first byte): %c: %w", - remainder, + "%w at byte %d: %w", + ErrInvalidUTF8, + utf8x.FirstInvalid(data), ErrDefaultWriter, ), ) - return + return nil, false } - w.jw.writeSingleByte(quote) + + *scratch = utf8x.Sanitize((*scratch)[:0], data) + + return *scratch, true +} + +// finishRemainder is generated by writegen from commonWriter.finishRemainder; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) finishRemainder(remainder []byte) bool { + switch w.jw.escapePolicy() { + case UTF8Strict: + w.jw.SetErr(fmt.Errorf("%w: input ends inside a multi-byte sequence (% x): %w", + ErrInvalidUTF8, remainder, ErrDefaultWriter)) + + return false + + case UTF8Passthrough: + w.jw.writeBinary(remainder) + + default: // UTF8Replace + var buf [utf8.UTFMax * utf8.UTFMax]byte + w.jw.writeBinary(utf8x.Sanitize(buf[:0], remainder)) + } + + return w.jw.Ok() +} + +// checkUTF8Chunk is generated by writegen from commonWriter.checkUTF8Chunk; DO NOT EDIT (edit commonWriter, re-run go generate). +func (w *Unbuffered) checkUTF8Chunk(chunk []byte) bool { + if w.jw.escapePolicy() != UTF8Strict { + return true + } + + idx := utf8x.FirstInvalid(chunk) + if idx < 0 || !utf8.FullRune(chunk[idx:]) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d of this chunk: %w", ErrInvalidUTF8, idx, ErrDefaultWriter), + ) + + return false } diff --git a/json/writers/default-writer/unbuffered_options.go b/json/writers/default-writer/unbuffered_options.go index 164369f..736cf06 100644 --- a/json/writers/default-writer/unbuffered_options.go +++ b/json/writers/default-writer/unbuffered_options.go @@ -1,10 +1,30 @@ package writer -// UnbufferedOption configures the [Unbuffered] writer. -// -// [Unbuffered] currently exposes no configuration knobs; the type exists for API symmetry with the -// other writers and for forward compatibility. Like the other writer options it threads the -// configuration value through, so it never allocates. +// UnbufferedOption configures the [Unbuffered] writer. Like the other writer options it threads the configuration +// value through, so it never allocates. type UnbufferedOption func(unbufferedOptions) unbufferedOptions -type unbufferedOptions struct{} +// WithUnbufferedUTF8Policy is [WithUTF8Policy] for the [Unbuffered] writer. +// +// It is a separate function only because Go options are typed per writer and [Unbuffered] does not take +// [BufferedOption]s; the knob and its meaning are identical. +func WithUnbufferedUTF8Policy(policy UTF8Policy) UnbufferedOption { + return func(o unbufferedOptions) unbufferedOptions { + o.utf8Policy = policy + + return o + } +} + +type unbufferedOptions struct { + utf8Policy UTF8Policy +} + +func unbufferedOptionsWithDefaults(opts []UnbufferedOption) unbufferedOptions { + var o unbufferedOptions // the zero value is UTF8Strict + for _, apply := range opts { + o = apply(o) + } + + return o +} diff --git a/json/writers/default-writer/utf8.go b/json/writers/default-writer/utf8.go new file mode 100644 index 0000000..f9104e3 --- /dev/null +++ b/json/writers/default-writer/utf8.go @@ -0,0 +1,129 @@ +package writer + +import ( + "fmt" + "unicode/utf8" + + "github.com/go-openapi/core/json/internal/utf8x" +) + +// UTF8Policy selects what a writer does with caller-supplied data that is not valid UTF-8. +// +// RFC 8259 §8.1 requires JSON text to be UTF-8, so a writer that emits an ill-formed sequence emits a document no +// conforming parser has to accept. It is the same type the lexers use ([lexer.UTF8Policy]) — a document refused on +// the way out and one rejected on the way in mean the same thing. +type UTF8Policy = utf8x.Policy + +const ( + // UTF8Strict refuses to write ill-formed input: the writer goes into an error state reporting + // [ErrInvalidUTF8], and every subsequent operation short-circuits. This is the default. + // + // Note that Go strings may legally hold ill-formed UTF-8, so String() and StringBytes() can trip this on data + // that arrived from a non-Unicode source. + UTF8Strict = utf8x.PolicyStrict + + // UTF8Replace substitutes U+FFFD for each invalid byte, so the output is always valid UTF-8 and no error is + // raised. This is what the writers did unconditionally before the policy existed. + UTF8Replace = utf8x.PolicyReplace + + // UTF8Passthrough writes ill-formed bytes through untouched and performs no validation. + // + // UNSAFE: the result is not valid JSON text. It exists for callers who have already validated their data and + // want neither the check nor the substitution. + UTF8Passthrough = utf8x.PolicyPassthrough +) + +// checkUTF8 applies the policy to caller-supplied data before it is written. +// +// It reports whether writing may proceed; under [UTF8Strict] an ill-formed input puts the writer into an error state +// and returns false. Under the other policies it is a no-op — the escaper substitutes (or passes through) as it goes, +// so there is nothing to check up front. +func (w *commonWriter[T]) checkUTF8(data []byte) bool { + if w.jw.escapePolicy() != UTF8Strict || utf8x.Valid(data) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d: %w", ErrInvalidUTF8, utf8x.FirstInvalid(data), ErrDefaultWriter), + ) + + return false +} + +// rawUTF8 applies the policy to caller-supplied bytes that will be written WITHOUT escaping. +// +// It returns the bytes to write and whether writing may proceed. Well-formed input is returned unchanged, so the +// zero-copy pass-through of Raw is preserved for every valid payload; only an ill-formed one is rewritten, and only +// under [UTF8Replace], into the borrowed scratch buffer. +func (w *commonWriter[T]) rawUTF8(data []byte, scratch *[]byte) ([]byte, bool) { + policy := w.jw.escapePolicy() + if policy == UTF8Passthrough || utf8x.Valid(data) { + return data, true + } + + if policy == UTF8Strict { + w.jw.SetErr( + fmt.Errorf( + "%w at byte %d: %w", + ErrInvalidUTF8, + utf8x.FirstInvalid(data), + ErrDefaultWriter, + ), + ) + + return nil, false + } + + *scratch = utf8x.Sanitize((*scratch)[:0], data) + + return *scratch, true +} + +// finishRemainder deals with the bytes left at the end of a value that do not form a complete UTF-8 sequence. +// +// The escapers hand these back rather than guessing, because in a streamed value a chunk may legitimately end inside +// a sequence. Once there is nothing left to read, though, the sequence is truncated, and that is an ill-formed input +// like any other: refused under [UTF8Strict], substituted per byte under [UTF8Replace], written as-is under +// [UTF8Passthrough]. +// +// It reports whether writing may continue. +func (w *commonWriter[T]) finishRemainder(remainder []byte) bool { + switch w.jw.escapePolicy() { + case UTF8Strict: + w.jw.SetErr(fmt.Errorf("%w: input ends inside a multi-byte sequence (% x): %w", + ErrInvalidUTF8, remainder, ErrDefaultWriter)) + + return false + + case UTF8Passthrough: + w.jw.writeBinary(remainder) + + default: // UTF8Replace + var buf [utf8.UTFMax * utf8.UTFMax]byte + w.jw.writeBinary(utf8x.Sanitize(buf[:0], remainder)) + } + + return w.jw.Ok() +} + +// checkUTF8Chunk is [commonWriter.checkUTF8] for one chunk of a streamed value. +// +// A chunk may legitimately end mid-sequence — the caller's reader knows nothing of rune boundaries — so a fault that +// is merely a truncated tail is tolerated here and left to the rune-stitching in [commonWriter.StringCopy], which +// reports it if the input really does end there. Any other fault is a genuine error. +func (w *commonWriter[T]) checkUTF8Chunk(chunk []byte) bool { + if w.jw.escapePolicy() != UTF8Strict { + return true + } + + idx := utf8x.FirstInvalid(chunk) + if idx < 0 || !utf8.FullRune(chunk[idx:]) { + return true + } + + w.jw.SetErr( + fmt.Errorf("%w at byte %d of this chunk: %w", ErrInvalidUTF8, idx, ErrDefaultWriter), + ) + + return false +} diff --git a/json/writers/default-writer/utf8_test.go b/json/writers/default-writer/utf8_test.go new file mode 100644 index 0000000..e76e7ba --- /dev/null +++ b/json/writers/default-writer/utf8_test.go @@ -0,0 +1,402 @@ +package writer + +import ( + "bytes" + "fmt" + "io" + "strings" + "testing" + "unicode/utf8" + + "github.com/go-openapi/testify/v2/assert" + "github.com/go-openapi/testify/v2/require" + + "github.com/go-openapi/core/json/internal/utf8x" + lexer "github.com/go-openapi/core/json/lexers/default-lexer" + "github.com/go-openapi/core/json/lexers/token" + "github.com/go-openapi/core/json/stores/values" +) + +// UTF-8 handling on the way OUT (see .claude/plans/utf8-validation.md, round 3). +// +// Before the policy existed the writers substituted U+FFFD for ill-formed input silently and unconditionally, with +// one inconsistency: a rune truncated at the very end of a StringCopy errored while every other fault was rewritten. +// These tests pin the three policies across all four writers and every entry point, and — because the same +// substitution rule is now claimed on both sides — cross-check the output against the lexer's utf8x.Sanitize. + +// utf8WriterCase is an input and what each policy must make of it. +type utf8WriterCase struct { + name string + in string + // valid inputs are written unchanged by every policy. + valid bool +} + +func utf8WriterCases() []utf8WriterCase { + return []utf8WriterCase{ + {name: "ascii", in: "plain text", valid: true}, + {name: "2-byte", in: "café", valid: true}, + {name: "3-byte", in: "\u65e5\u672c\u8a9e", valid: true}, + {name: "4-byte", in: "\U0001D11E", valid: true}, + {name: "needs escaping", in: "a\"b\\c\nd\te", valid: true}, + {name: "control chars", in: "a\x00b\x1fc", valid: true}, + + {name: "lone ff", in: "\xff"}, + {name: "lone continuation", in: "a\x80b"}, + {name: "overlong", in: "a\xc0\xafb"}, + {name: "encoded surrogate", in: "a\xed\xa0\x80b"}, + {name: "beyond U+10FFFF", in: "a\xf4\xbf\xbf\xbfb"}, + {name: "truncated at end", in: "abc\xe2\x82"}, + {name: "truncated then text", in: "a\xe0\xffb"}, + {name: "invalid amid escapes", in: "a\"b\xffc\nd"}, + } +} + +// utf8Writer is one writer implementation configured with a policy, reduced to what the tests need. +type utf8Writer struct { + name string + build func(w io.Writer, policy UTF8Policy) utf8WriterUnderTest +} + +type utf8WriterUnderTest interface { + String(string) + StringBytes([]byte) + StringRunes([]rune) + StringCopy(io.Reader) + Raw([]byte) + RawCopy(io.Reader) + Token(token.T) + VerbatimToken([]byte, token.T) + Err() error +} + +// flusher is implemented by the buffered writers; the unbuffered one needs no flush. +type flusher interface{ Flush() error } + +func utf8Writers() []utf8Writer { + return []utf8Writer{ + {name: "Unbuffered", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewUnbuffered(w, WithUnbufferedUTF8Policy(p)) + }}, + {name: "Buffered", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewBuffered(w, WithUTF8Policy(p)) + }}, + {name: "Buffered/tiny", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + // a buffer smaller than the values forces flushes in the middle of escaping + return NewBuffered(w, WithUTF8Policy(p), WithBufferSize(minBufferSize)) + }}, + {name: "Indented", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewIndented(w, WithIndentBufferedOptions(WithUTF8Policy(p))) + }}, + {name: "YAML", build: func(w io.Writer, p UTF8Policy) utf8WriterUnderTest { + return NewYAML(w, WithYAMLBufferedOptions(WithUTF8Policy(p))) + }}, + } +} + +// writeWith runs one entry point and returns what reached the underlying writer. +func writeWith( + t *testing.T, + wr utf8Writer, + policy UTF8Policy, + in string, + call func(utf8WriterUnderTest, string), +) (string, error) { + t.Helper() + + var buf bytes.Buffer + w := wr.build(&buf, policy) + call(w, in) + if f, ok := w.(flusher); ok { + _ = f.Flush() + } + + return buf.String(), w.Err() +} + +// entryPoints are the caller-supplied ways in. Each is subject to the policy. +func entryPoints() []struct { + name string + quoted bool // the output is a quoted, escaped JSON string + call func(utf8WriterUnderTest, string) + skipFor func(utf8WriterCase) bool +} { + return []struct { + name string + quoted bool + call func(utf8WriterUnderTest, string) + skipFor func(utf8WriterCase) bool + }{ + {name: "String", quoted: true, call: func(w utf8WriterUnderTest, s string) { w.String(s) }}, + { + name: "StringBytes", + quoted: true, + call: func(w utf8WriterUnderTest, s string) { w.StringBytes([]byte(s)) }, + }, + {name: "StringCopy", quoted: true, call: func(w utf8WriterUnderTest, s string) { + w.StringCopy(strings.NewReader(s)) + }}, + {name: "Raw", call: func(w utf8WriterUnderTest, s string) { w.Raw([]byte(s)) }}, + { + name: "RawCopy", + call: func(w utf8WriterUnderTest, s string) { w.RawCopy(strings.NewReader(s)) }, + }, + } +} + +// TestWriterUTF8Strict pins the default: ill-formed input is refused, well-formed input is written. +func TestWriterUTF8Strict(t *testing.T) { + for _, wr := range utf8Writers() { + for _, ep := range entryPoints() { + for _, tc := range utf8WriterCases() { + t.Run(wr.name+"/"+ep.name+"/"+tc.name, func(t *testing.T) { + out, err := writeWith(t, wr, UTF8Strict, tc.in, ep.call) + + if !tc.valid { + require.Errorf(t, err, "ill-formed input must be refused, got %q", out) + assert.ErrorIs(t, err, ErrInvalidUTF8) + + return + } + + require.NoError(t, err) + assert.Truef(t, utf8.ValidString(out), "output must be valid UTF-8: %q", out) + if !ep.quoted { + assert.Equal(t, tc.in, out) + } + }) + } + } + } +} + +// TestWriterUTF8ReplaceAlwaysEmitsValidUTF8 pins the promise of UTF8Replace: it never errors on ill-formed input, and +// what comes out is always valid UTF-8 — on every writer and every entry point, including the streaming ones where a +// sequence can be split across reads. +func TestWriterUTF8ReplaceAlwaysEmitsValidUTF8(t *testing.T) { + for _, wr := range utf8Writers() { + for _, ep := range entryPoints() { + for _, tc := range utf8WriterCases() { + t.Run(wr.name+"/"+ep.name+"/"+tc.name, func(t *testing.T) { + out, err := writeWith(t, wr, UTF8Replace, tc.in, ep.call) + + require.NoErrorf(t, err, "UTF8Replace must not error on %q", tc.in) + assert.Truef(t, utf8.ValidString(out), + "UTF8Replace must emit valid UTF-8, got %q for input %q", out, tc.in) + if tc.valid && !ep.quoted { + assert.Equal(t, tc.in, out) + } + }) + } + } + } +} + +// TestWriterSubstitutionMatchesLexer is the point of sharing utf8x: the bytes a writer substitutes must be the bytes +// the lexer would have produced for the same input, so a value can round-trip through both without changing again. +func TestWriterSubstitutionMatchesLexer(t *testing.T) { + for _, wr := range utf8Writers() { + for _, tc := range utf8WriterCases() { + if tc.valid { + continue + } + t.Run(wr.name+"/"+tc.name, func(t *testing.T) { + want := string(utf8x.Sanitize(nil, []byte(tc.in))) + + out, err := writeWith(t, wr, UTF8Replace, tc.in, + func(w utf8WriterUnderTest, s string) { w.Raw([]byte(s)) }) + require.NoError(t, err) + assert.Equalf(t, want, out, "Raw under UTF8Replace must agree with utf8x.Sanitize") + + // the escaped path reaches the same text, wrapped in quotes and with escapes applied + quoted, err := writeWith(t, wr, UTF8Replace, tc.in, + func(w utf8WriterUnderTest, s string) { w.StringBytes([]byte(s)) }) + require.NoError(t, err) + assert.Equalf(t, expectedJSONString(want), quoted, + "the escaped path must substitute the same way") + }) + } + } +} + +// TestWriterUTF8Passthrough pins that passthrough really passes bytes through rather than quietly substituting, +// which is the only thing that distinguishes it from UTF8Replace. +func TestWriterUTF8Passthrough(t *testing.T) { + for _, wr := range utf8Writers() { + for _, tc := range utf8WriterCases() { + if tc.valid { + continue + } + t.Run(wr.name+"/"+tc.name, func(t *testing.T) { + out, err := writeWith(t, wr, UTF8Passthrough, tc.in, + func(w utf8WriterUnderTest, s string) { w.Raw([]byte(s)) }) + require.NoError(t, err) + assert.Equal(t, tc.in, out, "Raw must not touch the bytes") + + esc, err := writeWith(t, wr, UTF8Passthrough, tc.in, + func(w utf8WriterUnderTest, s string) { w.StringBytes([]byte(s)) }) + require.NoError(t, err) + assert.Falsef( + t, + utf8.ValidString(esc), + "passthrough must NOT sanitize: the ill-formed bytes should survive escaping, got %q", + esc, + ) + assert.NotContainsf(t, esc, string(utf8.RuneError), + "passthrough must not substitute U+FFFD, got %q", esc) + }) + } + } +} + +// TestWriterStreamingSplitSequence forces a multi-byte sequence to straddle a read boundary at every offset. A chunk +// ending mid-sequence is NOT an error, and the validator must not mistake it for one — nor miss a genuine fault that +// only shows up once the chunks are joined. +func TestWriterStreamingSplitSequence(t *testing.T) { + valid := strings.Repeat("a", 40) + "\u65e5\U0001D11E" + strings.Repeat("b", 40) + broken := strings.Repeat( + "a", + 40, + ) + "\xe6\x41" + strings.Repeat( + "b", + 40, + ) // lead byte then a non-continuation + + for _, wr := range utf8Writers() { + for chunk := 1; chunk <= 8; chunk++ { + t.Run(fmt.Sprintf("%s/chunk=%d", wr.name, chunk), func(t *testing.T) { + for _, ep := range []string{"StringCopy", "RawCopy"} { + call := func(w utf8WriterUnderTest, s string) { w.StringCopy(&chunkReader{data: []byte(s), chunk: chunk}) } + if ep == "RawCopy" { + call = func(w utf8WriterUnderTest, s string) { + w.RawCopy(&chunkReader{data: []byte(s), chunk: chunk}) + } + } + + out, err := writeWith(t, wr, UTF8Strict, valid, call) + require.NoErrorf( + t, + err, + "%s: a sequence split across reads is not an error", + ep, + ) + assert.Truef(t, utf8.ValidString(out), "%s: %q", ep, out) + + _, err = writeWith(t, wr, UTF8Strict, broken, call) + require.Errorf( + t, + err, + "%s: a fault spanning the read boundary must be caught", + ep, + ) + + out, err = writeWith(t, wr, UTF8Replace, broken, call) + require.NoError(t, err) + assert.Truef(t, utf8.ValidString(out), "%s: %q", ep, out) + } + }) + } + } +} + +// TestWriterTrustsTokenValues pins the deliberate loophole: a value coming from a lexer token is NOT re-validated, +// because the lexers already guarantee valid UTF-8. A lexer relaxed to UTF8Passthrough can therefore hand the writer +// ill-formed bytes that a strict writer will emit — the documented cost of not paying for a second check. +func TestWriterTrustsTokenValues(t *testing.T) { + const doc = "[\"a\xffb\"]" + + // the lexer must be relaxed for such a token to exist at all + lx := lexer.NewWithBytes([]byte(doc), lexer.WithUTF8Policy(lexer.UTF8Passthrough)) + var value token.T + for tok := range lx.Tokens() { + if tok.Kind() == token.String { + value = token.MakeWithValue(token.String, bytes.Clone(tok.Value())) + } + } + require.NoError(t, lx.Err()) + require.False( + t, + utf8.Valid(value.Value()), + "the relaxed lexer should have produced ill-formed bytes", + ) + + // Token: not re-validated, so no error — but the escaper still decodes as it goes, so the ill-formed byte is + // silently substituted. The loophole is the SILENCE, not invalid output. + var buf bytes.Buffer + w := NewUnbuffered(&buf) // UTF8Strict + w.Token(value) + require.NoError(t, w.Err(), "token values are trusted, not re-validated") + assert.Equal(t, "\"a\ufffdb\"", buf.String(), + "the escaper substitutes as it goes, so the output stays valid UTF-8") + + // VerbatimToken writes the raw value with no escaping at all, so there ill-formed bytes really do reach the + // output. This is the sharp edge of trusting tokens and is documented as such. + var verbatim bytes.Buffer + vw := NewUnbuffered(&verbatim) // UTF8Strict + vw.VerbatimToken(nil, value) + require.NoError(t, vw.Err()) + assert.Equal(t, "\"a\xffb\"", verbatim.String(), + "verbatim writing is byte-for-byte by definition: nothing inspects or rewrites the value") + + // VerbatimValue is NOT in the trusted group: it renders through the ordinary string path + var vv bytes.Buffer + vvw := NewUnbuffered(&vv) + vvw.VerbatimValue(values.MakeVerbatimValue(nil, values.MakeStringValue(string(value.Value())))) + require.ErrorIs(t, vvw.Err(), ErrInvalidUTF8, + "VerbatimValue goes through the string path, so it is checked like any caller data") + + // the same bytes offered as caller data ARE checked + var caller bytes.Buffer + cw := NewUnbuffered(&caller) + cw.StringBytes(value.Value()) + require.ErrorIs(t, cw.Err(), ErrInvalidUTF8) +} + +// TestWriterStringRunesRejectsNonScalar pins that a []rune carrying a surrogate or an out-of-range value is refused +// under UTF8Strict rather than silently encoded as U+FFFD by utf8.AppendRune. +func TestWriterStringRunesRejectsNonScalar(t *testing.T) { + for _, tc := range []struct { + name string + runes []rune + }{ + {name: "surrogate", runes: []rune{'a', 0xD800, 'b'}}, + {name: "beyond max", runes: []rune{'a', 0x110000, 'b'}}, + {name: "negative", runes: []rune{'a', -1, 'b'}}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + w := NewUnbuffered(&buf) + w.StringRunes(tc.runes) + require.ErrorIs(t, w.Err(), ErrInvalidUTF8) + + var replaced bytes.Buffer + rw := NewUnbuffered(&replaced, WithUnbufferedUTF8Policy(UTF8Replace)) + rw.StringRunes(tc.runes) + require.NoError(t, rw.Err()) + assert.Equal(t, "\"a�b\"", replaced.String()) + }) + } +} + +// TestWriterDefaultIsStrict pins the default so it cannot regress to the old silent substitution. +func TestWriterDefaultIsStrict(t *testing.T) { + for _, wr := range utf8Writers() { + t.Run(wr.name, func(t *testing.T) { + var buf bytes.Buffer + w := wr.build(&buf, UTF8Strict) // what a caller passing no option gets + w.StringBytes([]byte("a\xffb")) + require.ErrorIs(t, w.Err(), ErrInvalidUTF8) + }) + } + + // and literally with no option at all + var buf bytes.Buffer + u := NewUnbuffered(&buf) + u.StringBytes([]byte("a\xffb")) + require.ErrorIs(t, u.Err(), ErrInvalidUTF8) + + var buf2 bytes.Buffer + b := NewBuffered(&buf2) + b.StringBytes([]byte("a\xffb")) + require.ErrorIs(t, b.Err(), ErrInvalidUTF8) +}