Fix/utf8 validation prototype

.gitattributes

@@ -0,0 +1,13 @@
+# Conformance fixtures and golden files are DATA, not source: their exact bytes ARE the test.
+#
+# Without this, a checkout with core.autocrlf=true (the Windows default) rewrites every LF to
+# CRLF and silently changes what is being tested — a YAML fixture that hinges on a trailing
+# space or a tab becomes a different document, a JSON fixture that must be rejected may start
+# being accepted, and every golden file mismatches. A harness may normalise line endings
+# defensively as well, but the fixtures should never have been translated in the first place.
+testdata/** -text
+**/testdata/** -text
+*.golden -text
+
+# Generated assembly is checked in and must match what `go generate` produces byte for byte.
+*.s -text

NOTICE

@@ -0,0 +1,44 @@
+Copyright 2015-2025 go-swagger maintainers
+
+// SPDX-FileCopyrightText: Copyright 2015-2025 go-swagger maintainers
+// SPDX-License-Identifier: Apache-2.0
+
+This software library, github.com/go-openapi/core, includes software developed
+by the go-swagger and go-openapi maintainers ("go-swagger maintainers").
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this software except in compliance with the License.
+
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0.
+
+It ships with copies of other software which license terms are recalled below.
+
+https://github.com/nst/JSONTestSuite
+===========================
+
+// SPDX-FileCopyrightText: Copyright (c) 2016 Nicolas Seriot
+// SPDX-License-Identifier: MIT
+
+MIT License
+
+Copyright (c) 2016 Nicolas Seriot
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

go.work

@@ -4,6 +4,7 @@ use (
 	.
 	./json
 	./json/benchmarks
+	./json/internal/utf8x/_asm
 	./json/lexers/default-lexer/internal/strscan/_asm
 	./json/lexers/yaml-lexer
 )

json/benchmarks/lexers/lanes/utf8scratch_test.go

@@ -0,0 +1,132 @@
+package lane
+
+// SCRATCH (utf-8 validation prototype): measures the cost of the three prototype
+// validation modes across the corpus. Delete when the design is settled.
+
+import (
+	"fmt"
+	"io"
+	"testing"
+	"unicode/utf8"
+
+	lexer "github.com/go-openapi/core/json/lexers/default-lexer"
+	"github.com/go-openapi/core/json/lexers/token"
+	"github.com/go-openapi/core/json/testdata/workloads"
+)
+
+func BenchmarkUTF8Validate(b *testing.B) {
+	suite, err := workloads.Corpus()
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	modes := []struct {
+		name   string
+		policy lexer.UTF8Policy
+	}{
+		{"passthrough", lexer.UTF8Passthrough}, // the pre-validation baseline
+		{"strict", lexer.UTF8Strict},           // the new default
+		{"replace", lexer.UTF8Replace},
+	}
+
+	for _, wl := range suite {
+		data := wl.Data
+		b.Run(wl.Name, func(b *testing.B) {
+			for _, m := range modes {
+				b.Run("L/"+m.name, func(b *testing.B) {
+					var sink int
+					lx := lexer.NewWithBytes(data, lexer.WithUTF8Policy(m.policy))
+					b.SetBytes(int64(len(data)))
+					b.ReportAllocs()
+					b.ResetTimer()
+					for b.Loop() {
+						lx.ResetWithBytes(data)
+						for t := range lx.Tokens() {
+							sink += int(t.Kind())
+						}
+					}
+					fmt.Fprint(io.Discard, sink)
+				})
+				b.Run("VL/"+m.name, func(b *testing.B) {
+					var sink int
+					lx := lexer.NewVerbatimWithBytes(data, lexer.WithUTF8Policy(m.policy))
+					b.SetBytes(int64(len(data)))
+					b.ReportAllocs()
+					b.ResetTimer()
+					for b.Loop() {
+						lx.ResetWithBytes(data)
+						for t := range lx.Tokens() {
+							sink += int(t.Kind())
+						}
+					}
+					fmt.Fprint(io.Discard, sink)
+				})
+			}
+		})
+	}
+}
+
+// TestCorpusStringProfile reports, per workload, how much of the payload is string
+// bytes and what share of those strings carry a non-ASCII byte -- i.e. how many
+// values a fused detector would hand to the validator.
+func TestCorpusStringProfile(t *testing.T) {
+	suite, err := workloads.Corpus()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, wl := range suite {
+		var (
+			strings, nonASCII, strBytes, nonASCIIBytes int
+			longStrings, longNonASCII                  int
+		)
+		lx := lexer.NewWithBytes(wl.Data)
+		for tok := range lx.Tokens() {
+			if tok.Kind() != token.String && tok.Kind() != token.Key {
+				continue
+			}
+			v := tok.Value()
+			strings++
+			strBytes += len(v)
+			if len(v) >= 16 {
+				longStrings++
+			}
+			if !isASCII(v) {
+				nonASCII++
+				nonASCIIBytes += len(v)
+				if len(v) >= 16 {
+					longNonASCII++
+				}
+			}
+			if !utf8.Valid(v) {
+				t.Errorf("%s: corpus carries invalid utf8", wl.Name)
+			}
+		}
+		if !lx.Ok() {
+			t.Fatalf("%s: %v", wl.Name, lx.Err())
+		}
+		t.Logf(
+			"%-20s bytes=%d strings=%d (%d long) strBytes=%d (%.1f%% of payload) nonASCII=%d (%.2f%% of strings, %d long) nonASCIIBytes=%d",
+			wl.Name,
+			len(wl.Data),
+			strings,
+			longStrings,
+			strBytes,
+			100*float64(strBytes)/float64(len(wl.Data)),
+			nonASCII,
+			100*float64(nonASCII)/float64(max(strings, 1)),
+			longNonASCII,
+			nonASCIIBytes,
+		)
+	}
+}
+
+func isASCII(b []byte) bool {
+	for _, c := range b {
+		if c >= 0x80 {
+			return false
+		}
+	}
+
+	return true
+}

json/internal/utf8x/_asm/asm.go

@@ -0,0 +1,195 @@
+//nolint:revive,mnd
+package main
+
+import (
+	"github.com/mmcloughlin/avo/attr"
+	. "github.com/mmcloughlin/avo/build"
+	. "github.com/mmcloughlin/avo/operand"
+)
+
+func main() {
+	validateUTF8()
+
+	Generate()
+}
+
+// table16 emits a 16-byte read-only lookup table for VPSHUFB.
+//
+// VPSHUFB indexes within each 128-bit lane, so the table is broadcast into both lanes of a YMM at load time
+// (VBROADCASTI128) and one 16-byte copy is all that needs to live in rodata.
+func table16(name string, b ...byte) Mem {
+	if len(b) != 16 {
+		panic(name + ": a VPSHUFB table must be exactly 16 bytes")
+	}
+	m := GLOBL(name, attr.RODATA|attr.NOPTR)
+	for i, v := range b {
+		DATA(i, U8(v))
+	}
+
+	return m
+}
+
+// UTF-8 error classes of the Keiser-Lemire "lookup4" algorithm (simdutf
+// src/generic/utf8_validation/utf8_lookup4_algorithm.h). Each is one bit of a per-byte bitset; a byte is ill-formed
+// iff its three table lookups agree on at least one violated rule.
+const (
+	tooShort     = 1 << 0 // 11______ followed by 0_______ or 11______
+	tooLong      = 1 << 1 // 0_______ followed by 10______
+	overlong3    = 1 << 2 // 11100000 100_____
+	tooLarge     = 1 << 3 // above U+10FFFF
+	surrogate    = 1 << 4 // 11101101 101_____ : an encoded UTF-16 surrogate
+	overlong2    = 1 << 5 // 1100000_ 10______
+	tooLarge1000 = 1 << 6
+	overlong4    = 1 << 6 // 11110000 1000____ (shares the bit with tooLarge1000)
+	twoConts     = 1 << 7 // 10______ 10______
+
+	// carry marks the classes that depend only on the high nibble of the leading byte, so the low-nibble table must
+	// let them through.
+	carry = tooShort | tooLong | twoConts
+)
+
+// validateUTF8 emits the AVX2 UTF-8 validator: three VPSHUFB table lookups classify every (previous byte, byte) pair,
+// and a saturating-subtract pass asserts that positions requiring a 2nd or 3rd continuation byte have one.
+//
+// It deliberately does NOT handle a partial trailing block or the end-of-input "sequence left incomplete" check: the
+// Go caller re-validates the last few bytes scalar-ly from a sequence boundary (see ScanUTF8), which is both simpler
+// and how simdutf itself locates errors.
+func validateUTF8() {
+	TEXT("validateUTF8BlocksAVX2", NOSPLIT, "func(data []byte) bool")
+	Doc(
+		"validateUTF8BlocksAVX2 reports whether the whole 32-byte blocks of data are valid UTF-8, ignoring any sequence that continues past the end. len(data) must be a non-zero multiple of 32. AVX2, 32 bytes/iter.",
+	)
+	ptr := Load(Param("data").Base(), GP64())
+	n := Load(Param("data").Len(), GP64())
+
+	// byte_1_high: indexed by the HIGH nibble of the previous byte.
+	tableHigh := YMM()
+	VBROADCASTI128(table16(
+		"utf8t1h",
+		tooLong,
+		tooLong,
+		tooLong,
+		tooLong,
+		tooLong,
+		tooLong,
+		tooLong,
+		tooLong, // 0_______ : ASCII lead
+		twoConts,
+		twoConts,
+		twoConts,
+		twoConts,                     // 10______ : continuation
+		tooShort|overlong2,           // 1100____
+		tooShort,                     // 1101____
+		tooShort|overlong3|surrogate, // 1110____
+		tooShort|tooLarge|tooLarge1000|overlong4, // 1111____
+	), tableHigh)
+
+	// byte_1_low: indexed by the LOW nibble of the previous byte.
+	tableLow := YMM()
+	VBROADCASTI128(table16("utf8t1l",
+		carry|overlong3|overlong2|overlong4, // ____0000
+		carry|overlong2,                     // ____0001
+		carry, carry,                        // ____001_
+		carry|tooLarge,              // ____0100
+		carry|tooLarge|tooLarge1000, // ____0101
+		carry|tooLarge|tooLarge1000, // ____011_
+		carry|tooLarge|tooLarge1000,
+		carry|tooLarge|tooLarge1000, // ____1___
+		carry|tooLarge|tooLarge1000,
+		carry|tooLarge|tooLarge1000,
+		carry|tooLarge|tooLarge1000,
+		carry|tooLarge|tooLarge1000,
+		carry|tooLarge|tooLarge1000|surrogate, // ____1101
+		carry|tooLarge|tooLarge1000,
+		carry|tooLarge|tooLarge1000,
+	), tableLow)
+
+	// byte_2_high: indexed by the HIGH nibble of the current byte.
+	tableCur := YMM()
+	VBROADCASTI128(table16("utf8t2h",
+		tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, tooShort, // 0_______
+		tooLong|overlong2|twoConts|overlong3|tooLarge1000|overlong4, // 1000____
+		tooLong|overlong2|twoConts|overlong3|tooLarge,               // 1001____
+		tooLong|overlong2|twoConts|surrogate|tooLarge,               // 101_____
+		tooLong|overlong2|twoConts|surrogate|tooLarge,
+		tooShort, tooShort, tooShort, tooShort, // 11______
+	), tableCur)
+
+	nibble := YMM()
+	VPBROADCASTB(ConstData("utf8n", U8(0x0f)), nibble)
+	high := YMM()
+	VPBROADCASTB(ConstData("utf8h", U8(0x80)), high)
+	// 0xe0-0x80 and 0xf0-0x80: a saturating subtract leaves the high bit set exactly where a 3rd (resp. 4th) byte of a
+	// sequence is required.
+	third := YMM()
+	VPBROADCASTB(ConstData("utf8c3", U8(0xe0-0x80)), third)
+	fourth := YMM()
+	VPBROADCASTB(ConstData("utf8c4", U8(0xf0-0x80)), fourth)
+
+	// prev holds the previous block; it starts as zeros, which is the correct "nothing precedes the input" state
+	// (NUL is ASCII, so it imposes no continuation).
+	prev := YMM()
+	VPXOR(prev, prev, prev)
+	errAcc := YMM()
+	VPXOR(errAcc, errAcc, errAcc)
+
+	i := GP64()
+	XORQ(i, i)
+
+	Label("utf8loop")
+	input := YMM()
+	VMOVDQU(Mem{Base: ptr, Index: i, Scale: 1}, input)
+
+	// The three "previous byte" views. All three shift the same concatenation of the previous block's high lane with
+	// this block's low lane, so the permute is computed once.
+	perm := YMM()
+	VPERM2I128(Imm(0x21), input, prev, perm)
+	prev1, prev2, prev3 := YMM(), YMM(), YMM()
+	VPALIGNR(Imm(15), perm, input, prev1)
+	VPALIGNR(Imm(14), perm, input, prev2)
+	VPALIGNR(Imm(13), perm, input, prev3)
+
+	// check_special_cases: AND of the three lookups leaves a bit set only where every view agrees the pair is illegal.
+	idx := YMM()
+	VPSRLW(Imm(4), prev1, idx)
+	VPAND(nibble, idx, idx)
+	b1h := YMM()
+	VPSHUFB(idx, tableHigh, b1h)
+
+	VPAND(nibble, prev1, idx)
+	b1l := YMM()
+	VPSHUFB(idx, tableLow, b1l)
+
+	VPSRLW(Imm(4), input, idx)
+	VPAND(nibble, idx, idx)
+	b2h := YMM()
+	VPSHUFB(idx, tableCur, b2h)
+
+	sc := YMM()
+	VPAND(b1h, b1l, sc)
+	VPAND(b2h, sc, sc)
+
+	// check_multibyte_lengths: a position two bytes after a 3-byte lead, or three after a 4-byte lead, MUST be a
+	// continuation. XOR-ing that requirement against the special-case bits leaves a set bit exactly where the two
+	// disagree — i.e. a missing or unexpected continuation.
+	must := YMM()
+	VPSUBUSB(third, prev2, must)
+	tmp := YMM()
+	VPSUBUSB(fourth, prev3, tmp)
+	VPOR(tmp, must, must)
+	VPAND(high, must, must)
+	VPXOR(sc, must, must)
+	VPOR(must, errAcc, errAcc)
+
+	VMOVDQA(input, prev)
+	ADDQ(Imm(32), i)
+	CMPQ(i, n)
+	JB(LabelRef("utf8loop"))
+
+	ok := GP8()
+	VPTEST(errAcc, errAcc)
+	SETEQ(ok) // ZF is set when the accumulator is all zero: no error was ever flagged
+	Store(ok, ReturnIndex(0))
+	VZEROUPPER()
+	RET()
+}