From 126578e84a376fdc96c9879d10c84ec788ca7b7f Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Mon, 19 Oct 2026 16:23:13 +0000 Subject: [PATCH 1/2] feat(vindex/mapfn): add WASM guest SDK, wazero sandbox host, examples, and CLI - Add zero-allocation WASM guest SDK (vindex/mapfn/sdk) and zero-I/O WASI preview1 Wazero sandbox host (vindex/internal/wasmhost). - Add canonical MapFn plugins for Starter, Go SumDB, and Certificate Transparency (vindex/mapfn/examples). - Add offline vindex-wasm CLI for inspecting, testing, determinism-checking, and benchmarking WASM mappers (vindex/cmd/vindex-wasm). --- go.mod | 4 + go.sum | 6 + vindex/cmd/vindex-wasm/bench_cmd.go | 176 ++++ vindex/cmd/vindex-wasm/inspect_cmd.go | 104 +++ vindex/cmd/vindex-wasm/main.go | 78 ++ vindex/cmd/vindex-wasm/main_test.go | 235 ++++++ vindex/cmd/vindex-wasm/test_cmd.go | 152 ++++ .../wasmhost/testdata/testguest/main.go | 56 ++ vindex/internal/wasmhost/wasm.go | 550 +++++++++++++ vindex/internal/wasmhost/wasm_test.go | 752 ++++++++++++++++++ vindex/internal/wasmhost/wasmtest.go | 69 ++ vindex/mapfn/README.md | 4 +- vindex/mapfn/examples/README.md | 89 +++ vindex/mapfn/examples/ct/README.md | 37 + vindex/mapfn/examples/ct/ct_test.go | 407 ++++++++++ vindex/mapfn/examples/ct/main.go | 489 ++++++++++++ .../examples/ct/testdata/golden_leaves.json | 72 ++ vindex/mapfn/examples/starter/main.go | 69 ++ vindex/mapfn/examples/starter/starter_test.go | 58 ++ vindex/mapfn/examples/sumdb/main.go | 109 +++ vindex/mapfn/examples/sumdb/sumdb_test.go | 142 ++++ vindex/mapfn/sdk/arena.go | 124 +++ vindex/mapfn/sdk/sdk.go | 43 + vindex/mapfn/sdk/sdk_test.go | 217 +++++ 24 files changed, 4040 insertions(+), 2 deletions(-) create mode 100644 vindex/cmd/vindex-wasm/bench_cmd.go create mode 100644 vindex/cmd/vindex-wasm/inspect_cmd.go create mode 100644 vindex/cmd/vindex-wasm/main.go create mode 100644 vindex/cmd/vindex-wasm/main_test.go create mode 100644 vindex/cmd/vindex-wasm/test_cmd.go create mode 100644 vindex/internal/wasmhost/testdata/testguest/main.go create mode 100644 vindex/internal/wasmhost/wasm.go create mode 100644 vindex/internal/wasmhost/wasm_test.go create mode 100644 vindex/internal/wasmhost/wasmtest.go create mode 100644 vindex/mapfn/examples/README.md create mode 100644 vindex/mapfn/examples/ct/README.md create mode 100644 vindex/mapfn/examples/ct/ct_test.go create mode 100644 vindex/mapfn/examples/ct/main.go create mode 100644 vindex/mapfn/examples/ct/testdata/golden_leaves.json create mode 100644 vindex/mapfn/examples/starter/main.go create mode 100644 vindex/mapfn/examples/starter/starter_test.go create mode 100644 vindex/mapfn/examples/sumdb/main.go create mode 100644 vindex/mapfn/examples/sumdb/sumdb_test.go create mode 100644 vindex/mapfn/sdk/arena.go create mode 100644 vindex/mapfn/sdk/sdk.go create mode 100644 vindex/mapfn/sdk/sdk_test.go diff --git a/go.mod b/go.mod index a633f23b..fac53ddf 100644 --- a/go.mod +++ b/go.mod @@ -5,10 +5,13 @@ go 1.27.0 toolchain go1.27.3 require ( + github.com/google/go-cmp v0.7.0 github.com/prometheus/client_golang v1.25.0 + github.com/tetratelabs/wazero v1.12.0 github.com/transparency-dev/formats v0.1.2-0.20261027211732-55fe9bf0fa53 github.com/transparency-dev/merkle v0.0.2 golang.org/x/mod v0.42.0 + golang.org/x/net v0.62.0 k8s.io/klog/v2 v2.140.0 ) @@ -24,5 +27,6 @@ require ( github.com/prometheus/procfs v0.22.0 // indirect golang.org/x/crypto v0.58.0 // indirect golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/go.sum b/go.sum index 57a04acf..50085470 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0o github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= +github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= github.com/transparency-dev/formats v0.1.2-0.20261027211732-55fe9bf0fa53 h1:yhYk19lcG42d9r9m957D5HpDIeP+NgpuTKy/S50++y0= github.com/transparency-dev/formats v0.1.2-0.20261027211732-55fe9bf0fa53/go.mod h1:AcGPVj62+dtCMlD/pdeG1LeUkYo0D+IHRWg1LmQazGE= github.com/transparency-dev/merkle v0.0.2 h1:Q9nBoQcZcgPamMkGn7ghV8XiTZ/kRxn1yCG81+twTK4= @@ -36,8 +38,12 @@ golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +golang.org/x/net v0.62.0 h1:18ELalfM5+XNRwzM2nYqXsZQ9ByS1uhj2O/U7+5jqRY= +golang.org/x/net v0.62.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= diff --git a/vindex/cmd/vindex-wasm/bench_cmd.go b/vindex/cmd/vindex-wasm/bench_cmd.go new file mode 100644 index 00000000..b8396bcf --- /dev/null +++ b/vindex/cmd/vindex-wasm/bench_cmd.go @@ -0,0 +1,176 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "slices" + "sync" + "time" + + "github.com/transparency-dev/incubator/vindex/internal/wasmhost" +) + +func runBenchCmd(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("bench", flag.ContinueOnError) + wasmPath := fs.String("wasm", "", "Path to compiled WASM plugin file (required).") + inputStr := fs.String("input", "github.com/example/benchmodule v1.2.3 h1:benchHash123456789=\n", "Benchmark input leaf payload.") + inputFile := fs.String("input_file", "", "Path to file containing input leaf payload.") + bundleSize := fs.Int("bundle_size", 256, "Number of leaves per bundle (defaults to 256).") + iterations := fs.Int("iterations", 1000, "Total number of bundle executions.") + workers := fs.Int("workers", 4, "Number of concurrent worker goroutines and pool size.") + + if err := fs.Parse(args); err != nil { + return err + } + + if *wasmPath == "" { + return errors.New("--wasm flag is required") + } + if *iterations <= 0 { + return errors.New("--iterations must be greater than 0") + } + if *bundleSize <= 0 { + *bundleSize = 256 + } + if *workers <= 0 { + *workers = 1 + } + + wasmBytes, err := os.ReadFile(*wasmPath) + if err != nil { + return fmt.Errorf("failed to read WASM file %q: %w", *wasmPath, err) + } + + var inputBytes []byte + if *inputFile != "" { + b, err := os.ReadFile(*inputFile) + if err != nil { + return fmt.Errorf("failed to read input file %q: %w", *inputFile, err) + } + inputBytes = b + } else { + inputBytes = []byte(*inputStr) + } + + bundle := make([][]byte, *bundleSize) + for i := range bundle { + bundle[i] = inputBytes + } + + wasmHost, err := wasmhost.NewWASMHost(ctx, wasmBytes, *workers) + if err != nil { + return fmt.Errorf("failed to initialize WASM host: %w", err) + } + defer func() { _ = wasmHost.Close(ctx) }() + + // Warmup + for i := 0; i < 10; i++ { + if _, err := wasmHost.MapBundle(ctx, bundle); err != nil { + return fmt.Errorf("warmup MapBundle failed: %w", err) + } + } + + latencies := make([]time.Duration, *iterations) + taskChan := make(chan int, *iterations) + for i := 0; i < *iterations; i++ { + taskChan <- i + } + close(taskChan) + + start := time.Now() + var wg sync.WaitGroup + var execErr error + var errMu sync.Mutex + + for w := 0; w < *workers; w++ { + wg.Add(1) + go func() { + defer wg.Done() + runner, err := wasmHost.NewRunner(ctx) + if err != nil { + errMu.Lock() + if execErr == nil { + execErr = err + } + errMu.Unlock() + return + } + defer func() { _ = runner.Close(ctx) }() + + for idx := range taskChan { + t0 := time.Now() + _, err := runner.MapBundle(ctx, bundle) + d := time.Since(t0) + if err != nil { + errMu.Lock() + if execErr == nil { + execErr = err + } + errMu.Unlock() + return + } + latencies[idx] = d + } + }() + } + + wg.Wait() + totalElapsed := time.Since(start) + + if execErr != nil { + return fmt.Errorf("benchmark execution failed: %w", execErr) + } + + // Calculate stats + slices.Sort(latencies) + n := len(latencies) + var sum time.Duration + for _, l := range latencies { + sum += l + } + + minLat := latencies[0] + p50Lat := latencies[n*50/100] + p90Lat := latencies[n*90/100] + p99Lat := latencies[n*99/100] + maxLat := latencies[n-1] + meanLat := sum / time.Duration(n) + + totalLeaves := (*iterations) * (*bundleSize) + bundlesPerSec := float64(*iterations) / totalElapsed.Seconds() + leavesPerSec := float64(totalLeaves) / totalElapsed.Seconds() + mbPerSec := (float64(len(inputBytes)*totalLeaves) / (1024 * 1024)) / totalElapsed.Seconds() + + fmt.Printf("--- WASM MapFn Benchmark Results ---\n") + fmt.Printf("Bundle Iterations:%d (%d leaves/bundle, %d total leaves)\n", *iterations, *bundleSize, totalLeaves) + fmt.Printf("Workers (pool): %d\n", *workers) + fmt.Printf("Input payload: %d bytes/leaf\n", len(inputBytes)) + fmt.Printf("Total Elapsed: %v\n", totalElapsed) + fmt.Printf("Throughput: %.2f leaves/sec (%.2f bundles/sec, %.2f MB/s)\n", leavesPerSec, bundlesPerSec, mbPerSec) + fmt.Printf("Latencies:\n") + fmt.Printf(" Min: %v\n", minLat) + fmt.Printf(" Mean: %v\n", meanLat) + fmt.Printf(" p50: %v\n", p50Lat) + fmt.Printf(" p90: %v\n", p90Lat) + fmt.Printf(" p99: %v\n", p99Lat) + fmt.Printf(" Max: %v\n", maxLat) + + return nil +} diff --git a/vindex/cmd/vindex-wasm/inspect_cmd.go b/vindex/cmd/vindex-wasm/inspect_cmd.go new file mode 100644 index 00000000..94f95532 --- /dev/null +++ b/vindex/cmd/vindex-wasm/inspect_cmd.go @@ -0,0 +1,104 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "errors" + "flag" + "fmt" + "os" + + "github.com/transparency-dev/incubator/vindex/internal/wasmhost" +) + +func runInspectCmd(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("inspect", flag.ContinueOnError) + wasmPath := fs.String("wasm", "", "Path to compiled WASM plugin file (required).") + + if err := fs.Parse(args); err != nil { + return err + } + + if *wasmPath == "" { + if fs.NArg() > 0 { + *wasmPath = fs.Arg(0) + } else { + return errors.New("--wasm flag is required") + } + } + + wasmBytes, err := os.ReadFile(*wasmPath) + if err != nil { + return fmt.Errorf("failed to read WASM file %q: %w", *wasmPath, err) + } + + compiled, err := wasmhost.CompileModule(ctx, wasmBytes) + if err != nil { + return fmt.Errorf("failed to compile WASM module: %w", err) + } + + exports := compiled.ExportedFunctions() + imports := compiled.ImportedFunctions() + memories := compiled.ExportedMemories() + + hasMemory := memories["memory"] != nil + hasInputBuf := exports["input_buffer"] != nil + hasMapBundle := exports["map_bundle"] != nil + + digest := sha256.Sum256(wasmBytes) + fmt.Printf("=== VIndex WASM MapFn Inspection: %s ===\n", *wasmPath) + fmt.Printf("Module Size: %d bytes (%.2f KB)\n", len(wasmBytes), float64(len(wasmBytes))/1024.0) + fmt.Printf("MapFn (base64): %s\n", base64.StdEncoding.EncodeToString(digest[:])) + fmt.Printf("SHA-256 (hex): %x\n\n", digest) + + fmt.Println("Required ABI Exports:") + if hasMemory { + fmt.Println(" [✓] memory") + } else { + fmt.Println(" [✗] memory: MISSING") + } + if hasInputBuf { + fmt.Println(" [✓] input_buffer () -> i32 (static 4 MB input slab)") + } else { + fmt.Println(" [✗] input_buffer () -> i32: MISSING") + } + if hasMapBundle { + fmt.Println(" [✓] map_bundle (i32, i32) -> i64 (batch tile ABI)") + } else { + fmt.Println(" [✗] map_bundle (i32, i32) -> i64: MISSING") + } + + fmt.Printf("\nImports (%d):\n", len(imports)) + for _, imp := range imports { + mod, name, _ := imp.Import() + fmt.Printf(" - %s.%s\n", mod, name) + } + + fmt.Println("\nABI Compatibility Status:") + if hasMemory && hasInputBuf && hasMapBundle { + host, err := wasmhost.NewWASMHost(ctx, wasmBytes, 1) + if err != nil { + return fmt.Errorf("module failed VIndex host instantiation: %w", err) + } + _ = host.Close(ctx) + fmt.Println(" >>> PASSED: Compatible with VIndex host runtime (vindexd).") + return nil + } + + return errors.New("module failed VIndex MapFn ABI requirements") +} diff --git a/vindex/cmd/vindex-wasm/main.go b/vindex/cmd/vindex-wasm/main.go new file mode 100644 index 00000000..34cd8ff9 --- /dev/null +++ b/vindex/cmd/vindex-wasm/main.go @@ -0,0 +1,78 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// vindex-wasm is a developer tool and test harness for VIndex WASM MapFn plugins. +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/signal" + "syscall" +) + +func main() { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + if err := run(ctx, os.Args[1:]); err != nil { + if !errors.Is(err, context.Canceled) { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + } + os.Exit(1) + } +} + +func run(ctx context.Context, args []string) error { + if len(args) == 0 { + printUsage() + return errors.New("subcommand required") + } + + subcmd := args[0] + subArgs := args[1:] + + switch subcmd { + case "test": + return runTestCmd(ctx, subArgs) + case "inspect": + return runInspectCmd(ctx, subArgs) + case "bench": + return runBenchCmd(ctx, subArgs) + case "-h", "--help", "help": + printUsage() + return nil + default: + printUsage() + return fmt.Errorf("unknown subcommand %q", subcmd) + } +} + +func printUsage() { + fmt.Fprintf(os.Stderr, `vindex-wasm is a utility tool for testing and benchmarking VIndex WASM MapFn plugins. + +Usage: + vindex-wasm [options] + +Commands: + inspect Inspect a WASM binary and verify VIndex MapFn ABI exports and imports. + test Execute a WASM MapFn against an input leaf and inspect mapped entries. + bench Benchmark execution throughput and latency percentiles of a WASM MapFn. + help Display this help message. + +Use "vindex-wasm -h" for more information about a command. +`) +} diff --git a/vindex/cmd/vindex-wasm/main_test.go b/vindex/cmd/vindex-wasm/main_test.go new file mode 100644 index 00000000..5a643f7f --- /dev/null +++ b/vindex/cmd/vindex-wasm/main_test.go @@ -0,0 +1,235 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "math/big" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/transparency-dev/incubator/vindex/internal/wasmhost" +) + +func TestCLI_TestCmd(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + wasmPath := filepath.Join(tmpDir, "test.wasm") + if err := os.WriteFile(wasmPath, wasmhost.TestWASMBytes(), 0o644); err != nil { + t.Fatalf("failed to write wasm file: %v", err) + } + + // 1. Plain output + err := run(ctx, []string{"test", "--wasm=" + wasmPath, "--input=hello"}) + if err != nil { + t.Fatalf("test command failed: %v", err) + } + + // 2. JSON output + err = run(ctx, []string{"test", "--wasm=" + wasmPath, "--input=hello", "--format=json"}) + if err != nil { + t.Fatalf("test json command failed: %v", err) + } + + // 3. Hex input + err = run(ctx, []string{"test", "--wasm=" + wasmPath, "--hex=68656c6c6f"}) + if err != nil { + t.Fatalf("test hex command failed: %v", err) + } + + // 4. Input file + inFile := filepath.Join(tmpDir, "input.txt") + if err := os.WriteFile(inFile, []byte("file_input"), 0o644); err != nil { + t.Fatalf("failed to write input file: %v", err) + } + err = run(ctx, []string{"test", "--wasm=" + wasmPath, "--input_file=" + inFile}) + if err != nil { + t.Fatalf("test input_file command failed: %v", err) + } + + // 5. Positional input + err = run(ctx, []string{"test", "--wasm=" + wasmPath, "positional_arg_leaf"}) + if err != nil { + t.Fatalf("test positional command failed: %v", err) + } +} + +func TestCLI_InspectCmd(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + wasmBytes := wasmhost.TestWASMBytes() + wasmPath := filepath.Join(tmpDir, "test.wasm") + if err := os.WriteFile(wasmPath, wasmBytes, 0o644); err != nil { + t.Fatalf("failed to write wasm file: %v", err) + } + + origStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe failed: %v", err) + } + os.Stdout = w + runErr := run(ctx, []string{"inspect", "--wasm=" + wasmPath}) + _ = w.Close() + os.Stdout = origStdout + outBytes, _ := io.ReadAll(r) + _ = r.Close() + if runErr != nil { + t.Fatalf("inspect command failed: %v", runErr) + } + + digest := sha256.Sum256(wasmBytes) + wantB64 := fmt.Sprintf("MapFn (base64): %s", base64.StdEncoding.EncodeToString(digest[:])) + wantHex := fmt.Sprintf("SHA-256 (hex): %x", digest) + outStr := string(outBytes) + if !strings.Contains(outStr, wantB64) { + t.Errorf("inspect output missing %q:\n%s", wantB64, outStr) + } + if !strings.Contains(outStr, wantHex) { + t.Errorf("inspect output missing %q:\n%s", wantHex, outStr) + } + + // Positional syntax + err = run(ctx, []string{"inspect", wasmPath}) + if err != nil { + t.Fatalf("inspect positional failed: %v", err) + } +} + +func TestCLI_BenchCmd(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + wasmPath := filepath.Join(tmpDir, "test.wasm") + if err := os.WriteFile(wasmPath, wasmhost.TestWASMBytes(), 0o644); err != nil { + t.Fatalf("failed to write wasm file: %v", err) + } + + err := run(ctx, []string{"bench", "--wasm=" + wasmPath, "--iterations=50", "--workers=2"}) + if err != nil { + t.Fatalf("bench command failed: %v", err) + } +} + +func TestCLI_ErrorsAndHelp(t *testing.T) { + ctx := context.Background() + + // Empty args + if err := run(ctx, []string{}); err == nil { + t.Fatal("expected error on empty args, got nil") + } + + // Help flag + if err := run(ctx, []string{"--help"}); err != nil { + t.Fatalf("expected nil for help, got %v", err) + } + + // Unknown subcommand + if err := run(ctx, []string{"unknown-cmd"}); err == nil { + t.Fatal("expected error on unknown subcommand, got nil") + } + + // Missing wasm flag + if err := run(ctx, []string{"test"}); err == nil { + t.Fatal("expected error on missing --wasm, got nil") + } +} + +func TestCLI_WithCompiledPlugins(t *testing.T) { + if testing.Short() { + t.Skip("skipping slow wasm compilation test in short mode") + } + ctx := context.Background() + tmpDir := t.TempDir() + + sumdbWasm := filepath.Join(tmpDir, "sumdb.wasm") + cmd := exec.Command("go", "build", "-buildmode=c-shared", "-o", sumdbWasm, "github.com/transparency-dev/incubator/vindex/mapfn/examples/sumdb") + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("failed to build sumdb wasm: %v\nOutput: %s", err, string(out)) + } + + ctWasm := filepath.Join(tmpDir, "ct.wasm") + cmd = exec.Command("go", "build", "-buildmode=c-shared", "-o", ctWasm, "github.com/transparency-dev/incubator/vindex/mapfn/examples/ct") + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("failed to build ct wasm: %v\nOutput: %s", err, string(out)) + } + + // 1. Test sumdb plugin via CLI + err := run(ctx, []string{"test", "--wasm=" + sumdbWasm, "--input=golang.org/x/mod v0.40.0 h1:abc="}) + if err != nil { + t.Fatalf("sumdb test failed: %v", err) + } + + // 2. Test CT plugin via CLI with a real DER X.509 certificate + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + tpl := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "foo.bar.example.com"}, + DNSNames: []string{"foo.bar.example.com", "sub.example.org"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + } + certDer, err := x509.CreateCertificate(rand.Reader, &tpl, &tpl, &priv.PublicKey, priv) + if err != nil { + t.Fatalf("CreateCertificate: %v", err) + } + certHex := hex.EncodeToString(certDer) + + origStdout := os.Stdout + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + os.Stdout = w + ctErr := run(ctx, []string{"test", "--wasm=" + ctWasm, "--hex=" + certHex, "--format=json"}) + _ = w.Close() + os.Stdout = origStdout + ctOut, _ := io.ReadAll(r) + _ = r.Close() + if ctErr != nil { + t.Fatalf("ct test failed: %v", ctErr) + } + if !strings.Contains(string(ctOut), "com.example.bar.foo.") || !strings.Contains(string(ctOut), "org.example.sub.") { + t.Fatalf("ct test output missing expected canonical keys:\n%s", string(ctOut)) + } + + // 3. Bench CT plugin + certFile := filepath.Join(tmpDir, "cert.der") + if err := os.WriteFile(certFile, certDer, 0o644); err != nil { + t.Fatalf("WriteFile(cert.der): %v", err) + } + err = run(ctx, []string{"bench", "--wasm=" + ctWasm, "--input_file=" + certFile, "--iterations=50", "--workers=2"}) + if err != nil { + t.Fatalf("ct bench failed: %v", err) + } +} diff --git a/vindex/cmd/vindex-wasm/test_cmd.go b/vindex/cmd/vindex-wasm/test_cmd.go new file mode 100644 index 00000000..8cd9479b --- /dev/null +++ b/vindex/cmd/vindex-wasm/test_cmd.go @@ -0,0 +1,152 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "slices" + "strings" + + "github.com/transparency-dev/incubator/vindex/internal/wasmhost" +) + +type testJSONEntry struct { + Index int `json:"index"` + Key string `json:"key,omitempty"` + KeyHash string `json:"key_hash"` +} + +type testJSONOutput struct { + Total int `json:"total"` + Entries []testJSONEntry `json:"entries"` +} + +func runTestCmd(ctx context.Context, args []string) error { + fs := flag.NewFlagSet("test", flag.ContinueOnError) + wasmPath := fs.String("wasm", "", "Path to compiled WASM plugin file (required).") + inputStr := fs.String("input", "", "Literal string input payload.") + inputFile := fs.String("input_file", "", "Path to file containing input leaf payload.") + inputHex := fs.String("hex", "", "Hex-encoded input leaf payload.") + bundleSize := fs.Int("bundle_size", 256, "Number of leaves per bundle execution (defaults to 256).") + format := fs.String("format", "plain", "Output format: plain or json.") + + if err := fs.Parse(args); err != nil { + return err + } + if *bundleSize <= 0 { + *bundleSize = 256 + } + + if *wasmPath == "" { + if fs.NArg() > 0 && (strings.HasSuffix(fs.Arg(0), ".wasm") || *inputStr != "") { + *wasmPath = fs.Arg(0) + } else { + return errors.New("--wasm flag is required") + } + } + + wasmBytes, err := os.ReadFile(*wasmPath) + if err != nil { + return fmt.Errorf("failed to read WASM file %q: %w", *wasmPath, err) + } + + var inputBytes []byte + if *inputHex != "" { + b, err := hex.DecodeString(*inputHex) + if err != nil { + return fmt.Errorf("invalid hex input %q: %w", *inputHex, err) + } + inputBytes = b + } else if *inputFile == "-" { + b, err := io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read input from stdin: %w", err) + } + inputBytes = b + } else if *inputFile != "" { + b, err := os.ReadFile(*inputFile) + if err != nil { + return fmt.Errorf("failed to read input file %q: %w", *inputFile, err) + } + inputBytes = b + } else if *inputStr != "" { + inputBytes = []byte(*inputStr) + } else if fs.NArg() > 0 { + // Positional argument as input string + arg := fs.Arg(0) + if arg == *wasmPath && fs.NArg() > 1 { + arg = fs.Arg(1) + } + if arg != *wasmPath { + inputBytes = []byte(arg) + } + } + + wasmHost, err := wasmhost.NewWASMHost(ctx, wasmBytes, 1) + if err != nil { + return fmt.Errorf("failed to initialize WASM host: %w", err) + } + defer func() { _ = wasmHost.Close(ctx) }() + + bundle := make([][]byte, *bundleSize) + for i := range bundle { + bundle[i] = inputBytes + } + + bundleResults, err := wasmHost.MapBundle(ctx, bundle) + if err != nil { + return fmt.Errorf("MapBundle execution failed: %w", err) + } + if len(bundleResults) != *bundleSize { + return fmt.Errorf("MapBundle returned %d leaf results, want %d", len(bundleResults), *bundleSize) + } + entries := bundleResults[0] + for i := 1; i < len(bundleResults); i++ { + if !slices.Equal(bundleResults[i], entries) { + return fmt.Errorf("non-deterministic bundle output at leaf %d: got %v, want %v", i, bundleResults[i], entries) + } + } + + if *format == "json" { + out := testJSONOutput{ + Total: len(entries), + Entries: make([]testJSONEntry, len(entries)), + } + for i, e := range entries { + out.Entries[i] = testJSONEntry{ + Index: i, + Key: e.Key, + KeyHash: hex.EncodeToString(e.KeyHash[:]), + } + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(out) + } + + fmt.Printf("Mapped %d entries from input (%d bytes):\n", len(entries), len(inputBytes)) + for i, e := range entries { + fmt.Printf(" [%d] Key: %s | KeyHash: %x\n", i, e.Key, e.KeyHash) + } + + return nil +} diff --git a/vindex/internal/wasmhost/testdata/testguest/main.go b/vindex/internal/wasmhost/testdata/testguest/main.go new file mode 100644 index 00000000..8e7e1f22 --- /dev/null +++ b/vindex/internal/wasmhost/testdata/testguest/main.go @@ -0,0 +1,56 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package main implements the default test WASM guest using the VIndex MapFn SDK. +// For each newline-delimited line in a leaf, it emits the first whitespace-delimited +// token as a search key. To support fault-injection tests: +// - If a leaf contains 0x00 (NUL), it panics (triggering a WASM trap). +// - If a leaf contains 0x02 (STX), it spins indefinitely until cancelled by context timeout. +// - If a leaf contains 0x03 (ETX), it overflows the 4 MB output slab so map_bundle returns (0, 0). +package main + +import ( + "bytes" + + "github.com/transparency-dev/incubator/vindex/mapfn/sdk" +) + +func main() {} + +func init() { + sdk.RegisterEmit(MapLeaf) +} + +func MapLeaf(leaf []byte, emit func(key []byte)) { + if bytes.IndexByte(leaf, 0x00) >= 0 { + panic("injected trap: NUL byte in leaf") + } + if bytes.IndexByte(leaf, 0x02) >= 0 { + for { + } + } + if bytes.IndexByte(leaf, 0x03) >= 0 { + bigKey := bytes.Repeat([]byte{'a'}, 1000) + for range 4500 { + emit(bigKey) + } + return + } + for line := range bytes.SplitSeq(leaf, []byte{'\n'}) { + fields := bytes.Fields(line) + if len(fields) > 0 { + emit(fields[0]) + } + } +} diff --git a/vindex/internal/wasmhost/wasm.go b/vindex/internal/wasmhost/wasm.go new file mode 100644 index 00000000..193d1a7e --- /dev/null +++ b/vindex/internal/wasmhost/wasm.go @@ -0,0 +1,550 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package wasmhost provides a pure Wazero WebAssembly host runtime and ABI decoder +// for VIndex MapFn guest modules without HTTP or log-fetching dependencies. +package wasmhost + +import ( + "bytes" + "cmp" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "io" + "runtime" + "slices" + "sync" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" + "github.com/tetratelabs/wazero/sys" +) + +const ( + // DefaultBundleSize is the default number of leaves per bundle execution. + DefaultBundleSize uint64 = 256 +) + +var ( + ErrWasmHalt = errors.New("wasm execution halted due to violation or trap") + ErrInvalidMemory = errors.New("wasm returned memory pointer out of bounds") + ErrMalformedOutput = errors.New("wasm output format is malformed") + ErrHostClosed = errors.New("wasm host is closed") +) + +// MappedEntry represents a single canonical search key and its SHA-256 digest produced by a WASM MapFn. +type MappedEntry struct { + Key string + KeyHash [sha256.Size]byte +} + +// ValidatePreimageKey validates that a WASM-emitted preimage key satisfies the canonical +// key invariants: 1 <= len <= 1024 and printable ASCII (0x21..0x7E, no NUL or whitespace). +func ValidatePreimageKey(keyBytes []byte) error { + if len(keyBytes) == 0 || len(keyBytes) > 1024 { + return fmt.Errorf("%w: canonical key length %d out of range [1, 1024]", ErrMalformedOutput, len(keyBytes)) + } + for i, b := range keyBytes { + if b < 0x21 || b > 0x7E { + return fmt.Errorf("%w: canonical key contains invalid byte 0x%02x at offset %d", ErrMalformedOutput, b, i) + } + } + return nil +} + +// WASMHost manages the lifecycle and execution of a compiled Wazero WebAssembly module. +type WASMHost struct { + runtime wazero.Runtime + compiled wazero.CompiledModule + pool chan *wasmInstance + mu sync.Mutex + closed bool +} + +type zeroReader struct{} + +func (zeroReader) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} + +type wasmInstance struct { + // Required for correctness: module instance and exported guest bindings. + mod api.Module + mapBundleFn api.Function + mem api.Memory + staticInPtr uint32 + + // Performance optimizations (NOT functional requirements): + // Reusable per-instance buffers to eliminate heap allocation churn during bundle processing. + resultsBuf [][]MappedEntry // Reusable outer slice buffer for decoded leaf entries. + entriesBuf []MappedEntry // Reusable contiguous backing array for all mapped entries in a bundle. +} + +var ( + sharedRuntimeOnce sync.Once + sharedRuntime wazero.Runtime + compileMu sync.Mutex + compiledModules = make(map[[sha256.Size]byte]wazero.CompiledModule) +) + +func getSharedRuntime() wazero.Runtime { + sharedRuntimeOnce.Do(func() { + bgCtx := context.Background() + sharedRuntime = wazero.NewRuntimeWithConfig(bgCtx, wazero.NewRuntimeConfig(). + WithDebugInfoEnabled(false). + WithMemoryLimitPages(1024). + WithCloseOnContextDone(true)) + wasi_snapshot_preview1.MustInstantiate(bgCtx, sharedRuntime) + }) + return sharedRuntime +} + +// CompileModule compiles wasmBytes on the shared Wazero runtime, returning a cached CompiledModule when available. +func CompileModule(ctx context.Context, wasmBytes []byte) (wazero.CompiledModule, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + r := getSharedRuntime() + hash := sha256.Sum256(wasmBytes) + + compileMu.Lock() + defer compileMu.Unlock() + if compiled, ok := compiledModules[hash]; ok { + return compiled, nil + } + compiled, err := r.CompileModule(ctx, wasmBytes) + if err != nil { + return nil, fmt.Errorf("failed to compile wasm module: %w", err) + } + compiledModules[hash] = compiled + return compiled, nil +} + +// NewWASMHost compiles the guest bytecode and initializes a pool of instantiated WASM modules. +// If poolSize <= 0, it defaults to runtime.GOMAXPROCS(0) - 1 (minimum 1). +func NewWASMHost(ctx context.Context, wasmBytes []byte, poolSize int) (*WASMHost, error) { + if poolSize <= 0 { + poolSize = runtime.GOMAXPROCS(0) - 1 + if poolSize < 1 { + poolSize = 1 + } + } + + compiled, err := CompileModule(ctx, wasmBytes) + if err != nil { + return nil, err + } + + h := &WASMHost{ + runtime: getSharedRuntime(), + compiled: compiled, + pool: make(chan *wasmInstance, poolSize), + } + + inst, err := h.instantiateInstance(ctx) + if err != nil { + _ = h.Close(ctx) + return nil, fmt.Errorf("failed to instantiate initial wasm module 0: %w", err) + } + h.pool <- inst + + return h, nil +} + +func (h *WASMHost) instantiateInstance(ctx context.Context) (*wasmInstance, error) { + var id [8]byte + _, _ = rand.Read(id[:]) + modName := fmt.Sprintf("vindex_mod_%x", id) + + wasiConfig := wazero.NewModuleConfig(). + WithName(modName). + WithStartFunctions("_initialize"). + WithStdout(io.Discard). + WithStderr(io.Discard). + WithStdin(bytes.NewReader(nil)). + WithArgs(). + WithWalltime(func() (sec int64, nsec int32) { return 1, 0 }, sys.ClockResolution(1)). + WithNanotime(func() int64 { return 1_000_000 }, sys.ClockResolution(1)). + WithNanosleep(func(int64) {}). + WithRandSource(zeroReader{}) + + mod, err := h.runtime.InstantiateModule(ctx, h.compiled, wasiConfig) + if err != nil { + return nil, err + } + + mapBundleFn := mod.ExportedFunction("map_bundle") + if mapBundleFn == nil { + _ = mod.Close(ctx) + return nil, fmt.Errorf("%w: 'map_bundle' not exported", ErrWasmHalt) + } + + mem := mod.Memory() + if mem == nil { + _ = mod.Close(ctx) + return nil, fmt.Errorf("%w: module memory not found", ErrWasmHalt) + } + + inBufFn := mod.ExportedFunction("input_buffer") + if inBufFn == nil { + _ = mod.Close(ctx) + return nil, fmt.Errorf("%w: 'input_buffer' not exported", ErrWasmHalt) + } + res, err := inBufFn.Call(ctx) + if err != nil { + _ = mod.Close(ctx) + return nil, fmt.Errorf("%w: input_buffer call failed: %v", ErrWasmHalt, err) + } + if len(res) == 0 { + _ = mod.Close(ctx) + return nil, fmt.Errorf("%w: input_buffer returned no result", ErrWasmHalt) + } + staticInPtr := uint32(res[0]) + if staticInPtr == 0 { + _ = mod.Close(ctx) + return nil, fmt.Errorf("%w: input_buffer returned null pointer (0)", ErrInvalidMemory) + } + if _, ok := mem.Read(staticInPtr, 4*1024*1024); !ok { + _ = mod.Close(ctx) + return nil, fmt.Errorf("%w: input_buffer (ptr=%d, len=4MB) exceeds guest memory %d", ErrInvalidMemory, staticInPtr, mem.Size()) + } + + return &wasmInstance{ + mod: mod, + mapBundleFn: mapBundleFn, + mem: mem, + staticInPtr: staticInPtr, + }, nil +} + +func (inst *wasmInstance) executeBundle(ctx context.Context, leaves [][]byte) ([][]MappedEntry, error) { + n := len(leaves) + if n == 0 { + return nil, nil + } + if n > 256 { + return nil, fmt.Errorf("%w: bundle size %d exceeds 256", ErrMalformedOutput, n) + } + + headerLen := 4 + (n+1)*4 + var payloadLen int + for _, l := range leaves { + payloadLen += len(l) + } + totalLen := headerLen + payloadLen + if totalLen > 4*1024*1024 { + return nil, fmt.Errorf("%w: bundle input size %d exceeds 4 MB static slab capacity", ErrWasmHalt, totalLen) + } + inPtr := inst.staticInPtr + inputLen := uint32(totalLen) + destSlice, ok := inst.mem.Read(inPtr, inputLen) + if !ok { + return nil, fmt.Errorf("%w: failed to read guest buffer at ptr %d len %d", ErrInvalidMemory, inPtr, totalLen) + } + binary.LittleEndian.PutUint32(destSlice[0:4], uint32(n)) + var currentOffset uint32 + binary.LittleEndian.PutUint32(destSlice[4:8], 0) + payloadDst := destSlice[headerLen:] + var writePos int + for i, l := range leaves { + copy(payloadDst[writePos:], l) + writePos += len(l) + currentOffset += uint32(len(l)) + binary.LittleEndian.PutUint32(destSlice[4+(i+1)*4:4+(i+2)*4], currentOffset) + } + + callRes, err := inst.mapBundleFn.Call(ctx, uint64(inPtr), uint64(inputLen)) + if err != nil { + return nil, fmt.Errorf("%w: map_bundle execution failed: %v", ErrWasmHalt, err) + } + if len(callRes) == 0 { + return nil, fmt.Errorf("%w: map_bundle returned no result", ErrMalformedOutput) + } + + packed := callRes[0] + outPtr := uint32(packed >> 32) + outLen := uint32(packed & 0xFFFFFFFF) + if outPtr == 0 || outLen == 0 { + return nil, fmt.Errorf("%w: map_bundle returned (ptr=%d, len=%d): output slab overflow or invalid input", ErrWasmHalt, outPtr, outLen) + } + + outBuf, ok := inst.mem.Read(outPtr, outLen) + if !ok { + return nil, fmt.Errorf("%w: out_ptr=%d out_len=%d exceeds guest memory %d", + ErrInvalidMemory, outPtr, outLen, inst.mem.Size()) + } + + return inst.decodeBundleOutput(outBuf, n) +} + +func decodeBundleOutputInto(resultsBuf [][]MappedEntry, entriesBuf []MappedEntry, outBuf []byte, expectedCount int) ([][]MappedEntry, []MappedEntry, error) { + if len(outBuf) < 4+expectedCount*4 { + return nil, nil, fmt.Errorf("%w: bundle output length %d is too short for %d leaves", ErrMalformedOutput, len(outBuf), expectedCount) + } + + leafCount := binary.LittleEndian.Uint32(outBuf[0:4]) + if int(leafCount) != expectedCount { + return nil, nil, fmt.Errorf("%w: bundle output leaf_count %d != expected %d", ErrMalformedOutput, leafCount, expectedCount) + } + + keyCountsOffset := 4 + preimagesOffset := 4 + expectedCount*4 + + var totalKeysU64 uint64 + for i := 0; i < expectedCount; i++ { + totalKeysU64 += uint64(binary.LittleEndian.Uint32(outBuf[keyCountsOffset+i*4 : keyCountsOffset+(i+1)*4])) + } + + maxPossibleKeys := uint64(len(outBuf)-preimagesOffset) / 4 + if totalKeysU64 > maxPossibleKeys { + return nil, nil, fmt.Errorf("%w: total keys %d exceeds maximum possible %d for buffer length %d", ErrMalformedOutput, totalKeysU64, maxPossibleKeys, len(outBuf)) + } + totalKeys := int(totalKeysU64) + + if cap(resultsBuf) < expectedCount { + resultsBuf = make([][]MappedEntry, expectedCount) + } else { + resultsBuf = resultsBuf[:expectedCount] + clear(resultsBuf) + } + if totalKeys == 0 { + if preimagesOffset != len(outBuf) { + return nil, nil, fmt.Errorf("%w: trailing unconsumed bytes in bundle output (%d != %d)", ErrMalformedOutput, preimagesOffset, len(outBuf)) + } + return resultsBuf, entriesBuf[:0], nil + } + + if cap(entriesBuf) < totalKeys { + entriesBuf = make([]MappedEntry, totalKeys) + } else { + entriesBuf = entriesBuf[:totalKeys] + } + allEntries := entriesBuf + var curEntryIdx int + + for i := 0; i < expectedCount; i++ { + kCount := int(binary.LittleEndian.Uint32(outBuf[keyCountsOffset+i*4 : keyCountsOffset+(i+1)*4])) + if kCount == 0 { + continue + } + + startEntryIdx := curEntryIdx + for j := 0; j < kCount; j++ { + if preimagesOffset+4 > len(outBuf) { + return nil, nil, fmt.Errorf("%w: truncated key length header", ErrMalformedOutput) + } + keyLen := binary.LittleEndian.Uint32(outBuf[preimagesOffset : preimagesOffset+4]) + preimagesOffset += 4 + + if uint64(preimagesOffset)+uint64(keyLen) > uint64(len(outBuf)) { + return nil, nil, fmt.Errorf("%w: truncated key bytes", ErrMalformedOutput) + } + keyBytes := outBuf[preimagesOffset : preimagesOffset+int(keyLen)] + preimagesOffset += int(keyLen) + if err := ValidatePreimageKey(keyBytes); err != nil { + return nil, nil, err + } + + allEntries[curEntryIdx] = MappedEntry{ + Key: string(keyBytes), + KeyHash: sha256.Sum256(keyBytes), + } + curEntryIdx++ + } + + entries := allEntries[startEntryIdx:curEntryIdx] + if len(entries) > 1 { + slices.SortFunc(entries, func(a, b MappedEntry) int { + return cmp.Compare(a.Key, b.Key) + }) + entries = slices.CompactFunc(entries, func(a, b MappedEntry) bool { + return a.Key == b.Key + }) + } + curEntryIdx = startEntryIdx + len(entries) + resultsBuf[i] = entries[:len(entries):len(entries)] + } + + if preimagesOffset != len(outBuf) { + return nil, nil, fmt.Errorf("%w: trailing unconsumed bytes in bundle output (%d != %d)", ErrMalformedOutput, preimagesOffset, len(outBuf)) + } + + return resultsBuf, allEntries[:curEntryIdx], nil +} + +func (inst *wasmInstance) decodeBundleOutput(outBuf []byte, expectedCount int) ([][]MappedEntry, error) { + results, entries, err := decodeBundleOutputInto(inst.resultsBuf, inst.entriesBuf, outBuf, expectedCount) + if err != nil { + return nil, err + } + inst.resultsBuf = results + inst.entriesBuf = entries + return results, nil +} + +func decodeBundleOutput(outBuf []byte, expectedCount int) ([][]MappedEntry, error) { + results, _, err := decodeBundleOutputInto(nil, nil, outBuf, expectedCount) + return results, err +} + +// MapBundle executes WASM mapping across a contiguous bundle of leaves using a checked-out runner. +func (h *WASMHost) MapBundle(ctx context.Context, leaves [][]byte) ([][]MappedEntry, error) { + runner, err := h.NewRunner(ctx) + if err != nil { + return nil, err + } + defer func() { _ = runner.Close(ctx) }() + + results, err := runner.MapBundle(ctx, leaves) + if err != nil { + return nil, err + } + return cloneBundleResults(results), nil +} + +func cloneBundleResults(results [][]MappedEntry) [][]MappedEntry { + if results == nil { + return nil + } + cloned := make([][]MappedEntry, len(results)) + for i, r := range results { + if len(r) > 0 { + cloned[i] = append([]MappedEntry(nil), r...) + } + } + return cloned +} + +// Close closes all pooled module instances. +func (h *WASMHost) Close(ctx context.Context) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.closed { + return nil + } + h.closed = true + + close(h.pool) + for inst := range h.pool { + if inst != nil && inst.mod != nil { + _ = inst.mod.Close(ctx) + } + } + return nil +} + +// WASMRunner is a dedicated, unshared WASM execution environment bound to a single worker. +// It executes mapping calls lock-free without instance pool or mutex contention. +type WASMRunner struct { + host *WASMHost + inst *wasmInstance + closed bool + hadErr bool +} + +// NewRunner creates a dedicated WASMRunner from the host. +// The runner has exclusive access to its WASM module instance, eliminating pool and mutex overhead. +func (h *WASMHost) NewRunner(ctx context.Context) (*WASMRunner, error) { + h.mu.Lock() + if h.closed { + h.mu.Unlock() + return nil, ErrHostClosed + } + h.mu.Unlock() + + var inst *wasmInstance + select { + case inst = <-h.pool: + default: + } + if inst == nil { + var err error + inst, err = h.instantiateInstance(ctx) + if err != nil { + return nil, fmt.Errorf("failed to instantiate wasm runner: %w", err) + } + } + + return &WASMRunner{ + host: h, + inst: inst, + }, nil +} + +// MapBundle executes WASM mapping across a contiguous bundle of leaves on this dedicated runner. +// The returned slice and sub-slices reference internal runner buffers and remain valid until +// the next invocation of MapBundle on this runner. +func (r *WASMRunner) MapBundle(ctx context.Context, leaves [][]byte) ([][]MappedEntry, error) { + if r.closed { + return nil, ErrHostClosed + } + if r.hadErr { + return nil, fmt.Errorf("%w: wasm runner already failed", ErrWasmHalt) + } + + results, err := r.inst.executeBundle(ctx, leaves) + if err != nil { + r.hadErr = true + return nil, err + } + + return results, nil +} + +// Close releases this runner's WASM module instance, returning it to the host pool if healthy. +func (r *WASMRunner) Close(ctx context.Context) error { + if r.closed { + return nil + } + r.closed = true + + if r.inst == nil { + return nil + } + + if r.hadErr { + if r.inst.mod != nil { + err := r.inst.mod.Close(ctx) + r.inst.mod = nil + return err + } + return nil + } + + r.host.mu.Lock() + defer r.host.mu.Unlock() + + if r.host.closed { + if r.inst.mod != nil { + return r.inst.mod.Close(ctx) + } + return nil + } + + select { + case r.host.pool <- r.inst: + return nil + default: + if r.inst.mod != nil { + return r.inst.mod.Close(ctx) + } + return nil + } +} diff --git a/vindex/internal/wasmhost/wasm_test.go b/vindex/internal/wasmhost/wasm_test.go new file mode 100644 index 00000000..315320f9 --- /dev/null +++ b/vindex/internal/wasmhost/wasm_test.go @@ -0,0 +1,752 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasmhost + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/binary" + "errors" + "fmt" + "slices" + "sync" + "testing" + "time" + + "github.com/google/go-cmp/cmp" +) + +func TestWASMHost_PreimageKeysAndDeduplication(t *testing.T) { + ctx := context.Background() + + hBeta := sha256.Sum256([]byte("key_beta")) + hAlpha := sha256.Sum256([]byte("key_alpha")) + + host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + results, err := host.MapBundle(ctx, [][]byte{[]byte("key_beta\nkey_alpha\nkey_alpha")}) + if err != nil { + t.Fatalf("MapBundle failed: %v", err) + } + + expected := [][]MappedEntry{{ + {Key: "key_alpha", KeyHash: hAlpha}, + {Key: "key_beta", KeyHash: hBeta}, + }} + + if diff := cmp.Diff(expected, results); diff != "" { + t.Fatalf("MapBundle diff (-want +got):\n%s", diff) + } +} + +func TestWASMHost_EmptyOutput(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + results, err := host.MapBundle(ctx, [][]byte{[]byte(" \n\t ")}) + if err != nil { + t.Fatalf("MapBundle failed: %v", err) + } + if len(results) != 1 || len(results[0]) != 0 { + t.Fatalf("expected 1 empty leaf result, got %v", results) + } +} + +func TestWASMHost_UnreachableTrap(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + _, err = host.MapBundle(ctx, [][]byte{[]byte("\x00trap_leaf")}) + if err == nil { + t.Fatal("expected error on unreachable trap, got nil") + } + if !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt, got: %v", err) + } +} + +func TestWASMHost_MalformedOutput(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + _, err = host.MapBundle(ctx, [][]byte{[]byte("\x01malformed")}) + if err == nil { + t.Fatal("expected malformed output error, got nil") + } + if !errors.Is(err, ErrMalformedOutput) { + t.Fatalf("expected ErrMalformedOutput, got: %v", err) + } +} + +func TestWASMHost_ConcurrentExecution(t *testing.T) { + ctx := context.Background() + + h1 := sha256.Sum256([]byte("concurrent_key")) + host, err := NewWASMHost(ctx, TestWASMBytes(), 4) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 50; j++ { + res, err := host.MapBundle(ctx, [][]byte{[]byte("concurrent_key")}) + if err != nil { + t.Errorf("concurrent MapBundle failed: %v", err) + return + } + if len(res) != 1 || len(res[0]) != 1 || res[0][0].Key != "concurrent_key" || res[0][0].KeyHash != h1 { + t.Errorf("unexpected MapBundle result: %v", res) + return + } + } + }() + } + wg.Wait() +} + +func TestWASMHost_ClosedHost(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + + if err := host.Close(ctx); err != nil { + t.Fatalf("Close failed: %v", err) + } + + _, err = host.MapBundle(ctx, [][]byte{[]byte("leaf")}) + if !errors.Is(err, ErrHostClosed) { + t.Fatalf("expected ErrHostClosed after Close(), got: %v", err) + } +} + +func TestWASMHost_MissingInputBufferExport(t *testing.T) { + ctx := context.Background() + + // Rename the 12-byte export "input_buffer" to "other_buffer" so the binary + // remains valid WASM but lacks the required "input_buffer" export. + wasmBytes := bytes.ReplaceAll(slices.Clone(TestWASMBytes()), []byte("input_buffer"), []byte("other_buffer")) + + _, err := NewWASMHost(ctx, wasmBytes, 1) + if err == nil { + t.Fatal("expected error on missing input_buffer export, got nil") + } + if !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt, got: %v", err) + } +} + +func TestWASMHost_MissingMapBundleExport(t *testing.T) { + ctx := context.Background() + + // Rename the 10-byte export "map_bundle" to "other_func" so the binary + // remains valid WASM but lacks the required "map_bundle" export. + wasmBytes := bytes.ReplaceAll(slices.Clone(TestWASMBytes()), []byte("map_bundle"), []byte("other_func")) + + _, err := NewWASMHost(ctx, wasmBytes, 1) + if err == nil { + t.Fatal("expected error on missing map_bundle export, got nil") + } + if !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt, got: %v", err) + } +} + +func TestWASMHost_Timeout(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + leafCtx, leafCancel := context.WithTimeout(ctx, 100*time.Millisecond) + defer leafCancel() + + start := time.Now() + _, err = host.MapBundle(leafCtx, [][]byte{[]byte("\x02infinite_loop")}) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected timeout error for infinite loop, got nil") + } + if !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt, got: %v", err) + } + if elapsed < 80*time.Millisecond || elapsed > 2000*time.Millisecond { + t.Logf("execution elapsed: %v (expected ~100ms timeout)", elapsed) + } +} + +func TestWASMHost_MapBundle(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + leaves := [][]byte{ + []byte("leaf0"), + []byte("leaf1"), + []byte("leaf2"), + []byte("leaf3"), + } + + results, err := host.MapBundle(ctx, leaves) + if err != nil { + t.Fatalf("MapBundle failed: %v", err) + } + if len(results) != len(leaves) { + t.Fatalf("expected %d bundle results, got %d", len(leaves), len(results)) + } + for i, r := range results { + if len(r) != 1 { + t.Fatalf("leaf %d expected 1 entry, got %d", i, len(r)) + } + } +} + +func TestWASMHost_MapBundle_PreimagesAndHardwareHashing(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + k0 := []byte("golang.org/x/mod") + k1 := []byte("example.com") + k2 := []byte("my-package") + + leaves := [][]byte{ + []byte("golang.org/x/mod\nexample.com"), + []byte(" "), + []byte("my-package"), + } + + results, err := host.MapBundle(ctx, leaves) + if err != nil { + t.Fatalf("MapBundle failed: %v", err) + } + + if len(results) != 3 { + t.Fatalf("expected 3 results, got %d", len(results)) + } + + h0 := sha256.Sum256(k0) + h1 := sha256.Sum256(k1) + if len(results[0]) != 2 { + t.Fatalf("leaf 0 expected 2 entries, got %d", len(results[0])) + } + // Lexicographical sort by canonical Key: "example.com" (k1) < "golang.org/x/mod" (k0). + if results[0][0].KeyHash != h1 || results[0][1].KeyHash != h0 { + t.Fatalf("leaf 0 key hashes mismatch: got %+v", results[0]) + } + + if len(results[1]) != 0 { + t.Fatalf("leaf 1 expected 0 entries, got %d", len(results[1])) + } + + h2 := sha256.Sum256(k2) + if len(results[2]) != 1 { + t.Fatalf("leaf 2 expected 1 entry, got %d", len(results[2])) + } + if results[2][0].KeyHash != h2 { + t.Fatalf("leaf 2 hash mismatch: got %x, want %x", results[2][0].KeyHash, h2) + } +} + +func BenchmarkWASMHost_MapBundle(b *testing.B) { + ctx := context.Background() + host, err := NewWASMHost(ctx, TestWASMBytes(), 4) + if err != nil { + b.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + leaves := make([][]byte, 256) + for i := range leaves { + leaves[i] = []byte("benchmark_leaf_payload") + } + + b.ReportAllocs() + + for b.Loop() { + _, err := host.MapBundle(ctx, leaves) + if err != nil { + b.Fatalf("MapBundle failed: %v", err) + } + } +} + +func BenchmarkWASMRunner_MapBundle(b *testing.B) { + ctx := context.Background() + host, err := NewWASMHost(ctx, TestWASMBytes(), 4) + if err != nil { + b.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + runner, err := host.NewRunner(ctx) + if err != nil { + b.Fatalf("NewRunner failed: %v", err) + } + defer func() { _ = runner.Close(ctx) }() + + leaves := make([][]byte, 256) + for i := range leaves { + leaves[i] = []byte("benchmark_leaf_payload") + } + + b.ReportAllocs() + + for b.Loop() { + _, err := runner.MapBundle(ctx, leaves) + if err != nil { + b.Fatalf("runner.MapBundle failed: %v", err) + } + } +} + +func BenchmarkDecodeBundleOutput(b *testing.B) { + const leavesCount = 256 + const keysPerLeaf = 2 + var outBuf bytes.Buffer + var countBuf [4]byte + binary.LittleEndian.PutUint32(countBuf[:], leavesCount) + outBuf.Write(countBuf[:]) + + for i := 0; i < leavesCount; i++ { + var kc [4]byte + binary.LittleEndian.PutUint32(kc[:], keysPerLeaf) + outBuf.Write(kc[:]) + } + + domain1 := []byte("example.com") + domain2 := []byte("sub.example.org") + for i := 0; i < leavesCount; i++ { + var k1Len, k2Len [4]byte + binary.LittleEndian.PutUint32(k1Len[:], uint32(len(domain1))) + outBuf.Write(k1Len[:]) + outBuf.Write(domain1) + + binary.LittleEndian.PutUint32(k2Len[:], uint32(len(domain2))) + outBuf.Write(k2Len[:]) + outBuf.Write(domain2) + } + + rawOutput := outBuf.Bytes() + b.ReportAllocs() + + for b.Loop() { + res, err := decodeBundleOutput(rawOutput, leavesCount) + if err != nil || len(res) != leavesCount { + b.Fatalf("decodeBundleOutput failed: %v", err) + } + } +} + +func BenchmarkDecodeBundleOutputInto(b *testing.B) { + const leavesCount = 256 + const keysPerLeaf = 2 + var outBuf bytes.Buffer + var countBuf [4]byte + binary.LittleEndian.PutUint32(countBuf[:], leavesCount) + outBuf.Write(countBuf[:]) + + for i := 0; i < leavesCount; i++ { + var kc [4]byte + binary.LittleEndian.PutUint32(kc[:], keysPerLeaf) + outBuf.Write(kc[:]) + } + + domain1 := []byte("example.com") + domain2 := []byte("sub.example.org") + for i := 0; i < leavesCount; i++ { + var k1Len, k2Len [4]byte + binary.LittleEndian.PutUint32(k1Len[:], uint32(len(domain1))) + outBuf.Write(k1Len[:]) + outBuf.Write(domain1) + + binary.LittleEndian.PutUint32(k2Len[:], uint32(len(domain2))) + outBuf.Write(k2Len[:]) + outBuf.Write(domain2) + } + + rawOutput := outBuf.Bytes() + var resultsBuf [][]MappedEntry + var entriesBuf []MappedEntry + + b.ReportAllocs() + + for b.Loop() { + var err error + resultsBuf, entriesBuf, err = decodeBundleOutputInto(resultsBuf, entriesBuf, rawOutput, leavesCount) + if err != nil || len(resultsBuf) != leavesCount { + b.Fatalf("decodeBundleOutputInto failed: %v", err) + } + } +} + +func TestWASMRunner_DedicatedExecution(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + runner, err := host.NewRunner(ctx) + if err != nil { + t.Fatalf("NewRunner failed: %v", err) + } + defer func() { _ = runner.Close(ctx) }() + + leaves := [][]byte{[]byte("bundle_leaf_0")} + bundleRes, err := runner.MapBundle(ctx, leaves) + if err != nil { + t.Fatalf("runner.MapBundle failed: %v", err) + } + if len(bundleRes) != 1 || len(bundleRes[0]) == 0 { + t.Fatalf("expected 1 non-empty bundle result, got %v", bundleRes) + } +} + +func TestWASMRunner_ConcurrentDedicatedRunners(t *testing.T) { + ctx := context.Background() + + concurrency := 8 + host, err := NewWASMHost(ctx, TestWASMBytes(), concurrency) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + var wg sync.WaitGroup + errCh := make(chan error, concurrency) + + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + runner, err := host.NewRunner(ctx) + if err != nil { + errCh <- fmt.Errorf("worker %d: NewRunner failed: %w", workerID, err) + return + } + defer func() { _ = runner.Close(ctx) }() + + for j := 0; j < 50; j++ { + leaves := [][]byte{ + []byte(fmt.Sprintf("worker_%d_bundle_%d_leaf_0", workerID, j)), + []byte(fmt.Sprintf("worker_%d_bundle_%d_leaf_1", workerID, j)), + } + res, err := runner.MapBundle(ctx, leaves) + if err != nil { + errCh <- fmt.Errorf("worker %d: MapBundle failed on iter %d: %w", workerID, j, err) + return + } + if len(res) != 2 { + errCh <- fmt.Errorf("worker %d: expected 2 results, got %d", workerID, len(res)) + return + } + } + }(i) + } + + wg.Wait() + close(errCh) + + for err := range errCh { + t.Fatal(err) + } +} + +func TestDecodeBundleOutput_BoundsCheck(t *testing.T) { + outBuf := make([]byte, 20) + binary.LittleEndian.PutUint32(outBuf[0:4], 1) + binary.LittleEndian.PutUint32(outBuf[4:8], 1_000_000) + + _, err := decodeBundleOutput(outBuf, 1) + if err == nil { + t.Fatal("expected error for totalKeys exceeding buffer capacity, got nil") + } + if !errors.Is(err, ErrMalformedOutput) { + t.Fatalf("expected ErrMalformedOutput, got: %v", err) + } +} + +func TestDecodeBundleOutputInto_CapacityProtection(t *testing.T) { + var buf bytes.Buffer + _ = binary.Write(&buf, binary.LittleEndian, uint32(2)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(2)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(5)) + buf.WriteString("key_a") + _ = binary.Write(&buf, binary.LittleEndian, uint32(5)) + buf.WriteString("key_a") + _ = binary.Write(&buf, binary.LittleEndian, uint32(5)) + buf.WriteString("key_b") + + results, entries, err := decodeBundleOutputInto(nil, nil, buf.Bytes(), 2) + if err != nil { + t.Fatalf("decodeBundleOutputInto failed: %v", err) + } + if len(results) != 2 { + t.Fatalf("expected 2 leaf results, got %d", len(results)) + } + if len(results[0]) != 1 { + t.Fatalf("expected 1 compacted key in leaf 0, got %d", len(results[0])) + } + if cap(results[0]) != len(results[0]) { + t.Errorf("expected cap(results[0]) == len(results[0]) (%d), got cap %d", len(results[0]), cap(results[0])) + } + if len(results[1]) != 1 { + t.Fatalf("expected 1 key in leaf 1, got %d", len(results[1])) + } + if cap(results[1]) != len(results[1]) { + t.Errorf("expected cap(results[1]) == len(results[1]) (%d), got cap %d", len(results[1]), cap(results[1])) + } + if len(entries) != 2 { + t.Fatalf("expected 2 total compacted entries, got %d", len(entries)) + } + expectedHashA := sha256.Sum256([]byte("key_a")) + expectedHashB := sha256.Sum256([]byte("key_b")) + if results[0][0].KeyHash != expectedHashA { + t.Errorf("leaf 0 hash mismatch: got %x, want %x", results[0][0].KeyHash, expectedHashA) + } + if results[1][0].KeyHash != expectedHashB { + t.Errorf("leaf 1 hash mismatch: got %x, want %x", results[1][0].KeyHash, expectedHashB) + } +} + +func TestDecodeBundleOutputInto_KeyLenOverflow(t *testing.T) { + var buf bytes.Buffer + _ = binary.Write(&buf, binary.LittleEndian, uint32(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(0xFFFFFFFF)) + buf.WriteString("short_data") + + _, _, err := decodeBundleOutputInto(nil, nil, buf.Bytes(), 1) + if err == nil { + t.Fatal("expected ErrMalformedOutput for overflowing keyLen, got nil") + } + if !errors.Is(err, ErrMalformedOutput) { + t.Fatalf("expected ErrMalformedOutput, got: %v", err) + } +} + +func TestWASMHost_MapBundle_ZeroLenSlabOverflowFailsFast(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + // 0x03 causes testguest to overflow the 4 MB output slab so map_bundle returns (0, 0). + _, err = host.MapBundle(ctx, [][]byte{[]byte("\x03overflow")}) + if err == nil { + t.Fatal("expected ErrWasmHalt when map_bundle returns (0, 0), got nil") + } + if !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt, got: %v", err) + } +} + +func TestWASMRunner_TrapDiscardsBrokenInstanceAndRecovers(t *testing.T) { + ctx := context.Background() + + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + runner, err := host.NewRunner(ctx) + if err != nil { + t.Fatalf("NewRunner failed: %v", err) + } + if _, err := runner.MapBundle(ctx, [][]byte{[]byte("\x00trap")}); !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt on trap, got: %v", err) + } + if err := runner.Close(ctx); err != nil { + t.Fatalf("runner.Close failed: %v", err) + } + if got := len(host.pool); got != 0 { + t.Fatalf("expected broken runner not to be returned to pool, got pool len %d", got) + } + + // Next MapBundle lazily instantiates a fresh healthy instance. + res, err := host.MapBundle(ctx, [][]byte{[]byte("healthy_key")}) + if err != nil { + t.Fatalf("MapBundle after runner trap failed: %v", err) + } + if len(res) != 1 || len(res[0]) != 1 || res[0][0].Key != "healthy_key" { + t.Fatalf("unexpected result after recovery: %+v", res) + } +} + +func TestDecodeBundleOutputInto_NonCanonicalKeys(t *testing.T) { + badKeys := [][]byte{ + {}, + bytes.Repeat([]byte("a"), 1025), + []byte("has space"), + []byte("has\x00nul"), + []byte("has\x7fdel"), + []byte{0x20}, + []byte{0xff}, + } + for i, badKey := range badKeys { + var buf bytes.Buffer + _ = binary.Write(&buf, binary.LittleEndian, uint32(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(len(badKey))) + buf.Write(badKey) + + _, _, err := decodeBundleOutputInto(nil, nil, buf.Bytes(), 1) + if !errors.Is(err, ErrMalformedOutput) { + t.Fatalf("case %d (%q): expected ErrMalformedOutput, got %v", i, badKey, err) + } + } +} + +func TestDecodeBundleOutputInto_TrailingUnconsumedBytes(t *testing.T) { + var buf bytes.Buffer + _ = binary.Write(&buf, binary.LittleEndian, uint32(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(5)) + buf.WriteString("key_a") + buf.WriteByte(0x99) // trailing unconsumed byte + + _, _, err := decodeBundleOutputInto(nil, nil, buf.Bytes(), 1) + if !errors.Is(err, ErrMalformedOutput) { + t.Fatalf("expected ErrMalformedOutput for trailing bytes, got %v", err) + } +} + +func TestWASMRunner_FailsSubsequentCallsAfterError(t *testing.T) { + ctx := context.Background() + host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + if err != nil { + t.Fatalf("NewWASMHost failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + runner, err := host.NewRunner(ctx) + if err != nil { + t.Fatalf("NewRunner failed: %v", err) + } + defer func() { _ = runner.Close(ctx) }() + + if _, err := runner.MapBundle(ctx, [][]byte{[]byte("\x00trap")}); !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt on first call, got %v", err) + } + if _, err := runner.MapBundle(ctx, [][]byte{[]byte("second")}); !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt on subsequent MapBundle after failure, got %v", err) + } +} + +func TestWASMHost_TestWASMBytes(t *testing.T) { + ctx := context.Background() + host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + if err != nil { + t.Fatalf("NewWASMHost(TestWASMBytes) failed: %v", err) + } + defer func() { _ = host.Close(ctx) }() + + leaves := [][]byte{ + []byte("single_key"), + []byte("shared_key #leaf1"), + []byte("news\ntech\nnews"), + []byte(" \n\t "), + nil, + } + res, err := host.MapBundle(ctx, leaves) + if err != nil { + t.Fatalf("MapBundle failed: %v", err) + } + if len(res) != 5 { + t.Fatalf("got %d results, want 5", len(res)) + } + if len(res[0]) != 1 || res[0][0].Key != "single_key" { + t.Fatalf("leaf 0 = %+v, want [single_key]", res[0]) + } + if len(res[1]) != 1 || res[1][0].Key != "shared_key" { + t.Fatalf("leaf 1 = %+v, want [shared_key]", res[1]) + } + if len(res[2]) != 2 { + t.Fatalf("leaf 2 = %+v, want 2 deduplicated keys", res[2]) + } + if len(res[3]) != 0 || len(res[4]) != 0 { + t.Fatalf("leaves 3,4 = %+v, %+v, want empty", res[3], res[4]) + } + + // Control byte 0x01 triggers ErrMalformedOutput + if _, err := host.MapBundle(ctx, [][]byte{[]byte("\x01bad")}); !errors.Is(err, ErrMalformedOutput) { + t.Fatalf("expected ErrMalformedOutput on 0x01 byte, got %v", err) + } + // NUL byte 0x00 triggers ErrWasmHalt + if _, err := host.MapBundle(ctx, [][]byte{[]byte("\x00trap")}); !errors.Is(err, ErrWasmHalt) { + t.Fatalf("expected ErrWasmHalt on 0x00 byte, got %v", err) + } +} + +func TestZeroReader_Infinite(t *testing.T) { + var zr zeroReader + buf := bytes.Repeat([]byte{0xaa}, 4096) + n, err := zr.Read(buf) + if err != nil || n != 4096 { + t.Fatalf("zeroReader.Read = (%d, %v), want (4096, nil)", n, err) + } + for i, b := range buf { + if b != 0 { + t.Fatalf("buf[%d] = 0x%02x, want 0", i, b) + } + } +} diff --git a/vindex/internal/wasmhost/wasmtest.go b/vindex/internal/wasmhost/wasmtest.go new file mode 100644 index 00000000..541f3b08 --- /dev/null +++ b/vindex/internal/wasmhost/wasmtest.go @@ -0,0 +1,69 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package wasmhost + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sync" +) + +var compileTestGuestOnce = sync.OnceValues(func() ([]byte, error) { + tmpDir, err := os.MkdirTemp("", "vindex-testguest-*") + if err != nil { + return nil, err + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + wasmPath := filepath.Join(tmpDir, "testguest.wasm") + _, thisFile, _, _ := runtime.Caller(0) + guestDir := filepath.Join(filepath.Dir(thisFile), "testdata", "testguest") + + cmd := exec.Command("go", "build", "-trimpath", "-ldflags=-s -w -buildid=", "-buildmode=c-shared", "-o", wasmPath, ".") + cmd.Dir = guestDir + cmd.Env = append(os.Environ(), "GOOS=wasip1", "GOARCH=wasm", "CGO_ENABLED=0") + if out, err := cmd.CombinedOutput(); err != nil { + return nil, fmt.Errorf("failed to build testdata/testguest: %w\n%s", err, string(out)) + } + return os.ReadFile(wasmPath) +}) + +var sharedTestHostOnce = sync.OnceValues(func() (*WASMHost, error) { + return NewWASMHost(context.Background(), TestWASMBytes(), 0) +}) + +// TestWASMBytes compiles and returns the WebAssembly guest module in testdata/testguest +// built with the VIndex MapFn SDK (sdk.RegisterEmit). +func TestWASMBytes() []byte { + b, err := compileTestGuestOnce() + if err != nil { + panic(err) + } + return b +} + +// TestWASMHost returns a shared, process-wide WASMHost running TestWASMBytes() +// for tests that do not close the host directly. +func TestWASMHost() *WASMHost { + h, err := sharedTestHostOnce() + if err != nil { + panic(err) + } + return h +} diff --git a/vindex/mapfn/README.md b/vindex/mapfn/README.md index f8e66a1a..47cb31bd 100644 --- a/vindex/mapfn/README.md +++ b/vindex/mapfn/README.md @@ -35,7 +35,7 @@ type EmitMapFunc func(entry []byte, emit func(key []byte)) func RegisterEmit(fn EmitMapFunc) ``` -- **Build & Reactor Initialization**: Go guest plugins compile as WASI reactors (`GOOS=wasip1 GOARCH=wasm -buildmode=c-shared`, ~1.9 to 2.5 MB with the standard Go runtime or ~200 KB with TinyGo). `//go:wasmexport` bindings in `mapfn/sdk` are isolated behind `//go:build wasm` so native `go test` runs cleanly on host OSes. On startup, `internal/wasmhost` instantiates the module without running `_start` / `main()` and invokes `_initialize` if exported so both WASI reactors and `wasm32-unknown-unknown` guests initialize deterministically. +- **Build & Reactor Initialization**: Go guest plugins compile as WASI reactors (`GOOS=wasip1 GOARCH=wasm -buildmode=c-shared`, ~1.9 to 2.5 MB with the standard Go runtime or ~200 KB with TinyGo). Go 1.24+ ignores `//go:wasmexport` directives on non-WASM targets so native `go test` runs cleanly on host OSes without build tags. On startup, `internal/wasmhost` instantiates the module without running `_start` / `main()` and invokes `_initialize` if exported so both WASI reactors and `wasm32-unknown-unknown` guests initialize deterministically. - **Canonical Key Validation & Fatal `HALT`**: Every emitted `key` MUST be a valid canonical key (`1..1024` printable ASCII bytes in `0x21..0x7E`, containing no `NUL` or whitespace bytes; see [`../docs/tlog-vindex.md`](../docs/tlog-vindex.md#canonical-keys) and [`../docs/APPLICATIONS.md`](../docs/APPLICATIONS.md)). Guest mappers must skip malformed or unindexable fields before calling `emit`: if the guest traps, overflows its linear memory slabs, or emits an invalid canonical key (`len == 0`, `len > 1024`, or any byte outside `0x21..0x7E`), `internal/wasmhost` returns a fatal validation error that immediately **`HALT`s** the indexer or auditor rather than silently dropping the key. - **Zero-Allocation Streaming**: Each `emit(key)` call writes the length-prefixed canonical key directly into the pre-allocated output slab in-place, avoiding intermediate `[]string` or `[][]byte` slice allocations inside the guest. @@ -51,7 +51,7 @@ func input_buffer() uint32 func map_bundle(inputPtr uint32, inputLen uint32) uint64 ``` -1. **Pack (Host)**: `internal/wasmhost` queries `input_buffer()` once on instance initialization (falling back to an exported `allocate(size)` function if `input_buffer` is not exported) and serializes the 256-entry bundle into the guest's static input slab. If the framed bundle exceeds the input slab capacity, the host returns a fatal error (`HALT`). +1. **Pack (Host)**: `internal/wasmhost` queries `input_buffer()` once on instance initialization and serializes the 256-entry bundle into the guest's static input slab. If the framed bundle exceeds the input slab capacity, the host returns a fatal error (`HALT`). 2. **Execute & Stream (Guest)**: The host invokes `map_bundle(inputPtr, inputLen)`. The guest reads the `[Count+1]uint32` offset table (`entry := payload[offsets[i]:offsets[i+1]]`), invokes the mapping callback per entry, and appends length-prefixed keys into the static output slab. If input decoding fails or `emit` overflows the output slab capacity, `map_bundle` aborts and returns `0`. 3. **Validate, Deduplicate & Wipe (Host)**: The guest returns `uint64(outputPtr)<<32 | uint64(outputLen)`. If the return value is `0` or out of bounds, the host returns a fatal error (`HALT`). Otherwise, the host reads all emitted keys from guest memory, validates canonical key bounds (`1..1024` bytes in `0x21..0x7E`), and **sorts and deduplicates canonical keys per entry** before passing the batch downstream. Output slab cursors reset to `0` on the next `map_bundle` call with zero guest heap allocations. diff --git a/vindex/mapfn/examples/README.md b/vindex/mapfn/examples/README.md new file mode 100644 index 00000000..bbceadb5 --- /dev/null +++ b/vindex/mapfn/examples/README.md @@ -0,0 +1,89 @@ +# VIndex WASM MapFn Examples + +This directory contains standalone example implementations of VIndex WASM MapFn binaries. + +These examples demonstrate how external developers can implement and compile custom indexing logic using the [VIndex Guest SDK](../sdk) without needing to upstream code into the core VIndex repository. + +--- + +## Examples Overview + +1. **`sumdb`** (`mapfn/examples/sumdb`): + - Parses Go Checksum Database (`go.sum`) leaves. + - Validates module path and semver formats. + - Filters out ephemeral pseudo-versions. + - Emits canonical case-escaped Go module paths (`module.EscapePath`) via `sdk.RegisterEmit`. + +2. **`ct`** (`mapfn/examples/ct`): + - Parses Certificate Transparency (RFC 6962) `MerkleTreeLeaf`, C2SP Static-CT `TileLeaf`, raw X.509 DER certificates, and PreCertificates. + - Extracts Subject Common Name (CN) and Subject Alternative Names (SANs). + - Normalizes domains (IDNA2008 Punycode, lowercase) into reversed-label canonical keys with a mandatory trailing dot (`com.google.mail.`, `com.google.*.`), without map-time parent expansion. + - Emits canonical domain keys via `sdk.RegisterEmit` for direct MPT prefix/point indexing. + +3. **`starter`** (`mapfn/examples/starter`): + - Minimal boilerplate demonstrating zero-allocation leaf ingestion and key emission. + - Ideal starting point for writing a custom MapFn from scratch. + +--- + +## Building the Examples + +### 1. Using `go generate` +Build all example WASM binaries from the repository root: + +```bash +go generate ./vindex/mapfn/examples/... +``` + +### 2. Manual Invocation +Or compile individually using standard Go flags: + +```bash +# Build starter template +GOOS=wasip1 GOARCH=wasm CGO_ENABLED=0 go build \ + -trimpath \ + -ldflags="-buildid=" \ + -buildmode=c-shared \ + -o vindex/mapfn/examples/starter/starter.wasm \ + github.com/transparency-dev/incubator/vindex/mapfn/examples/starter + +# Build SumDB example +GOOS=wasip1 GOARCH=wasm CGO_ENABLED=0 go build \ + -trimpath \ + -ldflags="-buildid=" \ + -buildmode=c-shared \ + -o vindex/mapfn/examples/sumdb/sumdb.wasm \ + github.com/transparency-dev/incubator/vindex/mapfn/examples/sumdb + +# Build CT example +GOOS=wasip1 GOARCH=wasm CGO_ENABLED=0 go build \ + -trimpath \ + -ldflags="-buildid=" \ + -buildmode=c-shared \ + -o vindex/mapfn/examples/ct/ct.wasm \ + github.com/transparency-dev/incubator/vindex/mapfn/examples/ct +``` + +--- + +## Testing with `vindex-wasm` + +You can test and benchmark the precompiled binaries using `vindex-wasm`: + +```bash +# 1. Test single leaf +go run ./vindex/cmd/vindex-wasm test \ + --wasm=./vindex/mapfn/examples/sumdb/sumdb.wasm \ + --input="golang.org/x/mod v0.14.0 h1:abc=" \ + --format=json + +# 2. Inspect ABI compliance and map-fn commitment digest +go run ./vindex/cmd/vindex-wasm inspect \ + --wasm=./vindex/mapfn/examples/ct/ct.wasm + +# 3. Benchmark throughput and latency percentiles +go run ./vindex/cmd/vindex-wasm bench \ + --wasm=./vindex/mapfn/examples/sumdb/sumdb.wasm \ + --iterations=1000 \ + --workers=4 +``` diff --git a/vindex/mapfn/examples/ct/README.md b/vindex/mapfn/examples/ct/README.md new file mode 100644 index 00000000..1c40818c --- /dev/null +++ b/vindex/mapfn/examples/ct/README.md @@ -0,0 +1,37 @@ +# CT MapFn Example (`ct.wasm`) + +This example implements a VIndex WASM MapFn for **Certificate Transparency (CT)** logs. + +## Supported Leaf Formats + +- **RFC 6962 CT Logs**: Parses TLS `MerkleTreeLeaf` structures containing both `x509_entry` (full X.509 certificate) and `precert_entry` (PreCertificate TBS DER). +- **Static CT Logs (C2SP / Sunlight / TesseraCT)**: Parses individual leaf entries unpacked from static data tiles (standard X.509 DER or RFC 6962 `MerkleTreeLeaf`). + +> [!NOTE] +> **Merkle Tree Certificates (MTC) are not supported** by this MapFn. MTC issuance logs use a distinct ASN.1 `TBSCertificateLogEntry` schema with `SubjectPublicKeyInfoHash` rather than standard X.509 `SubjectPublicKeyInfo`. + +## Indexing & Reversed-Label Canonical Keys + +For each certificate or precertificate leaf, the mapper: +1. Extracts all DNS names from the **Subject Common Name (CN)** and **Subject Alternative Name (SAN)** extension using a zero-allocation ASN.1 TLV scanner. +2. Normalizes domains via IDNA2008 Punycode (`idna.Lookup.ToASCII`) and ASCII lowercasing, preserving leading wildcard labels (`*.`) as a terminal wildcard label (`*`). +3. Reverses domain labels and appends a mandatory trailing dot (`ToReversedDomainKey`), e.g.: + - `auth.service.example.com` -> `com.example.service.auth.` + - `*.example.com` -> `com.example.*.` +4. Emits each unique reversed-label canonical key via `sdk.RegisterEmit` without map-time parent domain expansion (hierarchical subdomain discovery is served natively at query time via MPT prefix queries `GET /keys?prefix=com.example.`). + +## Building & Testing + +Compile the WASM binary: +```bash +go generate ./... +``` + +Inspect and test against sample inputs: +```bash +# Verify ABI compliance +vindex-wasm inspect --wasm=ct.wasm + +# Test against a leaf payload +vindex-wasm test --wasm=ct.wasm --input_file=leaf.bin +``` diff --git a/vindex/mapfn/examples/ct/ct_test.go b/vindex/mapfn/examples/ct/ct_test.go new file mode 100644 index 00000000..64938c47 --- /dev/null +++ b/vindex/mapfn/examples/ct/ct_test.go @@ -0,0 +1,407 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + _ "embed" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + "slices" + "testing" + "time" +) + +func generateTestCert(t *testing.T, cn string, sans []string) []byte { + priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("failed to generate key: %v", err) + } + + template := x509.Certificate{ + SerialNumber: big.NewInt(12345), + Subject: pkix.Name{ + CommonName: cn, + }, + DNSNames: sans, + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + BasicConstraintsValid: true, + } + + der, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + t.Fatalf("failed to create cert: %v", err) + } + return der +} + +func buildRFC6962Leaf(entryType uint16, payload []byte) []byte { + var buf []byte + buf = append(buf, 0x00) // version 0 + buf = append(buf, 0x00) // leaf_type = timestamped_entry + var ts [8]byte + buf = append(buf, ts[:]...) + var et [2]byte + binary.BigEndian.PutUint16(et[:], entryType) + buf = append(buf, et[:]...) + + switch entryType { + case 0: // x509_entry + certLen := len(payload) + buf = append(buf, byte(certLen>>16), byte(certLen>>8), byte(certLen)) + buf = append(buf, payload...) + buf = append(buf, 0x00, 0x00) // extensions length + case 1: // precert_entry + var keyHash [32]byte + buf = append(buf, keyHash[:]...) + tbsLen := len(payload) + buf = append(buf, byte(tbsLen>>16), byte(tbsLen>>8), byte(tbsLen)) + buf = append(buf, payload...) + buf = append(buf, 0x00, 0x00) // extensions length + } + return buf +} + +func TestMapCTLeaf_X509Certificate(t *testing.T) { + certDer := generateTestCert(t, "example.com", []string{"foo.bar.example.com", "*.sub.example.org"}) + leaf := buildRFC6962Leaf(0, certDer) + + var domains []string + MapCTLeaf(leaf, func(key []byte) { + domains = append(domains, string(key)) + }) + domainMap := make(map[string]bool) + for _, d := range domains { + domainMap[d] = true + } + + expectedDomains := []string{ + "com.example.", + "com.example.bar.foo.", + "org.example.sub.*.", + } + + if len(domains) != len(expectedDomains) { + t.Errorf("got %d domains %v, want %d %v", len(domains), domains, len(expectedDomains), expectedDomains) + } + for _, d := range expectedDomains { + if !domainMap[d] { + t.Errorf("expected domain %q in mapped entries, got %v", d, domains) + } + } +} + +func TestMapCTLeaf_StaticCTTileLeaf_HeaderDisambiguation(t *testing.T) { + certDer := generateTestCert(t, "static-ct.example.com", []string{"api.static-ct.example.com"}) + parsedCert, err := x509.ParseCertificate(certDer) + if err != nil { + t.Fatalf("ParseCertificate: %v", err) + } + + // Build a Static-CT precert TileLeaf whose 8-byte timestamp begins with 0x00, 0x00 + // and whose issuer_key_hash begins with 0x00, 0x01 (colliding with RFC 6962 entry_type at offset 10..12). + var leaf []byte + ts := [8]byte{0x00, 0x00, 0x01, 0x98, 0xcf, 0x15, 0xff, 0xf7} + leaf = append(leaf, ts[:]...) + leaf = append(leaf, 0x00, 0x01) // entry_type = precert_entry (at offset 8..10) + var keyHash [32]byte + keyHash[0] = 0x00 + keyHash[1] = 0x01 // at offset 10..12 in leaf! + leaf = append(leaf, keyHash[:]...) + tbs := parsedCert.RawTBSCertificate + tbsLen := len(tbs) + leaf = append(leaf, byte(tbsLen>>16), byte(tbsLen>>8), byte(tbsLen)) + leaf = append(leaf, tbs...) + leaf = append(leaf, 0x00, 0x00) // extensions + leaf = append(leaf, 0x00, 0x00, 0x00) + leaf = append(leaf, 0x00, 0x00) // chain + + var got []string + MapCTLeaf(leaf, func(key []byte) { + got = append(got, string(key)) + }) + want := []string{ + "com.example.static-ct.", + "com.example.static-ct.api.", + } + if !slices.Equal(got, want) { + t.Fatalf("MapCTLeaf(Static-CT precert with 0x0001 issuer_key_hash prefix) = %v, want %v", got, want) + } +} + +func TestMapCTLeaf_MultiLabelTLD_And_IDNA(t *testing.T) { + // Subject.CommonName is marshaled as UTF8String by x509.CreateCertificate, allowing IDN test input. + certDer := generateTestCert(t, "*.münchen.de.", []string{"deep.sub.service.co.uk.", "MAPS.GOOGLE.COM"}) + var domains []string + MapCTLeaf(certDer, func(key []byte) { + domains = append(domains, string(key)) + }) + domainMap := make(map[string]bool) + for _, d := range domains { + domainMap[d] = true + } + + expectedDomains := []string{ + "de.xn--mnchen-3ya.*.", + "uk.co.service.sub.deep.", + "com.google.maps.", + } + + if len(domains) != len(expectedDomains) { + t.Errorf("got %d domains %v, want %d %v", len(domains), domains, len(expectedDomains), expectedDomains) + } + for _, d := range expectedDomains { + if !domainMap[d] { + t.Errorf("expected domain %q in mapped entries, got %v", d, domains) + } + } +} + +func TestMapCTLeaf_NoPlaintextFallback(t *testing.T) { + plain := []byte("alpha.example.com\n*.beta.example.com\n") + var domains []string + MapCTLeaf(plain, func(key []byte) { + domains = append(domains, string(key)) + }) + if len(domains) != 0 { + t.Errorf("expected 0 keys for non-ASN.1 plaintext input, got %v", domains) + } +} + +func TestMapCTLeaf_LargeSANOverflow(t *testing.T) { + var sans []string + for i := 0; i < 100; i++ { + sans = append(sans, fmt.Sprintf("host-%03d.long-subdomain-segment.example.com", i)) + } + certDer := generateTestCert(t, "", sans) + leaf := buildRFC6962Leaf(0, certDer) + + var domains []string + MapCTLeaf(leaf, func(key []byte) { + domains = append(domains, string(key)) + }) + if len(domains) != 100 { + t.Fatalf("expected 100 domains, got %d", len(domains)) + } +} + +func TestMapCTLeaf_Empty(t *testing.T) { + var count int + emitCount := func(key []byte) { count++ } + + count = 0 + MapCTLeaf(nil, emitCount) + if count != 0 { + t.Fatalf("expected 0 results for nil input, got %d", count) + } + + count = 0 + MapCTLeaf([]byte(" \n\t "), emitCount) + if count != 0 { + t.Fatalf("expected 0 results for whitespace input, got %d", count) + } +} + +func TestCanonicalizeDomainBytes(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want string + wantOK bool + }{ + {name: "IPv4 literal 192.0.2.1", in: "192.0.2.1", want: "", wantOK: false}, + {name: "IPv4 literal 127.0.0.1", in: "127.0.0.1", want: "", wantOK: false}, + {name: "IPv4 literal trailing dot", in: "192.0.2.1.", want: "", wantOK: false}, + {name: "IPv6 literal", in: "2001:db8::1", want: "", wantOK: false}, + {name: "IPv6 bracketed literal", in: "[2001:db8::1]", want: "", wantOK: false}, + {name: "leading hyphen label", in: "-bad.example.com", want: "", wantOK: false}, + {name: "trailing hyphen label", in: "bad-.example.com", want: "", wantOK: false}, + {name: "leading hyphen TLD", in: "example.-com", want: "", wantOK: false}, + {name: "trailing hyphen TLD", in: "example.com-", want: "", wantOK: false}, + {name: "numeric subdomains", in: "1.2.example.com", want: "com.example.2.1.", wantOK: true}, + {name: "wildcard subdomain", in: "*.mail.example.com", want: "com.example.mail.*.", wantOK: true}, + {name: "IDN domain", in: "bücher.example.com", want: "com.example.xn--bcher-kva.", wantOK: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + var buf [256]byte + got, ok := canonicalizeDomainBytes([]byte(tc.in), &buf) + if ok != tc.wantOK || string(got) != tc.want { + t.Errorf("canonicalizeDomainBytes(%q) = (%q, %v), want (%q, %v)", tc.in, string(got), ok, tc.want, tc.wantOK) + } + }) + } +} + +func TestMapCTLeaf_SkipIPAndInvalidLabels(t *testing.T) { + invalidCert := generateTestCert(t, "192.0.2.1", []string{ + "127.0.0.1", + "2001:db8::1", + "[2001:db8::1]", + "-bad.example.com", + "bad-.example.com", + "example.-com", + }) + invalidLeaf := buildRFC6962Leaf(0, invalidCert) + + var gotInvalid []string + MapCTLeaf(invalidLeaf, func(key []byte) { + gotInvalid = append(gotInvalid, string(key)) + }) + if len(gotInvalid) != 0 { + t.Errorf("MapCTLeaf(invalid IP/hyphen labels) emitted %v, want 0 keys", gotInvalid) + } + + // Subject.CommonName is marshaled as UTF8String by x509.CreateCertificate, allowing IDN test input. + mixedCert := generateTestCert(t, "bücher.example.com", []string{ + "192.0.2.1", + "127.0.0.1", + "2001:db8::1", + "[2001:db8::1]", + "-bad.example.com", + "bad-.example.com", + "example.-com", + "1.2.example.com", + "*.mail.example.com", + }) + mixedLeaf := buildRFC6962Leaf(0, mixedCert) + + var gotMixed []string + MapCTLeaf(mixedLeaf, func(key []byte) { + gotMixed = append(gotMixed, string(key)) + }) + slices.Sort(gotMixed) + wantMixed := []string{ + "com.example.2.1.", + "com.example.mail.*.", + "com.example.xn--bcher-kva.", + } + if !slices.Equal(gotMixed, wantMixed) { + t.Errorf("MapCTLeaf(mixed valid/invalid) = %v, want %v", gotMixed, wantMixed) + } +} + +//go:embed testdata/golden_leaves.json +var goldenLeavesJSON []byte + +type goldenCase struct { + Index int `json:"index"` + EntryType string `json:"entry_type"` + Category string `json:"category"` + LeafHex string `json:"leaf_hex"` + ByteLength int `json:"byte_length"` +} + +func TestMapCTLeaf_GoldenDataset(t *testing.T) { + var cases []goldenCase + if err := json.Unmarshal(goldenLeavesJSON, &cases); err != nil { + t.Fatalf("failed to unmarshal golden_leaves.json: %v", err) + } + if len(cases) == 0 { + t.Fatal("empty golden dataset") + } + + for _, tc := range cases { + leafBytes, err := hex.DecodeString(tc.LeafHex) + if err != nil { + t.Fatalf("case %d: failed to decode hex: %v", tc.Index, err) + } + if len(leafBytes) != tc.ByteLength { + t.Fatalf("case %d: byte length mismatch: got %d, want %d", tc.Index, len(leafBytes), tc.ByteLength) + } + + var domains []string + MapCTLeaf(leafBytes, func(key []byte) { + domains = append(domains, string(key)) + }) + + want := []string{ + "com.flowers-to-the-world.", + fmt.Sprintf("com.flowers-to-the-world.letsencrypt_sycamore2026h2.2025.08.22.%02d.", tc.Index&^1), + } + if !slices.Equal(domains, want) { + t.Errorf("case %d [%s/%s]: got %v, want %v", tc.Index, tc.EntryType, tc.Category, domains, want) + } + } +} + +func BenchmarkMapCTLeaf(b *testing.B) { + var cases []goldenCase + if err := json.Unmarshal(goldenLeavesJSON, &cases); err != nil { + b.Fatalf("failed to unmarshal golden_leaves.json: %v", err) + } + if len(cases) == 0 { + b.Fatal("empty golden dataset") + } + leaf, err := hex.DecodeString(cases[0].LeafHex) + if err != nil { + b.Fatalf("failed to decode hex: %v", err) + } + + b.ReportAllocs() + emitSink := func(key []byte) {} + for b.Loop() { + MapCTLeaf(leaf, emitSink) + } +} + +func TestMapCTLeaf_ConcurrentSafety(t *testing.T) { + certDer := generateTestCert(t, "example.com", []string{"foo.bar.example.com", "*.sub.example.org"}) + leaf := buildRFC6962Leaf(0, certDer) + + done := make(chan struct{}) + for i := 0; i < 8; i++ { + go func() { + defer func() { done <- struct{}{} }() + for j := 0; j < 50; j++ { + var count int + MapCTLeaf(leaf, func(key []byte) { + count++ + }) + if count != 3 { + t.Errorf("got %d domains, want 3", count) + } + } + }() + } + for i := 0; i < 8; i++ { + <-done + } +} + +func TestReadASN1Header_OverflowGuard(t *testing.T) { + // 4-byte length 0x80000000 (negative on 32-bit int) or 0x7fffffff + malformed := []byte{0x30, 0x84, 0x80, 0x00, 0x00, 0x00, 0x01, 0x02} + if _, _, _, ok := readASN1Header(malformed, 0); ok { + t.Fatal("expected readASN1Header to reject negative/overflowing 4-byte length") + } + malformedMax := []byte{0x30, 0x84, 0x7f, 0xff, 0xff, 0xff, 0x01, 0x02} + if _, _, _, ok := readASN1Header(malformedMax, 0); ok { + t.Fatal("expected readASN1Header to reject 0x7fffffff length") + } +} diff --git a/vindex/mapfn/examples/ct/main.go b/vindex/mapfn/examples/ct/main.go new file mode 100644 index 00000000..a14d6d26 --- /dev/null +++ b/vindex/mapfn/examples/ct/main.go @@ -0,0 +1,489 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate sh -c "GOOS=wasip1 GOARCH=wasm CGO_ENABLED=0 go build -trimpath -ldflags=\"-buildid=\" -buildmode=c-shared -o ct.wasm ." + +// Package main implements an optimized, zero-alloc CT (Certificate Transparency) WASM MapFn plugin. +package main + +import ( + "bytes" + "encoding/binary" + "sync/atomic" + + "github.com/transparency-dev/incubator/vindex/mapfn/sdk" + "golang.org/x/net/idna" +) + +func main() {} + +func init() { + sdk.RegisterEmit(MapCTLeaf) +} + +var ( + oidSANRaw = []byte{0x55, 0x1d, 0x11} // 2.5.29.17 (Subject Alternative Name) + oidCNRaw = []byte{0x55, 0x04, 0x03} // 2.5.4.3 (Common Name) +) + +type domainSpan struct { + start uint16 + end uint16 +} + +// Package-level scratch buffers avoid a 2 KB heap allocation per leaf: slicing a local +// array into the indirect `emit func([]byte)` parameter causes Go's escape analysis +// to move the backing array to the heap. +var ( + leafScratchInUse uint32 + leafStorage [2048]byte + leafSpans [64]domainSpan +) + +// MapCTLeaf extracts all unique reversed-label canonical domain keys from a CT log leaf. +func MapCTLeaf(data []byte, emit func(key []byte)) { + if len(data) == 0 { + return + } + + storage := &leafStorage + spans := &leafSpans + if atomic.CompareAndSwapUint32(&leafScratchInUse, 0, 1) { + defer atomic.StoreUint32(&leafScratchInUse, 0) + } else { + var localStorage [2048]byte + var localSpans [64]domainSpan + storage = &localStorage + spans = &localSpans + } + storageUsed := 0 + spanCount := 0 + var overflowDomains [][]byte + var outBuf [256]byte + + extractRawDomainNames(data, func(rawName []byte) { + canon, ok := canonicalizeDomainBytes(rawName, &outBuf) + if !ok { + return + } + + for i := 0; i < spanCount; i++ { + sp := spans[i] + if bytes.Equal(storage[sp.start:sp.end], canon) { + return + } + } + for _, ov := range overflowDomains { + if bytes.Equal(ov, canon) { + return + } + } + + if storageUsed+len(canon) <= len(storage) && spanCount < len(spans) { + baseStart := uint16(storageUsed) + baseEnd := baseStart + uint16(len(canon)) + copy(storage[baseStart:], canon) + storageUsed = int(baseEnd) + spans[spanCount] = domainSpan{start: baseStart, end: baseEnd} + spanCount++ + } else { + overflowDomains = append(overflowDomains, append([]byte(nil), canon...)) + } + }) + + for i := 0; i < spanCount; i++ { + sp := spans[i] + emit(storage[sp.start:sp.end]) + } + for _, ov := range overflowDomains { + emit(ov) + } +} + +func isASCIIWhitespace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\v' || b == '\f' +} + +func canonicalizeDomainBytes(raw []byte, outBuf *[256]byte) ([]byte, bool) { + for len(raw) > 0 && isASCIIWhitespace(raw[0]) { + raw = raw[1:] + } + for len(raw) > 0 && isASCIIWhitespace(raw[len(raw)-1]) { + raw = raw[:len(raw)-1] + } + if len(raw) > 0 && raw[len(raw)-1] == '.' { + raw = raw[:len(raw)-1] + } + if len(raw) == 0 { + return nil, false + } + + hasWildcard := false + if len(raw) >= 2 && raw[0] == '*' && raw[1] == '.' { + hasWildcard = true + raw = raw[2:] + } else if raw[0] == '*' { + return nil, false + } + if len(raw) == 0 { + return nil, false + } + + hasNonASCII := false + for i := 0; i < len(raw); i++ { + if raw[i] >= 0x80 { + hasNonASCII = true + break + } + } + if hasNonASCII { + ascii, err := idna.Lookup.ToASCII(string(raw)) + if err != nil || len(ascii) == 0 { + return nil, false + } + if ascii[len(ascii)-1] == '.' { + ascii = ascii[:len(ascii)-1] + } + if len(ascii) == 0 { + return nil, false + } + raw = []byte(ascii) + } + + needed := len(raw) + 1 + if hasWildcard { + needed += 2 + } + if len(raw) > 253 || needed > len(outBuf) { + return nil, false + } + + outLen := 0 + dotCount := 0 + labelEnd := len(raw) + + for i := len(raw) - 1; i >= -1; i-- { + if i == -1 || raw[i] == '.' { + if i >= 0 { + dotCount++ + } + labelStart := i + 1 + labelLen := labelEnd - labelStart + if labelLen < 1 || labelLen > 63 { + return nil, false + } + if raw[labelStart] == '-' || raw[labelEnd-1] == '-' { + return nil, false + } + isTLD := labelEnd == len(raw) + allDigits := true + for j := labelStart; j < labelEnd; j++ { + c := raw[j] + if c >= 'A' && c <= 'Z' { + c += 'a' - 'A' + } + if (c < 'a' || c > 'z') && (c < '0' || c > '9') && c != '-' && c != '_' { + return nil, false + } + if c < '0' || c > '9' { + allDigits = false + } + outBuf[outLen] = c + outLen++ + } + if isTLD && allDigits { + return nil, false + } + outBuf[outLen] = '.' + outLen++ + labelEnd = i + } + } + + if dotCount < 1 { + return nil, false + } + if hasWildcard { + outBuf[outLen] = '*' + outLen++ + outBuf[outLen] = '.' + outLen++ + } + return outBuf[:outLen], true +} + +// readASN1Header parses tag and length at pos. Returns (tag, contentStart, contentEnd, ok). +func readASN1Header(b []byte, pos int) (tag byte, start, end int, ok bool) { + if pos < 0 || pos >= len(b) { + return 0, 0, 0, false + } + tag = b[pos] + pos++ + if pos >= len(b) { + return 0, 0, 0, false + } + lByte := b[pos] + pos++ + var length int + if lByte < 0x80 { + length = int(lByte) + } else { + numBytes := int(lByte & 0x7f) + if numBytes == 0 || numBytes > 4 || numBytes > len(b)-pos { + return 0, 0, 0, false + } + for i := 0; i < numBytes; i++ { + length = (length << 8) | int(b[pos+i]) + } + pos += numBytes + } + if length < 0 || length > len(b)-pos { + return 0, 0, 0, false + } + return tag, pos, pos + length, true +} + +// fastExtractDomainsFromTBS extracts CommonName and SAN DNSNames from raw TBSCertificate DER. +func fastExtractDomainsFromTBS(tbs []byte, emit func(raw []byte)) bool { + tag, start, end, ok := readASN1Header(tbs, 0) + if !ok || tag != 0x30 || end != len(tbs) { + return false + } + payload := tbs[start:end] + + pos := 0 + + // 1. Optional version [0] EXPLICIT + if pos < len(payload) && payload[pos] == 0xa0 { + _, _, next, ok := readASN1Header(payload, pos) + if !ok { + return false + } + pos = next + } + + // 2. serialNumber INTEGER (0x02) + if tag, _, next, ok := readASN1Header(payload, pos); ok && tag == 0x02 { + pos = next + } else { + return false + } + + // 3. signature AlgorithmIdentifier SEQUENCE (0x30) + if tag, _, next, ok := readASN1Header(payload, pos); ok && tag == 0x30 { + pos = next + } else { + return false + } + + // 4. issuer Name SEQUENCE (0x30) + if tag, _, next, ok := readASN1Header(payload, pos); ok && tag == 0x30 { + pos = next + } else { + return false + } + + // 5. validity Validity SEQUENCE (0x30) + if tag, _, next, ok := readASN1Header(payload, pos); ok && tag == 0x30 { + pos = next + } else { + return false + } + + // 6. subject Name SEQUENCE (0x30) + var subjectBytes []byte + if tag, sStart, sEnd, ok := readASN1Header(payload, pos); ok && tag == 0x30 { + pos = sEnd + subjectBytes = payload[sStart:sEnd] + } else { + return false + } + + // 7. subjectPublicKeyInfo SEQUENCE (0x30) - SKIP without parsing keys! + if tag, _, next, ok := readASN1Header(payload, pos); ok && tag == 0x30 { + pos = next + } else { + return false + } + + // Optional: issuerUniqueID [1] + if pos < len(payload) && payload[pos] == 0x81 { + _, _, next, ok := readASN1Header(payload, pos) + if !ok { + return false + } + pos = next + } + + // Optional: subjectUniqueID [2] + if pos < len(payload) && payload[pos] == 0x82 { + _, _, next, ok := readASN1Header(payload, pos) + if !ok { + return false + } + pos = next + } + + // Emit Subject CN if present + sPos := 0 + for sPos < len(subjectBytes) { + _, setStart, setEnd, ok := readASN1Header(subjectBytes, sPos) + if !ok { + break + } + setBytes := subjectBytes[setStart:setEnd] + subPos := 0 + for subPos < len(setBytes) { + _, seqStart, seqEnd, ok := readASN1Header(setBytes, subPos) + if !ok { + break + } + seqBytes := setBytes[seqStart:seqEnd] + oTag, oStart, oEnd, ok := readASN1Header(seqBytes, 0) + if ok && oTag == 0x06 && bytes.Equal(seqBytes[oStart:oEnd], oidCNRaw) { + _, vStart, vEnd, ok := readASN1Header(seqBytes, oEnd) + if ok && vEnd > vStart { + emit(seqBytes[vStart:vEnd]) + } + } + subPos = seqEnd + } + sPos = setEnd + } + + // Optional: extensions [3] EXPLICIT (0xa3) + if pos < len(payload) && payload[pos] == 0xa3 { + _, eStart, eEnd, ok := readASN1Header(payload, pos) + if ok { + extContainer := payload[eStart:eEnd] + tag, sStart, sEnd, ok := readASN1Header(extContainer, 0) + if ok && tag == 0x30 { + extList := extContainer[sStart:sEnd] + ePos := 0 + for ePos < len(extList) { + _, itemStart, itemEnd, ok := readASN1Header(extList, ePos) + if !ok { + break + } + itemBytes := extList[itemStart:itemEnd] + iPos := 0 + tag, oStart, oEnd, ok := readASN1Header(itemBytes, iPos) + if ok && tag == 0x06 { + isSAN := bytes.Equal(itemBytes[oStart:oEnd], oidSANRaw) + iPos = oEnd + if iPos < len(itemBytes) && itemBytes[iPos] == 0x01 { // critical BOOLEAN + _, _, next, ok := readASN1Header(itemBytes, iPos) + if ok { + iPos = next + } + } + tag, vStart, vEnd, ok := readASN1Header(itemBytes, iPos) // extnValue OCTET STRING + if ok && tag == 0x04 && isSAN { + sanVal := itemBytes[vStart:vEnd] + tag, gnStart, gnEnd, ok := readASN1Header(sanVal, 0) + if ok && tag == 0x30 { + gnList := sanVal[gnStart:gnEnd] + gnPos := 0 + for gnPos < len(gnList) { + tag, dStart, dEnd, ok := readASN1Header(gnList, gnPos) + if !ok { + break + } + if tag == 0x82 && dEnd > dStart { // dNSName [2] + emit(gnList[dStart:dEnd]) + } + gnPos = dEnd + } + } + } + } + ePos = itemEnd + } + } + } + } + + return true +} + +// fastExtractDomainsFromCert unwraps the outer Certificate SEQUENCE and parses TBSCertificate. +func fastExtractDomainsFromCert(certDer []byte, emit func(raw []byte)) bool { + tag, start, end, ok := readASN1Header(certDer, 0) + if !ok || tag != 0x30 || end != len(certDer) { + return false + } + outer := certDer[start:end] + tag, _, tbsEnd, ok := readASN1Header(outer, 0) + if !ok || tag != 0x30 { + return false + } + return fastExtractDomainsFromTBS(outer[:tbsEnd], emit) +} + +func extractRawDomainNames(data []byte, emit func(raw []byte)) { + if len(data) == 0 { + return + } + + // 1. Try RFC 6962 MerkleTreeLeaf parsing (starts with 0x00, 0x00 version & leaf_type, + // and ends with a 2-byte extensions length matching len(data)). + if len(data) >= 17 && data[0] == 0 && data[1] == 0 { + entryType := binary.BigEndian.Uint16(data[10:12]) + if entryType == 0 { // x509_entry + certLen := int(data[12])<<16 | int(data[13])<<8 | int(data[14]) + if certLen > 0 && len(data) >= 17+certLen { + extLen := int(binary.BigEndian.Uint16(data[15+certLen : 17+certLen])) + if 17+certLen+extLen == len(data) && fastExtractDomainsFromCert(data[15:15+certLen], emit) { + return + } + } + } else if entryType == 1 && len(data) >= 49 { // precert_entry + tbsLen := int(data[44])<<16 | int(data[45])<<8 | int(data[46]) + if tbsLen > 0 && len(data) >= 49+tbsLen { + extLen := int(binary.BigEndian.Uint16(data[47+tbsLen : 49+tbsLen])) + if 49+tbsLen+extLen == len(data) && fastExtractDomainsFromTBS(data[47:47+tbsLen], emit) { + return + } + } + } + } + + // 2. Try C2SP Static-CT TileLeaf parsing (starts directly with 8-byte timestamp) + if len(data) >= 13 { + entryType := binary.BigEndian.Uint16(data[8:10]) + if entryType == 0 { // x509_entry + certLen := int(data[10])<<16 | int(data[11])<<8 | int(data[12]) + if certLen > 0 && len(data) >= 13+certLen { + if fastExtractDomainsFromCert(data[13:13+certLen], emit) { + return + } + } + } else if entryType == 1 && len(data) >= 45 { // precert_entry + tbsLen := int(data[42])<<16 | int(data[43])<<8 | int(data[44]) + if tbsLen > 0 && len(data) >= 45+tbsLen { + if fastExtractDomainsFromTBS(data[45:45+tbsLen], emit) { + return + } + } + } + } + + // 3. Try raw X.509 Certificate DER + if fastExtractDomainsFromCert(data, emit) { + return + } + + // 4. Try raw TBSCertificate DER + _ = fastExtractDomainsFromTBS(data, emit) +} diff --git a/vindex/mapfn/examples/ct/testdata/golden_leaves.json b/vindex/mapfn/examples/ct/testdata/golden_leaves.json new file mode 100644 index 00000000..df4c9749 --- /dev/null +++ b/vindex/mapfn/examples/ct/testdata/golden_leaves.json @@ -0,0 +1,72 @@ +[ + { + "index": 0, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198cf15fff7000000053a308205363082031ea00302010202081067e403d67b3d17300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236303832393033303835365a170d3236303833303033303835365a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100e43f2d9125d82fa376ab533c03a0de2f094d9b38a7c4c31ed61b799463a40799b34cadb208191d5001e27512a7d663db2c58f3aa8c3ede35aabc2553b4605d43830c263f123d7b7988e09818e9be4bf3d4e5b99aef8f874d0ddd355210e0c25b4d10ced86276614df27501c39ccd46ed9aae46722015578caaf51a0cd91780df2ec6d3789c2a1a48e0e41e15ce726b140628670e96fc55d686fd4aa2c5478c99983ded0aba8ae97be36fec55c8577dc09652206acb38c31f5864b9919c2780b541164cdbdf9a766abc78ebb41fb20cb65e6b9622b226f8afe049b346f06ee0a9846a451f98137edcb1412f51e92169c903e136b6d9d0bc4eb307a90c6cc917cf0203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130302e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b05000382020100b616aba68152383580a1c89476cf7ad2949db8c117727152e7c65a080fe225c871bce1595329d9713f6aa1e60fc89ad5a4234f7d81e2bf7d769fb462c112cfe1aa82e23c707c77a1f53f4723ef666fe9f0d0e53a4d1ae479d2f3ddd039026c064e38fc3eac6e2e13ff0d6d63f7ee2377893bb3a046f2362193afb8eba514acb9e2c0afe30e317e69c3e3f750976647e90334fbec9ebe0349aa8322db1c4bc3979313e97e7b284f927f3031afefe6014d9f6754425a196a1f895d443b1c0722775230cf3181ecdc2e41bd5bcc2c8c9693e010f5e53d3a9a3385a1a50596bd6039618f9db142d8bbd3f8553a0f1b9380e610aceb8d2a6247f1789d1d30473af87a928c76d4fda5b262ba2247c27bac88badc471346446cb9db74890080a0b43f516ad4ab050084ebc621c84c7a3a9fb8efb99265243222036f9b770cc25fcef35387c3c68090caecc783f9680c5c292024e47952690cc399ae7dabedae9a6f0eca7574cb4c5c6ca31fe9be7531c6213ce8340ea9b7b3ddb2c11dba62c400e30043f9811d27105dc8aed79aab02bb1841b46bfe2e51f068f26c7464b5c72b108919a6e0cc72a43586dbc02e8070576d2a35c8b1d3e1fdcd80571e1c778e073a041b05aa8856067a3c70ddd2dae37811c4d63679b6aa9caed7c943f3c64cb34fbcaca42717a2187a8b9be7474b0231c099b58d6ae0bb403a33775e418bc801453931000800000500000000000040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1427 + }, + { + "index": 1, + "entry_type": "precert", + "category": "precert_standard", + "leaf_hex": "00000198cf184dd00001e37689003073a0c649cc656de946c03174d25c566fe3c3805b846f52369437980003233082031fa003020102020900f6e95b0c30d1740f300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236313231323231313230305a170d3236313231333231313230305a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100cdccbea8b71cc640858b77690d542964c6effe4146c6b5581514e3522755002e96ea4df44f9893cf32b9dba0b5deefa8c3f2bfeef3f1110882c2aab89e55a8fbe4ee5ebb51091ce97df7f9b58f8ec11414c10f5ec77f9eb7a9d7de14fd157424582e878e6d292a113f73995ea5e27b9e64df4fbd913a1eb4b487864532ff0c89978984da47931ccad76ef7190883f041623ec7e59af88c9b4a6ef2bb8b4b083c5f5f881e90018a64c693cd21cb65b8a80cac045cc65a101edccfc1a7cd4431532d85706f2a6f600d98b9ac4013394ac7095d6d18fe01f0611846dd5e05b6abbcf8b85f7ad8b6b81e1909691266d0878dcb2debf0633924a5f76dcb8ecdebabf10203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130302e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d000800000500000000010005503082054c30820334a003020102020900f6e95b0c30d1740f300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236313231323231313230305a170d3236313231333231313230305a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100cdccbea8b71cc640858b77690d542964c6effe4146c6b5581514e3522755002e96ea4df44f9893cf32b9dba0b5deefa8c3f2bfeef3f1110882c2aab89e55a8fbe4ee5ebb51091ce97df7f9b58f8ec11414c10f5ec77f9eb7a9d7de14fd157424582e878e6d292a113f73995ea5e27b9e64df4fbd913a1eb4b487864532ff0c89978984da47931ccad76ef7190883f041623ec7e59af88c9b4a6ef2bb8b4b083c5f5f881e90018a64c693cd21cb65b8a80cac045cc65a101edccfc1a7cd4431532d85706f2a6f600d98b9ac4013394ac7095d6d18fe01f0611846dd5e05b6abbcf8b85f7ad8b6b81e1909691266d0878dcb2debf0633924a5f76dcb8ecdebabf10203010001a381d43081d1300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130302e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d3013060a2b06010401d6790204030101ff04020500300d06092a864886f70d01010b0500038202010092c1b02e0377318d0ada54e4061c205e06da43c5efb49db1677a42fa5297c2ed9029a83de235bb509861f3d917591df62d45ba3aa32c30bf65d67affacc0998bf008edb5ce0ba6c150f5e54ece4525c3cdb26573465e8346f8e469e288b62a7495fb4878b3147fae32df7e783385b769298c9fb8d0d667498629d665f3850428731ba1ef6678e684a6988d5d219fc32f58b16fb4d06795c6e85c6a5b0f08f84b062d731a0c341acadf367f4a97cfa29ffa8d7d161ab9c128db695014e5e317c6c08f5626b1b7dbe754af92ebb6a86c25dd33c02e4e35f16a3ce4b535cf5def06d5967cecde323da0915c58fbc7c3bea0d8048c5a9febdd43c9e3cfe0acd15a54d6d66793b43a1fd32d1d5db5fc1a48142cab5756e582961b076de27f72806bb6d68a7174349c2ed9e865be9093a93e65bffce1bdc8c7b770de182079d31934f640b10625eb41a9bd67981afa6a7220f6fed2fa211c85cef9198476b13c91eda4215a1a80eb64f269e1ff6ad3d99e638140b2260335c1349ff2e497dbe89738ce153e4b07783283569de59e016b02d08775c0a6de628efa4ee739268ca6987c7c1f9e334b5f3a68979f97723532e8988f9a7964f5562c07e42714b9a517c16fbb56b0b4fd51f3d102d39a3a80f1ec0a1bdad3c8de101d98bde886c9854717229c7df2a29d2b0122de4153da9d5feb0928b83e270c08abc285e76ff18bfe74251b0040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 2287 + }, + { + "index": 2, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198cf83ddf5000000053b308205373082031fa003020102020900d15e397663f764f3300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236303831363139353532365a170d3236303831373139353532365a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100c4da3e34c0bb70bf5bdbef65c0f48ff453d9df8d469bed8054433d775a77501ddeec4dab5a0bf4ef205b02cfa39e9d6e7f17687cd251eddc749d0313f7303ce614cc21256d537e6f6baa4ef0400cee35dfc3798785afb3ae7449ff880287031c56cfdffd8c20619d0a5d8b426583d1cb941f01e63740eeebecdcbf9f205242e317d4ba8c867dbff4541a10192ec0079088f7fd38541e9914b6b816a675db38eed4b7679cab7390cfb52bc20f6f5ed72e313e173d7445c048bbf18fd52cef0f1e7c156c183d86ef1b76988ba7fee291a0f955eb558261e3831c7b7f727ae07de0d01a25e36d486dc66ba532d97d4e044ad41e9870f12162210341f59c0a7b5e190203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130322e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b0500038202010063b7c598f2fd976896a1c769af90500c6f510bbdecbb8325579f12de7a1a30b8408f17092e6538e1d6993c4ee038c95abd289454bc4345cc54d59870e0ef040ad55c28315f4a8940a083327405da0b42a02a1d71c45179a4f4d7e535b576a37c4d0e3ab607d3b15a036200d12fb436f3df9da7b6e1356e9773b967113535b81aca274c7852fd2feb3f0ebbf1408825a66d3af054666668b749c93901c6cace1ab1e783c8f75ac28686c009e65ead96a28b0b7bd3d1d15cef5bb8100d0eaf59aad2eed4764ed53b7b23a89a0ada8998e622fe5f43346e8d8805130f2e6942f28ee6ef4c32df37d326f4651bf9994295180e52246190fd38f26b85fff8fb3f9d0ea70545506c0f6c93720615e24525f416504b3765653f73fb1cb57e4cd2807e6446d91c26e530b9ad6fe73cea482819fb30df8594e53083aa5241c9c0be4c75d1c47af86e21bcc97dc80a2c503c2cc8caa00deec0cf7bccd57fd2e42e3b6f11149287b647b15adb99318375a1ee3b8b42e516dabb3159b0121c038a7a0319594925055a5fc33588a6689c4421f19193cc88ebf06ba3e2571c1197124b01a1ad56cf7ebeafd1801464b37fa6cbfa762b9de53292389bd6e320052454a5c0806e7c4ab93015819b121233f5f2aadfa8abfd692904ef48d5c60c9275cdedf98ad5fa2c75810288951782fde76131062dd60871ce22dbc4c344a1ae3ef5f84d29e84b000800000500000000020040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1428 + }, + { + "index": 3, + "entry_type": "precert", + "category": "precert_standard", + "leaf_hex": "00000198cf862bcd0001e37689003073a0c649cc656de946c03174d25c566fe3c3805b846f52369437980003233082031fa003020102020900a1cea161fc07a476300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236313131313131323831345a170d3236313131323131323831345a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100e2073b62ecb99fa6c34ac8001fd72f85aaa5c65d48e913e89c037800ec8149c680d52b5488fe5675910ed29c0d80e0af8a413c1f6bec560b54d0cf82374dece92eda20ff31b5ae30730ab34b1f4c20846ff22e6d85ad2148c10e17be47ca0194335d69153857e48d702341dc0fd66ea8dfa89956ab0268ef318bb7961ab05002a91226142e09dc4a45d8dedc4d7de1ce6c29c54dcd5c46774ce7478b259d1a8d39e12490395ed27a532af145fedcb4a8e4ec702a4aca4c3c6a1742babe3ea7840719208e58cdaad819acc1912b30657f3965402e5e1b3421b9d97b20bd830549375f84969d4bf748cfa860607de73583e48948b713e331b79b4ab65ed10789670203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130322e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d000800000500000000030005503082054c30820334a003020102020900a1cea161fc07a476300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236313131313131323831345a170d3236313131323131323831345a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100e2073b62ecb99fa6c34ac8001fd72f85aaa5c65d48e913e89c037800ec8149c680d52b5488fe5675910ed29c0d80e0af8a413c1f6bec560b54d0cf82374dece92eda20ff31b5ae30730ab34b1f4c20846ff22e6d85ad2148c10e17be47ca0194335d69153857e48d702341dc0fd66ea8dfa89956ab0268ef318bb7961ab05002a91226142e09dc4a45d8dedc4d7de1ce6c29c54dcd5c46774ce7478b259d1a8d39e12490395ed27a532af145fedcb4a8e4ec702a4aca4c3c6a1742babe3ea7840719208e58cdaad819acc1912b30657f3965402e5e1b3421b9d97b20bd830549375f84969d4bf748cfa860607de73583e48948b713e331b79b4ab65ed10789670203010001a381d43081d1300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130322e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d3013060a2b06010401d6790204030101ff04020500300d06092a864886f70d01010b05000382020100202e5282d28c4feec470592b0b65ec12e84126b6a0b0932b4230198fb5778fbfe0453bca4499106302f22650f6d8b3b53893b6bfda7a02294a5c780ee671bad9dad166fc560c539e2e1d1dfa9a07dd53addab42318a009ed56e938d93705dc2fee666b9bd1404df3cff362931de4ab5c9976889b80bf521de8ffe7c2ad5eb3937e9091e0dc534f902669001c599524d9ab432f3fe87ceec8efbc9202e2f991b14c644498555d46b759317495d37d690ab299a55f21f97e45578ab8dcc8aa906ba0337450b70b89c73dc7c6fcb8d39a22ce8930a7adab998cf52ac2478dd5682e0a4903e78496ce3003af500be28c458bcb01d75c442ea3747cd7082ae68b0e6d287f8c39d68704772d51718776bda81e17851f8ff3c96f2b15cd53ddf1ed7eeae2a95d61a76c08412b4461c30b25845727f6b81e6e84da6013f3968aef4a27248ca10edf4c5c2158953d54bd94b0031d187decc2060e11816a5c9223a6ba2d919f8164587dcd460b17a38df056d06c3ad13738dc01e48634ee106cde7ba1b10f0dfb6616ef59a8da3877feddedef582b1547a1bd7288b0b217ea820612eaeb098ccbb5ba92e329d4f1e07f1d8244fbd6102d82c98d2b6960161356a973dfa006f1a4a2d5a9158336286a25d8679774fe6a6a6ef61c4cdf8568eb7e144514a4c1f7f0a00f650cf1b1aec7fe9f7226d543b80ec7ca41d556129db32f47b142b5620040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 2287 + }, + { + "index": 4, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198cff1baf5000000053a308205363082031ea00302010202080f8bc9ef65c79ac3300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236313230333134343232325a170d3236313230343134343232325a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100dfa6217ce1bd8cb2ba23c943b6e0e486f96965a1bb1f4b5386ccf27c703c8f396feef9189f3dad7fe87de5cb624bc45aac6375b8ff19b6a4e76089a3d7b66d6e10a31ab53848c07e79b6b7ca1011c93372c6ab82d8695d459bd14e0951b621c6d5df456f964fdc894d0bfad0f638adc8bed83bef85b537abea354825bc13f39243d8e7285f3bd1b7eae8a3bc799fb61ed981e8af3ceef8a6fb9d37da2deecff249f7ac141c57e2e8d30f2eff55c32ac268b6657537daae803567f3d795c422ea6bd2c9b9813d231f6d36fffff74f06e85d0c52980b41294354010f05c873659e0c29217870237b7471a2f5a6a3861ef22c00cc49c19c58b9c052ad8f4f6b29fb0203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130342e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b050003820201005707691507bad31268e82c7a1b2ddf7061c63f3c3fd11c4dad897c38dc62f257b0e3e4aa9e1799634bcb0bf33d31218538f68ed7faccb04c527e5b13e2af6c7783e39e637b0d98981428902de3205f3657874a2c943961eff01b3ba1ca958b4f98d76ca902461e99fee37d6757cbc95d65225c4d27ca1a81d0144926818aed7ca0034147c94b9d167d8b87e280faee6322a3991cdb06a0911fedc140ab450f4687d5afd20d5702818df503e1b1c2b45d99aea51f9ebc26d44c772616fec67a53c0114cb17b0a09b7fa42287828fec123b03f7fbed857975bde0fd7a8178ddaccd0bed9f5e0d13a2764344f3176023b6ba64d56b3a6334a2ca0e986321e0db05d5c4336f78cea86c7755eb16b6e85f0c22fe99c0701e17a486fe131ab1e5afc8e6df0f6ba73498f5825c615a49683e006f7c2be434264e5faf726ff0319ac8da00030c9b5b399859c4640f8a623e267e9c33df6676e9202f079d3ec114056226578bfa708a264f10dca744c30952a673aea416f5f223efca77c2f60dbb9b4c1d8827ec3e646eb6876bab0d621fea0f0ef6f7d0a39b8ecc435e7388ad50bde3e3b0202e0e5d56bf08409ce4f90515864d581881541707a05b2f11b1b0bcd200be702dfb4b5768e9bd5d26f6d2a7a31fdb336cc897b761b8a7da94a91fede72f452b5f22f0ced204b5d2f16aa898ed976b68a1cea4f759bc45706f8c73d19a15881000800000500000000040040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1427 + }, + { + "index": 6, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198d05f97f5000000053b308205373082031fa003020102020900b9fc40043b489b8a300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236303831353132313231305a170d3236303831363132313231305a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a02820101009fbb9c7d3543c3d62fbf6d2cbd57f4a3dca3e542fe96094f6b1d04bc9172b4db369e2d4030bfbf61d20b32f87cd4c9f2f5082de107a549d1f16fcec75310a3b9a2d4fa11bb2e24e8c6b81b7ba20ea5414d847cb8d1f0cc78bc2048a73860ac3565317b8ee3cdc29bb30d90661018acf58bb048480a9e8fd976d46dd60a4d29164208d22a0a4a2006a124baad6348aa83b79706c32f1fcb7b61d6446245b7eddd3cc6567797ff8efb8619d7027a04d8e09592037cf66c5217afa906255b4118e1cb28e8c09afb6c33c9e81243cd2b21ae4f9383d514a0a1871d86ed9e4cc34690ce1ab2df6b316c1421d9311168b5aedbe950a3f97d19dd615ef9fa8d840a63750203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130362e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b0500038202010001284958faa59e556e73e86aa8342c1ed735c257976e94f7107b6035f89e6f7f160e484ed3fe473d8c811504996e25f61b3030a820887b1414417f82a18a73c0375267054c2446ee85783dab325ed700d28bdfaf795f43b70c755643e9de396b77956bcffcb8e69bb9f837ff972502b753ff9b3d57c12386d66a8bb1d5e6cb5a765dc455e4967115f22f81a89d708e30b555b7c9ea6e0dfccc01fe0106e191593745ccb8fb1e007683924e5a755f2121b26cfb7ec053397593e9f1e7e7014cc722fdddaebb3424ee1318afbaf628b0eca7c7115586be9bc06e885fd3a7e3491bafd2cfe7f7f55cd55232cf1bbf0257bd15ac6a1ee4b83baf03b382a4628d70e99cf79f3e19ff4415bb14a7ad57bf3b99cfc920fa81e5ab14094dbee838d1802417362ca0265694d90d8bfecc2c14d3fd6d86777411b5ad6330e5d87f57ff989e9325b1a5f036094485cb5b3f65d9baa14997e51580d5ce291b0dae46efbcb719c035287e2844056b77f4d390f84651a29b09897ed1f288620a3ac4053bf70692dd08ec04ffc1c56121f82bbd097127102de667a9b2b70ab54b16f78852744815f5035f162b49b8c256d899cb2bcbb04b0c6838d50d802176b50cba7530c930bdc8d87f0c820820d5e8919bdff1e37bdf12b855e0c67493aa814debb7ec1808cdf1dcee0d6f129231fe7929f3df5a334e3e69b6cebea27a532846c2b6d3d7ba6c000800000500000000060040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1428 + }, + { + "index": 8, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198d0cd74f5000000053b308205373082031fa003020102020900b1367c8e5abd2d75300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236303632383039313832345a170d3236303632393039313832345a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100d8e133d1e1eac3f9090f4250a7eebc83967e885edce4c080db42af76032dd674a755861776759a9c0a1b3693f43e80313080817d1765421ad0a45ebad0cd00e7cb89bfbd165594fe797034e85843c612fe7aa8a783e9b946481d38d6f7eabf808cad1ce0a61f1147539204f06a65850a22b157a75bef2dc1952b4d9f21f3fa3ac025f362fbd350954e1a8b4c1fd61ef16f74bb5c5947bd8e484a8f51d9b245a5a638e3af69fa5558f309207d5ee48899f1ce021062d60d331dc2c9c102b75b0e1ca300965f3eb8b493b490dbf4815a2dfc0765bdf6848d3714091429c888a30b7123c932ebb759191eb340406c9f1f079eac706635c9a311071ca93f54e3357d0203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824130382e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b05000382020100711cac7dbb578dd4b1b99d5039c93127a6405026a3686abbef55392620b9286dc593fd4461a355b041f6f051872da30bd2e7ffbb4c77ec9779cdf099577e597dffa9850f137afa7220138fa658cc53e589ac27d9ed30c3c849888a3fb75908aa42b259c75e15acd19c597ab248273247919ee225165a683b7c035d63d0211f5567a0b003ef9413528f9822ec41d006845fa31f4b6b471ec11e26695c8b7c92999bceddfda51523f5431fd8905dd84409880c42fd7ca8ffc155889157675e9c7f557c83ba9c8f8c6bb35678c7b0ef9ee75eb56bd946751cb14f481575e45fabcf3bd9d6a883a64986a19e41eaa9d61a2f4e486c6e00d15e93b3ff458a6249ecb89636fb2573081ccf103db1563e43dc286b13ad1edc9d349bb9a5e2c7ee61582e6f1670e946772d74179f6207d98312ea626786754f0589aa9f2b6e414668cbf5b35a1ee75fa3c353061287827fa2887a89928b3923937bf13dcb3405f0fd42bba388c71749950381733a0f8361ea82c58946b27bb045fc5d160aab0fc59f49ce8842152fc37eb220bd5a62b220c28dd97cef23fba19c7ec741d4095095768fcb446178076e92995d315d277f4f291e59e977b683a2b2a5eae38b775244d91e6ad3bf567e82493221dd32c73427ec8d01e64e5c937c7863d0d3bd2f0de19aa3f62a30f9df4caa316f657bd2a69d5810b02874b5cc66649b038d9e5ef56a312910000800000500000000080040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1428 + }, + { + "index": 10, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198d13b51f5000000053a308205363082031ea00302010202082948a4a859ee2e1f300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236313130393034343831355a170d3236313131303034343831355a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100bbfcc8679bd4f58c5b8f0bf82b9fd34c076c59646ba7febca13436caa2214604e81986dec43e410581e7494a686018b561362f4c0e7f2ed77e514c9d276508b181d433a544633e602835298ccf88a8fa6c8836292965da1bfe19c4b3a785f6b6cc48387fbdf7a4a14cfcefd15c6b7f90a903c52c07bb17a2a921ed5847b7f34a5503651a34512ec9fffae75d6fd6c67e9fd30369d9f759119bf3d6f85f24e2e707ab28231d10a6bd95b39cfd0c27efb6a1679f18d43a594c710eac91fc74c204f3a83e34c8d65fd9c77b94b330bad5be2cf8de36a706745c7a149833c204ff490ea03f7d4c112eb17957e4994513019171d459a95aa1c7701d49d5d987984f8b0203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824131302e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b05000382020100705e3b6495cdf281424f81b3f5b00e1149b3842c729fc5a24fe451a482da36b28d5fce1e7ea06c07855b51a41633a217b40df89ae8ce775dcc5e0254e82d503bdf16d1cfde29a15bcc8ffd4d3c10321c08b2fd118719cb8ce32a819bc5e45a0838e618d7e138a3cab12898b0602ff24b7fd6bf72e9aed0e73321b9e640810855a11d1df6cd1aacf876572cb5a5ca712f39f5d0d8c93b6af51281d83235ed9db4d153c18a5aec4592b88e288cfc4790cae1c114cd797456179f4905bf2652f0545fcca5b4fbd6e1b2669dd4f83aa4c26e3ad003ce69e86ed2be204012ff7deac164f9921a1654651690f12b4a1d56902863b89ac25af5760ecb73b34aa821fa1cb6a205e60fc8bd714a88b9c27cc0b03fc141d110b13ebc920a03941407006697d5cacf2dec0331633f725822cf1fdb234166eb2e7bec840075f573d3b35d6c0e4c99b746a143e0d6d3eb0a849153bc872d6afe3e20b291e4594412ab27a0ce4e4aa133bc477138a17ff12e9b9ead3a0ea90c836b9038efb3269f6ad0fc9a8a71281742187704d4ceba52a405fc5c21715311b8d2bd5d34e6b97259dcc1094c4bd55c613759e4538b4d6169adafa09dee92472652f52f5e2d2e2385f2748ffb4d2694e36965fc9eb26d36b2fb75720b87c5804627f6ff00c0360167b0ef8a904699bcf2582ddb5ab9280764c716efa1d1a952170e5bf7878b038a5337af7313fa0008000005000000000a0040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1427 + }, + { + "index": 12, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198d1a92ef5000000053b308205373082031fa003020102020900a6eb738e991068c0300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236303731333131323234305a170d3236303731343131323234305a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100c5a1a3b65fe47dbb9098b9d26749f33d4506af4bb209d7dc4ca4f8a45ab963c5d7e6823c4d1861fb0d71628dad2d232f6d9fef53b3b5bd75ecc9ee6ddad2ad430866495b710b0931da05affacf66be1e28a3bab998d3bb7dec3aca3c42856509065a7acc2641d9715267587098ee095f542a0a2a6bec0f2b8d443bf4c07a7d89ba5a04f963b583203def13808e9083af1ebd86de8c3f1e01428d70c931b92a0a06ae12450d698c507f53c0dd735d037fbe5c8ade64a60016165144c5e4433567d4d667c316c5852324bcb46b352b21066cb55e24bd2859d20b4892276542d4090b22c47bcb65eebfe269708de24fbeb3d62cb62b393f5a3b0c4a42ef26e3242b0203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824131322e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b050003820201004f197d6d6ade9cab9ad5890d55bcc9c5365ea0310c8ed8421defeb8d77bce5f8f7656a8dcb2ef8e52eff546bfb0ce4aa2add1a1c672995f3dc55051b053694518ee98d3100388727135a9b602b67995ae2dc50f463e64e5592058637e7ccff4cac18dd18ab20878e2f4ceb748175584c4d63431f345a76985d083f5a30f9ddedb802647352ff0a98878636a5887e24e8ee0f8230a05832830295167444472cbc2e9b3e5d6a5c467ce2906e639ba94b686546a37d6e87598433d24d4c10765fcf948f883a2e4cea496bc32337b26f4b18a371e0b31a25d5ca68e68aff9deca242eaeb5e62eba61420afdb673a1827956e24b123bd3fcf32ef14d0dae02102f5ecfa0d7f65ef48d9e20b865e4fa1888f48307d3385a0e241d2dba8fe4562896a5288a35942112b3bfebc43de62db651df8ee1fe61099f44b3a784ab7b49eacb9c156642567b76965b0029191e552b5686a2593886afd1e0c425aab4bdb5e13c1fe8905ad1dbdc526f8ea1b28972e5adb71d79a6727f104170b17540fba4bd32203090ed6a3e3fdd2574365b688e24ec9edf284b5adbd78a01f03214ad8e09be781f6f847234ce7df96a9abbb9ee62b41880b72444ca364f16fe58dc7219c5c3003064eed3d67e9b94c5506e45c89bb4b0be48339b5feeea03f3b54366954c621634887dc47d9807f810b6e37301d840278c3ecfc4a23f1d760d3d812d92c405fc80008000005000000000c0040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1428 + }, + { + "index": 14, + "entry_type": "x509", + "category": "x509_standard", + "leaf_hex": "00000198d2170fdd000000053a308205363082031ea00302010202085fda6b8eb19e8218300d06092a864886f70d01010b0500307f310b3009060355040613024742310f300d06035504080c064c6f6e646f6e31173015060355040a0c0e476f6f676c6520554b204c74642e3121301f060355040b0c184365727469666963617465205472616e73706172656e63793123302106035504030c1a4d657267652044656c617920496e7465726d6564696174652031301e170d3236303830393138343432385a170d3236303831303138343432385a3075310b3009060355040613024742310f300d060355040713064c6f6e646f6e310f300d060355040a1306476f6f676c653121301f060355040b13184365727469666963617465205472616e73706172656e63793121301f06035504031318666c6f776572732d746f2d7468652d776f726c642e636f6d30820122300d06092a864886f70d01010105000382010f003082010a0282010100be2130c183baaa1fabc1d9159e775b409212beeb3983c7004264da6d0396efd1267e0745e8cfd83e7d5ba15b24ae2fa58c40a00c0e8332f5c512b38ea4b13e1a7f1636858b1970f221db2e6fa219e0f0ef4fdbac51f9664590fb06bf7cd9c58dbd971f7c1a6f580ec1ca66252e12de707140c4388ca517c01b056a5e65a674ec38cf616efdcb1a1241876806849196b6c1a69fa9d272001c1e1732f0a531b25b2115ee48a1f69234546a609b4141f6a3f6da45cf72cf5709c33bbd73bcf76a1dc6d768b6ed9c28d7d6d5cba804ee3995b11078e9d8f9f631d4456a4b65a1862e657b3f431aa2e25bc4b8076860faa7875aeca85db06c6cef4ec31a20650203e90203010001a381bf3081bc300e0603551d0f0101ff04040302078030130603551d25040c300a06082b06010505070301300c0603551d130101ff04023000301f0603551d23041830168014e93c04e1802fc284132d26709ef2fd1acfaafec630660603551d11045f305d8218666c6f776572732d746f2d7468652d776f726c642e636f6d824131342e32322e30382e323032352e6c657473656e63727970745f737963616d6f72653230323668322e666c6f776572732d746f2d7468652d776f726c642e636f6d300d06092a864886f70d01010b05000382020100aa8080676855ec9ef02866bf6069797b0510ce6bad434dc9062d477e763153493c4e37cc06d339a90ad091dd3ce7944a70f13e9c124ac94f25ee188f5e12123e4772e2579cc2a2afa09372c130dc04ad232d730ca6175ca53384889414f6b901266160d9e2688941b707f42821e01710623ec966f0f44209147354c10e9434da00914793528bd4ae78091e477b9ea6b902006811da05e6fa196febc561dfbe284c63d0cce80e626de1022b640bc636d05f69e9ae4abcb2e05460877a3748f133cfa649c3b4d2cb62062ccc53f8d7bfd09da0d39c7612b2fa000429829994728df3fbc0f1398f7a6c4318d4f9b3bf61c723afd7776a648b47ed27424dc47f927d7728f82a89317e4ea199e1eec67004b393781d8c9500d1ca383a8c215d6f60b2bd4b5c8f7d514e9187ec48a2ed03a944bd55875cf5f63eb1735e2c5693e0c47bc5ac318705ad580b43a1cd8bf302fd4d9d7bb9137dc3301e968174671c72fdab5c986f85a9b1cb391a32d5f5f1783173e68c0f54f393d22cec7a93406aa93ec2d777ff4eb79cbedb069f545e36851c92f5bcc75b561433ced6c74f0c9b361e08911498587f8392903f9e9b132d8a3b9ccf7f74fc0292a31818080bbfd157118d73ad20e86198b0d25c69c2d604a5637f5b5d02a43a21c36c47160ce5325f82f9527455ae77fa9e86413d6d6f18ba2554790f75bbd175e9e10c31f1fc5fb2fcf40008000005000000000e0040adb4a7e96552b132901fa13917b030bb8e0f9afe391416c4abd6598a63d0992586d8219c7e2b6009e37eb14356268489b81379e076e8f372e3dde8c162a34134", + "byte_length": 1427 + } +] \ No newline at end of file diff --git a/vindex/mapfn/examples/starter/main.go b/vindex/mapfn/examples/starter/main.go new file mode 100644 index 00000000..057e1aef --- /dev/null +++ b/vindex/mapfn/examples/starter/main.go @@ -0,0 +1,69 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate sh -c "GOOS=wasip1 GOARCH=wasm CGO_ENABLED=0 go build -trimpath -ldflags=\"-buildid=\" -buildmode=c-shared -o starter.wasm ." + +// Package main demonstrates a minimal starter template for authoring a VIndex WASM MapFn. +package main + +import ( + "bytes" + + "github.com/transparency-dev/incubator/vindex/mapfn/sdk" +) + +func main() {} + +func init() { + // Register the leaf mapping function using the zero-allocation streaming callback. + sdk.RegisterEmit(MapLeaf) +} + +// MapLeaf inspects a single log leaf and extracts one or more canonical search key preimages. +// Canonical keys must be 1..1024 printable ASCII bytes in 0x21..0x7E (no NUL or whitespace). +func MapLeaf(leaf []byte, emit func(key []byte)) { + leaf = bytes.TrimSpace(leaf) + if len(leaf) == 0 { + return + } + + var buf [1024]byte + n := 0 + for _, b := range leaf { + if n >= len(buf) { + break + } + if b == ' ' || b == '\t' || b == '\n' || b == '\r' { + if n > 0 && buf[n-1] != '_' { + buf[n] = '_' + n++ + } + continue + } + if b >= 0x21 && b <= 0x7E { + buf[n] = b + n++ + } + } + for n > 0 && buf[n-1] == '_' { + n-- + } + if n == 0 { + return + } + + // Example: emit the normalized leaf payload as the search key preimage. + // Replace this logic with your custom leaf parsing / key extraction. + emit(buf[:n]) +} diff --git a/vindex/mapfn/examples/starter/starter_test.go b/vindex/mapfn/examples/starter/starter_test.go new file mode 100644 index 00000000..6679b4d0 --- /dev/null +++ b/vindex/mapfn/examples/starter/starter_test.go @@ -0,0 +1,58 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "testing" +) + +func TestMapLeaf(t *testing.T) { + input := []byte("sample leaf entry") + var keys [][]byte + MapLeaf(input, func(key []byte) { + for i, b := range key { + if b < 0x21 || b > 0x7E { + t.Fatalf("emitted non-canonical byte 0x%02x at offset %d", b, i) + } + } + keys = append(keys, append([]byte(nil), key...)) + }) + if len(keys) != 1 { + t.Fatalf("expected 1 key, got %d", len(keys)) + } + want := []byte("sample_leaf_entry") + if !bytes.Equal(keys[0], want) { + t.Fatalf("expected key %q, got %q", string(want), string(keys[0])) + } +} + +func TestMapLeaf_Empty(t *testing.T) { + var count int + emitCount := func(key []byte) { count++ } + + count = 0 + MapLeaf(nil, emitCount) + if count != 0 { + t.Fatalf("expected 0 keys for nil input, got %d", count) + } + + count = 0 + MapLeaf([]byte(" \n\t "), emitCount) + if count != 0 { + t.Fatalf("expected 0 keys for whitespace input, got %d", count) + } +} + diff --git a/vindex/mapfn/examples/sumdb/main.go b/vindex/mapfn/examples/sumdb/main.go new file mode 100644 index 00000000..bc5166fb --- /dev/null +++ b/vindex/mapfn/examples/sumdb/main.go @@ -0,0 +1,109 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:generate sh -c "GOOS=wasip1 GOARCH=wasm CGO_ENABLED=0 go build -trimpath -ldflags=\"-buildid=\" -buildmode=c-shared -o sumdb.wasm ." + +// Package main implements the SumDB WASM MapFn plugin. +package main + +import ( + "bytes" + "unsafe" + + "github.com/transparency-dev/incubator/vindex/mapfn/sdk" + "golang.org/x/mod/module" + "golang.org/x/mod/semver" +) + +func main() {} + +func init() { + sdk.RegisterEmit(MapSumDBLeaf) +} + +var goModSuffix = []byte("/go.mod") + +// MapSumDBLeaf parses a Go SumDB log leaf, filtering pseudo-versions and streaming canonical module path preimages. +func MapSumDBLeaf(data []byte, emit func(key []byte)) { + var emitted [8][]byte + n := 0 + var overflow [][]byte + + for len(data) > 0 { + var line []byte + idx := bytes.IndexByte(data, '\n') + if idx >= 0 { + line = data[:idx] + data = data[idx+1:] + } else { + line = data + data = nil + } + line = bytes.TrimSpace(line) + if len(line) == 0 { + continue + } + + modEnd := bytes.IndexByte(line, ' ') + if modEnd <= 0 { + continue + } + rawModPath := line[:modEnd] + + verStart := modEnd + 1 + if verStart >= len(line) { + continue + } + verLen := bytes.IndexByte(line[verStart:], ' ') + if verLen <= 0 || verStart+verLen+1 >= len(line) { + continue + } + verBytes := bytes.TrimSuffix(line[verStart:verStart+verLen], goModSuffix) + verStr := unsafe.String(unsafe.SliceData(verBytes), len(verBytes)) + if !semver.IsValid(verStr) || module.IsPseudoVersion(verStr) { + continue + } + + escaped, err := module.EscapePath(unsafe.String(unsafe.SliceData(rawModPath), len(rawModPath))) + if err != nil || len(escaped) == 0 || len(escaped) > 1024 { + continue + } + modPath := unsafe.Slice(unsafe.StringData(escaped), len(escaped)) + + duplicate := false + for i := 0; i < n; i++ { + if bytes.Equal(emitted[i], modPath) { + duplicate = true + break + } + } + if !duplicate { + for _, ov := range overflow { + if bytes.Equal(ov, modPath) { + duplicate = true + break + } + } + } + if !duplicate { + if n < len(emitted) { + emitted[n] = modPath + n++ + } else { + overflow = append(overflow, modPath) + } + emit(modPath) + } + } +} diff --git a/vindex/mapfn/examples/sumdb/sumdb_test.go b/vindex/mapfn/examples/sumdb/sumdb_test.go new file mode 100644 index 00000000..ba3d4765 --- /dev/null +++ b/vindex/mapfn/examples/sumdb/sumdb_test.go @@ -0,0 +1,142 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "testing" +) + +func TestMapSumDBLeaf_StandardRelease(t *testing.T) { + leaf := []byte("golang.org/x/mod v0.40.0 h1:testHash123=\ngolang.org/x/mod v0.40.0/go.mod h1:testHash456=\n") + var keys [][]byte + MapSumDBLeaf(leaf, func(key []byte) { + keys = append(keys, key) + }) + + if len(keys) != 1 { + t.Fatalf("expected 1 key, got %d", len(keys)) + } + + if string(keys[0]) != "golang.org/x/mod" { + t.Fatalf("key mismatch: got %q, want %q", string(keys[0]), "golang.org/x/mod") + } +} + +func TestMapSumDBLeaf_PseudoVersionFiltered(t *testing.T) { + leaf := []byte( + "github.com/example/repo v0.0.0-20230101000000-0123456789ab h1:testHash=\n" + + "github.com/example/repo2 v1.2.4-0.20190528180746-1234567890ab+incompatible h1:testHash=\n" + + "github.com/example/repo3 v1.2.3-pre.0.20190528180746-1234567890ab h1:testHash=\n", + ) + var keys [][]byte + MapSumDBLeaf(leaf, func(key []byte) { + keys = append(keys, key) + }) + + if len(keys) != 0 { + t.Fatalf("expected 0 keys for pseudo-versions, got %d", len(keys)) + } +} + +func TestMapSumDBLeaf_MultipleModulesInSingleLeaf(t *testing.T) { + leaf := []byte("github.com/mod1 v1.0.0 h1:h1=\ngithub.com/mod2/v2 v2.1.0 h1:h2=\n") + var keys [][]byte + MapSumDBLeaf(leaf, func(key []byte) { + keys = append(keys, key) + }) + + if len(keys) != 2 { + t.Fatalf("expected 2 keys, got %d", len(keys)) + } + + if string(keys[0]) != "github.com/mod1" || string(keys[1]) != "github.com/mod2/v2" { + t.Fatalf("keys mismatch: got %q, %q", string(keys[0]), string(keys[1])) + } +} + +func TestMapSumDBLeaf_MoreThan8Modules(t *testing.T) { + var buf []byte + for i := 0; i < 12; i++ { + buf = append(buf, fmt.Sprintf("github.com/mod%d v1.0.0 h1:h%d=\n", i, i)...) + } + var keys [][]byte + MapSumDBLeaf(buf, func(key []byte) { + keys = append(keys, key) + }) + + if len(keys) != 12 { + t.Fatalf("expected 12 keys, got %d", len(keys)) + } + for i := 0; i < 12; i++ { + want := fmt.Sprintf("github.com/mod%d", i) + if string(keys[i]) != want { + t.Errorf("keys[%d] = %q, want %q", i, string(keys[i]), want) + } + } +} + +func TestMapSumDBLeaf_EmptyOrMalformed(t *testing.T) { + testCases := [][]byte{ + nil, + []byte(""), + []byte(" \n\n "), + []byte("invalidline_without_spaces"), + []byte("golang.org/x/mod v0.40.0"), // missing third (hash) field + []byte("golang.org/x/mod not-a-version h1:abc="), // invalid semver + []byte("github.com/foo/bar/v1 v1.0.0 h1:abc="), // disallowed /v1 suffix in module path + []byte("example.com/con v1.0.0 h1:abc="), // disallowed Windows reserved path element + } + + for _, tc := range testCases { + var res [][]byte + MapSumDBLeaf(tc, func(key []byte) { + res = append(res, key) + }) + if len(res) != 0 { + t.Fatalf("expected 0 results for input %q, got %d (%q)", string(tc), len(res), res) + } + } +} + +func TestMapSumDBLeaf_EscapePathAndValidation(t *testing.T) { + leaf := []byte( + "github.com/Azure/azure-sdk-for-go v1.0.0 h1:abc=\n" + + "github.com/Azure/azure-sdk-for-go v1.0.0/go.mod h1:def=\n" + + "invalid_no_dot_module v1.0.0 h1:bad=\n", + ) + var keys []string + MapSumDBLeaf(leaf, func(key []byte) { + keys = append(keys, string(key)) + }) + + if len(keys) != 1 { + t.Fatalf("expected 1 key, got %d (%v)", len(keys), keys) + } + want := "github.com/!azure/azure-sdk-for-go" + if keys[0] != want { + t.Errorf("got %q, want %q", keys[0], want) + } +} + +func BenchmarkMapSumDBLeaf(b *testing.B) { + leaf := []byte("golang.org/x/mod v0.40.0 h1:testHash123=\ngolang.org/x/mod v0.40.0/go.mod h1:testHash456=\n") + b.ReportAllocs() + + emitSink := func(key []byte) {} + for b.Loop() { + MapSumDBLeaf(leaf, emitSink) + } +} diff --git a/vindex/mapfn/sdk/arena.go b/vindex/mapfn/sdk/arena.go new file mode 100644 index 00000000..37ef81d5 --- /dev/null +++ b/vindex/mapfn/sdk/arena.go @@ -0,0 +1,124 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "encoding/binary" + "unsafe" +) + +const ( + // maxBufferSize is the static arena buffer size (4MB to accommodate 256-leaf bundles). + maxBufferSize = 4 * 1024 * 1024 +) + +var ( + inputBuf [maxBufferSize]byte + outputBuf [maxBufferSize]byte + emitOutOffset int + emitKeyCount int + emitOverflow bool +) + +func emitKey(k []byte) { + kLen := len(k) + if emitOverflow || emitOutOffset+4+kLen > maxBufferSize { + emitOverflow = true + return + } + binary.LittleEndian.PutUint32(outputBuf[emitOutOffset:emitOutOffset+4], uint32(kLen)) + copy(outputBuf[emitOutOffset+4:emitOutOffset+4+kLen], k) + emitOutOffset += 4 + kLen + emitKeyCount++ +} + +func getInputSlice(ptr, length uint32) []byte { + if length == 0 || length > maxBufferSize { + return nil + } + base := uint32(uintptr(unsafe.Pointer(&inputBuf[0]))) + if ptr < base || ptr-base > maxBufferSize-length { + return nil + } + offset := ptr - base + return inputBuf[offset : offset+length] +} + +func inputBufferPtr() uint32 { + return uint32(uintptr(unsafe.Pointer(&inputBuf[0]))) +} + +func executeBundle(inPtr, inLen uint32) (uint32, uint32) { + emitOverflow = false + if inLen < 8 { + return 0, 0 + } + + inSlice := getInputSlice(inPtr, inLen) + if len(inSlice) < 8 { + return 0, 0 + } + + leafCount := binary.LittleEndian.Uint32(inSlice[0:4]) + if leafCount == 0 || leafCount > 256 { + return 0, 0 + } + + headerLen := 4 + (leafCount+1)*4 + if uint32(len(inSlice)) < headerLen { + return 0, 0 + } + + offsetTable := inSlice[4:headerLen] + payload := inSlice[headerLen:] + + if binary.LittleEndian.Uint32(offsetTable[0:4]) != 0 { + return 0, 0 + } + if binary.LittleEndian.Uint32(offsetTable[leafCount*4:(leafCount+1)*4]) != uint32(len(payload)) { + return 0, 0 + } + + outBase := uint32(uintptr(unsafe.Pointer(&outputBuf[0]))) + binary.LittleEndian.PutUint32(outputBuf[0:4], leafCount) + + keyCountsOffset := 4 + outOffset := 4 + int(leafCount)*4 + + for i := uint32(0); i < leafCount; i++ { + start := binary.LittleEndian.Uint32(offsetTable[i*4 : (i+1)*4]) + end := binary.LittleEndian.Uint32(offsetTable[(i+1)*4 : (i+2)*4]) + + if end < start || end > uint32(len(payload)) { + return 0, 0 + } + leafBytes := payload[start:end] + + if registeredEmitMapFunc != nil { + emitOutOffset = outOffset + emitKeyCount = 0 + registeredEmitMapFunc(leafBytes, emitKey) + if emitOverflow { + return 0, 0 + } + binary.LittleEndian.PutUint32(outputBuf[keyCountsOffset+int(i)*4:], uint32(emitKeyCount)) + outOffset = emitOutOffset + } else { + binary.LittleEndian.PutUint32(outputBuf[keyCountsOffset+int(i)*4:], 0) + } + } + + return outBase, uint32(outOffset) +} diff --git a/vindex/mapfn/sdk/sdk.go b/vindex/mapfn/sdk/sdk.go new file mode 100644 index 00000000..d6b5ba88 --- /dev/null +++ b/vindex/mapfn/sdk/sdk.go @@ -0,0 +1,43 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sdk provides the guest SDK for authoring VIndex WASM MapFn plugins. +package sdk + +// EmitMapFunc maps a raw input log leaf to search key preimages via an emitter callback. +type EmitMapFunc func(leaf []byte, emit func(key []byte)) + +var registeredEmitMapFunc EmitMapFunc + +// RegisterEmit registers a streaming plugin using an emitter callback. +func RegisterEmit(fn EmitMapFunc) { + registeredEmitMapFunc = fn +} + +//go:wasmexport map_bundle +func mapBundle(inPtr, inLen uint32) uint64 { + outPtr, outLen := executeBundle(inPtr, inLen) + return (uint64(outPtr) << 32) | uint64(outLen) +} + +//go:wasmexport input_buffer +func inputBuffer() uint32 { + return inputBufferPtr() +} + +// Suppress unused warnings when compiled on non-WASM architectures. +var ( + _ = mapBundle + _ = inputBuffer +) diff --git a/vindex/mapfn/sdk/sdk_test.go b/vindex/mapfn/sdk/sdk_test.go new file mode 100644 index 00000000..4118845c --- /dev/null +++ b/vindex/mapfn/sdk/sdk_test.go @@ -0,0 +1,217 @@ +// Copyright 2026 The Transparency Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sdk + +import ( + "encoding/binary" + "testing" +) + +func packBundleInput(leaves [][]byte) []byte { + n := len(leaves) + if n == 0 || n > 256 { + return nil + } + + headerLen := 4 + (n+1)*4 + var payloadLen int + for _, l := range leaves { + payloadLen += len(l) + } + + buf := make([]byte, headerLen+payloadLen) + binary.LittleEndian.PutUint32(buf[0:4], uint32(n)) + + var currentOffset uint32 + binary.LittleEndian.PutUint32(buf[4:8], 0) + payloadDst := buf[headerLen:] + var writePos int + + for i, l := range leaves { + copy(payloadDst[writePos:], l) + writePos += len(l) + currentOffset += uint32(len(l)) + binary.LittleEndian.PutUint32(buf[4+(i+1)*4:4+(i+2)*4], currentOffset) + } + + return buf +} + +func TestArena_InputBufferAndSliceBounds(t *testing.T) { + base := inputBufferPtr() + if base == 0 { + t.Fatal("expected non-zero inputBufferPtr pointer") + } + if got := inputBuffer(); got != base { + t.Fatalf("inputBuffer() = %d, want %d", got, base) + } + + // Test writing and reading via getInputSlice + slice := getInputSlice(base, 100) + if len(slice) != 100 { + t.Fatalf("expected slice len 100, got %d", len(slice)) + } + for i := range slice { + slice[i] = byte(i) + } + readSlice := getInputSlice(base, 100) + for i := range readSlice { + if readSlice[i] != byte(i) { + t.Fatalf("mismatch at byte %d: got %d, want %d", i, readSlice[i], byte(i)) + } + } + + // Bounds and uint32 wrap-around guards in getInputSlice + if s := getInputSlice(base, 0); s != nil { + t.Fatalf("expected nil for zero length, got len %d", len(s)) + } + if s := getInputSlice(base, maxBufferSize+1); s != nil { + t.Fatalf("expected nil for oversized length, got len %d", len(s)) + } + if s := getInputSlice(base+50, ^uint32(0)-10); s != nil { + t.Fatalf("expected nil for uint32 wrap-around length in getInputSlice, got len %d", len(s)) + } + if s := getInputSlice(base-1, 10); s != nil { + t.Fatalf("expected nil for ptr < base in getInputSlice, got len %d", len(s)) + } +} + +func TestPackBundleInput_And_ExecuteBundle_Emit(t *testing.T) { + RegisterEmit(func(leaf []byte, emit func(key []byte)) { + if string(leaf) == "skip" { + return + } + emit([]byte("key_" + string(leaf))) + emit([]byte("extra_" + string(leaf))) + }) + + leaves := [][]byte{ + []byte("leaf0"), + []byte("leaf1"), + []byte("skip"), + []byte("leaf3"), + } + + packedInput := packBundleInput(leaves) + if len(packedInput) == 0 { + t.Fatal("packBundleInput returned empty buffer") + } + + ptr := inputBufferPtr() + copy(getInputSlice(ptr, uint32(len(packedInput))), packedInput) + + outPtr, outLen := executeBundle(ptr, uint32(len(packedInput))) + if outPtr == 0 || outLen == 0 { + t.Fatalf("executeBundle failed: outPtr=%d outLen=%d", outPtr, outLen) + } + + outBuf := outputBuf[:outLen] + leafCount := binary.LittleEndian.Uint32(outBuf[0:4]) + if leafCount != 4 { + t.Fatalf("leafCount = %d, want 4", leafCount) + } + + k0 := binary.LittleEndian.Uint32(outBuf[4:8]) + k1 := binary.LittleEndian.Uint32(outBuf[8:12]) + k2 := binary.LittleEndian.Uint32(outBuf[12:16]) + k3 := binary.LittleEndian.Uint32(outBuf[16:20]) + + if k0 != 2 || k1 != 2 || k2 != 0 || k3 != 2 { + t.Fatalf("key counts mismatch: [%d, %d, %d, %d], want [2, 2, 0, 2]", k0, k1, k2, k3) + } + + // Verify mapBundle wasm export + ret := mapBundle(ptr, uint32(len(packedInput))) + if ret == 0 { + t.Fatal("mapBundle returned 0") + } +} + +func TestExecuteBundle_EmitSingle(t *testing.T) { + RegisterEmit(func(leaf []byte, emit func(key []byte)) { + emit([]byte("domain_" + string(leaf))) + }) + + leaves := [][]byte{[]byte("example.com")} + packedInput := packBundleInput(leaves) + + ptr := inputBufferPtr() + copy(getInputSlice(ptr, uint32(len(packedInput))), packedInput) + + outPtr, outLen := executeBundle(ptr, uint32(len(packedInput))) + if outPtr == 0 || outLen == 0 { + t.Fatalf("executeBundle failed: outPtr=%d outLen=%d", outPtr, outLen) + } + + outBuf := outputBuf[:outLen] + leafCount := binary.LittleEndian.Uint32(outBuf[0:4]) + if leafCount != 1 { + t.Fatalf("leafCount = %d, want 1", leafCount) + } + kCount := binary.LittleEndian.Uint32(outBuf[4:8]) + if kCount != 1 { + t.Fatalf("kCount = %d, want 1", kCount) + } + + keyLen := binary.LittleEndian.Uint32(outBuf[8:12]) + keyBytes := string(outBuf[12 : 12+keyLen]) + if keyBytes != "domain_example.com" { + t.Fatalf("key = %q, want %q", keyBytes, "domain_example.com") + } +} + +func TestExecuteBundle_OffsetTableValidation(t *testing.T) { + RegisterEmit(func(leaf []byte, emit func(key []byte)) { + emit(leaf) + }) + + leaves := [][]byte{[]byte("example.com")} + packedInput := packBundleInput(leaves) + ptr := inputBufferPtr() + + // 1. OffsetTable[0] != 0 must fail + badStart := append([]byte(nil), packedInput...) + binary.LittleEndian.PutUint32(badStart[4:8], 1) + copy(getInputSlice(ptr, uint32(len(badStart))), badStart) + if outPtr, outLen := executeBundle(ptr, uint32(len(badStart))); outPtr != 0 || outLen != 0 { + t.Fatalf("expected (0, 0) for non-zero OffsetTable[0], got (%d, %d)", outPtr, outLen) + } + + // 2. OffsetTable[leafCount] != len(payload) must fail + badEnd := append([]byte(nil), packedInput...) + binary.LittleEndian.PutUint32(badEnd[8:12], uint32(len(leaves[0])-1)) + copy(getInputSlice(ptr, uint32(len(badEnd))), badEnd) + if outPtr, outLen := executeBundle(ptr, uint32(len(badEnd))); outPtr != 0 || outLen != 0 { + t.Fatalf("expected (0, 0) for OffsetTable[leafCount] != len(payload), got (%d, %d)", outPtr, outLen) + } +} + +func TestExecuteBundle_EmitOverflow(t *testing.T) { + RegisterEmit(func(leaf []byte, emit func(key []byte)) { + bigKey := make([]byte, maxBufferSize) + emit(bigKey) + }) + + leaves := [][]byte{[]byte("leaf")} + packedInput := packBundleInput(leaves) + + ptr := inputBufferPtr() + copy(getInputSlice(ptr, uint32(len(packedInput))), packedInput) + + outPtr, outLen := executeBundle(ptr, uint32(len(packedInput))) + if outPtr != 0 || outLen != 0 { + t.Fatalf("expected (0, 0) on overflow, got (%d, %d)", outPtr, outLen) + } +} From 461db98c9317a3729ac82ca0e5352ce04638957f Mon Sep 17 00:00:00 2001 From: Martin Hutchinson Date: Fri, 6 Nov 2026 17:31:37 +0000 Subject: [PATCH 2/2] refactor(vindex/mapfn): simplify wasmhost, guest SDK, and vindex-wasm CLI - Drop unused github.com/google/go-cmp dependency in favor of stdlib slices.EqualFunc. - Move test fixture builder into internal/wasmhost/wasmtest so production wasmhost avoids os/exec and unused TestWASMHost. - Remove duplicate wasmhost tests, dead decodeBundleOutput wrapper, and duplicate benchmark; wire DefaultBundleSize across wasmhost and vindex-wasm. - Require --wasm uniformly across vindex-wasm subcommands instead of positional-arg heuristics. - Consolidate mapfn/sdk into a single file and compute KeyHash after key deduplication. --- go.mod | 1 - vindex/cmd/vindex-wasm/bench_cmd.go | 4 +- vindex/cmd/vindex-wasm/inspect_cmd.go | 6 +- vindex/cmd/vindex-wasm/main_test.go | 20 +-- vindex/cmd/vindex-wasm/test_cmd.go | 20 +-- vindex/internal/wasmhost/wasm.go | 26 +-- vindex/internal/wasmhost/wasm_test.go | 155 +++--------------- .../{ => wasmtest}/testdata/testguest/main.go | 0 .../wasmhost/{ => wasmtest}/wasmtest.go | 22 +-- vindex/mapfn/sdk/arena.go | 124 -------------- vindex/mapfn/sdk/sdk.go | 109 +++++++++++- vindex/mapfn/sdk/sdk_test.go | 15 +- 12 files changed, 156 insertions(+), 346 deletions(-) rename vindex/internal/wasmhost/{ => wasmtest}/testdata/testguest/main.go (100%) rename vindex/internal/wasmhost/{ => wasmtest}/wasmtest.go (75%) delete mode 100644 vindex/mapfn/sdk/arena.go diff --git a/go.mod b/go.mod index fac53ddf..c5e29206 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,6 @@ go 1.27.0 toolchain go1.27.3 require ( - github.com/google/go-cmp v0.7.0 github.com/prometheus/client_golang v1.25.0 github.com/tetratelabs/wazero v1.12.0 github.com/transparency-dev/formats v0.1.2-0.20261027211732-55fe9bf0fa53 diff --git a/vindex/cmd/vindex-wasm/bench_cmd.go b/vindex/cmd/vindex-wasm/bench_cmd.go index b8396bcf..7f19a5dc 100644 --- a/vindex/cmd/vindex-wasm/bench_cmd.go +++ b/vindex/cmd/vindex-wasm/bench_cmd.go @@ -32,7 +32,7 @@ func runBenchCmd(ctx context.Context, args []string) error { wasmPath := fs.String("wasm", "", "Path to compiled WASM plugin file (required).") inputStr := fs.String("input", "github.com/example/benchmodule v1.2.3 h1:benchHash123456789=\n", "Benchmark input leaf payload.") inputFile := fs.String("input_file", "", "Path to file containing input leaf payload.") - bundleSize := fs.Int("bundle_size", 256, "Number of leaves per bundle (defaults to 256).") + bundleSize := fs.Int("bundle_size", int(wasmhost.DefaultBundleSize), "Number of leaves per bundle (defaults to 256).") iterations := fs.Int("iterations", 1000, "Total number of bundle executions.") workers := fs.Int("workers", 4, "Number of concurrent worker goroutines and pool size.") @@ -47,7 +47,7 @@ func runBenchCmd(ctx context.Context, args []string) error { return errors.New("--iterations must be greater than 0") } if *bundleSize <= 0 { - *bundleSize = 256 + *bundleSize = int(wasmhost.DefaultBundleSize) } if *workers <= 0 { *workers = 1 diff --git a/vindex/cmd/vindex-wasm/inspect_cmd.go b/vindex/cmd/vindex-wasm/inspect_cmd.go index 94f95532..b71d1c5d 100644 --- a/vindex/cmd/vindex-wasm/inspect_cmd.go +++ b/vindex/cmd/vindex-wasm/inspect_cmd.go @@ -35,11 +35,7 @@ func runInspectCmd(ctx context.Context, args []string) error { } if *wasmPath == "" { - if fs.NArg() > 0 { - *wasmPath = fs.Arg(0) - } else { - return errors.New("--wasm flag is required") - } + return errors.New("--wasm flag is required") } wasmBytes, err := os.ReadFile(*wasmPath) diff --git a/vindex/cmd/vindex-wasm/main_test.go b/vindex/cmd/vindex-wasm/main_test.go index 5a643f7f..b65613b9 100644 --- a/vindex/cmd/vindex-wasm/main_test.go +++ b/vindex/cmd/vindex-wasm/main_test.go @@ -34,14 +34,14 @@ import ( "testing" "time" - "github.com/transparency-dev/incubator/vindex/internal/wasmhost" + "github.com/transparency-dev/incubator/vindex/internal/wasmhost/wasmtest" ) func TestCLI_TestCmd(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() wasmPath := filepath.Join(tmpDir, "test.wasm") - if err := os.WriteFile(wasmPath, wasmhost.TestWASMBytes(), 0o644); err != nil { + if err := os.WriteFile(wasmPath, wasmtest.Bytes(), 0o644); err != nil { t.Fatalf("failed to write wasm file: %v", err) } @@ -72,18 +72,12 @@ func TestCLI_TestCmd(t *testing.T) { if err != nil { t.Fatalf("test input_file command failed: %v", err) } - - // 5. Positional input - err = run(ctx, []string{"test", "--wasm=" + wasmPath, "positional_arg_leaf"}) - if err != nil { - t.Fatalf("test positional command failed: %v", err) - } } func TestCLI_InspectCmd(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() - wasmBytes := wasmhost.TestWASMBytes() + wasmBytes := wasmtest.Bytes() wasmPath := filepath.Join(tmpDir, "test.wasm") if err := os.WriteFile(wasmPath, wasmBytes, 0o644); err != nil { t.Fatalf("failed to write wasm file: %v", err) @@ -114,19 +108,13 @@ func TestCLI_InspectCmd(t *testing.T) { if !strings.Contains(outStr, wantHex) { t.Errorf("inspect output missing %q:\n%s", wantHex, outStr) } - - // Positional syntax - err = run(ctx, []string{"inspect", wasmPath}) - if err != nil { - t.Fatalf("inspect positional failed: %v", err) - } } func TestCLI_BenchCmd(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() wasmPath := filepath.Join(tmpDir, "test.wasm") - if err := os.WriteFile(wasmPath, wasmhost.TestWASMBytes(), 0o644); err != nil { + if err := os.WriteFile(wasmPath, wasmtest.Bytes(), 0o644); err != nil { t.Fatalf("failed to write wasm file: %v", err) } diff --git a/vindex/cmd/vindex-wasm/test_cmd.go b/vindex/cmd/vindex-wasm/test_cmd.go index 8cd9479b..d1a2bff5 100644 --- a/vindex/cmd/vindex-wasm/test_cmd.go +++ b/vindex/cmd/vindex-wasm/test_cmd.go @@ -24,7 +24,6 @@ import ( "io" "os" "slices" - "strings" "github.com/transparency-dev/incubator/vindex/internal/wasmhost" ) @@ -46,22 +45,18 @@ func runTestCmd(ctx context.Context, args []string) error { inputStr := fs.String("input", "", "Literal string input payload.") inputFile := fs.String("input_file", "", "Path to file containing input leaf payload.") inputHex := fs.String("hex", "", "Hex-encoded input leaf payload.") - bundleSize := fs.Int("bundle_size", 256, "Number of leaves per bundle execution (defaults to 256).") + bundleSize := fs.Int("bundle_size", int(wasmhost.DefaultBundleSize), "Number of leaves per bundle execution (defaults to 256).") format := fs.String("format", "plain", "Output format: plain or json.") if err := fs.Parse(args); err != nil { return err } if *bundleSize <= 0 { - *bundleSize = 256 + *bundleSize = int(wasmhost.DefaultBundleSize) } if *wasmPath == "" { - if fs.NArg() > 0 && (strings.HasSuffix(fs.Arg(0), ".wasm") || *inputStr != "") { - *wasmPath = fs.Arg(0) - } else { - return errors.New("--wasm flag is required") - } + return errors.New("--wasm flag is required") } wasmBytes, err := os.ReadFile(*wasmPath) @@ -90,15 +85,6 @@ func runTestCmd(ctx context.Context, args []string) error { inputBytes = b } else if *inputStr != "" { inputBytes = []byte(*inputStr) - } else if fs.NArg() > 0 { - // Positional argument as input string - arg := fs.Arg(0) - if arg == *wasmPath && fs.NArg() > 1 { - arg = fs.Arg(1) - } - if arg != *wasmPath { - inputBytes = []byte(arg) - } } wasmHost, err := wasmhost.NewWASMHost(ctx, wasmBytes, 1) diff --git a/vindex/internal/wasmhost/wasm.go b/vindex/internal/wasmhost/wasm.go index 193d1a7e..66287131 100644 --- a/vindex/internal/wasmhost/wasm.go +++ b/vindex/internal/wasmhost/wasm.go @@ -20,7 +20,6 @@ import ( "bytes" "cmp" "context" - "crypto/rand" "crypto/sha256" "encoding/binary" "errors" @@ -70,7 +69,6 @@ func ValidatePreimageKey(keyBytes []byte) error { // WASMHost manages the lifecycle and execution of a compiled Wazero WebAssembly module. type WASMHost struct { - runtime wazero.Runtime compiled wazero.CompiledModule pool chan *wasmInstance mu sync.Mutex @@ -153,7 +151,6 @@ func NewWASMHost(ctx context.Context, wasmBytes []byte, poolSize int) (*WASMHost } h := &WASMHost{ - runtime: getSharedRuntime(), compiled: compiled, pool: make(chan *wasmInstance, poolSize), } @@ -169,12 +166,8 @@ func NewWASMHost(ctx context.Context, wasmBytes []byte, poolSize int) (*WASMHost } func (h *WASMHost) instantiateInstance(ctx context.Context) (*wasmInstance, error) { - var id [8]byte - _, _ = rand.Read(id[:]) - modName := fmt.Sprintf("vindex_mod_%x", id) - wasiConfig := wazero.NewModuleConfig(). - WithName(modName). + WithName(""). WithStartFunctions("_initialize"). WithStdout(io.Discard). WithStderr(io.Discard). @@ -185,7 +178,7 @@ func (h *WASMHost) instantiateInstance(ctx context.Context) (*wasmInstance, erro WithNanosleep(func(int64) {}). WithRandSource(zeroReader{}) - mod, err := h.runtime.InstantiateModule(ctx, h.compiled, wasiConfig) + mod, err := getSharedRuntime().InstantiateModule(ctx, h.compiled, wasiConfig) if err != nil { return nil, err } @@ -239,8 +232,8 @@ func (inst *wasmInstance) executeBundle(ctx context.Context, leaves [][]byte) ([ if n == 0 { return nil, nil } - if n > 256 { - return nil, fmt.Errorf("%w: bundle size %d exceeds 256", ErrMalformedOutput, n) + if uint64(n) > DefaultBundleSize { + return nil, fmt.Errorf("%w: bundle size %d exceeds %d", ErrMalformedOutput, n, DefaultBundleSize) } headerLen := 4 + (n+1)*4 @@ -363,8 +356,7 @@ func decodeBundleOutputInto(resultsBuf [][]MappedEntry, entriesBuf []MappedEntry } allEntries[curEntryIdx] = MappedEntry{ - Key: string(keyBytes), - KeyHash: sha256.Sum256(keyBytes), + Key: string(keyBytes), } curEntryIdx++ } @@ -378,6 +370,9 @@ func decodeBundleOutputInto(resultsBuf [][]MappedEntry, entriesBuf []MappedEntry return a.Key == b.Key }) } + for k := range entries { + entries[k].KeyHash = sha256.Sum256([]byte(entries[k].Key)) + } curEntryIdx = startEntryIdx + len(entries) resultsBuf[i] = entries[:len(entries):len(entries)] } @@ -399,11 +394,6 @@ func (inst *wasmInstance) decodeBundleOutput(outBuf []byte, expectedCount int) ( return results, nil } -func decodeBundleOutput(outBuf []byte, expectedCount int) ([][]MappedEntry, error) { - results, _, err := decodeBundleOutputInto(nil, nil, outBuf, expectedCount) - return results, err -} - // MapBundle executes WASM mapping across a contiguous bundle of leaves using a checked-out runner. func (h *WASMHost) MapBundle(ctx context.Context, leaves [][]byte) ([][]MappedEntry, error) { runner, err := h.NewRunner(ctx) diff --git a/vindex/internal/wasmhost/wasm_test.go b/vindex/internal/wasmhost/wasm_test.go index 315320f9..f64382ba 100644 --- a/vindex/internal/wasmhost/wasm_test.go +++ b/vindex/internal/wasmhost/wasm_test.go @@ -26,7 +26,7 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" + "github.com/transparency-dev/incubator/vindex/internal/wasmhost/wasmtest" ) func TestWASMHost_PreimageKeysAndDeduplication(t *testing.T) { @@ -35,7 +35,7 @@ func TestWASMHost_PreimageKeysAndDeduplication(t *testing.T) { hBeta := sha256.Sum256([]byte("key_beta")) hAlpha := sha256.Sum256([]byte("key_alpha")) - host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 2) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -51,15 +51,15 @@ func TestWASMHost_PreimageKeysAndDeduplication(t *testing.T) { {Key: "key_beta", KeyHash: hBeta}, }} - if diff := cmp.Diff(expected, results); diff != "" { - t.Fatalf("MapBundle diff (-want +got):\n%s", diff) + if !slices.EqualFunc(results, expected, slices.Equal) { + t.Fatalf("MapBundle = %+v, want %+v", results, expected) } } func TestWASMHost_EmptyOutput(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -77,7 +77,7 @@ func TestWASMHost_EmptyOutput(t *testing.T) { func TestWASMHost_UnreachableTrap(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -95,7 +95,7 @@ func TestWASMHost_UnreachableTrap(t *testing.T) { func TestWASMHost_MalformedOutput(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -114,7 +114,7 @@ func TestWASMHost_ConcurrentExecution(t *testing.T) { ctx := context.Background() h1 := sha256.Sum256([]byte("concurrent_key")) - host, err := NewWASMHost(ctx, TestWASMBytes(), 4) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 4) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -144,7 +144,7 @@ func TestWASMHost_ConcurrentExecution(t *testing.T) { func TestWASMHost_ClosedHost(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -164,7 +164,7 @@ func TestWASMHost_MissingInputBufferExport(t *testing.T) { // Rename the 12-byte export "input_buffer" to "other_buffer" so the binary // remains valid WASM but lacks the required "input_buffer" export. - wasmBytes := bytes.ReplaceAll(slices.Clone(TestWASMBytes()), []byte("input_buffer"), []byte("other_buffer")) + wasmBytes := bytes.ReplaceAll(slices.Clone(wasmtest.Bytes()), []byte("input_buffer"), []byte("other_buffer")) _, err := NewWASMHost(ctx, wasmBytes, 1) if err == nil { @@ -180,7 +180,7 @@ func TestWASMHost_MissingMapBundleExport(t *testing.T) { // Rename the 10-byte export "map_bundle" to "other_func" so the binary // remains valid WASM but lacks the required "map_bundle" export. - wasmBytes := bytes.ReplaceAll(slices.Clone(TestWASMBytes()), []byte("map_bundle"), []byte("other_func")) + wasmBytes := bytes.ReplaceAll(slices.Clone(wasmtest.Bytes()), []byte("map_bundle"), []byte("other_func")) _, err := NewWASMHost(ctx, wasmBytes, 1) if err == nil { @@ -194,7 +194,7 @@ func TestWASMHost_MissingMapBundleExport(t *testing.T) { func TestWASMHost_Timeout(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -218,40 +218,10 @@ func TestWASMHost_Timeout(t *testing.T) { } } -func TestWASMHost_MapBundle(t *testing.T) { - ctx := context.Background() - - host, err := NewWASMHost(ctx, TestWASMBytes(), 2) - if err != nil { - t.Fatalf("NewWASMHost failed: %v", err) - } - defer func() { _ = host.Close(ctx) }() - - leaves := [][]byte{ - []byte("leaf0"), - []byte("leaf1"), - []byte("leaf2"), - []byte("leaf3"), - } - - results, err := host.MapBundle(ctx, leaves) - if err != nil { - t.Fatalf("MapBundle failed: %v", err) - } - if len(results) != len(leaves) { - t.Fatalf("expected %d bundle results, got %d", len(leaves), len(results)) - } - for i, r := range results { - if len(r) != 1 { - t.Fatalf("leaf %d expected 1 entry, got %d", i, len(r)) - } - } -} - func TestWASMHost_MapBundle_PreimagesAndHardwareHashing(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 2) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -301,7 +271,7 @@ func TestWASMHost_MapBundle_PreimagesAndHardwareHashing(t *testing.T) { func BenchmarkWASMHost_MapBundle(b *testing.B) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 4) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 4) if err != nil { b.Fatalf("NewWASMHost failed: %v", err) } @@ -324,7 +294,7 @@ func BenchmarkWASMHost_MapBundle(b *testing.B) { func BenchmarkWASMRunner_MapBundle(b *testing.B) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 4) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 4) if err != nil { b.Fatalf("NewWASMHost failed: %v", err) } @@ -351,44 +321,6 @@ func BenchmarkWASMRunner_MapBundle(b *testing.B) { } } -func BenchmarkDecodeBundleOutput(b *testing.B) { - const leavesCount = 256 - const keysPerLeaf = 2 - var outBuf bytes.Buffer - var countBuf [4]byte - binary.LittleEndian.PutUint32(countBuf[:], leavesCount) - outBuf.Write(countBuf[:]) - - for i := 0; i < leavesCount; i++ { - var kc [4]byte - binary.LittleEndian.PutUint32(kc[:], keysPerLeaf) - outBuf.Write(kc[:]) - } - - domain1 := []byte("example.com") - domain2 := []byte("sub.example.org") - for i := 0; i < leavesCount; i++ { - var k1Len, k2Len [4]byte - binary.LittleEndian.PutUint32(k1Len[:], uint32(len(domain1))) - outBuf.Write(k1Len[:]) - outBuf.Write(domain1) - - binary.LittleEndian.PutUint32(k2Len[:], uint32(len(domain2))) - outBuf.Write(k2Len[:]) - outBuf.Write(domain2) - } - - rawOutput := outBuf.Bytes() - b.ReportAllocs() - - for b.Loop() { - res, err := decodeBundleOutput(rawOutput, leavesCount) - if err != nil || len(res) != leavesCount { - b.Fatalf("decodeBundleOutput failed: %v", err) - } - } -} - func BenchmarkDecodeBundleOutputInto(b *testing.B) { const leavesCount = 256 const keysPerLeaf = 2 @@ -434,7 +366,7 @@ func BenchmarkDecodeBundleOutputInto(b *testing.B) { func TestWASMRunner_DedicatedExecution(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 2) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 2) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -460,7 +392,7 @@ func TestWASMRunner_ConcurrentDedicatedRunners(t *testing.T) { ctx := context.Background() concurrency := 8 - host, err := NewWASMHost(ctx, TestWASMBytes(), concurrency) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), concurrency) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -511,7 +443,7 @@ func TestDecodeBundleOutput_BoundsCheck(t *testing.T) { binary.LittleEndian.PutUint32(outBuf[0:4], 1) binary.LittleEndian.PutUint32(outBuf[4:8], 1_000_000) - _, err := decodeBundleOutput(outBuf, 1) + _, _, err := decodeBundleOutputInto(nil, nil, outBuf, 1) if err == nil { t.Fatal("expected error for totalKeys exceeding buffer capacity, got nil") } @@ -583,7 +515,7 @@ func TestDecodeBundleOutputInto_KeyLenOverflow(t *testing.T) { func TestWASMHost_MapBundle_ZeroLenSlabOverflowFailsFast(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -602,7 +534,7 @@ func TestWASMHost_MapBundle_ZeroLenSlabOverflowFailsFast(t *testing.T) { func TestWASMRunner_TrapDiscardsBrokenInstanceAndRecovers(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -672,7 +604,7 @@ func TestDecodeBundleOutputInto_TrailingUnconsumedBytes(t *testing.T) { func TestWASMRunner_FailsSubsequentCallsAfterError(t *testing.T) { ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 1) + host, err := NewWASMHost(ctx, wasmtest.Bytes(), 1) if err != nil { t.Fatalf("NewWASMHost failed: %v", err) } @@ -692,51 +624,6 @@ func TestWASMRunner_FailsSubsequentCallsAfterError(t *testing.T) { } } -func TestWASMHost_TestWASMBytes(t *testing.T) { - ctx := context.Background() - host, err := NewWASMHost(ctx, TestWASMBytes(), 2) - if err != nil { - t.Fatalf("NewWASMHost(TestWASMBytes) failed: %v", err) - } - defer func() { _ = host.Close(ctx) }() - - leaves := [][]byte{ - []byte("single_key"), - []byte("shared_key #leaf1"), - []byte("news\ntech\nnews"), - []byte(" \n\t "), - nil, - } - res, err := host.MapBundle(ctx, leaves) - if err != nil { - t.Fatalf("MapBundle failed: %v", err) - } - if len(res) != 5 { - t.Fatalf("got %d results, want 5", len(res)) - } - if len(res[0]) != 1 || res[0][0].Key != "single_key" { - t.Fatalf("leaf 0 = %+v, want [single_key]", res[0]) - } - if len(res[1]) != 1 || res[1][0].Key != "shared_key" { - t.Fatalf("leaf 1 = %+v, want [shared_key]", res[1]) - } - if len(res[2]) != 2 { - t.Fatalf("leaf 2 = %+v, want 2 deduplicated keys", res[2]) - } - if len(res[3]) != 0 || len(res[4]) != 0 { - t.Fatalf("leaves 3,4 = %+v, %+v, want empty", res[3], res[4]) - } - - // Control byte 0x01 triggers ErrMalformedOutput - if _, err := host.MapBundle(ctx, [][]byte{[]byte("\x01bad")}); !errors.Is(err, ErrMalformedOutput) { - t.Fatalf("expected ErrMalformedOutput on 0x01 byte, got %v", err) - } - // NUL byte 0x00 triggers ErrWasmHalt - if _, err := host.MapBundle(ctx, [][]byte{[]byte("\x00trap")}); !errors.Is(err, ErrWasmHalt) { - t.Fatalf("expected ErrWasmHalt on 0x00 byte, got %v", err) - } -} - func TestZeroReader_Infinite(t *testing.T) { var zr zeroReader buf := bytes.Repeat([]byte{0xaa}, 4096) diff --git a/vindex/internal/wasmhost/testdata/testguest/main.go b/vindex/internal/wasmhost/wasmtest/testdata/testguest/main.go similarity index 100% rename from vindex/internal/wasmhost/testdata/testguest/main.go rename to vindex/internal/wasmhost/wasmtest/testdata/testguest/main.go diff --git a/vindex/internal/wasmhost/wasmtest.go b/vindex/internal/wasmhost/wasmtest/wasmtest.go similarity index 75% rename from vindex/internal/wasmhost/wasmtest.go rename to vindex/internal/wasmhost/wasmtest/wasmtest.go index 541f3b08..0f5e2b55 100644 --- a/vindex/internal/wasmhost/wasmtest.go +++ b/vindex/internal/wasmhost/wasmtest/wasmtest.go @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package wasmhost +// Package wasmtest provides compiled WASM guest fixtures for VIndex tests. +package wasmtest import ( - "context" "fmt" "os" "os/exec" @@ -44,26 +44,12 @@ var compileTestGuestOnce = sync.OnceValues(func() ([]byte, error) { return os.ReadFile(wasmPath) }) -var sharedTestHostOnce = sync.OnceValues(func() (*WASMHost, error) { - return NewWASMHost(context.Background(), TestWASMBytes(), 0) -}) - -// TestWASMBytes compiles and returns the WebAssembly guest module in testdata/testguest +// Bytes compiles and returns the WebAssembly guest module in testdata/testguest // built with the VIndex MapFn SDK (sdk.RegisterEmit). -func TestWASMBytes() []byte { +func Bytes() []byte { b, err := compileTestGuestOnce() if err != nil { panic(err) } return b } - -// TestWASMHost returns a shared, process-wide WASMHost running TestWASMBytes() -// for tests that do not close the host directly. -func TestWASMHost() *WASMHost { - h, err := sharedTestHostOnce() - if err != nil { - panic(err) - } - return h -} diff --git a/vindex/mapfn/sdk/arena.go b/vindex/mapfn/sdk/arena.go deleted file mode 100644 index 37ef81d5..00000000 --- a/vindex/mapfn/sdk/arena.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2026 The Transparency Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sdk - -import ( - "encoding/binary" - "unsafe" -) - -const ( - // maxBufferSize is the static arena buffer size (4MB to accommodate 256-leaf bundles). - maxBufferSize = 4 * 1024 * 1024 -) - -var ( - inputBuf [maxBufferSize]byte - outputBuf [maxBufferSize]byte - emitOutOffset int - emitKeyCount int - emitOverflow bool -) - -func emitKey(k []byte) { - kLen := len(k) - if emitOverflow || emitOutOffset+4+kLen > maxBufferSize { - emitOverflow = true - return - } - binary.LittleEndian.PutUint32(outputBuf[emitOutOffset:emitOutOffset+4], uint32(kLen)) - copy(outputBuf[emitOutOffset+4:emitOutOffset+4+kLen], k) - emitOutOffset += 4 + kLen - emitKeyCount++ -} - -func getInputSlice(ptr, length uint32) []byte { - if length == 0 || length > maxBufferSize { - return nil - } - base := uint32(uintptr(unsafe.Pointer(&inputBuf[0]))) - if ptr < base || ptr-base > maxBufferSize-length { - return nil - } - offset := ptr - base - return inputBuf[offset : offset+length] -} - -func inputBufferPtr() uint32 { - return uint32(uintptr(unsafe.Pointer(&inputBuf[0]))) -} - -func executeBundle(inPtr, inLen uint32) (uint32, uint32) { - emitOverflow = false - if inLen < 8 { - return 0, 0 - } - - inSlice := getInputSlice(inPtr, inLen) - if len(inSlice) < 8 { - return 0, 0 - } - - leafCount := binary.LittleEndian.Uint32(inSlice[0:4]) - if leafCount == 0 || leafCount > 256 { - return 0, 0 - } - - headerLen := 4 + (leafCount+1)*4 - if uint32(len(inSlice)) < headerLen { - return 0, 0 - } - - offsetTable := inSlice[4:headerLen] - payload := inSlice[headerLen:] - - if binary.LittleEndian.Uint32(offsetTable[0:4]) != 0 { - return 0, 0 - } - if binary.LittleEndian.Uint32(offsetTable[leafCount*4:(leafCount+1)*4]) != uint32(len(payload)) { - return 0, 0 - } - - outBase := uint32(uintptr(unsafe.Pointer(&outputBuf[0]))) - binary.LittleEndian.PutUint32(outputBuf[0:4], leafCount) - - keyCountsOffset := 4 - outOffset := 4 + int(leafCount)*4 - - for i := uint32(0); i < leafCount; i++ { - start := binary.LittleEndian.Uint32(offsetTable[i*4 : (i+1)*4]) - end := binary.LittleEndian.Uint32(offsetTable[(i+1)*4 : (i+2)*4]) - - if end < start || end > uint32(len(payload)) { - return 0, 0 - } - leafBytes := payload[start:end] - - if registeredEmitMapFunc != nil { - emitOutOffset = outOffset - emitKeyCount = 0 - registeredEmitMapFunc(leafBytes, emitKey) - if emitOverflow { - return 0, 0 - } - binary.LittleEndian.PutUint32(outputBuf[keyCountsOffset+int(i)*4:], uint32(emitKeyCount)) - outOffset = emitOutOffset - } else { - binary.LittleEndian.PutUint32(outputBuf[keyCountsOffset+int(i)*4:], 0) - } - } - - return outBase, uint32(outOffset) -} diff --git a/vindex/mapfn/sdk/sdk.go b/vindex/mapfn/sdk/sdk.go index d6b5ba88..f80e3978 100644 --- a/vindex/mapfn/sdk/sdk.go +++ b/vindex/mapfn/sdk/sdk.go @@ -15,10 +15,28 @@ // Package sdk provides the guest SDK for authoring VIndex WASM MapFn plugins. package sdk +import ( + "encoding/binary" + "unsafe" +) + +const ( + // maxBufferSize is the static arena buffer size (4MB to accommodate 256-leaf bundles). + maxBufferSize = 4 * 1024 * 1024 + maxBundleLeaves = 256 +) + // EmitMapFunc maps a raw input log leaf to search key preimages via an emitter callback. type EmitMapFunc func(leaf []byte, emit func(key []byte)) -var registeredEmitMapFunc EmitMapFunc +var ( + registeredEmitMapFunc EmitMapFunc + inputBuf [maxBufferSize]byte + outputBuf [maxBufferSize]byte + emitOutOffset int + emitKeyCount int + emitOverflow bool +) // RegisterEmit registers a streaming plugin using an emitter callback. func RegisterEmit(fn EmitMapFunc) { @@ -33,7 +51,7 @@ func mapBundle(inPtr, inLen uint32) uint64 { //go:wasmexport input_buffer func inputBuffer() uint32 { - return inputBufferPtr() + return uint32(uintptr(unsafe.Pointer(&inputBuf[0]))) } // Suppress unused warnings when compiled on non-WASM architectures. @@ -41,3 +59,90 @@ var ( _ = mapBundle _ = inputBuffer ) + +func emitKey(k []byte) { + kLen := len(k) + if emitOverflow || emitOutOffset+4+kLen > maxBufferSize { + emitOverflow = true + return + } + binary.LittleEndian.PutUint32(outputBuf[emitOutOffset:emitOutOffset+4], uint32(kLen)) + copy(outputBuf[emitOutOffset+4:emitOutOffset+4+kLen], k) + emitOutOffset += 4 + kLen + emitKeyCount++ +} + +func getInputSlice(ptr, length uint32) []byte { + if length == 0 || length > maxBufferSize { + return nil + } + base := inputBuffer() + if ptr < base || ptr-base > maxBufferSize-length { + return nil + } + offset := ptr - base + return inputBuf[offset : offset+length] +} + +func executeBundle(inPtr, inLen uint32) (uint32, uint32) { + emitOverflow = false + if inLen < 8 { + return 0, 0 + } + + inSlice := getInputSlice(inPtr, inLen) + if len(inSlice) < 8 { + return 0, 0 + } + + leafCount := binary.LittleEndian.Uint32(inSlice[0:4]) + if leafCount == 0 || leafCount > maxBundleLeaves { + return 0, 0 + } + + headerLen := 4 + (leafCount+1)*4 + if uint32(len(inSlice)) < headerLen { + return 0, 0 + } + + offsetTable := inSlice[4:headerLen] + payload := inSlice[headerLen:] + + if binary.LittleEndian.Uint32(offsetTable[0:4]) != 0 { + return 0, 0 + } + if binary.LittleEndian.Uint32(offsetTable[leafCount*4:(leafCount+1)*4]) != uint32(len(payload)) { + return 0, 0 + } + + outBase := uint32(uintptr(unsafe.Pointer(&outputBuf[0]))) + binary.LittleEndian.PutUint32(outputBuf[0:4], leafCount) + + keyCountsOffset := 4 + outOffset := 4 + int(leafCount)*4 + + for i := uint32(0); i < leafCount; i++ { + start := binary.LittleEndian.Uint32(offsetTable[i*4 : (i+1)*4]) + end := binary.LittleEndian.Uint32(offsetTable[(i+1)*4 : (i+2)*4]) + + if end < start || end > uint32(len(payload)) { + return 0, 0 + } + leafBytes := payload[start:end] + + if registeredEmitMapFunc != nil { + emitOutOffset = outOffset + emitKeyCount = 0 + registeredEmitMapFunc(leafBytes, emitKey) + if emitOverflow { + return 0, 0 + } + binary.LittleEndian.PutUint32(outputBuf[keyCountsOffset+int(i)*4:], uint32(emitKeyCount)) + outOffset = emitOutOffset + } else { + binary.LittleEndian.PutUint32(outputBuf[keyCountsOffset+int(i)*4:], 0) + } + } + + return outBase, uint32(outOffset) +} diff --git a/vindex/mapfn/sdk/sdk_test.go b/vindex/mapfn/sdk/sdk_test.go index 4118845c..7d77ae87 100644 --- a/vindex/mapfn/sdk/sdk_test.go +++ b/vindex/mapfn/sdk/sdk_test.go @@ -50,12 +50,9 @@ func packBundleInput(leaves [][]byte) []byte { } func TestArena_InputBufferAndSliceBounds(t *testing.T) { - base := inputBufferPtr() + base := inputBuffer() if base == 0 { - t.Fatal("expected non-zero inputBufferPtr pointer") - } - if got := inputBuffer(); got != base { - t.Fatalf("inputBuffer() = %d, want %d", got, base) + t.Fatal("expected non-zero inputBuffer pointer") } // Test writing and reading via getInputSlice @@ -109,7 +106,7 @@ func TestPackBundleInput_And_ExecuteBundle_Emit(t *testing.T) { t.Fatal("packBundleInput returned empty buffer") } - ptr := inputBufferPtr() + ptr := inputBuffer() copy(getInputSlice(ptr, uint32(len(packedInput))), packedInput) outPtr, outLen := executeBundle(ptr, uint32(len(packedInput))) @@ -147,7 +144,7 @@ func TestExecuteBundle_EmitSingle(t *testing.T) { leaves := [][]byte{[]byte("example.com")} packedInput := packBundleInput(leaves) - ptr := inputBufferPtr() + ptr := inputBuffer() copy(getInputSlice(ptr, uint32(len(packedInput))), packedInput) outPtr, outLen := executeBundle(ptr, uint32(len(packedInput))) @@ -179,7 +176,7 @@ func TestExecuteBundle_OffsetTableValidation(t *testing.T) { leaves := [][]byte{[]byte("example.com")} packedInput := packBundleInput(leaves) - ptr := inputBufferPtr() + ptr := inputBuffer() // 1. OffsetTable[0] != 0 must fail badStart := append([]byte(nil), packedInput...) @@ -207,7 +204,7 @@ func TestExecuteBundle_EmitOverflow(t *testing.T) { leaves := [][]byte{[]byte("leaf")} packedInput := packBundleInput(leaves) - ptr := inputBufferPtr() + ptr := inputBuffer() copy(getInputSlice(ptr, uint32(len(packedInput))), packedInput) outPtr, outLen := executeBundle(ptr, uint32(len(packedInput)))