From 34c6b206f96ccbed5bb2745c84f8fe1ec390a40b Mon Sep 17 00:00:00 2001 From: Ford Shaper Date: Fri, 24 Jul 2026 12:09:58 -0400 Subject: [PATCH 1/6] feat(benchmark): UUID coherency validation for dataset replay Presence + cross-conversation contamination checks on the --from-dataset replay path, modeled on the cache-coherency test. Reusable primitives in benchmark/replay_uuid.go (UUID gen, marker injection, recite instruction, validateReplayResponse) and FindLeakedUUIDs moved into the benchmark package. To be re-targeted to the router-replay path next. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/auto.go | 82 ++++++ benchmark/replay.go | 146 ++++++++- benchmark/replay_uuid.go | 245 +++++++++++++++ benchmark/replay_uuid_test.go | 360 +++++++++++++++++++++++ benchmark/types.go | 9 +- cli/benchmark_commands.go | 20 ++ cli/benchmark_options.go | 30 +- cli/command_misc_cache_coherency_test.go | 59 +--- cli/eval_commands.go | 25 +- 9 files changed, 881 insertions(+), 95 deletions(-) create mode 100644 benchmark/replay_uuid.go create mode 100644 benchmark/replay_uuid_test.go diff --git a/benchmark/auto.go b/benchmark/auto.go index 811ead8..4d06ce2 100644 --- a/benchmark/auto.go +++ b/benchmark/auto.go @@ -89,6 +89,30 @@ type AutoBenchmarkConfig struct { // remains the normal budget C (--concurrency). ReplayStopAtLowConcurrency bool + // UUID-based response validation (--replay-inject-uuids). DATASET PATH + // ONLY (cfg.FromDataset != ""); the CLI rejects this combined with + // --router-replay-file (see cli/benchmark_commands.go) — router replay + // reconstructs prefixes from block hashes+token counts, and injecting + // visible text there would break cache-hit reproduction. + ReplayInjectUUIDs bool // inject per-turn UUID markers and validate their presence in later responses + // ReplayUUIDsPerTurn is how many UUIDs each injectable turn carries. + ReplayUUIDsPerTurn int + // ReplayUUIDSeed seeds the UUID generator (see newUUIDGenerator); 0 = crypto/rand + // (non-deterministic across runs). + ReplayUUIDSeed int64 + // ReplayUUIDMode selects which turns are injectable: "human" (default) or + // "all-non-gpt" (also tool + stray system turns). See replayTurnInjectable. + ReplayUUIDMode string + // ReplayReciteEveryTurn: ask the model to recite every ref-id seen so far on + // EVERY turn (default true), not just the conversation's final turn. + ReplayReciteEveryTurn bool + // replayUUIDSets is the precomputed per-conversation UUID list (parallel to + // replayConversations), populated once by RunAutoBenchmark before any + // per-model goroutine spawns — see buildReplayUUIDSets and the comment on + // replayConversations above (same sharing rationale: avoid recomputing N + // times in parallel, and every model must see the identical assignment). + replayUUIDSets [][]string + // RunID is populated internally by RunAutoBenchmark at the start of each // run. It's the UUID injected into every conversation's system prompt // (when ReplayNoStamp is false). Per-run scope — conversations that share @@ -185,6 +209,17 @@ type requestDataRecord struct { Question string `json:"question,omitempty"` ResponseText string `json:"response_text,omitempty"` RawResponseTail string `json:"raw_response_tail,omitempty"` + + // UUID validation (dataset-replay --replay-inject-uuids only). The three + // counts are always populated (0 when the feature is off); the raw detail + // lists are populated ONLY on a miss or a leak (mirrors the + // failed-request-only policy above — avoid bloating every row). + UUIDExpected int `json:"uuid_expected"` + UUIDFound int `json:"uuid_found"` + UUIDLeaked int `json:"uuid_leaked"` + ExpectedUUIDsRaw []string `json:"expected_uuids_raw,omitempty"` + FoundMask []bool `json:"found_mask,omitempty"` + LeakedUUIDsRaw []string `json:"leaked_uuids_raw,omitempty"` } // requestDataWriter writes requestDataRecord entries as JSONL, safe for concurrent use. @@ -878,6 +913,17 @@ type autoState struct { coldStartTTFTCount atomic.Int64 // count of cold-start samples (used for series-scaling gate) ttftDegradedCount atomic.Int64 // requests disqualified from cache-hit by TTFT degradation + // UUID validation (replay --replay-inject-uuids only). All zero when the + // feature is off — recordReplayRequest only touches these when + // metrics.ExpectedUUIDs is non-empty. + valReqs atomic.Int64 // requests that carried >=1 expected UUID (i.e. validation ran) + valUUIDChecks atomic.Int64 // total per-UUID presence checks made + valUUIDFound atomic.Int64 // per-UUID presence checks that found the UUID + valPresenceMissUUIDs atomic.Int64 // per-UUID PRESENCE_MISS count (expected UUID absent) + valCrossContamUUIDs atomic.Int64 // per-UUID CROSS_CONTAMINATION count (other-conversation UUID present) + valPresenceMissReqs atomic.Int64 // requests with >=1 PRESENCE_MISS + valCrossContamReqs atomic.Int64 // requests with >=1 CROSS_CONTAMINATION + // Persistent early-sample buffers — never trimmed, survive stream eviction. printMu sync.Mutex // serialises --print-responses output across concurrent series @@ -971,6 +1017,16 @@ type autoBenchmarkResult struct { totalInputWarm int64 // input tokens from warm requests (prefix was cached) totalOutput int64 // output tokens across all requests totalCachedTokens int64 // server-reported cached prompt tokens + + // UUID validation (replay --replay-inject-uuids only); all zero when the + // feature is off. See autoState's val* atomics for field meanings. + valReqs int64 + valUUIDChecks int64 + valUUIDFound int64 + valPresenceMissUUIDs int64 + valCrossContamUUIDs int64 + valPresenceMissReqs int64 + valCrossContamReqs int64 } // displaySnapshot is an atomic snapshot of state for the display goroutine. @@ -1122,6 +1178,14 @@ func printAutoSummary(res autoBenchmarkResult, cfg AutoBenchmarkConfig) { fmt.Println(strings.Repeat("-", 62)) fmt.Printf(" Total completed : %d\n", res.totalCompleted) fmt.Printf(" Total errors : %d\n", res.totalErrors) + if cfg.ReplayInjectUUIDs { + fmt.Println(strings.Repeat("-", 62)) + fmt.Println(" UUID validation (replay)") + fmt.Printf(" Requests validated : %d\n", res.valReqs) + fmt.Printf(" UUID presence : %d/%d\n", res.valUUIDFound, res.valUUIDChecks) + fmt.Printf(" PRESENCE_MISS (expected UUID absent) : %d across %d requests\n", res.valPresenceMissUUIDs, res.valPresenceMissReqs) + fmt.Printf(" CROSS_CONTAMINATION (other-conv) : %d across %d requests\n", res.valCrossContamUUIDs, res.valCrossContamReqs) + } fmt.Println(strings.Repeat("=", 62)) } @@ -2263,6 +2327,13 @@ func runSingleModelBenchmark( res.totalInputWarm = tt.inputWarm res.totalOutput = tt.output res.totalCachedTokens = tt.cached + res.valReqs = st.valReqs.Load() + res.valUUIDChecks = st.valUUIDChecks.Load() + res.valUUIDFound = st.valUUIDFound.Load() + res.valPresenceMissUUIDs = st.valPresenceMissUUIDs.Load() + res.valCrossContamUUIDs = st.valCrossContamUUIDs.Load() + res.valPresenceMissReqs = st.valPresenceMissReqs.Load() + res.valCrossContamReqs = st.valCrossContamReqs.Load() // Send a final snapshot with termReason set so multi-model display shows DONE. { @@ -2436,6 +2507,17 @@ func RunAutoBenchmark(ctx context.Context, cfg AutoBenchmarkConfig) error { } cfg.replayConversations = convs fmt.Printf("Loaded %d conversations. Starting auto benchmark...\n\n", len(convs)) + + // Precompute the per-conversation UUID sets ONCE here, before any + // per-model goroutine spawns below, so every model's + // runSingleModelBenchmark sees the identical assignment (same + // sharing rationale as replayConversations itself — see its doc + // comment on AutoBenchmarkConfig). + if cfg.ReplayInjectUUIDs { + cfg.replayUUIDSets = buildReplayUUIDSets(cfg.replayConversations, cfg.ReplayUUIDSeed, cfg.ReplayUUIDsPerTurn, cfg.ReplayUUIDMode) + fmt.Printf("UUID validation enabled: %d conversation(s) prepared (mode=%s, per-turn=%d, seed=%d)\n", + len(cfg.replayUUIDSets), cfg.ReplayUUIDMode, cfg.ReplayUUIDsPerTurn, cfg.ReplayUUIDSeed) + } } // Tree-aware router replay: only the header (line 1) is read here so diff --git a/benchmark/replay.go b/benchmark/replay.go index 94d0020..b67f821 100644 --- a/benchmark/replay.go +++ b/benchmark/replay.go @@ -101,7 +101,7 @@ func runReplaySeriesLoop( seriesNum := convIdx + 1 seriesGUID := conv.ID - fullyWalked := runReplayConversation(benchCtx, cfg, st, rdw, conv, seriesNum, seriesGUID, endpointOverride, reqTimeout, gate) + fullyWalked := runReplayConversation(benchCtx, cfg, st, rdw, conv, convIdx, seriesNum, seriesGUID, endpointOverride, reqTimeout, gate) // The conversation's slot is retired — its context no longer counts // toward the active dataset. st.datasetTracker.Reset(seriesNum) @@ -116,6 +116,11 @@ func runReplaySeriesLoop( // benchmark request per gpt turn. Errors on individual requests are recorded // but don't abort the series — the next turn still runs. // +// convIdx is this conversation's index into cfg.replayConversations / +// cfg.replayUUIDSets (== seriesNum-1, passed explicitly rather than +// re-derived so the UUID-validation wiring below doesn't have to assume that +// relationship). +// // Returns true if the whole conversation was walked; false if a stop signal // (--total reached, context cancel) cut it short mid-walk. func runReplayConversation( @@ -124,6 +129,7 @@ func runReplayConversation( st *autoState, rdw *requestDataWriter, conv Conversation, + convIdx int, seriesNum int, seriesGUID string, endpointOverride string, @@ -200,7 +206,31 @@ func runReplayConversation( var pending strings.Builder turnNum := 0 - flush := func() bool { + // UUID validation setup (--replay-inject-uuids, dataset path only). All of + // this is inert when the flag is off: uuidSets stays nil, injecting is + // false, and every gate below short-circuits. + injecting := cfg.ReplayInjectUUIDs + var uuidSets []string + if injecting && convIdx >= 0 && convIdx < len(cfg.replayUUIDSets) { + uuidSets = cfg.replayUUIDSets[convIdx] + } + uuidCursor := 0 + var inScope []string + reciteTruncWarned := false // logs the max-tokens recite-cap WARNING at most once per conversation + + // lastGptIdx locates this conversation's FINAL 'gpt' turn so the recite + // instruction still goes out at least once (on that turn) even when + // --replay-recite-every-turn=false. + lastGptIdx := -1 + if injecting { + for i := firstIdx; i < len(conv.Turns); i++ { + if conv.Turns[i].From == "gpt" { + lastGptIdx = i + } + } + } + + flush := func(gptIdx int) bool { userContent := strings.TrimSpace(pending.String()) pending.Reset() if userContent == "" { @@ -232,13 +262,40 @@ func runReplayConversation( requestNum := int(st.totalCompleted.Load()) + 1 // Observe the accumulated history (including this turn's user message) - // with the content-level estimator BEFORE the server responds. + // with the content-level estimator BEFORE the server responds. The + // recite-instruction tail (below) is deliberately NOT part of what the + // estimator/history see — it's per-request boilerplate, not + // conversation content, and would otherwise skew the cache-ratio + // estimate every single turn (recite-every-turn is the default). history.WriteString(userContent) ratio := st.estimator.Observe(history.String()) + // Append the recite-every-seen-ref-id instruction, if injecting. Cap + // the recited list to a fraction of the output budget first — an + // uncapped list only grows every turn and can eventually ask the model + // to reproduce more ref-ids than max_tokens can hold, truncating the + // SEEN_REFS line itself; ExpectedUUIDs is set to the SAME (possibly + // capped) list so a truncation-induced gap is never misread as a + // PRESENCE_MISS. + outgoingContent := userContent + var expectedSnapshot []string + if injecting { + recited, truncated := capRecitedUUIDs(inScope, cfg.MaxOutputTokens) + if truncated && !reciteTruncWarned { + reciteTruncWarned = true + fmt.Fprintf(os.Stderr, + "[auto][%s] WARNING: replay UUID recite list capped to fit --max-output-tokens budget (conv=%d) — PRESENCE_MISS on ref-ids dropped from recitation is expected, not corruption\n", + shortModelName(cfg.Model), convIdx) + } + expectedSnapshot = append([]string(nil), recited...) + if cfg.ReplayReciteEveryTurn || gptIdx == lastGptIdx { + outgoingContent = userContent + replayReciteInstruction(recited) + } + } + reqCtx, reqCancel := context.WithTimeout(benchCtx, reqTimeout) resetTTFT(time.Now()) - response, err := chat.Request(reqCtx, llm.TextParts(userContent), nil) + response, err := chat.Request(reqCtx, llm.TextParts(outgoingContent), nil) totalTime := time.Since(startTime) reqCancel() gate.Release() @@ -253,8 +310,14 @@ func runReplayConversation( TotalResponseTime: totalTime, Error: err, } + // respThinking is captured separately from metrics.Response (which may + // get overwritten by response.Thinking below when content is empty) so + // UUID validation can always scan content ∪ thinking, exactly like the + // cache-coherency eval does. + var respThinking string if response != nil { metrics.Response = response.Content + respThinking = response.Thinking if strings.TrimSpace(metrics.Response) == "" { metrics.Response = response.Thinking } @@ -281,6 +344,19 @@ func runReplayConversation( } metrics.LocalCacheRatio = ratio + + if injecting { + metrics.ConvIdx = convIdx + metrics.ExpectedUUIDs = expectedSnapshot + // ERROR responses (including the synthetic "empty response" error + // above) are excluded from validation — no usable content/thinking + // to scan. + if metrics.Error == nil { + metrics.UUIDFound, metrics.LeakedUUIDs = validateReplayResponse( + metrics.Response, respThinking, metrics.ExpectedUUIDs, convIdx, cfg.replayUUIDSets) + } + } + recordReplayRequest(cfg, st, rdw, metrics, isFirstRequest, &coldStartTTFT) isFirstRequest = false return true @@ -289,7 +365,7 @@ func runReplayConversation( for i := firstIdx; i < len(conv.Turns); i++ { t := conv.Turns[i] if t.From == "gpt" { - if !flush() { + if !flush(i) { return false } continue @@ -297,7 +373,23 @@ func runReplayConversation( if pending.Len() > 0 { pending.WriteString("\n\n") } - pending.WriteString(t.Value) + turnValue := t.Value + // UUID injection: this cursor/slicing logic MUST mirror + // computeInScopeAtEachGptTurn (replay_uuid.go) exactly — the test + // suite (replay_uuid_test.go) asserts the two stay in lockstep. + if injecting && replayTurnInjectable(t, cfg.ReplayUUIDMode) { + end := uuidCursor + cfg.ReplayUUIDsPerTurn + if end > len(uuidSets) { + end = len(uuidSets) + } + if uuidCursor < end { + turnUUIDs := uuidSets[uuidCursor:end] + inScope = append(inScope, turnUUIDs...) + turnValue = injectUUIDMarker(turnValue, turnUUIDs) + } + uuidCursor = end + } + pending.WriteString(turnValue) } // Trailing non-gpt content (if any) is discarded — there's no assistant // response to measure for it. @@ -347,6 +439,31 @@ func recordReplayRequest( isErr := metrics.Error != nil explicitCache := metrics.UsageData.CachedTokens.Count > 0 + // UUID validation tallies (--replay-inject-uuids only). metrics.ExpectedUUIDs + // is nil/empty for every request when the feature is off (default), so this + // block — and the val* counters it touches — is fully inert then. + uuidExpectedCount := len(metrics.ExpectedUUIDs) + uuidFoundCount := 0 + for _, found := range metrics.UUIDFound { + if found { + uuidFoundCount++ + } + } + uuidLeakedCount := len(metrics.LeakedUUIDs) + if uuidExpectedCount > 0 { + st.valReqs.Add(1) + st.valUUIDChecks.Add(int64(uuidExpectedCount)) + st.valUUIDFound.Add(int64(uuidFoundCount)) + if missCount := uuidExpectedCount - uuidFoundCount; missCount > 0 { + st.valPresenceMissUUIDs.Add(int64(missCount)) + st.valPresenceMissReqs.Add(1) + } + if uuidLeakedCount > 0 { + st.valCrossContamUUIDs.Add(int64(uuidLeakedCount)) + st.valCrossContamReqs.Add(1) + } + } + earlyColdBaseline := st.earlyColdStartTTFT() st.mu.Lock() @@ -390,7 +507,7 @@ func recordReplayRequest( if metrics.Error != nil { errMsg = metrics.Error.Error() } - if writeErr := rdw.write(requestDataRecord{ + rec := requestDataRecord{ StartTime: reqStart, EndTime: reqEnd, TTFT: float64(metrics.TimeToFirstToken.Milliseconds()), @@ -409,7 +526,20 @@ func recordReplayRequest( ErrorMessage: errMsg, IsEmpty: metrics.IsEmpty, LocalCacheRatio: metrics.LocalCacheRatio, - }); writeErr != nil { + UUIDExpected: uuidExpectedCount, + UUIDFound: uuidFoundCount, + UUIDLeaked: uuidLeakedCount, + } + // Raw detail lists only on a miss or a leak — mirrors the + // failed-request-only policy on PromptText/ResponseText/RawResponseTail + // above (avoid bloating every row with data that matters only when + // something's wrong). + if uuidFoundCount < uuidExpectedCount || uuidLeakedCount > 0 { + rec.ExpectedUUIDsRaw = metrics.ExpectedUUIDs + rec.FoundMask = metrics.UUIDFound + rec.LeakedUUIDsRaw = metrics.LeakedUUIDs + } + if writeErr := rdw.write(rec); writeErr != nil { fmt.Fprintf(os.Stderr, "warning: failed to write request data: %v\n", writeErr) } } diff --git a/benchmark/replay_uuid.go b/benchmark/replay_uuid.go new file mode 100644 index 0000000..f09aa85 --- /dev/null +++ b/benchmark/replay_uuid.go @@ -0,0 +1,245 @@ +package benchmark + +// UUID-based response validation for the dataset-replay benchmark +// (--from-dataset, --replay-inject-uuids). Distinct per-conversation UUIDs +// are stamped into injectable turns as the conversation is walked; the model +// is periodically asked to recite every ref-id it has seen so far, and the +// response is scored for presence (did it recall its own conversation's +// UUIDs?) and cross-contamination (did it leak a UUID belonging to a +// DIFFERENT conversation, i.e. a KV/scheduling leak?). +// +// This is a presence-based check (Contains), not the cache-coherency eval's +// exact-conformity check (matchesExpectedUUIDList) — replay responses are +// real multi-turn chat turns with real prose, not a bare comma-joined list. +// +// DATASET PATH ONLY: this file must never be reached from the router-replay +// path (replay_router*.go), which reconstructs prefixes from block +// hashes+token counts rather than raw dataset text — injecting visible text +// there would break cache-hit reproduction. The CLI enforces this (see +// cli/benchmark_commands.go: --replay-inject-uuids requires --from-dataset +// and is rejected together with --router-replay-file). + +import ( + "fmt" + "strings" +) + +// replayTurnInjectable reports whether turn t should carry an injected UUID +// marker under the given --replay-uuid-mode. This is the SAME predicate used +// both to precompute each conversation's UUID count (buildReplayUUIDSets) and +// to walk turns for real (runReplayConversation in replay.go) — the two MUST +// agree exactly, or the precomputed UUID list and the turn walk's cursor +// diverge mid-conversation. +// +// Modes: +// - "human" (default): only turns from the human/user. +// - "all-non-gpt": human, plus tool-result turns and any stray system turn +// (a conversation's LEADING system turn is never walked at all — it +// becomes the cached system prompt instead — so "stray" here means any +// system turn that is NOT at index 0). +func replayTurnInjectable(t HermesTurn, mode string) bool { + if mode == "all-non-gpt" { + return t.From == "human" || t.From == "tool" || t.From == "system" + } + return t.From == "human" +} + +// buildReplayUUIDSets precomputes, for every conversation in convs, the full +// ordered list of UUIDs its injectable turns will carry over the course of +// the conversation: injectableTurns * perTurn UUIDs. Every UUID across every +// conversation is drawn from a SINGLE seeded generator in conversation-major, +// turn-minor order, which is what makes the result both deterministic (same +// seed -> same output) and disjoint by construction (each draw is unique — +// the seeded PCG generator never repeats a UUID within a run). +// +// The returned slice is indexed by conversation index (parallel to convs), +// matching cfg.replayUUIDSets / cfg.replayConversations elsewhere. +func buildReplayUUIDSets(convs []Conversation, seed int64, perTurn int, mode string) [][]string { + newUUID := newUUIDGenerator(seed) + sets := make([][]string, len(convs)) + for i, conv := range convs { + // Skip a LEADING system turn exactly like the real turn walk does + // (runReplayConversation / computeInScopeAtEachGptTurn, both in + // replay.go) — it becomes the cached system prompt, not a walked turn. + // Without this skip, "all-non-gpt" mode (which counts system turns) + // would count that turn here but the real walk would never consume a + // UUID for it, drifting the precomputed count out of lockstep with + // what's actually assigned. + turns := conv.Turns + if len(turns) > 0 && turns[0].From == "system" { + turns = turns[1:] + } + injectable := 0 + for _, t := range turns { + if replayTurnInjectable(t, mode) { + injectable++ + } + } + n := injectable * perTurn + if n <= 0 { + continue + } + uuids := make([]string, n) + for j := range uuids { + uuids[j] = newUUID() + } + sets[i] = uuids + } + return sets +} + +// injectUUIDMarker appends one visible marker per uuid to turnValue. Unlike +// the cache-coherency eval's ... filler, the model MUST see +// and be able to repeat this text — it's asked to recite every ref-id later +// — so the marker is plain, readable text. Detection downstream is a raw +// substring match on the UUID itself, so the wrapper text ("[ref-id: ...]") +// is purely cosmetic and not load-bearing for scoring. +func injectUUIDMarker(turnValue string, uuids []string) string { + if len(uuids) == 0 { + return turnValue + } + var b strings.Builder + b.WriteString(turnValue) + for _, u := range uuids { + b.WriteString("\n\n[ref-id: ") + b.WriteString(u) + b.WriteString("]") + } + return b.String() +} + +// replayReciteInstruction returns the boilerplate appended to an outgoing +// user turn, asking the model to FIRST recite every ref-id it has seen so +// far (inScope, in order) on a delimited line, THEN answer normally. +// Presence scoring is Contains-based (see validateReplayResponse), so the +// exact wording/format here is not load-bearing — the "SEEN_REFS:" delimiter +// just keeps the recited list easy to spot in a transcript/log. +func replayReciteInstruction(inScope []string) string { + return fmt.Sprintf("\n\n(Before your normal answer, first output one line in the exact form `SEEN_REFS: %s` listing every ref-id you have seen anywhere in this conversation so far, comma-separated. Then answer normally.)", + strings.Join(inScope, ",")) +} + +// replayReciteBudgetFraction caps how much of a request's max_tokens output +// budget the recited ref-id list is allowed to consume (see +// capRecitedUUIDs) — recite-every-turn means the list only grows, so without +// a cap a long conversation eventually asks the model to reproduce more +// ref-ids than max_tokens can hold, truncating the SEEN_REFS line itself. +const replayReciteBudgetFraction = 0.5 + +// capRecitedUUIDs trims inScope down to (at most) however many entries fit +// within replayReciteBudgetFraction of maxOutputTokens, keeping the MOST +// RECENT entries (dropping the oldest first — the model is more likely to +// have retained recent context). Returns the (possibly untouched) list and +// whether trimming occurred. +// +// This is deliberately a simple heuristic (reuses the standard +// len/4 estimateTokens idiom already used elsewhere in this package, see +// cache_sim.go) — not exact tokenizer accounting. maxOutputTokens <= 0 +// (budget unknown/unbounded) disables capping entirely. +func capRecitedUUIDs(inScope []string, maxOutputTokens int) ([]string, bool) { + if maxOutputTokens <= 0 || len(inScope) == 0 { + return inScope, false + } + budget := int(float64(maxOutputTokens) * replayReciteBudgetFraction) + if estimateTokens(strings.Join(inScope, ",")) <= budget { + return inScope, false + } + perUUID := estimateTokens(inScope[0] + ",") + if perUUID < 1 { + perUUID = 1 + } + n := budget / perUUID + if n < 1 { + n = 1 + } + if n >= len(inScope) { + return inScope, false + } + return inScope[len(inScope)-n:], true +} + +// computeInScopeAtEachGptTurn walks turns exactly like runReplayConversation's +// real turn loop (replay.go) — skipping turns[0] when it's the leading system +// turn (that becomes the cached system prompt, not a walked turn) — and +// returns, for each 'gpt' turn encountered in order, a snapshot of every UUID +// assigned to an injectable turn seen so far. len(result) == the number of +// 'gpt' turns in turns (after the leading-system skip). +// +// This exists purely so the cumulative in-scope tracking is unit-testable in +// isolation (see replay_uuid_test.go); replay.go's real loop additionally +// needs the PER-TURN uuid slice (to wrap the turn text via injectUUIDMarker), +// so it maintains the same cursor/slicing logic inline rather than calling +// this function directly — the two must be kept in lockstep, which the test +// suite verifies. +func computeInScopeAtEachGptTurn(turns []HermesTurn, sets []string, perTurn int, mode string) [][]string { + firstIdx := 0 + if len(turns) > 0 && turns[0].From == "system" { + firstIdx = 1 + } + + var result [][]string + var inScope []string + cursor := 0 + for i := firstIdx; i < len(turns); i++ { + t := turns[i] + if t.From == "gpt" { + result = append(result, append([]string(nil), inScope...)) + continue + } + if replayTurnInjectable(t, mode) { + end := cursor + perTurn + if end > len(sets) { + end = len(sets) + } + if cursor < end { + inScope = append(inScope, sets[cursor:end]...) + } + cursor = end + } + } + return result +} + +// validateReplayResponse scores one replay response/thinking pair: +// - found[i] reports whether expected[i] (this conversation's in-scope +// ref-ids at this point) appears in resp or thinking (Contains, mirroring +// the cache-coherency eval's presence check). +// - leaked reports any OTHER conversation's UUID found in resp/thinking — +// cross-contamination — via the shared FindLeakedUUIDs helper (moved +// here from cli/eval_commands.go so both the coherency CLI and replay +// validation share one implementation). +func validateReplayResponse(resp, thinking string, expected []string, convIdx int, allSets [][]string) (found []bool, leaked []string) { + found = make([]bool, len(expected)) + for i, u := range expected { + found[i] = strings.Contains(resp, u) || strings.Contains(thinking, u) + } + leaked = FindLeakedUUIDs(resp, thinking, convIdx, allSets) + return found, leaked +} + +// FindLeakedUUIDs scans resp and thinking for UUIDs belonging to a series/ +// conversation OTHER than ownIdx, per the ordered allSets list (allSets[i] = +// full UUID list "owned" by index i — this doubles as the uuid -> owner +// mapping without needing an actual map, keeping iteration order — and +// therefore leak-report order — deterministic for a given seed). Returns +// "uuid(series=N)" entries, one per leaked UUID found. +// +// Exported (moved from cli/eval_commands.go) so both the cache-coherency +// eval CLI (cli/eval_commands.go, where ownIdx is a coherency series index +// and allSets is CacheCoherencyResult.SeriesUUIDs) and dataset-replay UUID +// validation above (where ownIdx is a conversation index and allSets is +// AutoBenchmarkConfig.replayUUIDSets) share one implementation, not two. +func FindLeakedUUIDs(resp, thinking string, ownIdx int, allSets [][]string) []string { + var leaked []string + for si, uuids := range allSets { + if si == ownIdx { + continue + } + for _, u := range uuids { + if strings.Contains(resp, u) || strings.Contains(thinking, u) { + leaked = append(leaked, fmt.Sprintf("%s(series=%d)", u, si)) + } + } + } + return leaked +} diff --git a/benchmark/replay_uuid_test.go b/benchmark/replay_uuid_test.go new file mode 100644 index 0000000..3704303 --- /dev/null +++ b/benchmark/replay_uuid_test.go @@ -0,0 +1,360 @@ +package benchmark + +import ( + "strings" + "testing" +) + +// syntheticMixedRoleConvs builds a small synthetic conversation set with mixed +// From roles (human/gpt/tool/system, including one LEADING system turn and one +// STRAY mid-conversation system turn) — enough to exercise both +// --replay-uuid-mode values. +func syntheticMixedRoleConvs() []Conversation { + return []Conversation{ + {ID: "c0", Turns: []HermesTurn{ + {From: "system", Value: "c0 leading system prompt"}, + {From: "human", Value: "c0 h1"}, + {From: "gpt", Value: "c0 g1"}, + {From: "tool", Value: "c0 t1"}, + {From: "human", Value: "c0 h2"}, + {From: "gpt", Value: "c0 g2"}, + }}, + {ID: "c1", Turns: []HermesTurn{ + {From: "human", Value: "c1 h1"}, // no leading system turn at all + {From: "gpt", Value: "c1 g1"}, + {From: "system", Value: "c1 stray system"}, // NOT at index 0 + {From: "human", Value: "c1 h2"}, + {From: "gpt", Value: "c1 g2"}, + }}, + } +} + +func TestBuildReplayUUIDSets(t *testing.T) { + convs := syntheticMixedRoleConvs() + + t.Run("determinism under fixed seed", func(t *testing.T) { + a := buildReplayUUIDSets(convs, 42, 2, "human") + b := buildReplayUUIDSets(convs, 42, 2, "human") + if len(a) != len(b) { + t.Fatalf("length mismatch: %d vs %d", len(a), len(b)) + } + for i := range a { + if len(a[i]) != len(b[i]) { + t.Fatalf("conv %d length mismatch: %d vs %d", i, len(a[i]), len(b[i])) + } + for j := range a[i] { + if a[i][j] != b[i][j] { + t.Errorf("conv %d uuid %d mismatch across identical-seed calls: %q vs %q", i, j, a[i][j], b[i][j]) + } + } + } + }) + + t.Run("different seeds diverge", func(t *testing.T) { + a := buildReplayUUIDSets(convs, 1, 2, "human") + b := buildReplayUUIDSets(convs, 2, 2, "human") + same := true + for i := range a { + for j := range a[i] { + if a[i][j] != b[i][j] { + same = false + } + } + } + if same { + t.Errorf("different seeds produced identical uuid sets") + } + }) + + t.Run("disjoint across conversations", func(t *testing.T) { + sets := buildReplayUUIDSets(convs, 7, 2, "all-non-gpt") + owner := make(map[string]int) + for ci, uuids := range sets { + for _, u := range uuids { + if prevCi, ok := owner[u]; ok { + t.Errorf("uuid %q appears in both conversation %d and conversation %d", u, prevCi, ci) + } + owner[u] = ci + } + } + }) + + t.Run("count == injectableTurns*perTurn, mode human", func(t *testing.T) { + const perTurn = 3 + sets := buildReplayUUIDSets(convs, 1, perTurn, "human") + // c0: human turns = h1, h2 -> 2. c1: human turns = h1, h2 -> 2. + if got, want := len(sets[0]), 2*perTurn; got != want { + t.Errorf("conv0 human mode: len = %d, want %d", got, want) + } + if got, want := len(sets[1]), 2*perTurn; got != want { + t.Errorf("conv1 human mode: len = %d, want %d", got, want) + } + }) + + t.Run("count == injectableTurns*perTurn, mode all-non-gpt", func(t *testing.T) { + const perTurn = 2 + sets := buildReplayUUIDSets(convs, 1, perTurn, "all-non-gpt") + // c0: leading system turn (index 0) is stripped before counting, so + // injectable turns are h1, tool t1, h2 -> 3. + if got, want := len(sets[0]), 3*perTurn; got != want { + t.Errorf("conv0 all-non-gpt mode: len = %d, want %d", got, want) + } + // c1: no leading system turn to strip; injectable turns are h1, the + // STRAY mid-conversation system turn, h2 -> 3. + if got, want := len(sets[1]), 3*perTurn; got != want { + t.Errorf("conv1 all-non-gpt mode: len = %d, want %d", got, want) + } + }) +} + +func TestReplayTurnInjectable(t *testing.T) { + tests := []struct { + from string + wantHuman bool + wantAllNGpt bool + }{ + {"human", true, true}, + {"gpt", false, false}, + {"tool", false, true}, + {"system", false, true}, + } + for _, tt := range tests { + t.Run(tt.from, func(t *testing.T) { + turn := HermesTurn{From: tt.from, Value: "x"} + if got := replayTurnInjectable(turn, "human"); got != tt.wantHuman { + t.Errorf("replayTurnInjectable(%q, \"human\") = %v, want %v", tt.from, got, tt.wantHuman) + } + if got := replayTurnInjectable(turn, "all-non-gpt"); got != tt.wantAllNGpt { + t.Errorf("replayTurnInjectable(%q, \"all-non-gpt\") = %v, want %v", tt.from, got, tt.wantAllNGpt) + } + }) + } +} + +func TestInjectUUIDMarker(t *testing.T) { + original := "what is the weather today?" + uuids := []string{"uuid-aaa", "uuid-bbb"} + + got := injectUUIDMarker(original, uuids) + + if !strings.Contains(got, original) { + t.Errorf("injectUUIDMarker() dropped the original turn text: %q", got) + } + for _, u := range uuids { + if !strings.Contains(got, u) { + t.Errorf("injectUUIDMarker() result missing uuid %q: %q", u, got) + } + } + + // No uuids -> turnValue passed through unchanged. + if got := injectUUIDMarker(original, nil); got != original { + t.Errorf("injectUUIDMarker(_, nil) = %q, want unchanged %q", got, original) + } +} + +// TestComputeInScopeAtEachGptTurn walks a synthetic turn sequence and asserts +// the snapshot at each gpt-turn boundary equals the expected prefix union of +// uuids assigned to injectable turns seen so far — the invariant +// replay.go's real turn loop maintains inline (see the comment there pointing +// back at this test). +func TestComputeInScopeAtEachGptTurn(t *testing.T) { + turns := []HermesTurn{ + {From: "system", Value: "leading system prompt"}, // skipped entirely + {From: "human", Value: "h1"}, // injectable (human mode): uuids[0:2] + {From: "gpt", Value: "g1"}, // snapshot #0 -> uuids[0:2] + {From: "tool", Value: "t1"}, // NOT injectable in human mode + {From: "human", Value: "h2"}, // injectable: uuids[2:4] + {From: "gpt", Value: "g2"}, // snapshot #1 -> uuids[0:4] + {From: "human", Value: "h3"}, // injectable: uuids[4:6] + {From: "gpt", Value: "g3"}, // snapshot #2 -> uuids[0:6] + } + sets := []string{"u0", "u1", "u2", "u3", "u4", "u5"} + const perTurn = 2 + + got := computeInScopeAtEachGptTurn(turns, sets, perTurn, "human") + want := [][]string{ + {"u0", "u1"}, + {"u0", "u1", "u2", "u3"}, + {"u0", "u1", "u2", "u3", "u4", "u5"}, + } + if len(got) != len(want) { + t.Fatalf("computeInScopeAtEachGptTurn() = %d snapshots, want %d", len(got), len(want)) + } + for i := range want { + if strings.Join(got[i], ",") != strings.Join(want[i], ",") { + t.Errorf("snapshot[%d] = %v, want %v", i, got[i], want[i]) + } + } + + // Mutating a returned snapshot must not corrupt a previous snapshot + // (defensive copy per call) or later ones. + got[0][0] = "MUTATED" + got2 := computeInScopeAtEachGptTurn(turns, sets, perTurn, "human") + if got2[0][0] != "u0" { + t.Errorf("computeInScopeAtEachGptTurn() snapshots are not independently allocated: got2[0][0] = %q", got2[0][0]) + } + + // all-non-gpt mode additionally picks up the mid-conversation 'tool' turn, + // so the second snapshot's union grows by one more perTurn slice than in + // human mode (the leading system turn is still skipped). + gotAll := computeInScopeAtEachGptTurn(turns, sets, perTurn, "all-non-gpt") + if len(gotAll) != 3 { + t.Fatalf("all-non-gpt mode: got %d snapshots, want 3", len(gotAll)) + } + if len(gotAll[0]) != 2 { + t.Errorf("all-non-gpt mode snapshot[0] = %v, want len 2", gotAll[0]) + } + if len(gotAll[1]) != 6 { + // h1 (2) + tool t1 (2) + h2 (2) = 6, vs 4 in human-only mode. + t.Errorf("all-non-gpt mode snapshot[1] = %v, want len 6 (tool turn now counted)", gotAll[1]) + } +} + +func TestValidateReplayResponse(t *testing.T) { + allSets := [][]string{ + {"own-0", "own-1"}, // conversation 0 (this response's own conversation) + {"other-0", "other-1"}, // conversation 1 + } + + t.Run("presence in content only", func(t *testing.T) { + found, leaked := validateReplayResponse("SEEN_REFS: own-0,own-1", "", []string{"own-0", "own-1"}, 0, allSets) + if len(found) != 2 || !found[0] || !found[1] { + t.Errorf("found = %v, want both true", found) + } + if len(leaked) != 0 { + t.Errorf("leaked = %v, want none", leaked) + } + }) + + t.Run("presence in thinking only", func(t *testing.T) { + found, _ := validateReplayResponse("some unrelated prose", "I recall own-0 and own-1 from earlier", []string{"own-0", "own-1"}, 0, allSets) + if !found[0] || !found[1] { + t.Errorf("found = %v, want both true (present in thinking)", found) + } + }) + + t.Run("presence in general prose (not the SEEN_REFS line)", func(t *testing.T) { + found, _ := validateReplayResponse("Sure! Earlier you mentioned own-0, and also own-1 came up.", "", []string{"own-0", "own-1"}, 0, allSets) + if !found[0] || !found[1] { + t.Errorf("found = %v, want both true (Contains-based, no format requirement)", found) + } + }) + + t.Run("PRESENCE_MISS: expected uuid absent from both", func(t *testing.T) { + found, leaked := validateReplayResponse("I don't remember any ref ids.", "", []string{"own-0", "own-1"}, 0, allSets) + if found[0] || found[1] { + t.Errorf("found = %v, want both false", found) + } + if len(leaked) != 0 { + t.Errorf("leaked = %v, want none", leaked) + } + }) + + t.Run("CROSS_CONTAMINATION: another conversation's uuid appears", func(t *testing.T) { + found, leaked := validateReplayResponse("SEEN_REFS: own-0,own-1,other-0", "", []string{"own-0", "own-1"}, 0, allSets) + if !found[0] || !found[1] { + t.Errorf("found = %v, want both true", found) + } + if len(leaked) != 1 || !strings.Contains(leaked[0], "other-0") || !strings.Contains(leaked[0], "series=1") { + t.Errorf("leaked = %v, want one entry naming other-0 from series=1", leaked) + } + }) +} + +func TestFindLeakedUUIDs(t *testing.T) { + allSets := [][]string{ + {"uuid-0-a", "uuid-0-b"}, // 0 + {"uuid-1-a", "uuid-1-b"}, // 1 + {"uuid-2-a", "uuid-2-b"}, // 2 + } + + t.Run("no leak: response only contains own uuids", func(t *testing.T) { + got := FindLeakedUUIDs("uuid-0-a,uuid-0-b", "", 0, allSets) + if len(got) != 0 { + t.Errorf("expected no leaks, got %v", got) + } + }) + + t.Run("single leak from another set", func(t *testing.T) { + got := FindLeakedUUIDs("uuid-0-a,uuid-1-a", "", 0, allSets) + if len(got) != 1 { + t.Fatalf("expected 1 leak, got %v", got) + } + if !strings.Contains(got[0], "uuid-1-a") || !strings.Contains(got[0], "series=1") { + t.Errorf("leak entry %q missing uuid or owning index", got[0]) + } + }) + + t.Run("multiple leaks, deterministic order", func(t *testing.T) { + resp := "uuid-0-a,uuid-1-a,uuid-2-b" + got1 := FindLeakedUUIDs(resp, "", 0, allSets) + got2 := FindLeakedUUIDs(resp, "", 0, allSets) + if len(got1) != 2 { + t.Fatalf("expected 2 leaks, got %v", got1) + } + for i := range got1 { + if got1[i] != got2[i] { + t.Errorf("non-deterministic leak order: %v vs %v", got1, got2) + } + } + }) + + t.Run("leak detected in thinking, not just response", func(t *testing.T) { + got := FindLeakedUUIDs("uuid-0-a", "I recall uuid-2-a from earlier", 0, allSets) + if len(got) != 1 || !strings.Contains(got[0], "uuid-2-a") { + t.Errorf("expected leak of uuid-2-a via thinking, got %v", got) + } + }) + + t.Run("own index never reported as a leak", func(t *testing.T) { + got := FindLeakedUUIDs("uuid-1-a,uuid-1-b", "", 1, allSets) + if len(got) != 0 { + t.Errorf("expected no leaks for own index, got %v", got) + } + }) +} + +func TestCapRecitedUUIDs(t *testing.T) { + t.Run("unbounded budget (<=0) never trims", func(t *testing.T) { + in := []string{"a", "b", "c"} + got, trimmed := capRecitedUUIDs(in, 0) + if trimmed { + t.Errorf("expected no trim with maxOutputTokens<=0") + } + if len(got) != len(in) { + t.Errorf("got %v, want unchanged %v", got, in) + } + }) + + t.Run("small list well within budget is untouched", func(t *testing.T) { + in := []string{"11111111-1111-1111-1111-111111111111"} + got, trimmed := capRecitedUUIDs(in, 10000) + if trimmed { + t.Errorf("expected no trim for a single uuid against a large budget") + } + if len(got) != 1 { + t.Errorf("got %v, want unchanged", got) + } + }) + + t.Run("large list against a tiny budget is trimmed to the most recent entries", func(t *testing.T) { + in := make([]string, 200) + for i := range in { + in[i] = "11111111-1111-1111-1111-11111111111" + string(rune('0'+i%10)) + } + got, trimmed := capRecitedUUIDs(in, 20) // tiny budget forces a cap + if !trimmed { + t.Fatalf("expected trimming for a 200-uuid list against a 20-token budget") + } + if len(got) == 0 || len(got) >= len(in) { + t.Fatalf("got %d entries, want a proper subset of %d", len(got), len(in)) + } + // Kept entries must be the MOST RECENT (tail) of the input, in order. + wantTail := in[len(in)-len(got):] + for i := range got { + if got[i] != wantTail[i] { + t.Errorf("capRecitedUUIDs did not keep the most-recent tail: got[%d]=%q, want %q", i, got[i], wantTail[i]) + } + } + }) +} diff --git a/benchmark/types.go b/benchmark/types.go index 8dbc3b2..5510b9a 100644 --- a/benchmark/types.go +++ b/benchmark/types.go @@ -23,5 +23,12 @@ type RequestMetrics struct { CachedPrompt string // Full system prompt text (only populated on error/empty for diagnostics) Question string // The user question sent with the request (only populated on error/empty) RawResponseTail string // raw SSE tail (last bytes); only populated on error/empty for diagnostics -} + // UUID validation (dataset-replay --replay-inject-uuids only). All nil/zero + // when the feature is off (default) or on the router-replay/synthetic paths, + // which never populate these. + ConvIdx int // conversation index within cfg.replayConversations / cfg.replayUUIDSets + ExpectedUUIDs []string // this conversation's in-scope ref-id UUIDs at this turn (defensive copy) + UUIDFound []bool // parallel to ExpectedUUIDs: whether each was found in Response or thinking + LeakedUUIDs []string // "uuid(series=N)" entries for any OTHER conversation's UUID found here +} diff --git a/cli/benchmark_commands.go b/cli/benchmark_commands.go index 3fc8abf..2761e66 100644 --- a/cli/benchmark_commands.go +++ b/cli/benchmark_commands.go @@ -269,6 +269,11 @@ func (c *BenchmarkAutoCommand) Execute(args []string) error { ReplayNoStamp: c.ReplayNoStamp, AbortOnCollapse: c.AbortOnCollapse, ReplayStopAtLowConcurrency: c.ReplayStopAtLowConcurrency, + ReplayInjectUUIDs: c.ReplayInjectUUIDs, + ReplayUUIDsPerTurn: c.ReplayUUIDsPerTurn, + ReplayUUIDSeed: c.ReplayUUIDSeed, + ReplayUUIDMode: c.ReplayUUIDMode, + ReplayReciteEveryTurn: c.ReplayReciteEveryTurn != "false", RouterReplayFile: c.RouterReplayFile, RouterReplayRoles: c.RouterReplayRoles, DryRun: c.DryRun, @@ -282,6 +287,21 @@ func (c *BenchmarkAutoCommand) Execute(args []string) error { if c.FromDataset != "" && c.RouterReplayFile != "" { return fmt.Errorf("--from-dataset and --router-replay-file are mutually exclusive") } + + // --replay-inject-uuids is DATASET PATH ONLY: router replay reconstructs + // prefixes from block hashes+token counts, so injecting visible ref-id + // text there would diverge those hashes and break cache-hit reproduction. + if c.ReplayInjectUUIDs { + if c.FromDataset == "" { + return fmt.Errorf("--replay-inject-uuids requires --from-dataset") + } + if c.RouterReplayFile != "" { + return fmt.Errorf("--replay-inject-uuids and --router-replay-file are mutually exclusive") + } + } + if c.ReplayUUIDMode != "human" && c.ReplayUUIDMode != "all-non-gpt" { + return fmt.Errorf("--replay-uuid-mode must be 'human' or 'all-non-gpt', got %q", c.ReplayUUIDMode) + } if c.DryRun && c.RouterReplayFile == "" { return fmt.Errorf("--dry-run requires --router-replay-file") } diff --git a/cli/benchmark_options.go b/cli/benchmark_options.go index 409e2a6..585b719 100644 --- a/cli/benchmark_options.go +++ b/cli/benchmark_options.go @@ -63,16 +63,26 @@ type BenchmarkAutoOptions struct { ReplayNoStamp bool `long:"replay-no-stamp" description:"Disable per-run RUN_GUID stamping in replay mode. By default each replay run prepends a fresh UUID to every request's system prompt (both --from-dataset and --router-replay-file paths) so server prefix caches from prior runs can't be reused — pristine per-run cache state." env:"BENCHMARK_REPLAY_NO_STAMP"` AbortOnCollapse bool `long:"abort-on-collapse" description:"Abort the benchmark if the windowed cache hit rate stays below 50% for 2 minutes. Off by default — this heuristic fires on legitimate workloads with low cache reuse (e.g. replay across many distinct conversations)." env:"BENCHMARK_ABORT_ON_COLLAPSE"` ReplayStopAtLowConcurrency bool `long:"replay-stop-at-low-concurrency" description:"Terminate the replay run once the queue is drained AND the number of active worker goroutines has dropped below --concurrency. Avoids long-tail measurements where only a handful of long conversations remain and the gate is underutilized." env:"BENCHMARK_REPLAY_STOP_AT_LOW_CONCURRENCY"` - RouterReplayFile string `long:"router-replay-file" description:"Path to a tree-aware replay file produced by 'wekai router replay-prepare'. Each series = one CLI session; within the session sub-agents fan out concurrently, honoring parent->child sequencing baked into the file. Mutually exclusive with --from-dataset." env:"BENCHMARK_ROUTER_REPLAY_FILE"` - RouterReplayRoles string `long:"router-replay-roles" description:"Comma-separated list of instance roles to replay (default: all). E.g. 'main,sub-agent' excludes the CLI's background helper:title / helper:summarize / ephemeral (haiku side-calls) instances and keeps only the agentic workload — gives a cleaner 'in_flight ~= series' steady state. Other values: 'helper-or-isolated', 'ephemeral (no system)', 'other'." env:"BENCHMARK_ROUTER_REPLAY_ROLES"` - RouterReplaySeriesIndices string `long:"replay-series-indices" description:"Comma-separated list of 0-based session indices to replay from --router-replay-file (e.g. '3,7,42'). Only sessions at those line positions (0 = first session after the header) are dispatched; others are skipped. Mutually exclusive with --replay-series-range. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_INDICES"` - RouterReplaySeriesRange string `long:"replay-series-range" description:"Inclusive range of 0-based session indices to replay from --router-replay-file (e.g. '0-50' or '100-199'). Mutually exclusive with --replay-series-indices. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_RANGE"` - DryRun bool `long:"dry-run" description:"Dry run: skip remote HTTP requests; drive the router-replay pipeline with synthetic timing so gcache evolution can be observed offline. Requires --router-replay-file." env:"BENCHMARK_DRY_RUN"` - DryRunColdTPS int `long:"dry-run-cold-tps" default:"1000000" description:"Dry-run: cold (uncached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_COLD_TPS"` - DryRunWarmTPS int `long:"dry-run-warm-tps" default:"10000000" description:"Dry-run: warm (cached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_WARM_TPS"` - DryRunOutputTPS int `long:"dry-run-output-tps" default:"100000" description:"Dry-run: output tokens generated per second." env:"BENCHMARK_DRY_RUN_OUTPUT_TPS"` - CacheSimChunkBytes int `long:"cache-sim-chunk-bytes" description:"Chunk size in bytes for the content-level cache estimator (0 = default 1024)." default:"0" env:"BENCHMARK_CACHE_SIM_CHUNK_BYTES"` - RandomGateOrder string `long:"random-gate-order" choice:"true" choice:"false" default:"true" optional:"yes" optional-value:"true" description:"Wake the concurrency gate's waiting series in uniformly random order when oversubscribed (the DEFAULT). Strict FIFO forces every series to wait behind all other waiting series before its next turn -- the adversarial worst case for GPU prefix-cache LRU. Pass --random-gate-order=false for the legacy exact-FIFO order. Cold-start waiters are unaffected (always served first, FIFO)." env:"BENCHMARK_RANDOM_GATE_ORDER"` + ReplayInjectUUIDs bool `long:"replay-inject-uuids" description:"Inject per-turn UUID markers into --from-dataset replay conversations and validate their presence in later responses -- a coherency check (PRESENCE_MISS / CROSS_CONTAMINATION) for the KV-offload path under realistic multi-turn traffic. DATASET PATH ONLY: requires --from-dataset; rejected together with --router-replay-file (router replay reconstructs prefixes from block hashes+token counts, and injecting visible text there would break cache-hit reproduction)." env:"BENCHMARK_REPLAY_INJECT_UUIDS"` + ReplayUUIDsPerTurn int `long:"replay-uuids-per-turn" description:"Number of UUID ref-ids injected per injectable turn (see --replay-uuid-mode). Only used with --replay-inject-uuids." default:"1" env:"BENCHMARK_REPLAY_UUIDS_PER_TURN"` + ReplayUUIDSeed int64 `long:"replay-uuid-seed" description:"PRNG seed for --replay-inject-uuids' UUID generation (0 = crypto/rand, non-deterministic across runs)." default:"0" env:"BENCHMARK_REPLAY_UUID_SEED"` + ReplayUUIDMode string `long:"replay-uuid-mode" description:"Which turns get UUID markers under --replay-inject-uuids: 'human' (default, only human/user turns) or 'all-non-gpt' (also tool-result turns and stray system turns)." default:"human" env:"BENCHMARK_REPLAY_UUID_MODE"` + // ReplayReciteEveryTurn is a string (not bool) choice, same workaround as + // RandomGateOrder above: a plain bool flag defaulting true can never be + // turned OFF via the CLI with this parser (go-flags only accepts + // `--flag=false` for bool options when the parser is built with + // AllowBoolValues, which main.go's flags.Default does not set). + ReplayReciteEveryTurn string `long:"replay-recite-every-turn" choice:"true" choice:"false" default:"true" optional:"yes" optional-value:"true" description:"With --replay-inject-uuids, ask the model to recite every ref-id seen so far on EVERY turn (the DEFAULT) rather than only on the conversation's final turn. Pass --replay-recite-every-turn=false to recite only on the final turn." env:"BENCHMARK_REPLAY_RECITE_EVERY_TURN"` + RouterReplayFile string `long:"router-replay-file" description:"Path to a tree-aware replay file produced by 'wekai router replay-prepare'. Each series = one CLI session; within the session sub-agents fan out concurrently, honoring parent->child sequencing baked into the file. Mutually exclusive with --from-dataset." env:"BENCHMARK_ROUTER_REPLAY_FILE"` + RouterReplayRoles string `long:"router-replay-roles" description:"Comma-separated list of instance roles to replay (default: all). E.g. 'main,sub-agent' excludes the CLI's background helper:title / helper:summarize / ephemeral (haiku side-calls) instances and keeps only the agentic workload — gives a cleaner 'in_flight ~= series' steady state. Other values: 'helper-or-isolated', 'ephemeral (no system)', 'other'." env:"BENCHMARK_ROUTER_REPLAY_ROLES"` + RouterReplaySeriesIndices string `long:"replay-series-indices" description:"Comma-separated list of 0-based session indices to replay from --router-replay-file (e.g. '3,7,42'). Only sessions at those line positions (0 = first session after the header) are dispatched; others are skipped. Mutually exclusive with --replay-series-range. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_INDICES"` + RouterReplaySeriesRange string `long:"replay-series-range" description:"Inclusive range of 0-based session indices to replay from --router-replay-file (e.g. '0-50' or '100-199'). Mutually exclusive with --replay-series-indices. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_RANGE"` + DryRun bool `long:"dry-run" description:"Dry run: skip remote HTTP requests; drive the router-replay pipeline with synthetic timing so gcache evolution can be observed offline. Requires --router-replay-file." env:"BENCHMARK_DRY_RUN"` + DryRunColdTPS int `long:"dry-run-cold-tps" default:"1000000" description:"Dry-run: cold (uncached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_COLD_TPS"` + DryRunWarmTPS int `long:"dry-run-warm-tps" default:"10000000" description:"Dry-run: warm (cached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_WARM_TPS"` + DryRunOutputTPS int `long:"dry-run-output-tps" default:"100000" description:"Dry-run: output tokens generated per second." env:"BENCHMARK_DRY_RUN_OUTPUT_TPS"` + CacheSimChunkBytes int `long:"cache-sim-chunk-bytes" description:"Chunk size in bytes for the content-level cache estimator (0 = default 1024)." default:"0" env:"BENCHMARK_CACHE_SIM_CHUNK_BYTES"` + RandomGateOrder string `long:"random-gate-order" choice:"true" choice:"false" default:"true" optional:"yes" optional-value:"true" description:"Wake the concurrency gate's waiting series in uniformly random order when oversubscribed (the DEFAULT). Strict FIFO forces every series to wait behind all other waiting series before its next turn -- the adversarial worst case for GPU prefix-cache LRU. Pass --random-gate-order=false for the legacy exact-FIFO order. Cold-start waiters are unaffected (always served first, FIFO)." env:"BENCHMARK_RANDOM_GATE_ORDER"` // Positional arguments Args struct { diff --git a/cli/command_misc_cache_coherency_test.go b/cli/command_misc_cache_coherency_test.go index 8be1a9c..0248ff6 100644 --- a/cli/command_misc_cache_coherency_test.go +++ b/cli/command_misc_cache_coherency_test.go @@ -61,58 +61,7 @@ func TestResolveGarbageChars(t *testing.T) { } } -func TestFindLeakedUUIDs(t *testing.T) { - seriesUUIDs := [][]string{ - {"uuid-0-a", "uuid-0-b"}, // series 0 - {"uuid-1-a", "uuid-1-b"}, // series 1 - {"uuid-2-a", "uuid-2-b"}, // series 2 - } - - t.Run("no leak: response only contains own series uuids", func(t *testing.T) { - resp := "uuid-0-a,uuid-0-b" - got := findLeakedUUIDs(resp, "", 0, seriesUUIDs) - if len(got) != 0 { - t.Errorf("expected no leaks, got %v", got) - } - }) - - t.Run("single leak from another series in response", func(t *testing.T) { - resp := "uuid-0-a,uuid-1-a" - got := findLeakedUUIDs(resp, "", 0, seriesUUIDs) - if len(got) != 1 { - t.Fatalf("expected 1 leak, got %v", got) - } - if !strings.Contains(got[0], "uuid-1-a") || !strings.Contains(got[0], "series=1") { - t.Errorf("leak entry %q missing uuid or owning series", got[0]) - } - }) - - t.Run("multiple leaks from multiple series, deterministic order", func(t *testing.T) { - resp := "uuid-0-a,uuid-1-a,uuid-2-b" - got1 := findLeakedUUIDs(resp, "", 0, seriesUUIDs) - got2 := findLeakedUUIDs(resp, "", 0, seriesUUIDs) - if len(got1) != 2 { - t.Fatalf("expected 2 leaks, got %v", got1) - } - // Order must be deterministic (series-index order) across repeated calls. - for i := range got1 { - if got1[i] != got2[i] { - t.Errorf("non-deterministic leak order: %v vs %v", got1, got2) - } - } - }) - - t.Run("leak detected in thinking, not just response", func(t *testing.T) { - got := findLeakedUUIDs("uuid-0-a", "I recall uuid-2-a from earlier", 0, seriesUUIDs) - if len(got) != 1 || !strings.Contains(got[0], "uuid-2-a") { - t.Errorf("expected leak of uuid-2-a via thinking, got %v", got) - } - }) - - t.Run("own series never reported as a leak", func(t *testing.T) { - got := findLeakedUUIDs("uuid-1-a,uuid-1-b", "", 1, seriesUUIDs) - if len(got) != 0 { - t.Errorf("expected no leaks for own series, got %v", got) - } - }) -} +// TestFindLeakedUUIDs moved to benchmark.FindLeakedUUIDs's own test +// (benchmark/replay_uuid_test.go) — the function itself moved to the +// benchmark package (benchmark/replay_uuid.go) so both this CLI and +// dataset-replay UUID validation share one implementation. diff --git a/cli/eval_commands.go b/cli/eval_commands.go index 186aa3d..6a9010e 100644 --- a/cli/eval_commands.go +++ b/cli/eval_commands.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "os" - "strings" "github.com/weka/wekai/benchmark" "github.com/weka/wekai/config" @@ -197,7 +196,7 @@ func (c *EvalCacheCoherencyCommand) Execute(args []string) error { } // Any uuid belonging to a DIFFERENT series found in this response/thinking // is cross-contamination (KV/scheduling leak across series). - leaked = findLeakedUUIDs(r.Response, r.Thinking, r.SeriesIdx, result.SeriesUUIDs) + leaked = benchmark.FindLeakedUUIDs(r.Response, r.Thinking, r.SeriesIdx, result.SeriesUUIDs) } uuidMissingFlakyCount += len(missing) crossContamCount += len(leaked) @@ -333,22 +332,6 @@ func resolveGarbageChars(garbageCharacters, garbageTokens int, w io.Writer) int } } -// findLeakedUUIDs scans resp and thinking for UUIDs belonging to a series OTHER than -// ownSeries, per the ordered seriesUUIDs list (seriesUUIDs[i] = full UUID stamp list of -// series i — this doubles as the uuid -> owning-series mapping without needing an actual -// map, keeping iteration order — and therefore leak-report order — deterministic for a -// given seed). Returns "uuid(series=N)" entries, one per leaked UUID found. -func findLeakedUUIDs(resp, thinking string, ownSeries int, seriesUUIDs [][]string) []string { - var leaked []string - for si, uuids := range seriesUUIDs { - if si == ownSeries { - continue - } - for _, u := range uuids { - if strings.Contains(resp, u) || strings.Contains(thinking, u) { - leaked = append(leaked, fmt.Sprintf("%s(series=%d)", u, si)) - } - } - } - return leaked -} +// findLeakedUUIDs moved to benchmark.FindLeakedUUIDs (benchmark/replay_uuid.go) +// so both this CLI and the dataset-replay UUID validation path share one +// implementation. From d1f131520da25ed225ea85409ecc3ebdcde56b44 Mon Sep 17 00:00:00 2001 From: Ford Shaper Date: Fri, 24 Jul 2026 12:33:37 -0400 Subject: [PATCH 2/6] feat(benchmark): retarget UUID cache-coherency injection to router-replay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-targets the per-turn UUID coherency check from the dataset-replay path (--from-dataset) to the tree-aware router-replay path (--router-replay-file), composing it with the existing forced-output feature (ignore_eos + continue-generating instruction). Design (Option C — boundary injection, with tail fallback): one deterministic UUID is injected per SESSION at the boundary between its cross-session-shared leading blocks (system/tools) and its per-session content. This lands the marker in a region cached WITHIN a session (later requests repeat it, giving a genuine KV-coherency signal) while leaving the cross-session shared prefix byte-identical, preserving cache-hit reproduction against the original capture. A session with no shared leading block falls back to tail injection. New benchmark/replay_router_uuid.go: computeBlockSessionCounts (streams a replay-v3 file once, counting distinct-session references per block hash), sharedPrefixBlockCount, buildSessionUUIDs, replayReciteFromContextInstruction (a recite ask that never embeds the UUID itself, so presence reflects genuine recall), and a max_tokens recite floor (with a one-time warning) so a tiny captured output budget can't truncate the recite line into a false PRESENCE_MISS. benchmark/replay_router_wire.go: buildAnthropicMessagesBody / buildOpenAIChatCompletionsBody take a new *uuidInjection param (nil = unchanged); the marker splices into the system prefix at the boundary, or falls back to the message tail, preserving Anthropic's strict user/ assistant role alternation. benchmark/replay_router_post.go / replay_router.go: replayPoster carries the per-session UUID/marker state and builds the injection per request; after a successful response, validates it via the existing validateReplayResponse/FindLeakedUUIDs primitives (unchanged, shared with the eval CLI). benchmark/auto.go precomputes the per-session UUID array and block-hash counts once before per-model goroutines spawn, sized from the header + --replay-series/--replay-series-indices (the same formula the router stream itself uses), so no lazy-growth is needed. Drops the dataset-path UUID wiring (replay.go, replay_uuid.go) that --replay-inject-uuids used before: buildReplayUUIDSets, computeInScopeAtEachGptTurn, replayTurnInjectable, replayReciteInstruction, capRecitedUUIDs, and the --replay-uuid-mode / --replay-uuids-per-turn flags are all removed. --replay-inject-uuids now requires --router-replay-file and is rejected with --from-dataset (flip of the previous gate). --replay-recite-every-turn is renamed --replay-recite-every-request. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/auto.go | 111 +++-- benchmark/replay.go | 91 +--- benchmark/replay_double_count_test.go | 6 +- benchmark/replay_router.go | 21 +- benchmark/replay_router_post.go | 80 +++- benchmark/replay_router_post_test.go | 30 +- benchmark/replay_router_uuid.go | 294 +++++++++++++ benchmark/replay_router_uuid_test.go | 489 +++++++++++++++++++++ benchmark/replay_router_wire.go | 155 ++++++- benchmark/replay_router_wire_test.go | 32 +- benchmark/replay_router_wire_tools_test.go | 12 +- benchmark/replay_uuid.go | 185 +------- benchmark/replay_uuid_test.go | 229 ---------- benchmark/types.go | 12 +- cli/benchmark_commands.go | 23 +- cli/benchmark_options.go | 38 +- 16 files changed, 1196 insertions(+), 612 deletions(-) create mode 100644 benchmark/replay_router_uuid.go create mode 100644 benchmark/replay_router_uuid_test.go diff --git a/benchmark/auto.go b/benchmark/auto.go index 2719842..0774949 100644 --- a/benchmark/auto.go +++ b/benchmark/auto.go @@ -89,29 +89,38 @@ type AutoBenchmarkConfig struct { // remains the normal budget C (--concurrency). ReplayStopAtLowConcurrency bool - // UUID-based response validation (--replay-inject-uuids). DATASET PATH - // ONLY (cfg.FromDataset != ""); the CLI rejects this combined with - // --router-replay-file (see cli/benchmark_commands.go) — router replay - // reconstructs prefixes from block hashes+token counts, and injecting - // visible text there would break cache-hit reproduction. - ReplayInjectUUIDs bool // inject per-turn UUID markers and validate their presence in later responses - // ReplayUUIDsPerTurn is how many UUIDs each injectable turn carries. - ReplayUUIDsPerTurn int + // UUID-based cache-coherency validation (--replay-inject-uuids). ROUTER + // PATH ONLY (cfg.RouterReplayFile != ""); the CLI rejects this combined + // with --from-dataset (see cli/benchmark_commands.go). One deterministic + // UUID is injected per SESSION at the boundary between its + // cross-session-shared leading blocks and its per-session content (see + // replay_router_uuid.go for the full design) — this puts the marker in + // a region cached WITHIN a session (later requests in the same session + // repeat it) while leaving the cross-session shared prefix byte- + // identical, so cache-hit reproduction against the original capture is + // preserved. + ReplayInjectUUIDs bool // ReplayUUIDSeed seeds the UUID generator (see newUUIDGenerator); 0 = crypto/rand // (non-deterministic across runs). ReplayUUIDSeed int64 - // ReplayUUIDMode selects which turns are injectable: "human" (default) or - // "all-non-gpt" (also tool + stray system turns). See replayTurnInjectable. - ReplayUUIDMode string - // ReplayReciteEveryTurn: ask the model to recite every ref-id seen so far on - // EVERY turn (default true), not just the conversation's final turn. - ReplayReciteEveryTurn bool - // replayUUIDSets is the precomputed per-conversation UUID list (parallel to - // replayConversations), populated once by RunAutoBenchmark before any - // per-model goroutine spawns — see buildReplayUUIDSets and the comment on - // replayConversations above (same sharing rationale: avoid recomputing N - // times in parallel, and every model must see the identical assignment). + // ReplayReciteEveryRequest: ask the model to recite the ref-id marker on + // EVERY request (default true), not just each instance's final request. + ReplayReciteEveryRequest bool + // replayUUIDSets is the precomputed per-session UUID list, populated + // once by RunAutoBenchmark before any per-model goroutine spawns — see + // buildSessionUUIDs. Index i = session i's owned singleton UUID set + // (index i corresponds to seriesNum-1, the order sessions are + // dispatched in — see the sizing note at the router-replay precompute + // call site). Shared, read-only, across every model in a multi-model + // run so every model sees the identical assignment (same sharing + // rationale as replayConversations below). replayUUIDSets [][]string + // replayBlockSessionCounts maps a replay-v3 block hash to the number of + // DISTINCT SESSIONS that reference it (see computeBlockSessionCounts) — + // a hash referenced by more than one session is safe to leave + // byte-identical across those sessions' requests. Populated once + // alongside replayUUIDSets. + replayBlockSessionCounts map[string]int // RunID is populated internally by RunAutoBenchmark at the start of each // run. It's the UUID injected into every conversation's system prompt @@ -226,7 +235,7 @@ type requestDataRecord struct { ResponseText string `json:"response_text,omitempty"` RawResponseTail string `json:"raw_response_tail,omitempty"` - // UUID validation (dataset-replay --replay-inject-uuids only). The three + // UUID validation (router-replay --replay-inject-uuids only). The three // counts are always populated (0 when the feature is off); the raw detail // lists are populated ONLY on a miss or a leak (mirrors the // failed-request-only policy above — avoid bloating every row). @@ -2523,17 +2532,6 @@ func RunAutoBenchmark(ctx context.Context, cfg AutoBenchmarkConfig) error { } cfg.replayConversations = convs fmt.Printf("Loaded %d conversations. Starting auto benchmark...\n\n", len(convs)) - - // Precompute the per-conversation UUID sets ONCE here, before any - // per-model goroutine spawns below, so every model's - // runSingleModelBenchmark sees the identical assignment (same - // sharing rationale as replayConversations itself — see its doc - // comment on AutoBenchmarkConfig). - if cfg.ReplayInjectUUIDs { - cfg.replayUUIDSets = buildReplayUUIDSets(cfg.replayConversations, cfg.ReplayUUIDSeed, cfg.ReplayUUIDsPerTurn, cfg.ReplayUUIDMode) - fmt.Printf("UUID validation enabled: %d conversation(s) prepared (mode=%s, per-turn=%d, seed=%d)\n", - len(cfg.replayUUIDSets), cfg.ReplayUUIDMode, cfg.ReplayUUIDsPerTurn, cfg.ReplayUUIDSeed) - } } // Tree-aware router replay: only the header (line 1) is read here so @@ -2553,6 +2551,57 @@ func RunAutoBenchmark(ctx context.Context, cfg AutoBenchmarkConfig) error { fmt.Printf("Header: %d sessions / %d instances / %d requests / %d fan-out turns / max fan-out %d\n\n", hdr.Summary.Sessions, hdr.Summary.Instances, hdr.Summary.Requests, hdr.Summary.FanOutTurns, hdr.Summary.MaxFanOutInOneTurn) + + // Precompute UUID cache-coherency injection ONCE here, before any + // per-model goroutine spawns below, so every model's + // runSingleModelBenchmark sees the identical per-session assignment + // (same sharing rationale as replayConversations above). + if cfg.ReplayInjectUUIDs { + // Effective session count: the SAME formula + // openRouterReplayStream uses to compute its own `total` (see + // replay_router.go), so the array is sized to exactly the + // number of sessions that will ever be dispatched (dispatch + // order == array index, since both walk the file in the same + // filtered, sequential order). This is what makes true + // lazy-growth unnecessary: the count is fully determined by + // the header + --replay-series + --replay-series-indices/ + // --replay-series-range before any session is ever pulled. + effectiveSessions := hdr.Summary.Sessions + if cfg.ReplaySeries > 0 && cfg.ReplaySeries < effectiveSessions { + effectiveSessions = cfg.ReplaySeries + } + if len(cfg.RouterReplaySeriesIndices) > 0 { + effectiveSessions = len(cfg.RouterReplaySeriesIndices) + if cfg.ReplaySeries > 0 && cfg.ReplaySeries < effectiveSessions { + effectiveSessions = cfg.ReplaySeries + } + } + if effectiveSessions <= 0 { + fmt.Fprintf(os.Stderr, + "[router-replay] WARNING: --replay-inject-uuids could not determine an effective session count (header sessions=%d) — UUID injection disabled for this run\n", + hdr.Summary.Sessions) + } else { + counts, cerr := computeBlockSessionCounts(cfg.RouterReplayFile, cfg.RouterReplaySeriesIndices, cfg.ReplaySeries) + if cerr != nil { + return fmt.Errorf("compute block session counts for --replay-inject-uuids: %w", cerr) + } + cfg.replayBlockSessionCounts = counts + cfg.replayUUIDSets = buildSessionUUIDs(effectiveSessions, cfg.ReplayUUIDSeed) + + sharedHashes := 0 + for _, n := range counts { + if n > 1 { + sharedHashes++ + } + } + usable, total, berr := countSessionsWithUsableBoundary(cfg.RouterReplayFile, cfg.RouterReplaySeriesIndices, cfg.ReplaySeries, counts) + if berr != nil { + return fmt.Errorf("compute usable-boundary diagnostic for --replay-inject-uuids: %w", berr) + } + fmt.Printf("UUID validation enabled: %d session(s) prepared, %d cross-session-shared block hash(es), %d/%d sessions have a usable boundary (%d fall back to tail injection) (recite-every-request=%v, seed=%d)\n", + effectiveSessions, sharedHashes, usable, total, total-usable, cfg.ReplayReciteEveryRequest, cfg.ReplayUUIDSeed) + } + } } // Create per-run subdirectory for request data if configured. diff --git a/benchmark/replay.go b/benchmark/replay.go index b67f821..3eab5bd 100644 --- a/benchmark/replay.go +++ b/benchmark/replay.go @@ -116,10 +116,8 @@ func runReplaySeriesLoop( // benchmark request per gpt turn. Errors on individual requests are recorded // but don't abort the series — the next turn still runs. // -// convIdx is this conversation's index into cfg.replayConversations / -// cfg.replayUUIDSets (== seriesNum-1, passed explicitly rather than -// re-derived so the UUID-validation wiring below doesn't have to assume that -// relationship). +// convIdx is this conversation's index into cfg.replayConversations +// (== seriesNum-1, passed explicitly rather than re-derived). // // Returns true if the whole conversation was walked; false if a stop signal // (--total reached, context cancel) cut it short mid-walk. @@ -206,30 +204,6 @@ func runReplayConversation( var pending strings.Builder turnNum := 0 - // UUID validation setup (--replay-inject-uuids, dataset path only). All of - // this is inert when the flag is off: uuidSets stays nil, injecting is - // false, and every gate below short-circuits. - injecting := cfg.ReplayInjectUUIDs - var uuidSets []string - if injecting && convIdx >= 0 && convIdx < len(cfg.replayUUIDSets) { - uuidSets = cfg.replayUUIDSets[convIdx] - } - uuidCursor := 0 - var inScope []string - reciteTruncWarned := false // logs the max-tokens recite-cap WARNING at most once per conversation - - // lastGptIdx locates this conversation's FINAL 'gpt' turn so the recite - // instruction still goes out at least once (on that turn) even when - // --replay-recite-every-turn=false. - lastGptIdx := -1 - if injecting { - for i := firstIdx; i < len(conv.Turns); i++ { - if conv.Turns[i].From == "gpt" { - lastGptIdx = i - } - } - } - flush := func(gptIdx int) bool { userContent := strings.TrimSpace(pending.String()) pending.Reset() @@ -270,32 +244,9 @@ func runReplayConversation( history.WriteString(userContent) ratio := st.estimator.Observe(history.String()) - // Append the recite-every-seen-ref-id instruction, if injecting. Cap - // the recited list to a fraction of the output budget first — an - // uncapped list only grows every turn and can eventually ask the model - // to reproduce more ref-ids than max_tokens can hold, truncating the - // SEEN_REFS line itself; ExpectedUUIDs is set to the SAME (possibly - // capped) list so a truncation-induced gap is never misread as a - // PRESENCE_MISS. - outgoingContent := userContent - var expectedSnapshot []string - if injecting { - recited, truncated := capRecitedUUIDs(inScope, cfg.MaxOutputTokens) - if truncated && !reciteTruncWarned { - reciteTruncWarned = true - fmt.Fprintf(os.Stderr, - "[auto][%s] WARNING: replay UUID recite list capped to fit --max-output-tokens budget (conv=%d) — PRESENCE_MISS on ref-ids dropped from recitation is expected, not corruption\n", - shortModelName(cfg.Model), convIdx) - } - expectedSnapshot = append([]string(nil), recited...) - if cfg.ReplayReciteEveryTurn || gptIdx == lastGptIdx { - outgoingContent = userContent + replayReciteInstruction(recited) - } - } - reqCtx, reqCancel := context.WithTimeout(benchCtx, reqTimeout) resetTTFT(time.Now()) - response, err := chat.Request(reqCtx, llm.TextParts(outgoingContent), nil) + response, err := chat.Request(reqCtx, llm.TextParts(userContent), nil) totalTime := time.Since(startTime) reqCancel() gate.Release() @@ -310,14 +261,8 @@ func runReplayConversation( TotalResponseTime: totalTime, Error: err, } - // respThinking is captured separately from metrics.Response (which may - // get overwritten by response.Thinking below when content is empty) so - // UUID validation can always scan content ∪ thinking, exactly like the - // cache-coherency eval does. - var respThinking string if response != nil { metrics.Response = response.Content - respThinking = response.Thinking if strings.TrimSpace(metrics.Response) == "" { metrics.Response = response.Thinking } @@ -345,18 +290,6 @@ func runReplayConversation( metrics.LocalCacheRatio = ratio - if injecting { - metrics.ConvIdx = convIdx - metrics.ExpectedUUIDs = expectedSnapshot - // ERROR responses (including the synthetic "empty response" error - // above) are excluded from validation — no usable content/thinking - // to scan. - if metrics.Error == nil { - metrics.UUIDFound, metrics.LeakedUUIDs = validateReplayResponse( - metrics.Response, respThinking, metrics.ExpectedUUIDs, convIdx, cfg.replayUUIDSets) - } - } - recordReplayRequest(cfg, st, rdw, metrics, isFirstRequest, &coldStartTTFT) isFirstRequest = false return true @@ -373,23 +306,7 @@ func runReplayConversation( if pending.Len() > 0 { pending.WriteString("\n\n") } - turnValue := t.Value - // UUID injection: this cursor/slicing logic MUST mirror - // computeInScopeAtEachGptTurn (replay_uuid.go) exactly — the test - // suite (replay_uuid_test.go) asserts the two stay in lockstep. - if injecting && replayTurnInjectable(t, cfg.ReplayUUIDMode) { - end := uuidCursor + cfg.ReplayUUIDsPerTurn - if end > len(uuidSets) { - end = len(uuidSets) - } - if uuidCursor < end { - turnUUIDs := uuidSets[uuidCursor:end] - inScope = append(inScope, turnUUIDs...) - turnValue = injectUUIDMarker(turnValue, turnUUIDs) - } - uuidCursor = end - } - pending.WriteString(turnValue) + pending.WriteString(t.Value) } // Trailing non-gpt content (if any) is discarded — there's no assistant // response to measure for it. diff --git a/benchmark/replay_double_count_test.go b/benchmark/replay_double_count_test.go index 22e78e6..bcabfff 100644 --- a/benchmark/replay_double_count_test.go +++ b/benchmark/replay_double_count_test.go @@ -125,7 +125,7 @@ func TestReplayConsumerInputTokensAreNetOfCache_Streaming(t *testing.T) { defer ts.Close() st := &autoState{stream: newCompletionStream(200)} - metrics := p.do(context.Background(), minReplayReq(true), strings.Repeat("x", 300), 1, "s1", "i1", 1, st) + metrics := p.do(context.Background(), minReplayReq(true), strings.Repeat("x", 300), 1, "s1", "i1", 1, st, true) if metrics.Error != nil { t.Fatalf("unexpected error: %v", metrics.Error) } @@ -180,7 +180,7 @@ func TestReplayConsumerInputTokensAreNetOfCache_Plain(t *testing.T) { defer ts.Close() st := &autoState{stream: newCompletionStream(200)} - metrics := p.do(context.Background(), minReplayReq(false), strings.Repeat("x", 300), 1, "s1", "i1", 1, st) + metrics := p.do(context.Background(), minReplayReq(false), strings.Repeat("x", 300), 1, "s1", "i1", 1, st, true) if metrics.Error != nil { t.Fatalf("unexpected error: %v", metrics.Error) } @@ -247,7 +247,7 @@ func TestReplayConsumerInputTokensAreNetOfCache_AnthropicPlain(t *testing.T) { {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, }, } - metrics := p.do(context.Background(), req, strings.Repeat("x", 300), 1, "s1", "i1", 1, st) + metrics := p.do(context.Background(), req, strings.Repeat("x", 300), 1, "s1", "i1", 1, st, true) if metrics.Error != nil { t.Fatalf("unexpected error: %v", metrics.Error) } diff --git a/benchmark/replay_router.go b/benchmark/replay_router.go index 054b3be..7d11c18 100644 --- a/benchmark/replay_router.go +++ b/benchmark/replay_router.go @@ -637,6 +637,21 @@ func runRouterReplayInstance( if err == nil { poster.outputRatio = cfg.ReplayOutputRatio poster.forceOutput = cfg.ReplayForceOutput + // UUID cache-coherency injection (--replay-inject-uuids, router + // path). sessionIdx == seriesNum-1: every instance of a session + // shares the session's seriesNum, so every instance's poster picks + // the SAME session UUID/marker. uuidEnabled stays false (and + // buildInjection nil) whenever the flag is off, or this session's + // index fell outside the precomputed array (see the sizing note on + // AutoBenchmarkConfig.replayUUIDSets) — degrading gracefully to + // "no injection" for that session rather than panicking. + if cfg.ReplayInjectUUIDs { + poster.uuidEnabled = true + poster.sessionIdx = seriesNum - 1 + poster.allUUIDSets = cfg.replayUUIDSets + poster.blockCounts = cfg.replayBlockSessionCounts + poster.reciteEveryRequest = cfg.ReplayReciteEveryRequest + } } if err != nil { // Configuration error — record one error per request in this @@ -693,12 +708,14 @@ func runRouterReplayInstance( } } + isLastRequest := ti == len(inst.Requests)-1 + reqCtx, reqCancel := context.WithTimeout(ctx, reqTimeout) var metrics RequestMetrics if poster.dryRun { - metrics = poster.dryDo(reqCtx, req, docs, ti+1, sessionID, inst.InstanceID, seriesNum, st) + metrics = poster.dryDo(reqCtx, req, docs, ti+1, sessionID, inst.InstanceID, seriesNum, st, isLastRequest) } else { - metrics = poster.do(reqCtx, req, docs, ti+1, sessionID, inst.InstanceID, seriesNum, st) + metrics = poster.do(reqCtx, req, docs, ti+1, sessionID, inst.InstanceID, seriesNum, st, isLastRequest) } reqCancel() gate.Release() diff --git a/benchmark/replay_router_post.go b/benchmark/replay_router_post.go index d37dfca..5e1891e 100644 --- a/benchmark/replay_router_post.go +++ b/benchmark/replay_router_post.go @@ -62,6 +62,52 @@ type replayPoster struct { // newReplayPoster, to avoid touching its many existing call sites. outputRatio float64 forceOutput bool + + // UUID cache-coherency injection (--replay-inject-uuids, router path — + // see replay_router_uuid.go). Set directly on the poster after + // construction, same rationale as outputRatio/forceOutput above. + // uuidEnabled gates everything: false leaves do()/dryDo() byte-for-byte + // identical to before this feature existed. + uuidEnabled bool + // sessionIdx is this instance's session's 0-based index into + // allUUIDSets/blockCounts (== seriesNum-1 — every instance of a session + // shares the same seriesNum, hence the same sessionIdx). + sessionIdx int + // allUUIDSets is cfg.replayUUIDSets: the full per-session UUID + // assignment (index i = session i's owned UUID set), shared read-only + // across every poster in the run — needed both to pick this session's + // own marker and to scan for OTHER sessions' UUIDs leaking into this + // response (cross-contamination). + allUUIDSets [][]string + // blockCounts is cfg.replayBlockSessionCounts: hash -> distinct-session + // count, used by sharedPrefixBlockCount to find each request's safe + // injection boundary. + blockCounts map[string]int + // reciteEveryRequest mirrors --replay-recite-every-request: true asks + // for the recite line on every request; false only on each instance's + // final request (see the isLastRequest parameter to do()/dryDo()). + reciteEveryRequest bool +} + +// buildInjection returns this call's *uuidInjection (nil when UUID +// injection is disabled, or when this session has no assigned UUID — e.g. +// sessionIdx fell outside the precomputed array). isLastRequest is whether +// req is the final request of the CURRENT instance's request list (see +// runRouterReplayInstance) — with --replay-recite-every-request=false, only +// that final request carries the recite ask. +func (p *replayPoster) buildInjection(req RouterReplayRequest, isLastRequest bool) *uuidInjection { + if !p.uuidEnabled || p.sessionIdx < 0 || p.sessionIdx >= len(p.allUUIDSets) { + return nil + } + uuids := p.allUUIDSets[p.sessionIdx] + if len(uuids) == 0 { + return nil + } + return &uuidInjection{ + Marker: injectUUIDMarker("", uuids), + Recite: p.reciteEveryRequest || isLastRequest, + SharedPrefixLen: sharedPrefixBlockCount(req, p.blockCounts), + } } func newReplayPoster(modelSpec string, keys llm.APIKeys, endpointOverride string, runID string, dryRun bool, coldTPS, warmTPS, outputTPS int, estimator *cacheEstimator) (*replayPoster, error) { @@ -270,7 +316,9 @@ func (p *replayPoster) sendOnce(ctx context.Context, url string, bodyBytes []byt // do issues one request and returns its metrics. Honors ctx for // cancellation / deadline. Dispatches to the appropriate body builder -// and response parser based on p.apiType. +// and response parser based on p.apiType. isLastRequest is whether req is +// the final request of the calling instance's request list — see +// buildInjection. func (p *replayPoster) do( ctx context.Context, req RouterReplayRequest, @@ -280,17 +328,20 @@ func (p *replayPoster) do( instanceID string, seriesNum int, st *autoState, + isLastRequest bool, ) RequestMetrics { startTime := time.Now() + inj := p.buildInjection(req, isLastRequest) + var bodyBytes []byte var canonical string var err error switch p.apiType { case "openai", "openai_vllm": - bodyBytes, canonical, err = buildOpenAIChatCompletionsBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput) + bodyBytes, canonical, err = buildOpenAIChatCompletionsBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput, inj) default: - bodyBytes, canonical, err = buildAnthropicMessagesBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput) + bodyBytes, canonical, err = buildAnthropicMessagesBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput, inj) } if err != nil { return RequestMetrics{ @@ -373,6 +424,17 @@ func (p *replayPoster) do( m.Error = fmt.Errorf("empty response from model") } m.LocalCacheRatio = localCacheRatio + + // UUID cache-coherency validation (--replay-inject-uuids, router path). + // consumeOpenAISSE/consumeOpenAIPlain/consumeSSE/consumePlain already + // merge reasoning/thinking into m.Response (see their doc comments), so + // a single Contains-scan of m.Response covers both — thinking is passed + // as "" here, mirroring the dataset path's own call shape. + if inj != nil && m.Error == nil && !m.IsEmpty { + m.ConvIdx = p.sessionIdx + m.ExpectedUUIDs = append([]string(nil), p.allUUIDSets[p.sessionIdx]...) + m.UUIDFound, m.LeakedUUIDs = validateReplayResponse(m.Response, "", m.ExpectedUUIDs, p.sessionIdx, p.allUUIDSets) + } return m } @@ -688,14 +750,20 @@ func (p *replayPoster) dryDo( instanceID string, seriesNum int, st *autoState, + isLastRequest bool, ) RequestMetrics { - // Build canonical string for estimator and compute ratio. + // Build canonical string for estimator and compute ratio. Injection is + // threaded through purely so the canonical text (and therefore the + // cache-ratio estimate) stays consistent with a real do() call for the + // same request — dry-run never makes a real request, so there's no + // response to validate. + inj := p.buildInjection(req, isLastRequest) var canonical string switch p.apiType { case "openai", "openai_vllm": - _, canonical, _ = buildOpenAIChatCompletionsBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput) + _, canonical, _ = buildOpenAIChatCompletionsBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput, inj) default: - _, canonical, _ = buildAnthropicMessagesBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput) + _, canonical, _ = buildAnthropicMessagesBody(req, docs, p.model, p.runID, p.outputRatio, p.forceOutput, inj) } var ratio float64 if p.estimator != nil { diff --git a/benchmark/replay_router_post_test.go b/benchmark/replay_router_post_test.go index 44187a9..2149a2b 100644 --- a/benchmark/replay_router_post_test.go +++ b/benchmark/replay_router_post_test.go @@ -64,7 +64,7 @@ func TestReplayEndpointResolution(t *testing.T) { ts, seen := newVLLMStyleServer() defer ts.Close() p := mustPoster(t, fmt.Sprintf("dynamic/%s,type=openai_vllm,model=m", ts.URL)) - if m := p.do(context.Background(), minimalReq, docs, 1, "s", "i", 1, newState()); m.Error != nil { + if m := p.do(context.Background(), minimalReq, docs, 1, "s", "i", 1, newState(), true); m.Error != nil { t.Fatalf("first request: %v", m.Error) } got := seen() @@ -75,7 +75,7 @@ func TestReplayEndpointResolution(t *testing.T) { t.Errorf("latch = %q fellBack=%v, want fallback latched", p.epResolved, p.epFellBack) } // Latched: the second request goes straight to /v1, one wire call. - if m := p.do(context.Background(), minimalReq, docs, 2, "s", "i", 1, newState()); m.Error != nil { + if m := p.do(context.Background(), minimalReq, docs, 2, "s", "i", 1, newState(), true); m.Error != nil { t.Fatalf("second request: %v", m.Error) } if got = seen(); len(got) != 3 || got[2] != "/v1/chat/completions" { @@ -87,7 +87,7 @@ func TestReplayEndpointResolution(t *testing.T) { ts, seen := newVLLMStyleServer() defer ts.Close() p := mustPoster(t, fmt.Sprintf("dynamic/%s/v1,type=openai_vllm,model=m", ts.URL)) - if m := p.do(context.Background(), minimalReq, docs, 1, "s", "i", 1, newState()); m.Error != nil { + if m := p.do(context.Background(), minimalReq, docs, 1, "s", "i", 1, newState(), true); m.Error != nil { t.Fatalf("request: %v", m.Error) } got := seen() @@ -110,7 +110,7 @@ func TestReplayEndpointResolution(t *testing.T) { })) defer ts.Close() p := mustPoster(t, fmt.Sprintf("dynamic/%s,type=openai_vllm,model=m", ts.URL)) - m := p.do(context.Background(), minimalReq, docs, 1, "s", "i", 1, newState()) + m := p.do(context.Background(), minimalReq, docs, 1, "s", "i", 1, newState(), true) if m.Error == nil || !strings.Contains(m.Error.Error(), "status 500") { t.Fatalf("expected status-500 error, got %v", m.Error) } @@ -136,7 +136,7 @@ func TestReplayEndpointResolution(t *testing.T) { wg.Add(1) go func(i int) { defer wg.Done() - m := p.do(context.Background(), minimalReq, docs, 1, "s", fmt.Sprintf("i%d", i), 1, newState()) + m := p.do(context.Background(), minimalReq, docs, 1, "s", fmt.Sprintf("i%d", i), 1, newState(), true) errs[i] = m.Error }(i) } @@ -152,7 +152,7 @@ func TestReplayEndpointResolution(t *testing.T) { // Duplicate probes during the race are allowed; once latched, a new // request adds exactly one wire call. before := len(seen()) - if m := p.do(context.Background(), minimalReq, docs, 2, "s", "i", 1, newState()); m.Error != nil { + if m := p.do(context.Background(), minimalReq, docs, 2, "s", "i", 1, newState(), true); m.Error != nil { t.Fatalf("post-latch request: %v", m.Error) } after := seen() @@ -378,7 +378,7 @@ func TestOpenAIReplayEndToEnd(t *testing.T) { } ctx := context.Background() - metrics := p.do(ctx, req, docs, 1, "session-1", "instance-1", 1, st) + metrics := p.do(ctx, req, docs, 1, "session-1", "instance-1", 1, st, true) // Verify no error. if metrics.Error != nil { @@ -459,7 +459,7 @@ func TestOpenAIReplayToolTranslation(t *testing.T) { }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, nil) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody: %v", err) } @@ -519,7 +519,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "run-42", 0, false) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "run-42", 0, false, nil) if err != nil { t.Fatalf("build: %v", err) } @@ -548,7 +548,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, nil) if err != nil { t.Fatalf("build: %v", err) } @@ -576,7 +576,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, nil) if err != nil { t.Fatalf("build: %v", err) } @@ -602,7 +602,7 @@ func TestOpenAIBodyBuilderExtra(t *testing.T) { Stream: true, OutputTokens: 100, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "test-model", "", 0, false, nil) if err != nil { t.Fatalf("build: %v", err) } @@ -652,7 +652,7 @@ func TestOpenAINonStreamingEndToEnd(t *testing.T) { } st := &autoState{stream: newCompletionStream(200)} - metrics := p.do(context.Background(), req, docs, 1, "s1", "i1", 1, st) + metrics := p.do(context.Background(), req, docs, 1, "s1", "i1", 1, st, true) if metrics.Error != nil { t.Fatalf("unexpected error: %v", metrics.Error) @@ -698,7 +698,7 @@ func TestOpenAIErrorResponse(t *testing.T) { } st := &autoState{stream: newCompletionStream(200)} - metrics := p.do(context.Background(), req, docs, 1, "s1", "i1", 1, st) + metrics := p.do(context.Background(), req, docs, 1, "s1", "i1", 1, st, true) if metrics.Error == nil { t.Fatal("expected error for 500 response, got nil") @@ -740,7 +740,7 @@ func TestOpenAISSEWithoutUsage(t *testing.T) { } st := &autoState{stream: newCompletionStream(200)} - metrics := p.do(context.Background(), req, docs, 1, "s1", "i1", 1, st) + metrics := p.do(context.Background(), req, docs, 1, "s1", "i1", 1, st, true) if metrics.Error != nil { // "empty response from model" is acceptable for no-usage streams. diff --git a/benchmark/replay_router_uuid.go b/benchmark/replay_router_uuid.go new file mode 100644 index 0000000..7bd719c --- /dev/null +++ b/benchmark/replay_router_uuid.go @@ -0,0 +1,294 @@ +package benchmark + +// UUID-based cache-coherency validation for the ROUTER-REPLAY path +// (--router-replay-file, --replay-inject-uuids). This is the router-path +// counterpart of the dataset-replay UUID validation (replay_uuid.go's +// path-agnostic primitives — injectUUIDMarker / validateReplayResponse / +// FindLeakedUUIDs — are shared with, not duplicated from, that file). +// +// Strategy (Option C — boundary injection, with tail fallback): every +// session in a replay-v3 capture opens with one or more blocks (system +// blocks, tools, or leading messages) whose content hash is shared across +// MANY OTHER sessions too — the router's own leading system prompt(s), +// repeated verbatim capture after capture. Everything AFTER that shared +// run is genuinely per-session (the user's actual turn). We inject exactly +// ONE deterministic UUID per session at that boundary: +// +// [ RUN_GUID ][ shared system blocks ][ MARKER ][ forceOutput instr ] [ messages... ] [ recite ask ] +// \_________________ byte-identical across sessions _________________/ \_ per-session, grows each turn _/ +// +// Putting the marker there means: +// - the cross-session shared prefix stays byte-identical (cache-hit +// reproduction against the original capture is preserved: two sessions +// that shared a system prompt still collide on the server's prefix +// cache exactly as they did in the original traffic) +// - the marker itself lands in a region that IS cached WITHIN a session +// (every subsequent request in the same session repeats it), so asking +// the model to recall it later is a genuine KV-coherency signal, not an +// artifact of it being freshly re-sent every turn. +// +// A session with no shared leading block at all (empirically none, across +// 5441 real sessions, lack one — but the file format doesn't guarantee it) +// falls back to tail injection: the marker is folded into the end of the +// request instead, forfeiting the "cached within a session" property but +// still producing a valid, scorable request. + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "sync" +) + +// uuidInjection describes the per-request UUID injection to apply when +// building a router-replay wire body. A nil *uuidInjection means "no +// injection" — buildAnthropicMessagesBody / buildOpenAIChatCompletionsBody +// must behave identically to before this feature existed. +type uuidInjection struct { + // Marker is the exact text to splice in (see injectUUIDMarker) — e.g. + // "\n\n[ref-id: ]". Empty means no marker this call (still allows + // Recite alone, though callers currently always set both together). + Marker string + // Recite asks the model to find and echo the marker verbatim from + // earlier in its own context (see replayReciteFromContextInstruction). + Recite bool + // SharedPrefixLen is this request's leading run of cross-session-shared + // prefix blocks (see sharedPrefixBlockCount). It tells the wire builder + // whether the marker can be spliced in at the natural system/message + // boundary (SharedPrefixLen covers every emitted system block) or must + // fall back to tail injection (SharedPrefixLen == 0). + SharedPrefixLen int +} + +// computeBlockSessionCounts streams a replay-v3 file once and returns, for +// every distinct block hash seen, the number of DISTINCT SESSIONS that +// reference it at least once (not the number of requests — a hash reused +// many times within one session still counts once for that session). A +// hash with count > 1 is shared across sessions, which is exactly the +// "safe to leave byte-identical" prefix content sharedPrefixBlockCount looks +// for. +// +// allowed and sessionLimit mirror openRouterReplayStream's filtering +// exactly (nil allowed = every session; sessionLimit <= 0 = no cap) so the +// set of sessions counted here matches the set the real run will dispatch. +// Only hashes are read here — synthText/synthesized content is never +// generated, so this pass is cheap even over large files. +func computeBlockSessionCounts(path string, allowed map[int]bool, sessionLimit int) (map[string]int, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + br := bufio.NewReaderSize(f, 1<<20) + if _, err := br.ReadBytes('\n'); err != nil { + return nil, fmt.Errorf("read header line: %w", err) + } + + counts := map[string]int{} + lineIdx := 0 + produced := 0 + for { + if sessionLimit > 0 && produced >= sessionLimit { + break + } + if allowed != nil && produced >= len(allowed) { + break + } + line, rerr := br.ReadBytes('\n') + if len(line) > 0 { + line = trimNL(line) + if len(line) > 0 { + currentIdx := lineIdx + lineIdx++ + if allowed != nil && !allowed[currentIdx] { + if rerr != nil { + break + } + continue + } + var sess RouterReplaySession + if jerr := json.Unmarshal(line, &sess); jerr == nil { + seen := map[string]bool{} + for _, inst := range sess.Instances { + for _, req := range inst.Requests { + hashes, _ := BuildReplayRequestPrefix(req) + for _, h := range hashes { + if h != "" { + seen[h] = true + } + } + } + } + for h := range seen { + counts[h]++ + } + produced++ + } + } + } + if rerr != nil { + break + } + } + return counts, nil +} + +// countSessionsWithUsableBoundary makes a second lightweight streaming pass +// (same filtering as computeBlockSessionCounts) purely to report, for the +// startup diagnostic, how many sessions have at least one request whose +// leading run of shared blocks is non-empty (a "usable boundary" — the +// marker can be spliced in at the natural system/message boundary) versus +// how many would fall back to tail injection for every one of their +// requests. Returns (usable, total, error); usable <= total. +func countSessionsWithUsableBoundary(path string, allowed map[int]bool, sessionLimit int, counts map[string]int) (usable int, total int, err error) { + f, ferr := os.Open(path) + if ferr != nil { + return 0, 0, ferr + } + defer f.Close() + + br := bufio.NewReaderSize(f, 1<<20) + if _, rerr := br.ReadBytes('\n'); rerr != nil { + return 0, 0, fmt.Errorf("read header line: %w", rerr) + } + + lineIdx := 0 + produced := 0 + for { + if sessionLimit > 0 && produced >= sessionLimit { + break + } + if allowed != nil && produced >= len(allowed) { + break + } + line, rerr := br.ReadBytes('\n') + if len(line) > 0 { + line = trimNL(line) + if len(line) > 0 { + currentIdx := lineIdx + lineIdx++ + if allowed != nil && !allowed[currentIdx] { + if rerr != nil { + break + } + continue + } + var sess RouterReplaySession + if jerr := json.Unmarshal(line, &sess); jerr == nil { + total++ + hasBoundary := false + for _, inst := range sess.Instances { + for _, req := range inst.Requests { + if sharedPrefixBlockCount(req, counts) > 0 { + hasBoundary = true + break + } + } + if hasBoundary { + break + } + } + if hasBoundary { + usable++ + } + produced++ + } + } + } + if rerr != nil { + break + } + } + return usable, total, nil +} + +// sharedPrefixBlockCount returns the length of req's LEADING run of blocks +// (per BuildReplayRequestPrefix's cache-order hash sequence: system blocks, +// then tools, then messages) whose hash is shared across more than one +// session, per counts (see computeBlockSessionCounts). The run stops at the +// first hash that is either unshared (counts[hash] <= 1) or empty. +// +// This is the offline analogue of "how much of this request's prefix is +// safe to leave byte-identical" — the wire builder uses it to decide +// whether the UUID marker can be spliced in at the natural boundary +// (SharedPrefixLen covers every system block) or must fall back to tail +// injection. +func sharedPrefixBlockCount(req RouterReplayRequest, counts map[string]int) int { + hashes, _ := BuildReplayRequestPrefix(req) + n := 0 + for _, h := range hashes { + if h == "" || counts[h] <= 1 { + break + } + n++ + } + return n +} + +// replayReciteFromContextInstruction returns the tail instruction asking +// the model to find and echo, verbatim, the ref-id marker planted earlier +// in its own context — WITHOUT restating it from the instruction itself. +// Unlike the dataset-path replayReciteInstruction (replay_uuid.go), this +// text never embeds the UUID: presence in the response therefore reflects +// genuine recall from cached KV, not an echo of the ask. +func replayReciteFromContextInstruction() string { + return "\n\n(Somewhere earlier in this conversation there is a line of the exact form `[ref-id: ]`. " + + "Find it and, before your normal answer, output one line in the exact form `SEEN_REF: ` reproducing " + + "that id verbatim from your context — do not invent one, and do not simply repeat this instruction. Then answer normally.)" +} + +// buildSessionUUIDs returns n singleton UUID sets (one UUID per session), +// drawn in order from a single seeded generator — same determinism/ +// disjointness rationale as the dataset path's buildReplayUUIDSets: same +// seed -> same per-session UUID assignment across runs and across every +// model in a multi-model run (see the precompute call site in +// RunAutoBenchmark, which populates cfg.replayUUIDSets once, before any +// per-model goroutine spawns, so every model sees the identical +// assignment). +func buildSessionUUIDs(n int, seed int64) [][]string { + if n <= 0 { + return nil + } + newUUID := newUUIDGenerator(seed) + sets := make([][]string, n) + for i := range sets { + sets[i] = []string{newUUID()} + } + return sets +} + +// ---- max_tokens recite floor ---- + +// replayReciteFloorTokens is the minimum max_tokens budget enforced on a +// request that carries the recite ask (see uuidInjection.Recite). A +// router-replay request's max_tokens is normally sized off the ORIGINAL +// capture's output_tokens (see pickMaxTokens) — which for a tool-call-only +// turn can be a handful of tokens, nowhere near enough to also emit the +// "SEEN_REF: " line the recite ask asks for. Without a floor, a tiny +// budget truncates the recite line itself, which would misread as a +// PRESENCE_MISS (coherency failure) when it's actually just an output-size +// artifact. +const replayReciteFloorTokens = 64 + +var reciteFloorWarnOnce sync.Once + +// applyReciteFloor raises maxTokens to replayReciteFloorTokens when recite +// is requested and the original budget falls short, warning once per +// process (mirrors the dataset path's reciteTruncWarned one-shot pattern, +// but this is a single global warning rather than per-conversation since +// the router path's floor is a fixed constant, not a per-conversation +// truncation computation). +func applyReciteFloor(maxTokens int, recite bool) int { + if !recite || maxTokens >= replayReciteFloorTokens { + return maxTokens + } + reciteFloorWarnOnce.Do(func() { + fmt.Fprintf(os.Stderr, + "[router-replay] WARNING: max_tokens raised to the UUID-recite floor (%d) for one or more requests — "+ + "a tiny captured output budget would otherwise truncate the recite line into a false PRESENCE_MISS, not real corruption\n", + replayReciteFloorTokens) + }) + return replayReciteFloorTokens +} diff --git a/benchmark/replay_router_uuid_test.go b/benchmark/replay_router_uuid_test.go new file mode 100644 index 0000000..a4ba4c8 --- /dev/null +++ b/benchmark/replay_router_uuid_test.go @@ -0,0 +1,489 @@ +package benchmark + +// Pure, offline unit tests for router-replay UUID cache-coherency injection +// (replay_router_uuid.go). No mocked LLM/Chat — these exercise data +// transforms and the wire builders directly, per repo testing policy. + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeReplayV3File marshals a header + sessions into a replay-v3 JSONL +// file under t.TempDir() and returns its path. +func writeReplayV3File(t *testing.T, sessions []RouterReplaySession) string { + t.Helper() + dir := t.TempDir() + path := filepath.Join(dir, "replay.jsonl") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create temp replay file: %v", err) + } + defer f.Close() + + hdr := RouterReplayHeader{ + Schema: "replay-v3", + Summary: RouterReplaySummary{ + Sessions: len(sessions), + }, + } + enc := json.NewEncoder(f) + if err := enc.Encode(hdr); err != nil { + t.Fatalf("encode header: %v", err) + } + for _, s := range sessions { + if err := enc.Encode(s); err != nil { + t.Fatalf("encode session: %v", err) + } + } + return path +} + +// syntheticSessionsForBoundaryTests builds 5 sessions exercising every +// sharedPrefixBlockCount regime: +// - sessions 0,1: share one system block (sys1) but each carries its own +// unique message -> leading run length 1, which equals the (single) +// emitted system block count -> "covers all system blocks" boundary case. +// - session 2: no block it carries is shared with any other session at +// all -> leading run length 0 -> tail-fallback case. +// - sessions 3,4: share BOTH their system block AND their message block +// (identical request shape) -> leading run length 2 == full prefix +// length -> "fully shared" edge case. +func syntheticSessionsForBoundaryTests() []RouterReplaySession { + mkSession := func(id string, sysHash string, msgHash string) RouterReplaySession { + return RouterReplaySession{ + SessionID: id, + Instances: []RouterReplayInstance{ + { + InstanceID: id + "-inst", + Requests: []RouterReplayRequest{ + { + RequestID: 1, + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{ + {Hash: sysHash, Bytes: 250, Tokens: 60}, + }, + Messages: []RouterReplayMessage{ + {Hash: msgHash, Role: "user", BlockTypes: []string{"text"}, Bytes: 100, Tokens: 25}, + }, + }, + }, + }, + }, + } + } + return []RouterReplaySession{ + mkSession("s0", "sys1", "msgUniq0"), + mkSession("s1", "sys1", "msgUniq1"), + mkSession("s2", "sys2only", "msgUniq2"), + mkSession("s3", "sysShared34", "msgShared34"), + mkSession("s4", "sysShared34", "msgShared34"), + } +} + +func TestComputeBlockSessionCounts(t *testing.T) { + path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) + + counts, err := computeBlockSessionCounts(path, nil, 0) + if err != nil { + t.Fatalf("computeBlockSessionCounts: %v", err) + } + + cases := []struct { + hash string + want int + }{ + {"sys1", 2}, // sessions 0 and 1 + {"msgUniq0", 1}, // only session 0 + {"msgUniq1", 1}, // only session 1 + {"sys2only", 1}, // only session 2 + {"msgUniq2", 1}, // only session 2 + {"sysShared34", 2}, // sessions 3 and 4 + {"msgShared34", 2}, // sessions 3 and 4 + {"never-appears", 0}, // absent hash + } + for _, c := range cases { + if got := counts[c.hash]; got != c.want { + t.Errorf("counts[%q] = %d, want %d", c.hash, got, c.want) + } + } +} + +func TestComputeBlockSessionCountsRespectsFilters(t *testing.T) { + path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) + + t.Run("sessionLimit caps to the first N sessions", func(t *testing.T) { + // sessionLimit=2 -> only s0, s1 counted; sys1 still shared (2), but + // session 2/3/4 hashes never appear. + counts, err := computeBlockSessionCounts(path, nil, 2) + if err != nil { + t.Fatalf("computeBlockSessionCounts: %v", err) + } + if got := counts["sys1"]; got != 2 { + t.Errorf("sys1 = %d, want 2", got) + } + if got := counts["sysShared34"]; got != 0 { + t.Errorf("sysShared34 = %d, want 0 (beyond sessionLimit)", got) + } + }) + + t.Run("allowed index set restricts to those sessions only", func(t *testing.T) { + // Only session index 3 and 4 (0-based) allowed -> sysShared34 still + // shows count 2, but sys1 (sessions 0,1) never counted. + allowed := map[int]bool{3: true, 4: true} + counts, err := computeBlockSessionCounts(path, allowed, 0) + if err != nil { + t.Fatalf("computeBlockSessionCounts: %v", err) + } + if got := counts["sysShared34"]; got != 2 { + t.Errorf("sysShared34 = %d, want 2", got) + } + if got := counts["sys1"]; got != 0 { + t.Errorf("sys1 = %d, want 0 (session 0/1 excluded)", got) + } + }) +} + +func TestSharedPrefixBlockCount(t *testing.T) { + path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) + counts, err := computeBlockSessionCounts(path, nil, 0) + if err != nil { + t.Fatalf("computeBlockSessionCounts: %v", err) + } + + reqBoundary := RouterReplayRequest{ + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msgUniq0", Role: "user", Bytes: 100}}, + } + if got := sharedPrefixBlockCount(reqBoundary, counts); got != 1 { + t.Errorf("boundary case: sharedPrefixBlockCount = %d, want 1 (covers the single system block)", got) + } + + reqNoShare := RouterReplayRequest{ + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys2only", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msgUniq2", Role: "user", Bytes: 100}}, + } + if got := sharedPrefixBlockCount(reqNoShare, counts); got != 0 { + t.Errorf("no-shared-block case: sharedPrefixBlockCount = %d, want 0 (tail fallback)", got) + } + + reqFullyShared := RouterReplayRequest{ + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sysShared34", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msgShared34", Role: "user", Bytes: 100}}, + } + if got, want := sharedPrefixBlockCount(reqFullyShared, counts), 2; got != want { + t.Errorf("fully-shared case: sharedPrefixBlockCount = %d, want %d (full prefix length)", got, want) + } +} + +// TestBuildSessionUUIDsDeterminism verifies buildSessionUUIDs matches the +// dataset path's determinism contract: same seed -> same per-session UUID +// assignment; different seed -> different assignment; every UUID unique. +func TestBuildSessionUUIDsDeterminism(t *testing.T) { + a := buildSessionUUIDs(5, 42) + b := buildSessionUUIDs(5, 42) + if len(a) != 5 || len(b) != 5 { + t.Fatalf("expected 5 sets each, got %d and %d", len(a), len(b)) + } + for i := range a { + if len(a[i]) != 1 || len(b[i]) != 1 { + t.Fatalf("session %d: expected singleton sets, got %v / %v", i, a[i], b[i]) + } + if a[i][0] != b[i][0] { + t.Errorf("session %d: same seed produced different UUIDs: %q vs %q", i, a[i][0], b[i][0]) + } + } + + c := buildSessionUUIDs(5, 43) + same := true + for i := range a { + if a[i][0] != c[i][0] { + same = false + } + } + if same { + t.Error("different seeds produced an identical UUID assignment") + } + + seen := map[string]bool{} + for _, set := range a { + if seen[set[0]] { + t.Errorf("uuid %q assigned to more than one session", set[0]) + } + seen[set[0]] = true + } + + if got := buildSessionUUIDs(0, 42); got != nil { + t.Errorf("buildSessionUUIDs(0, ...) = %v, want nil", got) + } +} + +// TestWireInjectionDeterminism verifies that, for a fixed session's marker, +// buildOpenAIChatCompletionsBody / buildAnthropicMessagesBody produce +// byte-identical bodies across repeated calls (same request + same +// injection in -> same bytes out), and that two DIFFERENT sessions' markers +// diverge the body. +func TestWireInjectionDeterminism(t *testing.T) { + docs := strings.Repeat("wire-injection-docs ", 100) + req := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msgUniq0", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + sets := buildSessionUUIDs(2, 7) + injA := &uuidInjection{Marker: injectUUIDMarker("", sets[0]), Recite: true, SharedPrefixLen: 1} + injA2 := &uuidInjection{Marker: injectUUIDMarker("", sets[0]), Recite: true, SharedPrefixLen: 1} + injB := &uuidInjection{Marker: injectUUIDMarker("", sets[1]), Recite: true, SharedPrefixLen: 1} + + for _, kind := range []string{"openai", "anthropic"} { + build := func(r RouterReplayRequest, inj *uuidInjection) []byte { + var body []byte + var err error + if kind == "openai" { + body, _, err = buildOpenAIChatCompletionsBody(r, docs, "model", "", 0, false, inj) + } else { + body, _, err = buildAnthropicMessagesBody(r, docs, "model", "", 0, false, inj) + } + if err != nil { + t.Fatalf("%s build: %v", kind, err) + } + return body + } + bodyA1 := build(req, injA) + bodyA2 := build(req, injA2) + bodyB := build(req, injB) + + if string(bodyA1) != string(bodyA2) { + t.Errorf("%s: identical injection produced different bytes", kind) + } + if string(bodyA1) == string(bodyB) { + t.Errorf("%s: different sessions' markers produced identical bytes", kind) + } + if !strings.Contains(string(bodyA1), sets[0][0]) { + t.Errorf("%s: body missing session A's own UUID", kind) + } + if strings.Contains(string(bodyA1), sets[1][0]) { + t.Errorf("%s: body A leaked session B's UUID into the wire body", kind) + } + } +} + +// TestCacheFidelityBoundaryInvariant verifies the core Option-C guarantee: +// two DIFFERENT sessions that share a leading system block emit +// byte-identical content for that shared block, diverging only at (or +// after) the injected per-session marker — i.e. injection never perturbs +// the cross-session-shared prefix a real server would prefix-cache on. +func TestCacheFidelityBoundaryInvariant(t *testing.T) { + docs := strings.Repeat("fidelity-docs ", 100) + reqA := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msgUniq-A", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + reqB := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, // SAME shared system block + Messages: []RouterReplayMessage{{Hash: "msgUniq-B", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + injA := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-session-A"}), Recite: false, SharedPrefixLen: 1} + injB := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-session-B"}), Recite: false, SharedPrefixLen: 1} + + bodyA, _, err := buildAnthropicMessagesBody(reqA, docs, "model", "", 0, false, injA) + if err != nil { + t.Fatalf("build A: %v", err) + } + bodyB, _, err := buildAnthropicMessagesBody(reqB, docs, "model", "", 0, false, injB) + if err != nil { + t.Fatalf("build B: %v", err) + } + + var parsedA, parsedB map[string]interface{} + if err := json.Unmarshal(bodyA, &parsedA); err != nil { + t.Fatalf("unmarshal A: %v", err) + } + if err := json.Unmarshal(bodyB, &parsedB); err != nil { + t.Fatalf("unmarshal B: %v", err) + } + + sysA, _ := parsedA["system"].([]interface{}) + sysB, _ := parsedB["system"].([]interface{}) + if len(sysA) != 2 || len(sysB) != 2 { + t.Fatalf("expected system = [shared block, marker], got lens %d and %d", len(sysA), len(sysB)) + } + // Index 0 (the shared system block, "sys1") must be byte-identical. + sharedA, _ := json.Marshal(sysA[0]) + sharedB, _ := json.Marshal(sysB[0]) + if string(sharedA) != string(sharedB) { + t.Errorf("shared leading system block diverged between sessions:\nA: %s\nB: %s", sharedA, sharedB) + } + // Index 1 (the injected marker) MUST diverge — that's the whole point. + markerA, _ := json.Marshal(sysA[1]) + markerB, _ := json.Marshal(sysB[1]) + if string(markerA) == string(markerB) { + t.Error("injected markers were identical across two different sessions") + } + if !strings.Contains(string(markerA), "uuid-session-A") { + t.Errorf("session A's marker missing its own uuid: %s", markerA) + } + if !strings.Contains(string(markerB), "uuid-session-B") { + t.Errorf("session B's marker missing its own uuid: %s", markerB) + } +} + +// TestTailFallbackInjection verifies that when SharedPrefixLen == 0 (no +// usable boundary), the marker is folded into the tail (messages array) +// rather than the system array, and the request remains well-formed. +func TestTailFallbackInjection(t *testing.T) { + docs := strings.Repeat("tail-fallback-docs ", 100) + req := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys-unique", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msg-unique", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + inj := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-tail"}), Recite: true, SharedPrefixLen: 0} + + body, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) + if err != nil { + t.Fatalf("build: %v", err) + } + var parsed map[string]interface{} + if err := json.Unmarshal(body, &parsed); err != nil { + t.Fatalf("unmarshal: %v", err) + } + sys, _ := parsed["system"].([]interface{}) + if len(sys) != 1 { + t.Fatalf("expected system to carry ONLY the original block (no boundary splice), got %d entries", len(sys)) + } + if strings.Contains(string(body), "uuid-tail") == false { + t.Fatal("marker missing from body entirely") + } + msgs, _ := parsed["messages"].([]interface{}) + if len(msgs) == 0 { + t.Fatal("expected messages to carry the tail-injected marker/recite content") + } + last := msgs[len(msgs)-1].(map[string]interface{}) + if last["role"] != "user" { + t.Errorf("tail-injected message role = %v, want user", last["role"]) + } +} + +// TestUUIDValidationEndToEnd exercises buildSessionUUIDs + injectUUIDMarker + +// validateReplayResponse together, mirroring how replayPoster.do() wires +// them: a response containing the OWN session's uuid scores found/no-leak; +// a response containing ANOTHER session's uuid scores CROSS_CONTAMINATION +// against the correct series index. +func TestUUIDValidationEndToEnd(t *testing.T) { + sets := buildSessionUUIDs(3, 123) + + t.Run("own uuid present, no leak", func(t *testing.T) { + resp := "Sure, the ref-id I recall is " + sets[0][0] + ". Anyway, here's your answer." + found, leaked := validateReplayResponse(resp, "", sets[0], 0, sets) + if len(found) != 1 || !found[0] { + t.Errorf("found = %v, want [true]", found) + } + if len(leaked) != 0 { + t.Errorf("leaked = %v, want none", leaked) + } + }) + + t.Run("cross contamination from another session", func(t *testing.T) { + resp := "SEEN_REF: " + sets[0][0] + " and also " + sets[2][0] + found, leaked := validateReplayResponse(resp, "", sets[0], 0, sets) + if len(found) != 1 || !found[0] { + t.Errorf("found = %v, want [true]", found) + } + if len(leaked) != 1 || !strings.Contains(leaked[0], sets[2][0]) || !strings.Contains(leaked[0], "series=2") { + t.Errorf("leaked = %v, want one entry naming session 2's uuid", leaked) + } + }) + + t.Run("presence miss", func(t *testing.T) { + found, leaked := validateReplayResponse("no ref ids here", "", sets[1], 1, sets) + if found[0] { + t.Error("expected PRESENCE_MISS (found=false)") + } + if len(leaked) != 0 { + t.Errorf("leaked = %v, want none", leaked) + } + }) +} + +// TestApplyReciteFloor verifies the max_tokens recite-floor helper: raises +// a too-small budget to replayReciteFloorTokens only when recite is +// requested; leaves larger budgets and non-recite calls untouched. +func TestApplyReciteFloor(t *testing.T) { + cases := []struct { + name string + tokens int + recite bool + want int + }{ + {"below floor, recite -> raised", 10, true, replayReciteFloorTokens}, + {"at floor, recite -> unchanged", replayReciteFloorTokens, true, replayReciteFloorTokens}, + {"above floor, recite -> unchanged", 1000, true, 1000}, + {"below floor, no recite -> unchanged", 10, false, 10}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := applyReciteFloor(c.tokens, c.recite); got != c.want { + t.Errorf("applyReciteFloor(%d, %v) = %d, want %d", c.tokens, c.recite, got, c.want) + } + }) + } +} + +// TestMaxTokensFloorAppliedInWireBuilders verifies the floor is actually +// wired into both body builders' emitted max_tokens when a recite +// injection is present and the original/recorded budget is tiny — the +// scenario a real tool-call-only turn would hit. +func TestMaxTokensFloorAppliedInWireBuilders(t *testing.T) { + docs := strings.Repeat("floor-docs ", 100) + req := RouterReplayRequest{ + InputTokens: 500, + OutputTokens: 5, // tiny recorded budget -- would truncate the recite line + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msg1", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + inj := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-floor"}), Recite: true, SharedPrefixLen: 1} + + anthBody, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) + if err != nil { + t.Fatalf("anthropic build: %v", err) + } + var anthParsed map[string]interface{} + if err := json.Unmarshal(anthBody, &anthParsed); err != nil { + t.Fatalf("anthropic unmarshal: %v", err) + } + if got, want := anthParsed["max_tokens"].(float64), float64(replayReciteFloorTokens); got != want { + t.Errorf("anthropic max_tokens = %v, want %v (floor)", got, want) + } + + openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, inj) + if err != nil { + t.Fatalf("openai build: %v", err) + } + var openaiParsed map[string]interface{} + if err := json.Unmarshal(openaiBody, &openaiParsed); err != nil { + t.Fatalf("openai unmarshal: %v", err) + } + if got, want := openaiParsed["max_tokens"].(float64), float64(replayReciteFloorTokens); got != want { + t.Errorf("openai max_tokens = %v, want %v (floor)", got, want) + } + + // Without injection (nil), the tiny recorded output_tokens is honored + // as before -- the floor must never apply when there's no recite ask. + plainBody, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, nil) + if err != nil { + t.Fatalf("anthropic build (no injection): %v", err) + } + var plainParsed map[string]interface{} + if err := json.Unmarshal(plainBody, &plainParsed); err != nil { + t.Fatalf("anthropic unmarshal (no injection): %v", err) + } + if got, want := plainParsed["max_tokens"].(float64), float64(5); got != want { + t.Errorf("anthropic max_tokens (no injection) = %v, want %v (unfloored)", got, want) + } +} diff --git a/benchmark/replay_router_wire.go b/benchmark/replay_router_wire.go index 272badd..43d36c8 100644 --- a/benchmark/replay_router_wire.go +++ b/benchmark/replay_router_wire.go @@ -41,10 +41,14 @@ const verboseOutputInstruction = "Provide a thorough, detailed response and keep // as possible. Returns the marshaled JSON bytes and the canonical text // string (all synthesized content concatenated in generation order) for // feeding into the content-level cache estimator. -func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool) ([]byte, string, error) { +// +// inj carries the UUID cache-coherency injection (--replay-inject-uuids, +// router path — see replay_router_uuid.go); nil means "no injection", +// leaving the body byte-for-byte identical to before this feature existed. +func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool, inj *uuidInjection) ([]byte, string, error) { body := map[string]interface{}{ "model": modelName, - "max_tokens": pickMaxTokens(req, outputRatio), + "max_tokens": applyReciteFloor(pickMaxTokens(req, outputRatio), inj != nil && inj.Recite), "stream": req.Stream, } if req.Temperature != nil { @@ -71,6 +75,22 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName } systemArr = append([]map[string]interface{}{stamp}, systemArr...) } + + // UUID marker injection at the system/conversation boundary (Option C — + // see replay_router_uuid.go). Only spliced in here when the leading run + // of cross-session-shared blocks covers every emitted system block; + // otherwise it falls back to tail injection below, alongside the + // messages array, so it never lands ahead of genuinely per-session + // system content (which would poison that session's OWN cache key, + // not just cross-session sharing). + markerAtBoundary := inj != nil && inj.Marker != "" && + inj.SharedPrefixLen > 0 && inj.SharedPrefixLen >= len(effectiveSystemBlocks(req.SystemBlocks)) + if markerAtBoundary { + systemArr = append(systemArr, map[string]interface{}{ + "type": "text", + "text": inj.Marker, + }) + } if forceOutput { systemArr = append(systemArr, map[string]interface{}{ "type": "text", @@ -83,8 +103,20 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName if req.Tools != nil && req.Tools.Count > 0 { body["tools"] = buildTools(req.Tools, docs) } + var msgs []map[string]interface{} if len(req.Messages) > 0 { - body["messages"] = buildMessages(req.Messages, docs) + msgs = buildMessages(req.Messages, docs) + } + if inj != nil { + if !markerAtBoundary && inj.Marker != "" { + msgs = appendTailMessageAnthropic(msgs, inj.Marker) + } + if inj.Recite { + msgs = appendTailMessageAnthropic(msgs, replayReciteFromContextInstruction()) + } + } + if len(msgs) > 0 { + body["messages"] = msgs } // Collect canonical text for the cache estimator. @@ -95,6 +127,9 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName for _, b := range effectiveSystemBlocks(req.SystemBlocks) { canonical.WriteString(synthText(b.Hash, b.Bytes, docs)) } + if markerAtBoundary { + canonical.WriteString(inj.Marker) + } if req.Tools != nil && req.Tools.Count > 0 { n := req.Tools.Count if n <= 0 { @@ -132,6 +167,14 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName } } } + if inj != nil { + if !markerAtBoundary && inj.Marker != "" { + canonical.WriteString(inj.Marker) + } + if inj.Recite { + canonical.WriteString(replayReciteFromContextInstruction()) + } + } bodyBytes, err := json.Marshal(body) return bodyBytes, canonical.String(), err @@ -264,6 +307,40 @@ func roleOrUser(role string) string { return "user" } +// appendTailMessageAnthropic appends text (a UUID marker or the recite ask) +// to the end of an Anthropic messages array, preserving strict user/ +// assistant role alternation (Anthropic rejects consecutive same-role +// messages, and "system" is not a valid role inside `messages` at all): +// - if the array is empty, or the last message is role "assistant", a +// NEW role="user" message carrying the text is appended (valid +// alternation; a fresh assistant turn should never have text +// injected into it after the fact). +// - if the last message is role "user" (the common case — router-replay +// requests carry history up to, but not including, the response being +// generated), the text is appended as an additional content block on +// THAT message instead of a new one. +func appendTailMessageAnthropic(msgs []map[string]interface{}, text string) []map[string]interface{} { + if text == "" { + return msgs + } + newUserMsg := map[string]interface{}{ + "role": "user", + "content": []map[string]interface{}{{"type": "text", "text": text}}, + } + if len(msgs) == 0 { + return append(msgs, newUserMsg) + } + last := msgs[len(msgs)-1] + if last["role"] != "user" { + return append(msgs, newUserMsg) + } + content, _ := last["content"].([]map[string]interface{}) + content = append(content, map[string]interface{}{"type": "text", "text": text}) + last["content"] = content + msgs[len(msgs)-1] = last + return msgs +} + // buildMessageContent materializes per-block content for one message. // block_types lists the kinds in order; we use tool_use_ids and // tool_result_ids to populate ids on the matching blocks (in order of @@ -460,10 +537,13 @@ func buildOpenAITools(spec *RouterReplayToolsSpec, docs string) []map[string]int // - Anthropic-specific fields (top_k, thinking) are dropped. // - Stream options with include_usage are set so we get token counts in // the final SSE chunk. -func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool) ([]byte, string, error) { +// +// inj carries the UUID cache-coherency injection (--replay-inject-uuids, +// router path — see replay_router_uuid.go); nil means "no injection". +func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool, inj *uuidInjection) ([]byte, string, error) { body := map[string]interface{}{ "model": modelName, - "max_tokens": pickMaxTokens(req, outputRatio), + "max_tokens": applyReciteFloor(pickMaxTokens(req, outputRatio), inj != nil && inj.Recite), "stream": req.Stream, } if req.Temperature != nil { @@ -502,6 +582,20 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN } messages = append([]map[string]interface{}{stamp}, messages...) } + + // UUID marker injection at the system/conversation boundary (Option C — + // see replay_router_uuid.go and the mirrored comment in + // buildAnthropicMessagesBody). Only spliced in here when the leading + // run of cross-session-shared blocks covers every emitted system + // block; otherwise it falls back to tail injection below. + markerAtBoundary := inj != nil && inj.Marker != "" && + inj.SharedPrefixLen > 0 && inj.SharedPrefixLen >= len(effectiveSystemBlocks(req.SystemBlocks)) + if markerAtBoundary { + messages = append(messages, map[string]interface{}{ + "role": "system", + "content": inj.Marker, + }) + } if forceOutput { // Force the model to generate up to max_tokens: append a short // continue-generating instruction as a system message AND set vLLM's @@ -528,6 +622,15 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN messages = append(messages, openaiMsgs...) } + if inj != nil { + if !markerAtBoundary && inj.Marker != "" { + messages = appendTailMessageOpenAI(messages, inj.Marker) + } + if inj.Recite { + messages = appendTailMessageOpenAI(messages, replayReciteFromContextInstruction()) + } + } + body["messages"] = messages if req.Tools != nil && req.Tools.Count > 0 { body["tools"] = buildOpenAITools(req.Tools, docs) @@ -541,6 +644,9 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN for _, b := range effectiveSystemBlocks(req.SystemBlocks) { canonical.WriteString(synthText(b.Hash, b.Bytes, docs)) } + if markerAtBoundary { + canonical.WriteString(inj.Marker) + } for _, m := range req.Messages { blocks := buildMessageContent(m, docs) for _, blk := range blocks { @@ -555,11 +661,50 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN } } } + if inj != nil { + if !markerAtBoundary && inj.Marker != "" { + canonical.WriteString(inj.Marker) + } + if inj.Recite { + canonical.WriteString(replayReciteFromContextInstruction()) + } + } bodyBytes, err := json.Marshal(body) return bodyBytes, canonical.String(), err } +// appendTailMessageOpenAI appends text (a UUID marker or the recite ask) to +// the end of an OpenAI messages array. Unlike Anthropic, OpenAI has no +// strict role-alternation requirement, but we still fold the text into the +// last message's content when that message is one the model would read as +// its own turn's input (user/system/tool) rather than always creating a +// new message — keeping the shape close to a real client's behavior. If +// the last message is "assistant" (or there are no messages at all), a new +// role="user" message carrying the text is appended instead. +func appendTailMessageOpenAI(msgs []map[string]interface{}, text string) []map[string]interface{} { + if text == "" { + return msgs + } + newUserMsg := map[string]interface{}{"role": "user", "content": text} + if len(msgs) == 0 { + return append(msgs, newUserMsg) + } + last := msgs[len(msgs)-1] + role, _ := last["role"].(string) + if role == "assistant" { + return append(msgs, newUserMsg) + } + existing, _ := last["content"].(string) + if existing != "" { + last["content"] = existing + "\n\n" + text + } else { + last["content"] = text + } + msgs[len(msgs)-1] = last + return msgs +} + // buildOpenAIMessages converts router-replay messages into OpenAI chat // messages. tool_use blocks become tool_calls on the assistant message; // tool_result blocks become separate role="tool" messages. Orphaned diff --git a/benchmark/replay_router_wire_test.go b/benchmark/replay_router_wire_test.go index 460f214..e461b52 100644 --- a/benchmark/replay_router_wire_test.go +++ b/benchmark/replay_router_wire_test.go @@ -26,7 +26,7 @@ func TestBuildAnthropicMessagesBodyRunGUIDStamp(t *testing.T) { // --- runID set --- runID := "test-run-id" - body, _, err := buildAnthropicMessagesBody(req, docs, modelName, runID, 0, false) + body, _, err := buildAnthropicMessagesBody(req, docs, modelName, runID, 0, false, nil) if err != nil { t.Fatalf("buildAnthropicMessagesBody with runID: %v", err) } @@ -53,7 +53,7 @@ func TestBuildAnthropicMessagesBodyRunGUIDStamp(t *testing.T) { } // --- runID empty --- - body, _, err = buildAnthropicMessagesBody(req, docs, modelName, "", 0, false) + body, _, err = buildAnthropicMessagesBody(req, docs, modelName, "", 0, false, nil) if err != nil { t.Fatalf("buildAnthropicMessagesBody empty runID: %v", err) } @@ -74,7 +74,7 @@ func TestBuildAnthropicMessagesBodyRunGUIDStamp(t *testing.T) { // --- runID set, 0 system blocks --- reqNoSys := RouterReplayRequest{} // no SystemBlocks - body, _, err = buildAnthropicMessagesBody(reqNoSys, docs, modelName, runID, 0, false) + body, _, err = buildAnthropicMessagesBody(reqNoSys, docs, modelName, runID, 0, false, nil) if err != nil { t.Fatalf("buildAnthropicMessagesBody no system blocks: %v", err) } @@ -107,8 +107,8 @@ func TestBuildAnthropicMessagesBodyCanonicalDeterminism(t *testing.T) { {Hash: "m1", Role: "user", BlockTypes: []string{"text"}, Bytes: 30}, }, } - _, c1, err1 := buildAnthropicMessagesBody(req, docs, "model", "run1", 0, false) - _, c2, err2 := buildAnthropicMessagesBody(req, docs, "model", "run1", 0, false) + _, c1, err1 := buildAnthropicMessagesBody(req, docs, "model", "run1", 0, false, nil) + _, c2, err2 := buildAnthropicMessagesBody(req, docs, "model", "run1", 0, false, nil) if err1 != nil || err2 != nil { t.Fatalf("errors: %v %v", err1, err2) } @@ -131,7 +131,7 @@ func TestBuildAnthropicMessagesBodyCanonicalContainsAllBlocks(t *testing.T) { {Hash: "msg1", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}, }, } - _, canonical, err := buildAnthropicMessagesBody(req, docs, "model", "runX", 0, false) + _, canonical, err := buildAnthropicMessagesBody(req, docs, "model", "runX", 0, false, nil) if err != nil { t.Fatalf("error: %v", err) } @@ -174,12 +174,12 @@ func TestEffectiveSystemBlocksSkipsHeader(t *testing.T) { hdrText := synthText("uniq-header-per-req", 106, "") for _, builder := range []struct { name string - fn func(RouterReplayRequest, string, string, string, float64, bool) ([]byte, string, error) + fn func(RouterReplayRequest, string, string, string, float64, bool, *uuidInjection) ([]byte, string, error) }{ {"openai", buildOpenAIChatCompletionsBody}, {"anthropic", buildAnthropicMessagesBody}, } { - body, canonical, err := builder.fn(req, "", "m", "", 0, false) + body, canonical, err := builder.fn(req, "", "m", "", 0, false, nil) if err != nil { t.Fatalf("%s build: %v", builder.name, err) } @@ -274,7 +274,7 @@ func TestBuildAnthropicMessagesBodyForceOutput(t *testing.T) { } // --- force-output off (--replay-natural-output) --- - body, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false) + body, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, nil) if err != nil { t.Fatalf("build (force-output off): %v", err) } @@ -283,7 +283,7 @@ func TestBuildAnthropicMessagesBodyForceOutput(t *testing.T) { } // --- force-output on (default) --- - body, _, err = buildAnthropicMessagesBody(req, docs, "model", "", 0, true) + body, _, err = buildAnthropicMessagesBody(req, docs, "model", "", 0, true, nil) if err != nil { t.Fatalf("build (force-output on): %v", err) } @@ -305,7 +305,7 @@ func TestBuildAnthropicMessagesBodyForceOutput(t *testing.T) { // --- force-output on, no system blocks at all: instruction must still be injected --- reqNoSys := RouterReplayRequest{InputTokens: 100} - body, _, err = buildAnthropicMessagesBody(reqNoSys, docs, "model", "", 0, true) + body, _, err = buildAnthropicMessagesBody(reqNoSys, docs, "model", "", 0, true, nil) if err != nil { t.Fatalf("build (force-output on, no system blocks): %v", err) } @@ -328,7 +328,7 @@ func TestBuildOpenAIChatCompletionsBodyForceOutput(t *testing.T) { } // force-output off: neither ignore_eos nor the instruction in the body. - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, nil) if err != nil { t.Fatalf("build (force-output off): %v", err) } @@ -344,7 +344,7 @@ func TestBuildOpenAIChatCompletionsBodyForceOutput(t *testing.T) { } // force-output on (default): ignore_eos=true AND the instruction is present. - body, _, err = buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, true) + body, _, err = buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, true, nil) if err != nil { t.Fatalf("build (force-output on): %v", err) } @@ -367,7 +367,7 @@ func TestBuildAnthropicMessagesBodyOutputRatioMaxTokens(t *testing.T) { docs := strings.Repeat("doc content ", 50) req := RouterReplayRequest{InputTokens: 2000, OutputTokens: 10} - anthBody, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0.25, false) + anthBody, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0.25, false, nil) if err != nil { t.Fatalf("anthropic build: %v", err) } @@ -379,7 +379,7 @@ func TestBuildAnthropicMessagesBodyOutputRatioMaxTokens(t *testing.T) { t.Errorf("anthropic max_tokens = %v, want %v", got, want) } - openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0.25, false) + openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0.25, false, nil) if err != nil { t.Fatalf("openai build: %v", err) } @@ -392,7 +392,7 @@ func TestBuildAnthropicMessagesBodyOutputRatioMaxTokens(t *testing.T) { } // Without a ratio, max_tokens falls back to the original output_tokens. - anthBodyNoRatio, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false) + anthBodyNoRatio, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, nil) if err != nil { t.Fatalf("anthropic build (no ratio): %v", err) } diff --git a/benchmark/replay_router_wire_tools_test.go b/benchmark/replay_router_wire_tools_test.go index 412ae55..2c60109 100644 --- a/benchmark/replay_router_wire_tools_test.go +++ b/benchmark/replay_router_wire_tools_test.go @@ -43,11 +43,11 @@ func TestOpenAIVsAnthropicBodySize(t *testing.T) { Messages: msgs, } - anthBody, _, err := buildAnthropicMessagesBody(req, docs, "model-a", "", 0, false) + anthBody, _, err := buildAnthropicMessagesBody(req, docs, "model-a", "", 0, false, nil) if err != nil { t.Fatalf("buildAnthropicMessagesBody: %v", err) } - openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-a", "", 0, false) + openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-a", "", 0, false, nil) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody: %v", err) } @@ -134,11 +134,11 @@ func TestOpenAIVsAnthropicBodySize(t *testing.T) { sessCount++ for _, inst := range sess.Instances { for _, r := range inst.Requests { - ab, _, err := buildAnthropicMessagesBody(r, docs, "model-a", "", 0, false) + ab, _, err := buildAnthropicMessagesBody(r, docs, "model-a", "", 0, false, nil) if err != nil { continue } - ob, _, err := buildOpenAIChatCompletionsBody(r, docs, "model-a", "", 0, false) + ob, _, err := buildOpenAIChatCompletionsBody(r, docs, "model-a", "", 0, false, nil) if err != nil { continue } @@ -186,7 +186,7 @@ func TestOpenAIToolUseConversion(t *testing.T) { }, } - body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-x", "", 0, false) + body, _, err := buildOpenAIChatCompletionsBody(req, docs, "model-x", "", 0, false, nil) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody: %v", err) } @@ -279,7 +279,7 @@ func TestOpenAIToolUseConversion(t *testing.T) { }, }, } - orphanBody, _, err := buildOpenAIChatCompletionsBody(reqOrphan, docs, "model-x", "", 0, false) + orphanBody, _, err := buildOpenAIChatCompletionsBody(reqOrphan, docs, "model-x", "", 0, false, nil) if err != nil { t.Fatalf("buildOpenAIChatCompletionsBody orphan: %v", err) } diff --git a/benchmark/replay_uuid.go b/benchmark/replay_uuid.go index f09aa85..a11d179 100644 --- a/benchmark/replay_uuid.go +++ b/benchmark/replay_uuid.go @@ -1,93 +1,23 @@ package benchmark -// UUID-based response validation for the dataset-replay benchmark -// (--from-dataset, --replay-inject-uuids). Distinct per-conversation UUIDs -// are stamped into injectable turns as the conversation is walked; the model -// is periodically asked to recite every ref-id it has seen so far, and the -// response is scored for presence (did it recall its own conversation's -// UUIDs?) and cross-contamination (did it leak a UUID belonging to a -// DIFFERENT conversation, i.e. a KV/scheduling leak?). +// Path-agnostic UUID-based response validation primitives, shared by the +// router-replay UUID cache-coherency check (replay_router_uuid.go, +// --replay-inject-uuids + --router-replay-file) and the cache-coherency eval +// CLI (cli/eval_commands.go). Distinct UUIDs are stamped somewhere in a +// request's content; the model is later asked (directly or implicitly) to +// recall them, and the response is scored for presence (did it recall its +// own UUID?) and cross-contamination (did it leak a UUID belonging to a +// DIFFERENT session/series, i.e. a KV/scheduling leak?). // // This is a presence-based check (Contains), not the cache-coherency eval's // exact-conformity check (matchesExpectedUUIDList) — replay responses are // real multi-turn chat turns with real prose, not a bare comma-joined list. -// -// DATASET PATH ONLY: this file must never be reached from the router-replay -// path (replay_router*.go), which reconstructs prefixes from block -// hashes+token counts rather than raw dataset text — injecting visible text -// there would break cache-hit reproduction. The CLI enforces this (see -// cli/benchmark_commands.go: --replay-inject-uuids requires --from-dataset -// and is rejected together with --router-replay-file). import ( "fmt" "strings" ) -// replayTurnInjectable reports whether turn t should carry an injected UUID -// marker under the given --replay-uuid-mode. This is the SAME predicate used -// both to precompute each conversation's UUID count (buildReplayUUIDSets) and -// to walk turns for real (runReplayConversation in replay.go) — the two MUST -// agree exactly, or the precomputed UUID list and the turn walk's cursor -// diverge mid-conversation. -// -// Modes: -// - "human" (default): only turns from the human/user. -// - "all-non-gpt": human, plus tool-result turns and any stray system turn -// (a conversation's LEADING system turn is never walked at all — it -// becomes the cached system prompt instead — so "stray" here means any -// system turn that is NOT at index 0). -func replayTurnInjectable(t HermesTurn, mode string) bool { - if mode == "all-non-gpt" { - return t.From == "human" || t.From == "tool" || t.From == "system" - } - return t.From == "human" -} - -// buildReplayUUIDSets precomputes, for every conversation in convs, the full -// ordered list of UUIDs its injectable turns will carry over the course of -// the conversation: injectableTurns * perTurn UUIDs. Every UUID across every -// conversation is drawn from a SINGLE seeded generator in conversation-major, -// turn-minor order, which is what makes the result both deterministic (same -// seed -> same output) and disjoint by construction (each draw is unique — -// the seeded PCG generator never repeats a UUID within a run). -// -// The returned slice is indexed by conversation index (parallel to convs), -// matching cfg.replayUUIDSets / cfg.replayConversations elsewhere. -func buildReplayUUIDSets(convs []Conversation, seed int64, perTurn int, mode string) [][]string { - newUUID := newUUIDGenerator(seed) - sets := make([][]string, len(convs)) - for i, conv := range convs { - // Skip a LEADING system turn exactly like the real turn walk does - // (runReplayConversation / computeInScopeAtEachGptTurn, both in - // replay.go) — it becomes the cached system prompt, not a walked turn. - // Without this skip, "all-non-gpt" mode (which counts system turns) - // would count that turn here but the real walk would never consume a - // UUID for it, drifting the precomputed count out of lockstep with - // what's actually assigned. - turns := conv.Turns - if len(turns) > 0 && turns[0].From == "system" { - turns = turns[1:] - } - injectable := 0 - for _, t := range turns { - if replayTurnInjectable(t, mode) { - injectable++ - } - } - n := injectable * perTurn - if n <= 0 { - continue - } - uuids := make([]string, n) - for j := range uuids { - uuids[j] = newUUID() - } - sets[i] = uuids - } - return sets -} - // injectUUIDMarker appends one visible marker per uuid to turnValue. Unlike // the cache-coherency eval's ... filler, the model MUST see // and be able to repeat this text — it's asked to recite every ref-id later @@ -108,98 +38,6 @@ func injectUUIDMarker(turnValue string, uuids []string) string { return b.String() } -// replayReciteInstruction returns the boilerplate appended to an outgoing -// user turn, asking the model to FIRST recite every ref-id it has seen so -// far (inScope, in order) on a delimited line, THEN answer normally. -// Presence scoring is Contains-based (see validateReplayResponse), so the -// exact wording/format here is not load-bearing — the "SEEN_REFS:" delimiter -// just keeps the recited list easy to spot in a transcript/log. -func replayReciteInstruction(inScope []string) string { - return fmt.Sprintf("\n\n(Before your normal answer, first output one line in the exact form `SEEN_REFS: %s` listing every ref-id you have seen anywhere in this conversation so far, comma-separated. Then answer normally.)", - strings.Join(inScope, ",")) -} - -// replayReciteBudgetFraction caps how much of a request's max_tokens output -// budget the recited ref-id list is allowed to consume (see -// capRecitedUUIDs) — recite-every-turn means the list only grows, so without -// a cap a long conversation eventually asks the model to reproduce more -// ref-ids than max_tokens can hold, truncating the SEEN_REFS line itself. -const replayReciteBudgetFraction = 0.5 - -// capRecitedUUIDs trims inScope down to (at most) however many entries fit -// within replayReciteBudgetFraction of maxOutputTokens, keeping the MOST -// RECENT entries (dropping the oldest first — the model is more likely to -// have retained recent context). Returns the (possibly untouched) list and -// whether trimming occurred. -// -// This is deliberately a simple heuristic (reuses the standard -// len/4 estimateTokens idiom already used elsewhere in this package, see -// cache_sim.go) — not exact tokenizer accounting. maxOutputTokens <= 0 -// (budget unknown/unbounded) disables capping entirely. -func capRecitedUUIDs(inScope []string, maxOutputTokens int) ([]string, bool) { - if maxOutputTokens <= 0 || len(inScope) == 0 { - return inScope, false - } - budget := int(float64(maxOutputTokens) * replayReciteBudgetFraction) - if estimateTokens(strings.Join(inScope, ",")) <= budget { - return inScope, false - } - perUUID := estimateTokens(inScope[0] + ",") - if perUUID < 1 { - perUUID = 1 - } - n := budget / perUUID - if n < 1 { - n = 1 - } - if n >= len(inScope) { - return inScope, false - } - return inScope[len(inScope)-n:], true -} - -// computeInScopeAtEachGptTurn walks turns exactly like runReplayConversation's -// real turn loop (replay.go) — skipping turns[0] when it's the leading system -// turn (that becomes the cached system prompt, not a walked turn) — and -// returns, for each 'gpt' turn encountered in order, a snapshot of every UUID -// assigned to an injectable turn seen so far. len(result) == the number of -// 'gpt' turns in turns (after the leading-system skip). -// -// This exists purely so the cumulative in-scope tracking is unit-testable in -// isolation (see replay_uuid_test.go); replay.go's real loop additionally -// needs the PER-TURN uuid slice (to wrap the turn text via injectUUIDMarker), -// so it maintains the same cursor/slicing logic inline rather than calling -// this function directly — the two must be kept in lockstep, which the test -// suite verifies. -func computeInScopeAtEachGptTurn(turns []HermesTurn, sets []string, perTurn int, mode string) [][]string { - firstIdx := 0 - if len(turns) > 0 && turns[0].From == "system" { - firstIdx = 1 - } - - var result [][]string - var inScope []string - cursor := 0 - for i := firstIdx; i < len(turns); i++ { - t := turns[i] - if t.From == "gpt" { - result = append(result, append([]string(nil), inScope...)) - continue - } - if replayTurnInjectable(t, mode) { - end := cursor + perTurn - if end > len(sets) { - end = len(sets) - } - if cursor < end { - inScope = append(inScope, sets[cursor:end]...) - } - cursor = end - } - } - return result -} - // validateReplayResponse scores one replay response/thinking pair: // - found[i] reports whether expected[i] (this conversation's in-scope // ref-ids at this point) appears in resp or thinking (Contains, mirroring @@ -226,9 +64,10 @@ func validateReplayResponse(resp, thinking string, expected []string, convIdx in // // Exported (moved from cli/eval_commands.go) so both the cache-coherency // eval CLI (cli/eval_commands.go, where ownIdx is a coherency series index -// and allSets is CacheCoherencyResult.SeriesUUIDs) and dataset-replay UUID -// validation above (where ownIdx is a conversation index and allSets is -// AutoBenchmarkConfig.replayUUIDSets) share one implementation, not two. +// and allSets is CacheCoherencyResult.SeriesUUIDs) and replay UUID +// validation above (where ownIdx is a conversation/session index and +// allSets is AutoBenchmarkConfig.replayUUIDSets) share one implementation, +// not two. func FindLeakedUUIDs(resp, thinking string, ownIdx int, allSets [][]string) []string { var leaked []string for si, uuids := range allSets { diff --git a/benchmark/replay_uuid_test.go b/benchmark/replay_uuid_test.go index 3704303..68f80bf 100644 --- a/benchmark/replay_uuid_test.go +++ b/benchmark/replay_uuid_test.go @@ -5,132 +5,6 @@ import ( "testing" ) -// syntheticMixedRoleConvs builds a small synthetic conversation set with mixed -// From roles (human/gpt/tool/system, including one LEADING system turn and one -// STRAY mid-conversation system turn) — enough to exercise both -// --replay-uuid-mode values. -func syntheticMixedRoleConvs() []Conversation { - return []Conversation{ - {ID: "c0", Turns: []HermesTurn{ - {From: "system", Value: "c0 leading system prompt"}, - {From: "human", Value: "c0 h1"}, - {From: "gpt", Value: "c0 g1"}, - {From: "tool", Value: "c0 t1"}, - {From: "human", Value: "c0 h2"}, - {From: "gpt", Value: "c0 g2"}, - }}, - {ID: "c1", Turns: []HermesTurn{ - {From: "human", Value: "c1 h1"}, // no leading system turn at all - {From: "gpt", Value: "c1 g1"}, - {From: "system", Value: "c1 stray system"}, // NOT at index 0 - {From: "human", Value: "c1 h2"}, - {From: "gpt", Value: "c1 g2"}, - }}, - } -} - -func TestBuildReplayUUIDSets(t *testing.T) { - convs := syntheticMixedRoleConvs() - - t.Run("determinism under fixed seed", func(t *testing.T) { - a := buildReplayUUIDSets(convs, 42, 2, "human") - b := buildReplayUUIDSets(convs, 42, 2, "human") - if len(a) != len(b) { - t.Fatalf("length mismatch: %d vs %d", len(a), len(b)) - } - for i := range a { - if len(a[i]) != len(b[i]) { - t.Fatalf("conv %d length mismatch: %d vs %d", i, len(a[i]), len(b[i])) - } - for j := range a[i] { - if a[i][j] != b[i][j] { - t.Errorf("conv %d uuid %d mismatch across identical-seed calls: %q vs %q", i, j, a[i][j], b[i][j]) - } - } - } - }) - - t.Run("different seeds diverge", func(t *testing.T) { - a := buildReplayUUIDSets(convs, 1, 2, "human") - b := buildReplayUUIDSets(convs, 2, 2, "human") - same := true - for i := range a { - for j := range a[i] { - if a[i][j] != b[i][j] { - same = false - } - } - } - if same { - t.Errorf("different seeds produced identical uuid sets") - } - }) - - t.Run("disjoint across conversations", func(t *testing.T) { - sets := buildReplayUUIDSets(convs, 7, 2, "all-non-gpt") - owner := make(map[string]int) - for ci, uuids := range sets { - for _, u := range uuids { - if prevCi, ok := owner[u]; ok { - t.Errorf("uuid %q appears in both conversation %d and conversation %d", u, prevCi, ci) - } - owner[u] = ci - } - } - }) - - t.Run("count == injectableTurns*perTurn, mode human", func(t *testing.T) { - const perTurn = 3 - sets := buildReplayUUIDSets(convs, 1, perTurn, "human") - // c0: human turns = h1, h2 -> 2. c1: human turns = h1, h2 -> 2. - if got, want := len(sets[0]), 2*perTurn; got != want { - t.Errorf("conv0 human mode: len = %d, want %d", got, want) - } - if got, want := len(sets[1]), 2*perTurn; got != want { - t.Errorf("conv1 human mode: len = %d, want %d", got, want) - } - }) - - t.Run("count == injectableTurns*perTurn, mode all-non-gpt", func(t *testing.T) { - const perTurn = 2 - sets := buildReplayUUIDSets(convs, 1, perTurn, "all-non-gpt") - // c0: leading system turn (index 0) is stripped before counting, so - // injectable turns are h1, tool t1, h2 -> 3. - if got, want := len(sets[0]), 3*perTurn; got != want { - t.Errorf("conv0 all-non-gpt mode: len = %d, want %d", got, want) - } - // c1: no leading system turn to strip; injectable turns are h1, the - // STRAY mid-conversation system turn, h2 -> 3. - if got, want := len(sets[1]), 3*perTurn; got != want { - t.Errorf("conv1 all-non-gpt mode: len = %d, want %d", got, want) - } - }) -} - -func TestReplayTurnInjectable(t *testing.T) { - tests := []struct { - from string - wantHuman bool - wantAllNGpt bool - }{ - {"human", true, true}, - {"gpt", false, false}, - {"tool", false, true}, - {"system", false, true}, - } - for _, tt := range tests { - t.Run(tt.from, func(t *testing.T) { - turn := HermesTurn{From: tt.from, Value: "x"} - if got := replayTurnInjectable(turn, "human"); got != tt.wantHuman { - t.Errorf("replayTurnInjectable(%q, \"human\") = %v, want %v", tt.from, got, tt.wantHuman) - } - if got := replayTurnInjectable(turn, "all-non-gpt"); got != tt.wantAllNGpt { - t.Errorf("replayTurnInjectable(%q, \"all-non-gpt\") = %v, want %v", tt.from, got, tt.wantAllNGpt) - } - }) - } -} - func TestInjectUUIDMarker(t *testing.T) { original := "what is the weather today?" uuids := []string{"uuid-aaa", "uuid-bbb"} @@ -152,64 +26,6 @@ func TestInjectUUIDMarker(t *testing.T) { } } -// TestComputeInScopeAtEachGptTurn walks a synthetic turn sequence and asserts -// the snapshot at each gpt-turn boundary equals the expected prefix union of -// uuids assigned to injectable turns seen so far — the invariant -// replay.go's real turn loop maintains inline (see the comment there pointing -// back at this test). -func TestComputeInScopeAtEachGptTurn(t *testing.T) { - turns := []HermesTurn{ - {From: "system", Value: "leading system prompt"}, // skipped entirely - {From: "human", Value: "h1"}, // injectable (human mode): uuids[0:2] - {From: "gpt", Value: "g1"}, // snapshot #0 -> uuids[0:2] - {From: "tool", Value: "t1"}, // NOT injectable in human mode - {From: "human", Value: "h2"}, // injectable: uuids[2:4] - {From: "gpt", Value: "g2"}, // snapshot #1 -> uuids[0:4] - {From: "human", Value: "h3"}, // injectable: uuids[4:6] - {From: "gpt", Value: "g3"}, // snapshot #2 -> uuids[0:6] - } - sets := []string{"u0", "u1", "u2", "u3", "u4", "u5"} - const perTurn = 2 - - got := computeInScopeAtEachGptTurn(turns, sets, perTurn, "human") - want := [][]string{ - {"u0", "u1"}, - {"u0", "u1", "u2", "u3"}, - {"u0", "u1", "u2", "u3", "u4", "u5"}, - } - if len(got) != len(want) { - t.Fatalf("computeInScopeAtEachGptTurn() = %d snapshots, want %d", len(got), len(want)) - } - for i := range want { - if strings.Join(got[i], ",") != strings.Join(want[i], ",") { - t.Errorf("snapshot[%d] = %v, want %v", i, got[i], want[i]) - } - } - - // Mutating a returned snapshot must not corrupt a previous snapshot - // (defensive copy per call) or later ones. - got[0][0] = "MUTATED" - got2 := computeInScopeAtEachGptTurn(turns, sets, perTurn, "human") - if got2[0][0] != "u0" { - t.Errorf("computeInScopeAtEachGptTurn() snapshots are not independently allocated: got2[0][0] = %q", got2[0][0]) - } - - // all-non-gpt mode additionally picks up the mid-conversation 'tool' turn, - // so the second snapshot's union grows by one more perTurn slice than in - // human mode (the leading system turn is still skipped). - gotAll := computeInScopeAtEachGptTurn(turns, sets, perTurn, "all-non-gpt") - if len(gotAll) != 3 { - t.Fatalf("all-non-gpt mode: got %d snapshots, want 3", len(gotAll)) - } - if len(gotAll[0]) != 2 { - t.Errorf("all-non-gpt mode snapshot[0] = %v, want len 2", gotAll[0]) - } - if len(gotAll[1]) != 6 { - // h1 (2) + tool t1 (2) + h2 (2) = 6, vs 4 in human-only mode. - t.Errorf("all-non-gpt mode snapshot[1] = %v, want len 6 (tool turn now counted)", gotAll[1]) - } -} - func TestValidateReplayResponse(t *testing.T) { allSets := [][]string{ {"own-0", "own-1"}, // conversation 0 (this response's own conversation) @@ -313,48 +129,3 @@ func TestFindLeakedUUIDs(t *testing.T) { } }) } - -func TestCapRecitedUUIDs(t *testing.T) { - t.Run("unbounded budget (<=0) never trims", func(t *testing.T) { - in := []string{"a", "b", "c"} - got, trimmed := capRecitedUUIDs(in, 0) - if trimmed { - t.Errorf("expected no trim with maxOutputTokens<=0") - } - if len(got) != len(in) { - t.Errorf("got %v, want unchanged %v", got, in) - } - }) - - t.Run("small list well within budget is untouched", func(t *testing.T) { - in := []string{"11111111-1111-1111-1111-111111111111"} - got, trimmed := capRecitedUUIDs(in, 10000) - if trimmed { - t.Errorf("expected no trim for a single uuid against a large budget") - } - if len(got) != 1 { - t.Errorf("got %v, want unchanged", got) - } - }) - - t.Run("large list against a tiny budget is trimmed to the most recent entries", func(t *testing.T) { - in := make([]string, 200) - for i := range in { - in[i] = "11111111-1111-1111-1111-11111111111" + string(rune('0'+i%10)) - } - got, trimmed := capRecitedUUIDs(in, 20) // tiny budget forces a cap - if !trimmed { - t.Fatalf("expected trimming for a 200-uuid list against a 20-token budget") - } - if len(got) == 0 || len(got) >= len(in) { - t.Fatalf("got %d entries, want a proper subset of %d", len(got), len(in)) - } - // Kept entries must be the MOST RECENT (tail) of the input, in order. - wantTail := in[len(in)-len(got):] - for i := range got { - if got[i] != wantTail[i] { - t.Errorf("capRecitedUUIDs did not keep the most-recent tail: got[%d]=%q, want %q", i, got[i], wantTail[i]) - } - } - }) -} diff --git a/benchmark/types.go b/benchmark/types.go index 5510b9a..38f50ee 100644 --- a/benchmark/types.go +++ b/benchmark/types.go @@ -24,11 +24,11 @@ type RequestMetrics struct { Question string // The user question sent with the request (only populated on error/empty) RawResponseTail string // raw SSE tail (last bytes); only populated on error/empty for diagnostics - // UUID validation (dataset-replay --replay-inject-uuids only). All nil/zero - // when the feature is off (default) or on the router-replay/synthetic paths, - // which never populate these. - ConvIdx int // conversation index within cfg.replayConversations / cfg.replayUUIDSets - ExpectedUUIDs []string // this conversation's in-scope ref-id UUIDs at this turn (defensive copy) + // UUID validation (router-replay --replay-inject-uuids only). All nil/zero + // when the feature is off (default) or on the synthetic path, which never + // populates these. + ConvIdx int // session index within cfg.replayUUIDSets (== seriesNum-1) + ExpectedUUIDs []string // this session's own UUID (singleton slice) UUIDFound []bool // parallel to ExpectedUUIDs: whether each was found in Response or thinking - LeakedUUIDs []string // "uuid(series=N)" entries for any OTHER conversation's UUID found here + LeakedUUIDs []string // "uuid(series=N)" entries for any OTHER session's UUID found here } diff --git a/cli/benchmark_commands.go b/cli/benchmark_commands.go index 3cebe45..15a9193 100644 --- a/cli/benchmark_commands.go +++ b/cli/benchmark_commands.go @@ -270,10 +270,8 @@ func (c *BenchmarkAutoCommand) Execute(args []string) error { AbortOnCollapse: c.AbortOnCollapse, ReplayStopAtLowConcurrency: c.ReplayStopAtLowConcurrency, ReplayInjectUUIDs: c.ReplayInjectUUIDs, - ReplayUUIDsPerTurn: c.ReplayUUIDsPerTurn, ReplayUUIDSeed: c.ReplayUUIDSeed, - ReplayUUIDMode: c.ReplayUUIDMode, - ReplayReciteEveryTurn: c.ReplayReciteEveryTurn != "false", + ReplayReciteEveryRequest: c.ReplayReciteEveryRequest != "false", RouterReplayFile: c.RouterReplayFile, RouterReplayRoles: c.RouterReplayRoles, ReplayOutputRatio: c.ReplayOutputRatio, @@ -292,20 +290,19 @@ func (c *BenchmarkAutoCommand) Execute(args []string) error { return fmt.Errorf("--from-dataset and --router-replay-file are mutually exclusive") } - // --replay-inject-uuids is DATASET PATH ONLY: router replay reconstructs - // prefixes from block hashes+token counts, so injecting visible ref-id - // text there would diverge those hashes and break cache-hit reproduction. + // --replay-inject-uuids is ROUTER-REPLAY PATH ONLY: it splices a + // per-session UUID marker at the boundary between cross-session-shared + // prefix blocks (computed from the replay-v3 block-hash schema) and + // per-session content — the dataset-replay path has no such block-hash + // schema to compute that boundary from. if c.ReplayInjectUUIDs { - if c.FromDataset == "" { - return fmt.Errorf("--replay-inject-uuids requires --from-dataset") + if c.RouterReplayFile == "" { + return fmt.Errorf("--replay-inject-uuids requires --router-replay-file") } - if c.RouterReplayFile != "" { - return fmt.Errorf("--replay-inject-uuids and --router-replay-file are mutually exclusive") + if c.FromDataset != "" { + return fmt.Errorf("--replay-inject-uuids and --from-dataset are mutually exclusive") } } - if c.ReplayUUIDMode != "human" && c.ReplayUUIDMode != "all-non-gpt" { - return fmt.Errorf("--replay-uuid-mode must be 'human' or 'all-non-gpt', got %q", c.ReplayUUIDMode) - } if c.DryRun && c.RouterReplayFile == "" { return fmt.Errorf("--dry-run requires --router-replay-file") } diff --git a/cli/benchmark_options.go b/cli/benchmark_options.go index b98f5e9..0dc0687 100644 --- a/cli/benchmark_options.go +++ b/cli/benchmark_options.go @@ -63,28 +63,26 @@ type BenchmarkAutoOptions struct { ReplayNoStamp bool `long:"replay-no-stamp" description:"Disable per-run RUN_GUID stamping in replay mode. By default each replay run prepends a fresh UUID to every request's system prompt (both --from-dataset and --router-replay-file paths) so server prefix caches from prior runs can't be reused — pristine per-run cache state." env:"BENCHMARK_REPLAY_NO_STAMP"` AbortOnCollapse bool `long:"abort-on-collapse" description:"Abort the benchmark if the windowed cache hit rate stays below 50% for 2 minutes. Off by default — this heuristic fires on legitimate workloads with low cache reuse (e.g. replay across many distinct conversations)." env:"BENCHMARK_ABORT_ON_COLLAPSE"` ReplayStopAtLowConcurrency bool `long:"replay-stop-at-low-concurrency" description:"Terminate the replay run once the queue is drained AND the number of active worker goroutines has dropped below --concurrency. Avoids long-tail measurements where only a handful of long conversations remain and the gate is underutilized." env:"BENCHMARK_REPLAY_STOP_AT_LOW_CONCURRENCY"` - ReplayInjectUUIDs bool `long:"replay-inject-uuids" description:"Inject per-turn UUID markers into --from-dataset replay conversations and validate their presence in later responses -- a coherency check (PRESENCE_MISS / CROSS_CONTAMINATION) for the KV-offload path under realistic multi-turn traffic. DATASET PATH ONLY: requires --from-dataset; rejected together with --router-replay-file (router replay reconstructs prefixes from block hashes+token counts, and injecting visible text there would break cache-hit reproduction)." env:"BENCHMARK_REPLAY_INJECT_UUIDS"` - ReplayUUIDsPerTurn int `long:"replay-uuids-per-turn" description:"Number of UUID ref-ids injected per injectable turn (see --replay-uuid-mode). Only used with --replay-inject-uuids." default:"1" env:"BENCHMARK_REPLAY_UUIDS_PER_TURN"` - ReplayUUIDSeed int64 `long:"replay-uuid-seed" description:"PRNG seed for --replay-inject-uuids' UUID generation (0 = crypto/rand, non-deterministic across runs)." default:"0" env:"BENCHMARK_REPLAY_UUID_SEED"` - ReplayUUIDMode string `long:"replay-uuid-mode" description:"Which turns get UUID markers under --replay-inject-uuids: 'human' (default, only human/user turns) or 'all-non-gpt' (also tool-result turns and stray system turns)." default:"human" env:"BENCHMARK_REPLAY_UUID_MODE"` - // ReplayReciteEveryTurn is a string (not bool) choice, same workaround as - // RandomGateOrder above: a plain bool flag defaulting true can never be - // turned OFF via the CLI with this parser (go-flags only accepts + ReplayInjectUUIDs bool `long:"replay-inject-uuids" description:"Inject one deterministic UUID marker per SESSION into --router-replay-file replay sessions, at the boundary between cross-session-shared leading blocks and per-session content, and validate its presence in later responses -- a coherency check (PRESENCE_MISS / CROSS_CONTAMINATION) for the KV-offload path under realistic multi-turn agentic traffic. ROUTER-REPLAY PATH ONLY: requires --router-replay-file; rejected together with --from-dataset." env:"BENCHMARK_REPLAY_INJECT_UUIDS"` + ReplayUUIDSeed int64 `long:"replay-uuid-seed" description:"PRNG seed for --replay-inject-uuids' per-session UUID generation (0 = crypto/rand, non-deterministic across runs)." default:"0" env:"BENCHMARK_REPLAY_UUID_SEED"` + // ReplayReciteEveryRequest is a string (not bool) choice, same workaround + // as RandomGateOrder above: a plain bool flag defaulting true can never + // be turned OFF via the CLI with this parser (go-flags only accepts // `--flag=false` for bool options when the parser is built with // AllowBoolValues, which main.go's flags.Default does not set). - ReplayReciteEveryTurn string `long:"replay-recite-every-turn" choice:"true" choice:"false" default:"true" optional:"yes" optional-value:"true" description:"With --replay-inject-uuids, ask the model to recite every ref-id seen so far on EVERY turn (the DEFAULT) rather than only on the conversation's final turn. Pass --replay-recite-every-turn=false to recite only on the final turn." env:"BENCHMARK_REPLAY_RECITE_EVERY_TURN"` - RouterReplayFile string `long:"router-replay-file" description:"Path to a tree-aware replay file produced by 'wekai router replay-prepare'. Each series = one CLI session; within the session sub-agents fan out concurrently, honoring parent->child sequencing baked into the file. Mutually exclusive with --from-dataset." env:"BENCHMARK_ROUTER_REPLAY_FILE"` - ReplayOutputRatio float64 `long:"replay-output-ratio" description:"Router-replay only: retarget each request's max_tokens to InputTokens * this ratio, overriding the recorded original output_tokens (which otherwise pins max_tokens to what the model produced in the original capture, so the model stops almost immediately on replay). 0 = off (default: use original output_tokens/max_tokens)." default:"0" env:"BENCHMARK_REPLAY_OUTPUT_RATIO"` - ReplayNaturalOutput bool `long:"replay-natural-output" description:"Let the model stop generation naturally instead of forcing it to fill max_tokens (disables the continue-generating instruction and vLLM ignore_eos). Router-replay only." env:"BENCHMARK_REPLAY_NATURAL_OUTPUT"` - RouterReplayRoles string `long:"router-replay-roles" description:"Comma-separated list of instance roles to replay (default: all). E.g. 'main,sub-agent' excludes the CLI's background helper:title / helper:summarize / ephemeral (haiku side-calls) instances and keeps only the agentic workload — gives a cleaner 'in_flight ~= series' steady state. Other values: 'helper-or-isolated', 'ephemeral (no system)', 'other'." env:"BENCHMARK_ROUTER_REPLAY_ROLES"` - RouterReplaySeriesIndices string `long:"replay-series-indices" description:"Comma-separated list of 0-based session indices to replay from --router-replay-file (e.g. '3,7,42'). Only sessions at those line positions (0 = first session after the header) are dispatched; others are skipped. Mutually exclusive with --replay-series-range. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_INDICES"` - RouterReplaySeriesRange string `long:"replay-series-range" description:"Inclusive range of 0-based session indices to replay from --router-replay-file (e.g. '0-50' or '100-199'). Mutually exclusive with --replay-series-indices. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_RANGE"` - DryRun bool `long:"dry-run" description:"Dry run: skip remote HTTP requests; drive the router-replay pipeline with synthetic timing so gcache evolution can be observed offline. Requires --router-replay-file." env:"BENCHMARK_DRY_RUN"` - DryRunColdTPS int `long:"dry-run-cold-tps" default:"1000000" description:"Dry-run: cold (uncached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_COLD_TPS"` - DryRunWarmTPS int `long:"dry-run-warm-tps" default:"10000000" description:"Dry-run: warm (cached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_WARM_TPS"` - DryRunOutputTPS int `long:"dry-run-output-tps" default:"100000" description:"Dry-run: output tokens generated per second." env:"BENCHMARK_DRY_RUN_OUTPUT_TPS"` - CacheSimChunkBytes int `long:"cache-sim-chunk-bytes" description:"Chunk size in bytes for the content-level cache estimator (0 = default 1024)." default:"0" env:"BENCHMARK_CACHE_SIM_CHUNK_BYTES"` - RandomGateOrder string `long:"random-gate-order" choice:"true" choice:"false" default:"true" optional:"yes" optional-value:"true" description:"Wake the concurrency gate's waiting series in uniformly random order when oversubscribed (the DEFAULT). Strict FIFO forces every series to wait behind all other waiting series before its next turn -- the adversarial worst case for GPU prefix-cache LRU. Pass --random-gate-order=false for the legacy exact-FIFO order. Cold-start waiters are unaffected (always served first, FIFO)." env:"BENCHMARK_RANDOM_GATE_ORDER"` + ReplayReciteEveryRequest string `long:"replay-recite-every-request" choice:"true" choice:"false" default:"true" optional:"yes" optional-value:"true" description:"With --replay-inject-uuids, ask the model to recite the ref-id marker on EVERY request (the DEFAULT) rather than only on each agent instance's final request. Pass --replay-recite-every-request=false to recite only on each instance's last request." env:"BENCHMARK_REPLAY_RECITE_EVERY_REQUEST"` + RouterReplayFile string `long:"router-replay-file" description:"Path to a tree-aware replay file produced by 'wekai router replay-prepare'. Each series = one CLI session; within the session sub-agents fan out concurrently, honoring parent->child sequencing baked into the file. Mutually exclusive with --from-dataset." env:"BENCHMARK_ROUTER_REPLAY_FILE"` + ReplayOutputRatio float64 `long:"replay-output-ratio" description:"Router-replay only: retarget each request's max_tokens to InputTokens * this ratio, overriding the recorded original output_tokens (which otherwise pins max_tokens to what the model produced in the original capture, so the model stops almost immediately on replay). 0 = off (default: use original output_tokens/max_tokens)." default:"0" env:"BENCHMARK_REPLAY_OUTPUT_RATIO"` + ReplayNaturalOutput bool `long:"replay-natural-output" description:"Let the model stop generation naturally instead of forcing it to fill max_tokens (disables the continue-generating instruction and vLLM ignore_eos). Router-replay only." env:"BENCHMARK_REPLAY_NATURAL_OUTPUT"` + RouterReplayRoles string `long:"router-replay-roles" description:"Comma-separated list of instance roles to replay (default: all). E.g. 'main,sub-agent' excludes the CLI's background helper:title / helper:summarize / ephemeral (haiku side-calls) instances and keeps only the agentic workload — gives a cleaner 'in_flight ~= series' steady state. Other values: 'helper-or-isolated', 'ephemeral (no system)', 'other'." env:"BENCHMARK_ROUTER_REPLAY_ROLES"` + RouterReplaySeriesIndices string `long:"replay-series-indices" description:"Comma-separated list of 0-based session indices to replay from --router-replay-file (e.g. '3,7,42'). Only sessions at those line positions (0 = first session after the header) are dispatched; others are skipped. Mutually exclusive with --replay-series-range. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_INDICES"` + RouterReplaySeriesRange string `long:"replay-series-range" description:"Inclusive range of 0-based session indices to replay from --router-replay-file (e.g. '0-50' or '100-199'). Mutually exclusive with --replay-series-indices. Overrides --replay-series." env:"BENCHMARK_REPLAY_SERIES_RANGE"` + DryRun bool `long:"dry-run" description:"Dry run: skip remote HTTP requests; drive the router-replay pipeline with synthetic timing so gcache evolution can be observed offline. Requires --router-replay-file." env:"BENCHMARK_DRY_RUN"` + DryRunColdTPS int `long:"dry-run-cold-tps" default:"1000000" description:"Dry-run: cold (uncached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_COLD_TPS"` + DryRunWarmTPS int `long:"dry-run-warm-tps" default:"10000000" description:"Dry-run: warm (cached) input tokens processed per second." env:"BENCHMARK_DRY_RUN_WARM_TPS"` + DryRunOutputTPS int `long:"dry-run-output-tps" default:"100000" description:"Dry-run: output tokens generated per second." env:"BENCHMARK_DRY_RUN_OUTPUT_TPS"` + CacheSimChunkBytes int `long:"cache-sim-chunk-bytes" description:"Chunk size in bytes for the content-level cache estimator (0 = default 1024)." default:"0" env:"BENCHMARK_CACHE_SIM_CHUNK_BYTES"` + RandomGateOrder string `long:"random-gate-order" choice:"true" choice:"false" default:"true" optional:"yes" optional-value:"true" description:"Wake the concurrency gate's waiting series in uniformly random order when oversubscribed (the DEFAULT). Strict FIFO forces every series to wait behind all other waiting series before its next turn -- the adversarial worst case for GPU prefix-cache LRU. Pass --random-gate-order=false for the legacy exact-FIFO order. Cold-start waiters are unaffected (always served first, FIFO)." env:"BENCHMARK_RANDOM_GATE_ORDER"` // Positional arguments Args struct { From 716d583a8e77843ace177d405895b3883afc7b90 Mon Sep 17 00:00:00 2001 From: Ford Shaper Date: Fri, 24 Jul 2026 15:35:55 -0400 Subject: [PATCH 3/6] feat(benchmark): retarget router-replay UUID injection to coherency-test mechanics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes the --replay-inject-uuids feature to mirror cache_coherency.go's --shared-prefix-per-series mode instead of its own single-marker scheme: - N bare UUIDs per session (was 1 wrapped "[ref-id: ...]" marker), sized via computeStampsPerSeries off each session's per-session cached-region bytes (the root request's prefix blocks at/after sharedPrefixBlockCount) — see the new computePerSessionCachedChars/requestPrefixBytes helpers. - Injection is spliced as one system block/message of N bare, space-separated UUIDs at the boundary (or tail-fallback), mirroring buildCoherencySharedSeriesPrompt's tail, byte-identical across a session's own requests. - Recite instruction now asks for the exact ordered UUID list on the FIRST line, then lets the model continue normally (replayReciteFirstLineInstruction), replacing the old "find and echo the marker" ask. - Validation adds first-line output conformity (firstLineConformity, reusing matchesExpectedUUIDList) alongside the existing per-UUID presence and cross-session leak checks; RequestMetrics/requestDataRecord gain ExactMatch/ uuid_exact_match, and the auto-summary prints both UUID correctness and output conformity, matching the coherency CLI's two-test layout. - The recite max_tokens floor now scales with N via computeMaxOutputTokens instead of a fixed 64-token constant. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/auto.go | 74 ++++-- benchmark/replay.go | 4 + benchmark/replay_router.go | 25 ++ benchmark/replay_router_post.go | 9 +- benchmark/replay_router_uuid.go | 284 ++++++++++++++++----- benchmark/replay_router_uuid_test.go | 354 +++++++++++++++++++++------ benchmark/replay_router_wire.go | 60 +++-- benchmark/types.go | 3 +- 8 files changed, 625 insertions(+), 188 deletions(-) diff --git a/benchmark/auto.go b/benchmark/auto.go index 0774949..77212a5 100644 --- a/benchmark/auto.go +++ b/benchmark/auto.go @@ -91,29 +91,33 @@ type AutoBenchmarkConfig struct { // UUID-based cache-coherency validation (--replay-inject-uuids). ROUTER // PATH ONLY (cfg.RouterReplayFile != ""); the CLI rejects this combined - // with --from-dataset (see cli/benchmark_commands.go). One deterministic - // UUID is injected per SESSION at the boundary between its - // cross-session-shared leading blocks and its per-session content (see - // replay_router_uuid.go for the full design) — this puts the marker in - // a region cached WITHIN a session (later requests in the same session - // repeat it) while leaving the cross-session shared prefix byte- - // identical, so cache-hit reproduction against the original capture is - // preserved. + // with --from-dataset (see cli/benchmark_commands.go). Mirrors the + // cache-coherency eval's --shared-prefix-per-series mechanics: N + // deterministic, bare, space-separated UUIDs (N sized off the session's + // per-session cached-region bytes, via computeStampsPerSeries — see + // computePerSessionCachedChars) are injected per SESSION at the boundary + // between its cross-session-shared leading blocks and its per-session + // content (see replay_router_uuid.go for the full design) — this puts + // the UUID block in a region cached WITHIN a session (later requests in + // the same session repeat it, byte-identical) while leaving the + // cross-session shared prefix byte-identical, so cache-hit reproduction + // against the original capture is preserved. ReplayInjectUUIDs bool // ReplayUUIDSeed seeds the UUID generator (see newUUIDGenerator); 0 = crypto/rand // (non-deterministic across runs). ReplayUUIDSeed int64 - // ReplayReciteEveryRequest: ask the model to recite the ref-id marker on - // EVERY request (default true), not just each instance's final request. + // ReplayReciteEveryRequest: ask the model to recite the first-line UUID + // list on EVERY request (default true), not just each instance's final + // request. ReplayReciteEveryRequest bool // replayUUIDSets is the precomputed per-session UUID list, populated // once by RunAutoBenchmark before any per-model goroutine spawns — see - // buildSessionUUIDs. Index i = session i's owned singleton UUID set - // (index i corresponds to seriesNum-1, the order sessions are - // dispatched in — see the sizing note at the router-replay precompute - // call site). Shared, read-only, across every model in a multi-model - // run so every model sees the identical assignment (same sharing - // rationale as replayConversations below). + // buildSessionUUIDs. Index i = session i's owned N-UUID list (index i + // corresponds to seriesNum-1, the order sessions are dispatched in — + // see the sizing note at the router-replay precompute call site). + // Shared, read-only, across every model in a multi-model run so every + // model sees the identical assignment (same sharing rationale as + // replayConversations below). replayUUIDSets [][]string // replayBlockSessionCounts maps a replay-v3 block hash to the number of // DISTINCT SESSIONS that reference it (see computeBlockSessionCounts) — @@ -242,6 +246,7 @@ type requestDataRecord struct { UUIDExpected int `json:"uuid_expected"` UUIDFound int `json:"uuid_found"` UUIDLeaked int `json:"uuid_leaked"` + UUIDExactMatch bool `json:"uuid_exact_match"` ExpectedUUIDsRaw []string `json:"expected_uuids_raw,omitempty"` FoundMask []bool `json:"found_mask,omitempty"` LeakedUUIDsRaw []string `json:"leaked_uuids_raw,omitempty"` @@ -944,6 +949,7 @@ type autoState struct { valReqs atomic.Int64 // requests that carried >=1 expected UUID (i.e. validation ran) valUUIDChecks atomic.Int64 // total per-UUID presence checks made valUUIDFound atomic.Int64 // per-UUID presence checks that found the UUID + valExactMatchReqs atomic.Int64 // requests whose first line was the exact ordered UUID list (output conformity) valPresenceMissUUIDs atomic.Int64 // per-UUID PRESENCE_MISS count (expected UUID absent) valCrossContamUUIDs atomic.Int64 // per-UUID CROSS_CONTAMINATION count (other-conversation UUID present) valPresenceMissReqs atomic.Int64 // requests with >=1 PRESENCE_MISS @@ -1048,6 +1054,7 @@ type autoBenchmarkResult struct { valReqs int64 valUUIDChecks int64 valUUIDFound int64 + valExactMatchReqs int64 valPresenceMissUUIDs int64 valCrossContamUUIDs int64 valPresenceMissReqs int64 @@ -1207,7 +1214,12 @@ func printAutoSummary(res autoBenchmarkResult, cfg AutoBenchmarkConfig) { fmt.Println(strings.Repeat("-", 62)) fmt.Println(" UUID validation (replay)") fmt.Printf(" Requests validated : %d\n", res.valReqs) - fmt.Printf(" UUID presence : %d/%d\n", res.valUUIDFound, res.valUUIDChecks) + // Two tests, mirroring the cache-coherency eval CLI's layout: UUID + // correctness (per-stamp presence, Contains anywhere in the response) + // and output conformity (first line is exactly the ordered, + // comma-joined UUID list — see firstLineConformity). + fmt.Printf(" UUID correctness (presence) : %d/%d\n", res.valUUIDFound, res.valUUIDChecks) + fmt.Printf(" Output conformity (first-line exact) : %d/%d\n", res.valExactMatchReqs, res.valReqs) fmt.Printf(" PRESENCE_MISS (expected UUID absent) : %d across %d requests\n", res.valPresenceMissUUIDs, res.valPresenceMissReqs) fmt.Printf(" CROSS_CONTAMINATION (other-conv) : %d across %d requests\n", res.valCrossContamUUIDs, res.valCrossContamReqs) } @@ -2355,6 +2367,7 @@ func runSingleModelBenchmark( res.valReqs = st.valReqs.Load() res.valUUIDChecks = st.valUUIDChecks.Load() res.valUUIDFound = st.valUUIDFound.Load() + res.valExactMatchReqs = st.valExactMatchReqs.Load() res.valPresenceMissUUIDs = st.valPresenceMissUUIDs.Load() res.valCrossContamUUIDs = st.valCrossContamUUIDs.Load() res.valPresenceMissReqs = st.valPresenceMissReqs.Load() @@ -2586,7 +2599,16 @@ func RunAutoBenchmark(ctx context.Context, cfg AutoBenchmarkConfig) error { return fmt.Errorf("compute block session counts for --replay-inject-uuids: %w", cerr) } cfg.replayBlockSessionCounts = counts - cfg.replayUUIDSets = buildSessionUUIDs(effectiveSessions, cfg.ReplayUUIDSeed) + // Per-session cached-region byte size (the blocks AFTER each + // session's cross-session-shared boundary — see + // computePerSessionCachedChars) sizes N stamps per session + // exactly as the cache-coherency eval turns --garbage-chars + // into --stamps-per-series (computeStampsPerSeries, min 2). + perSessionChars, perr := computePerSessionCachedChars(cfg.RouterReplayFile, cfg.RouterReplaySeriesIndices, cfg.ReplaySeries, counts) + if perr != nil { + return fmt.Errorf("compute per-session cached chars for --replay-inject-uuids: %w", perr) + } + cfg.replayUUIDSets = buildSessionUUIDs(perSessionChars, cfg.ReplayUUIDSeed) sharedHashes := 0 for _, n := range counts { @@ -2598,8 +2620,20 @@ func RunAutoBenchmark(ctx context.Context, cfg AutoBenchmarkConfig) error { if berr != nil { return fmt.Errorf("compute usable-boundary diagnostic for --replay-inject-uuids: %w", berr) } - fmt.Printf("UUID validation enabled: %d session(s) prepared, %d cross-session-shared block hash(es), %d/%d sessions have a usable boundary (%d fall back to tail injection) (recite-every-request=%v, seed=%d)\n", - effectiveSessions, sharedHashes, usable, total, total-usable, cfg.ReplayReciteEveryRequest, cfg.ReplayUUIDSeed) + minStamps, maxStamps := 0, 0 + if len(cfg.replayUUIDSets) > 0 { + minStamps, maxStamps = len(cfg.replayUUIDSets[0]), len(cfg.replayUUIDSets[0]) + for _, set := range cfg.replayUUIDSets { + if len(set) < minStamps { + minStamps = len(set) + } + if len(set) > maxStamps { + maxStamps = len(set) + } + } + } + fmt.Printf("UUID validation enabled: %d session(s) prepared, %d cross-session-shared block hash(es), %d/%d sessions have a usable boundary (%d fall back to tail injection), %d-%d UUID stamps/session (recite-every-request=%v, seed=%d)\n", + effectiveSessions, sharedHashes, usable, total, total-usable, minStamps, maxStamps, cfg.ReplayReciteEveryRequest, cfg.ReplayUUIDSeed) } } } diff --git a/benchmark/replay.go b/benchmark/replay.go index 3eab5bd..dd91d96 100644 --- a/benchmark/replay.go +++ b/benchmark/replay.go @@ -371,6 +371,9 @@ func recordReplayRequest( st.valReqs.Add(1) st.valUUIDChecks.Add(int64(uuidExpectedCount)) st.valUUIDFound.Add(int64(uuidFoundCount)) + if metrics.ExactMatch { + st.valExactMatchReqs.Add(1) + } if missCount := uuidExpectedCount - uuidFoundCount; missCount > 0 { st.valPresenceMissUUIDs.Add(int64(missCount)) st.valPresenceMissReqs.Add(1) @@ -446,6 +449,7 @@ func recordReplayRequest( UUIDExpected: uuidExpectedCount, UUIDFound: uuidFoundCount, UUIDLeaked: uuidLeakedCount, + UUIDExactMatch: metrics.ExactMatch, } // Raw detail lists only on a miss or a leak — mirrors the // failed-request-only policy on PromptText/ResponseText/RawResponseTail diff --git a/benchmark/replay_router.go b/benchmark/replay_router.go index 7d11c18..b284904 100644 --- a/benchmark/replay_router.go +++ b/benchmark/replay_router.go @@ -763,3 +763,28 @@ func BuildReplayRequestPrefix(req RouterReplayRequest) (hashes []string, tokens } return } + +// requestPrefixBytes mirrors BuildReplayRequestPrefix's exact block sequence +// (same skip-the-tiny-header-block rule, same system-blocks/tools/messages +// order) but returns each entry's Bytes instead of its hash/token count, so +// index i here lines up 1:1 with index i of BuildReplayRequestPrefix's +// hashes. Used by computePerSessionCachedChars to sum the byte size of a +// request's prefix blocks AT OR AFTER a given boundary index (e.g. +// sharedPrefixBlockCount) — the per-session cached region --replay-inject- +// uuids sizes its UUID-stamp count off (see replay_router_uuid.go). +func requestPrefixBytes(req RouterReplayRequest) []int { + var out []int + for i, sb := range req.SystemBlocks { + if i == 0 && sb.Bytes < 200 { + continue + } + out = append(out, sb.Bytes) + } + if req.Tools != nil && req.Tools.Hash != "" { + out = append(out, req.Tools.Bytes) + } + for _, m := range req.Messages { + out = append(out, m.Bytes) + } + return out +} diff --git a/benchmark/replay_router_post.go b/benchmark/replay_router_post.go index 5e1891e..f8e9848 100644 --- a/benchmark/replay_router_post.go +++ b/benchmark/replay_router_post.go @@ -104,7 +104,7 @@ func (p *replayPoster) buildInjection(req RouterReplayRequest, isLastRequest boo return nil } return &uuidInjection{ - Marker: injectUUIDMarker("", uuids), + UUIDs: uuids, Recite: p.reciteEveryRequest || isLastRequest, SharedPrefixLen: sharedPrefixBlockCount(req, p.blockCounts), } @@ -430,10 +430,17 @@ func (p *replayPoster) do( // merge reasoning/thinking into m.Response (see their doc comments), so // a single Contains-scan of m.Response covers both — thinking is passed // as "" here, mirroring the dataset path's own call shape. + // + // Two independent checks, mirroring the cache-coherency eval's two + // reported tests: per-UUID PRESENCE (Contains anywhere in the response, + // via validateReplayResponse) and output CONFORMITY (the FIRST LINE of + // the response is exactly the ordered, comma-joined UUID list — see + // firstLineConformity/matchesExpectedUUIDList). if inj != nil && m.Error == nil && !m.IsEmpty { m.ConvIdx = p.sessionIdx m.ExpectedUUIDs = append([]string(nil), p.allUUIDSets[p.sessionIdx]...) m.UUIDFound, m.LeakedUUIDs = validateReplayResponse(m.Response, "", m.ExpectedUUIDs, p.sessionIdx, p.allUUIDSets) + m.ExactMatch = firstLineConformity(m.Response, m.ExpectedUUIDs) } return m } diff --git a/benchmark/replay_router_uuid.go b/benchmark/replay_router_uuid.go index 7bd719c..3d52b8b 100644 --- a/benchmark/replay_router_uuid.go +++ b/benchmark/replay_router_uuid.go @@ -1,10 +1,17 @@ package benchmark // UUID-based cache-coherency validation for the ROUTER-REPLAY path -// (--router-replay-file, --replay-inject-uuids). This is the router-path -// counterpart of the dataset-replay UUID validation (replay_uuid.go's -// path-agnostic primitives — injectUUIDMarker / validateReplayResponse / -// FindLeakedUUIDs — are shared with, not duplicated from, that file). +// (--router-replay-file, --replay-inject-uuids). This mirrors the mechanics +// of the cache-coherency eval's --shared-prefix-per-series mode +// (cache_coherency.go's buildCoherencySharedSeriesPrompt/userMessage/ +// matchesExpectedUUIDList) as closely as the router-replay wire shape +// allows: N bare, space-separated UUIDs stamped once per session, a recite- +// the-list instruction, and exact first-line conformity scoring — rather +// than a single wrapped "[ref-id: ...]" marker recited anywhere in the +// response. (Path-agnostic primitives — injectUUIDMarker/ +// validateReplayResponse/FindLeakedUUIDs — are still shared with, not +// duplicated from, replay_uuid.go; injectUUIDMarker itself remains the +// dataset-replay path's own wrapper and is untouched here.) // // Strategy (Option C — boundary injection, with tail fallback): every // session in a replay-v3 capture opens with one or more blocks (system @@ -12,32 +19,37 @@ package benchmark // MANY OTHER sessions too — the router's own leading system prompt(s), // repeated verbatim capture after capture. Everything AFTER that shared // run is genuinely per-session (the user's actual turn). We inject exactly -// ONE deterministic UUID per session at that boundary: +// N deterministic, bare, space-separated UUIDs per session at that +// boundary — mirroring buildCoherencySharedSeriesPrompt's tail +// ("UUID0 UUID1 … UUIDlast") rather than the dataset path's wrapped +// "[ref-id: ]" marker: // -// [ RUN_GUID ][ shared system blocks ][ MARKER ][ forceOutput instr ] [ messages... ] [ recite ask ] -// \_________________ byte-identical across sessions _________________/ \_ per-session, grows each turn _/ +// [ RUN_GUID ][ shared system blocks ][ UUID0 UUID1 … UUIDlast ][ forceOutput instr ] [ messages... ] [ recite-first-line ask ] +// \_______________________ byte-identical across sessions ________________________/ \_ per-session, grows each turn _/ // -// Putting the marker there means: +// Putting the UUID block there means: // - the cross-session shared prefix stays byte-identical (cache-hit // reproduction against the original capture is preserved: two sessions // that shared a system prompt still collide on the server's prefix // cache exactly as they did in the original traffic) -// - the marker itself lands in a region that IS cached WITHIN a session -// (every subsequent request in the same session repeats it), so asking -// the model to recall it later is a genuine KV-coherency signal, not an -// artifact of it being freshly re-sent every turn. +// - the UUID block itself lands in a region that IS cached WITHIN a +// session (every subsequent request in the same session repeats it, +// byte-identical), so asking the model to recall it later is a genuine +// KV-coherency signal, not an artifact of it being freshly re-sent +// every turn. // // A session with no shared leading block at all (empirically none, across // 5441 real sessions, lack one — but the file format doesn't guarantee it) -// falls back to tail injection: the marker is folded into the end of the -// request instead, forfeiting the "cached within a session" property but -// still producing a valid, scorable request. +// falls back to tail injection: the UUID block is folded into the end of +// the request instead, forfeiting the "cached within a session" property +// but still producing a valid, scorable request. import ( "bufio" "encoding/json" "fmt" "os" + "strings" "sync" ) @@ -46,21 +58,50 @@ import ( // injection" — buildAnthropicMessagesBody / buildOpenAIChatCompletionsBody // must behave identically to before this feature existed. type uuidInjection struct { - // Marker is the exact text to splice in (see injectUUIDMarker) — e.g. - // "\n\n[ref-id: ]". Empty means no marker this call (still allows - // Recite alone, though callers currently always set both together). - Marker string - // Recite asks the model to find and echo the marker verbatim from - // earlier in its own context (see replayReciteFromContextInstruction). + // UUIDs is the session's full ordered N-UUID list (see buildSessionUUIDs), + // spliced bare and space-separated (see bareUUIDBlock) — mirrors the + // cache-coherency eval's buildCoherencySharedSeriesPrompt tail. Nil/empty + // means no UUID block this call (still allows Recite alone, though + // callers currently always set both together). + UUIDs []string + // Recite asks the model to output, as the FIRST line of its response, + // the exact ordered UUID list (see replayReciteFirstLineInstruction), + // then continue normally. Recite bool // SharedPrefixLen is this request's leading run of cross-session-shared // prefix blocks (see sharedPrefixBlockCount). It tells the wire builder - // whether the marker can be spliced in at the natural system/message - // boundary (SharedPrefixLen covers every emitted system block) or must - // fall back to tail injection (SharedPrefixLen == 0). + // whether the UUID block can be spliced in at the natural system/ + // message boundary (SharedPrefixLen covers every emitted system block) + // or must fall back to tail injection (SharedPrefixLen == 0). SharedPrefixLen int } +// bareUUIDBlock returns uuids joined bare and space-separated — mirrors the +// cache-coherency eval's buildCoherencySharedSeriesPrompt tail +// ("UUID0 UUID1 … UUIDlast", no wrapper text) — for splicing as one system +// message/block at the session boundary (or the tail, on fallback). Bare +// UUIDs (vs. the dataset path's "[ref-id: ]" wrapper) both match the +// coherency test's mechanics and avoid the model treating a marker-shaped +// wrapper as instruction text to echo verbatim. +func bareUUIDBlock(uuids []string) string { + return strings.Join(uuids, " ") +} + +// firstLineConformity implements this feature's output-conformity check: +// whether the FIRST LINE of resp (up to the first '\n', trimmed) is EXACTLY +// the comma-separated expected UUID list, in order (see +// matchesExpectedUUIDList) — the router-replay analogue of the coherency +// eval's ExactMatch, adapted for a recite-FIRST instruction that lets the +// model keep generating after line 1 (so forced-output/ignore_eos still +// fills the remainder of the output budget with a normal continuation). +func firstLineConformity(resp string, expected []string) bool { + line := resp + if idx := strings.Index(resp, "\n"); idx >= 0 { + line = resp[:idx] + } + return matchesExpectedUUIDList(strings.TrimSpace(line), expected) +} + // computeBlockSessionCounts streams a replay-v3 file once and returns, for // every distinct block hash seen, the number of DISTINCT SESSIONS that // reference it at least once (not the number of requests — a hash reused @@ -204,6 +245,80 @@ func countSessionsWithUsableBoundary(path string, allowed map[int]bool, sessionL return usable, total, nil } +// computePerSessionCachedChars makes a second streaming pass (same filtering +// as computeBlockSessionCounts) and returns, in dispatch order (index i = +// the i-th session encountered — the SAME order buildSessionUUIDs' returned +// slice is indexed by, and the same order cfg.replayUUIDSets ends up in), +// each session's per-session cached-region byte size: the ROOT request's +// (first instance, first request) prefix bytes (see requestPrefixBytes) AT +// OR AFTER that request's cross-session-shared boundary +// (sharedPrefixBlockCount, using counts from computeBlockSessionCounts). +// +// This is a proxy for "how much content is genuinely per-session and reused, +// byte-identical, across the session's own later requests" — the region the +// injected UUID block actually sits within once spliced at the boundary +// (see the package doc comment above). computeStampsPerSeries then turns +// this byte count into a stamp count exactly as the cache-coherency eval +// turns --garbage-chars into --stamps-per-series. A session with no +// requests at all (degenerate/empty) contributes 0, which +// computeStampsPerSeries floors to its minimum of 2. +func computePerSessionCachedChars(path string, allowed map[int]bool, sessionLimit int, counts map[string]int) ([]int, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + br := bufio.NewReaderSize(f, 1<<20) + if _, err := br.ReadBytes('\n'); err != nil { + return nil, fmt.Errorf("read header line: %w", err) + } + + var perSession []int + lineIdx := 0 + produced := 0 + for { + if sessionLimit > 0 && produced >= sessionLimit { + break + } + if allowed != nil && produced >= len(allowed) { + break + } + line, rerr := br.ReadBytes('\n') + if len(line) > 0 { + line = trimNL(line) + if len(line) > 0 { + currentIdx := lineIdx + lineIdx++ + if allowed != nil && !allowed[currentIdx] { + if rerr != nil { + break + } + continue + } + var sess RouterReplaySession + if jerr := json.Unmarshal(line, &sess); jerr == nil { + chars := 0 + if len(sess.Instances) > 0 && len(sess.Instances[0].Requests) > 0 { + root := sess.Instances[0].Requests[0] + boundary := sharedPrefixBlockCount(root, counts) + prefixBytes := requestPrefixBytes(root) + for i := boundary; i < len(prefixBytes); i++ { + chars += prefixBytes[i] + } + } + perSession = append(perSession, chars) + produced++ + } + } + } + if rerr != nil { + break + } + } + return perSession, nil +} + // sharedPrefixBlockCount returns the length of req's LEADING run of blocks // (per BuildReplayRequestPrefix's cache-order hash sequence: system blocks, // then tools, then messages) whose hash is shared across more than one @@ -227,68 +342,103 @@ func sharedPrefixBlockCount(req RouterReplayRequest, counts map[string]int) int return n } -// replayReciteFromContextInstruction returns the tail instruction asking -// the model to find and echo, verbatim, the ref-id marker planted earlier -// in its own context — WITHOUT restating it from the instruction itself. -// Unlike the dataset-path replayReciteInstruction (replay_uuid.go), this -// text never embeds the UUID: presence in the response therefore reflects -// genuine recall from cached KV, not an echo of the ask. -func replayReciteFromContextInstruction() string { - return "\n\n(Somewhere earlier in this conversation there is a line of the exact form `[ref-id: ]`. " + - "Find it and, before your normal answer, output one line in the exact form `SEEN_REF: ` reproducing " + - "that id verbatim from your context — do not invent one, and do not simply repeat this instruction. Then answer normally.)" +// replayReciteFirstLineInstruction returns the tail instruction asking the +// model to output, as the FIRST line of its response, every UUID injected +// at the session boundary — in order, comma-separated — then continue +// normally. Mirrors the cache-coherency eval's userMessage ("List every +// UUID shown in the request, in order, separated by commas. Output only the +// UUIDs and commas, nothing else.") but front-loads the ask to line 1 +// specifically (rather than the entire response), so a forced-output / +// ignore_eos budget can still fill the remainder with the model's normal +// continuation. Like the dataset-path instruction it replaces, this text +// never embeds the UUIDs themselves: first-line conformity in the response +// therefore reflects genuine recall from cached KV context, not an echo of +// the ask. +func replayReciteFirstLineInstruction() string { + return "\n\nBefore anything else, output on the FIRST line every UUID shown above in the request, in order, " + + "separated by commas, and nothing else on that first line. Then continue with your normal response." } -// buildSessionUUIDs returns n singleton UUID sets (one UUID per session), -// drawn in order from a single seeded generator — same determinism/ -// disjointness rationale as the dataset path's buildReplayUUIDSets: same -// seed -> same per-session UUID assignment across runs and across every -// model in a multi-model run (see the precompute call site in -// RunAutoBenchmark, which populates cfg.replayUUIDSets once, before any -// per-model goroutine spawns, so every model sees the identical +// buildSessionUUIDs returns len(perSessionChars) UUID sets, one per session, +// drawn in order (session-major, stamp-minor) from a single seeded +// generator — same determinism/disjointness rationale as the dataset path's +// buildReplayUUIDSets: same seed -> same per-session UUID assignment across +// runs and across every model in a multi-model run (see the precompute call +// site in RunAutoBenchmark, which populates cfg.replayUUIDSets once, before +// any per-model goroutine spawns, so every model sees the identical // assignment). -func buildSessionUUIDs(n int, seed int64) [][]string { - if n <= 0 { +// +// Session i's set size is N = computeStampsPerSeries(perSessionChars[i]) — +// the SAME sizing rule the cache-coherency eval uses to turn a garbage-char +// budget into a stamp count (min 2) — applied here to perSessionChars[i], +// session i's per-session cached-region byte size (see +// computePerSessionCachedChars): the blocks AFTER the cross-session-shared +// boundary that get reused, byte-identical, across every one of the +// session's own requests. A larger reused region gets more UUID stamps +// spread across it, mirroring the coherency test's +// garbageChars -> numStamps relationship. +func buildSessionUUIDs(perSessionChars []int, seed int64) [][]string { + if len(perSessionChars) == 0 { return nil } newUUID := newUUIDGenerator(seed) - sets := make([][]string, n) - for i := range sets { - sets[i] = []string{newUUID()} + sets := make([][]string, len(perSessionChars)) + for i, chars := range perSessionChars { + n := computeStampsPerSeries(chars) + uuids := make([]string, n) + for j := range uuids { + uuids[j] = newUUID() + } + sets[i] = uuids } return sets } // ---- max_tokens recite floor ---- -// replayReciteFloorTokens is the minimum max_tokens budget enforced on a -// request that carries the recite ask (see uuidInjection.Recite). A -// router-replay request's max_tokens is normally sized off the ORIGINAL -// capture's output_tokens (see pickMaxTokens) — which for a tool-call-only -// turn can be a handful of tokens, nowhere near enough to also emit the -// "SEEN_REF: " line the recite ask asks for. Without a floor, a tiny -// budget truncates the recite line itself, which would misread as a -// PRESENCE_MISS (coherency failure) when it's actually just an output-size -// artifact. -const replayReciteFloorTokens = 64 +// replayReciteFloorMultiplier mirrors the cache-coherency eval's default +// --max-output-multiplier (see computeMaxOutputTokens): the recite floor is +// sized at multiplier x the expected N-UUID first-line list size, giving +// the model headroom to emit the full list without truncation. +const replayReciteFloorMultiplier = 3.0 + +// replayReciteFloorTokens returns the minimum max_tokens budget to enforce +// on a request that carries the recite ask (see uuidInjection.Recite), sized +// to fit the FIRST-LINE numUUIDs-UUID comma-joined list this feature asks +// for (reuses the cache-coherency eval's computeMaxOutputTokens sizing: +// numUUIDs*36 chars + separating commas, /4 for an approximate token count, +// x replayReciteFloorMultiplier). A router-replay request's max_tokens is +// normally sized off the ORIGINAL capture's output_tokens (see +// pickMaxTokens) — which for a tool-call-only turn can be a handful of +// tokens, nowhere near enough to also emit the first-line UUID list. Without +// this floor, a tiny budget truncates that first line, which would misread +// as PRESENCE_MISS/NOT_EXACT (coherency failure) when it's actually just an +// output-size artifact. +func replayReciteFloorTokens(numUUIDs int) int { + return computeMaxOutputTokens(numUUIDs, replayReciteFloorMultiplier) +} var reciteFloorWarnOnce sync.Once -// applyReciteFloor raises maxTokens to replayReciteFloorTokens when recite -// is requested and the original budget falls short, warning once per -// process (mirrors the dataset path's reciteTruncWarned one-shot pattern, -// but this is a single global warning rather than per-conversation since -// the router path's floor is a fixed constant, not a per-conversation -// truncation computation). -func applyReciteFloor(maxTokens int, recite bool) int { - if !recite || maxTokens >= replayReciteFloorTokens { +// applyReciteFloor raises maxTokens to replayReciteFloorTokens(numUUIDs) +// when recite is requested and the original budget falls short, warning +// once per process (mirrors the dataset path's reciteTruncWarned one-shot +// pattern, but this is a single global warning rather than per-conversation +// since the router path computes one floor value per call, not a per- +// conversation truncation computation). +func applyReciteFloor(maxTokens int, recite bool, numUUIDs int) int { + if !recite { + return maxTokens + } + floor := replayReciteFloorTokens(numUUIDs) + if maxTokens >= floor { return maxTokens } reciteFloorWarnOnce.Do(func() { fmt.Fprintf(os.Stderr, "[router-replay] WARNING: max_tokens raised to the UUID-recite floor (%d) for one or more requests — "+ - "a tiny captured output budget would otherwise truncate the recite line into a false PRESENCE_MISS, not real corruption\n", - replayReciteFloorTokens) + "a tiny captured output budget would otherwise truncate the first-line recite list into a false PRESENCE_MISS, not real corruption\n", + floor) }) - return replayReciteFloorTokens + return floor } diff --git a/benchmark/replay_router_uuid_test.go b/benchmark/replay_router_uuid_test.go index a4ba4c8..fb29047 100644 --- a/benchmark/replay_router_uuid_test.go +++ b/benchmark/replay_router_uuid_test.go @@ -179,25 +179,59 @@ func TestSharedPrefixBlockCount(t *testing.T) { } } +// TestComputePerSessionCachedChars verifies the per-session cached-region +// byte size computation: the ROOT request's (first instance, first +// request) prefix bytes AT OR AFTER its sharedPrefixBlockCount boundary, +// using the same synthetic 5-session fixture TestSharedPrefixBlockCount +// exercises (sys=250 bytes, msg=100 bytes each request). +func TestComputePerSessionCachedChars(t *testing.T) { + path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) + counts, err := computeBlockSessionCounts(path, nil, 0) + if err != nil { + t.Fatalf("computeBlockSessionCounts: %v", err) + } + + got, err := computePerSessionCachedChars(path, nil, 0, counts) + if err != nil { + t.Fatalf("computePerSessionCachedChars: %v", err) + } + // s0,s1: boundary=1 (only the shared sys1 block) -> chars = msg bytes (100). + // s2: boundary=0 (nothing shared) -> chars = sys(250) + msg(100) = 350. + // s3,s4: boundary=2 (both blocks shared, full prefix) -> chars = 0. + want := []int{100, 100, 350, 0, 0} + if len(got) != len(want) { + t.Fatalf("computePerSessionCachedChars len = %d, want %d (got %v)", len(got), len(want), got) + } + for i, w := range want { + if got[i] != w { + t.Errorf("session %d: chars = %d, want %d", i, got[i], w) + } + } +} + // TestBuildSessionUUIDsDeterminism verifies buildSessionUUIDs matches the // dataset path's determinism contract: same seed -> same per-session UUID -// assignment; different seed -> different assignment; every UUID unique. +// assignment; different seed -> different assignment; every UUID unique +// across the whole run (not just within a session). func TestBuildSessionUUIDsDeterminism(t *testing.T) { - a := buildSessionUUIDs(5, 42) - b := buildSessionUUIDs(5, 42) + perSessionChars := []int{0, 0, 0, 0, 0} // all -> computeStampsPerSeries floors to 2 + a := buildSessionUUIDs(perSessionChars, 42) + b := buildSessionUUIDs(perSessionChars, 42) if len(a) != 5 || len(b) != 5 { t.Fatalf("expected 5 sets each, got %d and %d", len(a), len(b)) } for i := range a { - if len(a[i]) != 1 || len(b[i]) != 1 { - t.Fatalf("session %d: expected singleton sets, got %v / %v", i, a[i], b[i]) + if len(a[i]) != 2 || len(b[i]) != 2 { + t.Fatalf("session %d: expected 2-UUID sets (min floor), got %v / %v", i, a[i], b[i]) } - if a[i][0] != b[i][0] { - t.Errorf("session %d: same seed produced different UUIDs: %q vs %q", i, a[i][0], b[i][0]) + for j := range a[i] { + if a[i][j] != b[i][j] { + t.Errorf("session %d stamp %d: same seed produced different UUIDs: %q vs %q", i, j, a[i][j], b[i][j]) + } } } - c := buildSessionUUIDs(5, 43) + c := buildSessionUUIDs(perSessionChars, 43) same := true for i := range a { if a[i][0] != c[i][0] { @@ -210,22 +244,45 @@ func TestBuildSessionUUIDsDeterminism(t *testing.T) { seen := map[string]bool{} for _, set := range a { - if seen[set[0]] { - t.Errorf("uuid %q assigned to more than one session", set[0]) + for _, u := range set { + if seen[u] { + t.Errorf("uuid %q assigned to more than one stamp", u) + } + seen[u] = true } - seen[set[0]] = true } - if got := buildSessionUUIDs(0, 42); got != nil { - t.Errorf("buildSessionUUIDs(0, ...) = %v, want nil", got) + if got := buildSessionUUIDs(nil, 42); got != nil { + t.Errorf("buildSessionUUIDs(nil, ...) = %v, want nil", got) } } -// TestWireInjectionDeterminism verifies that, for a fixed session's marker, -// buildOpenAIChatCompletionsBody / buildAnthropicMessagesBody produce +// TestBuildSessionUUIDsScalesWithBytes verifies each session's N is exactly +// computeStampsPerSeries(perSessionChars[i]) -- min 2, scaling with bytes -- +// mirroring the cache-coherency eval's garbageChars -> numStamps rule. +func TestBuildSessionUUIDsScalesWithBytes(t *testing.T) { + perSessionChars := []int{0, 8192 * 5, 8192*10 + 100} + sets := buildSessionUUIDs(perSessionChars, 7) + want := []int{2, 5, 10} + for i, w := range want { + if got := len(sets[i]); got != w { + t.Errorf("session %d: len = %d, want %d (computeStampsPerSeries(%d))", i, got, w, perSessionChars[i]) + } + } + for _, chars := range []int{0, 1, 8192, 8192 * 3, 100000} { + got := len(buildSessionUUIDs([]int{chars}, 1)[0]) + want := computeStampsPerSeries(chars) + if got != want { + t.Errorf("chars=%d: N = %d, want %d (computeStampsPerSeries)", chars, got, want) + } + } +} + +// TestWireInjectionDeterminism verifies that, for a fixed session's N-UUID +// block, buildOpenAIChatCompletionsBody / buildAnthropicMessagesBody produce // byte-identical bodies across repeated calls (same request + same -// injection in -> same bytes out), and that two DIFFERENT sessions' markers -// diverge the body. +// injection in -> same bytes out), and that two DIFFERENT sessions' UUID +// blocks diverge the body. func TestWireInjectionDeterminism(t *testing.T) { docs := strings.Repeat("wire-injection-docs ", 100) req := RouterReplayRequest{ @@ -233,10 +290,10 @@ func TestWireInjectionDeterminism(t *testing.T) { SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, Messages: []RouterReplayMessage{{Hash: "msgUniq0", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, } - sets := buildSessionUUIDs(2, 7) - injA := &uuidInjection{Marker: injectUUIDMarker("", sets[0]), Recite: true, SharedPrefixLen: 1} - injA2 := &uuidInjection{Marker: injectUUIDMarker("", sets[0]), Recite: true, SharedPrefixLen: 1} - injB := &uuidInjection{Marker: injectUUIDMarker("", sets[1]), Recite: true, SharedPrefixLen: 1} + sets := buildSessionUUIDs([]int{0, 0}, 7) + injA := &uuidInjection{UUIDs: sets[0], Recite: true, SharedPrefixLen: 1} + injA2 := &uuidInjection{UUIDs: sets[0], Recite: true, SharedPrefixLen: 1} + injB := &uuidInjection{UUIDs: sets[1], Recite: true, SharedPrefixLen: 1} for _, kind := range []string{"openai", "anthropic"} { build := func(r RouterReplayRequest, inj *uuidInjection) []byte { @@ -260,22 +317,94 @@ func TestWireInjectionDeterminism(t *testing.T) { t.Errorf("%s: identical injection produced different bytes", kind) } if string(bodyA1) == string(bodyB) { - t.Errorf("%s: different sessions' markers produced identical bytes", kind) + t.Errorf("%s: different sessions' UUID blocks produced identical bytes", kind) + } + for _, u := range sets[0] { + if !strings.Contains(string(bodyA1), u) { + t.Errorf("%s: body missing session A's own UUID %q", kind, u) + } } - if !strings.Contains(string(bodyA1), sets[0][0]) { - t.Errorf("%s: body missing session A's own UUID", kind) + for _, u := range sets[1] { + if strings.Contains(string(bodyA1), u) { + t.Errorf("%s: body A leaked session B's UUID %q into the wire body", kind, u) + } } - if strings.Contains(string(bodyA1), sets[1][0]) { - t.Errorf("%s: body A leaked session B's UUID into the wire body", kind) + if strings.Contains(string(bodyA1), "ref-id") { + t.Errorf("%s: injected block still carries the old [ref-id: ...] wrapper", kind) } } } +// TestBoundaryInjectionEmitsBareSpaceSeparatedUUIDs verifies the injected +// block is exactly N bare, space-separated UUIDs (no wrapper text) — +// mirroring the cache-coherency eval's buildCoherencySharedSeriesPrompt tail +// — and that it is byte-identical across two DIFFERENT requests belonging +// to the SAME session (the within-session cache-reuse property), while the +// cross-session-shared leading system block stays untouched. +func TestBoundaryInjectionEmitsBareSpaceSeparatedUUIDs(t *testing.T) { + docs := strings.Repeat("boundary-multi-docs ", 100) + uuids := []string{"uuid-0", "uuid-1", "uuid-2"} + inj := &uuidInjection{UUIDs: uuids, Recite: false, SharedPrefixLen: 1} + + req1 := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, + Messages: []RouterReplayMessage{{Hash: "msgTurn1", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + req2 := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, // same shared system block + Messages: []RouterReplayMessage{{Hash: "msgTurn2", Role: "user", BlockTypes: []string{"text"}, Bytes: 140}}, + } + + body1, _, err := buildAnthropicMessagesBody(req1, docs, "model", "", 0, false, inj) + if err != nil { + t.Fatalf("build 1: %v", err) + } + body2, _, err := buildAnthropicMessagesBody(req2, docs, "model", "", 0, false, inj) + if err != nil { + t.Fatalf("build 2: %v", err) + } + + var parsed1, parsed2 map[string]interface{} + if err := json.Unmarshal(body1, &parsed1); err != nil { + t.Fatalf("unmarshal 1: %v", err) + } + if err := json.Unmarshal(body2, &parsed2); err != nil { + t.Fatalf("unmarshal 2: %v", err) + } + + sys1, _ := parsed1["system"].([]interface{}) + sys2, _ := parsed2["system"].([]interface{}) + if len(sys1) != 2 || len(sys2) != 2 { + t.Fatalf("expected system = [shared block, uuid block], got lens %d and %d", len(sys1), len(sys2)) + } + + wantText := "uuid-0 uuid-1 uuid-2" + block1 := sys1[1].(map[string]interface{}) + block2 := sys2[1].(map[string]interface{}) + if block1["text"] != wantText { + t.Errorf("uuid block 1 text = %q, want %q (bare, space-separated)", block1["text"], wantText) + } + if block2["text"] != wantText { + t.Errorf("uuid block diverged across two requests in the SAME session: %v vs %v", block2["text"], wantText) + } + + // The shared leading system block (index 0) must stay byte-identical — + // injection must never perturb the cross-session-shared prefix. + shared1, _ := json.Marshal(sys1[0]) + shared2, _ := json.Marshal(sys2[0]) + if string(shared1) != string(shared2) { + t.Errorf("shared leading system block diverged across requests: %s vs %s", shared1, shared2) + } +} + // TestCacheFidelityBoundaryInvariant verifies the core Option-C guarantee: // two DIFFERENT sessions that share a leading system block emit // byte-identical content for that shared block, diverging only at (or -// after) the injected per-session marker — i.e. injection never perturbs -// the cross-session-shared prefix a real server would prefix-cache on. +// after) the injected per-session UUID block — i.e. injection never +// perturbs the cross-session-shared prefix a real server would +// prefix-cache on. func TestCacheFidelityBoundaryInvariant(t *testing.T) { docs := strings.Repeat("fidelity-docs ", 100) reqA := RouterReplayRequest{ @@ -288,8 +417,8 @@ func TestCacheFidelityBoundaryInvariant(t *testing.T) { SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, // SAME shared system block Messages: []RouterReplayMessage{{Hash: "msgUniq-B", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, } - injA := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-session-A"}), Recite: false, SharedPrefixLen: 1} - injB := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-session-B"}), Recite: false, SharedPrefixLen: 1} + injA := &uuidInjection{UUIDs: []string{"uuid-session-A"}, Recite: false, SharedPrefixLen: 1} + injB := &uuidInjection{UUIDs: []string{"uuid-session-B"}, Recite: false, SharedPrefixLen: 1} bodyA, _, err := buildAnthropicMessagesBody(reqA, docs, "model", "", 0, false, injA) if err != nil { @@ -311,7 +440,7 @@ func TestCacheFidelityBoundaryInvariant(t *testing.T) { sysA, _ := parsedA["system"].([]interface{}) sysB, _ := parsedB["system"].([]interface{}) if len(sysA) != 2 || len(sysB) != 2 { - t.Fatalf("expected system = [shared block, marker], got lens %d and %d", len(sysA), len(sysB)) + t.Fatalf("expected system = [shared block, uuid block], got lens %d and %d", len(sysA), len(sysB)) } // Index 0 (the shared system block, "sys1") must be byte-identical. sharedA, _ := json.Marshal(sysA[0]) @@ -319,22 +448,22 @@ func TestCacheFidelityBoundaryInvariant(t *testing.T) { if string(sharedA) != string(sharedB) { t.Errorf("shared leading system block diverged between sessions:\nA: %s\nB: %s", sharedA, sharedB) } - // Index 1 (the injected marker) MUST diverge — that's the whole point. - markerA, _ := json.Marshal(sysA[1]) - markerB, _ := json.Marshal(sysB[1]) - if string(markerA) == string(markerB) { - t.Error("injected markers were identical across two different sessions") + // Index 1 (the injected uuid block) MUST diverge — that's the whole point. + blockA, _ := json.Marshal(sysA[1]) + blockB, _ := json.Marshal(sysB[1]) + if string(blockA) == string(blockB) { + t.Error("injected uuid blocks were identical across two different sessions") } - if !strings.Contains(string(markerA), "uuid-session-A") { - t.Errorf("session A's marker missing its own uuid: %s", markerA) + if !strings.Contains(string(blockA), "uuid-session-A") { + t.Errorf("session A's block missing its own uuid: %s", blockA) } - if !strings.Contains(string(markerB), "uuid-session-B") { - t.Errorf("session B's marker missing its own uuid: %s", markerB) + if !strings.Contains(string(blockB), "uuid-session-B") { + t.Errorf("session B's block missing its own uuid: %s", blockB) } } // TestTailFallbackInjection verifies that when SharedPrefixLen == 0 (no -// usable boundary), the marker is folded into the tail (messages array) +// usable boundary), the UUID block is folded into the tail (messages array) // rather than the system array, and the request remains well-formed. func TestTailFallbackInjection(t *testing.T) { docs := strings.Repeat("tail-fallback-docs ", 100) @@ -343,7 +472,7 @@ func TestTailFallbackInjection(t *testing.T) { SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys-unique", Bytes: 250}}, Messages: []RouterReplayMessage{{Hash: "msg-unique", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, } - inj := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-tail"}), Recite: true, SharedPrefixLen: 0} + inj := &uuidInjection{UUIDs: []string{"uuid-tail"}, Recite: true, SharedPrefixLen: 0} body, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) if err != nil { @@ -358,11 +487,11 @@ func TestTailFallbackInjection(t *testing.T) { t.Fatalf("expected system to carry ONLY the original block (no boundary splice), got %d entries", len(sys)) } if strings.Contains(string(body), "uuid-tail") == false { - t.Fatal("marker missing from body entirely") + t.Fatal("uuid block missing from body entirely") } msgs, _ := parsed["messages"].([]interface{}) if len(msgs) == 0 { - t.Fatal("expected messages to carry the tail-injected marker/recite content") + t.Fatal("expected messages to carry the tail-injected uuid block/recite content") } last := msgs[len(msgs)-1].(map[string]interface{}) if last["role"] != "user" { @@ -370,19 +499,26 @@ func TestTailFallbackInjection(t *testing.T) { } } -// TestUUIDValidationEndToEnd exercises buildSessionUUIDs + injectUUIDMarker + -// validateReplayResponse together, mirroring how replayPoster.do() wires -// them: a response containing the OWN session's uuid scores found/no-leak; -// a response containing ANOTHER session's uuid scores CROSS_CONTAMINATION -// against the correct series index. +// TestUUIDValidationEndToEnd exercises validateReplayResponse (presence + +// cross-session leak) directly against N-stamp sessions (N=2), mirroring +// how replayPoster.do() wires it: a response containing ALL of the OWN +// session's uuids scores found/no-leak; a response containing ANOTHER +// session's uuid scores CROSS_CONTAMINATION against the correct series +// index; a response missing one of its own uuids scores a partial +// PRESENCE_MISS. Semantics are unchanged from the single-uuid path — only +// the stamp count (N) is now typically > 1. func TestUUIDValidationEndToEnd(t *testing.T) { - sets := buildSessionUUIDs(3, 123) + sets := [][]string{ + {"uuid-s0-a", "uuid-s0-b"}, + {"uuid-s1-a", "uuid-s1-b"}, + {"uuid-s2-a", "uuid-s2-b"}, + } - t.Run("own uuid present, no leak", func(t *testing.T) { - resp := "Sure, the ref-id I recall is " + sets[0][0] + ". Anyway, here's your answer." + t.Run("own uuids present, no leak", func(t *testing.T) { + resp := "Sure, the ids I recall are " + sets[0][0] + " and " + sets[0][1] + ". Anyway, here's your answer." found, leaked := validateReplayResponse(resp, "", sets[0], 0, sets) - if len(found) != 1 || !found[0] { - t.Errorf("found = %v, want [true]", found) + if len(found) != 2 || !found[0] || !found[1] { + t.Errorf("found = %v, want [true true]", found) } if len(leaked) != 0 { t.Errorf("leaked = %v, want none", leaked) @@ -390,20 +526,31 @@ func TestUUIDValidationEndToEnd(t *testing.T) { }) t.Run("cross contamination from another session", func(t *testing.T) { - resp := "SEEN_REF: " + sets[0][0] + " and also " + sets[2][0] + resp := sets[0][0] + ", " + sets[0][1] + ", and also " + sets[2][0] found, leaked := validateReplayResponse(resp, "", sets[0], 0, sets) - if len(found) != 1 || !found[0] { - t.Errorf("found = %v, want [true]", found) + if len(found) != 2 || !found[0] || !found[1] { + t.Errorf("found = %v, want [true true]", found) } if len(leaked) != 1 || !strings.Contains(leaked[0], sets[2][0]) || !strings.Contains(leaked[0], "series=2") { t.Errorf("leaked = %v, want one entry naming session 2's uuid", leaked) } }) - t.Run("presence miss", func(t *testing.T) { + t.Run("partial presence miss", func(t *testing.T) { + resp := "only " + sets[1][0] + " here" + found, leaked := validateReplayResponse(resp, "", sets[1], 1, sets) + if len(found) != 2 || !found[0] || found[1] { + t.Errorf("found = %v, want [true false]", found) + } + if len(leaked) != 0 { + t.Errorf("leaked = %v, want none", leaked) + } + }) + + t.Run("total presence miss", func(t *testing.T) { found, leaked := validateReplayResponse("no ref ids here", "", sets[1], 1, sets) - if found[0] { - t.Error("expected PRESENCE_MISS (found=false)") + if found[0] || found[1] { + t.Error("expected PRESENCE_MISS on both stamps (found=[false false])") } if len(leaked) != 0 { t.Errorf("leaked = %v, want none", leaked) @@ -411,30 +558,81 @@ func TestUUIDValidationEndToEnd(t *testing.T) { }) } +// TestFirstLineConformity exercises firstLineConformity (the output- +// conformity check --replay-inject-uuids scores, mirroring the coherency +// eval's matchesExpectedUUIDList/ExactMatch): pass on an exact ordered +// comma-joined first line; fail on missing, reordered, or chatty first +// lines; pass when line 1 is exact even though LATER lines contain filler +// (the whole point of front-loading the ask to line 1 while forced-output +// keeps generating). +func TestFirstLineConformity(t *testing.T) { + expected := []string{"uuid-a", "uuid-b", "uuid-c"} + + cases := []struct { + name string + resp string + want bool + }{ + {"exact single line", "uuid-a, uuid-b, uuid-c", true}, + {"exact with surrounding whitespace tolerated", " uuid-a, uuid-b, uuid-c ", true}, + {"exact first line, filler after", "uuid-a, uuid-b, uuid-c\nHere is more detail about your request...", true}, + {"exact first line, multiple filler lines after", "uuid-a, uuid-b, uuid-c\nline2\nline3 with more text", true}, + {"missing a uuid", "uuid-a, uuid-c", false}, + {"reordered", "uuid-b, uuid-a, uuid-c", false}, + {"chatty first line", "Sure! The UUIDs are uuid-a, uuid-b, uuid-c", false}, + {"uuids only on line 2, not line 1", "Sure, here you go:\nuuid-a, uuid-b, uuid-c", false}, + {"empty response", "", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := firstLineConformity(c.resp, expected); got != c.want { + t.Errorf("firstLineConformity(%q) = %v, want %v", c.resp, got, c.want) + } + }) + } +} + // TestApplyReciteFloor verifies the max_tokens recite-floor helper: raises -// a too-small budget to replayReciteFloorTokens only when recite is -// requested; leaves larger budgets and non-recite calls untouched. +// a too-small budget to replayReciteFloorTokens(numUUIDs) only when recite +// is requested; leaves larger budgets and non-recite calls untouched. func TestApplyReciteFloor(t *testing.T) { cases := []struct { - name string - tokens int - recite bool - want int + name string + tokens int + recite bool + numUUIDs int }{ - {"below floor, recite -> raised", 10, true, replayReciteFloorTokens}, - {"at floor, recite -> unchanged", replayReciteFloorTokens, true, replayReciteFloorTokens}, - {"above floor, recite -> unchanged", 1000, true, 1000}, - {"below floor, no recite -> unchanged", 10, false, 10}, + {"below floor, recite -> raised", 5, true, 2}, + {"at floor, recite -> unchanged", replayReciteFloorTokens(2), true, 2}, + {"above floor, recite -> unchanged", 100000, true, 2}, + {"below floor, no recite -> unchanged", 5, false, 2}, + {"below floor, more uuids, recite -> raised higher", 5, true, 20}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - if got := applyReciteFloor(c.tokens, c.recite); got != c.want { - t.Errorf("applyReciteFloor(%d, %v) = %d, want %d", c.tokens, c.recite, got, c.want) + floor := replayReciteFloorTokens(c.numUUIDs) + want := c.tokens + if c.recite && c.tokens < floor { + want = floor + } + if got := applyReciteFloor(c.tokens, c.recite, c.numUUIDs); got != want { + t.Errorf("applyReciteFloor(%d, %v, %d) = %d, want %d", c.tokens, c.recite, c.numUUIDs, got, want) } }) } } +// TestReciteFloorScalesWithN verifies the recite floor grows with numUUIDs — +// more UUIDs to recite on the first line needs a bigger budget — the +// N-per-session analogue of the old fixed-64-token constant. +func TestReciteFloorScalesWithN(t *testing.T) { + small := replayReciteFloorTokens(2) + large := replayReciteFloorTokens(20) + if large <= small { + t.Errorf("replayReciteFloorTokens(20) = %d, want > replayReciteFloorTokens(2) = %d", large, small) + } +} + // TestMaxTokensFloorAppliedInWireBuilders verifies the floor is actually // wired into both body builders' emitted max_tokens when a recite // injection is present and the original/recorded budget is tiny — the @@ -447,7 +645,9 @@ func TestMaxTokensFloorAppliedInWireBuilders(t *testing.T) { SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, Messages: []RouterReplayMessage{{Hash: "msg1", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, } - inj := &uuidInjection{Marker: injectUUIDMarker("", []string{"uuid-floor"}), Recite: true, SharedPrefixLen: 1} + uuids := []string{"uuid-floor-0", "uuid-floor-1"} + inj := &uuidInjection{UUIDs: uuids, Recite: true, SharedPrefixLen: 1} + wantFloor := float64(replayReciteFloorTokens(len(uuids))) anthBody, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) if err != nil { @@ -457,8 +657,8 @@ func TestMaxTokensFloorAppliedInWireBuilders(t *testing.T) { if err := json.Unmarshal(anthBody, &anthParsed); err != nil { t.Fatalf("anthropic unmarshal: %v", err) } - if got, want := anthParsed["max_tokens"].(float64), float64(replayReciteFloorTokens); got != want { - t.Errorf("anthropic max_tokens = %v, want %v (floor)", got, want) + if got := anthParsed["max_tokens"].(float64); got != wantFloor { + t.Errorf("anthropic max_tokens = %v, want %v (floor)", got, wantFloor) } openaiBody, _, err := buildOpenAIChatCompletionsBody(req, docs, "model", "", 0, false, inj) @@ -469,8 +669,8 @@ func TestMaxTokensFloorAppliedInWireBuilders(t *testing.T) { if err := json.Unmarshal(openaiBody, &openaiParsed); err != nil { t.Fatalf("openai unmarshal: %v", err) } - if got, want := openaiParsed["max_tokens"].(float64), float64(replayReciteFloorTokens); got != want { - t.Errorf("openai max_tokens = %v, want %v (floor)", got, want) + if got := openaiParsed["max_tokens"].(float64); got != wantFloor { + t.Errorf("openai max_tokens = %v, want %v (floor)", got, wantFloor) } // Without injection (nil), the tiny recorded output_tokens is honored diff --git a/benchmark/replay_router_wire.go b/benchmark/replay_router_wire.go index 43d36c8..604ffd7 100644 --- a/benchmark/replay_router_wire.go +++ b/benchmark/replay_router_wire.go @@ -46,9 +46,13 @@ const verboseOutputInstruction = "Provide a thorough, detailed response and keep // router path — see replay_router_uuid.go); nil means "no injection", // leaving the body byte-for-byte identical to before this feature existed. func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool, inj *uuidInjection) ([]byte, string, error) { + injNumUUIDs := 0 + if inj != nil { + injNumUUIDs = len(inj.UUIDs) + } body := map[string]interface{}{ "model": modelName, - "max_tokens": applyReciteFloor(pickMaxTokens(req, outputRatio), inj != nil && inj.Recite), + "max_tokens": applyReciteFloor(pickMaxTokens(req, outputRatio), inj != nil && inj.Recite, injNumUUIDs), "stream": req.Stream, } if req.Temperature != nil { @@ -76,19 +80,23 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName systemArr = append([]map[string]interface{}{stamp}, systemArr...) } - // UUID marker injection at the system/conversation boundary (Option C — + // UUID block injection at the system/conversation boundary (Option C — // see replay_router_uuid.go). Only spliced in here when the leading run // of cross-session-shared blocks covers every emitted system block; // otherwise it falls back to tail injection below, alongside the // messages array, so it never lands ahead of genuinely per-session // system content (which would poison that session's OWN cache key, // not just cross-session sharing). - markerAtBoundary := inj != nil && inj.Marker != "" && + injUUIDText := "" + if inj != nil { + injUUIDText = bareUUIDBlock(inj.UUIDs) + } + markerAtBoundary := inj != nil && injUUIDText != "" && inj.SharedPrefixLen > 0 && inj.SharedPrefixLen >= len(effectiveSystemBlocks(req.SystemBlocks)) if markerAtBoundary { systemArr = append(systemArr, map[string]interface{}{ "type": "text", - "text": inj.Marker, + "text": injUUIDText, }) } if forceOutput { @@ -108,11 +116,11 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName msgs = buildMessages(req.Messages, docs) } if inj != nil { - if !markerAtBoundary && inj.Marker != "" { - msgs = appendTailMessageAnthropic(msgs, inj.Marker) + if !markerAtBoundary && injUUIDText != "" { + msgs = appendTailMessageAnthropic(msgs, injUUIDText) } if inj.Recite { - msgs = appendTailMessageAnthropic(msgs, replayReciteFromContextInstruction()) + msgs = appendTailMessageAnthropic(msgs, replayReciteFirstLineInstruction()) } } if len(msgs) > 0 { @@ -128,7 +136,7 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName canonical.WriteString(synthText(b.Hash, b.Bytes, docs)) } if markerAtBoundary { - canonical.WriteString(inj.Marker) + canonical.WriteString(injUUIDText) } if req.Tools != nil && req.Tools.Count > 0 { n := req.Tools.Count @@ -168,11 +176,11 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName } } if inj != nil { - if !markerAtBoundary && inj.Marker != "" { - canonical.WriteString(inj.Marker) + if !markerAtBoundary && injUUIDText != "" { + canonical.WriteString(injUUIDText) } if inj.Recite { - canonical.WriteString(replayReciteFromContextInstruction()) + canonical.WriteString(replayReciteFirstLineInstruction()) } } @@ -541,9 +549,13 @@ func buildOpenAITools(spec *RouterReplayToolsSpec, docs string) []map[string]int // inj carries the UUID cache-coherency injection (--replay-inject-uuids, // router path — see replay_router_uuid.go); nil means "no injection". func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool, inj *uuidInjection) ([]byte, string, error) { + injNumUUIDs := 0 + if inj != nil { + injNumUUIDs = len(inj.UUIDs) + } body := map[string]interface{}{ "model": modelName, - "max_tokens": applyReciteFloor(pickMaxTokens(req, outputRatio), inj != nil && inj.Recite), + "max_tokens": applyReciteFloor(pickMaxTokens(req, outputRatio), inj != nil && inj.Recite, injNumUUIDs), "stream": req.Stream, } if req.Temperature != nil { @@ -583,17 +595,21 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN messages = append([]map[string]interface{}{stamp}, messages...) } - // UUID marker injection at the system/conversation boundary (Option C — + // UUID block injection at the system/conversation boundary (Option C — // see replay_router_uuid.go and the mirrored comment in // buildAnthropicMessagesBody). Only spliced in here when the leading // run of cross-session-shared blocks covers every emitted system // block; otherwise it falls back to tail injection below. - markerAtBoundary := inj != nil && inj.Marker != "" && + injUUIDText := "" + if inj != nil { + injUUIDText = bareUUIDBlock(inj.UUIDs) + } + markerAtBoundary := inj != nil && injUUIDText != "" && inj.SharedPrefixLen > 0 && inj.SharedPrefixLen >= len(effectiveSystemBlocks(req.SystemBlocks)) if markerAtBoundary { messages = append(messages, map[string]interface{}{ "role": "system", - "content": inj.Marker, + "content": injUUIDText, }) } if forceOutput { @@ -623,11 +639,11 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN } if inj != nil { - if !markerAtBoundary && inj.Marker != "" { - messages = appendTailMessageOpenAI(messages, inj.Marker) + if !markerAtBoundary && injUUIDText != "" { + messages = appendTailMessageOpenAI(messages, injUUIDText) } if inj.Recite { - messages = appendTailMessageOpenAI(messages, replayReciteFromContextInstruction()) + messages = appendTailMessageOpenAI(messages, replayReciteFirstLineInstruction()) } } @@ -645,7 +661,7 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN canonical.WriteString(synthText(b.Hash, b.Bytes, docs)) } if markerAtBoundary { - canonical.WriteString(inj.Marker) + canonical.WriteString(injUUIDText) } for _, m := range req.Messages { blocks := buildMessageContent(m, docs) @@ -662,11 +678,11 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN } } if inj != nil { - if !markerAtBoundary && inj.Marker != "" { - canonical.WriteString(inj.Marker) + if !markerAtBoundary && injUUIDText != "" { + canonical.WriteString(injUUIDText) } if inj.Recite { - canonical.WriteString(replayReciteFromContextInstruction()) + canonical.WriteString(replayReciteFirstLineInstruction()) } } diff --git a/benchmark/types.go b/benchmark/types.go index 38f50ee..b4583e3 100644 --- a/benchmark/types.go +++ b/benchmark/types.go @@ -28,7 +28,8 @@ type RequestMetrics struct { // when the feature is off (default) or on the synthetic path, which never // populates these. ConvIdx int // session index within cfg.replayUUIDSets (== seriesNum-1) - ExpectedUUIDs []string // this session's own UUID (singleton slice) + ExpectedUUIDs []string // this session's own N-UUID list, in order UUIDFound []bool // parallel to ExpectedUUIDs: whether each was found in Response or thinking LeakedUUIDs []string // "uuid(series=N)" entries for any OTHER session's UUID found here + ExactMatch bool // first line of Response is exactly the ordered, comma-joined ExpectedUUIDs list (output conformity) } From 0b8339ec22d35908e717444bccb586a875eb94f1 Mon Sep 17 00:00:00 2001 From: Ford Shaper Date: Mon, 27 Jul 2026 11:05:17 -0400 Subject: [PATCH 4/6] fix(benchmark): gate router-replay UUID scoring on inj.Recite buildInjection returns a non-nil *uuidInjection on EVERY request once --replay-inject-uuids is on -- injection happens every turn so the stamp stays warm in KV, but only inj.Recite (reciteEveryRequest || isLastRequest) says the model was actually asked to recite it. With --replay-recite-every-request=false, non-final requests were being scored anyway, guaranteeing a false PRESENCE_MISS/conformity failure on every turn the model was never asked to recite. Add inj.Recite to the scoring gate in do() while leaving injection itself ungated. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/replay_router_post.go | 45 ++++++-- benchmark/replay_router_post_test.go | 147 +++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 7 deletions(-) diff --git a/benchmark/replay_router_post.go b/benchmark/replay_router_post.go index f8e9848..2ff3fad 100644 --- a/benchmark/replay_router_post.go +++ b/benchmark/replay_router_post.go @@ -436,7 +436,16 @@ func (p *replayPoster) do( // via validateReplayResponse) and output CONFORMITY (the FIRST LINE of // the response is exactly the ordered, comma-joined UUID list — see // firstLineConformity/matchesExpectedUUIDList). - if inj != nil && m.Error == nil && !m.IsEmpty { + // + // Gated on inj.Recite, NOT just inj != nil: buildInjection returns a + // non-nil *uuidInjection on EVERY request once the feature is on (it + // always carries the UUID block, so the stamp stays warm in KV across a + // session's turns — see the package doc in replay_router_uuid.go), but + // only inj.Recite (reciteEveryRequest || isLastRequest) says the model + // was actually ASKED to recite this turn. Scoring a non-recite turn + // would count "the model didn't volunteer the UUID list" as a + // PRESENCE_MISS/conformity failure even though nothing asked it to. + if inj != nil && inj.Recite && m.Error == nil && !m.IsEmpty { m.ConvIdx = p.sessionIdx m.ExpectedUUIDs = append([]string(nil), p.allUUIDSets[p.sessionIdx]...) m.UUIDFound, m.LeakedUUIDs = validateReplayResponse(m.Response, "", m.ExpectedUUIDs, p.sessionIdx, p.allUUIDSets) @@ -654,6 +663,12 @@ func consumeOpenAISSE(body io.Reader, startTime time.Time, m *RequestMetrics) { } // consumeOpenAIPlain reads a non-streaming OpenAI chat/completions response. +// Like consumeOpenAISSE, this merges message.reasoning_content (or +// message.reasoning, the vLLM field name) into m.Response ahead of +// message.content — a non-streaming reasoning-model response carries its +// full reasoning text on the message object rather than as incremental +// deltas, and skipping it here would make a UUID recited/leaked only in the +// reasoning channel invisible to the presence/leak scan. func consumeOpenAIPlain(body io.Reader, startTime time.Time, m *RequestMetrics) { b, err := io.ReadAll(body) if err != nil { @@ -664,7 +679,9 @@ func consumeOpenAIPlain(body io.Reader, startTime time.Time, m *RequestMetrics) var resp struct { Choices []struct { Message struct { - Content string `json:"content"` + Content string `json:"content"` + ReasoningContent string `json:"reasoning_content"` + Reasoning string `json:"reasoning"` // vLLM uses "reasoning" } `json:"message"` } `json:"choices"` Usage struct { @@ -681,7 +698,12 @@ func consumeOpenAIPlain(body io.Reader, startTime time.Time, m *RequestMetrics) return } if len(resp.Choices) > 0 { - m.Response = resp.Choices[0].Message.Content + msg := resp.Choices[0].Message + reasoning := msg.ReasoningContent + if reasoning == "" { + reasoning = msg.Reasoning + } + m.Response = reasoning + msg.Content } cached := 0 if resp.Usage.PromptTokensDetails != nil { @@ -690,7 +712,12 @@ func consumeOpenAIPlain(body io.Reader, startTime time.Time, m *RequestMetrics) m.UsageData = buildReplayUsage(resp.Usage.PromptTokens, cached, resp.Usage.CompletionTokens) } -// consumePlain reads a non-streaming Anthropic response. +// consumePlain reads a non-streaming Anthropic response. Like consumeSSE, +// this merges "thinking" content blocks into m.Response alongside "text" +// blocks — a non-streaming extended-thinking response carries its thinking +// block(s) as ordinary entries in the content array (field "thinking", not +// "text"), and skipping them here would make a UUID recited/leaked only in +// the thinking channel invisible to the presence/leak scan. func consumePlain(body io.Reader, startTime time.Time, m *RequestMetrics) { b, err := io.ReadAll(body) if err != nil { @@ -700,8 +727,9 @@ func consumePlain(body io.Reader, startTime time.Time, m *RequestMetrics) { m.TimeToFirstToken = time.Since(startTime) var resp struct { Content []struct { - Type string `json:"type"` - Text string `json:"text"` + Type string `json:"type"` + Text string `json:"text"` + Thinking string `json:"thinking"` } `json:"content"` StopReason string `json:"stop_reason"` Usage struct { @@ -717,8 +745,11 @@ func consumePlain(body io.Reader, startTime time.Time, m *RequestMetrics) { } var sb strings.Builder for _, c := range resp.Content { - if c.Type == "text" { + switch c.Type { + case "text": sb.WriteString(c.Text) + case "thinking": + sb.WriteString(c.Thinking) } } m.Response = sb.String() diff --git a/benchmark/replay_router_post_test.go b/benchmark/replay_router_post_test.go index 2149a2b..6f86144 100644 --- a/benchmark/replay_router_post_test.go +++ b/benchmark/replay_router_post_test.go @@ -750,3 +750,150 @@ func TestOpenAISSEWithoutUsage(t *testing.T) { t.Errorf("response = %q, want %q", metrics.Response, "ok") } } + +// TestUUIDScoringGatedOnRecite covers the H1 fix: buildInjection returns a +// non-nil *uuidInjection on EVERY request once UUID injection is on (see +// its doc comment), but only inj.Recite says the model was actually ASKED +// to recite this turn. With --replay-recite-every-request=false, a +// non-final request must still get the UUID block injected (so it stays +// warm in KV — see replay_router_uuid.go's package doc) but must NOT be +// scored: scoring it would count "the model didn't volunteer the UUID +// list" as a false PRESENCE_MISS/conformity failure even though nothing +// asked it to. This drives the poster through p.do() end-to-end (real HTTP +// round trip against an httptest server) rather than calling buildInjection +// directly, so the assertion covers the actual gate in do(). +func TestUUIDScoringGatedOnRecite(t *testing.T) { + docs := strings.Repeat("uuid-gate-docs ", 100) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + // Response deliberately omits the UUID list entirely — if this + // non-recite request were (wrongly) scored, it would register as a + // presence-miss on every expected UUID. + fmt.Fprintf(w, `{ + "id": "chatcmpl-1", + "object": "chat.completion", + "model": "test-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "just a normal answer, no uuids here"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 30, "completion_tokens": 5, "total_tokens": 35} + }`) + })) + defer ts.Close() + + modelSpec := fmt.Sprintf("dynamic/%s,type=openai,model=test-model", ts.URL) + keys := llm.APIKeys{OpenAI: "sk-test"} + p, err := newReplayPoster(modelSpec, keys, "", "", false, 0, 0, 0, nil) + if err != nil { + t.Fatalf("newReplayPoster: %v", err) + } + // Wire up UUID injection exactly as runRouterReplayInstance does (see + // replay_router.go), with reciteEveryRequest=false so only the FINAL + // request of an instance's list carries the recite ask. + p.uuidEnabled = true + p.sessionIdx = 0 + p.allUUIDSets = [][]string{{"uuid-alpha", "uuid-beta"}} + p.blockCounts = map[string]int{} + p.reciteEveryRequest = false + + req := RouterReplayRequest{ + Stream: false, + OutputTokens: 100, + Messages: []RouterReplayMessage{ + {Role: "user", Hash: "h1", Bytes: 50, BlockTypes: []string{"text"}}, + }, + } + st := &autoState{stream: newCompletionStream(200)} + + // isLastRequest=false -> inj.Recite=false (reciteEveryRequest is also + // false) -> the H1 gate must skip scoring entirely. + metrics := p.do(context.Background(), req, docs, 1, "s1", "i1", 1, st, false) + + if metrics.Error != nil { + t.Fatalf("unexpected error: %v", metrics.Error) + } + if len(metrics.ExpectedUUIDs) != 0 { + t.Errorf("ExpectedUUIDs = %v, want empty (non-recite request must not be scored)", metrics.ExpectedUUIDs) + } + if len(metrics.UUIDFound) != 0 { + t.Errorf("UUIDFound = %v, want empty (non-recite request must not be scored)", metrics.UUIDFound) + } + if len(metrics.LeakedUUIDs) != 0 { + t.Errorf("LeakedUUIDs = %v, want empty (non-recite request must not be scored)", metrics.LeakedUUIDs) + } + if metrics.ExactMatch { + t.Error("ExactMatch = true, want false (non-recite request must not be scored)") + } + + // Sanity check the OTHER half of the invariant: the SAME request, with + // isLastRequest=true (recite asked), DOES get scored — otherwise this + // test could be vacuously passing because scoring never fires at all. + metrics2 := p.do(context.Background(), req, docs, 1, "s1", "i2", 1, st, true) + if metrics2.Error != nil { + t.Fatalf("unexpected error on recite request: %v", metrics2.Error) + } + if len(metrics2.ExpectedUUIDs) == 0 { + t.Fatal("ExpectedUUIDs empty on a recite=true request — scoring gate is broken (never scores) rather than fixed") + } +} + +// TestConsumeOpenAIPlainMergesReasoning covers the M2 fix: a non-streaming +// OpenAI response's message.reasoning_content must be merged into +// m.Response alongside message.content, matching consumeOpenAISSE's +// streaming behavior — otherwise a UUID recited/leaked only in the +// reasoning channel would be invisible to the presence/leak scan. +func TestConsumeOpenAIPlainMergesReasoning(t *testing.T) { + body := strings.NewReader(`{ + "choices": [{"index": 0, "message": {"role": "assistant", "content": "the answer is 42", "reasoning_content": "let me think about uuid-in-reasoning first"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5} + }`) + var m RequestMetrics + consumeOpenAIPlain(body, time.Now(), &m) + + if !strings.Contains(m.Response, "uuid-in-reasoning") { + t.Errorf("m.Response = %q, want it to contain the reasoning_content text", m.Response) + } + if !strings.Contains(m.Response, "the answer is 42") { + t.Errorf("m.Response = %q, want it to also contain message.content", m.Response) + } +} + +// TestConsumeOpenAIPlainMergesReasoningVLLMField covers vLLM's alternate +// "reasoning" field name (used when reasoning_content is absent). +func TestConsumeOpenAIPlainMergesReasoningVLLMField(t *testing.T) { + body := strings.NewReader(`{ + "choices": [{"index": 0, "message": {"role": "assistant", "content": "final text", "reasoning": "uuid-in-vllm-reasoning-field"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 10, "completion_tokens": 5} + }`) + var m RequestMetrics + consumeOpenAIPlain(body, time.Now(), &m) + + if !strings.Contains(m.Response, "uuid-in-vllm-reasoning-field") { + t.Errorf("m.Response = %q, want it to contain the reasoning field text", m.Response) + } +} + +// TestConsumePlainMergesThinking covers the M2 fix: a non-streaming +// Anthropic response's "thinking" content block must be merged into +// m.Response alongside "text" blocks, matching consumeSSE's streaming +// behavior — otherwise a UUID recited/leaked only in the thinking channel +// would be invisible to the presence/leak scan. +func TestConsumePlainMergesThinking(t *testing.T) { + body := strings.NewReader(`{ + "content": [ + {"type": "thinking", "thinking": "pondering uuid-in-thinking-block"}, + {"type": "text", "text": "here is my answer"} + ], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 5} + }`) + var m RequestMetrics + consumePlain(body, time.Now(), &m) + + if !strings.Contains(m.Response, "uuid-in-thinking-block") { + t.Errorf("m.Response = %q, want it to contain the thinking block text", m.Response) + } + if !strings.Contains(m.Response, "here is my answer") { + t.Errorf("m.Response = %q, want it to also contain the text block", m.Response) + } +} From 746c51b57261ea411981a1aef0e2fd4d0ef5dd31 Mon Sep 17 00:00:00 2001 From: Ford Shaper Date: Mon, 27 Jul 2026 11:05:26 -0400 Subject: [PATCH 5/6] fix(benchmark): don't splice router-replay UUID ahead of shared tools/messages markerAtBoundary required SharedPrefixLen >= len(effectiveSystemBlocks), i.e. "the shared run covers at least all system blocks" -- which stays true even when the cross-session-shared run extends PAST the system blocks into shared tools or a shared leading message. Splicing the per-session UUID marker as a system block in that case puts it ahead of those shared tools/messages on the wire, making them diverge per session and losing their cross-session prefix-cache hit even though their content stays byte-identical. Flip the comparison to SharedPrefixLen <= len(effectiveSystemBlocks): boundary injection is now eligible only when the shared run ends at or within the system blocks, falling back to tail injection whenever it reaches into tools/messages. Applied to both the Anthropic and OpenAI body builders; behavior is unchanged for the common case where the shared run is contained within the system blocks. Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/replay_router_uuid.go | 14 ++- benchmark/replay_router_uuid_test.go | 124 +++++++++++++++++++++++++++ benchmark/replay_router_wire.go | 23 +++-- 3 files changed, 148 insertions(+), 13 deletions(-) diff --git a/benchmark/replay_router_uuid.go b/benchmark/replay_router_uuid.go index 3d52b8b..80afe0d 100644 --- a/benchmark/replay_router_uuid.go +++ b/benchmark/replay_router_uuid.go @@ -71,8 +71,12 @@ type uuidInjection struct { // SharedPrefixLen is this request's leading run of cross-session-shared // prefix blocks (see sharedPrefixBlockCount). It tells the wire builder // whether the UUID block can be spliced in at the natural system/ - // message boundary (SharedPrefixLen covers every emitted system block) - // or must fall back to tail injection (SharedPrefixLen == 0). + // message boundary (SharedPrefixLen > 0 and does NOT extend past the + // emitted system blocks — i.e. the first tool/message block is not + // itself part of the shared run) or must fall back to tail injection + // (SharedPrefixLen == 0, or the shared run extends into tools/messages, + // in which case boundary-splicing would poison THEIR cross-session + // cache key too). SharedPrefixLen int } @@ -328,8 +332,10 @@ func computePerSessionCachedChars(path string, allowed map[int]bool, sessionLimi // This is the offline analogue of "how much of this request's prefix is // safe to leave byte-identical" — the wire builder uses it to decide // whether the UUID marker can be spliced in at the natural boundary -// (SharedPrefixLen covers every system block) or must fall back to tail -// injection. +// (SharedPrefixLen > 0 and does not extend past the system blocks) or must +// fall back to tail injection (SharedPrefixLen == 0, or it extends into +// tools/messages, which would otherwise poison their cross-session cache +// key too). func sharedPrefixBlockCount(req RouterReplayRequest, counts map[string]int) int { hashes, _ := BuildReplayRequestPrefix(req) n := 0 diff --git a/benchmark/replay_router_uuid_test.go b/benchmark/replay_router_uuid_test.go index fb29047..b30b801 100644 --- a/benchmark/replay_router_uuid_test.go +++ b/benchmark/replay_router_uuid_test.go @@ -6,6 +6,7 @@ package benchmark import ( "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -462,6 +463,129 @@ func TestCacheFidelityBoundaryInvariant(t *testing.T) { } } +// TestCacheFidelityBoundaryDoesNotExtendPastSystemBlocks covers the M1 fix: +// SharedPrefixLen counts the leading run of cross-session-shared blocks over +// the FULL cache order (system blocks, then tools, then messages — see +// BuildReplayRequestPrefix). When that shared run extends PAST the system +// blocks into shared tools (or a shared leading message), splicing the +// per-session UUID marker at the system/tools boundary would land it ahead +// of the shared tools, making them diverge per session on the wire and +// losing their cross-session prefix-cache hit — even though the tools +// themselves stay byte-identical. The fix requires SharedPrefixLen <= +// len(effectiveSystemBlocks(...)) for boundary splicing; when it's greater +// (shared run reaches into tools), injection must fall back to the tail +// instead. Exercises both the Anthropic and OpenAI builders. +func TestCacheFidelityBoundaryDoesNotExtendPastSystemBlocks(t *testing.T) { + docs := strings.Repeat("shared-tools-docs ", 100) + + // Two sessions share BOTH the leading system block (index 0) AND the + // tools block (index 1) — SharedPrefixLen=2 — but diverge starting at + // their own message. len(effectiveSystemBlocks(...)) is only 1, so the + // shared run extends past the system blocks into tools. + sharedTools := &RouterReplayToolsSpec{Count: 2, Bytes: 300, Hash: "toolshash"} + reqA := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, + Tools: sharedTools, + Messages: []RouterReplayMessage{{Hash: "msgUniq-A", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + reqB := RouterReplayRequest{ + InputTokens: 500, + SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, // same shared system block + Tools: sharedTools, // same shared tools + Messages: []RouterReplayMessage{{Hash: "msgUniq-B", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + } + injA := &uuidInjection{UUIDs: []string{"uuid-session-A"}, Recite: false, SharedPrefixLen: 2} + injB := &uuidInjection{UUIDs: []string{"uuid-session-B"}, Recite: false, SharedPrefixLen: 2} + + t.Run("anthropic", func(t *testing.T) { + bodyA, _, err := buildAnthropicMessagesBody(reqA, docs, "model", "", 0, false, injA) + if err != nil { + t.Fatalf("build A: %v", err) + } + bodyB, _, err := buildAnthropicMessagesBody(reqB, docs, "model", "", 0, false, injB) + if err != nil { + t.Fatalf("build B: %v", err) + } + var parsedA, parsedB map[string]interface{} + if err := json.Unmarshal(bodyA, &parsedA); err != nil { + t.Fatalf("unmarshal A: %v", err) + } + if err := json.Unmarshal(bodyB, &parsedB); err != nil { + t.Fatalf("unmarshal B: %v", err) + } + + // The marker must NOT be spliced into the system array — it must + // carry ONLY the original shared block. + sysA, _ := parsedA["system"].([]interface{}) + sysB, _ := parsedB["system"].([]interface{}) + if len(sysA) != 1 || len(sysB) != 1 { + t.Fatalf("expected system to carry ONLY the original shared block (no boundary splice), got lens %d and %d", len(sysA), len(sysB)) + } + + // The shared tools array must stay byte-identical across sessions — + // nothing per-session was spliced ahead of it. + toolsA, _ := json.Marshal(parsedA["tools"]) + toolsB, _ := json.Marshal(parsedB["tools"]) + if string(toolsA) != string(toolsB) { + t.Errorf("shared tools diverged between sessions:\nA: %s\nB: %s", toolsA, toolsB) + } + + // The UUID marker must have landed in the tail (messages) instead. + if !strings.Contains(string(bodyA), "uuid-session-A") { + t.Error("session A's uuid missing from the body entirely — expected tail injection") + } + msgsA, _ := parsedA["messages"].([]interface{}) + if len(msgsA) == 0 { + t.Fatal("expected messages to carry the tail-injected uuid block") + } + lastA := msgsA[len(msgsA)-1].(map[string]interface{}) + if !strings.Contains(fmt.Sprintf("%v", lastA["content"]), "uuid-session-A") { + t.Errorf("uuid marker not found in the tail message: %v", lastA["content"]) + } + }) + + t.Run("openai", func(t *testing.T) { + bodyA, _, err := buildOpenAIChatCompletionsBody(reqA, docs, "model", "", 0, false, injA) + if err != nil { + t.Fatalf("build A: %v", err) + } + bodyB, _, err := buildOpenAIChatCompletionsBody(reqB, docs, "model", "", 0, false, injB) + if err != nil { + t.Fatalf("build B: %v", err) + } + var parsedA, parsedB map[string]interface{} + if err := json.Unmarshal(bodyA, &parsedA); err != nil { + t.Fatalf("unmarshal A: %v", err) + } + if err := json.Unmarshal(bodyB, &parsedB); err != nil { + t.Fatalf("unmarshal B: %v", err) + } + + // The shared tools array must stay byte-identical across sessions. + toolsA, _ := json.Marshal(parsedA["tools"]) + toolsB, _ := json.Marshal(parsedB["tools"]) + if string(toolsA) != string(toolsB) { + t.Errorf("shared tools diverged between sessions:\nA: %s\nB: %s", toolsA, toolsB) + } + + // No system-role message besides the original system block should + // carry the uuid marker (i.e. it must not be boundary-spliced as an + // extra system message). + messagesA, _ := parsedA["messages"].([]interface{}) + for _, raw := range messagesA { + msg, _ := raw.(map[string]interface{}) + if msg["role"] == "system" && strings.Contains(fmt.Sprintf("%v", msg["content"]), "uuid-session-A") { + t.Errorf("uuid marker was boundary-spliced into a system message: %v", msg) + } + } + // It must still appear somewhere in the body (tail injection). + if !strings.Contains(string(bodyA), "uuid-session-A") { + t.Error("session A's uuid missing from the body entirely — expected tail injection") + } + }) +} + // TestTailFallbackInjection verifies that when SharedPrefixLen == 0 (no // usable boundary), the UUID block is folded into the tail (messages array) // rather than the system array, and the request remains well-formed. diff --git a/benchmark/replay_router_wire.go b/benchmark/replay_router_wire.go index 604ffd7..9347336 100644 --- a/benchmark/replay_router_wire.go +++ b/benchmark/replay_router_wire.go @@ -82,17 +82,20 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName // UUID block injection at the system/conversation boundary (Option C — // see replay_router_uuid.go). Only spliced in here when the leading run - // of cross-session-shared blocks covers every emitted system block; - // otherwise it falls back to tail injection below, alongside the - // messages array, so it never lands ahead of genuinely per-session - // system content (which would poison that session's OWN cache key, - // not just cross-session sharing). + // of cross-session-shared blocks does NOT extend past the emitted + // system blocks — i.e. the first tool/message block is not itself part + // of the shared run; otherwise splicing the per-session marker here + // would land it ahead of shared tools/messages and poison THEIR + // cross-session cache key too, not just add a per-session block after + // genuinely-shared content. When the shared run does extend into + // tools/messages, fall back to tail injection below instead, so the + // marker never lands ahead of content this session shares with others. injUUIDText := "" if inj != nil { injUUIDText = bareUUIDBlock(inj.UUIDs) } markerAtBoundary := inj != nil && injUUIDText != "" && - inj.SharedPrefixLen > 0 && inj.SharedPrefixLen >= len(effectiveSystemBlocks(req.SystemBlocks)) + inj.SharedPrefixLen > 0 && inj.SharedPrefixLen <= len(effectiveSystemBlocks(req.SystemBlocks)) if markerAtBoundary { systemArr = append(systemArr, map[string]interface{}{ "type": "text", @@ -598,14 +601,16 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN // UUID block injection at the system/conversation boundary (Option C — // see replay_router_uuid.go and the mirrored comment in // buildAnthropicMessagesBody). Only spliced in here when the leading - // run of cross-session-shared blocks covers every emitted system - // block; otherwise it falls back to tail injection below. + // run of cross-session-shared blocks does NOT extend past the emitted + // system blocks; otherwise it falls back to tail injection below, so it + // never lands ahead of shared tools/messages and poisons their + // cross-session cache key too. injUUIDText := "" if inj != nil { injUUIDText = bareUUIDBlock(inj.UUIDs) } markerAtBoundary := inj != nil && injUUIDText != "" && - inj.SharedPrefixLen > 0 && inj.SharedPrefixLen >= len(effectiveSystemBlocks(req.SystemBlocks)) + inj.SharedPrefixLen > 0 && inj.SharedPrefixLen <= len(effectiveSystemBlocks(req.SystemBlocks)) if markerAtBoundary { messages = append(messages, map[string]interface{}{ "role": "system", From 041d456f859db0b8fb72cde22326467e6c261d76 Mon Sep 17 00:00:00 2001 From: Ford Shaper Date: Mon, 27 Jul 2026 16:54:43 -0400 Subject: [PATCH 6/6] feat(benchmark): rework router-replay UUID injection to per-turn windowed recite Replace the "N UUID stamps per session, clustered at the shared->per-session boundary, recite ALL on line 1" scheme with one deterministic UUID injected inline per qualifying user turn (role==user, has a text block, hash used by exactly one session), spread through the conversation, and a bounded recite window per request: first turn + up to 3 most-recent turns, excluding the current turn, capped at 4. This keeps recite cost/response-budget constant regardless of session length while spreading coverage across the whole conversation over time; every visible turn still gets its own inline marker stamped (keeping it warm in KV) even when it falls outside the recite window. - replay_router_uuid.go: computeSessionTurnHashes/isQualifyingUserTurn replace computePerSessionCachedChars/countSessionsWithUsableBoundary/ sharedPrefixBlockCount/bareUUIDBlock; buildSessionTurnUUIDs replaces buildSessionUUIDs and also returns a uuid->owning-session reverse map; uuidInjection is now {StampByHash, Recite, ReciteLabels, ReciteUUIDs}; replayReciteWindowInstruction replaces replayReciteFirstLineInstruction. - replay_router_wire.go: buildMessageContent/buildMessages/ buildOpenAIMessages take a stampByHash param and append the inline "[turn-N id: ]" marker to the stamped message's own synthesized text, keyed by its hash for byte-identical replay; boundary/tail-splice logic removed from both body builders. - replay_router_post.go: replayPoster gains turnHashes/hashToTurn/owner (drops blockCounts); buildInjection now walks a request's visible turns to build StampByHash and derive the recite window; do()'s validation scores against inj.ReciteUUIDs and calls the new findLeakedUUIDsByOwner scanner instead of validateReplayResponse. - replay_uuid.go: add uuidRe + findLeakedUUIDsByOwner, an O(response) contamination scan via the reverse owner map (FindLeakedUUIDs, whose O(population) scan the dataset path and the coherency CLI still use, is left untouched). - replay_router.go: wire the new poster fields in runRouterReplayInstance; drop the now-dead requestPrefixBytes helper (its only caller, computePerSessionCachedChars, is removed). - auto.go: precompute replaySessionTurnHashes/replayUUIDSets/ replayUUIDOwner once per run; startup diagnostic reports turn counts instead of the retired boundary-usability stats. - cli/benchmark_options.go: update --replay-inject-uuids' description. - Tests rewritten for the new turn/window/contamination model (turn identification across instances, window selection at turns 1-6, determinism/fidelity, reverse-map contamination scanning). Co-Authored-By: Claude Opus 4.8 (1M context) --- benchmark/auto.go | 104 +-- benchmark/replay_router.go | 48 +- benchmark/replay_router_post.go | 143 +++- benchmark/replay_router_post_test.go | 3 +- benchmark/replay_router_uuid.go | 422 +++++----- benchmark/replay_router_uuid_test.go | 1065 +++++++++++++------------- benchmark/replay_router_wire.go | 154 ++-- benchmark/replay_uuid.go | 44 ++ cli/benchmark_options.go | 2 +- 9 files changed, 1030 insertions(+), 955 deletions(-) diff --git a/benchmark/auto.go b/benchmark/auto.go index 77212a5..c61341e 100644 --- a/benchmark/auto.go +++ b/benchmark/auto.go @@ -91,40 +91,48 @@ type AutoBenchmarkConfig struct { // UUID-based cache-coherency validation (--replay-inject-uuids). ROUTER // PATH ONLY (cfg.RouterReplayFile != ""); the CLI rejects this combined - // with --from-dataset (see cli/benchmark_commands.go). Mirrors the - // cache-coherency eval's --shared-prefix-per-series mechanics: N - // deterministic, bare, space-separated UUIDs (N sized off the session's - // per-session cached-region bytes, via computeStampsPerSeries — see - // computePerSessionCachedChars) are injected per SESSION at the boundary - // between its cross-session-shared leading blocks and its per-session - // content (see replay_router_uuid.go for the full design) — this puts - // the UUID block in a region cached WITHIN a session (later requests in - // the same session repeat it, byte-identical) while leaving the - // cross-session shared prefix byte-identical, so cache-hit reproduction - // against the original capture is preserved. + // with --from-dataset (see cli/benchmark_commands.go). One deterministic + // UUID is injected per user turn, spread through the conversation + // (rather than clustered at a session boundary) — see the package doc + // in replay_router_uuid.go for the full design. Each request recites a + // bounded WINDOW (first turn + up to 3 most-recent turns, excluding the + // current turn, capped at 4), keeping the recite cost/response budget + // constant regardless of session length. ReplayInjectUUIDs bool // ReplayUUIDSeed seeds the UUID generator (see newUUIDGenerator); 0 = crypto/rand // (non-deterministic across runs). ReplayUUIDSeed int64 // ReplayReciteEveryRequest: ask the model to recite the first-line UUID - // list on EVERY request (default true), not just each instance's final - // request. + // window on EVERY request (default true), not just each instance's + // final request. ReplayReciteEveryRequest bool - // replayUUIDSets is the precomputed per-session UUID list, populated - // once by RunAutoBenchmark before any per-model goroutine spawns — see - // buildSessionUUIDs. Index i = session i's owned N-UUID list (index i - // corresponds to seriesNum-1, the order sessions are dispatched in — - // see the sizing note at the router-replay precompute call site). - // Shared, read-only, across every model in a multi-model run so every - // model sees the identical assignment (same sharing rationale as - // replayConversations below). + // replaySessionTurnHashes is the precomputed per-session ordered list of + // qualifying user-turn hashes, populated once by RunAutoBenchmark before + // any per-model goroutine spawns — see computeSessionTurnHashes. + // replaySessionTurnHashes[i][t] = session i's turn-t message hash. + // Shared, read-only, across every model in a multi-model run (same + // sharing rationale as replayConversations below). + replaySessionTurnHashes [][]string + // replayUUIDSets is the precomputed per-session-per-turn UUID + // assignment, populated once alongside replaySessionTurnHashes — see + // buildSessionTurnUUIDs. Index i = session i's ordered turn-UUID list + // (index t = turn t's UUID); i corresponds to seriesNum-1, the order + // sessions are dispatched in (see the sizing note at the router-replay + // precompute call site). Shared, read-only, across every model in a + // multi-model run so every model sees the identical assignment. replayUUIDSets [][]string // replayBlockSessionCounts maps a replay-v3 block hash to the number of // DISTINCT SESSIONS that reference it (see computeBlockSessionCounts) — - // a hash referenced by more than one session is safe to leave - // byte-identical across those sessions' requests. Populated once + // a hash referenced by more than one session is never eligible for + // turn-stamping (isQualifyingUserTurn requires count==1). Populated once // alongside replayUUIDSets. replayBlockSessionCounts map[string]int + // replayUUIDOwner is the precomputed reverse uuid -> owning-session-index + // map, populated once alongside replayUUIDSets — see + // buildSessionTurnUUIDs. Read-only after precompute; used by + // findLeakedUUIDsByOwner (replay_uuid.go) to flag cross-session + // contamination in O(response) time. + replayUUIDOwner map[string]int // RunID is populated internally by RunAutoBenchmark at the start of each // run. It's the UUID injected into every conversation's system prompt @@ -2599,16 +2607,23 @@ func RunAutoBenchmark(ctx context.Context, cfg AutoBenchmarkConfig) error { return fmt.Errorf("compute block session counts for --replay-inject-uuids: %w", cerr) } cfg.replayBlockSessionCounts = counts - // Per-session cached-region byte size (the blocks AFTER each - // session's cross-session-shared boundary — see - // computePerSessionCachedChars) sizes N stamps per session - // exactly as the cache-coherency eval turns --garbage-chars - // into --stamps-per-series (computeStampsPerSeries, min 2). - perSessionChars, perr := computePerSessionCachedChars(cfg.RouterReplayFile, cfg.RouterReplaySeriesIndices, cfg.ReplaySeries, counts) - if perr != nil { - return fmt.Errorf("compute per-session cached chars for --replay-inject-uuids: %w", perr) + + // Per-session ordered list of qualifying user-turn hashes + // (role=="user", has a text block, hash referenced by + // exactly this one session — see isQualifyingUserTurn), + // each turn getting its own UUID (buildSessionTurnUUIDs) + // rather than N stamps clustered at a session boundary. + turnHashes, terr := computeSessionTurnHashes(cfg.RouterReplayFile, cfg.RouterReplaySeriesIndices, cfg.ReplaySeries, counts) + if terr != nil { + return fmt.Errorf("compute session turn hashes for --replay-inject-uuids: %w", terr) + } + cfg.replaySessionTurnHashes = turnHashes + + turnCounts := make([]int, len(turnHashes)) + for i, h := range turnHashes { + turnCounts[i] = len(h) } - cfg.replayUUIDSets = buildSessionUUIDs(perSessionChars, cfg.ReplayUUIDSeed) + cfg.replayUUIDSets, cfg.replayUUIDOwner = buildSessionTurnUUIDs(turnCounts, cfg.ReplayUUIDSeed) sharedHashes := 0 for _, n := range counts { @@ -2616,24 +2631,21 @@ func RunAutoBenchmark(ctx context.Context, cfg AutoBenchmarkConfig) error { sharedHashes++ } } - usable, total, berr := countSessionsWithUsableBoundary(cfg.RouterReplayFile, cfg.RouterReplaySeriesIndices, cfg.ReplaySeries, counts) - if berr != nil { - return fmt.Errorf("compute usable-boundary diagnostic for --replay-inject-uuids: %w", berr) - } - minStamps, maxStamps := 0, 0 - if len(cfg.replayUUIDSets) > 0 { - minStamps, maxStamps = len(cfg.replayUUIDSets[0]), len(cfg.replayUUIDSets[0]) - for _, set := range cfg.replayUUIDSets { - if len(set) < minStamps { - minStamps = len(set) + totalTurns, minTurns, maxTurns := 0, 0, 0 + if len(turnCounts) > 0 { + minTurns, maxTurns = turnCounts[0], turnCounts[0] + for _, n := range turnCounts { + totalTurns += n + if n < minTurns { + minTurns = n } - if len(set) > maxStamps { - maxStamps = len(set) + if n > maxTurns { + maxTurns = n } } } - fmt.Printf("UUID validation enabled: %d session(s) prepared, %d cross-session-shared block hash(es), %d/%d sessions have a usable boundary (%d fall back to tail injection), %d-%d UUID stamps/session (recite-every-request=%v, seed=%d)\n", - effectiveSessions, sharedHashes, usable, total, total-usable, minStamps, maxStamps, cfg.ReplayReciteEveryRequest, cfg.ReplayUUIDSeed) + fmt.Printf("UUID validation enabled: %d session(s) prepared, %d turn(s) total, %d-%d turns/session, %d cross-session-shared block hash(es), seed=%d, recite-every-request=%v\n", + effectiveSessions, totalTurns, minTurns, maxTurns, sharedHashes, cfg.ReplayUUIDSeed, cfg.ReplayReciteEveryRequest) } } } diff --git a/benchmark/replay_router.go b/benchmark/replay_router.go index b284904..accf035 100644 --- a/benchmark/replay_router.go +++ b/benchmark/replay_router.go @@ -640,17 +640,28 @@ func runRouterReplayInstance( // UUID cache-coherency injection (--replay-inject-uuids, router // path). sessionIdx == seriesNum-1: every instance of a session // shares the session's seriesNum, so every instance's poster picks - // the SAME session UUID/marker. uuidEnabled stays false (and - // buildInjection nil) whenever the flag is off, or this session's - // index fell outside the precomputed array (see the sizing note on - // AutoBenchmarkConfig.replayUUIDSets) — degrading gracefully to - // "no injection" for that session rather than panicking. + // the SAME session's turn-UUID assignment. uuidEnabled stays false + // (and buildInjection nil) whenever the flag is off, or this + // session's index fell outside the precomputed array (see the + // sizing note on AutoBenchmarkConfig.replayUUIDSets) — degrading + // gracefully to "no injection" for that session rather than + // panicking. turnHashes/hashToTurn are session-global (every + // instance of the session sees the SAME turn numbering, even though + // any one instance's requests typically only surface its own + // subset of turns) — built once here rather than per request. if cfg.ReplayInjectUUIDs { poster.uuidEnabled = true poster.sessionIdx = seriesNum - 1 poster.allUUIDSets = cfg.replayUUIDSets - poster.blockCounts = cfg.replayBlockSessionCounts + poster.owner = cfg.replayUUIDOwner poster.reciteEveryRequest = cfg.ReplayReciteEveryRequest + if poster.sessionIdx >= 0 && poster.sessionIdx < len(cfg.replaySessionTurnHashes) { + poster.turnHashes = cfg.replaySessionTurnHashes[poster.sessionIdx] + poster.hashToTurn = make(map[string]int, len(poster.turnHashes)) + for t, h := range poster.turnHashes { + poster.hashToTurn[h] = t + } + } } } if err != nil { @@ -763,28 +774,3 @@ func BuildReplayRequestPrefix(req RouterReplayRequest) (hashes []string, tokens } return } - -// requestPrefixBytes mirrors BuildReplayRequestPrefix's exact block sequence -// (same skip-the-tiny-header-block rule, same system-blocks/tools/messages -// order) but returns each entry's Bytes instead of its hash/token count, so -// index i here lines up 1:1 with index i of BuildReplayRequestPrefix's -// hashes. Used by computePerSessionCachedChars to sum the byte size of a -// request's prefix blocks AT OR AFTER a given boundary index (e.g. -// sharedPrefixBlockCount) — the per-session cached region --replay-inject- -// uuids sizes its UUID-stamp count off (see replay_router_uuid.go). -func requestPrefixBytes(req RouterReplayRequest) []int { - var out []int - for i, sb := range req.SystemBlocks { - if i == 0 && sb.Bytes < 200 { - continue - } - out = append(out, sb.Bytes) - } - if req.Tools != nil && req.Tools.Hash != "" { - out = append(out, req.Tools.Bytes) - } - for _, m := range req.Messages { - out = append(out, m.Bytes) - } - return out -} diff --git a/benchmark/replay_router_post.go b/benchmark/replay_router_post.go index 2ff3fad..b376f88 100644 --- a/benchmark/replay_router_post.go +++ b/benchmark/replay_router_post.go @@ -70,19 +70,29 @@ type replayPoster struct { // identical to before this feature existed. uuidEnabled bool // sessionIdx is this instance's session's 0-based index into - // allUUIDSets/blockCounts (== seriesNum-1 — every instance of a session - // shares the same seriesNum, hence the same sessionIdx). + // allUUIDSets/cfg.replaySessionTurnHashes (== seriesNum-1 — every + // instance of a session shares the same seriesNum, hence the same + // sessionIdx). sessionIdx int - // allUUIDSets is cfg.replayUUIDSets: the full per-session UUID - // assignment (index i = session i's owned UUID set), shared read-only - // across every poster in the run — needed both to pick this session's - // own marker and to scan for OTHER sessions' UUIDs leaking into this - // response (cross-contamination). + // allUUIDSets is cfg.replayUUIDSets: the full per-session-per-turn UUID + // assignment (index i = session i's ordered turn-UUID list, index t = + // turn t's UUID), shared read-only across every poster in the run. allUUIDSets [][]string - // blockCounts is cfg.replayBlockSessionCounts: hash -> distinct-session - // count, used by sharedPrefixBlockCount to find each request's safe - // injection boundary. - blockCounts map[string]int + // turnHashes is this poster's session's ordered turn-hash list + // (cfg.replaySessionTurnHashes[sessionIdx] — see + // computeSessionTurnHashes): turnHashes[t] is the hash of turn t. + // Session-global (spans every instance of the session), even though any + // one instance's requests typically only ever surface its own subset of + // turns. + turnHashes []string + // hashToTurn is turnHashes inverted (hash -> turn index), computed once + // per instance (see runRouterReplayInstance) rather than per request. + hashToTurn map[string]int + // owner is cfg.replayUUIDOwner: the reverse uuid -> owning-session-index + // map (see buildSessionTurnUUIDs), shared read-only across every poster + // in the run — used by findLeakedUUIDsByOwner to flag cross-session + // contamination in O(response) time. + owner map[string]int // reciteEveryRequest mirrors --replay-recite-every-request: true asks // for the recite line on every request; false only on each instance's // final request (see the isLastRequest parameter to do()/dryDo()). @@ -90,23 +100,84 @@ type replayPoster struct { } // buildInjection returns this call's *uuidInjection (nil when UUID -// injection is disabled, or when this session has no assigned UUID — e.g. -// sessionIdx fell outside the precomputed array). isLastRequest is whether -// req is the final request of the CURRENT instance's request list (see +// injection is disabled, this session has no assigned turns — e.g. +// sessionIdx fell outside the precomputed array, or this session had zero +// qualifying turns — or this particular request has no qualifying turn +// visible in its message history yet). isLastRequest is whether req is the +// final request of the CURRENT instance's request list (see // runRouterReplayInstance) — with --replay-recite-every-request=false, only // that final request carries the recite ask. +// +// Every VISIBLE qualifying turn in req.Messages gets stamped into +// StampByHash (keeping every turn's marker warm in KV as later requests +// repeat it) — see uuidInjection's doc. The recite WINDOW is separate and +// bounded: the first (visible) turn plus up to 3 most-recent turns +// EXCLUDING the current turn (the highest-index turn visible in THIS +// request), deduplicated and capped at 4 (see the package doc in +// replay_router_uuid.go for the design rationale and edge cases at turns +// 1-3). func (p *replayPoster) buildInjection(req RouterReplayRequest, isLastRequest bool) *uuidInjection { - if !p.uuidEnabled || p.sessionIdx < 0 || p.sessionIdx >= len(p.allUUIDSets) { + if !p.uuidEnabled || p.sessionIdx < 0 || p.sessionIdx >= len(p.allUUIDSets) || len(p.turnHashes) == 0 { return nil } uuids := p.allUUIDSets[p.sessionIdx] if len(uuids) == 0 { return nil } + + stampByHash := map[string]turnStamp{} + var visible []int // turn indices visible in this request, in first-appearance order + seenTurn := map[int]bool{} + for _, m := range req.Messages { + t, ok := p.hashToTurn[m.Hash] + if !ok || t < 0 || t >= len(uuids) { + continue + } + stampByHash[m.Hash] = turnStamp{Idx: t, UUID: uuids[t], Label: fmt.Sprintf("turn-%d", t+1)} + if !seenTurn[t] { + seenTurn[t] = true + visible = append(visible, t) + } + } + if len(visible) == 0 { + return nil + } + + // Window: first visible turn, plus up to 3 most-recent turns EXCLUDING + // the current turn (visible's last entry — the highest turn index + // present, since turns only ever get appended to a growing history). + first := visible[0] + var recentCandidates []int + if len(visible) > 1 { + recentCandidates = visible[:len(visible)-1] + } + recent := recentCandidates + if len(recent) > 3 { + recent = recent[len(recent)-3:] + } + window := []int{first} + for _, t := range recent { + if t == first { + continue + } + window = append(window, t) + } + if len(window) > 4 { + window = window[:4] + } + + labels := make([]string, len(window)) + reciteUUIDs := make([]string, len(window)) + for i, t := range window { + labels[i] = fmt.Sprintf("turn-%d", t+1) + reciteUUIDs[i] = uuids[t] + } + return &uuidInjection{ - UUIDs: uuids, - Recite: p.reciteEveryRequest || isLastRequest, - SharedPrefixLen: sharedPrefixBlockCount(req, p.blockCounts), + StampByHash: stampByHash, + Recite: p.reciteEveryRequest || isLastRequest, + ReciteLabels: labels, + ReciteUUIDs: reciteUUIDs, } } @@ -433,23 +504,33 @@ func (p *replayPoster) do( // // Two independent checks, mirroring the cache-coherency eval's two // reported tests: per-UUID PRESENCE (Contains anywhere in the response, - // via validateReplayResponse) and output CONFORMITY (the FIRST LINE of - // the response is exactly the ordered, comma-joined UUID list — see - // firstLineConformity/matchesExpectedUUIDList). + // scored against inj.ReciteUUIDs — this request's recite WINDOW, not the + // session's full turn history) and output CONFORMITY (the FIRST LINE of + // the response is exactly the ordered, comma-joined window UUID list — + // see firstLineConformity/matchesExpectedUUIDList). Cross-contamination + // uses findLeakedUUIDsByOwner (replay_uuid.go), an O(response) reverse- + // map scan — NOT FindLeakedUUIDs, whose O(population) iteration over + // every session's UUID set no longer fits once turns (not sessions) are + // the stamping unit. // // Gated on inj.Recite, NOT just inj != nil: buildInjection returns a - // non-nil *uuidInjection on EVERY request once the feature is on (it - // always carries the UUID block, so the stamp stays warm in KV across a - // session's turns — see the package doc in replay_router_uuid.go), but - // only inj.Recite (reciteEveryRequest || isLastRequest) says the model - // was actually ASKED to recite this turn. Scoring a non-recite turn - // would count "the model didn't volunteer the UUID list" as a - // PRESENCE_MISS/conformity failure even though nothing asked it to. + // non-nil *uuidInjection whenever ANY qualifying turn is visible in this + // request (it always stamps every visible turn inline, so those stamps + // stay warm in KV across the session — see the package doc in + // replay_router_uuid.go), but only inj.Recite (reciteEveryRequest || + // isLastRequest) says the model was actually ASKED to recite this turn. + // Scoring a non-recite turn would count "the model didn't volunteer the + // UUID list" as a PRESENCE_MISS/conformity failure even though nothing + // asked it to. if inj != nil && inj.Recite && m.Error == nil && !m.IsEmpty { m.ConvIdx = p.sessionIdx - m.ExpectedUUIDs = append([]string(nil), p.allUUIDSets[p.sessionIdx]...) - m.UUIDFound, m.LeakedUUIDs = validateReplayResponse(m.Response, "", m.ExpectedUUIDs, p.sessionIdx, p.allUUIDSets) - m.ExactMatch = firstLineConformity(m.Response, m.ExpectedUUIDs) + m.ExpectedUUIDs = append([]string(nil), inj.ReciteUUIDs...) + m.UUIDFound = make([]bool, len(inj.ReciteUUIDs)) + for i, u := range inj.ReciteUUIDs { + m.UUIDFound[i] = strings.Contains(m.Response, u) + } + m.LeakedUUIDs = findLeakedUUIDsByOwner(m.Response, "", p.sessionIdx, p.owner) + m.ExactMatch = firstLineConformity(m.Response, inj.ReciteUUIDs) } return m } diff --git a/benchmark/replay_router_post_test.go b/benchmark/replay_router_post_test.go index 6f86144..39c8d15 100644 --- a/benchmark/replay_router_post_test.go +++ b/benchmark/replay_router_post_test.go @@ -793,7 +793,8 @@ func TestUUIDScoringGatedOnRecite(t *testing.T) { p.uuidEnabled = true p.sessionIdx = 0 p.allUUIDSets = [][]string{{"uuid-alpha", "uuid-beta"}} - p.blockCounts = map[string]int{} + p.turnHashes = []string{"h1", "h2"} + p.hashToTurn = map[string]int{"h1": 0, "h2": 1} p.reciteEveryRequest = false req := RouterReplayRequest{ diff --git a/benchmark/replay_router_uuid.go b/benchmark/replay_router_uuid.go index 80afe0d..75d835c 100644 --- a/benchmark/replay_router_uuid.go +++ b/benchmark/replay_router_uuid.go @@ -1,48 +1,37 @@ package benchmark // UUID-based cache-coherency validation for the ROUTER-REPLAY path -// (--router-replay-file, --replay-inject-uuids). This mirrors the mechanics -// of the cache-coherency eval's --shared-prefix-per-series mode -// (cache_coherency.go's buildCoherencySharedSeriesPrompt/userMessage/ -// matchesExpectedUUIDList) as closely as the router-replay wire shape -// allows: N bare, space-separated UUIDs stamped once per session, a recite- -// the-list instruction, and exact first-line conformity scoring — rather -// than a single wrapped "[ref-id: ...]" marker recited anywhere in the -// response. (Path-agnostic primitives — injectUUIDMarker/ -// validateReplayResponse/FindLeakedUUIDs — are still shared with, not -// duplicated from, replay_uuid.go; injectUUIDMarker itself remains the -// dataset-replay path's own wrapper and is untouched here.) +// (--router-replay-file, --replay-inject-uuids). // -// Strategy (Option C — boundary injection, with tail fallback): every -// session in a replay-v3 capture opens with one or more blocks (system -// blocks, tools, or leading messages) whose content hash is shared across -// MANY OTHER sessions too — the router's own leading system prompt(s), -// repeated verbatim capture after capture. Everything AFTER that shared -// run is genuinely per-session (the user's actual turn). We inject exactly -// N deterministic, bare, space-separated UUIDs per session at that -// boundary — mirroring buildCoherencySharedSeriesPrompt's tail -// ("UUID0 UUID1 … UUIDlast") rather than the dataset path's wrapped -// "[ref-id: ]" marker: +// Strategy: one deterministic UUID is injected inline into EVERY qualifying +// user turn — a role=="user" message with >=1 text block whose hash is +// referenced by exactly one session (see isQualifyingUserTurn, which reuses +// computeBlockSessionCounts' distinct-session-count map to exclude blocks +// shared across sessions) — spread through the conversation rather than +// clustered at a session boundary. Turn i's marker +// ("\n\n[turn-N id: ]") is appended to that turn's own synthesized +// text content (see buildMessageContent's stampByHash param in +// replay_router_wire.go), keyed by the turn's session-global +// first-appearance index (see computeSessionTurnHashes/ +// buildSessionTurnUUIDs below). Content synthesis is already deterministic +// in the block hash (synthText, seeded ":block:"), so every request +// that repeats a given turn in its growing history re-emits byte-identical +// content for it — the fidelity invariant that makes cache-hit reproduction +// against the original capture possible: two sessions sharing a leading +// block still collide on the server's prefix cache exactly as in the +// original traffic, because only count==1 (genuinely per-session) turns +// ever carry a stamp. // -// [ RUN_GUID ][ shared system blocks ][ UUID0 UUID1 … UUIDlast ][ forceOutput instr ] [ messages... ] [ recite-first-line ask ] -// \_______________________ byte-identical across sessions ________________________/ \_ per-session, grows each turn _/ -// -// Putting the UUID block there means: -// - the cross-session shared prefix stays byte-identical (cache-hit -// reproduction against the original capture is preserved: two sessions -// that shared a system prompt still collide on the server's prefix -// cache exactly as they did in the original traffic) -// - the UUID block itself lands in a region that IS cached WITHIN a -// session (every subsequent request in the same session repeats it, -// byte-identical), so asking the model to recall it later is a genuine -// KV-coherency signal, not an artifact of it being freshly re-sent -// every turn. -// -// A session with no shared leading block at all (empirically none, across -// 5441 real sessions, lack one — but the file format doesn't guarantee it) -// falls back to tail injection: the UUID block is folded into the end of -// the request instead, forfeiting the "cached within a session" property -// but still producing a valid, scorable request. +// Each request asks the model to recite a WINDOW of turns rather than the +// whole history: the first (visible) turn plus up to 3 most-recent turns, +// EXCLUDING the current turn, deduplicated and capped at 4 (see +// replayPoster.buildInjection in replay_router_post.go and +// replayReciteWindowInstruction below). This keeps the recite cost/response +// budget constant regardless of session length while still spreading +// coverage across the whole conversation over time — turn N's stamp gets +// asked about at turns N, N+1, N+2, N+3, then ages out of the window (but +// stays warm in KV via the always-embedded StampByHash markers above, which +// cover every visible turn, not just the recited window). import ( "bufio" @@ -53,42 +42,40 @@ import ( "sync" ) +// turnStamp is the per-user-turn UUID marker injected inline into that +// turn's own message content (see buildMessageContent's stampByHash param +// in replay_router_wire.go). +type turnStamp struct { + Idx int // this session's global turn index (0-based, first-appearance order) + UUID string // this turn's deterministic UUID + Label string // "turn-N" (Idx+1 — 1-based, for human-readable instructions) +} + // uuidInjection describes the per-request UUID injection to apply when // building a router-replay wire body. A nil *uuidInjection means "no // injection" — buildAnthropicMessagesBody / buildOpenAIChatCompletionsBody // must behave identically to before this feature existed. type uuidInjection struct { - // UUIDs is the session's full ordered N-UUID list (see buildSessionUUIDs), - // spliced bare and space-separated (see bareUUIDBlock) — mirrors the - // cache-coherency eval's buildCoherencySharedSeriesPrompt tail. Nil/empty - // means no UUID block this call (still allows Recite alone, though - // callers currently always set both together). - UUIDs []string - // Recite asks the model to output, as the FIRST line of its response, - // the exact ordered UUID list (see replayReciteFirstLineInstruction), - // then continue normally. + // StampByHash carries one turnStamp per user-turn message VISIBLE in + // this request (keyed by RouterReplayMessage.Hash) — every qualifying + // turn gets its UUID marker embedded inline in its own synthesized + // content (see buildMessageContent), keeping every turn's stamp warm in + // KV as later requests repeat that history in full, not just the turns + // named in the recite window below. + StampByHash map[string]turnStamp + // Recite asks the model to output, on the FIRST line of its response, + // the ReciteUUIDs values (identified to the model by ReciteLabels' + // inline tags — see replayReciteWindowInstruction), then continue + // normally. Recite bool - // SharedPrefixLen is this request's leading run of cross-session-shared - // prefix blocks (see sharedPrefixBlockCount). It tells the wire builder - // whether the UUID block can be spliced in at the natural system/ - // message boundary (SharedPrefixLen > 0 and does NOT extend past the - // emitted system blocks — i.e. the first tool/message block is not - // itself part of the shared run) or must fall back to tail injection - // (SharedPrefixLen == 0, or the shared run extends into tools/messages, - // in which case boundary-splicing would poison THEIR cross-session - // cache key too). - SharedPrefixLen int -} - -// bareUUIDBlock returns uuids joined bare and space-separated — mirrors the -// cache-coherency eval's buildCoherencySharedSeriesPrompt tail -// ("UUID0 UUID1 … UUIDlast", no wrapper text) — for splicing as one system -// message/block at the session boundary (or the tail, on fallback). Bare -// UUIDs (vs. the dataset path's "[ref-id: ]" wrapper) both match the -// coherency test's mechanics and avoid the model treating a marker-shaped -// wrapper as instruction text to echo verbatim. -func bareUUIDBlock(uuids []string) string { - return strings.Join(uuids, " ") + // ReciteLabels is the ordered window of turn labels ("turn-N") the + // instruction names — first (visible) turn, then up to 3 most-recent + // turns EXCLUDING the current turn, deduplicated, capped at 4. + ReciteLabels []string + // ReciteUUIDs is ReciteLabels' matching ordered UUID values — what + // firstLineConformity/matchesExpectedUUIDList checks the response's + // first line against. + ReciteUUIDs []string } // firstLineConformity implements this feature's output-conformity check: @@ -98,6 +85,8 @@ func bareUUIDBlock(uuids []string) string { // eval's ExactMatch, adapted for a recite-FIRST instruction that lets the // model keep generating after line 1 (so forced-output/ignore_eos still // fills the remainder of the output budget with a normal continuation). +// expected is inj.ReciteUUIDs — the current request's recite WINDOW, not +// every UUID ever stamped in the session. func firstLineConformity(resp string, expected []string) bool { line := resp if idx := strings.Index(resp, "\n"); idx >= 0 { @@ -109,10 +98,15 @@ func firstLineConformity(resp string, expected []string) bool { // computeBlockSessionCounts streams a replay-v3 file once and returns, for // every distinct block hash seen, the number of DISTINCT SESSIONS that // reference it at least once (not the number of requests — a hash reused -// many times within one session still counts once for that session). A -// hash with count > 1 is shared across sessions, which is exactly the -// "safe to leave byte-identical" prefix content sharedPrefixBlockCount looks -// for. +// many times within one session still counts once for that session). +// +// A hash with count > 1 is shared across sessions — the router's own +// leading system prompt(s), repeated verbatim capture after capture, are +// the common case — and is never eligible for UUID stamping: stamping it +// would perturb content this session shares with others, poisoning their +// cross-session prefix-cache key too. computeSessionTurnHashes uses this +// map (via isQualifyingUserTurn) to restrict turn-stamping to hashes with +// count == 1: genuinely per-session, per-turn content. // // allowed and sessionLimit mirror openRouterReplayStream's filtering // exactly (nil allowed = every session; sessionLimit <= 0 = no cap) so the @@ -180,93 +174,41 @@ func computeBlockSessionCounts(path string, allowed map[int]bool, sessionLimit i return counts, nil } -// countSessionsWithUsableBoundary makes a second lightweight streaming pass -// (same filtering as computeBlockSessionCounts) purely to report, for the -// startup diagnostic, how many sessions have at least one request whose -// leading run of shared blocks is non-empty (a "usable boundary" — the -// marker can be spliced in at the natural system/message boundary) versus -// how many would fall back to tail injection for every one of their -// requests. Returns (usable, total, error); usable <= total. -func countSessionsWithUsableBoundary(path string, allowed map[int]bool, sessionLimit int, counts map[string]int) (usable int, total int, err error) { - f, ferr := os.Open(path) - if ferr != nil { - return 0, 0, ferr +// isQualifyingUserTurn reports whether m is a "user-input turn" for the +// purposes of UUID stamping: role=="user", at least one "text" content +// block, and a hash referenced by exactly one session (counts[m.Hash]==1 — +// see computeBlockSessionCounts). tool_result-only messages, assistant/ +// system messages, and any message whose hash is shared across sessions +// never qualify. +func isQualifyingUserTurn(m RouterReplayMessage, counts map[string]int) bool { + if m.Role != "user" { + return false } - defer f.Close() - - br := bufio.NewReaderSize(f, 1<<20) - if _, rerr := br.ReadBytes('\n'); rerr != nil { - return 0, 0, fmt.Errorf("read header line: %w", rerr) + if m.Hash == "" || counts[m.Hash] != 1 { + return false } - - lineIdx := 0 - produced := 0 - for { - if sessionLimit > 0 && produced >= sessionLimit { - break - } - if allowed != nil && produced >= len(allowed) { - break - } - line, rerr := br.ReadBytes('\n') - if len(line) > 0 { - line = trimNL(line) - if len(line) > 0 { - currentIdx := lineIdx - lineIdx++ - if allowed != nil && !allowed[currentIdx] { - if rerr != nil { - break - } - continue - } - var sess RouterReplaySession - if jerr := json.Unmarshal(line, &sess); jerr == nil { - total++ - hasBoundary := false - for _, inst := range sess.Instances { - for _, req := range inst.Requests { - if sharedPrefixBlockCount(req, counts) > 0 { - hasBoundary = true - break - } - } - if hasBoundary { - break - } - } - if hasBoundary { - usable++ - } - produced++ - } - } - } - if rerr != nil { - break + for _, t := range m.BlockTypes { + if t == "text" { + return true } } - return usable, total, nil + return false } -// computePerSessionCachedChars makes a second streaming pass (same filtering -// as computeBlockSessionCounts) and returns, in dispatch order (index i = -// the i-th session encountered — the SAME order buildSessionUUIDs' returned -// slice is indexed by, and the same order cfg.replayUUIDSets ends up in), -// each session's per-session cached-region byte size: the ROOT request's -// (first instance, first request) prefix bytes (see requestPrefixBytes) AT -// OR AFTER that request's cross-session-shared boundary -// (sharedPrefixBlockCount, using counts from computeBlockSessionCounts). -// -// This is a proxy for "how much content is genuinely per-session and reused, -// byte-identical, across the session's own later requests" — the region the -// injected UUID block actually sits within once spliced at the boundary -// (see the package doc comment above). computeStampsPerSeries then turns -// this byte count into a stamp count exactly as the cache-coherency eval -// turns --garbage-chars into --stamps-per-series. A session with no -// requests at all (degenerate/empty) contributes 0, which -// computeStampsPerSeries floors to its minimum of 2. -func computePerSessionCachedChars(path string, allowed map[int]bool, sessionLimit int, counts map[string]int) ([]int, error) { +// computeSessionTurnHashes makes a second streaming pass (same filtering as +// computeBlockSessionCounts — nil allowed = every session; sessionLimit <= 0 +// = no cap) and returns, in dispatch order (index i = the i-th session +// encountered, the SAME order buildSessionTurnUUIDs' returned slice is +// indexed by and cfg.replayUUIDSets ends up in), that session's ordered list +// of DISTINCT qualifying user-turn hashes (see isQualifyingUserTurn), in +// first-appearance order across the session's instances/requests/messages +// (walked in file order — Instances, then each instance's Requests, then +// each request's Messages). A request's Messages carries the FULL growing +// conversation history, so the same turn hash reappears in every later +// request of the same instance; only its FIRST appearance contributes an +// entry here — turnHashes[i][t] is session i's turn-t hash, and len(...) is +// that session's total turn count. +func computeSessionTurnHashes(path string, allowed map[int]bool, sessionLimit int, counts map[string]int) ([][]string, error) { f, err := os.Open(path) if err != nil { return nil, err @@ -278,7 +220,7 @@ func computePerSessionCachedChars(path string, allowed map[int]bool, sessionLimi return nil, fmt.Errorf("read header line: %w", err) } - var perSession []int + var turnHashes [][]string lineIdx := 0 produced := 0 for { @@ -302,16 +244,23 @@ func computePerSessionCachedChars(path string, allowed map[int]bool, sessionLimi } var sess RouterReplaySession if jerr := json.Unmarshal(line, &sess); jerr == nil { - chars := 0 - if len(sess.Instances) > 0 && len(sess.Instances[0].Requests) > 0 { - root := sess.Instances[0].Requests[0] - boundary := sharedPrefixBlockCount(root, counts) - prefixBytes := requestPrefixBytes(root) - for i := boundary; i < len(prefixBytes); i++ { - chars += prefixBytes[i] + seen := map[string]bool{} + var hashes []string + for _, inst := range sess.Instances { + for _, req := range inst.Requests { + for _, m := range req.Messages { + if !isQualifyingUserTurn(m, counts) { + continue + } + if seen[m.Hash] { + continue + } + seen[m.Hash] = true + hashes = append(hashes, m.Hash) + } } } - perSession = append(perSession, chars) + turnHashes = append(turnHashes, hashes) produced++ } } @@ -320,84 +269,70 @@ func computePerSessionCachedChars(path string, allowed map[int]bool, sessionLimi break } } - return perSession, nil + return turnHashes, nil } -// sharedPrefixBlockCount returns the length of req's LEADING run of blocks -// (per BuildReplayRequestPrefix's cache-order hash sequence: system blocks, -// then tools, then messages) whose hash is shared across more than one -// session, per counts (see computeBlockSessionCounts). The run stops at the -// first hash that is either unshared (counts[hash] <= 1) or empty. -// -// This is the offline analogue of "how much of this request's prefix is -// safe to leave byte-identical" — the wire builder uses it to decide -// whether the UUID marker can be spliced in at the natural boundary -// (SharedPrefixLen > 0 and does not extend past the system blocks) or must -// fall back to tail injection (SharedPrefixLen == 0, or it extends into -// tools/messages, which would otherwise poison their cross-session cache -// key too). -func sharedPrefixBlockCount(req RouterReplayRequest, counts map[string]int) int { - hashes, _ := BuildReplayRequestPrefix(req) - n := 0 - for _, h := range hashes { - if h == "" || counts[h] <= 1 { - break - } - n++ - } - return n -} - -// replayReciteFirstLineInstruction returns the tail instruction asking the -// model to output, as the FIRST line of its response, every UUID injected -// at the session boundary — in order, comma-separated — then continue -// normally. Mirrors the cache-coherency eval's userMessage ("List every -// UUID shown in the request, in order, separated by commas. Output only the -// UUIDs and commas, nothing else.") but front-loads the ask to line 1 -// specifically (rather than the entire response), so a forced-output / -// ignore_eos budget can still fill the remainder with the model's normal -// continuation. Like the dataset-path instruction it replaces, this text -// never embeds the UUIDs themselves: first-line conformity in the response -// therefore reflects genuine recall from cached KV context, not an echo of -// the ask. -func replayReciteFirstLineInstruction() string { - return "\n\nBefore anything else, output on the FIRST line every UUID shown above in the request, in order, " + - "separated by commas, and nothing else on that first line. Then continue with your normal response." -} - -// buildSessionUUIDs returns len(perSessionChars) UUID sets, one per session, -// drawn in order (session-major, stamp-minor) from a single seeded -// generator — same determinism/disjointness rationale as the dataset path's -// buildReplayUUIDSets: same seed -> same per-session UUID assignment across -// runs and across every model in a multi-model run (see the precompute call -// site in RunAutoBenchmark, which populates cfg.replayUUIDSets once, before -// any per-model goroutine spawns, so every model sees the identical +// buildSessionTurnUUIDs returns, for each session, one UUID per turn +// (sets[i][t] = session i's turn-t UUID), drawn session-major/turn-minor +// from a single seeded generator (see newUUIDGenerator) — same determinism/ +// disjointness rationale as the dataset path's buildReplayUUIDSets: same +// seed -> same per-session-per-turn UUID assignment across runs and across +// every model in a multi-model run (see the precompute call site in +// RunAutoBenchmark, which populates cfg.replayUUIDSets once, before any +// per-model goroutine spawns, so every model sees the identical // assignment). // -// Session i's set size is N = computeStampsPerSeries(perSessionChars[i]) — -// the SAME sizing rule the cache-coherency eval uses to turn a garbage-char -// budget into a stamp count (min 2) — applied here to perSessionChars[i], -// session i's per-session cached-region byte size (see -// computePerSessionCachedChars): the blocks AFTER the cross-session-shared -// boundary that get reused, byte-identical, across every one of the -// session's own requests. A larger reused region gets more UUID stamps -// spread across it, mirroring the coherency test's -// garbageChars -> numStamps relationship. -func buildSessionUUIDs(perSessionChars []int, seed int64) [][]string { - if len(perSessionChars) == 0 { - return nil +// owner is the reverse mapping (uuid -> the owning session's index i) used +// by findLeakedUUIDsByOwner (replay_uuid.go) to flag cross-session +// contamination in O(response) time — a substring scan of the response +// plus map lookups, rather than iterating every session's UUID set. +func buildSessionTurnUUIDs(turnCounts []int, seed int64) (sets [][]string, owner map[string]int) { + owner = map[string]int{} + if len(turnCounts) == 0 { + return nil, owner } newUUID := newUUIDGenerator(seed) - sets := make([][]string, len(perSessionChars)) - for i, chars := range perSessionChars { - n := computeStampsPerSeries(chars) + sets = make([][]string, len(turnCounts)) + for i, n := range turnCounts { + if n <= 0 { + continue + } uuids := make([]string, n) for j := range uuids { - uuids[j] = newUUID() + u := newUUID() + uuids[j] = u + owner[u] = i } sets[i] = uuids } - return sets + return sets, owner +} + +// replayReciteWindowInstruction returns the tail instruction asking the +// model to output, as the FIRST line of its response, the id VALUES for the +// given ordered window of inline "[turn-N id: ...]" tags — in the same +// order, comma-separated — then continue normally. Unlike the retired +// per-session replayReciteFirstLineInstruction (which asked for "every UUID +// shown above" verbatim), this instruction names the exact turns to recite +// by LABEL, not position or value, so the model must locate each tagged +// turn in its own (possibly long) context rather than simply copying +// whatever happens to be nearby. Mirrors the cache-coherency eval's +// userMessage ("List every UUID shown in the request...") but (a) +// front-loads the ask to line 1 specifically, so a forced-output/ +// ignore_eos budget can still fill the remainder with the model's normal +// continuation, and (b) references labels instead of embedding the UUIDs +// themselves, so first-line conformity in the response reflects genuine +// recall from cached KV context, not an echo of the ask. +func replayReciteWindowInstruction(labels []string) string { + if len(labels) == 0 { + return "" + } + tagged := make([]string, len(labels)) + for i, l := range labels { + tagged[i] = "[" + l + "]" + } + return "\n\nSomewhere above, several ids are tagged like [turn-N id: ...]. On the FIRST line output ONLY the id values for these tags, " + + "in this exact order, comma-separated and nothing else: " + strings.Join(tagged, ", ") + ". Then continue normally." } // ---- max_tokens recite floor ---- @@ -413,13 +348,24 @@ const replayReciteFloorMultiplier = 3.0 // to fit the FIRST-LINE numUUIDs-UUID comma-joined list this feature asks // for (reuses the cache-coherency eval's computeMaxOutputTokens sizing: // numUUIDs*36 chars + separating commas, /4 for an approximate token count, -// x replayReciteFloorMultiplier). A router-replay request's max_tokens is -// normally sized off the ORIGINAL capture's output_tokens (see -// pickMaxTokens) — which for a tool-call-only turn can be a handful of -// tokens, nowhere near enough to also emit the first-line UUID list. Without -// this floor, a tiny budget truncates that first line, which would misread -// as PRESENCE_MISS/NOT_EXACT (coherency failure) when it's actually just an -// output-size artifact. +// x replayReciteFloorMultiplier). numUUIDs is now len(inj.ReciteUUIDs) — the +// current request's recite WINDOW, capped at 4 (see uuidInjection) — so the +// floor itself is now bounded and constant regardless of session length, +// unlike the retired per-session-N scheme where a long session's floor grew +// without bound. The tradeoff: for a request whose recorded output budget +// is tiny (a handful of tokens, e.g. a pure tool-call turn), this constant +// floor is a much LARGER ratio of the original budget than an N-scaled +// floor would have been at N=2 — i.e. the recite ask now perturbs a small +// turn's output-size profile proportionally more. Accepted: correctness +// (not truncating the recite line into a false PRESENCE_MISS) takes +// priority over preserving the exact captured output-size ratio. +// +// A router-replay request's max_tokens is normally sized off the ORIGINAL +// capture's output_tokens (see pickMaxTokens) — which for a tool-call-only +// turn can be a handful of tokens, nowhere near enough to also emit the +// first-line UUID list. Without this floor, a tiny budget truncates that +// first line, which would misread as PRESENCE_MISS/NOT_EXACT (coherency +// failure) when it's actually just an output-size artifact. func replayReciteFloorTokens(numUUIDs int) int { return computeMaxOutputTokens(numUUIDs, replayReciteFloorMultiplier) } diff --git a/benchmark/replay_router_uuid_test.go b/benchmark/replay_router_uuid_test.go index b30b801..12defbe 100644 --- a/benchmark/replay_router_uuid_test.go +++ b/benchmark/replay_router_uuid_test.go @@ -1,8 +1,10 @@ package benchmark // Pure, offline unit tests for router-replay UUID cache-coherency injection -// (replay_router_uuid.go). No mocked LLM/Chat — these exercise data -// transforms and the wire builders directly, per repo testing policy. +// (replay_router_uuid.go, replay_uuid.go, and the buildInjection/wire-body +// plumbing in replay_router_post.go/replay_router_wire.go). No mocked LLM/ +// Chat — these exercise data transforms and the wire builders directly, per +// repo testing policy. import ( "encoding/json" @@ -43,199 +45,222 @@ func writeReplayV3File(t *testing.T, sessions []RouterReplaySession) string { return path } -// syntheticSessionsForBoundaryTests builds 5 sessions exercising every -// sharedPrefixBlockCount regime: -// - sessions 0,1: share one system block (sys1) but each carries its own -// unique message -> leading run length 1, which equals the (single) -// emitted system block count -> "covers all system blocks" boundary case. -// - session 2: no block it carries is shared with any other session at -// all -> leading run length 0 -> tail-fallback case. -// - sessions 3,4: share BOTH their system block AND their message block -// (identical request shape) -> leading run length 2 == full prefix -// length -> "fully shared" edge case. -func syntheticSessionsForBoundaryTests() []RouterReplaySession { - mkSession := func(id string, sysHash string, msgHash string) RouterReplaySession { - return RouterReplaySession{ - SessionID: id, - Instances: []RouterReplayInstance{ - { - InstanceID: id + "-inst", - Requests: []RouterReplayRequest{ - { - RequestID: 1, - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{ - {Hash: sysHash, Bytes: 250, Tokens: 60}, - }, - Messages: []RouterReplayMessage{ - {Hash: msgHash, Role: "user", BlockTypes: []string{"text"}, Bytes: 100, Tokens: 25}, - }, - }, - }, - }, - }, - } - } - return []RouterReplaySession{ - mkSession("s0", "sys1", "msgUniq0"), - mkSession("s1", "sys1", "msgUniq1"), - mkSession("s2", "sys2only", "msgUniq2"), - mkSession("s3", "sysShared34", "msgShared34"), - mkSession("s4", "sysShared34", "msgShared34"), - } +// userText is a shorthand for a qualifying user-turn message (role=="user", +// one "text" block). +func userText(hash string, bytes int) RouterReplayMessage { + return RouterReplayMessage{Role: "user", Hash: hash, BlockTypes: []string{"text"}, Bytes: bytes} } -func TestComputeBlockSessionCounts(t *testing.T) { - path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) +// assistantText is a shorthand for an assistant message — never a +// qualifying turn (wrong role), included in fixtures purely to exercise +// that non-user messages are skipped. +func assistantText(hash string, bytes int) RouterReplayMessage { + return RouterReplayMessage{Role: "assistant", Hash: hash, BlockTypes: []string{"text"}, Bytes: bytes} +} - counts, err := computeBlockSessionCounts(path, nil, 0) - if err != nil { - t.Fatalf("computeBlockSessionCounts: %v", err) +// toolResultOnly is a shorthand for a role=="user" message carrying ONLY a +// tool_result block (no text) — the "exclude tool_result-only" case from +// isQualifyingUserTurn. +func toolResultOnly(hash string, bytes int) RouterReplayMessage { + return RouterReplayMessage{Role: "user", Hash: hash, BlockTypes: []string{"tool_result"}, Bytes: bytes, ToolResultIDs: []string{"tr1"}} +} + +// turnFixtureSessions builds two sessions exercising every +// isQualifyingUserTurn exclusion plus multi-instance turn ordering: +// +// - both sessions' first request opens with the SAME message hash +// ("shared-msg", a role=="user" text message) — shared across sessions +// (count==2) so it must NEVER qualify as a turn despite being +// role==user with a text block. +// - each session's main instance accumulates 3 genuine user turns +// (u1,u2,u3) interleaved with assistant replies (never turns) and, for +// session 0 only, a tool_result-only message (never a turn, even +// though role=="user"). +// - session 0 has a SECOND instance ("s0-sub") appearing AFTER the main +// instance in the Instances slice, contributing one more turn +// (s0-sub-u1) — verifies turn indices are session-global and ordered +// by instance/request/message file order, not per-instance. +func turnFixtureSessions() []RouterReplaySession { + mkMainInstance := func(id, prefix string, includeToolResultOnly bool) RouterReplayInstance { + req1 := RouterReplayRequest{ + RequestID: 1, + Messages: []RouterReplayMessage{userText("shared-msg", 60)}, + } + req2Msgs := []RouterReplayMessage{ + userText("shared-msg", 60), + assistantText(prefix+"-a1", 80), + userText(prefix+"-u1", 100), + } + req2 := RouterReplayRequest{RequestID: 2, Messages: req2Msgs} + + req3Msgs := append(append([]RouterReplayMessage{}, req2Msgs...), + assistantText(prefix+"-a2", 80), + userText(prefix+"-u2", 100), + ) + req3 := RouterReplayRequest{RequestID: 3, Messages: req3Msgs} + + req4Msgs := append([]RouterReplayMessage{}, req3Msgs...) + if includeToolResultOnly { + req4Msgs = append(req4Msgs, toolResultOnly(prefix+"-tool1", 40)) + } + req4 := RouterReplayRequest{RequestID: 4, Messages: req4Msgs} + + req5Msgs := append(append([]RouterReplayMessage{}, req4Msgs...), + assistantText(prefix+"-a3", 80), + userText(prefix+"-u3", 100), + ) + req5 := RouterReplayRequest{RequestID: 5, Messages: req5Msgs} + + return RouterReplayInstance{ + InstanceID: id, + Role: "main", + Requests: []RouterReplayRequest{req1, req2, req3, req4, req5}, + } + } + + subInstance := RouterReplayInstance{ + InstanceID: "s0-sub", + Role: "sub-agent", + Requests: []RouterReplayRequest{ + {RequestID: 6, Messages: []RouterReplayMessage{userText("s0-sub-u1", 90)}}, + }, } - cases := []struct { - hash string - want int - }{ - {"sys1", 2}, // sessions 0 and 1 - {"msgUniq0", 1}, // only session 0 - {"msgUniq1", 1}, // only session 1 - {"sys2only", 1}, // only session 2 - {"msgUniq2", 1}, // only session 2 - {"sysShared34", 2}, // sessions 3 and 4 - {"msgShared34", 2}, // sessions 3 and 4 - {"never-appears", 0}, // absent hash + s0 := RouterReplaySession{ + SessionID: "s0", + Instances: []RouterReplayInstance{ + mkMainInstance("s0-main", "s0", true), + subInstance, + }, } - for _, c := range cases { - if got := counts[c.hash]; got != c.want { - t.Errorf("counts[%q] = %d, want %d", c.hash, got, c.want) - } + s1 := RouterReplaySession{ + SessionID: "s1", + Instances: []RouterReplayInstance{ + mkMainInstance("s1-main", "s1", false), + }, } + return []RouterReplaySession{s0, s1} } -func TestComputeBlockSessionCountsRespectsFilters(t *testing.T) { - path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) - - t.Run("sessionLimit caps to the first N sessions", func(t *testing.T) { - // sessionLimit=2 -> only s0, s1 counted; sys1 still shared (2), but - // session 2/3/4 hashes never appear. - counts, err := computeBlockSessionCounts(path, nil, 2) - if err != nil { - t.Fatalf("computeBlockSessionCounts: %v", err) - } - if got := counts["sys1"]; got != 2 { - t.Errorf("sys1 = %d, want 2", got) - } - if got := counts["sysShared34"]; got != 0 { - t.Errorf("sysShared34 = %d, want 0 (beyond sessionLimit)", got) - } - }) - - t.Run("allowed index set restricts to those sessions only", func(t *testing.T) { - // Only session index 3 and 4 (0-based) allowed -> sysShared34 still - // shows count 2, but sys1 (sessions 0,1) never counted. - allowed := map[int]bool{3: true, 4: true} - counts, err := computeBlockSessionCounts(path, allowed, 0) - if err != nil { - t.Fatalf("computeBlockSessionCounts: %v", err) - } - if got := counts["sysShared34"]; got != 2 { - t.Errorf("sysShared34 = %d, want 2", got) - } - if got := counts["sys1"]; got != 0 { - t.Errorf("sys1 = %d, want 0 (session 0/1 excluded)", got) - } - }) -} - -func TestSharedPrefixBlockCount(t *testing.T) { - path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) +func TestComputeSessionTurnHashesIdentifiesQualifyingTurns(t *testing.T) { + path := writeReplayV3File(t, turnFixtureSessions()) counts, err := computeBlockSessionCounts(path, nil, 0) if err != nil { t.Fatalf("computeBlockSessionCounts: %v", err) } + if got := counts["shared-msg"]; got != 2 { + t.Fatalf("counts[shared-msg] = %d, want 2 (referenced by both sessions)", got) + } - reqBoundary := RouterReplayRequest{ - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msgUniq0", Role: "user", Bytes: 100}}, + turnHashes, err := computeSessionTurnHashes(path, nil, 0, counts) + if err != nil { + t.Fatalf("computeSessionTurnHashes: %v", err) } - if got := sharedPrefixBlockCount(reqBoundary, counts); got != 1 { - t.Errorf("boundary case: sharedPrefixBlockCount = %d, want 1 (covers the single system block)", got) + if len(turnHashes) != 2 { + t.Fatalf("len(turnHashes) = %d, want 2 sessions", len(turnHashes)) } - reqNoShare := RouterReplayRequest{ - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys2only", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msgUniq2", Role: "user", Bytes: 100}}, + wantS0 := []string{"s0-u1", "s0-u2", "s0-u3", "s0-sub-u1"} + if !equalStrSlices(turnHashes[0], wantS0) { + t.Errorf("session 0 turnHashes = %v, want %v", turnHashes[0], wantS0) } - if got := sharedPrefixBlockCount(reqNoShare, counts); got != 0 { - t.Errorf("no-shared-block case: sharedPrefixBlockCount = %d, want 0 (tail fallback)", got) + wantS1 := []string{"s1-u1", "s1-u2", "s1-u3"} + if !equalStrSlices(turnHashes[1], wantS1) { + t.Errorf("session 1 turnHashes = %v, want %v", turnHashes[1], wantS1) } - reqFullyShared := RouterReplayRequest{ - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sysShared34", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msgShared34", Role: "user", Bytes: 100}}, - } - if got, want := sharedPrefixBlockCount(reqFullyShared, counts), 2; got != want { - t.Errorf("fully-shared case: sharedPrefixBlockCount = %d, want %d (full prefix length)", got, want) + for _, session := range turnHashes { + for _, h := range session { + if h == "shared-msg" { + t.Error("shared-msg (count==2) qualified as a turn — cross-session-shared content must never be stamped") + } + if strings.Contains(h, "tool1") { + t.Errorf("tool_result-only hash %q qualified as a turn", h) + } + if strings.Contains(h, "-a") { + t.Errorf("assistant hash %q qualified as a turn", h) + } + } } } -// TestComputePerSessionCachedChars verifies the per-session cached-region -// byte size computation: the ROOT request's (first instance, first -// request) prefix bytes AT OR AFTER its sharedPrefixBlockCount boundary, -// using the same synthetic 5-session fixture TestSharedPrefixBlockCount -// exercises (sys=250 bytes, msg=100 bytes each request). -func TestComputePerSessionCachedChars(t *testing.T) { - path := writeReplayV3File(t, syntheticSessionsForBoundaryTests()) +func TestComputeSessionTurnHashesRespectsFilters(t *testing.T) { + path := writeReplayV3File(t, turnFixtureSessions()) counts, err := computeBlockSessionCounts(path, nil, 0) if err != nil { t.Fatalf("computeBlockSessionCounts: %v", err) } - got, err := computePerSessionCachedChars(path, nil, 0, counts) - if err != nil { - t.Fatalf("computePerSessionCachedChars: %v", err) - } - // s0,s1: boundary=1 (only the shared sys1 block) -> chars = msg bytes (100). - // s2: boundary=0 (nothing shared) -> chars = sys(250) + msg(100) = 350. - // s3,s4: boundary=2 (both blocks shared, full prefix) -> chars = 0. - want := []int{100, 100, 350, 0, 0} - if len(got) != len(want) { - t.Fatalf("computePerSessionCachedChars len = %d, want %d (got %v)", len(got), len(want), got) - } - for i, w := range want { - if got[i] != w { - t.Errorf("session %d: chars = %d, want %d", i, got[i], w) + t.Run("sessionLimit caps to the first N sessions", func(t *testing.T) { + turnHashes, err := computeSessionTurnHashes(path, nil, 1, counts) + if err != nil { + t.Fatalf("computeSessionTurnHashes: %v", err) } - } + if len(turnHashes) != 1 { + t.Fatalf("len(turnHashes) = %d, want 1 (sessionLimit=1)", len(turnHashes)) + } + }) + + t.Run("allowed index set restricts to those sessions only", func(t *testing.T) { + // counts computed globally (both sessions), but the turn-hash pass + // only walks session index 1 (s1). + allowed := map[int]bool{1: true} + turnHashes, err := computeSessionTurnHashes(path, allowed, 0, counts) + if err != nil { + t.Fatalf("computeSessionTurnHashes: %v", err) + } + if len(turnHashes) != 1 { + t.Fatalf("len(turnHashes) = %d, want 1 (only index 1 allowed)", len(turnHashes)) + } + want := []string{"s1-u1", "s1-u2", "s1-u3"} + if !equalStrSlices(turnHashes[0], want) { + t.Errorf("turnHashes[0] = %v, want %v (session s1)", turnHashes[0], want) + } + }) } -// TestBuildSessionUUIDsDeterminism verifies buildSessionUUIDs matches the -// dataset path's determinism contract: same seed -> same per-session UUID -// assignment; different seed -> different assignment; every UUID unique -// across the whole run (not just within a session). -func TestBuildSessionUUIDsDeterminism(t *testing.T) { - perSessionChars := []int{0, 0, 0, 0, 0} // all -> computeStampsPerSeries floors to 2 - a := buildSessionUUIDs(perSessionChars, 42) - b := buildSessionUUIDs(perSessionChars, 42) - if len(a) != 5 || len(b) != 5 { - t.Fatalf("expected 5 sets each, got %d and %d", len(a), len(b)) +func equalStrSlices(a, b []string) bool { + if len(a) != len(b) { + return false } for i := range a { - if len(a[i]) != 2 || len(b[i]) != 2 { - t.Fatalf("session %d: expected 2-UUID sets (min floor), got %v / %v", i, a[i], b[i]) + if a[i] != b[i] { + return false } - for j := range a[i] { - if a[i][j] != b[i][j] { - t.Errorf("session %d stamp %d: same seed produced different UUIDs: %q vs %q", i, j, a[i][j], b[i][j]) + } + return true +} + +// TestBuildSessionTurnUUIDsDeterminism verifies buildSessionTurnUUIDs +// matches the dataset path's determinism contract: same seed -> same +// per-session-per-turn UUID assignment; different seed -> different +// assignment; every UUID unique across the whole run; owner correctly +// reverse-maps each UUID to its issuing session. +func TestBuildSessionTurnUUIDsDeterminism(t *testing.T) { + turnCounts := []int{3, 2, 4} + setsA, ownerA := buildSessionTurnUUIDs(turnCounts, 42) + setsB, ownerB := buildSessionTurnUUIDs(turnCounts, 42) + if len(setsA) != 3 || len(setsB) != 3 { + t.Fatalf("expected 3 sets each, got %d and %d", len(setsA), len(setsB)) + } + for i := range setsA { + if len(setsA[i]) != turnCounts[i] || len(setsB[i]) != turnCounts[i] { + t.Fatalf("session %d: expected %d UUIDs, got %v / %v", i, turnCounts[i], setsA[i], setsB[i]) + } + for j := range setsA[i] { + if setsA[i][j] != setsB[i][j] { + t.Errorf("session %d turn %d: same seed produced different UUIDs: %q vs %q", i, j, setsA[i][j], setsB[i][j]) } } } + if len(ownerA) != len(ownerB) { + t.Errorf("owner map sizes differ: %d vs %d", len(ownerA), len(ownerB)) + } - c := buildSessionUUIDs(perSessionChars, 43) + setsC, _ := buildSessionTurnUUIDs(turnCounts, 43) same := true - for i := range a { - if a[i][0] != c[i][0] { + for i := range setsA { + if len(setsA[i]) > 0 && len(setsC[i]) > 0 && setsA[i][0] != setsC[i][0] { same = false } } @@ -244,57 +269,212 @@ func TestBuildSessionUUIDsDeterminism(t *testing.T) { } seen := map[string]bool{} - for _, set := range a { + for i, set := range setsA { for _, u := range set { if seen[u] { - t.Errorf("uuid %q assigned to more than one stamp", u) + t.Errorf("uuid %q assigned to more than one turn", u) } seen[u] = true + if got := ownerA[u]; got != i { + t.Errorf("owner[%q] = %d, want %d", u, got, i) + } } } - if got := buildSessionUUIDs(nil, 42); got != nil { - t.Errorf("buildSessionUUIDs(nil, ...) = %v, want nil", got) + if sets, owner := buildSessionTurnUUIDs(nil, 42); sets != nil || len(owner) != 0 { + t.Errorf("buildSessionTurnUUIDs(nil, ...) = (%v, %v), want (nil, empty)", sets, owner) } } -// TestBuildSessionUUIDsScalesWithBytes verifies each session's N is exactly -// computeStampsPerSeries(perSessionChars[i]) -- min 2, scaling with bytes -- -// mirroring the cache-coherency eval's garbageChars -> numStamps rule. -func TestBuildSessionUUIDsScalesWithBytes(t *testing.T) { - perSessionChars := []int{0, 8192 * 5, 8192*10 + 100} - sets := buildSessionUUIDs(perSessionChars, 7) - want := []int{2, 5, 10} - for i, w := range want { - if got := len(sets[i]); got != w { - t.Errorf("session %d: len = %d, want %d (computeStampsPerSeries(%d))", i, got, w, perSessionChars[i]) - } +// TestBuildSessionTurnUUIDsScalesWithTurnCount verifies each session gets +// EXACTLY turnCounts[i] UUIDs (one per turn, no floor/multiplier — unlike +// the retired per-session-N scheme) and a zero-turn session gets none. +func TestBuildSessionTurnUUIDsScalesWithTurnCount(t *testing.T) { + turnCounts := []int{0, 1, 5} + sets, owner := buildSessionTurnUUIDs(turnCounts, 7) + if len(sets[0]) != 0 { + t.Errorf("session 0 (0 turns): len = %d, want 0", len(sets[0])) + } + if len(sets[1]) != 1 { + t.Errorf("session 1 (1 turn): len = %d, want 1", len(sets[1])) + } + if len(sets[2]) != 5 { + t.Errorf("session 2 (5 turns): len = %d, want 5", len(sets[2])) + } + if len(owner) != 6 { + t.Errorf("len(owner) = %d, want 6 (1+5 issued UUIDs)", len(owner)) + } +} + +// newFixturePoster builds a replayPoster wired up exactly as +// runRouterReplayInstance does (see replay_router.go), for a single session +// with nTurns turns, without touching HTTP/newReplayPoster. +func newFixturePoster(nTurns int, seed int64, reciteEveryRequest bool) *replayPoster { + turnHashes := make([]string, nTurns) + for i := range turnHashes { + turnHashes[i] = fmt.Sprintf("h%d", i) + } + sets, owner := buildSessionTurnUUIDs([]int{nTurns}, seed) + hashToTurn := make(map[string]int, nTurns) + for i, h := range turnHashes { + hashToTurn[h] = i + } + return &replayPoster{ + uuidEnabled: true, + sessionIdx: 0, + allUUIDSets: sets, + turnHashes: turnHashes, + hashToTurn: hashToTurn, + owner: owner, + reciteEveryRequest: reciteEveryRequest, + } +} + +// visibleTurnsRequest builds a RouterReplayRequest whose Messages carry the +// qualifying-turn hashes h0..h(n-1) (in order), simulating a growing +// conversation history at turn n. +func visibleTurnsRequest(n int) RouterReplayRequest { + msgs := make([]RouterReplayMessage, 0, n) + for i := 0; i < n; i++ { + msgs = append(msgs, userText(fmt.Sprintf("h%d", i), 50)) + } + return RouterReplayRequest{Messages: msgs} +} + +// TestBuildInjectionWindowSelection verifies buildInjection's recite window: +// first (visible) turn + up to 3 most-recent turns EXCLUDING the current +// (highest-index visible) turn, deduplicated, capped at 4 — and that +// StampByHash always covers EVERY visible turn, not just the window. +// Exercises the edge cases at turns 1, 2, 3 explicitly (D1/D2/D3 in the +// plan) plus the steady-state 4-cap and window-sliding behavior beyond it. +func TestBuildInjectionWindowSelection(t *testing.T) { + p := newFixturePoster(6, 1, true) + + cases := []struct { + turn int // 1-based "current turn" being requested + wantLabels []string // expected inj.ReciteLabels + }{ + {1, []string{"turn-1"}}, + {2, []string{"turn-1"}}, + {3, []string{"turn-1", "turn-2"}}, + {4, []string{"turn-1", "turn-2", "turn-3"}}, + {5, []string{"turn-1", "turn-2", "turn-3", "turn-4"}}, + {6, []string{"turn-1", "turn-3", "turn-4", "turn-5"}}, + } + for _, c := range cases { + t.Run(fmt.Sprintf("turn-%d", c.turn), func(t *testing.T) { + req := visibleTurnsRequest(c.turn) + inj := p.buildInjection(req, false) + if inj == nil { + t.Fatal("buildInjection returned nil, want a non-nil injection") + } + if !equalStrSlices(inj.ReciteLabels, c.wantLabels) { + t.Errorf("ReciteLabels = %v, want %v", inj.ReciteLabels, c.wantLabels) + } + if len(inj.ReciteLabels) > 4 { + t.Errorf("ReciteLabels len = %d, want <= 4", len(inj.ReciteLabels)) + } + if len(inj.ReciteUUIDs) != len(inj.ReciteLabels) { + t.Fatalf("ReciteUUIDs len = %d, want %d (matching ReciteLabels)", len(inj.ReciteUUIDs), len(inj.ReciteLabels)) + } + // Every UUID in the window must be this session's own (index 0). + for i, u := range inj.ReciteUUIDs { + if p.owner[u] != 0 { + t.Errorf("ReciteUUIDs[%d] = %q owned by session %d, want session 0", i, u, p.owner[u]) + } + } + // StampByHash must cover EVERY visible turn (h0..h(turn-1)), not + // just the recite window — this is what keeps every turn's + // marker warm in KV regardless of whether it's being recited + // this request. + if len(inj.StampByHash) != c.turn { + t.Errorf("StampByHash covers %d turns, want %d (every visible turn)", len(inj.StampByHash), c.turn) + } + for i := 0; i < c.turn; i++ { + h := fmt.Sprintf("h%d", i) + stamp, ok := inj.StampByHash[h] + if !ok { + t.Errorf("StampByHash missing visible turn hash %q", h) + continue + } + if stamp.Idx != i { + t.Errorf("StampByHash[%q].Idx = %d, want %d", h, stamp.Idx, i) + } + if stamp.Label != fmt.Sprintf("turn-%d", i+1) { + t.Errorf("StampByHash[%q].Label = %q, want turn-%d", h, stamp.Label, i+1) + } + } + }) } - for _, chars := range []int{0, 1, 8192, 8192 * 3, 100000} { - got := len(buildSessionUUIDs([]int{chars}, 1)[0]) - want := computeStampsPerSeries(chars) - if got != want { - t.Errorf("chars=%d: N = %d, want %d (computeStampsPerSeries)", chars, got, want) +} + +// TestBuildInjectionNilCases verifies buildInjection degrades to "no +// injection" (nil) rather than panicking: disabled, session index out of +// range, session with zero turns, and a request with no visible qualifying +// turn at all. +func TestBuildInjectionNilCases(t *testing.T) { + t.Run("uuidEnabled false", func(t *testing.T) { + p := newFixturePoster(3, 1, true) + p.uuidEnabled = false + if got := p.buildInjection(visibleTurnsRequest(2), false); got != nil { + t.Errorf("buildInjection = %v, want nil (disabled)", got) + } + }) + t.Run("sessionIdx out of range", func(t *testing.T) { + p := newFixturePoster(3, 1, true) + p.sessionIdx = 5 + if got := p.buildInjection(visibleTurnsRequest(2), false); got != nil { + t.Errorf("buildInjection = %v, want nil (sessionIdx out of range)", got) } + }) + t.Run("zero-turn session", func(t *testing.T) { + p := newFixturePoster(0, 1, true) + if got := p.buildInjection(visibleTurnsRequest(0), false); got != nil { + t.Errorf("buildInjection = %v, want nil (no turns)", got) + } + }) + t.Run("no qualifying turn visible in this request", func(t *testing.T) { + p := newFixturePoster(3, 1, true) + req := RouterReplayRequest{Messages: []RouterReplayMessage{assistantText("unrelated", 30)}} + if got := p.buildInjection(req, false); got != nil { + t.Errorf("buildInjection = %v, want nil (nothing visible)", got) + } + }) +} + +// TestBuildInjectionRecite verifies Recite = reciteEveryRequest || +// isLastRequest, independent of window selection. +func TestBuildInjectionRecite(t *testing.T) { + req := visibleTurnsRequest(2) + + pAlways := newFixturePoster(3, 1, true) + if inj := pAlways.buildInjection(req, false); inj == nil || !inj.Recite { + t.Error("reciteEveryRequest=true, isLastRequest=false: expected Recite=true") + } + + pFinalOnly := newFixturePoster(3, 1, false) + if inj := pFinalOnly.buildInjection(req, false); inj == nil || inj.Recite { + t.Error("reciteEveryRequest=false, isLastRequest=false: expected Recite=false") + } + if inj := pFinalOnly.buildInjection(req, true); inj == nil || !inj.Recite { + t.Error("reciteEveryRequest=false, isLastRequest=true: expected Recite=true") } } -// TestWireInjectionDeterminism verifies that, for a fixed session's N-UUID -// block, buildOpenAIChatCompletionsBody / buildAnthropicMessagesBody produce -// byte-identical bodies across repeated calls (same request + same -// injection in -> same bytes out), and that two DIFFERENT sessions' UUID -// blocks diverge the body. +// TestWireInjectionDeterminism verifies that, for a fixed set of turn +// stamps, buildOpenAIChatCompletionsBody / buildAnthropicMessagesBody +// produce byte-identical bodies across repeated calls (same request + same +// injection in -> same bytes out) and across two DIFFERENT requests that +// both carry the SAME turn message (the within-session cache-reuse +// property), while two DIFFERENT sessions' turn stamps diverge the body. func TestWireInjectionDeterminism(t *testing.T) { docs := strings.Repeat("wire-injection-docs ", 100) + stampA := map[string]turnStamp{"msg1": {Idx: 0, UUID: "uuid-session-A", Label: "turn-1"}} + stampB := map[string]turnStamp{"msg1": {Idx: 0, UUID: "uuid-session-B", Label: "turn-1"}} req := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msgUniq0", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + InputTokens: 500, + Messages: []RouterReplayMessage{userText("msg1", 100)}, } - sets := buildSessionUUIDs([]int{0, 0}, 7) - injA := &uuidInjection{UUIDs: sets[0], Recite: true, SharedPrefixLen: 1} - injA2 := &uuidInjection{UUIDs: sets[0], Recite: true, SharedPrefixLen: 1} - injB := &uuidInjection{UUIDs: sets[1], Recite: true, SharedPrefixLen: 1} for _, kind := range []string{"openai", "anthropic"} { build := func(r RouterReplayRequest, inj *uuidInjection) []byte { @@ -310,6 +490,10 @@ func TestWireInjectionDeterminism(t *testing.T) { } return body } + injA := &uuidInjection{StampByHash: stampA} + injA2 := &uuidInjection{StampByHash: stampA} + injB := &uuidInjection{StampByHash: stampB} + bodyA1 := build(req, injA) bodyA2 := build(req, injA2) bodyB := build(req, injB) @@ -318,366 +502,210 @@ func TestWireInjectionDeterminism(t *testing.T) { t.Errorf("%s: identical injection produced different bytes", kind) } if string(bodyA1) == string(bodyB) { - t.Errorf("%s: different sessions' UUID blocks produced identical bytes", kind) + t.Errorf("%s: different sessions' turn stamps produced identical bytes", kind) } - for _, u := range sets[0] { - if !strings.Contains(string(bodyA1), u) { - t.Errorf("%s: body missing session A's own UUID %q", kind, u) - } + if !strings.Contains(string(bodyA1), "uuid-session-A") { + t.Errorf("%s: body missing session A's own turn UUID", kind) } - for _, u := range sets[1] { - if strings.Contains(string(bodyA1), u) { - t.Errorf("%s: body A leaked session B's UUID %q into the wire body", kind, u) - } + if strings.Contains(string(bodyA1), "uuid-session-B") { + t.Errorf("%s: body A leaked session B's turn UUID into the wire body", kind) + } + + // A SECOND, different request that repeats the SAME turn message + // (msg1) must emit the byte-identical stamped content for it. + req2 := RouterReplayRequest{ + InputTokens: 500, + Messages: []RouterReplayMessage{ + userText("msg1", 100), + assistantText("msg2", 60), + }, + } + bodyA3 := build(req2, injA) + var p1, p3 map[string]interface{} + if err := json.Unmarshal(bodyA1, &p1); err != nil { + t.Fatalf("unmarshal bodyA1: %v", err) } - if strings.Contains(string(bodyA1), "ref-id") { - t.Errorf("%s: injected block still carries the old [ref-id: ...] wrapper", kind) + if err := json.Unmarshal(bodyA3, &p3); err != nil { + t.Fatalf("unmarshal bodyA3: %v", err) + } + msgs1, _ := p1["messages"].([]interface{}) + msgs3, _ := p3["messages"].([]interface{}) + if len(msgs1) == 0 || len(msgs3) == 0 { + t.Fatalf("%s: expected non-empty messages in both bodies", kind) + } + first1, _ := json.Marshal(msgs1[0]) + first3, _ := json.Marshal(msgs3[0]) + if string(first1) != string(first3) { + t.Errorf("%s: turn msg1's stamped content diverged across two different requests:\n%s\nvs\n%s", kind, first1, first3) } } } -// TestBoundaryInjectionEmitsBareSpaceSeparatedUUIDs verifies the injected -// block is exactly N bare, space-separated UUIDs (no wrapper text) — -// mirroring the cache-coherency eval's buildCoherencySharedSeriesPrompt tail -// — and that it is byte-identical across two DIFFERENT requests belonging -// to the SAME session (the within-session cache-reuse property), while the -// cross-session-shared leading system block stays untouched. -func TestBoundaryInjectionEmitsBareSpaceSeparatedUUIDs(t *testing.T) { - docs := strings.Repeat("boundary-multi-docs ", 100) - uuids := []string{"uuid-0", "uuid-1", "uuid-2"} - inj := &uuidInjection{UUIDs: uuids, Recite: false, SharedPrefixLen: 1} - - req1 := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msgTurn1", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, - } - req2 := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, // same shared system block - Messages: []RouterReplayMessage{{Hash: "msgTurn2", Role: "user", BlockTypes: []string{"text"}, Bytes: 140}}, +// TestInlineMarkerAppendedToTurnMessage verifies the injected marker's exact +// shape ("\n\n[turn-N id: ]") lands inside the stamped message's OWN +// text content (both wire body and canonical text), and that an unrelated +// message in the same request is left untouched. +func TestInlineMarkerAppendedToTurnMessage(t *testing.T) { + docs := strings.Repeat("inline-marker-docs ", 100) + stampByHash := map[string]turnStamp{"stamped-msg": {Idx: 3, UUID: "abc-uuid", Label: "turn-4"}} + inj := &uuidInjection{StampByHash: stampByHash} + req := RouterReplayRequest{ + InputTokens: 500, + Messages: []RouterReplayMessage{ + userText("stamped-msg", 100), + assistantText("other-msg", 80), + }, } + wantMarker := "\n\n[turn-4 id: abc-uuid]" - body1, _, err := buildAnthropicMessagesBody(req1, docs, "model", "", 0, false, inj) - if err != nil { - t.Fatalf("build 1: %v", err) - } - body2, _, err := buildAnthropicMessagesBody(req2, docs, "model", "", 0, false, inj) + body, canonical, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) if err != nil { - t.Fatalf("build 2: %v", err) - } - - var parsed1, parsed2 map[string]interface{} - if err := json.Unmarshal(body1, &parsed1); err != nil { - t.Fatalf("unmarshal 1: %v", err) - } - if err := json.Unmarshal(body2, &parsed2); err != nil { - t.Fatalf("unmarshal 2: %v", err) - } - - sys1, _ := parsed1["system"].([]interface{}) - sys2, _ := parsed2["system"].([]interface{}) - if len(sys1) != 2 || len(sys2) != 2 { - t.Fatalf("expected system = [shared block, uuid block], got lens %d and %d", len(sys1), len(sys2)) + t.Fatalf("build: %v", err) } - - wantText := "uuid-0 uuid-1 uuid-2" - block1 := sys1[1].(map[string]interface{}) - block2 := sys2[1].(map[string]interface{}) - if block1["text"] != wantText { - t.Errorf("uuid block 1 text = %q, want %q (bare, space-separated)", block1["text"], wantText) + if !strings.Contains(canonical, wantMarker) { + t.Errorf("canonical text missing marker %q", wantMarker) } - if block2["text"] != wantText { - t.Errorf("uuid block diverged across two requests in the SAME session: %v vs %v", block2["text"], wantText) + var parsed map[string]interface{} + if err := json.Unmarshal(body, &parsed); err != nil { + t.Fatalf("unmarshal: %v", err) } - - // The shared leading system block (index 0) must stay byte-identical — - // injection must never perturb the cross-session-shared prefix. - shared1, _ := json.Marshal(sys1[0]) - shared2, _ := json.Marshal(sys2[0]) - if string(shared1) != string(shared2) { - t.Errorf("shared leading system block diverged across requests: %s vs %s", shared1, shared2) + msgs, _ := parsed["messages"].([]interface{}) + if len(msgs) != 2 { + t.Fatalf("expected 2 messages, got %d", len(msgs)) + } + // Compare against the UNMARSHALED content (fmt.Sprintf, not + // json.Marshal) so a "\n" in the marker is a real newline byte, not the + // two-character JSON escape sequence "\\n". + first, _ := msgs[0].(map[string]interface{}) + firstContent := fmt.Sprintf("%v", first["content"]) + if !strings.Contains(firstContent, wantMarker) { + t.Errorf("stamped message content missing marker: %s", firstContent) + } + second, _ := msgs[1].(map[string]interface{}) + secondContent := fmt.Sprintf("%v", second["content"]) + if strings.Contains(secondContent, "abc-uuid") { + t.Errorf("marker leaked into the unrelated (unstamped) message: %s", secondContent) } } -// TestCacheFidelityBoundaryInvariant verifies the core Option-C guarantee: -// two DIFFERENT sessions that share a leading system block emit -// byte-identical content for that shared block, diverging only at (or -// after) the injected per-session UUID block — i.e. injection never -// perturbs the cross-session-shared prefix a real server would -// prefix-cache on. -func TestCacheFidelityBoundaryInvariant(t *testing.T) { +// TestCacheFidelitySharedBlockUnaffectedByInjection verifies the core +// fidelity invariant: a message hash that is NOT a key in StampByHash +// (standing in for a cross-session-shared, count>1 block — see +// isQualifyingUserTurn, which excludes those from ever being stamped) emits +// byte-identical content whether or not UUID injection is active elsewhere +// in the SAME request. +func TestCacheFidelitySharedBlockUnaffectedByInjection(t *testing.T) { docs := strings.Repeat("fidelity-docs ", 100) - reqA := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msgUniq-A", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, - } - reqB := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, // SAME shared system block - Messages: []RouterReplayMessage{{Hash: "msgUniq-B", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + req := RouterReplayRequest{ + InputTokens: 500, + Messages: []RouterReplayMessage{ + userText("shared-unstamped-msg", 100), // stands in for a count>1 block + userText("own-msg", 80), + }, } - injA := &uuidInjection{UUIDs: []string{"uuid-session-A"}, Recite: false, SharedPrefixLen: 1} - injB := &uuidInjection{UUIDs: []string{"uuid-session-B"}, Recite: false, SharedPrefixLen: 1} - bodyA, _, err := buildAnthropicMessagesBody(reqA, docs, "model", "", 0, false, injA) + bodyNoInj, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, nil) if err != nil { - t.Fatalf("build A: %v", err) + t.Fatalf("build (no injection): %v", err) } - bodyB, _, err := buildAnthropicMessagesBody(reqB, docs, "model", "", 0, false, injB) + inj := &uuidInjection{StampByHash: map[string]turnStamp{"own-msg": {Idx: 0, UUID: "own-uuid", Label: "turn-1"}}} + bodyWithInj, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) if err != nil { - t.Fatalf("build B: %v", err) - } - - var parsedA, parsedB map[string]interface{} - if err := json.Unmarshal(bodyA, &parsedA); err != nil { - t.Fatalf("unmarshal A: %v", err) - } - if err := json.Unmarshal(bodyB, &parsedB); err != nil { - t.Fatalf("unmarshal B: %v", err) + t.Fatalf("build (with injection): %v", err) } - sysA, _ := parsedA["system"].([]interface{}) - sysB, _ := parsedB["system"].([]interface{}) - if len(sysA) != 2 || len(sysB) != 2 { - t.Fatalf("expected system = [shared block, uuid block], got lens %d and %d", len(sysA), len(sysB)) - } - // Index 0 (the shared system block, "sys1") must be byte-identical. - sharedA, _ := json.Marshal(sysA[0]) - sharedB, _ := json.Marshal(sysB[0]) - if string(sharedA) != string(sharedB) { - t.Errorf("shared leading system block diverged between sessions:\nA: %s\nB: %s", sharedA, sharedB) + var parsedNoInj, parsedWithInj map[string]interface{} + if err := json.Unmarshal(bodyNoInj, &parsedNoInj); err != nil { + t.Fatalf("unmarshal (no injection): %v", err) } - // Index 1 (the injected uuid block) MUST diverge — that's the whole point. - blockA, _ := json.Marshal(sysA[1]) - blockB, _ := json.Marshal(sysB[1]) - if string(blockA) == string(blockB) { - t.Error("injected uuid blocks were identical across two different sessions") + if err := json.Unmarshal(bodyWithInj, &parsedWithInj); err != nil { + t.Fatalf("unmarshal (with injection): %v", err) } - if !strings.Contains(string(blockA), "uuid-session-A") { - t.Errorf("session A's block missing its own uuid: %s", blockA) + msgsNoInj, _ := parsedNoInj["messages"].([]interface{}) + msgsWithInj, _ := parsedWithInj["messages"].([]interface{}) + if len(msgsNoInj) != 2 || len(msgsWithInj) != 2 { + t.Fatalf("expected 2 messages in both bodies, got %d and %d", len(msgsNoInj), len(msgsWithInj)) } - if !strings.Contains(string(blockB), "uuid-session-B") { - t.Errorf("session B's block missing its own uuid: %s", blockB) - } -} -// TestCacheFidelityBoundaryDoesNotExtendPastSystemBlocks covers the M1 fix: -// SharedPrefixLen counts the leading run of cross-session-shared blocks over -// the FULL cache order (system blocks, then tools, then messages — see -// BuildReplayRequestPrefix). When that shared run extends PAST the system -// blocks into shared tools (or a shared leading message), splicing the -// per-session UUID marker at the system/tools boundary would land it ahead -// of the shared tools, making them diverge per session on the wire and -// losing their cross-session prefix-cache hit — even though the tools -// themselves stay byte-identical. The fix requires SharedPrefixLen <= -// len(effectiveSystemBlocks(...)) for boundary splicing; when it's greater -// (shared run reaches into tools), injection must fall back to the tail -// instead. Exercises both the Anthropic and OpenAI builders. -func TestCacheFidelityBoundaryDoesNotExtendPastSystemBlocks(t *testing.T) { - docs := strings.Repeat("shared-tools-docs ", 100) - - // Two sessions share BOTH the leading system block (index 0) AND the - // tools block (index 1) — SharedPrefixLen=2 — but diverge starting at - // their own message. len(effectiveSystemBlocks(...)) is only 1, so the - // shared run extends past the system blocks into tools. - sharedTools := &RouterReplayToolsSpec{Count: 2, Bytes: 300, Hash: "toolshash"} - reqA := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, - Tools: sharedTools, - Messages: []RouterReplayMessage{{Hash: "msgUniq-A", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + sharedNoInj, _ := json.Marshal(msgsNoInj[0]) + sharedWithInj, _ := json.Marshal(msgsWithInj[0]) + if string(sharedNoInj) != string(sharedWithInj) { + t.Errorf("unstamped (shared) message diverged with injection active elsewhere:\nwithout: %s\nwith: %s", sharedNoInj, sharedWithInj) } - reqB := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, // same shared system block - Tools: sharedTools, // same shared tools - Messages: []RouterReplayMessage{{Hash: "msgUniq-B", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, - } - injA := &uuidInjection{UUIDs: []string{"uuid-session-A"}, Recite: false, SharedPrefixLen: 2} - injB := &uuidInjection{UUIDs: []string{"uuid-session-B"}, Recite: false, SharedPrefixLen: 2} - t.Run("anthropic", func(t *testing.T) { - bodyA, _, err := buildAnthropicMessagesBody(reqA, docs, "model", "", 0, false, injA) - if err != nil { - t.Fatalf("build A: %v", err) - } - bodyB, _, err := buildAnthropicMessagesBody(reqB, docs, "model", "", 0, false, injB) - if err != nil { - t.Fatalf("build B: %v", err) - } - var parsedA, parsedB map[string]interface{} - if err := json.Unmarshal(bodyA, &parsedA); err != nil { - t.Fatalf("unmarshal A: %v", err) - } - if err := json.Unmarshal(bodyB, &parsedB); err != nil { - t.Fatalf("unmarshal B: %v", err) - } - - // The marker must NOT be spliced into the system array — it must - // carry ONLY the original shared block. - sysA, _ := parsedA["system"].([]interface{}) - sysB, _ := parsedB["system"].([]interface{}) - if len(sysA) != 1 || len(sysB) != 1 { - t.Fatalf("expected system to carry ONLY the original shared block (no boundary splice), got lens %d and %d", len(sysA), len(sysB)) - } - - // The shared tools array must stay byte-identical across sessions — - // nothing per-session was spliced ahead of it. - toolsA, _ := json.Marshal(parsedA["tools"]) - toolsB, _ := json.Marshal(parsedB["tools"]) - if string(toolsA) != string(toolsB) { - t.Errorf("shared tools diverged between sessions:\nA: %s\nB: %s", toolsA, toolsB) - } + ownWithInj, _ := json.Marshal(msgsWithInj[1]) + if !strings.Contains(string(ownWithInj), "own-uuid") { + t.Errorf("stamped message missing its own marker: %s", ownWithInj) + } +} - // The UUID marker must have landed in the tail (messages) instead. - if !strings.Contains(string(bodyA), "uuid-session-A") { - t.Error("session A's uuid missing from the body entirely — expected tail injection") - } - msgsA, _ := parsedA["messages"].([]interface{}) - if len(msgsA) == 0 { - t.Fatal("expected messages to carry the tail-injected uuid block") - } - lastA := msgsA[len(msgsA)-1].(map[string]interface{}) - if !strings.Contains(fmt.Sprintf("%v", lastA["content"]), "uuid-session-A") { - t.Errorf("uuid marker not found in the tail message: %v", lastA["content"]) +// TestFindLeakedUUIDsByOwner exercises the router path's O(response) +// contamination scanner: a known OTHER-session UUID is flagged with the +// correct series index; the caller's OWN uuid is never flagged; a +// UUID-shaped string with no entry in owner is silently ignored; and +// multiple leaks are returned in deterministic (scan-order) sequence. +// Mirrors validateReplayResponse/FindLeakedUUIDs' semantics for the dataset +// path, but via the reverse uuid->owner map instead of iterating every +// session's UUID set. +func TestFindLeakedUUIDsByOwner(t *testing.T) { + owner := map[string]int{ + "11111111-1111-1111-1111-111111111111": 0, + "22222222-2222-2222-2222-222222222222": 0, + "33333333-3333-3333-3333-333333333333": 1, + "44444444-4444-4444-4444-444444444444": 2, + } + + t.Run("other-session uuid flagged with correct series", func(t *testing.T) { + resp := "here it is: 33333333-3333-3333-3333-333333333333" + got := findLeakedUUIDsByOwner(resp, "", 0, owner) + if len(got) != 1 || !strings.Contains(got[0], "33333333-3333-3333-3333-333333333333") || !strings.Contains(got[0], "series=1") { + t.Errorf("got %v, want one entry naming series=1", got) } }) - t.Run("openai", func(t *testing.T) { - bodyA, _, err := buildOpenAIChatCompletionsBody(reqA, docs, "model", "", 0, false, injA) - if err != nil { - t.Fatalf("build A: %v", err) - } - bodyB, _, err := buildOpenAIChatCompletionsBody(reqB, docs, "model", "", 0, false, injB) - if err != nil { - t.Fatalf("build B: %v", err) - } - var parsedA, parsedB map[string]interface{} - if err := json.Unmarshal(bodyA, &parsedA); err != nil { - t.Fatalf("unmarshal A: %v", err) - } - if err := json.Unmarshal(bodyB, &parsedB); err != nil { - t.Fatalf("unmarshal B: %v", err) - } - - // The shared tools array must stay byte-identical across sessions. - toolsA, _ := json.Marshal(parsedA["tools"]) - toolsB, _ := json.Marshal(parsedB["tools"]) - if string(toolsA) != string(toolsB) { - t.Errorf("shared tools diverged between sessions:\nA: %s\nB: %s", toolsA, toolsB) - } - - // No system-role message besides the original system block should - // carry the uuid marker (i.e. it must not be boundary-spliced as an - // extra system message). - messagesA, _ := parsedA["messages"].([]interface{}) - for _, raw := range messagesA { - msg, _ := raw.(map[string]interface{}) - if msg["role"] == "system" && strings.Contains(fmt.Sprintf("%v", msg["content"]), "uuid-session-A") { - t.Errorf("uuid marker was boundary-spliced into a system message: %v", msg) - } - } - // It must still appear somewhere in the body (tail injection). - if !strings.Contains(string(bodyA), "uuid-session-A") { - t.Error("session A's uuid missing from the body entirely — expected tail injection") + t.Run("own uuid never flagged", func(t *testing.T) { + resp := "own ids: 11111111-1111-1111-1111-111111111111, 22222222-2222-2222-2222-222222222222" + got := findLeakedUUIDsByOwner(resp, "", 0, owner) + if len(got) != 0 { + t.Errorf("got %v, want none (both are the caller's own uuids)", got) } }) -} - -// TestTailFallbackInjection verifies that when SharedPrefixLen == 0 (no -// usable boundary), the UUID block is folded into the tail (messages array) -// rather than the system array, and the request remains well-formed. -func TestTailFallbackInjection(t *testing.T) { - docs := strings.Repeat("tail-fallback-docs ", 100) - req := RouterReplayRequest{ - InputTokens: 500, - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys-unique", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msg-unique", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, - } - inj := &uuidInjection{UUIDs: []string{"uuid-tail"}, Recite: true, SharedPrefixLen: 0} - - body, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) - if err != nil { - t.Fatalf("build: %v", err) - } - var parsed map[string]interface{} - if err := json.Unmarshal(body, &parsed); err != nil { - t.Fatalf("unmarshal: %v", err) - } - sys, _ := parsed["system"].([]interface{}) - if len(sys) != 1 { - t.Fatalf("expected system to carry ONLY the original block (no boundary splice), got %d entries", len(sys)) - } - if strings.Contains(string(body), "uuid-tail") == false { - t.Fatal("uuid block missing from body entirely") - } - msgs, _ := parsed["messages"].([]interface{}) - if len(msgs) == 0 { - t.Fatal("expected messages to carry the tail-injected uuid block/recite content") - } - last := msgs[len(msgs)-1].(map[string]interface{}) - if last["role"] != "user" { - t.Errorf("tail-injected message role = %v, want user", last["role"]) - } -} -// TestUUIDValidationEndToEnd exercises validateReplayResponse (presence + -// cross-session leak) directly against N-stamp sessions (N=2), mirroring -// how replayPoster.do() wires it: a response containing ALL of the OWN -// session's uuids scores found/no-leak; a response containing ANOTHER -// session's uuid scores CROSS_CONTAMINATION against the correct series -// index; a response missing one of its own uuids scores a partial -// PRESENCE_MISS. Semantics are unchanged from the single-uuid path — only -// the stamp count (N) is now typically > 1. -func TestUUIDValidationEndToEnd(t *testing.T) { - sets := [][]string{ - {"uuid-s0-a", "uuid-s0-b"}, - {"uuid-s1-a", "uuid-s1-b"}, - {"uuid-s2-a", "uuid-s2-b"}, - } - - t.Run("own uuids present, no leak", func(t *testing.T) { - resp := "Sure, the ids I recall are " + sets[0][0] + " and " + sets[0][1] + ". Anyway, here's your answer." - found, leaked := validateReplayResponse(resp, "", sets[0], 0, sets) - if len(found) != 2 || !found[0] || !found[1] { - t.Errorf("found = %v, want [true true]", found) - } - if len(leaked) != 0 { - t.Errorf("leaked = %v, want none", leaked) + t.Run("unowned uuid-shaped string ignored", func(t *testing.T) { + resp := "random: 99999999-9999-9999-9999-999999999999" + got := findLeakedUUIDsByOwner(resp, "", 0, owner) + if len(got) != 0 { + t.Errorf("got %v, want none (not a real stamp)", got) } }) - t.Run("cross contamination from another session", func(t *testing.T) { - resp := sets[0][0] + ", " + sets[0][1] + ", and also " + sets[2][0] - found, leaked := validateReplayResponse(resp, "", sets[0], 0, sets) - if len(found) != 2 || !found[0] || !found[1] { - t.Errorf("found = %v, want [true true]", found) - } - if len(leaked) != 1 || !strings.Contains(leaked[0], sets[2][0]) || !strings.Contains(leaked[0], "series=2") { - t.Errorf("leaked = %v, want one entry naming session 2's uuid", leaked) + t.Run("thinking channel scanned too", func(t *testing.T) { + got := findLeakedUUIDsByOwner("no leak here", "but here: 44444444-4444-4444-4444-444444444444", 0, owner) + if len(got) != 1 || !strings.Contains(got[0], "series=2") { + t.Errorf("got %v, want one entry naming series=2 (found in thinking)", got) } }) - t.Run("partial presence miss", func(t *testing.T) { - resp := "only " + sets[1][0] + " here" - found, leaked := validateReplayResponse(resp, "", sets[1], 1, sets) - if len(found) != 2 || !found[0] || found[1] { - t.Errorf("found = %v, want [true false]", found) + t.Run("deterministic scan order for multiple leaks", func(t *testing.T) { + resp := "first 44444444-4444-4444-4444-444444444444 then 33333333-3333-3333-3333-333333333333" + got1 := findLeakedUUIDsByOwner(resp, "", 0, owner) + got2 := findLeakedUUIDsByOwner(resp, "", 0, owner) + if len(got1) != 2 { + t.Fatalf("got %v, want 2 leaked entries", got1) + } + for i := range got1 { + if got1[i] != got2[i] { + t.Errorf("non-deterministic order: %v vs %v", got1, got2) + } } - if len(leaked) != 0 { - t.Errorf("leaked = %v, want none", leaked) + if !strings.Contains(got1[0], "series=2") || !strings.Contains(got1[1], "series=1") { + t.Errorf("got %v, want series=2 before series=1 (scan order)", got1) } }) - t.Run("total presence miss", func(t *testing.T) { - found, leaked := validateReplayResponse("no ref ids here", "", sets[1], 1, sets) - if found[0] || found[1] { - t.Error("expected PRESENCE_MISS on both stamps (found=[false false])") - } - if len(leaked) != 0 { - t.Errorf("leaked = %v, want none", leaked) + t.Run("empty owner map returns nil", func(t *testing.T) { + if got := findLeakedUUIDsByOwner("11111111-1111-1111-1111-111111111111", "", 0, map[string]int{}); got != nil { + t.Errorf("got %v, want nil", got) } }) } @@ -688,7 +716,8 @@ func TestUUIDValidationEndToEnd(t *testing.T) { // comma-joined first line; fail on missing, reordered, or chatty first // lines; pass when line 1 is exact even though LATER lines contain filler // (the whole point of front-loading the ask to line 1 while forced-output -// keeps generating). +// keeps generating). expected here plays the role of inj.ReciteUUIDs — the +// request's recite WINDOW, not the session's full turn history. func TestFirstLineConformity(t *testing.T) { expected := []string{"uuid-a", "uuid-b", "uuid-c"} @@ -730,7 +759,7 @@ func TestApplyReciteFloor(t *testing.T) { {"at floor, recite -> unchanged", replayReciteFloorTokens(2), true, 2}, {"above floor, recite -> unchanged", 100000, true, 2}, {"below floor, no recite -> unchanged", 5, false, 2}, - {"below floor, more uuids, recite -> raised higher", 5, true, 20}, + {"below floor, more uuids (capped at 4), recite -> raised higher", 5, true, 4}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { @@ -746,32 +775,38 @@ func TestApplyReciteFloor(t *testing.T) { } } -// TestReciteFloorScalesWithN verifies the recite floor grows with numUUIDs — -// more UUIDs to recite on the first line needs a bigger budget — the -// N-per-session analogue of the old fixed-64-token constant. +// TestReciteFloorScalesWithN verifies the recite floor grows with numUUIDs +// up to the window cap (4) — more UUIDs to recite on the first line needs a +// bigger budget — but is now BOUNDED, unlike the retired per-session-N +// scheme where a long session's floor grew without bound. func TestReciteFloorScalesWithN(t *testing.T) { - small := replayReciteFloorTokens(2) - large := replayReciteFloorTokens(20) + small := replayReciteFloorTokens(1) + large := replayReciteFloorTokens(4) if large <= small { - t.Errorf("replayReciteFloorTokens(20) = %d, want > replayReciteFloorTokens(2) = %d", large, small) + t.Errorf("replayReciteFloorTokens(4) = %d, want > replayReciteFloorTokens(1) = %d", large, small) } } // TestMaxTokensFloorAppliedInWireBuilders verifies the floor is actually // wired into both body builders' emitted max_tokens when a recite // injection is present and the original/recorded budget is tiny — the -// scenario a real tool-call-only turn would hit. +// scenario a real tool-call-only turn would hit. numUUIDs is now +// len(inj.ReciteUUIDs) (the window, capped at 4), not the old per-session N. func TestMaxTokensFloorAppliedInWireBuilders(t *testing.T) { docs := strings.Repeat("floor-docs ", 100) req := RouterReplayRequest{ InputTokens: 500, OutputTokens: 5, // tiny recorded budget -- would truncate the recite line - SystemBlocks: []RouterReplaySystemBlock{{Hash: "sys1", Bytes: 250}}, - Messages: []RouterReplayMessage{{Hash: "msg1", Role: "user", BlockTypes: []string{"text"}, Bytes: 100}}, + Messages: []RouterReplayMessage{userText("msg1", 100)}, + } + reciteUUIDs := []string{"uuid-floor-0", "uuid-floor-1"} + inj := &uuidInjection{ + StampByHash: map[string]turnStamp{"msg1": {Idx: 0, UUID: reciteUUIDs[0], Label: "turn-1"}}, + Recite: true, + ReciteLabels: []string{"turn-1"}, + ReciteUUIDs: reciteUUIDs, } - uuids := []string{"uuid-floor-0", "uuid-floor-1"} - inj := &uuidInjection{UUIDs: uuids, Recite: true, SharedPrefixLen: 1} - wantFloor := float64(replayReciteFloorTokens(len(uuids))) + wantFloor := float64(replayReciteFloorTokens(len(reciteUUIDs))) anthBody, _, err := buildAnthropicMessagesBody(req, docs, "model", "", 0, false, inj) if err != nil { diff --git a/benchmark/replay_router_wire.go b/benchmark/replay_router_wire.go index 9347336..fae40c9 100644 --- a/benchmark/replay_router_wire.go +++ b/benchmark/replay_router_wire.go @@ -46,9 +46,11 @@ const verboseOutputInstruction = "Provide a thorough, detailed response and keep // router path — see replay_router_uuid.go); nil means "no injection", // leaving the body byte-for-byte identical to before this feature existed. func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool, inj *uuidInjection) ([]byte, string, error) { + var stampByHash map[string]turnStamp injNumUUIDs := 0 if inj != nil { - injNumUUIDs = len(inj.UUIDs) + stampByHash = inj.StampByHash + injNumUUIDs = len(inj.ReciteUUIDs) } body := map[string]interface{}{ "model": modelName, @@ -80,28 +82,6 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName systemArr = append([]map[string]interface{}{stamp}, systemArr...) } - // UUID block injection at the system/conversation boundary (Option C — - // see replay_router_uuid.go). Only spliced in here when the leading run - // of cross-session-shared blocks does NOT extend past the emitted - // system blocks — i.e. the first tool/message block is not itself part - // of the shared run; otherwise splicing the per-session marker here - // would land it ahead of shared tools/messages and poison THEIR - // cross-session cache key too, not just add a per-session block after - // genuinely-shared content. When the shared run does extend into - // tools/messages, fall back to tail injection below instead, so the - // marker never lands ahead of content this session shares with others. - injUUIDText := "" - if inj != nil { - injUUIDText = bareUUIDBlock(inj.UUIDs) - } - markerAtBoundary := inj != nil && injUUIDText != "" && - inj.SharedPrefixLen > 0 && inj.SharedPrefixLen <= len(effectiveSystemBlocks(req.SystemBlocks)) - if markerAtBoundary { - systemArr = append(systemArr, map[string]interface{}{ - "type": "text", - "text": injUUIDText, - }) - } if forceOutput { systemArr = append(systemArr, map[string]interface{}{ "type": "text", @@ -116,21 +96,19 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName } var msgs []map[string]interface{} if len(req.Messages) > 0 { - msgs = buildMessages(req.Messages, docs) + msgs = buildMessages(req.Messages, docs, stampByHash) } - if inj != nil { - if !markerAtBoundary && injUUIDText != "" { - msgs = appendTailMessageAnthropic(msgs, injUUIDText) - } - if inj.Recite { - msgs = appendTailMessageAnthropic(msgs, replayReciteFirstLineInstruction()) - } + if inj != nil && inj.Recite { + msgs = appendTailMessageAnthropic(msgs, replayReciteWindowInstruction(inj.ReciteLabels)) } if len(msgs) > 0 { body["messages"] = msgs } - // Collect canonical text for the cache estimator. + // Collect canonical text for the cache estimator. Uses the SAME + // buildMessageContent(m, docs, stampByHash) call as buildMessages above + // so the cache-estimate canonical matches the wire body exactly + // (including any inline turn-stamp markers). var canonical strings.Builder if runID != "" { canonical.WriteString(fmt.Sprintf("RUN_GUID: %s", runID)) @@ -138,9 +116,6 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName for _, b := range effectiveSystemBlocks(req.SystemBlocks) { canonical.WriteString(synthText(b.Hash, b.Bytes, docs)) } - if markerAtBoundary { - canonical.WriteString(injUUIDText) - } if req.Tools != nil && req.Tools.Count > 0 { n := req.Tools.Count if n <= 0 { @@ -158,7 +133,7 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName } } for _, m := range req.Messages { - blocks := buildMessageContent(m, docs) + blocks := buildMessageContent(m, docs, stampByHash) for _, blk := range blocks { if t, ok := blk["text"]; ok { canonical.WriteString(fmt.Sprintf("%v", t)) @@ -178,13 +153,8 @@ func buildAnthropicMessagesBody(req RouterReplayRequest, docs string, modelName } } } - if inj != nil { - if !markerAtBoundary && injUUIDText != "" { - canonical.WriteString(injUUIDText) - } - if inj.Recite { - canonical.WriteString(replayReciteFirstLineInstruction()) - } + if inj != nil && inj.Recite { + canonical.WriteString(replayReciteWindowInstruction(inj.ReciteLabels)) } bodyBytes, err := json.Marshal(body) @@ -289,10 +259,13 @@ func buildTools(spec *RouterReplayToolsSpec, docs string) []map[string]interface // share of the message's total Bytes. For tool_use and tool_result blocks // we preserve the original ids verbatim so the conversation maintains a // valid reference graph that matches what the original capture sent. -func buildMessages(msgs []RouterReplayMessage, docs string) []map[string]interface{} { +// stampByHash is inj.StampByHash (nil when --replay-inject-uuids is off) — +// threaded down to buildMessageContent, which appends each qualifying +// turn's inline UUID marker to its own synthesized content. +func buildMessages(msgs []RouterReplayMessage, docs string, stampByHash map[string]turnStamp) []map[string]interface{} { out := make([]map[string]interface{}, 0, len(msgs)) for _, m := range msgs { - content := buildMessageContent(m, docs) + content := buildMessageContent(m, docs, stampByHash) entry := map[string]interface{}{ "role": roleOrUser(m.Role), "content": content, @@ -318,8 +291,9 @@ func roleOrUser(role string) string { return "user" } -// appendTailMessageAnthropic appends text (a UUID marker or the recite ask) -// to the end of an Anthropic messages array, preserving strict user/ +// appendTailMessageAnthropic appends text (the windowed recite ask — see +// replayReciteWindowInstruction) to the end of an Anthropic messages array, +// preserving strict user/ // assistant role alternation (Anthropic rejects consecutive same-role // messages, and "system" is not a valid role inside `messages` at all): // - if the array is empty, or the last message is role "assistant", a @@ -356,7 +330,18 @@ func appendTailMessageAnthropic(msgs []map[string]interface{}, text string) []ma // block_types lists the kinds in order; we use tool_use_ids and // tool_result_ids to populate ids on the matching blocks (in order of // appearance in block_types). -func buildMessageContent(m RouterReplayMessage, docs string) []map[string]interface{} { +// +// stampByHash is inj.StampByHash (nil when --replay-inject-uuids is off, or +// when this message isn't a qualifying turn). When m.Hash is a key in +// stampByHash, the labeled marker "\n\n[turn-N id: ]" is appended to +// the LAST "text"-type block's synthesized content, at a position wholly +// determined by m.Hash (same seed synthText already keys on) — so two +// requests carrying the same turn emit byte-identical content for it, +// preserving the within-session cache-hit property. Only count==1 +// (genuinely per-session) messages are ever present in stampByHash — see +// isQualifyingUserTurn — so a cross-session-shared message is never +// perturbed. +func buildMessageContent(m RouterReplayMessage, docs string, stampByHash map[string]turnStamp) []map[string]interface{} { nText := 0 for _, t := range m.BlockTypes { if t == "text" { @@ -435,6 +420,15 @@ func buildMessageContent(m RouterReplayMessage, docs string) []map[string]interf }) } } + if stamp, ok := stampByHash[m.Hash]; ok { + marker := "\n\n[" + stamp.Label + " id: " + stamp.UUID + "]" + for i := len(out) - 1; i >= 0; i-- { + if out[i]["type"] == "text" { + out[i]["text"] = fmt.Sprintf("%v", out[i]["text"]) + marker + break + } + } + } return out } @@ -552,9 +546,11 @@ func buildOpenAITools(spec *RouterReplayToolsSpec, docs string) []map[string]int // inj carries the UUID cache-coherency injection (--replay-inject-uuids, // router path — see replay_router_uuid.go); nil means "no injection". func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelName string, runID string, outputRatio float64, forceOutput bool, inj *uuidInjection) ([]byte, string, error) { + var stampByHash map[string]turnStamp injNumUUIDs := 0 if inj != nil { - injNumUUIDs = len(inj.UUIDs) + stampByHash = inj.StampByHash + injNumUUIDs = len(inj.ReciteUUIDs) } body := map[string]interface{}{ "model": modelName, @@ -598,25 +594,6 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN messages = append([]map[string]interface{}{stamp}, messages...) } - // UUID block injection at the system/conversation boundary (Option C — - // see replay_router_uuid.go and the mirrored comment in - // buildAnthropicMessagesBody). Only spliced in here when the leading - // run of cross-session-shared blocks does NOT extend past the emitted - // system blocks; otherwise it falls back to tail injection below, so it - // never lands ahead of shared tools/messages and poisons their - // cross-session cache key too. - injUUIDText := "" - if inj != nil { - injUUIDText = bareUUIDBlock(inj.UUIDs) - } - markerAtBoundary := inj != nil && injUUIDText != "" && - inj.SharedPrefixLen > 0 && inj.SharedPrefixLen <= len(effectiveSystemBlocks(req.SystemBlocks)) - if markerAtBoundary { - messages = append(messages, map[string]interface{}{ - "role": "system", - "content": injUUIDText, - }) - } if forceOutput { // Force the model to generate up to max_tokens: append a short // continue-generating instruction as a system message AND set vLLM's @@ -639,17 +616,12 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN // each block and join them with newlines, and log a one-time warning so // the operator knows tool fidelity is lost. if len(req.Messages) > 0 { - openaiMsgs := buildOpenAIMessages(req.Messages, docs) + openaiMsgs := buildOpenAIMessages(req.Messages, docs, stampByHash) messages = append(messages, openaiMsgs...) } - if inj != nil { - if !markerAtBoundary && injUUIDText != "" { - messages = appendTailMessageOpenAI(messages, injUUIDText) - } - if inj.Recite { - messages = appendTailMessageOpenAI(messages, replayReciteFirstLineInstruction()) - } + if inj != nil && inj.Recite { + messages = appendTailMessageOpenAI(messages, replayReciteWindowInstruction(inj.ReciteLabels)) } body["messages"] = messages @@ -657,7 +629,11 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN body["tools"] = buildOpenAITools(req.Tools, docs) } - // Collect canonical text for the cache estimator (system blocks + messages). + // Collect canonical text for the cache estimator (system blocks + + // messages). Uses the SAME buildMessageContent(m, docs, stampByHash) + // call as buildOpenAIMessages above so the cache-estimate canonical + // matches the wire body exactly (including any inline turn-stamp + // markers). var canonical strings.Builder if runID != "" { canonical.WriteString(fmt.Sprintf("RUN_GUID: %s", runID)) @@ -665,11 +641,8 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN for _, b := range effectiveSystemBlocks(req.SystemBlocks) { canonical.WriteString(synthText(b.Hash, b.Bytes, docs)) } - if markerAtBoundary { - canonical.WriteString(injUUIDText) - } for _, m := range req.Messages { - blocks := buildMessageContent(m, docs) + blocks := buildMessageContent(m, docs, stampByHash) for _, blk := range blocks { if t, ok := blk["text"]; ok { canonical.WriteString(fmt.Sprintf("%v", t)) @@ -682,21 +655,17 @@ func buildOpenAIChatCompletionsBody(req RouterReplayRequest, docs string, modelN } } } - if inj != nil { - if !markerAtBoundary && injUUIDText != "" { - canonical.WriteString(injUUIDText) - } - if inj.Recite { - canonical.WriteString(replayReciteFirstLineInstruction()) - } + if inj != nil && inj.Recite { + canonical.WriteString(replayReciteWindowInstruction(inj.ReciteLabels)) } bodyBytes, err := json.Marshal(body) return bodyBytes, canonical.String(), err } -// appendTailMessageOpenAI appends text (a UUID marker or the recite ask) to -// the end of an OpenAI messages array. Unlike Anthropic, OpenAI has no +// appendTailMessageOpenAI appends text (the windowed recite ask — see +// replayReciteWindowInstruction) to the end of an OpenAI messages array. +// Unlike Anthropic, OpenAI has no // strict role-alternation requirement, but we still fold the text into the // last message's content when that message is one the model would read as // its own turn's input (user/system/tool) rather than always creating a @@ -730,12 +699,13 @@ func appendTailMessageOpenAI(msgs []map[string]interface{}, text string) []map[s // messages. tool_use blocks become tool_calls on the assistant message; // tool_result blocks become separate role="tool" messages. Orphaned // tool_result blocks (no matching prior tool_call) are folded into user text. -func buildOpenAIMessages(msgs []RouterReplayMessage, docs string) []map[string]interface{} { +// stampByHash is threaded through to buildMessageContent (see its doc). +func buildOpenAIMessages(msgs []RouterReplayMessage, docs string, stampByHash map[string]turnStamp) []map[string]interface{} { out := make([]map[string]interface{}, 0, len(msgs)) seenToolCallIDs := map[string]bool{} for _, m := range msgs { - blocks := buildMessageContent(m, docs) + blocks := buildMessageContent(m, docs, stampByHash) role := roleOrUser(m.Role) if role == "assistant" { diff --git a/benchmark/replay_uuid.go b/benchmark/replay_uuid.go index a11d179..b3d39ea 100644 --- a/benchmark/replay_uuid.go +++ b/benchmark/replay_uuid.go @@ -15,9 +15,53 @@ package benchmark import ( "fmt" + "regexp" "strings" ) +// uuidRe matches a canonical hyphenated UUID string (8-4-4-4-12 hex digits). +// Used by findLeakedUUIDsByOwner to pull UUID-shaped substrings out of a +// response/thinking blob without needing to know the candidate set up +// front. +var uuidRe = regexp.MustCompile(`[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}`) + +// findLeakedUUIDsByOwner scans resp and thinking for UUID-shaped substrings +// and flags any that are a KNOWN turn UUID (present in owner, the +// uuid -> owning-session-index reverse map — see buildSessionTurnUUIDs) +// belonging to a session OTHER than ownIdx. Unlike FindLeakedUUIDs (which +// iterates every UUID ever assigned across the whole population — O(total +// UUIDs)), this scans only the response/thinking text once via uuidRe and +// does an O(1) map lookup per match — O(response), not O(population) — +// which matters once turns (not sessions) are the stamping unit: the total +// UUID population grows with conversation length instead of session count. +// A UUID-shaped string with no entry in owner (or that happens to match a +// hex pattern but isn't a real stamp) is silently ignored, not flagged. +// Returns "uuid(series=N)" entries, one per distinct leaked UUID found, in +// first-appearance (scan) order — deterministic for a given response. +func findLeakedUUIDsByOwner(resp, thinking string, ownIdx int, owner map[string]int) []string { + if len(owner) == 0 { + return nil + } + combined := resp + if thinking != "" { + combined = resp + "\n" + thinking + } + var leaked []string + seen := map[string]bool{} + for _, u := range uuidRe.FindAllString(combined, -1) { + if seen[u] { + continue + } + seen[u] = true + si, ok := owner[u] + if !ok || si == ownIdx { + continue + } + leaked = append(leaked, fmt.Sprintf("%s(series=%d)", u, si)) + } + return leaked +} + // injectUUIDMarker appends one visible marker per uuid to turnValue. Unlike // the cache-coherency eval's ... filler, the model MUST see // and be able to repeat this text — it's asked to recite every ref-id later diff --git a/cli/benchmark_options.go b/cli/benchmark_options.go index 0dc0687..1c2cad6 100644 --- a/cli/benchmark_options.go +++ b/cli/benchmark_options.go @@ -63,7 +63,7 @@ type BenchmarkAutoOptions struct { ReplayNoStamp bool `long:"replay-no-stamp" description:"Disable per-run RUN_GUID stamping in replay mode. By default each replay run prepends a fresh UUID to every request's system prompt (both --from-dataset and --router-replay-file paths) so server prefix caches from prior runs can't be reused — pristine per-run cache state." env:"BENCHMARK_REPLAY_NO_STAMP"` AbortOnCollapse bool `long:"abort-on-collapse" description:"Abort the benchmark if the windowed cache hit rate stays below 50% for 2 minutes. Off by default — this heuristic fires on legitimate workloads with low cache reuse (e.g. replay across many distinct conversations)." env:"BENCHMARK_ABORT_ON_COLLAPSE"` ReplayStopAtLowConcurrency bool `long:"replay-stop-at-low-concurrency" description:"Terminate the replay run once the queue is drained AND the number of active worker goroutines has dropped below --concurrency. Avoids long-tail measurements where only a handful of long conversations remain and the gate is underutilized." env:"BENCHMARK_REPLAY_STOP_AT_LOW_CONCURRENCY"` - ReplayInjectUUIDs bool `long:"replay-inject-uuids" description:"Inject one deterministic UUID marker per SESSION into --router-replay-file replay sessions, at the boundary between cross-session-shared leading blocks and per-session content, and validate its presence in later responses -- a coherency check (PRESENCE_MISS / CROSS_CONTAMINATION) for the KV-offload path under realistic multi-turn agentic traffic. ROUTER-REPLAY PATH ONLY: requires --router-replay-file; rejected together with --from-dataset." env:"BENCHMARK_REPLAY_INJECT_UUIDS"` + ReplayInjectUUIDs bool `long:"replay-inject-uuids" description:"Inject one deterministic UUID per user turn into --router-replay-file replay sessions (spread through the conversation), and validate a windowed recite of first + up to 3 most recent turns (excluding the current turn) in later responses -- a coherency check (PRESENCE_MISS / CROSS_CONTAMINATION) for the KV-offload path under realistic multi-turn agentic traffic. ROUTER-REPLAY PATH ONLY: requires --router-replay-file; rejected together with --from-dataset." env:"BENCHMARK_REPLAY_INJECT_UUIDS"` ReplayUUIDSeed int64 `long:"replay-uuid-seed" description:"PRNG seed for --replay-inject-uuids' per-session UUID generation (0 = crypto/rand, non-deterministic across runs)." default:"0" env:"BENCHMARK_REPLAY_UUID_SEED"` // ReplayReciteEveryRequest is a string (not bool) choice, same workaround // as RandomGateOrder above: a plain bool flag defaulting true can never