From 61d27eb41ccadd9b862925a8b560602b60e5fbc0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 07:06:59 -0400 Subject: [PATCH 1/4] Audit the frozen run's near-zero and missing household latencies Every household that sums to under a second is a disk-cache replay (328 full, 33 partial, 2 untraced); GPT-5.5 ran through the batch API, and Claude Fable 5's board rows were rebuilt without usage columns. Adds the audit, its household evidence, the script that builds it, a shared household-latency helper, and paper accessors that state the causes from the data. WIP: paper prose, benchmark card, manifest note and tests to follow. Co-Authored-By: Claude Opus 5.5 --- docs/latency_replay_audit.md | 207 +++++++++++++ docs/latency_replay_audit/households.csv | 364 +++++++++++++++++++++++ paper/index.qmd | 54 +++- policybench/analysis.py | 31 +- policybench/paper_results.py | 184 +++++++++++- scripts/audit_latency_replays.py | 276 +++++++++++++++++ 6 files changed, 1097 insertions(+), 19 deletions(-) create mode 100644 docs/latency_replay_audit.md create mode 100644 docs/latency_replay_audit/households.csv create mode 100644 scripts/audit_latency_replays.py diff --git a/docs/latency_replay_audit.md b/docs/latency_replay_audit.md new file mode 100644 index 00000000..2152644b --- /dev/null +++ b/docs/latency_replay_audit.md @@ -0,0 +1,207 @@ +# Latency replay audit (frozen US run, October 2026) + +This audit explains every household in the frozen US run whose recorded latency +sums to about zero, and every model whose latency is missing. It then says how +the published figures treat them. The run is +`paper/snapshot/20260501/runs/us_full_run_20260612_policyengine_4_16_1_populace` +as of release dashboard-data-20261014, with 50 models and 100 households each. +The household-level evidence is `latency_replay_audit/households.csv`, written by +`scripts/audit_latency_replays.py`. + +## How latency is recorded + +The harness times each provider call. A household that needs several calls +(one or three outputs per request, a repair round, or a retry round) makes them +one after another in `run_single_no_tools`. Their times are summed, and the sum +is divided equally among the household's rows (`elapsed_seconds / batch_size` +in `policybench/eval_no_tools.py`; all three lines date from 2026-05-01, before +every board run). A household's latency is therefore the sum over its rows. +`analysis.household_latency_seconds` computes that sum. +`analysis.model_cost_latency` publishes the median across households as +`latencySeconds`. A household whose rows carry no time is left out of the +median rather than counted as zero. + +The harness also caches provider responses on disk (`policybench/cache.py`, +litellm's disk cache namespaced by the prompt contract version). A request +identical to an earlier one is answered from the cache. The timer measures the +lookup, a few hundredths of a second, and the row records it as if it were the +call. + +## Two models have no latency in the board rows + +**GPT-5.5** ran through OpenAI's batch API. All 716 of its board response ids +come from one batch (`batch_6a49d188…`, submitted 2026-07-05 03:37 UTC and +completed 11 minutes later; `results/local/gpt55_default_run_20260706`). Batch +rows leave `elapsed_seconds` and both request timestamps empty by design, +because a batch round trip includes queue time (`policybench/batch_eval.py`). +Its cost is reconstructed from token counts at standard synchronous rates. + +**Claude Fable 5** ran synchronously from 2026-07-01 22:26 to 2026-07-02 01:53 +UTC. The batch adapter (`batch_eval.py`, #88) first landed at 10:03 UTC on +2026-07-02, and the original rows record a per-call request time, which batch +rows never carry. Those rows, with raw responses, tokens, costs and timings, +are the `claude-fable-5-predictions.csv.gz` asset of release +dashboard-data-20260701. The board rows were rebuilt from that release on +2026-07-02 (57d26f1c) without the usage, timing and raw-response columns. Their +predictions and explanations equal the original rows'. The published +`costUsd` ($54.11), `totalTokens` (3,850,174) and `latencySeconds` (97.68 s) +come from the original rows exactly. The release drivers carry them forward as +`CARRIED_USAGE`, and 6cb7b488 restored them after a rebuild had zeroed them. +Three of the original households (scenario_044, scenario_114 and scenario_115) +sum to under a second. No cache or ledger from that run survives to show +whether they were replays. Without them the median is 97.71 s. + +The snapshot manifest, the release drivers' comments and the paper used to say +that Fable 5 was "served through the Anthropic batch adapter" or that its usage +was "recorded at the run level". Neither was true, and both are corrected. + +## Replayed households + +A household counts as replayed when its first recorded response existed +before the household's request began. The evidence is any of the following: + +- **Disk cache.** The response's id sits in a litellm cache entry stored more + than a minute before the request (`.policybench_cache` in the main checkout, + 121,648 entries). +- **OpenRouter id.** OpenRouter ids are `gen--…`, and the time + is more than a minute before the request. On live households it falls + within 4 seconds of `request_started_at`. +- **OpenAI id.** An OpenAI Responses id embeds its creation time in hex, after + litellm's base64 wrapper is removed. On live households it falls within 4 + seconds. +- **Spend ledger.** The board run's spend ledger records `cache_hit: true` for + the call. +- **Smoke.** The same response id was served live in the model's onboarding + smoke. + +A full replay sums to under a second: every call in the household came from +the cache. Every full replay in the run sums to under 0.12 s, while the fastest +live household takes 1.75 s (Gemini 3.5 Flash-Lite), so the one-second line +(`analysis.REPLAYED_HOUSEHOLD_SECONDS`) separates them cleanly. A partial replay +sums to a second or more. Its first recorded call was a replay, and a later +call (another chunk, a repair, or a retry round's new request) went to the +provider. Its time leaves out the replayed call's original duration. + +| Model | Full | Partial | Where the replayed responses came from | Published median (s) | Without sub-second households (s) | +|---|---|---|---|---|---| +| Qwen3.7-max | 93 | 1 | earlier passes of its run | 0.009 | 917.9 (7 households) | +| GLM-5.2 | 80 | 1 | its JSON-contract smoke (scenario_000 and scenario_001); earlier passes of its run | 0.007 | 384.8 (20 households) | +| Kimi K2.6 | 75 | 4 | its JSON-contract smoke (scenario_000 and scenario_001); earlier passes of its run | 0.033 | 1,194.2 (25 households) | +| MiniMax M3 | 53 | 1 | earlier passes of its run | 0.003 | 10.7 (47 households) | +| Claude Sonnet 5 | 9 | 2 | its smoke (scenario_000); two restarted chunks (8 full, 2 partial) | 63.9 | 64.8 | +| Muse Spark 1.3 | 4 | 0 | its onboarding smoke | 40.6 | 41.1 | +| Claude Haiku 5.5 | 2 | 0 | its onboarding smoke | 5.1 | 5.1 | +| Claude Opus 5.5 | 2 | 0 | its onboarding smoke | 24.0 | 24.0 | +| Muse Wonder | 2 | 0 | its onboarding smoke | 26.1 | 26.5 | +| DeepSeek V4 Pro | 2 | 0 | requests about an hour before its run; no file survives | 169.9 | 172.4 | +| GPT-5.6 Luna | 2 | 0 | requests 5 minutes before its run; no file survives | 14.4 | 14.5 | +| GPT-5.6 Sol | 2 | 0 | requests 7 minutes before its run; no file survives | 36.3 | 36.4 | +| GPT-5.6 Terra | 2 | 0 | requests 6 to 7 minutes before its run; no file survives | 18.9 | 19.1 | +| GLM-5.3 | 0 | 18 | earlier rounds of its supervised run | 134.2 | 134.2 | +| GLM-5.3-Flash (preview) | 0 | 4 | earlier rounds of its supervised run | 241.7 | 241.7 | +| Kimi K3 | 0 | 2 | earlier in its run | 1,023.3 | 1,023.3 | + +That is 328 full and 33 partial replays across 16 models. Two more households +sum to under a second with no surviving evidence either way: Grok 4.3's +scenario_000 (0.022 s) and Claude Haiku 4.5's scenario_009 (0.026 s). Both +were already that way in the June 13-model board. Those runs predate per-call +timestamps and spend ledgers, and their cache does not survive. Without them +the two medians move from 22.13 to 22.21 s and from 47.67 to 47.69 s. + +The 0.05-second line used before this audit missed four replays: Kimi K2.6's +scenario_008, scenario_026 and scenario_064 (0.056 to 0.068 s) and Claude Haiku +5.5's scenario_001 (0.118 s). The board ledger records `cache_hit: true` for +the last one. + +### The four open-weight models: a multi-pass run + +Kimi K2.6, GLM-5.2, MiniMax M3 and Qwen3.7-max were added on 2026-07-05 to 07 +(`results/local/openweight_run_20260705.md`). OpenRouter's connection +allowance collapsed under concurrent long-lived requests, so whole households +errored and wrote no rows. `results/local/rerun_openweight.sh` and +`results/local/grind_model.sh` then re-ran each model's full range several +times. Each pass deleted the part files and re-requested every household, and +the cache answered the households earlier passes had completed. The board rows +are each model's final pass. MiniMax M3's first pass completed 53 of its 100 +households, and 53 of its households are full replays. Qwen3.7-max reached +93/100 in the round before its last, and 93 are full replays. Kimi K2.6's +scenario_109 shows a partial replay at work. In earlier rounds its answer call +succeeded and was cached, but its explanation repair timed out at 300 s +(`part_84_100.grind*.log`). In the final pass the answer was replayed and only +the repair reached the provider, so the household records 777 s and the +replayed answer's id. + +For these four models the households left after removing replays are the ones +earlier passes failed on, which are the slow ones. Their medians (10.7 to +1,194 s) are not estimates of the model's latency. For OpenRouter responses, +the cache's store time minus the id's creation time approximates a replayed +call's original duration. It agrees with the recorded time to a median of +0.5 s on single-call live households. Applied to the replays it gives medians +of about 133 s (Qwen3.7-max), 117 s (GLM-5.2), 294 s (Kimi K2.6) and 10 s +(MiniMax M3). It measures only the call that succeeded, not the retries the +harness timer would include, so it is a different quantity, and nothing +publishes it. + +## What the published figures do with them + +- **`modelStats.latencySeconds` in the dashboard data** (release asset + `dashboard-data.json`, the same values as the frozen `data.json.gz`) is the + median of household sums with replays counted as calls. Every value + reproduces from the frozen predictions except Fable 5's, which is carried + from its original rows. For the four open-weight models it publishes 0.003 to + 0.033 seconds, which is cache-lookup time and not latency. For the twelve + other models with full replays it publishes medians lowered by up to 2.5 s + (DeepSeek V4 Pro); partial replays lower them further by an unknown amount. + GPT-5.5 has no `latencySeconds`. +- **The leaderboard** shows no latency: the column was dropped on 2026-07-05 + (#100), and `latencySeconds` is not rendered anywhere in the app. +- **The paper's cost-and-latency table** treats a sub-second or missing + median as unmeasured and falls back to `modelStats`. The four open-weight + models and GPT-5.5 therefore show a dash, and Fable 5 shows its carried + 98 s. The paragraph under the table used to attribute the dashes to "batch or + asynchronous dispatch whose logged times reflect submission rather than + generation". That fits no model in the board: GPT-5.5's batch rows log no + time at all, and the four sub-second models ran synchronously. The paragraph + and caption now give the real causes, computed from the frozen data + (`PaperResults.latency_dash_models` and its neighbours). The rounded table + values are unchanged; without sub-second households Claude Sonnet 5 would + read 65 s rather than 64 s and DeepSeek V4 Pro 172 s rather than 170 s. +- **`costUsd`.** A replayed response keeps the cost recorded when it was + generated. Spend ledgers record each cache hit's spend as $0 and the replayed + cost as `cached_response_cost_usd`, and the board row carries the latter + (163 hits). On 43 single-call MiniMax M3 replays the row cost equals the + cost embedded in the cached response exactly. A household's published cost + therefore counts each of its responses once, whether or not the board run + paid for it, and leaves out spend on attempts that did not make the board, + such as the open-weight models' failed passes. That matches the paper's + description ("each call's recorded total"), so no cost figure misleads + because of replays. Fable 5's `costUsd` is computed from its original rows, + so its NaN board rows do not reach it. `model_cost_latency` itself returns + $0 for a model whose rows carry no tokens when a price override exists, as + Fable 5's do. The release drivers overwrite that with the carried value, so + nothing published shows it. +- **GLM-5.3-Flash (preview)** (artifact key `ox-alpha`) records a + provider-reported charge of $0.00 on all 1,984 rows. The leaderboard prints + $0.000 per household and the paper prints a dash, which the caption now + explains. This is unrelated to replays. + +## Changing the published numbers + +Excluding replayed households from `latencySeconds`, or publishing none for the +four open-weight models, changes published numbers, so it is a methodology +choice for Max (decision queued with `cos add-decision`). This audit changes +only disclosures, comments and the tests that tie them to the data. + +## Reproducing + +```bash +uv run python scripts/audit_latency_replays.py \ + --cache-dir .policybench_cache \ + --ledger-root results/local \ + --ledger-root ~/reviews +``` + +The cache, ledgers and smoke files exist only on the machine that ran the +board. The id-time evidence and every sub-second household can be recomputed +from the frozen predictions alone, and `tests/test_latency_replay_audit.py` +does so. diff --git a/docs/latency_replay_audit/households.csv b/docs/latency_replay_audit/households.csv new file mode 100644 index 00000000..fc9066b0 --- /dev/null +++ b/docs/latency_replay_audit/households.csv @@ -0,0 +1,364 @@ +model,scenario_id,household_latency_seconds,replay,evidence,source,first_response_id,request_started_at,response_existed_at +claude-haiku-4.5,scenario_009,0.026461,untraced,,untraced,chatcmpl-9c7cea35-2052-496a-bf4e-3602ca12515c,, +claude-haiku-5.5,scenario_000,0.017322,full,smoke_file+spend_ledger,onboarding smoke,chatcmpl-992b7cba-d5e8-42da-9449-c06899393c28,1791495684.702,1791495644.564 +claude-haiku-5.5,scenario_001,0.117501,full,smoke_file+spend_ledger,onboarding smoke,chatcmpl-308e4847-b356-4499-a13a-3ab9f3208450,1791495684.777,1791495651.476 +claude-opus-5.5,scenario_000,0.029634,full,smoke_file+smoke_ledger+spend_ledger,onboarding smoke,chatcmpl-e65b82db-643d-47f9-b87a-d6143e484ca6,1790095878.103,1790095745.462 +claude-opus-5.5,scenario_001,0.029532,full,smoke_file+smoke_ledger+spend_ledger,onboarding smoke,chatcmpl-f5a49889-34ba-4ef9-8e78-3cd1cbd21ef7,1790095878.103,1790095771.107 +claude-sonnet-5,scenario_000,0.031374,full,cache+smoke_file,onboarding smoke,chatcmpl-d78ec31e-c962-43fd-a568-a6f2f8fc0214,1782978630.541,1782978530.271 +claude-sonnet-5,scenario_076,0.034917,full,cache,same run,chatcmpl-a1f0c372-ff8e-441d-9f94-18d544d87f47,1782980398.594,1782979796.859 +claude-sonnet-5,scenario_077,0.014105,full,cache,same run,chatcmpl-1ead1bd1-376b-45fe-8dad-599790a290be,1782980398.635,1782979899.118 +claude-sonnet-5,scenario_078,0.013461,full,cache,same run,chatcmpl-83be4c02-4aed-4c14-ac07-bd271ebb51c4,1782980398.652,1782979955.733 +claude-sonnet-5,scenario_079,0.014319,full,cache,same run,chatcmpl-89a48391-de8f-4e5b-a480-fe521adf292f,1782980398.669,1782980015.737 +claude-sonnet-5,scenario_080,60.086180,partial,cache,same run,chatcmpl-1a8790f3-4377-4791-8019-f8ec29b6b6cc,1782980398.686,1782980097.957 +claude-sonnet-5,scenario_108,0.028484,full,cache,same run,chatcmpl-0d161de2-daa8-474e-86ce-34c9856b4694,1782980402.165,1782979767.387 +claude-sonnet-5,scenario_109,0.040883,full,cache,same run,chatcmpl-d0c0f199-c820-4f87-bb5c-2efa48a36bde,1782980402.197,1782979823.052 +claude-sonnet-5,scenario_110,0.015177,full,cache,same run,chatcmpl-3d121844-95b6-43b5-b3cc-6c2704ffc17a,1782980402.247,1782979963.904 +claude-sonnet-5,scenario_111,0.017898,full,cache,same run,chatcmpl-24573c76-0a32-4835-b898-c519e810f0bd,1782980402.266,1782980022.908 +claude-sonnet-5,scenario_112,43.325586,partial,cache,same run,chatcmpl-f3062f00-4cf9-431b-afb0-dc821d65fb1c,1782980402.287,1782980090.591 +deepseek-v4-pro,scenario_000,0.008594,full,cache,before the board rows,9ef950f9-59da-4f99-9719-3ec6880ca9bd,1783264088.998,1783260231.726 +deepseek-v4-pro,scenario_001,0.001136,full,cache,before the board rows,902543e5-70c5-485d-a8c4-045a6f87e621,1783264089.007,1783260449.021 +glm-5.2,scenario_000,0.016873,full,cache+openrouter_id+smoke_file,onboarding smoke,gen-1783264675-wpwzQ9139F0ixJwnYjU6,1783331940.272,1783264675.000 +glm-5.2,scenario_001,0.007684,full,cache+openrouter_id+smoke_file,onboarding smoke,gen-1783264883-yVzMGCt6HL5zeDEZ1x21,1783331940.290,1783264883.000 +glm-5.2,scenario_002,0.007263,full,cache+openrouter_id,before the board rows,gen-1783266583-dTu6RztBLx3TX7bklky6,1783331940.299,1783266583.000 +glm-5.2,scenario_003,0.005803,full,cache+openrouter_id,before the board rows,gen-1783267235-LNoAViUajkhCjI10rkp8,1783331940.308,1783267235.000 +glm-5.2,scenario_004,0.006318,full,cache+openrouter_id,before the board rows,gen-1783267319-8HesUaIcOkl8n3N2jZQD,1783331940.315,1783267319.000 +glm-5.2,scenario_005,0.008563,full,cache+openrouter_id,before the board rows,gen-1783267850-H6Ou4TYXVgSzFKaVnry8,1783331940.322,1783267850.000 +glm-5.2,scenario_007,0.007492,full,cache+openrouter_id,before the board rows,gen-1783268397-EAIbLTW31jvbpSZcDl2R,1783331940.332,1783268397.000 +glm-5.2,scenario_008,0.030048,full,cache+openrouter_id,before the board rows,gen-1783269453-flMrbpFjayacmwkJJZ2S,1783331940.340,1783269453.000 +glm-5.2,scenario_009,0.005580,full,cache+openrouter_id,before the board rows,gen-1783271962-0K2VHknWlD1llXiMua1Y,1783331940.374,1783271962.000 +glm-5.2,scenario_012,0.007640,full,cache+openrouter_id,before the board rows,gen-1783272224-ppxmCCZVbJx83yRWUeVl,1783331940.380,1783272224.000 +glm-5.2,scenario_013,0.003717,full,cache+openrouter_id,before the board rows,gen-1783273142-UHDA014Q8pNNm0GLS4Pc,1783331940.389,1783273142.000 +glm-5.2,scenario_014,0.006413,full,cache+openrouter_id,before the board rows,gen-1783273348-Xj1UqttlBvGfyFHeDfb5,1783331940.393,1783273348.000 +glm-5.2,scenario_015,0.005958,full,cache+openrouter_id,before the board rows,gen-1783274195-dnJUUoeOYDxKX2w7cxGT,1783331940.401,1783274195.000 +glm-5.2,scenario_016,0.004191,full,cache+openrouter_id,before the board rows,gen-1783274669-dP0cmBPjXNPVCiuoNXtu,1783331940.407,1783274669.000 +glm-5.2,scenario_018,0.004627,full,cache+openrouter_id,before the board rows,gen-1783274711-S631hE9SOEtd3TadO5NN,1783331940.412,1783274711.000 +glm-5.2,scenario_020,0.004869,full,cache+openrouter_id,before the board rows,gen-1783274850-DWxn5YzbSYEEPLef22j1,1783331940.418,1783274850.000 +glm-5.2,scenario_021,0.004261,full,cache+openrouter_id,before the board rows,gen-1783275012-LdzAUp9Z8K3VntB1Tcje,1783331940.424,1783275012.000 +glm-5.2,scenario_022,0.007669,full,cache+openrouter_id,before the board rows,gen-1783275191-F8AF2A7iJv0aw7QxzoGo,1783331940.429,1783275191.000 +glm-5.2,scenario_023,0.007770,full,cache+openrouter_id,before the board rows,gen-1783275779-PfA1SQChuC6s5eqhGhdT,1783331940.437,1783275779.000 +glm-5.2,scenario_025,0.007165,full,cache+openrouter_id,before the board rows,gen-1783284666-lR2mGGVNlZZCEfA1GoHS,1783331940.446,1783284666.000 +glm-5.2,scenario_026,0.011444,full,cache+openrouter_id,before the board rows,gen-1783285120-qHCnsZjM8p00HlPDvcq9,1783331940.454,1783285120.000 +glm-5.2,scenario_027,0.012097,full,cache+openrouter_id,before the board rows,gen-1783285649-q6J5w9m1opbs0yAwdbUd,1783331940.468,1783285649.000 +glm-5.2,scenario_028,0.020727,full,cache+openrouter_id,before the board rows,gen-1783285698-jHyIHr6uFjh0SwpmLEFI,1783331940.482,1783285698.000 +glm-5.2,scenario_029,0.003923,full,cache+openrouter_id,before the board rows,gen-1783285783-m1bwLFWLkrdc3r6iu6Xm,1783331940.506,1783285783.000 +glm-5.2,scenario_030,0.003935,full,cache+openrouter_id,before the board rows,gen-1783285895-O8sipUdlWTV1rnSMK8m6,1783331940.511,1783285895.000 +glm-5.2,scenario_031,0.017882,full,cache+openrouter_id,before the board rows,gen-1783286076-BMAOcIJeOd1MfEPLqv41,1783331940.269,1783286076.000 +glm-5.2,scenario_032,0.023159,full,cache+openrouter_id,before the board rows,gen-1783286232-gAxiAq8YrTbMPsHXqM7U,1783331940.288,1783286232.000 +glm-5.2,scenario_033,0.007156,full,cache+openrouter_id,before the board rows,gen-1783287973-UvbqyCUGudTN6kBzTnmQ,1783331940.313,1783287973.000 +glm-5.2,scenario_036,0.007160,full,cache+openrouter_id,before the board rows,gen-1783288191-2Ko3ftXnBRVOTQdVabuL,1783331940.322,1783288191.000 +glm-5.2,scenario_037,0.004868,full,cache+openrouter_id,before the board rows,gen-1783289078-HSVaQRabeUmKRRnHWzMb,1783331940.330,1783289078.000 +glm-5.2,scenario_038,0.018407,full,cache+openrouter_id,before the board rows,gen-1783289456-0GCSJaw9iMhZShzp6Ucr,1783331940.335,1783289456.000 +glm-5.2,scenario_039,0.006487,full,cache+openrouter_id,before the board rows,gen-1783290940-JehQ5uSX1vbRkVxdX7wu,1783331940.356,1783290940.000 +glm-5.2,scenario_040,0.006169,full,cache+openrouter_id,before the board rows,gen-1783291214-HFIZyXsUnwrbzgNNKf4A,1783331940.363,1783291214.000 +glm-5.2,scenario_042,0.006445,full,cache+openrouter_id,before the board rows,gen-1783291447-f6YPEnMXI1OdCZfRdL49,1783331940.370,1783291447.000 +glm-5.2,scenario_043,0.004685,full,cache+openrouter_id,before the board rows,gen-1783291868-Jab45K33Zp8c5MT8pTVz,1783331940.378,1783291868.000 +glm-5.2,scenario_044,0.004609,full,cache+openrouter_id,before the board rows,gen-1783292488-ZL2duvKdwMKOD22SfKEH,1783331940.383,1783292488.000 +glm-5.2,scenario_045,0.006946,full,cache+openrouter_id,before the board rows,gen-1783292668-DUKUAUgjj8BOu920xIcQ,1783331940.389,1783292668.000 +glm-5.2,scenario_046,0.015504,full,cache+openrouter_id,before the board rows,gen-1783293350-eVBmjZqGx5ilW5av8q24,1783331940.396,1783293350.000 +glm-5.2,scenario_048,0.004639,full,cache+openrouter_id,before the board rows,gen-1783294113-gigBTYbMi87Slhljic2t,1783331940.414,1783294113.000 +glm-5.2,scenario_049,0.013697,full,cache+openrouter_id,before the board rows,gen-1783294772-VIbVIaIun5pCLxBqcI85,1783331940.419,1783294772.000 +glm-5.2,scenario_051,0.007044,full,cache+openrouter_id,before the board rows,gen-1783295024-4z0xWnmtg6rgIV4vMgAK,1783331940.435,1783295024.000 +glm-5.2,scenario_052,0.005729,full,cache+openrouter_id,before the board rows,gen-1783295505-J5mDMuhr39zvV9Pa46rN,1783331940.443,1783295505.000 +glm-5.2,scenario_053,0.003807,full,cache+openrouter_id,before the board rows,gen-1783295787-RkLKvfcewg0shPE6dOCl,1783331940.450,1783295787.000 +glm-5.2,scenario_054,0.011648,full,cache+openrouter_id,before the board rows,gen-1783295869-je6tQQQ8sm4RZaKTZu6O,1783331940.455,1783295869.000 +glm-5.2,scenario_055,0.004366,full,cache+openrouter_id,before the board rows,gen-1783296736-ZiHKeICIFbSE5NHxY9Np,1783331940.468,1783296736.000 +glm-5.2,scenario_056,0.008079,full,cache+openrouter_id,before the board rows,gen-1783296833-mndjxQmm5xxublVHP0RP,1783331940.473,1783296833.000 +glm-5.2,scenario_057,0.010171,full,cache+openrouter_id,before the board rows,gen-1783297498-unnyjX1F09SBP25meHPN,1783331940.482,1783297498.000 +glm-5.2,scenario_059,0.004785,full,cache+openrouter_id,before the board rows,gen-1783298873-ATvwj93EywbbNw0JP3Sr,1783331940.493,1783298873.000 +glm-5.2,scenario_060,0.006202,full,cache+openrouter_id,before the board rows,gen-1783298985-mzDG731e9oEjgBkIjxHw,1783331940.499,1783298985.000 +glm-5.2,scenario_062,0.004458,full,cache+openrouter_id,before the board rows,gen-1783299385-YOc1JzziCuCiyegqFuOO,1783331940.506,1783299385.000 +glm-5.2,scenario_064,0.037649,full,cache+openrouter_id,before the board rows,gen-1783266583-QdBf21Hct0zzY9pXzIZZ,1783331940.269,1783266583.000 +glm-5.2,scenario_066,0.007620,full,cache+openrouter_id,before the board rows,gen-1783268104-mXVnAaUCdHg4IMUH9S10,1783331940.311,1783268104.000 +glm-5.2,scenario_067,0.006880,full,cache+openrouter_id,before the board rows,gen-1783268952-ivYs5IMugKXLxPsj3qU1,1783331940.319,1783268952.000 +glm-5.2,scenario_068,0.006608,full,cache+openrouter_id,before the board rows,gen-1783269620-5G9p4Hu1BhI1ZZcROLMG,1783331940.327,1783269620.000 +glm-5.2,scenario_070,0.005462,full,cache+openrouter_id,before the board rows,gen-1783270125-HFLPRQj4UQC7NrWBcAc6,1783331940.335,1783270125.000 +glm-5.2,scenario_071,0.004727,full,cache+openrouter_id,before the board rows,gen-1783270368-OqxlidC2Sus8r9gXvZ2S,1783331940.341,1783270368.000 +glm-5.2,scenario_072,0.005811,full,cache+openrouter_id,before the board rows,gen-1783284434-C6r0fYsrE1BfqQriqVsP,1783331940.347,1783284434.000 +glm-5.2,scenario_073,0.005650,full,cache+openrouter_id,before the board rows,gen-1783284602-0b0Ked8ikI4CtMZrW1z1,1783331940.353,1783284602.000 +glm-5.2,scenario_074,0.005542,full,cache+openrouter_id,before the board rows,gen-1783284979-Mh64hhfSPmjlh910qTK6,1783331940.360,1783284979.000 +glm-5.2,scenario_075,0.007999,full,cache+openrouter_id,before the board rows,gen-1783285193-B1zfvrUkatvGpxORmdLM,1783331940.367,1783285193.000 +glm-5.2,scenario_076,0.007836,full,cache+openrouter_id,before the board rows,gen-1783285578-gwmPwYJr60p7p0bsbpsp,1783331940.375,1783285578.000 +glm-5.2,scenario_077,0.006071,full,cache+openrouter_id,before the board rows,gen-1783285905-MEbQM65Gj0leNbJPbH0L,1783331940.385,1783285905.000 +glm-5.2,scenario_078,0.004059,full,cache+openrouter_id,before the board rows,gen-1783286197-8bKqbLJsJxOrJKOIUwsk,1783331940.392,1783286197.000 +glm-5.2,scenario_079,0.009051,full,cache+openrouter_id,before the board rows,gen-1783286301-6tjVvXwGUY0OKeMvPTMG,1783331940.396,1783286301.000 +glm-5.2,scenario_080,0.010218,full,cache+openrouter_id,before the board rows,gen-1783286946-NGbwqQc0oX0CVmUQ2wWO,1783331940.406,1783286946.000 +glm-5.2,scenario_081,0.005579,full,cache+openrouter_id,before the board rows,gen-1783287433-kN9fZx22inubux95Wa7M,1783331940.418,1783287433.000 +glm-5.2,scenario_082,0.016309,full,cache+openrouter_id,before the board rows,gen-1783288326-mIJUhf8aXFhHcnwd1rIi,1783331940.425,1783288326.000 +glm-5.2,scenario_083,0.004758,full,cache+openrouter_id,before the board rows,gen-1783289434-n33lp4NDlIqs11pBA3Nk,1783331940.443,1783289434.000 +glm-5.2,scenario_084,0.005038,full,cache+openrouter_id,before the board rows,gen-1783289595-bVOV59e4DFEKfYfFbSlw,1783331940.449,1783289595.000 +glm-5.2,scenario_085,0.005083,full,cache+openrouter_id,before the board rows,gen-1783289830-KsJDuoKjQdmuL61aH94h,1783331940.454,1783289830.000 +glm-5.2,scenario_086,0.007000,full,cache+openrouter_id,before the board rows,gen-1783290022-D6zvgFVuW5HI1HmAGUaY,1783331940.460,1783290022.000 +glm-5.2,scenario_088,0.006379,full,cache+openrouter_id,before the board rows,gen-1783290330-GxRGZ71gWs0d88KAyJFy,1783331940.468,1783290330.000 +glm-5.2,scenario_089,0.006977,full,cache+openrouter_id,before the board rows,gen-1783291192-6Wrf9yE72w2bAD1MUOeK,1783331940.476,1783291192.000 +glm-5.2,scenario_090,0.003895,full,cache+openrouter_id,before the board rows,gen-1783291765-khoqjZLbGGQEp8Ez3B6B,1783331940.484,1783291765.000 +glm-5.2,scenario_091,0.004962,full,cache+openrouter_id,before the board rows,gen-1783292374-czt8mo2NroyfiRmGkZjQ,1783331940.488,1783292374.000 +glm-5.2,scenario_092,0.015390,full,cache+openrouter_id,before the board rows,gen-1783292562-BzyJY9NNHQF47MpCUI9N,1783333189.265,1783292562.000 +glm-5.2,scenario_093,0.007451,full,cache+openrouter_id,before the board rows,gen-1783292717-jSvawisBww1r8gqsMdu6,1783333189.281,1783292717.000 +glm-5.2,scenario_095,0.010188,full,cache+openrouter_id,before the board rows,gen-1783293828-bUEECmItUiDAZiJezOwC,1783333189.290,1783293828.000 +glm-5.2,scenario_098,0.005563,full,cache+openrouter_id,before the board rows,gen-1783294390-KH8cjOLjuqhmfkpVrPZK,1783333189.301,1783294390.000 +glm-5.2,scenario_099,0.013693,full,cache+openrouter_id,before the board rows,gen-1783331940-Aebly3eW1KYrgKWRZtu1,1783333189.308,1783331940.000 +glm-5.2,scenario_100,39.829285,partial,cache+openrouter_id,same run,gen-1783332255-zOaEFi0cUmXvZnyfHh4a,1783333189.323,1783332255.000 +glm-5.3,scenario_007,670.949044,partial,openrouter_id+spend_ledger,same run,gen-1788443437-kHB3di20R9eBcnvuAX6S,1788522468.992,1788443437.000 +glm-5.3,scenario_028,280.114137,partial,openrouter_id+spend_ledger,same run,gen-1788444666-BfwlvmD6jq7BRSDJbm3s,1788531688.961,1788444666.000 +glm-5.3,scenario_037,350.955777,partial,openrouter_id+spend_ledger,same run,gen-1788445259-ZoY2rZUiydfACfOec5d5,1788529640.906,1788445259.000 +glm-5.3,scenario_042,366.712537,partial,openrouter_id+spend_ledger,same run,gen-1788445513-9nhTScDFLYdxyAoTSg0n,1788535656.584,1788445513.000 +glm-5.3,scenario_043,55.408105,partial,openrouter_id+spend_ledger,same run,gen-1788445558-ZUZs55vSz5c1O3kHHDVM,1788523168.543,1788445558.000 +glm-5.3,scenario_046,102.070615,partial,openrouter_id+spend_ledger,same run,gen-1788445754-PFwn6BupdP1KRTp6UFKo,1788523263.040,1788445754.000 +glm-5.3,scenario_057,546.675098,partial,openrouter_id+spend_ledger,same run,gen-1788446232-xmkvi6ebZ6muMtxWdjmh,1788523615.857,1788446232.000 +glm-5.3,scenario_062,8.507851,partial,openrouter_id+spend_ledger,same run,gen-1788523847-iwtKxP7WTlCPcaYP5q2Y,1788556429.722,1788523847.000 +glm-5.3,scenario_064,3671.612833,partial,openrouter_id+spend_ledger,same run,gen-1788529640-XjMEO5vxBk5wuCwEr5NF,1788563482.334,1788529640.000 +glm-5.3,scenario_075,259.822257,partial,openrouter_id+spend_ledger,same run,gen-1788446387-clYogpRgQ6SiBl7JPxZA,1788524652.971,1788446387.000 +glm-5.3,scenario_086,112.633520,partial,openrouter_id+spend_ledger,same run,gen-1788525085-dTSkM98ErRDqCGEMumnx,1788530003.948,1788525085.000 +glm-5.3,scenario_091,45.313927,partial,openrouter_id+spend_ledger,same run,gen-1788525491-pK5oEjPwwzodUcGmRe9T,1788530129.901,1788525491.000 +glm-5.3,scenario_099,124.193102,partial,openrouter_id+spend_ledger,same run,gen-1788526049-zCakBsDZh7x7CoHgXNRV,1788530559.337,1788526049.000 +glm-5.3,scenario_102,158.606853,partial,openrouter_id+spend_ledger,same run,gen-1788526680-wE2KxUb2HCghIoIRzMxD,1788530559.338,1788526680.000 +glm-5.3,scenario_108,71.216645,partial,openrouter_id+spend_ledger,same run,gen-1788526914-2Ipat7Pm9uGb7w7LY3fH,1788531983.981,1788526914.000 +glm-5.3,scenario_117,84.563591,partial,openrouter_id+spend_ledger,same run,gen-1788446575-uo8BgHZi21mleE4CMwSr,1788527751.585,1788446575.000 +glm-5.3,scenario_119,616.636458,partial,openrouter_id+spend_ledger,same run,gen-1788527854-Emxg3msssl6gqhXCtU1w,1788532069.451,1788527854.000 +glm-5.3,scenario_122,389.279603,partial,openrouter_id+spend_ledger,same run,gen-1788446600-cgKyESrmHUNAnL2gAMOd,1788533524.876,1788446600.000 +gpt-5.6-luna,scenario_000,0.004324,full,openai_id,before the board rows,resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMDQ3OGVhN2ZkMDAyNjVhMDAwNmE0ZmVkMWRkNTY4ODE5ZWE3OGE5M2FhZmRmMWJlN2Q=,1783623249.474,1783622941.000 +gpt-5.6-luna,scenario_001,0.005954,full,openai_id,before the board rows,resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMDJjMDJiZDY3NWU5MWJjNjAwNmE0ZmVkMmE0YTcwODFhMWJlYmZkYjA4NjUwMzI3NDE=,1783623249.469,1783622954.000 +gpt-5.6-sol,scenario_000,0.006776,full,openai_id,before the board rows,resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMDhhMzE1ODVmNDJmZWUzMDAwNmE0ZmVjYzM4NTk4ODE5MTgyNWUzM2EwZmY4MDExZDA=,1783623303.475,1783622851.000 +gpt-5.6-sol,scenario_001,0.007025,full,openai_id,before the board rows,resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGUxYmY2N2JjNzM4MGY1NzAwNmE0ZmVjZGVjNjU4ODE5ZDg3OWNlYTAzMjIwZjhmMzU=,1783623303.479,1783622878.000 +gpt-5.6-terra,scenario_000,0.006326,full,openai_id,before the board rows,resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGE2ODVhMDk0NmQ1NTE1NDAwNmE0ZmVjZjk2MWE4ODE5Mjg5YTBjN2U1NzQxM2RkYzc=,1783623303.487,1783622905.000 +gpt-5.6-terra,scenario_001,0.006229,full,openai_id,before the board rows,resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMDEyYzY4OWMyMjQ0YTI4ZDAwNmE0ZmVkMGJlZWZjODE5Zjg2NmMwMTBlMWU2Mzg5OTE=,1783623303.477,1783622923.000 +grok-4.3,scenario_000,0.021931,untraced,,untraced,4a87ac1c-9f23-909f-9f8b-d652a150e386,, +kimi-k2.6,scenario_000,0.025560,full,cache+openrouter_id+smoke_file,onboarding smoke,gen-1783264675-MvBkFkbisFcUSVwUKEGM,1783420244.711,1783264675.000 +kimi-k2.6,scenario_001,0.022980,full,cache+openrouter_id+smoke_file,onboarding smoke,gen-1783265511-1BzQDC5BmQGyJzLlgkiw,1783420244.722,1783265511.000 +kimi-k2.6,scenario_002,0.020134,full,cache+openrouter_id,before the board rows,gen-1783266620-jkSxCNXhpwQX7nDLk4BK,1783420244.733,1783266620.000 +kimi-k2.6,scenario_003,0.020995,full,cache+openrouter_id,before the board rows,gen-1783267399-0lvWvDEXPVuWJ16HCkDt,1783420244.720,1783267399.000 +kimi-k2.6,scenario_004,0.019908,full,cache+openrouter_id,before the board rows,gen-1783268478-2A4MEh5EGJN6dACylNNS,1783420257.089,1783268478.000 +kimi-k2.6,scenario_005,0.030770,full,cache+openrouter_id,before the board rows,gen-1783268963-8QkfMfC6O7sMl3Limylo,1783420257.110,1783268963.000 +kimi-k2.6,scenario_007,0.031346,full,cache+openrouter_id,before the board rows,gen-1783271165-hX6BAmHVaFOBEgYBhN2P,1783420257.107,1783271165.000 +kimi-k2.6,scenario_008,0.067598,full,cache+openrouter_id,before the board rows,gen-1783272193-tgrp8KZffPV9vnXUEBnc,1783420257.093,1783272193.000 +kimi-k2.6,scenario_009,0.036505,full,cache+openrouter_id,before the board rows,gen-1783299623-eSJlBQJZNfVWcwKLMz88,1783420269.342,1783299623.000 +kimi-k2.6,scenario_012,0.047160,full,cache+openrouter_id,before the board rows,gen-1783300224-GPUwL5PUemor9fhyMDk3,1783420269.345,1783300224.000 +kimi-k2.6,scenario_013,0.023155,full,cache+openrouter_id,before the board rows,gen-1783301511-C9CR4I2NgE6Avdym8Ka4,1783420269.348,1783301511.000 +kimi-k2.6,scenario_014,0.036954,full,cache+openrouter_id,before the board rows,gen-1783362292-yupk4l3nmTvH2q949EqA,1783420269.342,1783362292.000 +kimi-k2.6,scenario_015,0.032797,full,cache+openrouter_id,before the board rows,gen-1783362896-CvVE77Y0kcpcq7ls9mhg,1783420282.559,1783362896.000 +kimi-k2.6,scenario_016,0.025906,full,cache+openrouter_id,before the board rows,gen-1783364817-0AYQSSQoYEYwj2ZkSuT7,1783420282.557,1783364817.000 +kimi-k2.6,scenario_018,0.030026,full,cache+openrouter_id,before the board rows,gen-1783365746-g0EGwxWi24XYDtwjkkMC,1783420282.559,1783365746.000 +kimi-k2.6,scenario_020,0.026624,full,cache+openrouter_id,before the board rows,gen-1783367534-LyfIwQjJNm2hrc0KYe2Y,1783420282.576,1783367534.000 +kimi-k2.6,scenario_021,0.036108,full,cache+openrouter_id,before the board rows,gen-1783368779-E2S1atFm3KGzYAk03zyT,1783420296.925,1783368779.000 +kimi-k2.6,scenario_022,0.037522,full,cache+openrouter_id,before the board rows,gen-1783355350-MkBJxY74vWFvuKlG7Xhp,1783420296.842,1783355350.000 +kimi-k2.6,scenario_023,0.039451,full,cache+openrouter_id,before the board rows,gen-1783357125-XbMLItNn1lOHOkXFvTlB,1783420296.844,1783357125.000 +kimi-k2.6,scenario_025,0.031495,full,cache+openrouter_id,before the board rows,gen-1783363013-hnaF3lovTZdUb7GYQAjt,1783420296.840,1783363013.000 +kimi-k2.6,scenario_026,0.056106,full,cache+openrouter_id,before the board rows,gen-1783364791-McGn6GLKUem52wjxgGAf,1783420309.768,1783364791.000 +kimi-k2.6,scenario_027,0.023750,full,cache+openrouter_id,before the board rows,gen-1783368433-8t5lHK3JUBnlp0n3kVHB,1783420309.774,1783368433.000 +kimi-k2.6,scenario_028,0.035226,full,cache+openrouter_id,before the board rows,gen-1783369848-4INgAbdDIuSemNHK4vXC,1783420309.768,1783369848.000 +kimi-k2.6,scenario_029,0.021021,full,cache+openrouter_id,before the board rows,gen-1783372098-TqKLVhyHVjQ5SeMVjsjt,1783420309.764,1783372098.000 +kimi-k2.6,scenario_030,0.028253,full,cache+openrouter_id,before the board rows,gen-1783387689-VlK6VB2iHZZgUunthzOL,1783420321.958,1783387689.000 +kimi-k2.6,scenario_031,0.018525,full,cache+openrouter_id,before the board rows,gen-1783266620-zn2a2MqqklxTWmVgi340,1783420321.956,1783266620.000 +kimi-k2.6,scenario_032,0.047279,full,cache+openrouter_id,before the board rows,gen-1783299409-kcjTZm7JVZ4XSH4kZW0R,1783420321.959,1783299409.000 +kimi-k2.6,scenario_033,0.023807,full,cache+openrouter_id,before the board rows,gen-1783388519-XpvVogYeMWers7WPzmqo,1783420321.958,1783388519.000 +kimi-k2.6,scenario_036,0.028435,full,cache+openrouter_id,before the board rows,gen-1783389273-Ylxv6Vaisri1QSSU7HtC,1783420455.259,1783389273.000 +kimi-k2.6,scenario_037,0.027188,full,cache+openrouter_id,before the board rows,gen-1783397356-uKARp2OvQialfu8XLeZj,1783420455.267,1783397356.000 +kimi-k2.6,scenario_038,0.045205,full,cache+openrouter_id,before the board rows,gen-1783403970-BnILa1kDEZNNSJTJEmcl,1783420455.259,1783403970.000 +kimi-k2.6,scenario_039,414.221534,partial,cache+openrouter_id,before the board rows,gen-1783407828-uPMHyKrXftW2GsWL7lMa,1783420455.262,1783407828.000 +kimi-k2.6,scenario_043,0.031529,full,cache+openrouter_id,before the board rows,gen-1783355350-wVYUwd3KlIwr62TNmEK9,1783420467.310,1783355350.000 +kimi-k2.6,scenario_044,0.026887,full,cache+openrouter_id,before the board rows,gen-1783356942-3K8MsVXqTCoxVeq0piNH,1783420476.395,1783356942.000 +kimi-k2.6,scenario_045,0.033757,full,cache+openrouter_id,before the board rows,gen-1783358186-OofNjEsOgja4q1FW2bAw,1783420486.332,1783358186.000 +kimi-k2.6,scenario_046,0.048324,full,cache+openrouter_id,before the board rows,gen-1783362292-1heJXaEJrIcaK3vkMHY6,1783420496.348,1783362292.000 +kimi-k2.6,scenario_048,0.023369,full,cache+openrouter_id,before the board rows,gen-1783363856-XQDv8RBTnfsjAz6Wpzxn,1783420506.712,1783363856.000 +kimi-k2.6,scenario_049,0.027492,full,cache+openrouter_id,before the board rows,gen-1783364291-9MscP3LsthSeEuqHe7s5,1783420516.718,1783364291.000 +kimi-k2.6,scenario_051,0.030375,full,cache+openrouter_id,before the board rows,gen-1783365389-QMJYe1zarXx0s5GDxxIm,1783420526.779,1783365389.000 +kimi-k2.6,scenario_052,0.029431,full,cache+openrouter_id,before the board rows,gen-1783366643-WEohM4ToBwZRv6jk4T74,1783420536.891,1783366643.000 +kimi-k2.6,scenario_053,0.024262,full,cache+openrouter_id,before the board rows,gen-1783367653-BjdCYhXUnAxgRX8z594w,1783420547.236,1783367653.000 +kimi-k2.6,scenario_054,0.037881,full,cache+openrouter_id,before the board rows,gen-1783369166-CzuxJ6WPLTfQysIPvOiw,1783420557.056,1783369166.000 +kimi-k2.6,scenario_055,0.019701,full,cache+openrouter_id,before the board rows,gen-1783371681-4L2RrcRDsL1gpNW55aPS,1783420566.870,1783371681.000 +kimi-k2.6,scenario_056,0.043097,full,cache+openrouter_id,before the board rows,gen-1783387959-cHoq0FsqgRA7uYEuSXfG,1783420576.901,1783387959.000 +kimi-k2.6,scenario_057,0.031445,full,cache+openrouter_id,before the board rows,gen-1783390372-mb7POQAXCoYPVLEHKHev,1783420586.812,1783390372.000 +kimi-k2.6,scenario_059,0.019348,full,cache+openrouter_id,before the board rows,gen-1783392683-WI3bfvIbtD9PfkEj3WNu,1783420596.721,1783392683.000 +kimi-k2.6,scenario_060,0.016183,full,cache+openrouter_id,before the board rows,gen-1783394046-tZ4skudh5inYY0CJwkaE,1783420606.849,1783394046.000 +kimi-k2.6,scenario_062,0.029063,full,cache+openrouter_id,before the board rows,gen-1783395526-hWQ8K7Lwt9zDzlsRTHDv,1783420616.797,1783395526.000 +kimi-k2.6,scenario_064,0.061694,full,cache+openrouter_id,before the board rows,gen-1783266620-06JXdv6urXLyaOi10cc2,1783420626.914,1783266620.000 +kimi-k2.6,scenario_066,0.021828,full,cache+openrouter_id,before the board rows,gen-1783299784-bAYcCcoQBPsKybwhzUWT,1783420636.966,1783299784.000 +kimi-k2.6,scenario_067,0.031418,full,cache+openrouter_id,before the board rows,gen-1783300618-Pt5iY9R6THgh171NHjKv,1783420646.901,1783300618.000 +kimi-k2.6,scenario_068,0.022989,full,cache+openrouter_id,before the board rows,gen-1783387176-i2JnoVlhpl9BKRQvs3tl,1783420656.971,1783387176.000 +kimi-k2.6,scenario_070,0.023721,full,cache+openrouter_id,before the board rows,gen-1783388718-UvkaYcPiOLsiUggnhzzu,1783420666.984,1783388718.000 +kimi-k2.6,scenario_071,0.020934,full,cache+openrouter_id,before the board rows,gen-1783396400-tjo1NPH4Xx49m4IvyV0V,1783420676.919,1783396400.000 +kimi-k2.6,scenario_072,0.024056,full,cache+openrouter_id,before the board rows,gen-1783397131-vmKGu6jqz92c0GHjI1Td,1783420687.177,1783397131.000 +kimi-k2.6,scenario_073,0.021378,full,cache+openrouter_id,before the board rows,gen-1783398699-it8NaXfXvd3vnX9yVdmF,1783420697.243,1783398699.000 +kimi-k2.6,scenario_074,0.029813,full,cache+openrouter_id,before the board rows,gen-1783399020-igoJsO6G8fmVZIbT1qeB,1783420707.170,1783399020.000 +kimi-k2.6,scenario_075,0.023099,full,cache+openrouter_id,before the board rows,gen-1783399775-PpDxK7O9uAbITO5ocuLj,1783420717.134,1783399775.000 +kimi-k2.6,scenario_076,0.036734,full,cache+openrouter_id,before the board rows,gen-1783404273-OcJ55m2J45uRkvjkDMMn,1783420727.193,1783404273.000 +kimi-k2.6,scenario_077,0.037854,full,cache+openrouter_id,before the board rows,gen-1783404763-cSYJjqxkdkuGE3SqIEg4,1783420737.325,1783404763.000 +kimi-k2.6,scenario_078,236.701054,partial,cache+openrouter_id,same run,gen-1783421113-OUloMQcpdEMLMoimeUDm,1783432001.450,1783421113.000 +kimi-k2.6,scenario_083,0.022446,full,cache+openrouter_id,before the board rows,gen-1783355350-P70VIc2gK4rj7XwO3GLE,1783423066.514,1783355350.000 +kimi-k2.6,scenario_084,0.023209,full,cache+openrouter_id,before the board rows,gen-1783356457-7sr7d87oOGyQso7wY1fh,1783423077.727,1783356457.000 +kimi-k2.6,scenario_085,0.024019,full,cache+openrouter_id,before the board rows,gen-1783358252-IveYSn7P4Uvb0GS7JLuK,1783423089.799,1783358252.000 +kimi-k2.6,scenario_086,0.040194,full,cache+openrouter_id,before the board rows,gen-1783359454-YsTX5gIFRb1yJumxlK8D,1783423099.744,1783359454.000 +kimi-k2.6,scenario_088,0.026725,full,cache+openrouter_id,before the board rows,gen-1783387443-K0Pxvh2OWOZLwUQvxcbL,1783423110.443,1783387443.000 +kimi-k2.6,scenario_089,0.034123,full,cache+openrouter_id,before the board rows,gen-1783388646-ChTE56zHNkyy5TDTp8jw,1783423121.315,1783388646.000 +kimi-k2.6,scenario_090,0.019432,full,cache+openrouter_id,before the board rows,gen-1783390934-eHovPWjELurt3tmSkvbY,1783423131.318,1783390934.000 +kimi-k2.6,scenario_091,0.032730,full,cache+openrouter_id,before the board rows,gen-1783396503-Ao1OayHmvfmRc8BjA0x5,1783423141.122,1783396503.000 +kimi-k2.6,scenario_092,0.029124,full,cache+openrouter_id,before the board rows,gen-1783266620-3ziZe5vQVmJXy2b1W9le,1783423151.088,1783266620.000 +kimi-k2.6,scenario_093,0.027950,full,cache+openrouter_id,before the board rows,gen-1783299409-nViiqzgc40F5W7xvIZk3,1783423161.109,1783299409.000 +kimi-k2.6,scenario_095,0.028090,full,cache+openrouter_id,before the board rows,gen-1783300266-4YWBfWhmqrwQbJgrrkF5,1783423171.313,1783300266.000 +kimi-k2.6,scenario_098,0.020831,full,cache+openrouter_id,before the board rows,gen-1783300802-hiHJAcilyIvlnxuz1wP0,1783423175.395,1783300802.000 +kimi-k2.6,scenario_099,0.044692,full,cache+openrouter_id,before the board rows,gen-1783398037-GWXnGXORP9FcoerMFGYg,1783423181.431,1783398037.000 +kimi-k2.6,scenario_100,0.044659,full,cache+openrouter_id,before the board rows,gen-1783404571-dsVPl6Vla5lrjQnaKSL5,1783423185.501,1783404571.000 +kimi-k2.6,scenario_107,0.033450,full,cache+openrouter_id,before the board rows,gen-1783355350-2YwODSzJOJBoA4GjLeyT,1783423908.758,1783355350.000 +kimi-k2.6,scenario_108,0.039933,full,cache+openrouter_id,before the board rows,gen-1783356685-Kn6JyttSXOAJuTnqS3fc,1783423921.083,1783356685.000 +kimi-k2.6,scenario_109,776.957269,partial,cache+openrouter_id,before the board rows,gen-1783358519-0nhzWBSTP5yVevGnCHWn,1783438481.948,1783358519.000 +kimi-k2.6,scenario_123,279.558510,partial,cache+openrouter_id,same run,gen-1783430400-6DcG4kkUriXTUnw43sNo,1783432713.641,1783430400.000 +kimi-k3,scenario_008,1591.501587,partial,cache+openrouter_id,same run,gen-1784382637-oyVCDk8yDTL7Df16gsc4,1784426539.404,1784382637.000 +kimi-k3,scenario_026,693.837113,partial,cache+openrouter_id,same run,gen-1784384047-3E45FwuF1bag4PUOEZNH,1784426539.401,1784384047.000 +minimax-m3,scenario_000,0.014041,full,cache+openrouter_id,before the board rows,gen-1783264298-9iQ10dYE9ySSbhPXFHuq,1783276494.117,1783264298.000 +minimax-m3,scenario_001,0.001252,full,cache+openrouter_id,before the board rows,gen-1783264305-fZUCVHiqrAEKj0tWQvsB,1783276494.132,1783264305.000 +minimax-m3,scenario_002,0.001052,full,cache+openrouter_id,before the board rows,gen-1783264313-4eTNra94ubxqvIfr7BdD,1783276494.133,1783264313.000 +minimax-m3,scenario_003,0.001275,full,cache+openrouter_id,before the board rows,gen-1783264325-Brw7uwILGQLGjSqSzBDU,1783276494.135,1783264325.000 +minimax-m3,scenario_004,0.001078,full,cache+openrouter_id,before the board rows,gen-1783264330-II0NZnN7LDc1zx2qhyUE,1783276494.136,1783264330.000 +minimax-m3,scenario_005,0.004243,full,cache+openrouter_id,before the board rows,gen-1783264341-xwX8ADdMN1vnme4E2f96,1783276494.138,1783264341.000 +minimax-m3,scenario_007,0.000990,full,cache+openrouter_id,before the board rows,gen-1783264355-xs03DPfLvEgYbnU5m34g,1783276494.143,1783264355.000 +minimax-m3,scenario_008,0.002376,full,cache+openrouter_id,before the board rows,gen-1783264362-oPNwqutlfVJHMEiWzoiq,1783276494.145,1783264362.000 +minimax-m3,scenario_009,0.001820,full,cache+openrouter_id,before the board rows,gen-1783264628-diJqtdgjjZJs1Yb8CcLS,1783276494.148,1783264628.000 +minimax-m3,scenario_012,0.001590,full,cache+openrouter_id,before the board rows,gen-1783264781-g9gBXVuoLlBIID4vC1yM,1783276494.150,1783264781.000 +minimax-m3,scenario_013,0.001412,full,cache+openrouter_id,before the board rows,gen-1783264860-nCOXnMppPB5wNbruELOM,1783276494.152,1783264860.000 +minimax-m3,scenario_014,0.000728,full,cache+openrouter_id,before the board rows,gen-1783264870-cBhmqfQNJ1F2XBYRTUd4,1783276494.154,1783264870.000 +minimax-m3,scenario_015,0.000689,full,cache+openrouter_id,before the board rows,gen-1783264883-OQzrFUHtLtid6JgXtPMR,1783276494.155,1783264883.000 +minimax-m3,scenario_016,0.000713,full,cache+openrouter_id,before the board rows,gen-1783264893-oq3eUcaRMtk7W0XxYat0,1783276494.156,1783264893.000 +minimax-m3,scenario_018,0.000704,full,cache+openrouter_id,before the board rows,gen-1783264905-OoKwsbJYdm3aVZGhu1gn,1783276494.157,1783264905.000 +minimax-m3,scenario_020,0.000783,full,cache+openrouter_id,before the board rows,gen-1783264910-yR0yOceot127VZuwneCd,1783276494.158,1783264910.000 +minimax-m3,scenario_021,0.000587,full,cache+openrouter_id,before the board rows,gen-1783264913-YiH6xq5Gra73oYTZleBO,1783276494.159,1783264913.000 +minimax-m3,scenario_022,0.000976,full,cache+openrouter_id,before the board rows,gen-1783264922-hxF931ATMcqMPCp30eo3,1783276494.160,1783264922.000 +minimax-m3,scenario_023,0.002535,full,cache+openrouter_id,before the board rows,gen-1783264929-v3bK3u3swxM6qYMMpgAJ,1783276494.161,1783264929.000 +minimax-m3,scenario_025,0.000733,full,cache+openrouter_id,before the board rows,gen-1783265126-KbnxrJaUFXGaeTQokDpf,1783276494.164,1783265126.000 +minimax-m3,scenario_026,15.547861,partial,cache+openrouter_id,before the board rows,gen-1783265136-hPy76ijxLRfWcnjxyb4P,1783276494.165,1783265136.000 +minimax-m3,scenario_064,0.015252,full,cache+openrouter_id,before the board rows,gen-1783264298-blf0WsiZOgsExfUmXTeW,1783276494.118,1783264298.000 +minimax-m3,scenario_066,0.001026,full,cache+openrouter_id,before the board rows,gen-1783264309-cHoPX5zcLrrDVACVBRTc,1783276494.134,1783264309.000 +minimax-m3,scenario_067,0.002441,full,cache+openrouter_id,before the board rows,gen-1783264315-d2qBotEM2cvL3WIFyb3K,1783276494.135,1783264315.000 +minimax-m3,scenario_068,0.002468,full,cache+openrouter_id,before the board rows,gen-1783264563-KJuskqm5bKLYiWkUjwB0,1783276494.138,1783264563.000 +minimax-m3,scenario_070,0.001029,full,cache+openrouter_id,before the board rows,gen-1783264720-tLcUSD4a86uSiMG5Cs6R,1783276494.141,1783264720.000 +minimax-m3,scenario_071,0.000836,full,cache+openrouter_id,before the board rows,gen-1783264728-OLjqoe1FQsVqUnBrAgcE,1783276494.142,1783264728.000 +minimax-m3,scenario_072,0.000832,full,cache+openrouter_id,before the board rows,gen-1783264734-vvIzLw8SEBCynOblFfVG,1783276494.143,1783264734.000 +minimax-m3,scenario_073,0.001550,full,cache+openrouter_id,before the board rows,gen-1783264746-p4OfKzCRa6VIiEDJh3Bv,1783276494.144,1783264746.000 +minimax-m3,scenario_074,0.000769,full,cache+openrouter_id,before the board rows,gen-1783264757-ulV63c3CWtxwkhylGD28,1783276494.146,1783264757.000 +minimax-m3,scenario_075,0.001613,full,cache+openrouter_id,before the board rows,gen-1783264765-38VvKOCnsROTRGpJ15V8,1783276494.148,1783264765.000 +minimax-m3,scenario_076,0.002518,full,cache+openrouter_id,before the board rows,gen-1783264861-5fYy30PJR8GKSMldc2K3,1783276494.150,1783264861.000 +minimax-m3,scenario_077,0.000735,full,cache+openrouter_id,before the board rows,gen-1783265117-7j33SN52q934PWGeiCX5,1783276494.153,1783265117.000 +minimax-m3,scenario_078,0.000681,full,cache+openrouter_id,before the board rows,gen-1783265125-RiAtULt1i3bEoZsb9Wze,1783276494.154,1783265125.000 +minimax-m3,scenario_079,0.000703,full,cache+openrouter_id,before the board rows,gen-1783265128-8NCDtaC1Jm2NBpUgdOcH,1783276494.154,1783265128.000 +minimax-m3,scenario_080,0.000694,full,cache+openrouter_id,before the board rows,gen-1783265137-kh7lGmbiE471wNFVEPfk,1783276494.155,1783265137.000 +minimax-m3,scenario_081,0.000482,full,cache+openrouter_id,before the board rows,gen-1783265147-SssVfKp0S3JjBZoCmt5S,1783276494.156,1783265147.000 +minimax-m3,scenario_082,0.001966,full,cache+openrouter_id,before the board rows,gen-1783265149-xQok5jaOvQVPkUstFBq8,1783276494.157,1783265149.000 +minimax-m3,scenario_083,0.000991,full,cache+openrouter_id,before the board rows,gen-1783265165-l3tczzxzDxDzjZmucKy9,1783276494.160,1783265165.000 +minimax-m3,scenario_084,0.000547,full,cache+openrouter_id,before the board rows,gen-1783265176-zBTk8o1Gyhhn4VzqHiwP,1783276494.161,1783265176.000 +minimax-m3,scenario_085,0.000940,full,cache+openrouter_id,before the board rows,gen-1783265184-FoJAvNiwLCSbIICf1BXU,1783276494.162,1783265184.000 +minimax-m3,scenario_086,0.001911,full,cache+openrouter_id,before the board rows,gen-1783265193-EtxmU31I1ETRLF7v70zW,1783276494.163,1783265193.000 +minimax-m3,scenario_088,0.000687,full,cache+openrouter_id,before the board rows,gen-1783265428-HHJ1Mk0Vwhhjr1RalLz3,1783276494.165,1783265428.000 +minimax-m3,scenario_089,0.001477,full,cache+openrouter_id,before the board rows,gen-1783265890-OfpppF9i0wJWZ2AsnU3V,1783276494.166,1783265890.000 +minimax-m3,scenario_090,0.000507,full,cache+openrouter_id,before the board rows,gen-1783266101-qzTPneeMSmphEjTxYH43,1783276494.168,1783266101.000 +minimax-m3,scenario_091,0.000852,full,cache+openrouter_id,before the board rows,gen-1783266113-WHcB7JtAn6p3hhzaIdYW,1783276494.169,1783266113.000 +minimax-m3,scenario_092,0.000783,full,cache+openrouter_id,before the board rows,gen-1783266119-oxOQf9soJEHcgQTyPKFy,1783276494.187,1783266119.000 +minimax-m3,scenario_093,0.000669,full,cache+openrouter_id,before the board rows,gen-1783266582-PuPeHRZ1lfIZ0OHCojFt,1783276494.189,1783266582.000 +minimax-m3,scenario_095,0.002148,full,cache+openrouter_id,before the board rows,gen-1783266591-26MwyBg9fxF9qHCS7w50,1783276494.190,1783266591.000 +minimax-m3,scenario_098,0.000572,full,cache+openrouter_id,before the board rows,gen-1783266825-MDSZrcB2ivWsIpGNrNuZ,1783276494.192,1783266825.000 +minimax-m3,scenario_099,0.000622,full,cache+openrouter_id,before the board rows,gen-1783266832-7O292eRUvYPNRaYBAo4n,1783276494.193,1783266832.000 +minimax-m3,scenario_100,0.000629,full,cache+openrouter_id,before the board rows,gen-1783266842-W4TeWKiyf6M0coeXN33V,1783276494.194,1783266842.000 +minimax-m3,scenario_101,0.002176,full,cache+openrouter_id,before the board rows,gen-1783266857-xENsYU7AEs49WC3EZKuj,1783276494.195,1783266857.000 +minimax-m3,scenario_102,0.001282,full,cache+openrouter_id,before the board rows,gen-1783266873-T0P09NrEJkNxDKrQeSeS,1783276494.198,1783266873.000 +muse-spark-1.3,scenario_000,0.021963,full,openrouter_id+smoke_file+smoke_ledger+spend_ledger,onboarding smoke,gen-1791913993-DpxohcSu78LDiLgPpcLx,1791914304.295,1791913993.000 +muse-spark-1.3,scenario_001,0.020273,full,openrouter_id+smoke_file+smoke_ledger+spend_ledger,onboarding smoke,gen-1791913993-5zJ3lqf7ETE5I6JRbs1T,1791914304.295,1791913993.000 +muse-spark-1.3,scenario_099,0.012316,full,openrouter_id+smoke_file+smoke_ledger+spend_ledger,onboarding smoke,gen-1791913993-PQn6iUoqkvByarTtbdyC,1791915179.386,1791913993.000 +muse-spark-1.3,scenario_119,0.010278,full,openrouter_id+smoke_file+smoke_ledger+spend_ledger,onboarding smoke,gen-1791913993-Hwjd8TvOupjGQoEWJMUH,1791915312.922,1791913993.000 +muse-wonder,scenario_000,0.010042,full,openrouter_id+smoke_file+smoke_ledger+spend_ledger,onboarding smoke,gen-1791859722-dbN2jdViiyGxcdRzn0eG,1791859801.894,1791859722.000 +muse-wonder,scenario_001,0.009422,full,openrouter_id+smoke_file+smoke_ledger+spend_ledger,onboarding smoke,gen-1791859722-nZ1hgpzJG024bIN2YP6L,1791859801.912,1791859722.000 +ox-alpha,scenario_008,111.679407,partial,cache+openrouter_id+spend_ledger,same run,gen-1787353297-fclFQjwEeGUWqE54QgTr,1787360471.044,1787353297.000 +ox-alpha,scenario_023,351.659923,partial,cache+openrouter_id+spend_ledger,same run,gen-1787353751-tROrBhYcebBFVfaY5slH,1787360471.043,1787353751.000 +ox-alpha,scenario_038,271.122913,partial,cache+openrouter_id+spend_ledger,same run,gen-1787365665-hVIDbJuHcN35XLjHBTr9,1787395308.191,1787365665.000 +ox-alpha,scenario_039,275.210712,partial,cache+openrouter_id+spend_ledger,same run,gen-1787370980-uTCF2Ylw6h5y2dHyBWuN,1787395308.195,1787370980.000 +qwen-3.7-max,scenario_000,0.020002,full,cache+openrouter_id,before the board rows,gen-1783333641-JiEAUm8237YHFmJUM8hN,1783381171.662,1783333641.000 +qwen-3.7-max,scenario_001,0.011061,full,cache+openrouter_id,before the board rows,gen-1783333839-EWobKn29RdTgB3goyMuK,1783381171.683,1783333839.000 +qwen-3.7-max,scenario_002,0.010231,full,cache+openrouter_id,before the board rows,gen-1783335088-83zwIyqsij6RGWvIsNwp,1783381171.696,1783335088.000 +qwen-3.7-max,scenario_003,0.007404,full,cache+openrouter_id,before the board rows,gen-1783335635-XgbKgfYbuP7Jfv20CdQd,1783381171.708,1783335635.000 +qwen-3.7-max,scenario_004,0.008426,full,cache+openrouter_id,before the board rows,gen-1783335904-6Lat06V7LDjdivbOjQzf,1783381171.716,1783335904.000 +qwen-3.7-max,scenario_005,0.009426,full,cache+openrouter_id,before the board rows,gen-1783336176-aaJrXGJ8Az8bKTEXx1VC,1783381171.726,1783336176.000 +qwen-3.7-max,scenario_007,0.009372,full,cache+openrouter_id,before the board rows,gen-1783336909-SS2xlgkmq5vnjd5j1Xtj,1783381171.737,1783336909.000 +qwen-3.7-max,scenario_008,0.022191,full,cache+openrouter_id,before the board rows,gen-1783348516-629V5MW7GTNItkGi5hrr,1783381171.747,1783348516.000 +qwen-3.7-max,scenario_009,0.007921,full,cache+openrouter_id,before the board rows,gen-1783349087-pCf1n706YpcI5v79tTs7,1783381171.773,1783349087.000 +qwen-3.7-max,scenario_012,0.014097,full,cache+openrouter_id,before the board rows,gen-1783350019-TTHw2FxbeQT8e07kN6dq,1783381171.782,1783350019.000 +qwen-3.7-max,scenario_013,0.007858,full,cache+openrouter_id,before the board rows,gen-1783350966-PnT8Ehb2FIVim6l0NMC6,1783381171.798,1783350966.000 +qwen-3.7-max,scenario_014,0.009577,full,cache+openrouter_id,before the board rows,gen-1783353600-lgoVcdniXAVRpoF7ldPs,1783381171.807,1783353600.000 +qwen-3.7-max,scenario_015,0.008274,full,cache+openrouter_id,before the board rows,gen-1783356907-Y4Dc4EmbwiW3muHRCTn0,1783381171.818,1783356907.000 +qwen-3.7-max,scenario_016,0.006466,full,cache+openrouter_id,before the board rows,gen-1783357376-0Crtw2Ewsess5MC4d8zZ,1783381171.827,1783357376.000 +qwen-3.7-max,scenario_018,0.006626,full,cache+openrouter_id,before the board rows,gen-1783357578-pxOA9NuOmbkVCVnFCGPg,1783381171.835,1783357578.000 +qwen-3.7-max,scenario_020,0.005361,full,cache+openrouter_id,before the board rows,gen-1783359343-sa2sUysnYDtTpa1qx2lN,1783381171.843,1783359343.000 +qwen-3.7-max,scenario_021,0.005762,full,cache+openrouter_id,before the board rows,gen-1783359570-HrdmhDcBIoEAF1g8wrUG,1783381171.849,1783359570.000 +qwen-3.7-max,scenario_022,0.019229,full,cache+openrouter_id,before the board rows,gen-1783353600-FyuOMlAZkhow2KmnxhBU,1783381171.664,1783353600.000 +qwen-3.7-max,scenario_023,0.011890,full,cache+openrouter_id,before the board rows,gen-1783359330-Z3eN0GATWrACWFsdpHtM,1783381171.684,1783359330.000 +qwen-3.7-max,scenario_025,0.009880,full,cache+openrouter_id,before the board rows,gen-1783362291-Xq7WqzynEK2CJGQebRgS,1783381171.697,1783362291.000 +qwen-3.7-max,scenario_026,0.019781,full,cache+openrouter_id,before the board rows,gen-1783363173-PqZdP5x0Lm6m4n3ItLTM,1783381171.708,1783363173.000 +qwen-3.7-max,scenario_027,0.009971,full,cache+openrouter_id,before the board rows,gen-1783364848-Q1vL3H0o3oIWP7NWLWis,1783381171.730,1783364848.000 +qwen-3.7-max,scenario_028,0.017020,full,cache+openrouter_id,before the board rows,gen-1783365278-WgRwtwpxD0bV9nANqjL8,1783381171.741,1783365278.000 +qwen-3.7-max,scenario_029,0.008184,full,cache+openrouter_id,before the board rows,gen-1783366648-Cv4ouGY4pY89XgifVsOj,1783381171.760,1783366648.000 +qwen-3.7-max,scenario_030,0.010005,full,cache+openrouter_id,before the board rows,gen-1783366968-sWroWKZ8tN11tAVPKdgd,1783381171.770,1783366968.000 +qwen-3.7-max,scenario_031,0.007897,full,cache+openrouter_id,before the board rows,gen-1783333189-FWUJCrbL31VTkN2n6cXj,1783381171.781,1783333189.000 +qwen-3.7-max,scenario_032,0.016950,full,cache+openrouter_id,before the board rows,gen-1783368223-5xZMOzbLbmKdNC2cc7xE,1783381171.790,1783368223.000 +qwen-3.7-max,scenario_033,0.014286,full,cache+openrouter_id,before the board rows,gen-1783368908-8Tri7gnqDHUTM8MoIesl,1783381171.809,1783368908.000 +qwen-3.7-max,scenario_036,0.009349,full,cache+openrouter_id,before the board rows,gen-1783370445-I2CtN6zjP8dpRhKXbKfI,1783381171.825,1783370445.000 +qwen-3.7-max,scenario_037,129.838813,partial,cache+openrouter_id,before the board rows,gen-1783371326-XS92eHifygpz54E7dQ3o,1783381171.835,1783371326.000 +qwen-3.7-max,scenario_043,0.020905,full,cache+openrouter_id,before the board rows,gen-1783337764-d0sZATWxmXHLichUcBfm,1783381171.664,1783337764.000 +qwen-3.7-max,scenario_044,0.007368,full,cache+openrouter_id,before the board rows,gen-1783338346-IYrCEavUum1CRhHUwc2V,1783381171.686,1783338346.000 +qwen-3.7-max,scenario_045,0.008963,full,cache+openrouter_id,before the board rows,gen-1783339022-ubhzP6IGxTHxiESMsTBd,1783381171.694,1783339022.000 +qwen-3.7-max,scenario_046,0.011745,full,cache+openrouter_id,before the board rows,gen-1783340108-V23BYBBgCvrL795aUIzB,1783381171.704,1783340108.000 +qwen-3.7-max,scenario_048,0.007667,full,cache+openrouter_id,before the board rows,gen-1783340573-pPo1SKJ2n6x2Gd25nIoW,1783381171.718,1783340573.000 +qwen-3.7-max,scenario_049,0.007503,full,cache+openrouter_id,before the board rows,gen-1783340832-vjWe6xo9WA5nOCfHLqhm,1783381171.727,1783340832.000 +qwen-3.7-max,scenario_051,0.008564,full,cache+openrouter_id,before the board rows,gen-1783341115-B5NXWPqvIEz4AniVO4fB,1783381171.735,1783341115.000 +qwen-3.7-max,scenario_052,0.005778,full,cache+openrouter_id,before the board rows,gen-1783342291-hTV9KgQGxcSR8L1H6wHN,1783381171.745,1783342291.000 +qwen-3.7-max,scenario_053,0.008001,full,cache+openrouter_id,before the board rows,gen-1783342527-0gYfy9mMwR8s468JATYG,1783381171.752,1783342527.000 +qwen-3.7-max,scenario_054,0.014798,full,cache+openrouter_id,before the board rows,gen-1783343589-vjayFl4ZjCRjKWv7AlMn,1783381171.761,1783343589.000 +qwen-3.7-max,scenario_055,0.006931,full,cache+openrouter_id,before the board rows,gen-1783344575-PfI17Qj0CBgBLJ2TRzDJ,1783381171.777,1783344575.000 +qwen-3.7-max,scenario_056,0.009784,full,cache+openrouter_id,before the board rows,gen-1783345020-UvtCyHw6I89pAfMD0ZeQ,1783381171.786,1783345020.000 +qwen-3.7-max,scenario_057,0.012571,full,cache+openrouter_id,before the board rows,gen-1783348735-iZeeLUvBWLmkW8pTq6wX,1783381171.797,1783348735.000 +qwen-3.7-max,scenario_059,0.009834,full,cache+openrouter_id,before the board rows,gen-1783350131-2xH1tofDVu14jas7fW44,1783381171.811,1783350131.000 +qwen-3.7-max,scenario_060,0.009146,full,cache+openrouter_id,before the board rows,gen-1783350638-Rhg5iNCGCLxNjdFiMnXA,1783381171.822,1783350638.000 +qwen-3.7-max,scenario_062,0.007454,full,cache+openrouter_id,before the board rows,gen-1783353148-t5V4OI2mZO1sjS7HLr1x,1783381171.833,1783353148.000 +qwen-3.7-max,scenario_064,0.030334,full,cache+openrouter_id,before the board rows,gen-1783333189-5FTFNAtd8bbVqUJgSoMi,1783381171.662,1783333189.000 +qwen-3.7-max,scenario_066,0.010746,full,cache+openrouter_id,before the board rows,gen-1783335102-k4EXSaZqRnapjmlDEqIE,1783381171.695,1783335102.000 +qwen-3.7-max,scenario_067,0.015826,full,cache+openrouter_id,before the board rows,gen-1783336327-8gMp34PSfwluuQSMhskf,1783381171.707,1783336327.000 +qwen-3.7-max,scenario_068,0.007575,full,cache+openrouter_id,before the board rows,gen-1783341830-3PqmBpy6hlnr9aTcL00V,1783381171.725,1783341830.000 +qwen-3.7-max,scenario_070,0.007975,full,cache+openrouter_id,before the board rows,gen-1783341980-lyMvbC2iW1cvxTtQpdzT,1783381171.734,1783341980.000 +qwen-3.7-max,scenario_071,0.006763,full,cache+openrouter_id,before the board rows,gen-1783342538-avU2Hzb65hk91Ea8dBIv,1783381171.743,1783342538.000 +qwen-3.7-max,scenario_072,0.008540,full,cache+openrouter_id,before the board rows,gen-1783342901-dC6sQh6FSylnz13MkvqR,1783381171.750,1783342901.000 +qwen-3.7-max,scenario_073,0.008586,full,cache+openrouter_id,before the board rows,gen-1783343414-8GwVaI6SLUw27jVgAgny,1783381171.760,1783343414.000 +qwen-3.7-max,scenario_074,0.007795,full,cache+openrouter_id,before the board rows,gen-1783343893-w1TDyAJTpdlSfmx6mapH,1783381171.770,1783343893.000 +qwen-3.7-max,scenario_075,0.005245,full,cache+openrouter_id,before the board rows,gen-1783344229-62E95lH6FfwNnlnIIkl9,1783381171.778,1783344229.000 +qwen-3.7-max,scenario_076,0.012421,full,cache+openrouter_id,before the board rows,gen-1783344928-z4cAmIP9onGpNnttGvto,1783381171.785,1783344928.000 +qwen-3.7-max,scenario_077,0.013032,full,cache+openrouter_id,before the board rows,gen-1783346525-0A1znkWkDDifABJr3Tdi,1783381171.799,1783346525.000 +qwen-3.7-max,scenario_078,0.007941,full,cache+openrouter_id,before the board rows,gen-1783353149-OgJxCuiUEgboC5nKMlxP,1783381171.814,1783353149.000 +qwen-3.7-max,scenario_079,0.008331,full,cache+openrouter_id,before the board rows,gen-1783353479-RWoC9dvfFrDJSv2lBZM2,1783381171.823,1783353479.000 +qwen-3.7-max,scenario_080,0.008497,full,cache+openrouter_id,before the board rows,gen-1783353894-QYaygV8oQt8mzkiYNEOo,1783381171.832,1783353894.000 +qwen-3.7-max,scenario_081,0.006726,full,cache+openrouter_id,before the board rows,gen-1783354533-JVv7KwbPvDC7TxyVrTqi,1783381171.842,1783354533.000 +qwen-3.7-max,scenario_082,0.007264,full,cache+openrouter_id,before the board rows,gen-1783356375-zVeCuaVX93Z8VqqqX7kI,1783381171.850,1783356375.000 +qwen-3.7-max,scenario_083,0.018920,full,cache+openrouter_id,before the board rows,gen-1783337312-dU2wng53YTRLXUsjK5XN,1783381171.664,1783337312.000 +qwen-3.7-max,scenario_084,0.008864,full,cache+openrouter_id,before the board rows,gen-1783337643-e28jNbvjlj6PXyam4bga,1783381171.684,1783337643.000 +qwen-3.7-max,scenario_085,0.007309,full,cache+openrouter_id,before the board rows,gen-1783338166-erSnRqaiFxizZMkQhY00,1783381171.694,1783338166.000 +qwen-3.7-max,scenario_086,0.011688,full,cache+openrouter_id,before the board rows,gen-1783356827-dwoIinxC5irNynlohtOB,1783381171.702,1783356827.000 +qwen-3.7-max,scenario_088,0.006498,full,cache+openrouter_id,before the board rows,gen-1783357253-MuDtGMeR1cLzWvnhZdGH,1783381171.716,1783357253.000 +qwen-3.7-max,scenario_089,0.008486,full,cache+openrouter_id,before the board rows,gen-1783363194-XnAH4cNBO8GyS6QkN4z8,1783381171.723,1783363194.000 +qwen-3.7-max,scenario_090,0.010636,full,cache+openrouter_id,before the board rows,gen-1783363293-gI3BKPC0wyFGSi7qqX7U,1783381171.733,1783363293.000 +qwen-3.7-max,scenario_091,0.008203,full,cache+openrouter_id,before the board rows,gen-1783364169-oSVbVilFU1k8oDTYdImU,1783381171.745,1783364169.000 +qwen-3.7-max,scenario_092,0.006172,full,cache+openrouter_id,before the board rows,gen-1783333189-EmFPfX0kS90tLowl9XYz,1783381171.754,1783333189.000 +qwen-3.7-max,scenario_093,0.009682,full,cache+openrouter_id,before the board rows,gen-1783333946-zWBDA42vUuUzSIBj8FWI,1783381171.761,1783333946.000 +qwen-3.7-max,scenario_095,0.008939,full,cache+openrouter_id,before the board rows,gen-1783334968-LSTgC6PjF1KqcvLSrBbY,1783381171.772,1783334968.000 +qwen-3.7-max,scenario_098,0.007741,full,cache+openrouter_id,before the board rows,gen-1783335618-mybG2kwjoUpYIemZfXtq,1783381171.783,1783335618.000 +qwen-3.7-max,scenario_099,0.014834,full,cache+openrouter_id,before the board rows,gen-1783335936-9toDxItkDiGt9yb54hrn,1783381171.792,1783335936.000 +qwen-3.7-max,scenario_100,0.021292,full,cache+openrouter_id,before the board rows,gen-1783364647-yc9ODjVDuGzB6ll5UgnU,1783381171.809,1783364647.000 +qwen-3.7-max,scenario_101,0.008238,full,cache+openrouter_id,before the board rows,gen-1783366638-5GSOMxW4Ub8odULIBiXM,1783381171.833,1783366638.000 +qwen-3.7-max,scenario_102,0.009545,full,cache+openrouter_id,before the board rows,gen-1783367171-RCWy5E7Z0DlImfDdTc50,1783381171.842,1783367171.000 +qwen-3.7-max,scenario_104,0.005236,full,cache+openrouter_id,before the board rows,gen-1783368135-wEfeEF6gh6oAlcypvsEc,1783381171.853,1783368135.000 +qwen-3.7-max,scenario_107,0.020234,full,cache+openrouter_id,before the board rows,gen-1783337312-LEjY7wjJ1JsMxo6SiJ69,1783381171.664,1783337312.000 +qwen-3.7-max,scenario_108,0.010732,full,cache+openrouter_id,before the board rows,gen-1783338219-6zje5n81g9h6ux9Zhic0,1783381171.685,1783338219.000 +qwen-3.7-max,scenario_109,0.016645,full,cache+openrouter_id,before the board rows,gen-1783339649-GOHfzCLwTlwG4dWMhAvG,1783381171.697,1783339649.000 +qwen-3.7-max,scenario_110,0.008435,full,cache+openrouter_id,before the board rows,gen-1783341830-KfruepW2NIBkhcvSVMeb,1783381171.716,1783341830.000 +qwen-3.7-max,scenario_111,0.008858,full,cache+openrouter_id,before the board rows,gen-1783341929-oPttFtkWILtf11zsWNlR,1783381171.725,1783341929.000 +qwen-3.7-max,scenario_112,0.008089,full,cache+openrouter_id,before the board rows,gen-1783342243-EO3hHHrOuPZfFoLLTxpT,1783381171.736,1783342243.000 +qwen-3.7-max,scenario_114,0.007903,full,cache+openrouter_id,before the board rows,gen-1783353148-s8oYKCb29HEejr38ZK4j,1783381171.745,1783353148.000 +qwen-3.7-max,scenario_115,0.006686,full,cache+openrouter_id,before the board rows,gen-1783353245-JsEybk2B0yx2KqDDrHBk,1783381171.754,1783353245.000 +qwen-3.7-max,scenario_116,0.005974,full,cache+openrouter_id,before the board rows,gen-1783353511-u61td3KKS1wkg6eLxnZM,1783381171.761,1783353511.000 +qwen-3.7-max,scenario_117,0.015305,full,cache+openrouter_id,before the board rows,gen-1783353662-wpmaReU8bZXXldonFU3U,1783381171.769,1783353662.000 +qwen-3.7-max,scenario_118,0.006185,full,cache+openrouter_id,before the board rows,gen-1783354527-n28tpgPPKgfDA1X0zOVs,1783381171.787,1783354527.000 +qwen-3.7-max,scenario_119,0.014331,full,cache+openrouter_id,before the board rows,gen-1783359783-hNpqBKAjtdnjgWB9vBVM,1783381171.794,1783359783.000 +qwen-3.7-max,scenario_120,0.009706,full,cache+openrouter_id,before the board rows,gen-1783362292-rkGAofEpzk8zrObJ2L57,1783381171.811,1783362292.000 +qwen-3.7-max,scenario_121,0.012724,full,cache+openrouter_id,before the board rows,gen-1783363205-tltwvED2zYu9KsWAbLUf,1783381171.822,1783363205.000 diff --git a/paper/index.qmd b/paper/index.qmd index cfd9e3e0..cdc29b1d 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -746,12 +746,14 @@ def fmt_cost_per_household(usd) -> str: def load_us_cost_latency() -> dict: # Cost and latency use the frozen run's recorded per-call costs and timings, # through the same model_cost_latency function as the leaderboard pipeline, - # then are reconciled against the frozen - # modelStats: Claude Fable 5's usage is recorded at the run level (its - # per-row usage columns are empty), and batch or asynchronous waves log - # submission times rather than generation time, so a zero or sub-second - # median is treated as unmeasured and falls back to modelStats or "—". - from policybench.analysis import model_cost_latency + # then are reconciled against the frozen modelStats. Claude Fable 5's board + # rows carry no usage or timing fields (they were rebuilt from the 20260701 + # release without them), so its modelStats carries its original run's + # per-call figures. A zero cost is a recorded charge of zero, and a + # sub-second median is a model whose median household was replayed from the + # disk cache (docs/latency_replay_audit.md); both fall back to modelStats + # or "—". + from policybench.analysis import REPLAYED_HOUSEHOLD_SECONDS, model_cost_latency from policybench.config import PRICE_OVERRIDES_PER_1M predictions = pd.read_csv( @@ -768,11 +770,13 @@ def load_us_cost_latency() -> dict: float(fallback) if fallback and fallback > 0 else None ) latency = entry.get("latencySeconds") - if latency is None or pd.isna(latency) or latency < 1: + if latency is None or pd.isna(latency) or latency < REPLAYED_HOUSEHOLD_SECONDS: fallback = None if row is None else row.get("latencySeconds") entry["latencySeconds"] = ( float(fallback) - if fallback is not None and not pd.isna(fallback) and fallback >= 1 + if fallback is not None + and not pd.isna(fallback) + and fallback >= REPLAYED_HOUSEHOLD_SECONDS else None ) return computed @@ -1164,12 +1168,38 @@ Sampling uncertainty is substantial at this sample size. In the frozen snapshot, Accuracy is one axis; the price and speed of a no-tools answer are another. @tbl-us-cost-latency reports each model's estimated cost to compute one household's full output vector and its median per-household latency, next to the headline exact rate. `{python} cost_latency_text` ```{python} -#| label: tbl-us-cost-latency -#| tbl-cap: "Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: reconstructed from token counts at the list price configured at request time, or the provider-reported charge where no reconstruction was available. List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. Latency is the median across households of each household's summed request time. A dash marks usage or wall-clock latency that was not measured per call." -md_table(cost_latency) +def cost_latency_caption() -> str: + """The table's caption, naming the model behind each dash and why.""" + shown = list(us_model["model"]) + cost_dash = [m for m in shown if us_cost_latency.get(m, {}).get("costPerHousehold") is None] + latency_dash = [m for m in shown if us_cost_latency.get(m, {}).get("latencySeconds") is None] + # Every dash is one the caption explains, or the render fails. + assert sorted(cost_dash) == r.zero_cost_models, cost_dash + assert sorted(latency_dash) == r.latency_dash_models, latency_dash + zero_cost = r.zero_cost_models_fmt + return ( + "Estimated cost and median latency per household for the frozen US snapshot, " + "alongside the headline exact-match rate. Each row uses its recorded per-call " + "cost: reconstructed from token counts at the list price configured at request " + "time, or the provider-reported charge where no reconstruction was available. " + "List-price overrides apply at request time, not retroactively to recorded " + "costs. Claude Fable 5's cost and latency come from its original run's per-call " + "rows, which its board rows do not carry. Latency is the median across " + "households of each household's summed request time. A cost dash marks a " + f"provider-reported charge of zero on every call ({zero_cost}). A latency dash " + f"marks rows that record no request time ({r.batch_latency_models_fmt}, through " + "OpenAI's batch API) or a median household answered from the harness's " + f"response cache rather than by the provider ({r.replayed_latency_models_fmt}; " + "see text)." + ) + + +md_table(cost_latency, caption=cost_latency_caption(), label="tbl-us-cost-latency") ``` -Cost preserves each call's recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5's usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run's providers, prompts, and serving configurations at evaluation time rather than fixed model constants. +Cost preserves each call's recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5's board rows carry no usage or timing fields, because they were rebuilt from the July 1, 2026 release without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (@tbl-model-runs) sends a household's calls one after another, and its household time is their sum. `{python} r.batch_latency_models_fmt` ran through OpenAI's batch API, which records no per-call request time, so it shows a dash. + +The harness also caches provider responses on disk, keyed on the request and the prompt contract version. A request identical to an earlier one is answered from the cache in hundredths of a second, and the timer records that as if it were a call. A replayed response keeps the cost recorded when it was generated, so a household's cost counts each of its responses once, whether or not the board run paid for it. `{python} r.replayed_latency_models_fmt` were finished over several passes, each of which replayed the earlier passes' successful responses. Each of them records a replay on `{python} r.replayed_latency_range_fmt` of its `{python} r.n_households_fmt` households, so its median falls under a second and it shows a dash as well. In `{python} r.other_subsecond_model_count_fmt` other models, `{python} r.other_subsecond_household_count` households sum to under a second. The audit traces `{python} r.traced_other_subsecond_count` of them to an earlier request on the same household, such as an onboarding smoke or a restarted chunk; the other `{python} r.untraced_subsecond_count_fmt` come from runs that predate per-call timestamps and left no cache or ledger behind. Dropping them would raise no model's median by more than `{python} r.subsecond_median_shift_fmt` seconds. A further `{python} r.partial_replay_count` households replay their first recorded call and then reach the provider, so their times leave out that first call. The benchmark card summarizes the replays, and the repository's latency replay audit lists each household. Both cost and latency are properties of this run's providers, prompts, and serving configurations at evaluation time rather than fixed model constants. ### Simple baselines diff --git a/policybench/analysis.py b/policybench/analysis.py index 0ae3b331..6e89f3a6 100644 --- a/policybench/analysis.py +++ b/policybench/analysis.py @@ -1411,6 +1411,32 @@ def _wall_clock_seconds(group: pd.Series) -> float: ) +# A household whose recorded request time sums to under this is a disk-cache +# replay, not a provider call: in the frozen US run every replayed household +# sums to under 0.12 s, and the fastest live one takes 1.75 s +# (docs/latency_replay_audit.md). +REPLAYED_HOUSEHOLD_SECONDS = 1.0 + + +def household_latency_seconds(predictions: pd.DataFrame) -> pd.Series: + """Each household's recorded request time, indexed by (model, scenario_id). + + A row's ``elapsed_seconds`` is its household's request time divided among + the household's rows, so the household's time is the sum over its rows, + not the max. A household whose rows carry no time is NaN rather than zero. + A response the disk cache replays records the lookup's time, a few + hundredths of a second, as if it were a provider call. + """ + if predictions.empty or "elapsed_seconds" not in predictions.columns: + return pd.Series( + dtype=float, + index=pd.MultiIndex.from_tuples([], names=["model", "scenario_id"]), + ) + return predictions.groupby(["model", "scenario_id"])["elapsed_seconds"].sum( + min_count=1 + ) + + def model_cost_latency( predictions: pd.DataFrame, price_overrides: dict[str, dict[str, float]] | None = None, @@ -1431,10 +1457,7 @@ def model_cost_latency( latency_median = pd.Series(dtype=float) if "elapsed_seconds" in predictions.columns: latency_median = ( - predictions.groupby(["model", "scenario_id"])["elapsed_seconds"] - .sum(min_count=1) - .groupby("model") - .median() + household_latency_seconds(predictions).groupby("model").median() ) overrides = price_overrides or {} out: dict[str, dict[str, float]] = {} diff --git a/policybench/paper_results.py b/policybench/paper_results.py index 1373b245..328bfd3b 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -54,6 +54,9 @@ # ``paper_results`` lives in ``policybench/``; the repo root is one level up. ROOT = Path(__file__).resolve().parents[1] SNAPSHOT_DIR = ROOT / "paper" / "snapshot" / "20260501" +# The frozen run's replayed households, traced from the disk caches, spend +# ledgers and smoke files (scripts/audit_latency_replays.py). +LATENCY_REPLAY_AUDIT = ROOT / "docs" / "latency_replay_audit" / "households.csv" # The 2026-09-29 move from policyengine-us 1.755.4 to 2.15.17. Its audit, # reference_audit/2026-09-28, holds the publication check, the sweep timing @@ -1256,9 +1259,11 @@ def contract_violation_count_fmt(self) -> str: def blank_raw_response_counts(self) -> Counter: """Frozen prediction rows whose raw_response is empty, by model. - Rows served through the Anthropic batch adapter carry no raw payload, - and some parse failures never captured one; the manuscript states the - preservation rule with these exceptions rather than as absolute. + Claude Fable 5's board rows were rebuilt from the 20260701 release + without their raw payloads (that release's + claude-fable-5-predictions.csv.gz asset keeps them), and some parse + failures never captured one; the manuscript states the preservation + rule with these exceptions rather than as absolute. """ run_dir = self.snapshot_dir / "runs" / self.us_run_label frame = pd.read_csv( @@ -1280,6 +1285,179 @@ def blank_raw_response_note(self) -> str: return "every row carries its raw provider response" return "no raw payload is retained for " + _ordinal_join(items) + # ----- latency: untimed rows and cache replays ------------------------- + @cached_property + def household_latencies(self) -> pd.Series: + """Each frozen household's summed request time, by (model, scenario).""" + from policybench.analysis import household_latency_seconds + + run_dir = self.snapshot_dir / "runs" / self.us_run_label + frame = pd.read_csv( + run_dir / "predictions.csv.gz", + usecols=["model", "scenario_id", "elapsed_seconds"], + ) + return household_latency_seconds(frame) + + def _published_latency(self, model_id: str) -> float | None: + value = self._stats_by_model.get(model_id, {}).get("latencySeconds") + return None if value is None or pd.isna(value) else float(value) + + @cached_property + def untimed_latency_models(self) -> list[str]: + """Models none of whose frozen rows records a request time.""" + timed = self.household_latencies.groupby("model").count() + return sorted(timed[timed == 0].index) + + @cached_property + def replayed_latency_models(self) -> list[str]: + """Models whose median household is a cache replay, not a call.""" + from policybench.analysis import REPLAYED_HOUSEHOLD_SECONDS + + median = self.household_latencies.groupby("model").median() + return sorted(median[median < REPLAYED_HOUSEHOLD_SECONDS].index) + + @cached_property + def latency_dash_models(self) -> list[str]: + """Models the cost-and-latency table shows without a latency. + + A model is dashed when its frozen rows give no latency of a second or + more and its published modelStats carries none either; Claude Fable + 5's rows are untimed but its published latency comes from its + original run's per-call rows, so it is not dashed. + """ + from policybench.analysis import REPLAYED_HOUSEHOLD_SECONDS + + dashed = [] + for model in sorted( + set(self.untimed_latency_models) | set(self.replayed_latency_models) + ): + published = self._published_latency(model) + if published is None or published < REPLAYED_HOUSEHOLD_SECONDS: + dashed.append(model) + return dashed + + @cached_property + def subsecond_household_counts(self) -> Counter: + """Households summing to under a second, by model.""" + from policybench.analysis import REPLAYED_HOUSEHOLD_SECONDS + + latency = self.household_latencies + fast = latency[latency < REPLAYED_HOUSEHOLD_SECONDS] + return Counter(fast.groupby("model").size().to_dict()) + + @cached_property + def zero_cost_models(self) -> list[str]: + """Models every one of whose frozen rows records a charge of zero.""" + run_dir = self.snapshot_dir / "runs" / self.us_run_label + frame = pd.read_csv( + run_dir / "predictions.csv.gz", usecols=["model", "total_cost_usd"] + ) + cost = frame.groupby("model")["total_cost_usd"] + recorded = cost.count() == cost.size() + zero = cost.apply(lambda values: bool((values == 0).all())) + return sorted(recorded[recorded & zero].index) + + @property + def zero_cost_models_fmt(self) -> str: + return _ordinal_join([self.model_name(m) for m in self.zero_cost_models]) + + @property + def batch_latency_models_fmt(self) -> str: + """Dashed models whose rows carry no request time (the batch run).""" + dashed = [ + m for m in self.latency_dash_models if m in self.untimed_latency_models + ] + return _ordinal_join([self.model_name(m) for m in dashed]) + + @property + def replayed_latency_models_fmt(self) -> str: + dashed = [ + m for m in self.latency_dash_models if m in self.replayed_latency_models + ] + return _ordinal_join([self.model_name(m) for m in dashed]) + + @property + def replayed_latency_range_fmt(self) -> str: + """Fewest to most replayed households among the dashed replay models.""" + counts = [ + self.subsecond_household_counts[m] + for m in self.latency_dash_models + if m in self.replayed_latency_models + ] + low, high = min(counts), max(counts) + return f"{low}" if low == high else f"{low} to {high}" + + @cached_property + def _other_subsecond_models(self) -> list[str]: + """Shown models with some households under a second.""" + return sorted( + m + for m, n in self.subsecond_household_counts.items() + if n and m not in self.latency_dash_models + ) + + @property + def other_subsecond_household_count(self) -> int: + return sum( + self.subsecond_household_counts[m] for m in self._other_subsecond_models + ) + + @property + def other_subsecond_model_count(self) -> int: + return len(self._other_subsecond_models) + + @property + def other_subsecond_model_count_fmt(self) -> str: + return count_word(self.other_subsecond_model_count) + + @property + def untraced_subsecond_count(self) -> int: + return self.other_subsecond_household_count - self.traced_other_subsecond_count + + @property + def untraced_subsecond_count_fmt(self) -> str: + return count_word(self.untraced_subsecond_count) + + @cached_property + def subsecond_median_shift_seconds(self) -> float: + """Largest rise in a shown model's median when its sub-second + households are dropped: how far the replays lower a published median.""" + from policybench.analysis import REPLAYED_HOUSEHOLD_SECONDS + + latency = self.household_latencies + shifts = [] + for model in self._other_subsecond_models: + households = latency.loc[model].dropna() + kept = households[households >= REPLAYED_HOUSEHOLD_SECONDS] + shifts.append(float(kept.median() - households.median())) + return max(shifts) if shifts else 0.0 + + @property + def subsecond_median_shift_fmt(self) -> str: + return f"{self.subsecond_median_shift_seconds:.1f}" + + @cached_property + def latency_replay_audit(self) -> pd.DataFrame: + """The audited replayed households (docs/latency_replay_audit).""" + return pd.read_csv(LATENCY_REPLAY_AUDIT) + + @property + def traced_other_subsecond_count(self) -> int: + """Of the shown models' sub-second households, those the audit traced + to an earlier request on the same household.""" + audit = self.latency_replay_audit + traced = audit[ + (audit["replay"] == "full") + & audit["model"].isin(self._other_subsecond_models) + ] + return len(traced) + + @property + def partial_replay_count(self) -> int: + """Households whose first recorded call is a replay and whose later + calls reached the provider.""" + return int((self.latency_replay_audit["replay"] == "partial").sum()) + # ----- populace dataset construction --------------------------------- @property def populace_people_fmt(self) -> str: diff --git a/scripts/audit_latency_replays.py b/scripts/audit_latency_replays.py new file mode 100644 index 00000000..26bb1ecd --- /dev/null +++ b/scripts/audit_latency_replays.py @@ -0,0 +1,276 @@ +"""Find the frozen board's households whose recorded latency is a cache replay. + +Writes ``docs/latency_replay_audit/households.csv``: one row for every +household of the frozen US run whose first recorded response was served again +from the harness's disk response cache (``policybench/cache.py``), and for +every household whose rows sum to under a second without such evidence. +``docs/latency_replay_audit.md`` explains the result. + +The evidence comes from artifacts that live outside the repository, so this +script runs only on the machine that holds them: + +- the litellm disk caches (``--cache-dir``, repeatable). Each entry's sqlite + ``store_time`` says when a response was first stored. Values are pickles; the + script searches their bytes for response ids and never unpickles them. +- the supervised runner's spend ledgers (``*.spend.jsonl`` under + ``--ledger-root``, repeatable), which record ``cache_hit`` per call. A ledger + under a path containing ``smoke`` belongs to an onboarding smoke. +- smoke prediction files (CSV under a path containing ``smoke``) below the + ledger roots, which keep each smoke response's id. + +Two kinds of evidence need only the frozen predictions. OpenRouter ids +(``gen--...``) and OpenAI Responses ids (``resp_`` + 16 hex + +``00`` + 8 hex seconds) embed their creation time. On live households that +time falls within a few seconds of the row's ``request_started_at``. A +creation time more than a minute before the request began means the response +already existed when the request was sent. + +A household counts as replayed when any evidence holds for its first recorded +response id. A full replay sums to under a second: every call came from the +cache. A partial replay sums to a second or more, so a later call in the +household (a repair, a retry round or another chunk) went to the provider. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import json +import os +import re +import sqlite3 +import sys +from pathlib import Path + +import pandas as pd + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) + +from policybench.analysis import household_latency_seconds # noqa: E402 + +FROZEN_PREDICTIONS = ( + ROOT + / "paper" + / "snapshot" + / "20260501" + / "runs" + / "us_full_run_20260612_policyengine_4_16_1_populace" + / "predictions.csv.gz" +) +OUTPUT = ROOT / "docs" / "latency_replay_audit" / "households.csv" +# A response that existed this long before the request began was not generated +# for it. Live households sit within 4 seconds of their request start. +MARGIN_SECONDS = 60.0 +# Every replayed household in the frozen run sums to under 0.12 s; the fastest +# live household (Gemini 3.5 Flash-Lite) takes 1.75 s. +FULL_REPLAY_SECONDS = 1.0 + +RESPONSE_ID = re.compile( + rb"(gen-\d{10}-[A-Za-z0-9]{10,40}" + rb"|chatcmpl-[A-Za-z0-9-]{8,80}" + rb"|msg_[A-Za-z0-9]{10,60}" + rb"|resp_[A-Za-z0-9]{20,80}" + rb"|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})" +) +OPENROUTER_ID = re.compile(r"^gen-(\d{10})-") +OPENAI_ID = re.compile(r"^resp_[0-9a-f]{16}00([0-9a-f]{8})[0-9a-f]{24}$") + +FIELDS = [ + "model", + "scenario_id", + "household_latency_seconds", + "replay", + "evidence", + "source", + "first_response_id", + "request_started_at", + "response_existed_at", +] + + +def provider_response_id(recorded: object) -> object: + """The provider's own id; litellm wraps OpenAI Responses ids in base64.""" + if not isinstance(recorded, str) or not recorded.startswith("resp_"): + return recorded + try: + decoded = base64.b64decode(recorded[5:] + "==").decode() + except (ValueError, UnicodeDecodeError): + return recorded + match = re.search(r"response_id:(resp_[0-9a-f]+)", decoded) + return match.group(1) if match else recorded + + +def id_created_at(response_id: object) -> float | None: + """Creation time embedded in an OpenRouter or OpenAI Responses id.""" + if not isinstance(response_id, str): + return None + for pattern, base in ((OPENROUTER_ID, 10), (OPENAI_ID, 16)): + match = pattern.match(response_id) + if match: + return float(int(match.group(1), base)) + return None + + +def cache_store_times(cache_dirs: list[Path]) -> dict[str, float]: + """Earliest store time of a cache entry holding each response id.""" + stored: dict[str, float] = {} + for cache_dir in cache_dirs: + con = sqlite3.connect(f"file:{cache_dir / 'cache.db'}?mode=ro", uri=True) + try: + rows = con.execute("select store_time, filename, value from Cache") + for store_time, filename, value in rows: + if filename: + blob = (cache_dir / filename).read_bytes() + else: + blob = bytes(value or b"") + for match in set(RESPONSE_ID.findall(blob)): + key = match.decode() + if key not in stored or store_time < stored[key]: + stored[key] = float(store_time) + finally: + con.close() + return stored + + +def ledger_records(roots: list[Path]) -> pd.DataFrame: + records = [] + for root in roots: + for path in root.rglob("*.spend.jsonl"): + for line in path.read_text().splitlines(): + try: + record = json.loads(line) + except json.JSONDecodeError: + continue + record["_smoke"] = "smoke" in str(path) + records.append(record) + return pd.DataFrame(records) + + +def smoke_file_ids(roots: list[Path]) -> set[str]: + ids: set[str] = set() + for root in roots: + for path in root.rglob("*.csv"): + if "smoke" not in str(path): + continue + try: + frame = pd.read_csv( + path, + usecols=lambda column: column == "provider_response_id", + low_memory=False, + ) + except (ValueError, pd.errors.ParserError, UnicodeDecodeError): + continue + if "provider_response_id" in frame: + ids.update(frame["provider_response_id"].dropna().astype(str)) + return ids + + +def build( + predictions: pd.DataFrame, + stored: dict[str, float], + ledgers: pd.DataFrame, + smoke_ids: set[str], +) -> list[dict]: + latency = household_latency_seconds(predictions) + grouped = predictions.groupby(["model", "scenario_id"], sort=True) + first_id = grouped["provider_response_id"].first() + started = grouped["request_started_at"].min() + board_start = predictions.groupby("model")["request_started_at"].min() + + ledger_hit: set[str] = set() + smoke_live: dict[str, float] = {} + if not ledgers.empty: + board = ledgers[~ledgers["_smoke"] & (ledgers["cache_hit"] == True)] # noqa: E712 + ledger_hit = set(board["provider_response_id"].dropna()) + smoke = ledgers[ledgers["_smoke"] & (ledgers["cache_hit"] != True)] # noqa: E712 + smoke_live = ( + smoke.dropna(subset=["provider_response_id"]) + .groupby("provider_response_id")["request_completed_at"] + .min() + .to_dict() + ) + + rows = [] + for (model, scenario), seconds in latency.items(): + if pd.isna(seconds): + continue + recorded = first_id.get((model, scenario)) + inner = provider_response_id(recorded) + start = started.get((model, scenario)) + existed: dict[str, float] = {} + if isinstance(inner, str) and inner in stored: + existed["cache"] = stored[inner] + created = id_created_at(inner) + if created is not None: + name = "openrouter_id" if inner.startswith("gen-") else "openai_id" + existed[name] = created + if recorded in smoke_live: + existed["smoke_ledger"] = float(smoke_live[recorded]) + evidence = sorted( + name + for name, when in existed.items() + if pd.notna(start) and when < start - MARGIN_SECONDS + ) + if recorded in ledger_hit: + evidence.append("spend_ledger") + if recorded in smoke_ids: + evidence.append("smoke_file") + evidence = sorted(set(evidence)) + if not evidence and seconds >= FULL_REPLAY_SECONDS: + continue + if not evidence: + replay, source = "untraced", "untraced" + else: + replay = "full" if seconds < FULL_REPLAY_SECONDS else "partial" + earliest = min(existed.values()) if existed else None + if "smoke_ledger" in evidence or "smoke_file" in evidence: + source = "onboarding smoke" + elif earliest is not None and earliest >= board_start[model]: + source = "same run" + else: + source = "before the board rows" + rows.append( + { + "model": model, + "scenario_id": scenario, + "household_latency_seconds": f"{seconds:.6f}", + "replay": replay, + "evidence": "+".join(evidence), + "source": source, + "first_response_id": recorded, + "request_started_at": "" if pd.isna(start) else f"{start:.3f}", + "response_existed_at": ( + f"{min(existed.values()):.3f}" if existed else "" + ), + } + ) + return rows + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--predictions", type=Path, default=FROZEN_PREDICTIONS) + parser.add_argument("--cache-dir", type=Path, action="append", default=[]) + parser.add_argument("--ledger-root", type=Path, action="append", default=[]) + parser.add_argument("--output", type=Path, default=OUTPUT) + args = parser.parse_args() + + predictions = pd.read_csv(args.predictions, low_memory=False) + rows = build( + predictions, + cache_store_times(args.cache_dir), + ledger_records(args.ledger_root), + smoke_file_ids(args.ledger_root), + ) + args.output.parent.mkdir(parents=True, exist_ok=True) + with open(args.output, "w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=FIELDS, lineterminator="\n") + writer.writeheader() + writer.writerows(rows) + print(f"wrote {len(rows)} households to {os.path.relpath(args.output, ROOT)}") + + +if __name__ == "__main__": + main() From dd4631da73cf5bea5f300a87ea4f5abc0ad5be16 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 07:25:13 -0400 Subject: [PATCH 2/4] State the real causes of the paper's latency dashes and Fable 5's missing usage The paper attributed its latency dashes to "batch or asynchronous dispatch whose logged times reflect submission", and said Fable 5's usage was "recorded at the run level"; the manifest and drivers said Fable 5 ran through the Anthropic batch adapter. None held. GPT-5.5 is the only batch run (its rows log no time); the four sub-second medians are cache replays of synchronous calls; Fable 5 ran synchronously, and its board rows were rebuilt without usage columns. - Paper: the caption and paragraph name each dashed model and its cause from PaperResults (computed from the frozen data; the render asserts every dash is explained). Re-rendered and re-pinned; no table value changes. - Benchmark card: cost basis and a new latency basis section. - Manifest note, release-driver comments and the gpt61sol design note corrected. - Tests: latency invariants as Hypothesis properties; the audit CSV, card and rendered paper checked against the frozen predictions. Co-Authored-By: Claude Opus 5.5 --- app/public/paper/policybench.pdf | Bin 260940 -> 262801 bytes app/public/paper/web/index.html | 19 +- app/src/paperSnapshot.json | 4 +- docs/benchmark_card.md | 68 ++++-- docs/gpt61sol/design.md | 7 +- docs/latency_replay_audit.md | 104 ++++++--- paper/index.qmd | 2 +- paper/snapshot/20260501/manifest.json | 6 +- scripts/audit_latency_replays.py | 25 +- scripts/finish_argon.py | 6 +- scripts/finish_gpt61sol.py | 6 +- scripts/finish_haiku55.py | 6 +- scripts/finish_musewonder.py | 6 +- scripts/freeze_snapshot.py | 8 +- scripts/release_20261006.py | 4 +- tests/test_cost_latency.py | 117 ++++++++++ tests/test_disclosures.py | 114 +++++----- tests/test_latency_replay_audit.py | 314 ++++++++++++++++++++++++++ 18 files changed, 671 insertions(+), 145 deletions(-) create mode 100644 tests/test_latency_replay_audit.py diff --git a/app/public/paper/policybench.pdf b/app/public/paper/policybench.pdf index 4f891c4d4c2da622d6224c230a060f418d6e39f1..a61214297baefa9d81bac525f3b932365762f08e 100644 GIT binary patch delta 56144 zcmV)EK)}Du_79PT5Rg9tIWe(88U=p_`+KvQGq;a~vrH?q2xh9=aI6V%du-zsfRu!s z(@Biuiwk)IIY@g&}>&q*3Uo1DCae3{Ro@}7Tn5#j#D@p!IW}~K!$ZKi*M`}(Fqi6&TQbJ%-Me$e?z$x z!kB;Kya5qsQsXGy2VBW~Q=A4C+~)b9$ETaW#}zH1g3ZMA?fK@*SO4G-|NnX744s|+ zjjulY7W`ZtrN94c`u*nD{&=TxlVZ-8uuIe&N3#sfBqTCWYUAkLr-SB`cRN~R_pJja z!Mh^K+e4vl<4`DydRDPJ*CR0({xkhD=R=sZ6E@S6uyw0fHsY6OEEhIeo+jKlria}3 zh{2$vQh4G}owwJ270b-@lY|C70Wz0SVgVI@TS<>2yA8h2ukat3dJ?q*Mgv{#_5*SZ z<`(3doZcn>|B#X>E>bP+&IE&o?J6y#NPgTj=JxNKx&8UW>^{xS-u_d6)GvSk^T+z@ z#~;s7Zq_WtZ2xY*{`JH5*V!#Tgj+IYG2Y&O{dN1t-Iy5Oep+M9_SMC=?6yz)VT^}= z?JsYyAKTyd+QHjT=jvO(8WZf>PsrO3c-B|j5P19NufOdGq^*NZOLTGG+m-rvIP;vv ziJh4J60&1pT`Q_aG|HbyZStf%btrt$uV_&kRl#ECwEbqK z>uVU~xIHhp8Gh$=Kl%z!lJ#q6GcwZj(f#JxSi6A9v?L31oz91}u%44!*|^%1w>GMd zXd?~TSU;T?^>rkq^5#D;dHds;&8!a^+c23rR!jgQR zmaB@<)Sw>=DSgA8(2nM#Z`h9oHfwR!Rw@|vFWad21&eM2N5429;fO_FmIRt6aO&U< zDjKTku+I*3x376x+vW{Fc=6!gpb*hnm`>X+0Lq3|9Os@@c}mxwWZpt1sP5Q6*w*p5 zk4|;1r|2AiH`|EPU_T*i9aC6;_dKYPZLPXf+p+1a?CMq&R3AkSq9_N^hy>cQGKvc^ z5lX=cw1pvN{i&3$lSj|$4)CP)2<(%mxkR{(sTu@D;Z-}?2MgCO1|}JruxtpyGCag2 zWYoDbpKY~rZ(d}hk4#$S`pWH10`Y3YdWA+bs>o5((sjF#-+uZO>yV6plRfCA32@N< zx*VX}q9^%nOGCOhRTNM&nt4$-Ub)MF2?xzAxr0%Ww-Jvz!gp;#E5tqN5ITr9i`*HW zFLv?SGjMf|eD_+!WuwE!m&DV##L=^e+tY#f_JjaZqg;B-dtbqVvtoM8bz48&(Yr7V zk~WgX!m3H3hzM39U4?UhbowIou2gNRwxZ4{kUrpIgHs#k)R75x+R{yJ!_84_DJ{iu z+(J0R!RdnlW7`HXtE+7li1wQlLpA>Sa8KAVer&nO`bU{ITw*m`0a z-Esi@o=UgPW)0pFa~kyH7od-;ySCaPGY_=d8Q^i=C3MT#RoL!-zW~Wjg^aIEkm=** zZL@?}uiBO)PDM$(!1-<6yvEGVrTbNY*sN5q^5`Q7TiX0Vv)dC=AdcHe#RcP&GIiEJ zCXmcxh8n+;s)#U}AWc)HK6^GShQp7lBI44R#TIFg{-P?LcBM6%$QVtKyrvMy2eV|y z@}iiB{IGL@!fj@MbHKXxo`Q@<%2RBG5etv!N#~0rsz+(4-mR(4c9M(sz`iLS9#gpE z2s}J`5<>G3q$PezCO{nzD1{zG*1%H*Ly>u-w5(DjR5V*kw2CKS!gq%PP&V5}ji*Sy zAb-ynoOnQ&7RX7<^edo*9EK2LmNPy9Wx3>PFkB2BcTjnMX*~PglM}{yGUpW-drLR;r0-dz$`;@iv-a1_fU zInTxmZb>eS&!o_b91)=m;ew~Io19b`nIs}aP-6%v*k)B|$CZl~xY-$8`;*;DlWVHK zTNwG!E`w)(IVBU+B|#L~W)Gq|Ja!!KD6)EFiJQ}bv0rL)iD4bfp;y*S&8Q>0C|x5- z&e6}7Oxn!pF{Q)g1l`~w&x0!pD2fX99eb|VV49NyoU`*X^yb9R5STH{CXznW@Kr$I zjUl51Nq4#*K}HgMrpZ=!*vdSW2?<>kCE1uxJ0k>!|D{E_@4a4 zh9u!t5H8`1Xcbo`Vr9;el<-YbmQC8!?9{kjl03VmzQXZoJ9FPxwU9osa7lx-wU1Co zoVn8&h$!hOaY`wHDpH8La^kWm8k)?|ZJ(xG=lfN!{Aq|OArp^#3M{x)B?yGQW|k6% z2ylFVUT&pO#1F+4{0mkGBUgBV!LaW;seN|=9?{y8&NkyDb>P>N(M1>u8h&LVt>Q6y zZgEo6;T)H0Tr5kFRs%W~NJLAbz$UAwOUd#hrzJxVXhTE+2px1yvs$nm@eqSjvf$O>4c> zb$lHQ)DWrpBD8DqMPzwll3xfdne6iO>lBDQ1RSX9&Xq^DBOiofu*(HK zs+f_t!SVJbWP$sL6ljxEnzpCOgQqOx9xd#dJY{=BRMdCX_io& zo^VEp#D0oLZdKc)4~N=o9#p^ba7Xs)I5EV$b?M4%^3YG(*@kXUv>BF)oL zNl@i7c5p4N3%yCtxd4|DXt!X06fpDZ?dY=H7uZ+6A?F2r)P1L;RHg^iUZm1u&U97Q z1*umKJ5C9((c8M7FxOr+JAVFtm=?>2lS1l^YW>xg>(!=7s z(uiECG%vJL!?#x)REK)*YY*)YI7uNO`6ZU5J5c20eJ?HR#sG{<>%QoJMdI0);%Muj z7fnOfsVN8aM(6+dWNe=cDwE|<@ft^S8O-PqklyM-S%m_8I%lY7G@?XP^iu_@?|kBD zJ-1|zZuM-oHen}3yI7xRQ3-UMUj0(hjP7U`4N4djxN3K(q==#H!Ht?bc=4no1JDr$ zg=N&z1Ww`IK}#tfW*hE*tiFjmFeUD5oPayEA*=OiS&1`rcPX@zf~Tq4=r~Y3s_Krt zTb5;-Wa-lqN|9S}Kj!&T89+s1|g8qN_8qT$)1(-df*v*=5o(tBw@@x{UbB!st78>Sj}E zmLAhfd7^T9)NRuD2w6dpb=B5#vD|ebZMbYPHI4wxGF-K2+Tus}g>~jrXKJUHGZ0+7 zN)6Vz-3^@K?k^O73Yv~K1E=>bG>Ymwd)gsk9(dJZoiE`@c43v=+Wo+XDw&nztkS(G zYvER1@vQ=z4b8-*1i8iiLS`4=DJh>!nG@^($L$tu1DxD$fs>ruo}~n=LtoxYYisDs zj*_oSH{Q;k(5<=vP076`fiEZwIjy;mw<*08AjY=hF-+QjvV9sZZsN5oE_Ad@_^70* z0|=_HaxvK3I!GyCr=pi7C}&hrf+E9RFavV?litwWqjF3H-Q}Q>1C~o2M9VMqdib~O zad*clWxeL8Y35sYXm@sKr(ngVvgF{d?6Cdz>ic__Zi{hi!tI`uvw~gS<-!0drZ%MF&uyhpt@7ba=Cr}eB0d=)a z5PG;cK4un=Nv3`L5NN#26z%Cos!xD3z>gmOQx5hyQ0t-Q+mAk8t(>QZ}1g zAh}zCC1}7rob>tY$A1BGh9>s28WSHw0XUIC6o1WHOOGVC4ZhE>s6Vh}y&phhfT^u% z1HL7ho8eP}_>w)?m;C*Rq$p8pb#HGru!F&%x~ufy@Q7C#Afm?-uS| zUw=PYo61OcDy2V{>j6*TZobqzCy#?WF1#8TZ|e%)!D_rp$mQ;pSAaYCT+!I5NE-ay z_V8uLz`G|J5ljl!c*^TW{tbh8;$4@VMS7!8L$bNU(uouaLWz~FR!Ii0aqL)(Lke4_WRBJS<}2kPiAA%`f?oNugyfCN!Kx$($~OGvxEUiaCBGJl@64E%+H>7 zi6?V_VN9lu5PG^oh!8MKF{GOAl;dEwFFQtU;Et9v)1?c~?gHn)J$6}DGgKo%oP}oK z1|N#NNTxyeO}-0hx7o4pKwO;Q$bWZi1fVj8G?9|M;S#%~QY(>(-8LM90} z(6n8e9SsOdai;j;YD=0ZX!Kr3NE5TR))Z}7+3-dHvPCqn6j(95CUfaQ5R(W@6RKrd zN#cJpL5pm26P$Plo+FvEv*0mUmsd?Q&yK{_O8rVCUh#(zso+4WmU1aAa^WooO)g1vtKwQl;?AEgMokc3XN2!6z* z93Nvz2yUJblt^eYyOMAsy9$xn;_)W`lY*(^ghH#$6iC8qt#GYRxJsH=F$X8T`p-O` z1qsFUybtTTPkLzxn_1BchZ5%_9>B)B+4kFG;BkJbnnFgTvN6qA$A8t%4y2m0>*}NR zTYN0-!3=0*nDxy#VAYmD?U=2oY}<}^l^4j zFJ`j$re4Jdx7!fddtNrM-%C9FZflg!1@$il+h4#d7wsVWjC z=)^@DW1*FpxMoA3s0>K%xD=(jiyvAoJ;;+%T4R5(y&`pv@kuaCMLTBY7R%nQZO#Q^ zWv)fF>uuakNG*feu34k{W1@YkV80!X_EDWN6v8(Sp>nsHlM7>KWNl@UHQHElBEeF0 zJMkzh0FI=mBzv?%yfKBnjOLLvhduW(?~fcbm*h!paSyiqEh=)a-ELej$@MzDmXN(u zrEq`Aq0Js$n%C3EnB>?YXcUj0{XF@ib#}xDtqH%!t$KY^T=Z6k>9=zi}W)48@u~ zM80HEbKT0&_U-e$J(tK_LU1+&i^+gN>w~DSH>h~roJfGUEQV}<=|NI5|0B3mJeNE6 zo$@Z0-3R%);I&1Hcqwuc@?&!9HZ-bZJlD#R?m>ctCEYuHx9J{PS+Do9*0*tH&vd;l zw(aD+goVi}{!wA^SeTX5mv$&sZ!vMNiFyui?*(!%D(iOVy6}^Se#I8puslDfs{Uuk zttmyK^?z{OE?rYNQhrFi5F8`Gi1VdWv}Sw;Wxg-$l8xZK;A)Urq1w@{3myWCqnVb0 zW^&xM4R^G7>LkN{^(5oJlc8NGW;1O9ccS$lSO4R&lktWVgIZs=T3-Q9FatI)G?!6g0Tq8*$&PEc5#9G! z=m(nSJOF|K-PcbByi3wM!z%~zCT|gM{QF^5vB=^bbk#`E5W2<7OCHv6YGCWj9~XW3 z`Ax5WFQ0EeJzTjMqe=)ST|WQyX6h4tF_-L=58;w@QQDi!=a0*;SFK&vI@C|R-`<1P zb;RE5h^ZsCe#-6L*;s!^%{Q(6M}7L>5ua%^mb!{=Cf$A zC+w8$&8D_d)N%&kb(?okdHsAeE9Bd|Pk8ct%eKw7j`ponu6*_? z1;18S^pm#b3&ks=)8++DsJi?RZj(qPAhU^p)y9~WF?|tOfXsnR#ewV-%6JiSwM_!` zjOG!Wd~up%izt7jad=G{v+WDoW#tYy%Icfv;O4$rY-zel8`~rVkG5{pGMdq2*h0m= z#++z2JoQ$9%Ru$^G zT!$%w^`9ReG#H+ORAFzD5vUthiK>!w?4?hblx2Jy*b`rFvM-#3c%4)OPx=cZ| zez*=|VtnXm4xR)zME|vO`yI1uyIE55pofmIuAT~UIsC@I`_mAJ*Sj=7*g1~;&DSxA zAI;ysw>?2JV3bwrE*lX)@OR|V?jAHpV1s$CVWX!{u<`qGm$K?p%?Kh{f-8P)()o!2 zGXhEFUr~QmTm0BWSseHN*K}-mZ*MHa=seD__!f(dm9z@w(I(|%?n;UROG67M@oD%p znkeA1S*nSM*piy!(h$=ez=IA9h@f$nueoj$;e`9~-Q+8m>b?_!QABJx$TvQ9nSV&hZ zMH#1xDXgfY7)2)R+?6aNIEdD5zratTVH@NTvOWIhzl)>}HcEZX4T;Geqg^pSt=25l zHd_jQ4!l0@qgc}s+LY&~D~IbZz2b6xv+m&ppN+O3CgfEfn)M~A=<3f-S{zD^WIabQ z0*HU+d=M?)Rd0x#dBf;V9vQt=a%~N{T1~VW}#iZ2qKTy{4KxeaggMy^F8Q~J^8V19pI=cUY0`g{Or`nLuXU}L5yc4l9;xmN;N zWSz_KZxQxaIti(|K?pU=@73W%gwDK>FF9uqs0Krk(CU=CjHu-*kr+Nu0 zDs_}nIkBUf>CK8_L{d2uvh90-C7>6|{J<_l&cwyrYN5pxUv{?QSjTZIehg8y?T>$S zP(*mt?mwq++kKs{$3V z^kx=!Fe|dNbVTmkbl_>qyR&|UgmA|-v$PcFJz&XmSU};shXoX(aZ`7s?Dc;dn%4^aPbe=}anKAPcpw>C>!cp2-#CpE>Rw_<_^-g>CAA?KbT z6tmKm&so(4_cMg_tk!n22xEW!#_sFswWgSu-TSOA9rXlV{A#NW*Fsbut^8SJK9J60 zlFK_(LTqWMx!qRTI2nyTWC7n(fVkE|^0jPe$#f735E+V@R&M83_!q1}HQrC4%5>}o_A z^2~C?WL5aJ;Oc#|rv!;nYo+fNpZE3YkGO&2UlEEZa{e~BY;y{KYS(^a*F4JNmkkTq z7o;YGEP%TAw@h%HJm?iKBcX#En(n0R6mBhipOg!|9|n<578|9LLTXhx0QRPJ#9Wps zBYq?2$xlfwzudUUJNJJ&vN1lB%q9rB&ApXd`<|Cdvy#-Z%vr1`Ze{K`7G*b+4n7!u zs?$bO@5m*bQ(5PAz?fUGwl&Ms)IxSs8(0!p+9fc~RG#v|B3s3qekOHvL+hQv{J`@u z*}cn7Eq$k3#{TY|IA^@GZAXXue5;2FXx-lE=TaH3GituzH3xqtx^SHOj9u1ND;vrr z>wYEDS!}PD^ZoPfKVvzal=c4Nbx^5<%kMwle*LXJ`Dgw7a?vU$clitF{vB|aBiG==|0dVuRLbMl=A|ZbNhNLi#^_1@Nh3F8Y8tHC zuCrA#tQr)p8@zl}tB^#sxX0|HBb?mcYuGakxFMt4qs2#A7ZGCG#)odhZ5noi|v ziVEf2Jk2sUGawG||NR7- zt%T`|eNE#~?H~8@S>s@n$vAd>MuNJc^>TmKy;RO(HI~=M>5gWgN0B1sn@jVx zGx!y{ssZH893ak^I?9>{%R8YR4B(wFjal~h&tB;=Qc8|^adkztKvwB(UhV(H@`%>w z1R}o>>`a1$Zi;rjo*T(%8OBuQ63YtLfswIX1pT?rif6b&`a)vU?uBKW9-KFojS5e4 zg_S+?o<@lJ-_!38zdp>b55K&G0%NG6 z;ct3>{p)&M9aYF_di^ndJ!$RFS_kv=Xfu4VFIqbbH}lh@F)3QO=Zx0LYi&Nk^I7Xy zo*tcZ@FWDd5?m2l>k@GQ8VwrRzF~~=$-UI)7jA=-$vvMDKpV;@o->w;>B94;*w620 zeu~Jwn+#jMuK#d5`L@Znv5%*G6TLJ(%{b6p=?0SJs{`R>~yB2o@Z z46yoO8&8q>AOeV^r^jHO#zzir1@7RG;;cR1)<8`i2mi;JABSv9?d5Wt)Dk;a3rJqU zLs@_WV*(B~&SJtkNepPJ7?;uczJI=niJ<$22_Xhsd+g$hTKSNATtwo`Hb;B=MqNCA zS8m)MnX%+J0LKO0A~W!KLZ-Cd1??laO1_UwG|h@kA=%%?gf4Y%n2<=k;(Ub5Ipspb zg%R7xxrhX`5YyFnDP9|DK$}Vuu)_Er*s!uZ0X-xT1TbTy-2>Fcr;Pter5v|(17H(I zz~@cCE>{EC2G~FM4s{DKD65S>6aI~VZYNG6cO-0xF@WnmEwf(*smApxII=jpIuYOj z4^|S2Rmqt5EM+U`mL;+CR_rm2hDo<|je4)$o{V#(Z6Ot9qJ5i&kVoI6AtYrYI@`bz zX(z~sq8DON5i-!*0EagM-icXPb*WS9!O#X!x$V7VuVT0f>TsV$x{| zj05!^T!x~4cM&c_?)Pt7#FD$OgIMn1C`t1u^cnk*x1IJKIeodx^oMHcp;pW> zy(^=3$dc7&1ibCs7?LP{opZZ?e20Vk?3FX=&N}bBE=^jNEc;S7D92G*0xqdiyt6F9 z1Vm^re7V(jqpwgStaSi>Be@W0$GwJ{>d>K~I91k^eL}6}rI3r)z0Q&mCDc-hSmk4a zMAIHs)MdqVi@+mI@ZPDIK2D}vZs~FFBtxm7I}bS~*=c30`B_=MtsY~4b~)A**rJAU zu?Ho~e1HA{X;dNEk18Z%Ht~M2?DK1n@-=97L2IYL#qX@vpX*MYmVhmmN_}xQ*iX9w z_>GEPSnhxyhJedO`P534a3<5zF^C*DEM;IVLuQ7ZK&5l=rh>!kKv;03B~uzRZ$nB! z73&W(^}J+inm-(tKwR>F9nK%E0t_FK$T0S6B14D=hXWM<>vGfOdu-^l%FC9Eh-y;6 z-^E@v)X(fgImHngO|}!j{*XNvluhv-heC6ZLv;W%BoTpBb_JC4>DBQLxq&@m%o>$! zdUu$X%`kQ4*c0=01$$IrIh~bBtD_VWxV(;wMgGQ|kiE{QZr6E#tR!7>I`l$Yyo$Db zNa=zH6_(Cj3GBS9*CNh+atU{2Qk#uhPNsaQWX*j7#k*~j2xOGf#y`+I*1BGdZ$*ra zNd;%#VAo4`!V9S_TNdFkG`=qnkDJ-e!(tnaGQPa&7+wzb>CiaqDXHk-uRk-+PWccf zYlBLzxsEsoX{zRb^KLrb{zL$|>%ZUY7{7g9&>E|f&*OMz=5esHK@;P6?PM$vqo3-G z^_1OWEMfslyI1}|Wd_f`era)iA`T_Mus{r5(}%|D^*}*Srz}z{EqN(`-;;i_=_L21 zwX^arqCS0*C$^?%Tp?WMB}zXsa-9KN5eB`y@1RngTa|@>4;c&Ce+)6v>EbwAnAuu_ zotW+z=l2u0F!;;Ti|llfcP!kS+emMhx^2ZgA_f~Zwz!XQA|3^IDI3vjRT)+WgS(Rf zF$*n;rFsY>CZVOI<`$R!4ko*Fk^&pg^$Jr)cfpt;p1ORJ$HQ9qe4*GqEkRC(q0z}f zUsecwt8wdp4AsdyykrV76(m_|DAsR2-zHgnsxAY&iUnyoEmpcokR;#vZou>w``2N( zlyu>-B(~Qxlg|!5BnyCr1rMH_*>4aSY9PB z+1}l+A&G__g_qk1QCcq2<$c)AoVFhE`^_VM%k|=a67am=cJ-|$!VzmOn{IKPT;04Q zV_w>medC&#MoNudZ0lBJFafPy=@V=Xn^|m`-C#>dihen$P8WG4%X~zM6PIL-V&aCo zymv(xh`gmmG@|sAfNpzj+W^0;cHsQhdD~DN1g(Pz zvgW#fMBZ^??w24H;bIkK0@JC~gM z{3nJCgNmbKQb}n4*598!jpVIf|>tp9{2Ix$JpGf|~Nb zEL8wYS~>g`LlR0_8ZL{ezS&-7D`)TJsu0Sdb%N$>BgqQtH8wUgvXVLudN;s@-0 zk;9hl9h+>dTxd+BzQ?puv1q#}o5y*^IQCNDxHOc5neD$-s!E{@E2H0e6stmLky?)n zXSO=Rrr>{pen1eH5n=%o0ys05p`igO0yjC4K@@+TTgi?bxe>kVE9wW7!+ik?4Ros1 z1Ndm@X87tk&BgzJaFW5;$jqveA+)+GlS?qx7ZGf8`|Du!wyGA3GM(&PAX$9KoiLNUgDeSG(> z9wUG4kK@ThK1p>vO&>1bVQzS|w`Gh^PL6HsdU_IenBk76!QuLHH+PIkX3*!xYTlf# z7LWJ2wjX@*`eg7tmXGft^i|8UYWVox57Q<$Qi1`pIU}N{C$!y#5b0X7>Q58NP+YT~2ub5~v zBv^-B%{cHlV(4q&Xy;L4pb1SQ%2+dh9?waxof-C2j{V7{R=``W1>-CvB{%{T-Bpz) z@-oQ6t`eEq1_}%k_Db2_A(;&1(It79VG*`>(0xnAZZ&w6WF8vXrF3u(w6-k^%7K5M zki^`!43Y@4jye>u$b+}JT3HA80<+{;^{19}w}Qt7-cd8D-cE+Rm92SjsIC;*L%I0f8 z>1ldC$V~w6?NG-~p_5O*#@r;uPPcy^_ZX@d_%m?Bt)7XNM3dNI5*{B!(wC0S(R~zu zU&PS0wx9UFe~qL6^H0SIe*~9ft%Vi)mU{*UGcj?GzR#?K}Qeaueo_&+S~S98Wua^dgg#0V0P@jq8UnB?<`^4EYk{U z1sOfsqRkH$@`D{m>_FZz58_B)e)gS3`A#$g^gmrxK_dYcIYD%?}^P)PLE zL7s$0;OU9({uKFwIDjms&r1_}$*1TAM-oz*BYY560t@pSBIFCLqJa@$bg=QR!D>#E z+XfQVN-EyNlV@UP=Ag}Y>ef}55gZ+|s{=bCn4^CH+9)x=-Vdd` z2nvyp0DdioMN?j#uaA%dlQaY;O#^8l{A9{q_z{D2k)jQFNoruq?6CWKD-{}3U}FoY#P=}@v&OEy-&l}wPZ(dKhA z=t!oWM3`jB!}{Pr(h`4X(t=deWsVY`JIrY{aMW^F9@C|l5~HeFy@$z`4fd2;6XfdB6V>hM6PfKxIrQ!S7s8c+!Mn+;^^ib1EmW#Bhjv85!*1I zcBKe*!Z&0m3c!^{lixYkBBLdON0o6^=3=mG69cUMJ9rCbOuT=XpFY4%AaH>M3>0|;~1UBObu4^e=_|&*A?dUi9yeN};`84fgd{pBFiHw&y-%1vp(y(QnR?nDT?^O+wzCo(wJl;mC*1 zsyNKab0agC$nbJeu6s; zBnIaY=*%K?X|pcRZz@l4Acqjg)%ReVOD`Y1d|PqXvUQ-*uNy-;rUL zzm3X`it4XX-IJ5kVd|H!@q{Xmo)~drw8wvdYCu@JX>b%k2flTvxFqS=4w@_Q zFh{s*0D>5k8<6y8Y7 zNmjhBszqbtx>l*FqqcMYfh;G7cp=Ubw~$Gk*a?8jCbE}GTh>g8Zn@9;P4E(C7CyF@ z=_7xvSrw%NT&hBa#pvLMRc>Mp*5+(U2!mX1i7YJW$TPOGuE-o#7Xm;n!T(PM?TF%x^^< z^AZo4tMYSylOJ4!;@;t_oOpB?mx`-dV8Va0`(xf?0pDwHS~Uo-tCZ9MIFny2n=e)h z5W=#(WRE>{2$WbBBp+M8h*(tf_96Q`YrV#viojmhUHly2VXPPR@bJ*JUQF4#=S#Ef zIMuaNGeXU4xVOwK&@!@5(;$hmRxO!UL$n$u8HTVG@8-gdZZ$iRc3bSH%U)EvBW-^d zT~t-1I=88Kg{lB(7Iu1!?bAhV1+s#7ES^hm&dDHrQT;lF?eXR5sf4Hsg0eKev|rFl zSUnO0Mik~88@=6JVUJolNWl_80|6ny@TNCNs+)6O+?S(PZiQDpZm*!t1B}`4?Cp`y zvcS)$U`G(|RL+R1m-9GVgJ@e?k70klC$gkLO9bq58r?aNhd(K^1)I0*NnjMUi*D3o z-)q;^1VT}=J;K!mY*YX7uFO+c^k88$J*Ze_2%0Wjv`Mn5 z=|pBJOS5hwX1%&KCD1;G?f4-rzS=&8_qHD+y~s{`B{fBEb2w^&4XuF_F(T)dwzL(qhL-~wY8K|Z_2#Bv%Mjp%vB8S0nUIAw-V%i+E&9nT>NHZKCNMLzEEV3mHp7A^8{sl z8g=D`;Msmfh8>|(fUSS9m}QbDa@r*$;^;V-f4Y0uZDq^4-L~Aqa&@^6Ja)8PblXv^Cy|M6Bbd>j%S&Ro>%YLz$?fYamMaYlLgZE4dewbR zsQoEW^LLKK6q$|qgNubKG*WdVLAZ$}^hcd5ccU{6^g>3=8AO44^H92syBQD4*K1m(w>h~R*QsbTU0|3MjZ`Exva)J-N z_Oq3L{`&SmE$H4pml0wC69PFjm$BOeC;~Symk<;I7JpjFZX`zzz3VIT2Pz%=0u%%& zyHo@CHnwhtPd3Jv>BG!ret#m!AY;p{Ds@*I4MFORjluPi46*#Rh~*zI;?D2w<*zsjvoADD zopDs<^=U6BaHiuu@>g@=xiO7j+tu3Rsyxq!a?XWTXSy;4>5B?^{iU9lS7mB4-7eA! zSLn;DU_{}GKwUarF)jS{Qg2A#@=aCx+lZb&Ab+b3cgka4AzeUqUcco%UbUdRD%{5g zluVMyAP^Vb8eO9l4TB`9M=hl{@(7v)jS^RoGWyyvS$Fy(sv3AlqrJ$gvJbB7{p0074IAwm zO=6=9YqWa#hj>RCC{p(4J=<;=CH4_q76`)l(d0Li9ASr;BTpQKbde@NxANToY zBB^7}pI?Y>kyGC*NXfzody{~j?)!Hr1OpXQJm!rEKSy~Zr zrrNy~s%>}2_H%Q`{F6WHa6{12uR@wn8Ayjof>`{T#Hw8fNs{-ncCiVXxU{l?1AI$S zJsAUwy!5|+l|6oG;fPv&qITF%7Ju(U{#0!Z#ITvm=1UFDaFWi9I~U(C$UnMWk3zqH zy6U9WGC2~PJ@R}@6IK;jsdZqspOOU1y-5q&D42hHLH}>`0VNv?neaadME=~W6D|NB zu~i3ZcmvuR0=4R{I2mdK9n!55l}qw8Ym{*xIhb?N(FVW}0dNBewoyrtCq7u2lbsXkYhK|cyObHu|GG-uBd95F$>}Dw! zdC6X0Ch9dPc~@Xay>74{=mCLsKcHO@2Yh7;GA5xoj%&AOZPlxC)_>t$(G1mRy{SLA z9U3!v=-nif>0!iL6N_sf<8#b!uvM7}zOH()mVvYo-6MVVs>OdqLSqFjUMaphiDFg0 zO7bB02y^a1XoQ_)l5BNH%RGYG)65#eW@WV{G!uDYJMMv>EpMo1o1Mb&ddhbgR^fOl?=!T>EB`X^d$@ zkx6N}+Nj1ohcZ*0olusYC@}s8#t3^DgH`1bClbzWDmp6OV?i^VJ2M%g%501WuIg^m zOPZZ!NcNBglDs9ewWP9?bGWQnnPo2K6V+Q&(&3^2FBT0Ns+Hj;eHjjkI07{Y$BF4k zFVpe>-ITeOEq{m5214-3+_l=t#Pvu^0cz-LvPO2@8-;wj4K(t92I9HnV zP^jEm?MFY4#oXgU-dvQgh3*u<zSB1z z(1>XdLhA&ftuUWT8XoThDCbrYpVtQ%_WQxf@|-7hrO#Y|Q7L{KXLpO`R-T7+bQd&P zbr7F%1FjXqJa8lMo$4HU1wE*E1S3$N?Wjgn>3@=?8%MReFw__I*3y~J)SmAnhK9bC zhfg}N9%msE-VCx<+zYaJc13J+y&AMK$N2hKg_5_N*k$YS^qeD70t@SeM7iWlm2cB6kKZ&D|ik zN`FCW|K>s)rakk|)A#u%W~Zz<%P0kZ-Mn9U=$^K_r`vx+aB|SbQBBRT>tG?mxTR&- zmTmKb=#JjSAQgF~BVBUmqk_RymrS&((XAYub*B&kx0>GP3^(cS*Gs7H6M}?tl&>#U z9QDLW8rX__{54EkVMo0yF9OQCt zf&7uOd!7z=+VkgyBV(E0Pba%aibk-G+iBMt0f{eM_x?f>jHYP>rSx7#vnm&AjRg&S z*S+P}1?b+i-E(j7Rq#0M<@tZMx*LRFwxBtJ0;54=?bV)Z&?Kf=M-7}%(&VWttbfhT z2TXSV0h7G)lM~yt?eYx?K4R6E_xP@tezeO`Kt>ZeaE~&G)i+J#u5C4zy@(4eOKiI} zFC@)YTVCDxg<3SVOLZGQ^ac${nrRRak;9n0v)>YT0hdvPLQ4y~1jos8Vo|;9aeb^k z%%&H2{|!ZWP__=CE431*kR}E+EPtI}`=@;rnmBpAv=*m^RF_-;+6U@Bx4c@{fN5{O z6X#err7$E}*D9MX+3KD_Xonpj^fFxP-g~5>3E|GzzkQ#0kKO7r_7wZ(%Jh(v`-e5R z8&Q{gFRU}<#x$Zdig{)bq1CHXN60bn{*dCllduY^c<&!ZknzHJ8OwVk2Y-?}C`r43 zn+&&ZMh-nYH%+r=ei4_ZzPIjPoP*&4zGN1nAjCwc6U@g({XhYJdJ8yovor z3T19&b98cLVQmU!Ze(v_Y6>$jm%)?+C<8J!Gm$_Qf6ZFSZX`Dnz3VIT2gD|WJ5VS< z*`+ez+l+NHe6lgVOdn=0^ZP>v8Dx;TsMJs!hES+dk_^W3Vqq}l^5-I#pPpp>Uf-U+ z+-+Qx6j7$XobJ(LxYq~N@>zZ-_w9f z|KHLFf65p=GeJtYU!S#>mpYn8)Yq~asX!i8c{JuV^q;xa@z|a~c?xT#z2+G->ML!H zv%Z&#*>F^7(i6D%RYWOElXe7up?$Kf{uQOcryl5OnM*p!6t%BUL;3XGL6;F&uc&k5 zCQbeJ_VZJ+UT~w$T(rVOAe)UsWOnpg)Q?7Me**hDH*e`pF)iq1?|31tmT!L}IX#mz zG&%EA)X}X-YD&u-c5coU69za>gzjuzQ*&lfqD?wR$F^_dkd?a;#A3t*R`x285{}ytE$EubYZll+;LKtyFXVi}1!&8?sz7SJ z$`LQKt?{;ErUtF1a|VRTfDUH3NjCoBLK|Q3y|`{$HQ5#E&$B}3`r=)TSNEW#iwyFG zMZCBCDkg?WzusI&LQudN&)&?$Ly2?DbHTt;GpvS@us$`relkB5nU4SXxc-K~vxtTq zv5zHbPQYrc#eC*ThhbIjBEJvjNyb0geN0i9eWIyduGx z=mKf+0&I4WsfwUQ`A^Zc!KM@*O2c##h}7k6qaf1jhU0=$NH!v z1I$0bLy^Qx-~r4djJ-M@KKYoh)6pn9R-u>r&j|&_$52iVG~Q_`#iVYTV_V&-{7G~n z6q|c{npY)M`3tml*jqaFOUob^E1=3?M}NTepg154p??PKDo13AFY?AM|JX`EIXLVg z%W8upZ%fgj3_U69=4ooVH?zXG|Go)O(zT9#XmJb5{9ljxfL;S>od4_FbZ9ulF2yTirdD8H&eahBGF|_`FR$jH|i9i zj)l2#C;+C$>(u#2N+0_vW0YXyxC7F1%R!=1rk=DaT1(ezs)@o-;jEuc_-Pm+?q4%6 z&m@~`O+puK>gd5HhNZBDt(YOwb%_iYZiQAh71Qj_nHv{ZzB#IJB0>j<#KUWpG5*Hm zrN&`aw{)9NySvV{1$dfj|5Os;*=DLB}^PP7s98_qPL(_4u>F* zNfs){Po2(oz))LA3Cc#{i#AsiH$!VPE^p?cS>cK~;`Q(Hn#$Xhxh=xr^G1ww^GWX3 z+R6Q_bNq9nR5ecC?B@7>(f6idP{$=kN6Ww_^3n-g7 z3T!b6I=YyZ6yt&K?THO-#KDGhqVYg!_M)gj40Gf=b5e#7i7lg6ZLY@Au^= z@);G`&j2*H+2=nZFM8--lqV@k3uGuQ4dI?~~%813Dip(v|B`RN0&&AQ%zvH@!jN4E{X zKVd;`KN*CI7v?~yBh``p@YUGF+gGgr`dr#NKgN>*v7&gwnxPJt!ZZYn!Rc%SK;~P# zC@@|UUgI)G*F(560rBgzt0_sty<;A!rY?fVv^Wv8+wJd9K|VRZGNjs1ke>SYsXQ?} zJC$A)xS(jhJpX)=q$YWW900B~n4^=)gq^Z;vT|pn*0y)DgIeuElF!YtYpq2GSb<-YTis^Ix_fTPJZ$MtS&Is)OdGEw!dqQ{ zzWqL_QH#IMt*ccBihv_UgO36k6E!3zaN`3dz-6)#BG>d-y=e0)v@MHUo)oz>&YQf< zN8euwqDi~5ZFy%qXL|OI)XAj zgx-35#X7}H`$|@vx7KwhZ{!kJ;Sy{?h;8=hEIRj?=n?kpY`{ezt;wGV`}?k9>nv1C zdEPa}84VUnv@H9%JHU;HB|naK?xM!PN5=SRn#?rQ^6qmV7jX+=ZAS~Fd$)SO^cg<` zrdRrR9U%xoutY+xOD=x_8xeJP-TYmnnyu(E-)4*0zs8v>e1QoJfEE1@!11*V29K(U zx;~Lfb9g~{9?;y5YRSBaj;>n)Ia`?p|!j=BbDq%UAuJHO@nAY}sfm@;%0}ch0R#2#d@mi>tZ|opuxOwrT{oM` za4rUk;#OfBoQ|rjLD_l@aD}!uYOkdVuK9{ctjN;50VL5sCYz1M(iY=2+d!PUddJc( z(V04@G-#cp27^b87`}Oq(*26BE`u1!7!sqj1{Z!05ZEtn+{xrH@hvy+3v>SAyPxpf zCUfxj=M1Q8>1D>wM0G_v{sB%>jPCd!=#zrs~SJ1fNXV-FR`pOb`XY#JEck&XNBoa|?QoL<4XimZo^$I zyhOJO9R2C#`MH1oQu2Cw(c_NB??rZ;@kq=ESf95S&r38m?_u*&h&;S0g#o+urTQLh z$w9vk>6$C_?A0vI@~1pJ&`ym7q#HTE2}~pf*5@;*0dt1EO5A=@YH|^ux^mrV!as+% ziSGch?%Q=C-TltxRvKr|S8ubQIRZLq1@wTGJr z6dk2G9kt*o(FN{KnB>nsklV-SY^5>k?)y}k7=z}TqwP<4DST6nJZCM5;yNGwUWnMi za$H654Hczy)aX8WtQX-EBDd2pnN|+3i%2byH%$H{()QR0`iKF?OCg+Gara$HOIiF%U`E*F_IUT{1pg z^GD@BZ>$Y(S-6>dJ?n;9t#0?f=ycpwqZXnTV^mtQJFWW-pdu^H<#Cu=pu1fFCeUKx z?-zFYxT*)}JD~a5cy8c*_{&SG{mdp;dL!;ZI4Q5E;^_{j4$FpbBxce(EA=pCSYMoF zGe7d$O_3^$lG8A&mS5w<^8zc;1b5omOxLC!#bl>oL0Z7LTpw)oTM&JqN#+R?fe*I- zh;lE8QbNA6c;@)$BB!cI&!?UOrdjRV(8@PXOiP~$Rq`-~^|&(fsHp1r-mAlb3C+? zry*5FV08_a*WM|G(A-q4QDrt9+e%<6F#??rN>2Rl?kz9cN=@6;lv&G)eo+FQs*t=m z%$gxvGdsgoPGFsx)#@l{{(EhFkK?8-59;K7aR|lv*&gH_)#`Gcuv~a?Us3;4f@p*qk_rA6J^yb%4a!mVHYxT_rs*+&# z{knNvT!gQN=N^n5-T^fd6v6SwE$RFXq!RtXo2EU^>6VH?I$IyswmHLU9&*}=J2h9) zD`&pSoSiKl!K+*Z11`Eo$t3j29AR9sk`YJBc-qc1H7p^o;%A!!Sh&1}(5YYbspXPt z?N!Sj>5#srr{HMWT8Go!tV7mP#&^UfQ<#u-zLv7n(vx4W5xvxKX+-SQ$h8FZ#_beV zKkX?YrNE%*no+AN_Q`yaTba?`bX$NcFPP9$-H;%#oZATRp?wkQUY3O$Io*$(?A&-` z@Mkzl+%=#lMnVh%bc(P#^yPP-NB4M>C}hbz8_hg4B!smc2kaJtGdnSuGeLk>qAY)k z5Z2HSUJ^p2WoV@^^Jn!74PyVYGlQ>Hnt&VL66GgieAKoXMTs2-3M_K+0NNr{^(Op_Khi^5|=3O=8 zlH;WL#b!wOr%4@YGo@Mm*cM?Bk492X#@t((s`v3EPYQKkP~z8$3AQ}!sXbh~9NL!e zs0JMDgEbCXJ0LI3-McMvk}S6VlwIur+w&)GpKf5(FSm7Y8|&7}H7~y$);5sJvr%m^ z+xE_0iMK`$VEKmtH+7Xc&R%<@*9?fbud1HVH%%Vf6HqUI1AE9A zB+chRGau?_X%7xeM~u8YRn(BaB<#$m_QE!?rqE zKhnbowEz`r`XSmJa0C^$=Ub4&2J6o?@e!d$Y)uyes8Ffl5xzl84*ej+Q!H6QDKPi5 zrc6L~G$sS!e!;&R%N6sOWE+48Qj~~LZs`w}DTAy!1S1ZCdPafgEaKgi9U_e~3mHZ; zc{X3sq=&G7n8YP94PU-ZaS)9^vW(=X)-6c6E7NzwcuC({rWd0MKLV!J!pZ0BGWqX{ zT$6PHyD%jWqOqwgd02uNG6!rqLFNI|A286ivTpI|P2o2|{0GwYp`Jf1d;R>-c68i*K`WA>5bc9lTr z%ost!>D;u;%i%ACV~qCd>2pblxz)JA1(q3dacd$kI4CMec~rLo{NUeF4_?-%7MIH&PVKU+g_WhJ962x9kA*jwz&=`8>o`*p2^2rV&^5@{&IOh zf=I-_c9xG}!?{LAKs*fT3izXf$a!_oUc*kR!^xQ+26~G3DbqkAC=S1Ag{D26!ugf! z!nEJ^*ctp+2Xlqy$=SMo!t$5TN2L9@n;t>M)b^F22n9C*WcO=W!Gpwtq+nTX-_V~p zLCg8_)=}G!Fsq}^cEVk#psE6TDsBk?r70)#c=GxCcf85Yso!Cygj$xveI#PPLsel) z>da!n$j^OK4Aw?Zk*x3Ec|e~WMuSoHt=Ij*9FC8K?Ux1sNMxe za3I%sxT09H>L=ShartYiLQRVW8Gm7`fp}uw^7+bjPqsX>(^pG{nc)JZ>1`Ka4Ct`$ zNs-Orv#)fz8H?y*=#MW3!wqoJjmuvXSlGn@YGQ>b3IR)uULeSBLEF|=S2r&!ddu4p zH&}hS#(Y!sgR)tHvUy8UEOlJuep@l8<@(0ky2$VhJxt7tsMVZb0B#qZ;_|^=w8ymu zm6`AtQ3lt~iWA=>!iqOUH~#`a{PK=M?<(O*hHD`9|j+PkS z_D@?Cxudziv@a1G-R0M9$a1uqkC7Sz5IpGUTW1x}n^qU*y@!|6%}KH;r85|-7xJic z{SIkYmJZT4zUQR*#s9)&9RI~+sU4dj_`f)qQun$+Pyxw`GIoIsu-&I>ukdkv+)z-8 z!~U+>En1|ui?WF;$QkcH^vu8mlH4-LOuny&Z=9gcgmaDkBaJ;shp_;58G>hUoCU=s(*FtT*+b?{uaQ$;FsdG zVi*!YS_iyuDDcm1;@PwAiTXe+iL+9qv^dUKu>JYi>HVhi2T>NO>@^I&wNM5TX-arTelGv#22~l*a9<*jTe}qUmalbQ*rpl&2ihY3rEf54LK*H* zZ9Rrm$g-|QObD5>ZFkH%M|;7?yJp7HsF#I2U+LYxJ2>Dxk-hW8tRXK&&BeU_ye?(h z{k6yt{vThFku{Wg1h|xWURk3`9M$L-yk)c z*FfvjL!AVbpo!C4!&9cd020L{isT9#2&gEr&PtL9JG=%-o$t1?$ym7x-ypSgnM%B8LyK*v&(k*O7_L@uqSky zZ%_S~+uIBd!*p2bE)!{Cq%qg7iuCGn^U4fQ3304jO4UDqDLa%>{`%uS6~5P7irQD@ z!iyMkt>%Kme!8)^XXlb8!;lPT<6N^>J&+E7n6^+ySqJePo0HYNRBspHxlmZ{qx%6GzteL9diNQv zUby1GjEipah^W4ol34;_tHj3`DyrO6=81+P1WpCo*@$#zu(OOvr5!>^hZ8$tYf2d~ zBjbX$m5BReUNv*GRK0e&-f>9IQl+h9@& zk95iXM!;lteFfWBPnG z@@}2hfdkDwk1BWs6$dH%#ge+~veFSixm9UwgRhmlINa8%&i{v5oKH>1D-QT0l>=Yq z1`f*xiIII9U#bFCcNmW~2-o%|9S!7gy<8b20)7%ppOhjseJt8L|$VlcYPX~1#{5$y4uEova)Gs65teEC6w|IvCg{8OzGK*qLO(rSE5j9 z%hU#GwV7BSQ@o|MmSs_lv)-!^ZmgjGaJM09q^il;@V=1j<%8PQx(tv^aq-CTb+BMP+Jm1U>XcT$>^-y zLe(%;CWL1pi1CUu|IHJL*5gD`p%5@T0ny1E4AjJXBQFI{G7!2C|M&zW&#tRRw=x?Ip zntLc5U327ek18?Kl{!TIsY=o&SokCnI4Is9*AHY|dYYDE6tc@Sb=c94 z`W_I*7+`B8nm<}a!1DEgp)OEOn$$}5y%1?y1Zaeu0>fn$r9?d-GKLZ^{Y+8==B4ZC z!zvmFF9PY=G78aU27@5apC|mE_U8gd=50ecAy~n|_}G=kVke zHl!?gUOM_hXGTv->^TOM04+L@|BLj81Y|8_`jh!=65CUE2>cDFc;^S+aDz|jPUH$6p|{ItL8@1B)46Jv`9-v%}F7W4X+ zpw+QOFY%srC+7GykM~pq6%EpOamry4>J;KS*XcuMkZ%eA8;=B1=x%7go_tS@B=mKK zm$8VQ@l}?!*(F_AlT6pj59=ND^3);gEX=dNfqk40r(SKV-jPy+xqP{B;rtu5SZY+j zw&z2y>gJ7;Hu82D8$45X)*=~GMd-bwxy}ewqlsa9xE1kOX^jfqKELO9rG8_Dor2@ zO!N05M)11iOiCl@eu@ehlLyLEuxUy$@;hgIX>L;iOjdHrVf2DA1GTfE{mFjS8w>lx z+C8x$JQ4!M2^6(hTVFO_(=OdEGMikp7}mjhD*kcbPfWcPiCBlxP)LEHy<}=H__|4- zIgt1{y6gd`WW$cD>Cqz7`Wv*&L4=JBth+ePGSvz!2Eb_XE3m;%?}Ypk?-;7!N8wqs zN&k8SQ0n+(%*X`HIcOJ%-g;)a!0;AO4EjB1$KrP9bjIcY19@@orEjo3oO7g?xe0F9 zVE_hut)EUl@|>JAw-17=g1;_uCFx#L6%1kcC4lg>F(1?$K8kRCYhF%n%*T9>nOhiE z_=l9M+gG;O7-A=ei4nCsR#p7lUT>bJYhvvXz$>Wa5T6z(vowp+<7$|E=ES^8u#_Nv zNakbIln4VF4h{yfFv&qOw;!$N+Nk0)Qk%1ep@%;oaygIh!6%(a_X0^nm>nw9ff;&b zB!i)uC22LiSU}|u4+34umd7VuOcwnKeCbzsSMb{hq{%U^! zFyw2kcmWDU1>hf|P7I|UJi@A({6;1;p5vzln zIBksW=A27CYnXRu1oXeutQx_>n0dQjMwzuSJ>i7?d&7rpMKWVd- zO`nD{>PQKlc_bQwbYciv{58;aOd%@)i_)!>)z=rum^NQ%h1?8^Tfa($zhTr@gXEks zTu`~OzqnpLMgj{s@}DX>%G~(CL_D52QIZ!J4hUpPo-4NizmEMc2-Hh>YTl;t2ww=+ zy(SxBcNdo=7-R`YcI|HDe$o#sqwtI*Q+Ip#zwWJi?!3M~ofWiTpYQp#-@kqVbnQlx zqW_fvd_Kk=e@H|t9XCRhMvkj6q@QpWAJ8FJtoUnYybm2uZ9Nj);P^U`)Gb;hdrzZZ zPQ98a9v|l(&+S`fp$^jgMqEZi!GYcH`=S(P(7 zWgFJ=0FFcpp#ea~(|=^s4Q@^X8e@TDQqhxF+wQ$SzqFs_1Ty4 zuhCW}eET)MF!S;ywTyrYrQh=S_F%qY_2_H}a^!oE){&DP`f|!@$fs({W5#ppAZ?}Q zb~V?|gV*5Ii<)kT#jWBAqGG?*EvnwV=ZN#t)eQS3+tV+$mDjE12*-bbaXTptQ|E|5 zP*UTWfGK`4vZmG({WG>Ur;o=RvLOiEd_#W;Fl$GMqA@V&%e;W5X~p&YK1m{IH-_Up zei=L~@9%>z^?M-g4qd&aM^1uStv+0?>5DzbAEZip${1{z zpxP<)W{Ls(ZPin>>InZDPN*%Yvtm@T&l+FhOS}lzMa-Yg>9N`_xp*-~aoHG|^du2a zbgl3w0_}qCGyE73zHssR>HSYcc1t;4r)!5(g}846xEoD1V?P!@)oynX9T4(U8;G)#zTeySI0{r2wBmipKX{Y6 z%Z95i0gERMLIhPweI6|lXo^vvmN;FOBs?v2xO^#hdBO$|tL@Onh8jLkDl$iPxk0qJ z8`SbYPsZJjz#}snq`~?X&9JbIIS{~W64p?i{0lMy7h_01X}p1Sr$H+O03)Gu%&ea; z-XiWyy_2WSnx@TO!PfEoG;CR@u40#!u<-lV)yH8*s)yK|;Z{0zo|_`|!P1DUmtH8y zSws+?^tS`RJth(1B*lM}3_>PtdIo#%&(0UDJ#Libn*T0b05;W)vV^#iOdMPv`kwSj z83@}qY#qsdzf50iQ5$%gAQ7nQTobXTZkViTjiXP=n36Ud37<@j%b{3&Y}*}5Ldh7D z38*BA$wQvO+FkfaE0%(L;BN(?5tlu!T>Hm7=Ee#@uH(ratoxtyh%&HK>st&C93V54 z5dLeP7`FauR1TBavy^0PWz%V%;BET=g}n15m@6QEG|37flxe#}@?EUzt`tRemz0Pj zA7TtCGm69*6w^J!sdn&lIE?%YFEDj4gX-&wFn;TI+QgUsw2izbDeu`%gfyCjLU1?1{x=)$*x0uIaD!3S`l<@iHYUX6tD7MQu3#Yq=y1Ulgu0F zU&!1f8?uw7e*K+u9?sjMiIXkaoB^qOUema{NjXw^jh~P^Xf}g7)tL${w+b~b?hpg& zo6AXegO<>q&54s}3Ae!Ju-)xfT+ll$;X{ys{Y7(yCiP1&7Ld4iZ=XBKcBP!IKqv$K z`n6}oM$P)@abi162=zh>l)cXr2W~=cu0X3O`vxnoHbq=ZFU6ph7BkW4)3zp*iT4@N z{r&_k?&psbGwvHMrwMSzX8mj_Ol2ajgM z_o-iP=Tp?Ow}QERUnlkct;V|m9iC$e_iY>t?>{_F6*dPR`I6HoX=wk-%c1Lp^RvtG z-n>C1-F~0ZIN;AYTBxbzDZrHG!w_%47uJW&M{J+wjfVRLInK44;0j#Px?#2v-HX|HN_n(;Mjj`x8hHKGf4M49 z6lHY&Eycw()3qPw57+Jx7X?jZyt zfCTmQWMGfgXpp(nc+eg2UbnX=#q;Isd0r4T#gV>4a^*F_mUD*%s=GoM&r+l7<`z!; z0p+n$G{PQ8u+8u?SmsK_7j`HyK*R%A1msa%l_*+jm@2uMs_SMIic5@tkQRgh_+Y;M zo)?xfnZ3M$Zx>?`Cn3$+G97y_4MJ-fJ^l{tj5dU!ybwF^ryJ%f4{O**w-y#~ySsYd zi?q|%Gas}hjb0|L{JveE&t4vPW^bQe<-DSuU$tys_3e3XMtO(5t$KRjZ%cm7(8B=_ zXhJ*P=LtRjMWW0EL^kBsg)58z40Y!=GAGVmTdV2vaS1ts!bPf8oq6%dz7ajViFJkb z<*NHs(!gj9Tamg>mAg8`owwd*M3ySG&dHePel)_Zmku&`$8P_lHnhzx_C8gAi4D^n zyRx3Mn%eQV4(nKabF_&=jKmIocy-F>o}LgCTpruNF@6Z$$g8qyRuqNEm5CLb2gWG#ZABGws6(e4H8?7j;+jqTt4q<3GgO4%SbPc;@^?9 z_v<$sTHqSA`oe@5DzyYnAvHf=mC<*R0v|UY22fNGAVCa(gm-9|M;pG8U*I*Pjn~93!N;BDJw$?C6W(_ES)F_sNy&4_ z*42eK42np3@Rh;s7+1rM3mt|?K?GU7e5{W!V%2;3M`H}N(KPjbzTWUyEE*ii?X1>- zJlw^&n9^XAma|a~IwRx3r&Yt3-*O7_pYm7D?39M`RS{?pA;(RCG?wAMi`>qI#$+__ ziFlu+;jY5~$lCdViKa3YmnZ)LlYy0#M=h;5hLl`oMwdml(a1ShY_z@XIl8VY&+Jz6XZDF%%J|DsjZJ;=K~NEUjY>M52+(rjy@+j|axb-_|6^FsGg1f&xetwQSC zDx^`7TZ8OS37!xaW94P18yB2{Mv?w$^7^u1*aFJUi%Z=6E;9IiKRFntBs4)!@yxqWLw5s~A-! zHB5{xgqTeLQpx9DX(%n>*58$6b)O^hnj%Q9$1@#JYF&sEWKu@#aJEmfL<=kq6IjxX zGz&@i*|nK&(KkvbBJ8L-zQ%}#CRW?JQO2gc)A}d&plU7@$bNN!b&ySk*t!TYp(^L? z2(M2h>5|KkZ6f5+5ufXrMc6H9BLa~KX;*MP8KMwCaS!2)=0}qW= zq5X{FT$|N{E2)7CZduyBF_dE{H0W=|hOM*-f?=}YeV@qE;|N2UdXB{f|fv>=)pRn(U{%&dMBqVpemhEnCiftIoJW3 zO;a|2xF_U0)0i&sJQVnz2x+0GidInHy#b@~$_%W7bA(Nu*GkG?RZI8`%o7bU8^fGQ z+VjSKXsrY?V-K_A%r#Fqo4@7xZy)N!!&;J9W?IC_j%oR{;2=Zg7jw27(ETr(jePL0 z*wbL3;gC{Rr9r}t(&lc)!)=CXrJ7#5OuQa|ESY*^cV23S1YTc#@akpAVpPW+P-n*d zqv!pM&ju__msw*tRk-$U|F4f2Qpn6=YbKf%UdW`-mG~XaATQw|KO~__XQ680 z1)crG!7`5$k*D$cu752u>7Qmtc8$M>g*PhAO&HyhhH~bw-cJOI>Or0KHhB@#h&3(& zPSxRgyh#1zcm3~OrwDQff%>Rb{`G@V^FbbULT28H2`jXam|buKTbuO#bakCUYLG0- z*Qk zhiEDEX@s~V4=e4pY*`R|w(e46CUYu4Ti1-#y}qA8|5;VpLF>UxB9pAUCBZ`0A#%n1 zDN$c41FbPC+{v|7cmX8$P*8A{7{{%SavJQhQoJ7tEwHnWc9PdIYaEx)TIHsHssp^y zGs?i0u-{U^um()_eVEd)Y+zjVZ@mutpFLEOm$C_i|o0yyCF!qFmuo;EiY2aNa$QBqylcvv?9CXYA8;j>f5Cw zb9+Xt43}?=*a5V+ZNi;1?brYgh%;1SAq|mFOw*Y9qcl8xT;RgTQCod}fp4_#rYt~S z8^%WhMc)Z$$5TUa_Tr4mqgDstfa`>^cXTn1{ibiu9kwcZiH9QdQUW+-;BRQzGtYFq zRC4&VU*M=EM0#yga zBWjF6?x)s(L+WLr6{z|Gsl?-$wwzE{xCQ+DEg6PJEWX@NyY93@TPm=9A;CJ{tlB4X z@C(O>!<{$e7L)Vqu!S;$ggZH?9*@O6(OoULU@%B#$Qa85QV)NBK~x+@C{l=aYr^X* z%~k4No7LcGN^-M8u+$}BvLk^K9;W@mqY#gl^Uk88D;I@QtG4lvb#wfoyg<2Ju?eF0 z?_kuFQYw*f#h8l^<;l^P`s=ZAyKFNX7n1J$pdYd(hthS*G0?@qk}~TWp|(5Em-rWB z(%yY3xKNGiFZY#3%T^7X-uMX5)?@*`DV%=10HhNqqXePdY|LMPrN%XVrT!oKCGt`@ zqu-O-o^hmf0(k*;{0D4g>-pMCl^i%fDA7+%WkDu%ohS-By9Wytc1TSNOhCx7vI#JI z$jrNi1ELX`noO(CFL-1QO-`;0?k;4eF%yjF8)r~V*;pc4TmcTGPvLi$2rD?9|hA{~e7GlqrB_df!56OTUQX`6_YOwm$xULi}BD>XI1-~p0V%KGxc!TY@Vi`ctX;}wo?C+1_8wN z_Cb{MkfA}2boVB`E_d?DC8qC(wHF6w+-w>w>O^7D5ZM<1E(JGfVF}G(bz~gZ==Mlk zM+m}{M@RFo5rVz!vb~Ko=LRn(Jqi?ATyUgXQroAc>ns%x8~WMnfVs3$jXdMu$5s?f ztM_)6`o%-)TN&DmJ4e$SD@F$n1_oCYr8FH%s72#$H&>vl1eN|%b^b+>&Q*HsiAG3? zBbcWuo;`tpqOC4^g=Me0fY>O?>9JCZ{C#Ev9+))IgAPhGp7I+j^U`Y*H)T|DU9zY&Dv7Y*Z8Ug`N~F4f;bgI0Pt zGe%<3^zJo#WrlATVWn+jD%*0mbz+UFsh>ID zp+N`8Plirxoj2?%m)kmSt}b6cV4R@T|3k`TW&W=qOpvIxg7=G=B~{+=A1|ae9Yfsy zZ&yR<6|6<|hYDB>D*YoU0=YkW1}f`$#>`1`$J~6-PH7s|=v_Ao2P1`H2CTdLut6p8664z#bWY zOyV$eR5g3j;`SQ9Yfue6co1{uMS(jU5-e3I6ETR1;O1Q01FgUuJi)ayc`wHpJ|JOon2@a_|Ph!8Ur9EG3 zG~Ia1$W^I|9_Re}#<8^sZ)~|P4Pnvw)*a1Ea#5}#mqG)Y61FF}^i)Xl<3vi`qaH!W zNa~e>!1~Xj`@e2r0D_XPb^HlrgtNBgq3wpLR_dzfT-bAUv?gF3qK0}~rqog*e(vW$ z$+f`b_RqyQojDReB!Wo!a&it!h72LLnLE-opGYG1?1?f0qqUd5Ac7a8bLnjS9&4p} z^9<=JTHh;^O3g{L=9SX$}aKGs| zrVhrGO$;=SmLXs?1Y1E>kY#lKf^`R6aZ8#X2YO;>-XHcl=;@k-%qvc*E|WU`NW>80 z8&qd4m3mvLsZW@^cz8iUOl!Xv@KpT$7OLCcR($!Odfmus%5gLx`I?m9eP$($wjpXV!x5SCz%+gT2td;W@F1Gyi7+ zyCsQ!2N@n2eJ?5aJ-vS3)pE6)~U`!BUYA^Whb zOO6J8=Yk|%fsVBF3&N7BChu1hVXpWqWc0- z3~lJ&CQd>~OZc~wxcGrZ8k8VuC&KcpR*VVQ@|1U-298#1@P?@vv8!MBt$zcLlQj7L zgjWNk_mprp_2FU&pQb)zd<+%%;`Ksj{BDC0P^B~UpLPtIsU_BjN9~?_TB6A)q1v-I zX5RI~YOm!KibX+;}97QdnN#`Y@h-Q_f_uZJ$nk!J(E`NbinkgUKI(4nT%owN6R{qRS)`e|kp z>?uzYlX!_!fM5C;{<7GG6WS0uVpe4fSe|DbvsmONRGrqBB;I)1nL*0R7avWAz`TUp zC1SUSx*3kfP)u+o4kh{Y+zQ4CY!Ezyhcn3MFRnC}5dn`E%4Or1nQGt4ZUqO!50wIl zByq1sSEKsv5wC_k#lN2u1Y|p^{W-}$@~E@{k%A$DyG7*~Go&O*l}ag<3Jm0F#OG;< zn`|Phbe6bk@z4qlrrOg4XkO?X*K>A*1-(TE7#m6=f3lWi%Bn5#0;TJ*h zlhOh+KCy1K5$f|pHtfDBzm34jQ~wm4fxrqp#eTI#s;M}p!o8VyP+btQLO%{H`Zu?^ zD|8D_P#x^O4qh+Zfm+G@k<8aUNi1=K;&Na>(-WszG&A>1rJ8?M9^T_Rfz{6Zm+I_sm~hm;mUP_}xMY-L~$y7urns zowEi9wBnrnWx!7a6YoC}&c5}QH>|gJQJarwkjF_S{xNlE zwJQZ%yUxPC<1ubPAY#zx=mUVj)dcgbkaNeB{L8Z;fm*s+=!7YfVt9IlMf{T!|CSH` z(JHrISKgFr!;4m#l!(bp=ug<6M3lwELqU9C6J5Yci#^ZvWpT6Z(tMUPe%$r0qUf1; zpKfoiu2Uz{z!Pq^NOeR;tMNh^NvmWNZy7dkw<4YVnDbb5;)iuK+A4sQfdCfx^1)Q% zEsHqgfl?*3IT;+zN(Ea1cCC}{Bye%(*uBQPX>eT3xxzTHrWB2wG-&T-dZ6SjCp}$(!Xmm5~pnT z%^#;d*PJ`Ka@HON!wXQw$;FI;-eT^$UO9B++8g$d_Hu?PZzDf0%IUqzPMq;G6Ncbo zLpf;!vDek9%rmBgAncOg83~zD<0(T@w0l9*wO_JpC!Ce!Hu`swiSs@MEyWTrHJ|!K z5zi4@XMdhhLd_%lA84={MaT)KuN-rCiG9 z@vFW5mCV!=rVlH=J-v|M-NAOQHVEMX_=b6!YO({X#6aAALScqNTR~PMniIP(cU*{Y zvcFnh)ab(+xXjb6@C(JWMAf8+=R{0SfelH_7W%#RYXB0aUnlbwdwWLzbPlC~HE~gB z3K}|d2Yfi>2cNBen2Kd@7EoJ-Q%Z{c*Bw>sH*+g6dG5Z{%A+laUd~H?jl?W50Au7s z1r0ItQ+pu&4R3?*TYKo{Bw-+w&R+~l>~bznFMhQnv1udaG!tHT+bq!eg>fORq2aK` zl7TgP69AC%R~cM8Uyp|t8#tRqs&6`$4+>i-`qvq!{}jsIMTU8ML5(q#O39-(x1~=g z(d2VgsdF`WUz*9GiAQ4f^q-2UEc$xQ-M|B%fl2s%M7)I2r{U*@A7o#KTROI*i)J|y&ddByS`Z+RgyX#$*Fh`mdu zGC&u#uuLWlbCpk7^cITbcq+c@_)a#_=+Xpy?&#*<(x{r)u#(rMUBKdta%0)@{%;q& zH=Y(9L%05cpd^_RndGn_Gv0$fgye@M7$W+s2@PPc9(i7N;1h zM-Q_I@aF)WY40 za1=l?Ii9+y-f|W}!#m#jTj{b01hD+V-zB6&LUqx~<9!;C(7~Ao?&fT5)rGpdlPXsl z$6l{lEN7$vei6BWd~o0luKHjSMxTfoJ?ReN5!@ENS@WqR&=L*owiuKcl1x)eTlZ(j zycW59*g^2Rj-qb3p`+{z@|WOe1RNlVDXjgJM{!G&@&Z%>$8N@aW_}DLihl#55#X=Z zt+Vr?iauM~AgGc<8{$rF8#zC@8?Q{zlkxhn+WG_RxP-{^KVDUA4F745qWs0d_}_se z9Cq05*?+owy#Cx-F-gL~`=4md z9y-gn$AXQIzhB3L*9Qg5SSs&YJ3sCc_(q;(Xx*TVnx`o;!zvoxT>z~f?IcG{N`PE# z8sFyk`ry+tMy~1^`)5nDT^+Rv!0sIE+@lf}ZT=X{=dZ?sB@iJB+y6Jc-KgbJ_Yzga0i`2S42| zzTJZECLmiX|hWy|T}o z{QWYusEv^;;GF^VA_+ZhVcS%Uk(I`}6=wA@WuF2C#|$2zT{)rY<^ z!=O>xsa1q(-8=`A)+yAd1ZXRMS;;&B1j89fixVOHWGz^hLKfp*bG5303In#WUFE|s zq~0#uHhE2_2z=55C^s+hpVfOjq++%`N-@`ib}Nl|r?##hsB-_%T*YStq=B0}M7dep zLEaR(NkIlwdjUyzT2o6v>Yf=r+)iguEJ#5DA>T{k7#}X@Q2pTCjNVg3&!U6&{@@gs zwnRM9quUc#6izwZS^<`P95+HJaEZNRpU3S8mxo#2wR$TLv(pzF(5U`<1r12QEz{BC zY4b^?wabt zl)_XxnFdz1$v_1g3sG7spEq0k@+I3uJekv!@F`Fl4!BJP_<^j>Z?Zo`vTUPfOEUS_ zK_0`AhVRS8nX)tipf6zXUfzN2Qy0_D8P;2IYJ>=?0Q2crS*REMu_qloO*+LRbPdd7 z7THC2WTXn)|FLyWL7D*DmhQ4`+qP}nwr&39sxI5MZQHi1tIM`Eea?-ExDhi?nfaP8 zxpVKe)`vzfUi0aULbb{d_Lj zh1y~z&dUI+;>;_p8TtjqrI~ynco-_3Bv+RFxZI3?1Wr$*$-3O^`HU(`VmzpIat|&% zS3N12SCQ%VrfNL-nCsj|VZ?o`$!kW~E_4Kz@e2lI@#D(Pd{`l7-C#NfyexD9xUjT7(OTf!lAtB_~C&jH^}Q#y!M> zXg;(x@n;VRh&X88V3V9ySc@%0awvsYpOOJY=ra*Fzho7Nkh1Nz$hr`bZIqR6{rq=j zf6V!WLgM*)58BWpjvgO+P__5n4)|U9Oe}!LYivpLFb4bPihb!p)Ml%U3hR7g+t*ji z)z2GB!lJkI)RBvHzX*?^PERAmHNvffgQvvpriDhBLZYVCiAf?zF|4)JVEQ9RV6ywt z*-m0WCK2$;Z}Q7mS+RP1_UOk)xo0*q8^gXE*{rAtc|IS=X--1ChOBo9Dg z`Yf4RvB_3@nCs`5{sJq1!^6PtLf!0U0)2&CEx4JwWKfLqCqaF;v~S|KRur+&$I<|9 zOa^EP!K0Y(&+GU7+5i#?JChPapP$d$y^Z&+kGG%xGPd&Lr%I~7%)4_{q}M^-6a#~R z9K{xiR=|oiPbFrSLynS!ypb&@Ff!oh5JVPi)nCqB0P?c>Ljjm#k1=pEa>X_k-$!YN z@2R7vCR2g59CAmzD()N64GS>kK0R+}J&m}nw>h2gw2{?LA3$MQJ7Xv|WW<};` zjSx>z2PuBRW^cqq6q-{|th@i2LER-!#!0)1PInAAp^H4~l zN|Kt;g!pN%!~1ed#cm0=xy608Sb^CRXEFtaAzeiA!NBH3c93OL7q^JB3U&E{9 z#@0o9!4hyV{mt{`@4R&9s|`>tgK#6$v)LaX_g1apdtY6~QF&7g48wEhmfqLUVYO>F z%A(QDWk~4|d<~O)JSa{Q0i|vIqQ2v;SMK{Oi?|~Y=e%+=RcxTWdR^~?QptfI2LCUDmQZczkyk!jc-GB@hNf{Ah%F>8!FBt)6Ys}rEts;#&3w={T6 zVrug8J`fbG_=?BNI;(}qK5&&78X#sO%#7Hq7$iv51pmwqPBlpXmFOD0W>C7W2ZOhL#d=x4imTN&8 z6iD@lxq>G=ixkK_E0iMhY1o%Ea8`swL6XYUhmqX2eSuB+9s-U>x2`QSP3+1jCqAFs{L zH(0ESRpkG#YWZ(m5GXS<7t?=I@=N~Z1(8U={-fx;Lu{*zYyu4guwjFx>mcueyGY0X z(*;hD_2q4j8`IHEFv1Esd(hF+$HiTuZ^g;k_VNDa**j?~`^URwr+Oy*yZ8LOmK5zV zOwM67ZJ)BC+ig|6v+HlaCoin|^H=RNH81oRS;(_6rKguVwH_Pd4-CWJ9-vp=XQ6Z8 zTKvLE?{#}-aDi|cUa4LQ8cTr4U-VP=@1@Jvk8*7VC4_E~>s?#IlXtfR>o@stH!c}{ zs7YCxTl-#@3LBujwCht}h`K+$?S1)#?5h)Ly;UY1$w6&yNUHLsfb>O&jF{ zp1(xiz`VT|mWEAMfKyY0HL~B+AdOKj8hTw8UxSl`b1gO*i)Op7y{Y zVl9PEVo8?ntd08wgci)*{bf`R7{`vEw0@;)9vbX8$n5LG1m_^9*pI}Ze{4@a`Q|T> zSr~WP0`q%;yfo8MQQ!QglH6oMO44AhFT_{LbikpT5`^Q+TCDZtB$4c7hNaCfqZw?m7q(4L3YLe|DgWsR%M!G9&U3UAs}1tqXQ zXj6^QQ8u-yZsjTQ$kth)QMVJNWVLR@BT?kIWp4W(YJ3TM!i^yNXY!<|>cs`ilHN}x zWXC4_B>kyCk>E>TZMRvYHDxuYrf^LYe#kP$nBtyx&$sr?9`LNADQ!axD?(_c@tgE4 z+&1nT54$_LSuC^XOy<+}e){B3J2*e@9VrcKyY}$MWKneZj@29=^~R2R^#UBk;N-j2 z+{Qb{<;G{$*3(ni4`Dzry55~=4g96q5WPsg>s_eTR_Vq;Mlt~ju}?bQ0MCWKBSX}< zXRto&nLxDH5&#RuilkkJaH}Z?Ki$Rhl$ewnsma2*loZ^Ytm+tdjlc`GoNM`Y!A5f0 z%f*Wpt_w9WV4uXA$GyZwrkGdLr$Z+orfMTKO#E(Otep66ujm;`Xb>q#a$Lf^=_`<^ zKs@PnAv zV;Cyt`~hBS4A^=eK8bECpP8pZwie5KH>6cu&O&_Hj$kyA`bu2WZ@-N8-494&GeG4) zQ%WIr1OVo`8L|{_z_x@L78xW0oS(CsLeyLXWk2d~Qsfy^)C=)u}JM_#+OUr=L4t2(UGK6;Jl4=Ng`{Gr_fMaky-9 zv$k{c^?kOvyt0V7mCThpV=IM=TNvG@H1naNJ>U#STtk=ZB3=L&{}n!T;O<$y&#aWP zVpew8fa-i$)!;L`$n7Ynkj&80b z2(TIY@8;Mz)oVt1&RMAlT97ro6l|((!n_+gb}C=nEIFx2JnC~j#?=w!{)uRw_6Xc! z_B3HsQ_rn66>@d(B&NZJc1+jarWd)AaqEUF-jg_$+*+;4gb%p z=6G5XitkpBVH{@C0nH;ykgmhzpkWF^2~znL1tBD(i03cExC_sMrn)b!GK^OVZZGID zo|L>=s5Zm7`S6Twbe){!jPLX7JD0F9%{ zX+>*X8g3=Na%FE^K^wv%L6DO!DsNoe)@!F;oT;tGLK`>rI;gefv<-{nuS6MT(_vE) zvja<>gxq9B#@w_4%EJ}n!?19k$$h?TrKmub)G$ib@l5PWug{t=YbkM~^RlAP<;~|O zQ?HQ@A~}kJ`&%Qp_-Gq1x%6FbWFo=C^+p9-E3r4lDBaw25xi$U z(<`6!ikrX^3(Z#X{Q~k9U1wHJ_Pav}oCU`wBsdQ(s%o4Ck#KOkgTY$2y?Vk(($q3^ zJeHL3np35{9`>}|TF*~IVw;tcOKUl%6`j9EsPQUv<4;+{!XUN$HzRqD0BLnIFzS(! z%6|9@;4XLq7?V{W!qjfq22O7f8+U@`E@*WOaj;IuD#Is}f%8ETuqq6SSyLxtXY3$g z40XI1VF=c)PHM>qU1~`^&tgKl!iOMM886NF)Vl`4P&FZK$(R!g9!g1!4tUthO1>#E zQ~0TKrtO$-#+BlId%5#6z&OIxI0ElI70XKI>bd`jv*URcN41quF z5rHQxSehCFQu3ajPpz-6S!39G3zCv7?xO33n&-;otF(xVCt%QN$E6L1${5N>ot#311;U`CjZD5u$d4QNw_ zORmi@pr?It<+&o8(yiEn_fb14k$0{`zW)64PiQBt{&OUd~DpGc@c}(~zCNH?H>l=at#2@bGzgggEFNz>CKy3es zP+ZYAaKPa}2CS9+G5HuI;~K)cM+B8zlEKLjhe^bpXTKG>JVoB&QnsEFe80NkSl&#O zk0rK4N3-oGPUAU*J^6{x3WlLKbEBpI-JZTqHiAeMR?;Iq7;7!8qf3A3?3zf8U-b`V z(X>>aRD9WNE~}v-B7~zSrzzcF8=YPk0*Iy4L`EbNrs$FcWsO8q4yK|%HY8aQ+vk)b zYcC%O*cgE(#WExc3IIL?T2~p_8h@=2+#09vPmDeeLK>Y`jW*6w3!WJs%uf-n4+X;; zJ%+65sl|*CM_12>w5IySF4d3nEd{6MI*ep#?UuZy+O0f`0_~|bfi4#dvMvY&3y?Qv z9BKG66^|nL<1)?8Vh9T84@k%kKq)jNB1xRD(Bu>!8z~3laQWq|2;~IHD0&;(m_S2+ z(i+=YEiM*GzXqC(MJGu?OFt^eDBa%&C4>r$w4uKc7(z*9YTHg^NyKc;%xzuA7{wGE zk#lH?H;oL70_JL~0gASOP0p@j4e%jTM2-h7S)iOqKV=O>5el2W*uOQMLGK~}PP1vo8@#c< zP>mp!X!AXepzC^M>!-&D=bp_A0Rg$YzgIq={3DY~Z}-^X4Vk+= z+4>Cjwe+QHGnrfS0;MpY>A#{_G!IymQ^a+fPtcb$Yq1*%B zvtjShW)mDScK3z;PBz2V14GjrK<}S_=X~e!Jf;R|wllBmWM_$^fe+IQqm==?5A_p; zE_VQ|b|LGp3|uZzaMSf~8(g2G&OG!wjM2NBREIf+GW|Xde7n0s3cvw>_bn%V?cE@2 z0EW71rN*g_>ND6gw9S2~yG2shPVeJRn z&OvGwoh5J5C}es2+&jT~M12nFBsy8>XY>BOm8z~PCh5LHznu(smfUsD60vn! z%Jc8iQ{Qcv%7ky3s@!tdnhMiHzq+b#n_Ap5gClue)kR!Nc-|S=RQ46je9@5k(fNW_ z4uPGTK}xMD1wgFB<7&#UurR;zq3ey+6V3hwnat2(FU-GB9ov9kx?HXBbYdhF( zTWsSN21J$sy|=1!#3*u@z9Gz5gS8-=+DI(6qVd3?#p1_((7I1I!u8H#m=2{Dktf?R zInn5n_SY<7RE`1WcK3AUzDBkbYRs`mf=!p>tmqbOuPVsAqd`zfJ5EV0m?3{)vIRPn$;{h!l9mNvkrZDX%%&^1un!Wg)c4K|t51#_1 z`OoKXW2ppJC*b-00c8dU&{x^iTX@U;jPf`X#O^;UgDwohg-B7TETfZ<%#7+h6|6PT zfcdQKag7;}r6nNOUwL@cOce<3C(@3{3yI&*ST_nuR>`bbqZS-q9{TJS21l&&k*|=! z>%iAJs*=04D+s~l_1Tr5PvH^6PHU#;9GuATLku+FH%4n%jj(JV-A?L89t$(u8DpHU z)%7Qx{CA0l;mBtNB}Z6XOjV(ihgZ+rfbHZRr>vG^YKXn^>gA*K_Mw+yJk;pS2Po;$ zZWkK!JBRYcBe{~PgOOBC5?A_72>h!~n$W)@WE30pt7lah{JwG1_edP%Tm3W{C58H$ zajxznN2FaPe)zhZGMn)#KSUvZtjxLHij@!vD-eiRMRcNq#lQo#P%q+!_(#e>Vs06F zU^_W46~rt2N;&ay6)5-i?z0>8To+J5m6!OrQ-6kBHl{_)j@)LyA=IAO+5bOHj(;S; z|A0L}xHwt<)8v@Z{%3jVfZN%tyG3w~*+uBL%-^v`h&OZe5Ngd(V}JV*6OAmUR7=_7 zNctsIWYqOoNO^x7Pwnz{U`*fl!}#d+@pjNAQH^2KYQ4F4bRyutKBKv(Ju{*oxhEW* zW_xZP)bsgya^#?4vOR=(6D^g(AH%f2`n~nyc$4>gb05&v)zjzYP}^e>gdZwC81X#q zAA3_fgL?ImMbn6ia02Fgt#Zex8BrFt|N60;H^AJ`-52`Di2v10>JehvrtT5+efK@P z`?szo+$Q8>aBFL+A8EpQY3t=`?l#N`Q#X!`f+1j|d(0J&W(Z!;`sGioFKCphBuU+_0KE?X<<4Ow5U+e+jp0Z=u2b)AP-%xP<_WwvoK;1TyZ@9+9VG` z{8t#*xrN93mC`hs!$g~aYsryj^qr)Y1jV+G^MRC}Y@OTy2bNJrX?t#xlyonguEuF-Yhp$qJ-+)#?a$P(BpI zcU+>YS7BcSZ!4E^JSPu@0}6p8gfE{d3j}5%&R6-n5SLW#M$?*+$&kh2oR|-vmo>A_ z9gOtPnO;l>E1_z^%A-S}N5h`5g(B@i*_@W9HNs^sAk2oB}T zG=s;FPLY>A9F;M-3xsVDY3I+LwGH`sZH8)8iqpt#ll1e}{)97V9}Q`4lSpEs&1V|{ z=Ow}D#AeJvp^q49O8K>KJZHT3kvJ*En8An4!xIN`u7jK3ba5K~krTZHIfl05JORu? zy%*$e>BPO|>XIPt230enuMWBmf=YE;5e%WL=zH|!H%4zpXWdb)g0I4jq_0h{ZZ=TN zbG3XQT8LWBUvN1x8-vc}!v3vta$@`ivl<3VXlY8c_4v3qMjq-yn*eIFbU;vJH4Y4w zG3puP*2ADv&6rlHW2%5Bu$(2FEZB#Gj0AkV@|anL6p=H;g$?10>BM|fcfZe!kS#%_YZaX5gYHF@a- zMV>BuR^(e=H6?lXbPH+Nqd|CL?=N5WvDEruvyC6}+qca?1XW^dsj+jBd2cDdu>key zR9!vx2kzX^=ZugSmvy8u5R$1}!cY)pd1_s=`h43i78o~gMmQTjWzHO}wrJOIqnI)B zszx#3YH*YGzwz@Zv`8-8#arr}u{ z=rkkWf-snzWDjoK4u+gnZg6oq*A@{gOpacC@mx6q%yYv0 zArI9$VzLcERA;a%pHu07zc|td<-tfP5`*)T8dSD(U%OVV9NlHi%a#E>OMQJp=_|2& zM;CKMig06BbLxZvhA}Rlg|TOd54M|`J(A`xi{=PO{jJFr;BB*>z(D!Qj3$DVZZ1^T zumOBhyK1$;2A+B7_Q;X;Qkk_@Yxz!Jw@POA#~eA_V4<}9)PqBift-#@2}DQ;Hc9mx zW7U_^us)d{kmwlAkOhF;d@;o4fOwqRqT=EU%_f#oiJJ;PfxWRC{pASi780>dp~wD& z529?vym?~n)}U|hmf*w`ngm8}Y9F`7!QRQTSxg*zYM7=yNewy+UWF2epKFiZCLx#doF2x%^y^pu5!ns(kkQe~H!^XPW`h(%_`*beu zWNIc~Jp}-cVpwnUE=$swXSPw3hb(USeg`)`Dsai?w&Z$y@>e@!;9kz7qN`RoD^+)I zJ@Xux6Sg{RY98aGsoM8ltPO0E6gqw|XtnbmN=$aWv>WV#<}`eo3ukL98+H37!vKeM zrfEL=m}F@Lm;>N%p**hZAz>kX8$LV6 zoA{zaH#t~1KO-<+*?Ima@2cOt^wLh)c&8#mfBK3HvLk?LIo_wT;Ff|lIV@~mJEevr z3Xbl#HA$Ozt@waC*~|&_=WdFvjI8y>%nzQU1;=;MZ05=e$Q00+ead<4FZ^RH4ZNKX zW}g%#4=L5kAduEIrqmM<1l*QMr49=R^LkUw5dYJ=HX@QK`oNQgvFd+-J+lE^Bn3%K z8GNigyIDXtI70z<7BY7_k#4Db#S)d}oprS%dt34#lzqK@+XJ%l^~6q|5AQ>4Ql~+1 ziS62s08|YN?b~Tv_7@VpmzaG#gN4}3D}12+QSb4Va4pcg9-QO$FK4`3fjgP2I)}VT zMdyFe&xkSI+^9dV47RKK5>Qy*1@@aduy?vXMZy4;zSiPOOp;>V%%h`4TaB`NbK-XM zn(!xz5*jpgF!=~80rfR6o-#2G)ACZ5<&onehcb-m)0q(jxb*s4N{)-ZRJ|go-F8sY+C240KmRD{oo+3{gAI+T~pt_iOyj9H)Jj7a- zT#&tEZ(?;M;Ot+)9gSB^;oAhgmmH*k9h%OsFFoZNPC)*}1lxoc+tQIwGm>6{eP9gO zsI2PBa)@(Mkt^yzcgZ+tv%mLVxteNoqEbp#ifxS6XY@a5YE%}hF0^b3^{gjCN@T^47)SBCB^fnzV2FedyaEzwPT=UX*>;={NK7>@&F`z*`F|1*g ztW!yBX!gCCQ!bpe>Dc67@pFDdHCo)x?X2$G8Fq3*QI4II)A1n-(H*v4phox^pDgQ)0*(I z2F)W)!@p&)-hvG>GYoEwIzs`wd60h_ zZ*|Yg#PL6zZo`ayqx0o-0@Z}}LU`}qK7TYh}3<1|WO z5=IZ!cXI>rkNwM-?r{Ta=PuFhQnRW z&V|&j0S;{4&J3-8|N5C6iQACrYgo|hcOdlL@w+#Pz~C7oG#4#TJm0G?RZTatUt;V}<25!6!bM}-%Di<83R7qufdVKrrk>Pya5#1H}70a1y zkfxE&)|lKmk}9fO{<~W5Hy2Kw-|=xejbLDx4PaV;rSKtuYG1ZrshITxkex*NX9nRK z8m=bbvk!U9lOVXMlA-)##`(*#<+k<{jH&EGBvb5=xcXkd zId8fIFrBG240>{QYbN*!L74?DV=Hb|gIRxwg3W<#0S)a$V{a?nNn`CtUw{6CAAzx= zEWmDSs`vU4H+T?O$Y}3b)fD#+geL8}HikC2k^BCeHLLtHynC}xnnFoXL+buRNuAl;HvHYv7R36|apj3WJWntP zD8slEW~#*;s+%}DICO!C5k7Q{gEzCO`~VloYAIUYShrzh7#$))6Hh;3kr77UP2a6N zHN`*0pTUV_VbM7SIHUbUK;Q)g0KN&R(V;H!eti);FB?!pk*~<_!2?A&XYQ}KH7JaEU0a~&%0%Q zn}ktB>a!jpiBTFXIVasnHFG*%@2^RZI9hD1Lhyou^oXWiN`g+22gwHF6L1zdC~Lu z?Yry-(#1Q1a|5@+of9gwF0uws^L0Dv?$V?Wr$G(_7TG?P836hEGrJl`VNrC@9LnJp zb=alYIf+pw+<`(ViFsH@k~BoO(2n4VGTOO*sn~~Opltg!jGSJ__ZngcwK==3=%${v zvGl7v0U~lVK^T2E9Y7XmBQaLZ0z|x&Gk7p?Zu(5Z&;M+|H^Bq7b(=o>YRut=lddOG zk8yE)4{4B?=fyXjxlf^w<{kDv+-|;V%&Nr&Rio4cVaK{biN}&0zneN7b9t_F3%6p| z@Ig9t)(f)N|E6VcU zoB%YidikU_-&cmjwh$n}v)+k$@mBfr+K7=@XxY*eM;_pj^_WuhEU<5xg~^)^gHkW8 z=Q+@-aob<Zl*^wQ#J^(26GI3GMofd8lwEk6? z7~SkM(#Ym*RjLRupcvKEM|YD}1F5D^?VG_spTuA|bcFytU5I@AN)WHgR8x11h1NFG$7EZa8EG%8S)c%}hl1?Dkh1Jpv}NS#{}dp>!=7p8B204)}GS)IJbWfpHQK~ugoP9v7&=tvXFar|YGgW#1n zb_pxg5kJnNFTZo%{gaL?tw4s`h0~Gpueo{K56x!7;z?UyGsymn!@1L%Y{FZ#Ay>6& z3w<9w+vC&qFMVhIQ6hw}YUdZRhRt0sM2New5TNx}afg9NCU4p$o@$txEifQtnb0IA zdJldvY_a8o!&T0hvGyT{eVh$!_we6?E@Z#W1riMGfsHNs2X&;v#AV<-RE4uyYcoM} zz%K|!!`@wCm(yK1Zh_&+x6)t5h7rt#x(8v5Von>|dZ-RX)=cM?ZuB2v(k}buH;M}B zDxi`)`-HpOiOCsp@4OU4$p4GHjOqg}Kbx)wN4?40=dP~GN-4|S=~R!IinGT*8PAx5h1&7%*(gEVFOIjQ%}1BubFLhtU{Qj|P!=uq<@?4_oGRj_Bp=Wm{g5%~A)>-S8cF9Btu`Y*e|K1T5x(f#c8~`ba z!MBT~C~6`nMk_IyP)E-QM&}nwi8-&k-c>qEpSkPQ%ED%MzX|AXbX1j}mHWR~Fx%y{ zPVy7merk0TJsb#TQCc-fN(XD`&W3l1C^o;)?I_Yqw}=qKPC+3i@9avb!sqk7l>H5`qVv?2_!;eTb=SNe z>ui4F)Ap8}*cuLUygUC{_t~C{oJdYgUA#i-T|wKUj5&{S5H}GR4TqOC5dhHPR+HtG zsf19eCG^U#j3?W~Mt~DLOFUQQ_=~aU0eU!c_iftoKnvpO9?}&dRXQ)tV>Ou4Kazb7 z_*18)5KF4wfbsWw>BIFi53`RU;u!(DBM5&YRbXC&j|tj#sE&>a;?KGa@r?YdS@e4mB3Wxx3`A6)4D$w}` zE}AS7Z5B33kUc-xJpWMYb7SpibMUFj)@JPh$2e0I1v_G`8cGcO3&2a4qey!4AW_OD z3dCN(EH&}FNQbI^$o_}|ZP%TuI{~1{x)q8K)hbOt#eJ8)&Ubff8_D=F7Ib*)ky$fx z=GpHfN66sBsT?g+)JxwGuff{#JESs?NdQuwx0L?+U%XBzoyFHT0ZVr{hydXGl>z1w zg8YAd3jVYG4}XT0^?zXAAdFo9{T8HXN!inIAntsib_Vd^MxqcxI(GH($D94pi??f> zB!4joVRKTZaW=$MZkjM|2-B#eba}flrWe?Y_PsW6=-U?Q&}>3e`Jp!e^lr>(YV2i3 z^&~f%Wid=7&jj@Ne)+w9S;P8bqFi3>9KHX!9`)3KXkuE}+2xA?*O7qta|XBxMLsP} z5$h`E$-Q>>1eZ|}17p1R{07nr7(&8GdFR{vetmz6Vf#E`P1!=D#CZa0poJYlRdK(lyNc zgvwz|c71{LFmKu}M_^r+wR!8w7Tk*e=&Cl{TEKpS^Y%jJ^3~x>h_nOr*0|PYLi412 zM|^5n&biY_xj*19qG?&?yXpLmmsD}N%c4W#d0eqS)br-6s3W^Q9u*@+BnuVw6NXQ4 z)9oJ6ORqc$DtQTF%!Rrjj`nqR zi5~!3-bvQ(oaOtgiLwDmNkWQ@@f&9y1By2Ofl0t7JZj%KT~#T7BiXR{7L8hRQG>cV z;XmS<&6{|dW92o$Txj1{_?WmM^5{6=rMVY6EcXq$^fX(V{}37V$Sr#x;-3;4TJU$; zRXA#Wz;ou`kaz-FdnvGBnbUb%u*fqt*r|K$$qOJ^pGlb?oQ?x%NkH&|kXqE{Kd=`CLszE@oB~64jBcY!@XYxkLo~?cZ%A#Y^jHaxLZ{T)jv! z=lmKVZRd7^bZvY=JjG<7$KF12x!rC?=YRi9RW*+`O|9|YM@=iN!(FP_RhpFzL6mW_ z`k<`g8FzFY)b2=^^|5X?9Gh8_Q zXR@lo>9;W;Kgm&vU(US6r9Z*H;^ADy0$%E4JK;pi)COzHsCoTGM)S6|H6|v$jI|jO z(?bddB#7^eWS{vwLY>{;^&kHG?w)pjhRk+`yv_?;$&ep^r`QFjh=(tgoo84eGy+dr zM9b_|92<(iOq2q~$6~0TjqcdQPDy{|b+E2G3^nH`(!O({f#u~ql zsNog>m-b1PUT=1QvloB->Sg*6##j6Ca1n7W_xf_+)yt|#5)e)CKpl3X6V1gtHh=lB znGT#YE~YoOg#srHEVCBzgLMi9f-~Hx{qYtWl;wn=T)QnG+fL!Zt5nX7Dc#b=+V@vL zc0gQfYnt|QNY7_V)(K#^G{@;beKL*stAGImbphBpuV^NOta82;ZJ-9k`AjjxcN{Y~9y8?@g{t;^S0* z!G^jLIy8ON&r*JLJ%=`Pu2#g-wc11VWYYENzFvX^2pNp-xXC&NG@4k(ElE1l6-fY+);>=VW|R%xlBHty8>&Y+WWEAiZo(65RwXbPbLfyc(PQfw;__yi|7Iw-Q#<1=6`t>tmsuAxkj-0Smu z<-OETt+4-L;3$|bDd#EKJ13-p6*&_+Iz8Wvd6AKlUf^W8wDDTV@D3N*-(G__&a1(_ z&b}ZiFvN1AvXWy+WJ@8-OKnsQG^3IkfO9CP9huHtW-MW6iA)-VC;}}_-1fTaG09Yb*uuT5br}=X*_3M$@xx>MOj^(9@Ty!CWD~kCwe-BK0zFrz zR^#{&N%yvBdus}O|0|+J0lzuB&K}qboOJ<27mHxp&+s_VK&SvhFeG&}`n>?wtSNMA zxr{cyP)j+wQ|lZeorlW+-MSTiF-in8878_&{-Vjw@pL-D_CIiXmYdIS3aMTT%fy0SF@t=Rb8p+Lrx4v`VZ=Sz@xZb7l5AcK~Us zC^{(SFWuVvi-)IQ!`Idv=(3AcBm9o$jHhR3*Jo!K<<-)jh1D{m@uzOYDKg1pn(3l0 z8J1@#N1n#b;ta@TwB8w@RP$A$w}LRj#KkKblV_rqV9B$jgYz_h&}Na4qvT*`3^AxI zN6D1OH24!%O#s0GffUlZlBfulmg?FrCEZfV$o-c>0nC=Xj!3^JK|)t8f%Pv0Jq5ED zJfWdMw2|t!seeJpd?MGAhzqjV9rPC-hv4}Z71)Uug4HC`FGLm#LGt|peO=QvQuoS~ zZ!QP}83oQbW?v90r=e3b=5$}%CKJa7evU65P{v0#K?4v4+w#>Q5(oC*N{0ZwodNmj$3?i*RCpK+^Nh2$i)Ds%wiFapk0e z>tR~?gtFu#+1Xi!7Bn#x=Z`Ky2{C!0wx_ZNv0GCn{E0nwWX06NM&d`9u#wU1E=8(= z6gIGu(FCpy3Y!KU2Uc_SvNVYf5QFrFu^WVxt_C=ng(wcQ>7fg~2rrQ#g{jZyASUoj zXm;OfFzAzRyxe$*CH7QalqxhbWVx#~3pnele$#)jf6#wGBM5%bPwb*j@xLWJ8_&ki zx`Z*&XPa{p^70(#58c!zUS9bxfY?2sH1@x_O=|`K5F<2?FTPeG*SZI`O+Z~hNBIYQn>t+iE~;LWLz)=Ijc4||Oat`; zLl(~;5+og=P5MAYN$u)UH9VpNFV_WX_-5TNOgR$1cq?3QuIHgxQ><_czUN@oaa#B} zcVGTq50jg*w+0W8Tn~%0aJL|;ee$mWJX4_6eLxLCU-I{#42TSM=uxza79((t1pxI3 zFzo}>VO?e-+Z8uTJL4{P%1)%Lv(-9VIsJuO8v0h}CR5cmS4dL6QJ3L4-NQl?w7HRQ z49xU^V2(t`^XM(D1Ax*YgLRt;=&I#p9_D1=6dcnED?^tAgoQEimI4<27S;RV8U7Gy zqhcYw7!d+@y9 zYY-r-Yye92jmr!-mZ5XM-mVs56l+nt&r7Mu89J?(QJZNtrY_cS5&1WaF3V0GBb=Un z(K&>81B~@o1C$CRA}VZlp`{wlb44O;v+2MXC0peHsYPu%6ou8*zN4X+0pQ3j&j6#U z93xzAw=A6a>iz57&)sq}>jE5}K@Y7>Y~`}`)>5mtbLN7qTDLPKie{4@OjesMyt#V7 zEkXZetC2d&p7q44t1gmW@5rxryz4LU3HZw>rJ{0s$07GVITms2UO23y!9J`laj+4- zED1}1hj{4#pSNEnA~<){0DvS3^I2~dT`!^Sb_BtoG)QCxfsJYJ$*@;o%8YV|bdUwA zjFuoBfz%b2>8WmR#`Bd*USNm)!*#8eKED9DZ3TN$lx$tN(ArU^=C#j6!gG#t?vC*o z)qq6AG5L7tl*R`tTwXh>cx;jaD&U`~I-snd8dxVnpXlHVvbjJJ3@}i@7Oi}nG4F_( zTsIrc?ih;ewIXXtUm)yF&Pp2IJ~I<44rv?jHXE$I$`sn~Ma~+;e{n3W=sne*EyF5UB5>%hN{*eov;NUQ z;LJ~r1lNPPANLT92zX4#RdKWQRJS;cG^ssmA2=%N#C4{p#(>(^b;?-RCW?BHH|Vd{ z8b%O~oT%1{^1Wydd4@^}bE*XB5j&LV#!T8X!V(2oA5y78axSDTzuEQb(3%8viqaw< z;6|$6Ghbzuo-gkB1H~m0!!guO^>+P&0q^*S3&Gnia^hxU1f0ruMl~sJAFuG=P_Y7W zc&(-A8SJ8;ArtKso~Zzr2EYStX$ir&&L4CVD*m>W^`Z|-M`0N^*I2e|x`|3XGFyET z(|^jU?vQz~Zku38xS}TYQRQaMM{^&w+?K85r_%;gBdmaydr?s`LOTcE>@ZZ7_+~3C ztDXBGzIxyZ0mxJp2CcqWG!W`V7=^26Qh5^L8JZaCG`mB(&egIzT{pt7hHjy{vmFWP zBb2hH=nfE@ zxnvNGJHq%Si5h+%Wid4XeGzA8mp+X+FQsi2MvXE!06v_54>PMMW1`r^r}9`eW;yGW za!{94Me4TdI=s`*Y^6pUwbg07eKgZlUbrr~E@;!gc=l~4yWzVmH`8@xB}ZbjCGMiT zH_kqhz{sLj70$>f7P-W^}E4l^RzVwWh`rhMD$B47}V9G$RA?ZC_WMINUheaK6Z zx`a#+MCi`_Hgyqb-OmaxInKP-%E(*NIF(;H0O|5LQ^0pcr1ovB$0`T0$l~wh0$tU& zkgw_Q5ao99E(J(t6NbmaDs*}u8S|s}9r3_SzcXZ-+>4uBtX7p-hE5i%$MpnspuNyD zd*^xot?-%7Gla*=#2q4+{AtKSPdr>@rHwd_PQR+$(XFvd62Ao&cfdh@3M4PG(C|y?RE~ z=KAxtL9&tN!QC~5*l;iLuaI@o`&}h~I9A_}!~ZGls)OQanm%yBEx2po;P5-#-GjSB zaJOJbU;_jKhf6rzL$DyhArPG4hXzl8U?C7J!2*0d&-=cAe>>G%GgH6m?yc&ot=a9K z-`St?F&R$fvXgmsBAmc%A{8}LbnQUe{ixx%cHnu zLE9~r#EwR9tPC1E_OOAQcVTaELlbfuHRk|ZmACQtqV|)-znM3*@owtktK5j#4!07s z`tCYDE@c?OU2=>%KdDIR@~waY%=8;B4i9I{pR~nz$eu{*1Q)oHoXQ>-?O%Wd?pm z%R9G8(`A{`oHGyC6^&cAriC9=uNH_Sk~iw!$!=D(qZ~SZTdodCIxL6tHMJz%VM`Fx z?xW2xNK;)ey}WADJ@rhxZMf<#o{+ieFe~>CrS6~FKXjIbIG^~-`cB&DH5QeZmcJ{1 z3x4NBvn%rn9@)TMq9JeskZ%Re%_XM_&(%~ceJy=kp0~ar+Zg|%V~oxX&#fwMIXoxc z;&wpz-JQT8iEH~+cH-P@Vs~E30@*&DTgz-WX!*58ZMpK=!1nbgzjm9aH?kZkZw`m6 zSDiVih1r#@ofX%eJG9ZbKm3f5Dixi?NjJE)p&LZ2K*DS1q=UL->yTKXx3gH=Y?VhF zJ7fFbSm(UVtsjoJ(Ycg(igx?l`eH^xA?~F2MMAOj+~(l`>J-XQJ^Zv=_QZjbY~j{^ zPUrUx2;+Kq{D|oZobB7u$*G#o3DnMobO=C;lnC@e7I%TjFa$;a%_>GRhk|6$#aeAc zK~>mDk7y79GTarNz7-Y)LZM>_{U`Mp$rBBdMJA+z2$AlgAfeWpXwVEU@*ovNkS-lY z-&&pw+C@hfPuE-EKsKj>gfWB#kPrG~Te;IfZ!nO@nGfsoVGm08Ge8LRfA|NHxtSni zBwp`>wSN^tk)T&#bkTn_w61l5ctFTwNe~xOuoonctXuh?+2&r5H5G=4;J+}bCD3yW z^nV}130>?TdvdL&tDs2C|2Wc$u?gx#2MLJ@JnV8fFvJD^-QT!HS~oc!W$LCGd5Pi0J8=MoZOrGTDTp4lc=3;1sJX z)_kRwid!s9pH2@_1;tv#JIGb~;OIV)?DMD1Qh+_dv&V>fz`1$56imR;Dj*uxF6>$QdM-J#VkMZ-2DV>_J2K__JA^f8ik+CtS??^hygpQ-_ z@;=uHkP70{VUa&3|En*Aiax>UrZzBDA+iHU>p9bUnMPO2$r;;+x9etEl#!b8rR=4& z^}*%jgDu38P~^`gi-BSyK1p0XPvs4Tx!4}aD2PYwy$`NaesM^QlXH>c)_|F&;11qy z1~Icgqkj!a%oB^8t3JUTBZn1#^*!c#->~us-J+x(M|}R&;BBJ~3Uq42I*QYumK%gD zB)TV9f_2BcyjBX9>m=s9eDY7Q+P-&58w`IcU41$pAiJoLe=1XHpsy9EM}8 zwC1%j9bW}YNzEOhA?mgy{l99X_vq8j*4h|CM zw&rebDxbL6@B%sT#_$qGD&W?g@q4_batN_W)V}hOTehFkI~R#N<&qlzsi2X4<&xi< z2bFR}NJu8fU$GpUv^Ns2HAxu3f~lQ99FAhZgK>`*AoWEjuxaCVew!GtIaO0PQm-nO zcGVdcRVBbKfc(4BX}Qi({>jMJWw`c2?TiajCp9lQFbTmc|{PS|c7X9*?@^WryPyoql9IA#ysJz*6^-FzuMexCN)7|eVs71H_1 zyWK+T`@4%XZ4NNrQZD(pr^4C~7l;+O^Onne+qL2dlQ^s|imT2)z zYzM5o2t3*9*R2pGVUt~>h!rRLK93VD9+`W;JP7_2nE!R-=xLYqpUYPIwcJoEMs^3Q!>Tsq4^kEJGFZ+$ zbB$d9gAW226{Pj}52iEihF<1Yt4ed4TpgYz0iHvyYdW*%i97Y5BElGVxR39r&TsA9 z?)|gcc*6&+nxB4OFLYwwJdPcYQlf__xhKtx7S#+ z`mhC# zAccFiBU}n$omY-lioTd8fAed5+)I>_kL*H)bmn$nAKz2J?=#}~L)ngof6tdQ&jKm6 z7ZXRCCwu#T{$|pAeBsNy)eM7o74f&wkmj}8DvKNfth$jCq7QT27&;8b(=N}{Oe*Gw zQsf->QnlJjDDq>tC3<*cTWkZtENu*=BKbu9)^_+i%sUQqiN%`FNM;j$XAwo_M14 zttr&L#yQCPOmu(%h->^+afI@|(g}Fq-~DK%;+On7!SJ$pS3L`E;#1n)A=W|ma$~hX zCBf;*z@>L|R(h%qvr0VffXkijK;GmvHMCYW;g&Jhu9#Vm#O;7h8c#**1CXf^Q3i9# zR;<}b(xJyNiB)_(`nH0|9R57+trl``YQ|pD6C+-prOdr+rlrMAJvk`7{P?@Li%^L_ zh_>tzT?;`kCjt_eLJ2ST*i-jMxu}<7Bj^_;^=2q1vG39Z<=>xeB-gfbN#Tpe8T)qz zoER4JO2?GHTVmTbqU=g)1}p~2Q-;WO&LflTc=mdwrH?V~RrivbtUKvld1(Fc5z33U zdYmjbhs#MSUCDRP)*m@0e1{TU&`w@o+3`TRz&F0SauF@PGb>R+P%4R4){*HO%2b>1 zklTLm`=8&&m#kVDMk*Ku;F2U&u@lv?`PHmH=R8#>$w|_wNZx-k0!YLet_&%B4U7h4 z+7RPqVTLMA8jeb1zgkC1sGxTCjCik44jDdGlk$AbK<7r+j1H3>wHTXjAviZ@{@f~z zv7#ey0b}}_LPN#hDjaF81c7@;y{&YdTM49j`DUZ!Nh*wLy&;P&y35p*Mt6Q6TRr+v zzD)x?^(8eqsCPfw0B{%f+Kl|FF3*D%n{k=8H3IHoSS$4%&J_tuJ=!!4rSeO2lSlb} zV!!II^Ire>G?bL@$UM#TaQ8XWH^c32FZGUe&9p-6G?nj~GsoV|Y&xnk4Qz=yrfDiN z!RnJ`VNGm=2gtH}M$HdaQ60ZRvefZ;2@(m!wkjK^c>+WZQ2;mwHHlhTtgU>&a{mV0XsK!S&rGaB#w%zt+!BTFmfE`u@{t$MUS5>X*t3&qPlmq2}W|;DCL*#46r5e>vDiaAR5U zOTpEG|BveaR3JXM35q0!O(};)$5&W?2sS)fWQ`eKPa0;BOm4Mg#AiruywDZ(NElT< z<14rcJ}MRennP71O_eujf6yJfA%!AjF>hpbk{{rS&B7#UWPX+l-po{0GK~|9I3M{8 zY{hu%v20mtuIHqegK!9y0mo2k}P<2;A9m2%d>C$ z-3DErz|)Pu-OeBO#al|rWg%&l;o1_D!-KL_Q(ZdkwbY&{N2h?wr(ZH`Nb=>8dGmv> zJ6`3_0PJup4;AETw$1dIrO7LfeZ+jl?3*%+Pi+NhB3(^6ZzvUY+Hz;uL#&Q9`*f+ECtBAjXSV^~JXUz}7teiAV6eI%1-vvMlS@N}*twO#W zz-Bv2gmTjsy3Xla-i+y@SfnX&3n_4qVw5*mfkZ9DxrNC(Yoif&f8SCQ{^lkbs4hfprVqJg9 z#QUQh{-L^datv;R4<2{2crP2xw+mgS7v6C*b`cw`c2objvUcjrVsEvdO({B0i@Y-G zy3@gpY3slu$Q>I)h~-$B)ivC60K6UpO*K`A-BmIv#mNY<%h;`|ss-fyQBEj-?&(=~ zH#?v7W{1pb28zXGKA^p$W+9$#llC<+64}WgPaNuZ(Bjckwz&8n z7l`CH@5hoI^!v>Nj6_3^aYQEUDfe1rJOi1k7<>~I%a#^BiL~mSMGD8}fj==%hdfoL z+SZEfZ}s2THME1Omqi^7SRez{J0StGl^HaWQ=XZ*I%^r(&;z2oTD9$PD=$0q5s#oTKSQ191 zX-2hj768+(m)fkP6pdd4So|zaT6@Yrn!gFfh;r~>-GQ&TD0)1Ld|%ms7I&Z+W&xxe zp5j$d*Cixxa|w76BR^lG$-4dg`3~M(b&(rTm)SO?_P1$?C+KxG_olhGSgAM-_UZ7% zjvV?gqH07Ht`;JbBMUXgn7ucj^PoA8It!GoBJZdu+Dw z7CibuvGokq|3V$n>!42_H6m3`T_dhy(Wf^RgyC^KK{&yMw@hedK;FcpV9DY#6630y zmibU@cjW&J=b7>wF3ki5ls;klJ03k4>Gz}qT%l|1MHzhegG=T5RUBM&h*5W{;Fz*d{KsBaQrO$1uz~`v(*E4Q z@{@b?r1U)I|9l`2gSPf;fz;74MPX0@D3WL!#Q5JXbQ|Oh5*GdE{~;Y7jG14>)zAMq zGZe|92V%y6!jRv(Konkr!mYXjx&qA)0{i}0jzE=6D}a^dS! zoad8ilKP*{Qk;WGXAR`t2zuSlZ#6ajU}rt7DvwD%4Oj|%!RW)RyqM5YpC{(kj^A$_xw5HHJg&zb?a&N9Q0`KunO?ikaWQ74FC^{Ltq&VYXm~G7CP) z!?eD)_w$Eofd>{5eP(`jFDLJZ5%j+!r3a*xf{>^P6ea)_QC3uv7f}*a6qXkif1r?8 zR1uU{QV|lC`F|i-|6;6=5uqR^kf6BeKULcdnVKF(M&vOU9oTIdCJY6G{Qnq-*ZN%X|2_BFZsAV){`c0y4z&M4m(#nv?(qYx@wS4yfe`gNZD29y@;f zJTRbEvTKpfzx%U!s{7{Wy=3Uu4E^i66nz%;4k)sXG$n%F7GjEpuodS-=kC#{#{+7x z+~pvq_@g}%xoCJD4ZnJTQu4pMid6o%RnC- zf*k>{*VEOkLQgD{UP~Fn)GvILy;!=SoE{d`1gAx>h%U&jH?aPJ;WU?>x09@~&OTgk zu@5ck8%7&}6GSKyuD*S<6}M)~0W9L@uz7}Z;7t?}K7VPfK4GA@Nc<;)4+W`om%zi8 z=b6;=p+ArGQU}?nA{l)M=&^QtaeYYhu)@-Y#!Ao7#4xXW1;nsp*e1r6kd5C6+R#+k zCK8qG$P=QV>Qh}bo>XReS<8+BEc;hFr9QY)m|*rUDOyhhTdb#=@VC;-K2b=?>8tX_3ziuELrO9YAD#c0RH9UCDe+~7l8#0wy+sWBSVpKkyYVU;XI+H zW_@)VnYi#vk4ukOuiPHWdij~P*o7*zBsy*{F^BoI?i-mZ3$sK{+w^%gbNYCU7&B=( zw(9irJumY5fya}&>w%irK&S!vB>{m=jh=(6f0J;8F^X0HPHb6V95t*vc&YDE`1wzC z#p!SAKAI3}KeN|rD-#u`%4Qrr0*ZVGl>4_VRb_6Rj$uWr4sCpRm(HIZZ&~dDZt>{Rm-KS%GfIt%4e7p4GY^%p1 zuAa}c(d=z$ZPO=?t@zNz9MW0q7L=U3jdHOK(()m3iO(RM=$NTNHpwlHw&I_AxOA}= zUQ)2OvR}W);_m$?e&pCWDXu((uHR$3KZF3LNYtWkmAv#KkqF|&Z)K?%twD3jme<@O zs7>vM(7+&4e6uHF=es1`ELudLI^&WPxt`x3zl=X;)j?x2r7T$S;odBDeQJCS>OA-M*a1XT@ zx7mc?^tO~3Hn>-LB&Sw&gOnP3_8VRs-f0T)Esa%QtR)km=hfVEnCH-Sv1QzrkNg?^ z+BcpAficQb5?sWMrp}%14q?`158i&IJ4rLaMz=y+${fWSmDcS?4l(Df-B*z-LRcb5 z9)$AW7J$G;h_8=PIB`dC&2XwbG#Fnc5Z(cO3esIRNr=>0y zD|1FCg7|WqbDDE%Ko~i9>xSG0(3N%&7zptsYy{;f$svi<%pQ5GTxnuBV>C6IW**0L z%5N#doPXb=LcHuR*_O6c$CseP4+rX~LY!NVvmVokvxQj*gDVbtS{SO!(D}?h?b<5b(W>5 zg^$PIO6fm6SEdMJ2r>bVnOoxT7`}RVjvQVN{?Gj!?L7aNVe9|g&hL4!pQ95VOauxQg5t5WE9 F{U5{pIEDZK delta 54455 zcmV)WK(4=$g%Hg450F0sI5n|B8U=s)zPkh=;Vje2EP|QpHXLgL+#cI_1t29M=X4U| z_~JsIKn~Jg5xEGP(dKUD7{n=TtYU)O1vau^W12;W>PuF(0c5*hv~W=wq;89{4SQR_ zp!M@lG0J%jT0a7(frU$OGJ+}P7J&@wS{C2fEus@B*qqtGLz%NR{)TcXgfV~r#(4uG z&ZNdsx(~RL`KCAxEV#|{L61*2e~&9#LIsD%+om#_Z8AO8RI#2Gp}{Tp9>_AU6i zI!b^4*Yx|%ul?~(<0i$NF=3aeIgVx-m`O-vpw!0EyH5wrC+~K&#_n4OOoDeslDCIK z-NvC%6!olPcdkb+{Ac=Q&W9^7X(w!^Ct>SWuWZCG&sZ*OvOG<=aZC@n?-7GRN2Tz@ zp*nA`{{Vcp%=eRu20oYZLIV|lty@cvBe@N}&#&+wm|BT?14aYw?(qV94CWT(nw)aU z|37Sr6e&tos`g|TgNE%YJxG!K_|ll$e{SaX`!{p=G&g(uU;RC^Led`~_1pD?K^8N#!_0>KE-v0ITj{||cb+BoRF3x+qvd?@!ow;UlVkc(5 zh3teKFYNf#*m%%qEW}#A@KB-vP5Wzf zPn#4xq`yYD3tNnlm$X%XBYgd9T2VCuDSr~R$+Pm*q42@Kq9tim1&fVS{4Gk?)-c2| zJ}2x$@`j`82pnnH#@gw; zsIMa-l{f$6lDFTU+1zmQKK%$BA?M8&F*K`WVn}uYS{9g)R9!Emsu-)u0^< zD?Q>)z@zzS8@6MC-CA6=l?q1f%RVZ8!J^y2(QnR2IAhV5ErF&9oH}?zibiTW>~jF! z{cE1qwt2@7Svc}mlsWZpt%sBYN6*w*m4jZSr~XV*FT zZuSucVLuUT4O2va_dKYPxK`Z>cSN1-ySfJns*Pd?QIvx~B7wK8jM73vgi>$@?aq*} z{!~iW$m3^q2YB*&4ED+MTqeAasTu@D;T4|jrwG?>1|}JruAGFWZ{K~2bx5Xv$sY953^-(eO%Bj((X;&a zr6Ju>6+5UI&3sokS-HtT2nWq9yMs}Y_Yt3Ugm2nJR!DpDAv6$0i#!;fFLv|U3ve}# zeDhk&CDLKzOX6u<(&$;l_;lbMpD;jbluMuU-dC{TY&SjO8rKhZ@-6~{tc`53@X(}i zL2`!RaRh z#=Z?!= zT7$RDoCf{)1?bc24p%#6=98>;26&ox3ElndDr~oZUqEE1!p2u7$nvefy>*vd5xK!OZTe)5v^3O^4Uf(w&46hv+)Tlki>1Q;vM6YGIh~EA&|{th8q81 zRq4V&K^j!0HhVTKfy0lgBHg74ix_E-{-P?LcBM6%$P`VmyrvMy2eV}-^5U3>{jhU^ z!);-IbHF-$&p`%~^6a)Eh(*Wq6} zTH>c<0@U$>LRXhU|y*m|vvRNE8o+A5#{A<2o;Q?LR zASW%;uYeMA7($3y&h!M7<+7`xa4|I8A?4+N@#1$ECrtDDoL6XYN;~*8uC#_$O+VPB z$T|mVB|a2LWL{af*$IV6NCJhpi-)4@0W4An(^K87}w&Zg6 znH*ZNBVv?cT<{!rvy&<#lT1Vx)EEMGY_lq~lgb4KZgvLO_T;eA3A>U>ZJ>paF9p_WW3H;bOUHI1v(r4zu(GvN4@G18kIa;Sz9`AN+S z5n(d67nm-cf%w^aI{AT$C~g&h;?frHw0r<&6jXh1X#U8`V<{(2Hm&ti*YR~MP(vj2 zMP%3A7qR7qNq!+mYW0UGTQoD#1n0CobrGP#~wD*s`QqjW3@Dl;?liE{`d zhIeqAH0Pya6s^8=?@c4D)qqS#WEIg)B^5)#&dk8;<4~{X!dslg2zkpzf|zb061F?I zUpsQbeREehd)b8LH3ffv@t3xDa&Pw1zU*BmEX1x?6*&rpU8}A2T}Sr@Q^kzE4UM-i zAq(6`q#&D|BL8Xke$Mb~c*n-*x)=?Vo3mkazQ>}0oJ$3?`iaiXbfDw!SdfeiM)`0& zdGP*S>y{;8n$cokY7AV&3VBEs@-pGbmkCE6^cJ62r{~D=Ajv_0NVAOM{6sQBBKAl3 z$XK;a`f#|t=0WuhdHXl zvandw*wpB+>Oh+f0q{v-mx&RNo2M265wto>cY2nzYEeP9sG@)GJuJy9O_wW|<~yz2 z@NE?b)uEpI+C%#T7AXWIzr~Vt2a27%Z>4437=UqU-50%oNIct85^bIIqIt+;YRUn> z(fL0;8QbR_mDzH*cuk|F3}$o)$ZvI_tU>`joio%GjU>?&{ZxT!J1-oqmzK=gt)AW1 zChUZ0H|z5(stz5?t6wUb(H-rgL5X4lSM83JlrU60xKVQlFP?N{06N0pu#8)pASt{% zX(`3iY{RX8>YKEKP~yJE3Al3`vRa>(Rdfhwpd@oz;9P+eCAM;9y@y!T} zv)bYi)q-wMbakdLm*!AHw$^uD4w-b!sv||euF!=Gd)1KkF#1lNy4h6F(qnolPgG8i zyG{BYp(qHkuG(5Imb)&b4VNvZ#u0$ohN~8+Eq;Vwc+7n2Ol^5NgTTeB)Zj6Sve+x?sD+R0n2+GM9VLvP1ms)%W)%U5s&S!tI`uvw}n2<-!0tr|xCSrr;I7wNHzG zG8C(Lau2N&%umnmX(TSvii9X(AA-B6{2b`9oz#4Ky)Fw?NnJGV=?jlJul}7!_t2y^ zNi4?Ane!_3FW3#-R~OgLON#5OFIR=lqb0!s!qQpve`Jf#oI$Ny38<@W9ifMd<6~y= zImxt*9|Db+nc_X&Nc93Z18gbu+H~$)USMi*E{{6ANEMnWMsWWZDmmTx)BlAB4mZCJ z)*#u-c~Mp0S{&=iq2Cle#+zfCHcvM^`;sTu`RM<}sGA&%`v_P6FJ-s64U)SR*n$Sk z(@8&me)|)X-X{gKArl`%0XdUFRu-3S!~qI_bXfZWBnqhLs5Ic4?YbF0&0u_)KG>J} z{Siq~qLj+&Y}ddZ6bg|Up@qxiBdJKW{kEy?`zKYe$J^6)SD!bn_N>a^_V%x*@-eyc zRBd`oc6Z)yQRQ7Zy}fYbCv!5tT#4UD&S1@B-rUM1vm_sT249el26Y*Zu-er|jCvSZ-g6OD)`je?ox z}t3Ml37gQ+QqAr@Gb0-ZBdDa^R z%Cq$X>D%na_9#~U_3eizy(=?%F=W4gyKLBtcUZN+nF=>4{4)ND!qE98D+}XCO{*#J;KmZ7*!wgT_zzkvyhLgbn z>yax7NqvruWF$1|0e2_F+;Lgbdv=yxW?#zevXX-dGH{D&8iogLxzvxyadv8a{P z0>7@}Frq1W9_vKkVx)D^5=QB3;HO!_03j_sgd1quF3pYx1f@7re0Q}a zO%ybGuOp<1SzBw0wybP;ApqGTnpX;}7@m{4^dN{y1f~hqvaBTWKbfFKwz&yTyaUgX zOxaoR7%a>L)F;oePb@#_xk!hTs><(5H^)LiGUrJHw!8uYGUP~Xyz9h&W7J&`2y-tw z(PV*Hay?iys^^xrg6eDVTxkN%7jh3dpMa;Ui6DGwSI5!Hx#{d#pP_`lLKCM;uLml1 z;0j#}&J#9@3}BcFabMJJfImqY?_no5sjGa7q`(oX+6DoBhh%-$y9>UNtf3$s5;W5V zsuJU+rR@4GB!V}Bz%sReRb;_lfBaTA{poutf-WSX(=37?F)7E#SQ3JpCj=!Dn#`^w zoXD<1WVU#`$^WEa>Nug$YBL3ruv#lz>l3b$=2gtWNw5Ahk7q$bF+J|Xy6%Hs8p38) zw8Ejp`G^Owv2M2g_853Szf?^jBU0IzW~}3CX9rSE*>&~N`Yk?xmiAxa&3Kthf$*>KLx%;HS3)vFZAmB;n;%H=HVQvRv->Fm;2@`bUB8{=oN=#g{Ay8B# zcU+26-Ng^Bmfp#eSz2R%ZhJ-Q9OIKCz#fbz<(e1>etN=KYo|5d*3h~Ai z_A;7B(j4~O$Gkss&|H!ywZ%Qy^0%nSy>`2Cy(HJ`^jbppPL;xcC5JY9bZK5sA7heZ zhoDhBdiL|=|KicgtL+2EmQ?C%*PHsB#WY1coAu_h*+pTLh+?O@j2&W@d$%QB-jnmw zlFeY#U>|m`=lyM^_+{xz`2UvjqCw(*qjUokzP`Ra{XS#FMJPYR_Kz=5zx-Mr{ImT3 zuoVuDcKZ(w{jxa=SK6Hn6aT&a{Pg3K#aku|9}(7ty_D55PTp;k_FF;(P|fhNlRR8D ze;(EEv?l?`G6EV~c#b-`Gk#=-%m5&v>@~1$0j0{xl8R1 z8jr{A^2m0CTsy9>$=R1(OFf~uYE511wGValyS^8tb7e2z5bIF9b$PT+cpc+c#^zKp zL%4ZwwzB>nT>m*ofPZkS4uMX3eXIUE5Y~-zE$7{N&P*Mith_R%OS{Vc@wlv|Yf$w2 z48w%lB9ff(-jSPCb~n*(*%YJoe%szV(fTi~xZ{(P0ag=(U|+XjUja@q0W_C!4+9o| zrCP~uB)Jj2>nrjH#NoaGg#wf#TG-3k=PSa5yTga38HfoD1dqce2vQP%0H?|p zI2idZvO5z_3i_!-ImpX?1>iNCXHa?dezeNP=c^5P^Z1Oqtk#Ck&3ll_Kq>$EvuNk9 zzxKMa(Td>gtSoN_ZTAl(E0heI2PC1Y@5F`EnPEmG(l|VS){|M+4fQ&i1CFx##x;1jEsRY?H(|p~LhxDkEm}qz zeKsy+>@(&-t6|p0erG3@#nt>T-n&`iGZ@WXga+)S3H}_Bx=G{XgUk=~+cb#>P{Cbt!KD1EkDSH&J!*1HV0wR`Z}W9X42J3O1TO!N$+~U6++-B_r^}1e5Gc z>3Bk5M$o(R@2IM5eoRq+7ROutYq+*~D^IMAbQV`=e2PWJN}58s*FjhxtCD<&rJ#kI z*wAbmDGIoB?A1h$Sd*IkS`gD3z=H-1h^=e>5|QE>7)7+64FJ~LALVv>jz_#BtAo|U z5f8@5WAX_gPf~kRFZ>R}U0Rkv5!15 wT@`#DAa{=vwH>_De~NAsItd>FoUXQHxm z87t{(Qj{_xslSzWXM5(o>cx+N{e9QxSLvAg2UNmbz#mFOQ~5KtbjZflTuaxKw0+}y87NG z-h5*vw9Pm6Ntk-$oE4fq7fa%_yC=6>dlAO{s-(&jElW_^2;-;Yjk-?9w0C=P5l4GI z(8!U;FS92un^sI|3lHB4;IuNdOp8W_^>^!dvk)hOo^#D-SUj-(>#yqb(nJrnz zS_i}~=Ukh0cR_#BNpP9A&gJh~8xC&Js2kamG5WyKU?A$WI%Mv+_iiTTWMRi9+po`L z8$m^din3RZuc$`nrn^$4cjWAjbv*-^fHv4?2YPLCCQrPr6-q?0Y3C@8bR4z(dlOYT z{zykw3!T${HMX3NLUzqSliT9hj2uomfEHN^MW(VzI0@n=^Zlfji+X{i7>$4(`Bcfj<#TrQSrEs!t2nPFGzL zU3I0S@UA(<%HOVhUwcOz^jNmH87l1Im;EN4hGeRLobs#*QL_g1gGO`7=xWDDH(%Mp z0^25f($F)ZB?%iIP~cbfj3MFnv+Q*r>0v)iyg1?8NjOfJ!iJ<~?IkDk?y>6A8c zW!BX=chW}Rv4Ne__Q(dfcPnGk%GaJkWM5fU#RQw{m$BQnt*k~X-qc3?{qgnk?{cYmN`1z_-@VaDya}TaW6$fQ5H8zw?e(xm)1quJ1QnPGz?hyF=eWOQAAnAR{ zS}8&b8_Bv{o&8^u)gfDO^_cg12`MAlv=6YyW>4yQ7pXlmRo^338J5aaHC5qqY6jRe`rHC)NngzURwxaOx?$FvsW zcMQ_sg9p<{OgGAPy@KbZ&?a8Pv;0Ely+Dg;BRO{{IWnU`a`xj2n)SlRbWi1f&^ABr z>9ERyltCG`eMX2-w!qPD;$TW9-2rZlP^NW2=Qc9Q(~%d+56KzNfgs^LEl9`88E@iqy>3t-CgRwD`9BE<{V3?4A0R z6VgcwoAiAjxiK1r)p|};@qC5O-g?kXfX``IHKs>*tV@ zmI>^cZ(Qjvy*#g+Pfi4dNki<$68mkEC;#5O^BIyN+wwb)T>;7A+NMLPUoL6 zkG%SrK*SrtPHQWI^}R%DzpkaDsgWtm#it3^0m)b{o%b`apxb|O3O>y`zsPbso!9r)r(mNMGF7np zqx}ydG)L8bH;DdoN@7cY0N)z7(^S!$7M;*#op_**9EMEq`~m?gkaUxTK$`0v zAAR)NF7Jn>W{%*lHK{Q%F-Prz{{e+;IrIu;Ze(+Ga%Ev{3T1AW9!&uT0yi|5Ayfk> z128u=m$8ol7k_RXHxRw+EBXh-a4nL+V1Q|j6u`G6x*0w>h%fOWx#ai5DzZrS-eVa; zV>E0okF~rivPKTy208rtBo98v_orXpTpW}XQKrA){jVpL-pD}>K?!Z$;L|&&V0iyN ze7;Bt;TlZ(R4-C$Bc%;e8hw2>23x^izr251EW=YO1%JZNXVfp#F(L-;u8lCTfM8Tw zNT(;J<h#9d-rS0P?Y-}ZGvtwZ#@wU64b%B)Y#QN#dpMi^ zi^;?07u$?WdGnzEB z#&JQiFn>b9lt!4bj4t9x>yvh-TJbIn5J5y}hS32wMqT(@F_pKxj4i-&FI`0DZRs@@ zt;(rZ^?22RX5_h15WP-ZxISAai*@9=*2Xw>B}Ms)vR4jNRJPbxYr)ZEBWzQW1(X$; zXG2RcvRXpkiXNMsZQ!m`<0L1&jf=Vn!JKr?Rz9e*8# z)>Q|(#E!AwZB0~mI9mg(cp6EcEKiNxr|M1To#fVi^PDslmS?iIa4u8xM}km<}I{eSTF`_tzy>CK<%`7}royczzZzTXEEeM;s{`v33Y zkEh>uk03m+3qfd8kyio>69T(FE9F6#9B`Ub)|lebgUNPMgOZVq=Qcegln_xv$C)g? zOtsGky#Bh+6DE}Os$^ZziR|=3vM!y_J|x+cbf{Rv8p@(t?!BG>7(9pTdw(BvO%lA1 zp13+ieL!&rIto6UATa?+t$E8fB`A1T1+S;Z%RpT+UUv?sfxRM z?`;*Y!3Y1>6|a58kw04B64-(tv4fv$#>Y)dqS3dBC6RW* z(UuYwrk&Jt(XGHA+X>aSOO4*5H@5>Z00SS3?U+sAh!n+BP3j16Vpi+SgMi(GfOmYB z{2)}oIO5P}ZSaYC>|>yt?v|iS*5x}W`Ac(*y7N;eLvFvj34d>eaqh8AIKAz} z+(H*0Nrs2#F&5>s=)>&A^PiR9NA zw+UwqTuY%`=;dGUh<{)KiR=O(&M=v{Pjxfq7>_VX5Fk&cB^ZGCUc>SkA})|8seui4 z27Dv95NMM}bq39}y$jw%Bc>j56j*06NIBT6tn ziWc5K&eAos^tg9op$QW?qk$aD7@UZpew3BB$zvW|jwvPO{(o-1*@KK_JRg3H)50qK zv+_s?3eJPgCO@?(TYP30vc@Umx6s17oy4q&j371|4NLt4z3-NBB`Kp#oWdEvT>!;iXYTy;|uk1a7Q z6WF8tDm_BBi+J=4BDgU1i$;I*m=LTCm+sJc%p_fKI)Ao8BZY;ZleyqUg{BKF0z2;V zHIs86T-=FBsuhS>%F@=OQ9|3 zOu}x~`CJ^HpJi8X;S`ph2vS7stBa zOW@FdzkiiEe(!%}p@p|$KA##}C6bG6^=8bGL|@u31ZX8wirI8{m6w{tu3ZQ(t{S__ z0B_J55hoF4^5YE38Mz$Dg#a!uO^SecFCN^ijKRhJ{APBI%_WPn%263j2j3OOxjaOP zI#%g*E_u#pTNW`&QFv0q*urqOESs^-OZjEat|pWpx>A0V$yO;p_gE<@@BN-9IThVN zM=e~toGSUvM%Cbd35jozmk~_?69PCjmmyRGCj>JvH8PPw6o0*2$&Mp861~?~=m$E@ zeE|{zTCI`+-^Q++;gbjB%k*L9GQU4$kP!?nnNqh6!zfG2u17|L12ogFn@btITWF)xHn4NC<1l;?c=Ac z&7gX1x@b2O6>2k2NQ-{bM(l;$!YDpzGF*_67GcvF5(d-gg)o{&y6a$)c zafbRVQ-5`<6vk@)Xld~Kh{7bIs>dh?JqEc&qfw8>x8(|9Wijuv>12~MC|CH!eHL|s zg3I5tO~3&34CBxS$t0>xYqH)v!d2OUtCQ{8Eq z@LR)aqTmAaRG~Q2h59J_*oCPX)GzRaI&F~x>3{3YvT}pZvMAFI=!=62UD$LqsKP$< zslC}sl08{?p-4^PwwFnM(l{)hhO~&@0r5rDZCUoozd{|? zIe)BZxd7}FJqqS#PeJSR#T#K2h(({Pjl&xdhNNA%9?Q%lt0-QMR#4XB2{arfP?~6n z2$V3Ggl|&ik*Sb`L>7|$L~>vxlY`JcUNu)m(ST#ZHwdfcS*5|NBUdzXQ_+$VD-Glv zvD#nYe4Axn!#Ae#>fd`s7myI1#cb8Z?0>Q%&(823(?ymLdH@<$!&f-}(B%~d#}0w1++7FoIzA*6tc8n(-J1b9E5%;X0qJ@Fb6KCEa>w+I5Md<<5%3w*IsF zhxTd_p@&zvN{LCh6GjaVikr(Gngh%o%%Q4ph@Ce0O+?ZjC3#xPtP&iguYUlZrFd1$ z$F~XwpLU{#3i(vjG>1eDPa1GzMLm_k)u_n@4VD?&;{qJoJ6g8Mp4rZ@;c=&dN({*9 zoJ7pZrPSo3W^tTkU(nv5I9usE@5Ak#d2RbTQnOT_ z{Yg99pnL=4)hs!v2b+O4+z|R29Xx=!eE=0s^P<5SOc+1`c2~%eM1P~TwH0fufR&VH z!3zO`uPmAyYYw;)!Yufq3)i6LTvD`?<^SJ5sv>^=k*md|%H0J$>P72Lv3!D0rM{4p zBF_q}rY9E07actl-R4x+MF<5>#@dF3wwkNRLX7eqJ6KM!x}1AgYlZdkOQQ1N+q%H) zJF>?$Va2@E@~I6`>VNuGGrX^QlrIoY(aXqXj}L(Y#j_`!Klyt9C*vnSXfAyUPn~tX zd2?&TN^`q<__HGYN#y_|(*xEB$iI_{$;TAph%=>nv7=mZ0!81Gyp2ZkXlP)jENB?D& zN&*+=yi6kL+D_}#74ar+DT`BVv?I|ZW@59KD`&?Yrsw32= zyoKZRVWO=V)WkSOA=CQd6_T59S_YZ0w-zCf>#kd_-{5Jms-vnOJ5th(6+@wwygYml z#Y3%o7k@RZk#^I{u1~+#2mk96x(X2$MiYI|7>AK_1v@2iULVgiObD^9qUI+2x@w+* zn%-L`rpzqHz96RxowV!dvM@1~yg^+2=2E%j78S3KD%B^R7ntPhqAjYv9C=IqDy7=I z3FCV5*7sy(%#6I99Ny8jG+XY4tn9qd9h<|Lx<(w_CqvU zdS1MwGPQ|RVm+l6X{5 z#u*;*bdvSvlfJakn-gKx6Xnn;oI)qPkFomOK$6~1niXZeBN#OGM%$box9T$(RPxJ! zU58wNpYI=U|6yWGSS3ep6*rk;SQ`{^PQe$kiz!Pu`C?ZhTUKC%1v z<(IdAEH4t$x=AfMt0%=r>6&`fc_fQYLSZ9U_#C91a=jE0aJ4+TxPup08Zcnf==|c2 z$Ybn2FM698*jjg8TLeSZvg zsx@-9k3`7Hq(rS*A(r5 z1+9Q*L1-Qcx*gT*flA`ilA$`?zkhTlfKza3epl!SOZJzDa(hW8SSjTiIPlX%#4or% zFd*&T7jgWIk#cI5Jx(hr+V1T_4nHbxcwu>SQNDO{5qDtsJ?ZX<#EYcyBx{$}uc`3b zy?WTfq^o1ttoDO=&HDtc@vKE82Q}mP?x-cMj#{L7<-&|K{BV>VkD1}1w0~XEUi3$# zPcG{GW*rgO+a5bHwwm20Lx;tC5naz{QSSS_$ItC>KP=UHcHUAC;wAt?J1^>vzqHTM zj_v9GOk9*})H+&Tb~AoZvzyh7my!mBSpzrFo+(UWkA;w5xnuO zs)E6PP=SV3_d34V7a6aw$A40Pz+LfWjL*vwQ%!BVp#$7!{O;7aHq!aIjp72ti6uJX z)2W#v;+s7|2f92JG{!wZC`DY?0rCPlS zBR#nlIxY%fRH0`=ruq6sTb)*zuXV>*chcz_amo+rLt5qP{&POZ-#=Y!T%=f|NdGw3 zGP+gju()8-aU0v=HRQv{ulWG%h#HRgKHVH?DlJzV)ku>>Hm(kgPs-j7r-(pqO0XZ@;$Bh_E{e<@$TBtte}b(bocGgx2c$)*)S zN_V)xZMPP@fdr-;Te*r5b{tz{zl&oen=Q^8WZ1{Vq=+_aGs(o8YIIxGcGlX~w6C^T zO=*r$|cgH;LsNtI8%x7;bW2I?EmBc5fZYtl&`zD!IF|sDx}s zRS8};x7C*>%6tZ(=W#c5x2zs(m~pB4{G26L{PYKeuk~jNpQQRU;iEo%JQTi^tMtv~ zHxj;5E#8@PgWeutInHI)eNxQ#3R(XrEL?XIe?cLeHzdJ{d-)m^hS@WZKIrgd<)+i! zM9?;-OhdJ>`qn{>-OBzLaKOSb*@%qA3PZXk54gsfYf4ed#ZQwYIQX%-{($C>NkQ&Z zfgv*~zuwUEOd%=D5I&$nbi@8k-E7?Y3(lu(0o8t>nkF|inJn*|T^J$d(^Y_0mp|X7 ze}da`Lz24shjL;bu@KeohyXpwnJQ8~V^t(o1X6|7Mf4!>T@uF=udJ40@9D?4bPZF$ zlCJZ30XnGZ&9UO1AX7wPRMRtLx(#ZI(mdb1w$3eQinDHwo)njbf}xwKpykt_p`cVT z*r&4&stR?*kAd4rih&!jzOlTfoh)|de^n8mGMVWHOii;CIxSDNn185fxW8Do(ylUu zrxnWsy-zNd_jST5RAVri%wkgfwze@CFx$@u0BGms=rn19%xD&$3ju>>4(k9$;JL26 z<~MQ$es7MDO14s-RNTyTQmF!GBq?2Rm$V^(U-P}!d%7O?P1#+=_p090G| z7EE4j%?N7B;+xLl_nAh&rUC)1%jG|ojfT#G6^~Z4T@IlWD3Tq)d-;2WOdkeNhV@u0 z5pPgir64uuTxdgQ&+_y5e9wvLe>pkOQ40R16(D!9X*)JEGWV<%LvzN|ItBq2yF}3M zDGA0kBLm7^Sx};Y8hZnDiu|TSQ*x)HegW?Z;aSy)DhGSr=tBTk=(xqr_ndV6^$JQL z7zPF9ATK3psP;DuYD2l=`UD#sY~rbbHub=i9nB8TPf)WlJJ|Ye8-+mFe>KBl(HPRe z_p&ZA9l+q@=(L@C(cU8*1%xAIXFyjd&EULBgf}nfeBI9Oo}LlBhfma{1wfPw?v259 zjnOm}%8-JW(W_2Dj9M(kzo+P?Eux#YGjxNeg1!e@p8m28ZA$iKYchwK!06DW^=eNo zW}?unE3RQos8w=NW;HZ8f2%okVWhgRl!d%F%mU$bm&PGxkEQ8LG%jqH>OlA`uLPxF zqnCA{2poXQQ9Ld#a)gd5ri_d^A*$Dx$G^WPeFAuYoIRXi^RjdxEt8OB(uC_$<~p^VI-my$=^Jkf2mZ#Yd4;-^!uPufuEN6 z8OGFNG+GF@_g;>>Lexl1XFacD(>Gw6< z>2*|6kaiqA8fNzGXoODZ7Mn}oE*u6na_`voEy2(aj$e>FaBx6eMWG37^7_+F!GzA7 z2SOK)Bw5Y}C7d>4wUREMkVPD4AZC>1S>>5wlVCv3f3nPmsMz6YV;6m{$PE+J2uQc< zNL(u>i{r;RhFipx_&u0HwcHd_fHQT^mvUCx~~bIuE;y0V&J&Q8Na*(oox&^LiGH5Q!VNOi~-6!Qu( z1z$;uf3Thp>6|$-<#?9?v1P4cXs2M+)Fzt3sU>4$p>0%4=;k|@#ETL)bpHd7RB|;k za*sr@q8MS?iapQ6N)d;ZsQ%jlzS^aAMXn4EyRLLwPiwJK;B`AbW4m|F7pxiMl_pTX zr_yZrYSJ8uJV*vsN(!XlREskyq>-UF7t7OYe|A5;Hot>lS~6C_Y|)^mQgQZI2<#>2 zU~X-AX*QeE!%MiQ4X1@)LmQ#f3^(?-4fT3z)|@IRD)<1_IHD$4GY)MZ$LL!6L-ts; zULus>m1ksqk#m>0+&xss?a~o+W0HW)KSi)18N0vH ze~HaFUbLvVsAyK*dU19ZR`^KbaewQ$LT(M(de_j#8P9x#2YnbUXKQ8Zwfm!;H`64{ zJaszefrp6ewLRTAu^~<7`+;)^JgJ5oEnj)y8hUcst%6v+r#|=`nm9Yn?C((&HAu#G zTp2~WQQGcjOdo2;)m|$Hwi_3|%KFt?fBm!z-+|u$+;p_GtI)}LwOb`)`|5P`EL;pT z&(hj8njPJ7Lb_GcGL>XbnxlghWhWR@f`uNu(QphDVJb<@ft!&}+aMREf07D> z=@9&(uI4_v@NbVWdE%qjhQyT!YqFE8;~vTPoR1I9h|6eK*?QY~L{i_0OEHz8|DEx1 z8A9?hdwqQT0bw9npa08*nvIXY-*;nl_lyR`*yG!(&lR4o6NYPPqEjCVSlh%DF7?4!`J)-<=nc-v@0G?lGW#huPR985IX3=w zb$jIhP?+11JJAFqiQ{0~LSOT1$`QG!VYeugD+RtRLl<)M{z6J%hvzm>c2*f(v_q z3xA(-*=1Lpq$jf)V9_Y*ZpSXy%k}s?YWi(b(>D)l_*`BdK0Ze^h1pyDn_hl>(D*`4 zdWv=i=cb_iY#~f9pQj%$N_nl6!^hw7QA(LbDQoca)Ags9Zy(CiKYxF#G`|0)k<;4D z0pdtcFNKazN`Wc#AZD7^57t?n0iWuQsqSR)jk@H)_62A8y8phMAD?YP7Utxf4m(o% zU}t3`b7heOhmUvH4^HJh>=W|(smlCZIPv*O$b|8i!j}n`@(ONQlxz8o(_!h$k=42+ zi^QY0Dl(6Gvl8axvqOL8)tnUmW#;i~?EK^O$%C7{*Ud_?Y)e{H?oxD$3NsPE2Xi&) zB4xn$s(jD-CvB@@k1U8?n2OKLlG-!-L0w&@*90@=yRmFT5~Z$ff*M{AOW`N`Ql8H~ zz`U@<9pn`t6c&JKxTUPmqi`z;bV@wp4H({;lg@lU$DP8XFnG6S_ReJvxK_!}1p1!c}IKbPQgUW^x) z0|;ipbmQNvvw*i6jlu`5`({R)xe_N4Z2Y3MTqzZ92x_W1}$msu#B zPC}FGE{=0hQ6hN>$$YtLQ84ci&JQia#{~Ge+8S2ODrQizZ=cceFR}6}IcAF5CW`+0 zWmTLKHu*}bU`rKlRx&r9-+Ga!s8Ro+`LK9lO=3T=%>wcm!McitTz-j!|GY6sZWCK# z#TM+gCed7_xQiE~U>0mwGBysyonJpK3NeDe>BK1cflMkclMCVCvEZ+P#tqMLW3#kR zbolyryUGIV_(M{8qB$Jv2Z#yN$r=O`M~z8_R}JX+-kMPBzVgQEmI1ruzd2WkXWm+4 zL&OW`2psUyvTJdx7w6*4jBnV?=f5uzJyG-f}O>#OSS^>5;aqbr{&arVz#-?^0XREkvnSXDm9qMj+7vpY*NM8Y=xyPChIgc_)f zeS%FVw4iCJk;b#~dc>bT<}fFOt%kORvsDpwxs-Q#BF$x&Uv9%;&@{5T@5E<>ufFa8 zs|tLt#h)rf`3~@MCmCuO2cuR#5HY2(FfQ6Okef~(7Fkre+WdaDI(#x}d@i5+Tl^=l zzm=Oz;nT~*{n;=LhBaS#Vz)KVH8HKLmE-3Zr$ygH*RBtWo-5{` z&V6Nq?Z_R9B4q{Vc9hXZsPym4;opj22T)YD#pn=zlke--yh8vX zN2|=P1zEV17p>0E=}D?v+7vU@Nuuz0Wu2OoHJ0W@okd7%iufvsoZu(USZJO@XWP;$ z*70Tita9+_wS@u3#OLf__;wz0Y3PPv904|LtIMU60zCxg!7+DBqZ$QxVm~$g1|rW7d*-r7dGd`cS#(W18kQ z9cJB~HiCVzEjg4v=!D-bep}QxoDNc1bSpD^kzlV;>nu-EKU(td0 z7HMqp8~A>+LDJcId z=h!=}s^@nv2T(&|oi^k=nJl@N2cl7%QjqeGD?pnE*CQ*88c6;eU}GtQN^7nBkzfT+ zmw%SVQkQ(u5n5ieeax$`oG}0{|JqYHHkZ`)!<FH*EFdw}eSd*k5QPMY)TxX0BM$pw-5B>Ls@yiV6+kK^bz2YV?>tK%AZt#DbE znfynQ%XRu8@ehtA>Q6xgC%XiqMwZC&+wban_mg1>a-ds%RRPfO3(`zL>1W~|l`EUV z0oErGD!5H2Xx`HSyfi-KKmw)R@ek~8ZxY*#d}!eLZ&m&|X!NlUW0t-+&mtQ!(wy|% zOLD0h;F)eB^q~4};&Gg$0LMwiF6427lxG5NU-+K|(UclANECx-@FjZdGp^3QhZ%cJ z8)w;6!@#&@H}cVjs1Z&;$kqgGM>ggOTXT0X!zSf{LeXzYdi>GE=&Q2HGD_H}OY@+a70*?@?Ot;Gelkt97xih`)3FjnQ6~VeW=eGk6oyv;Y z5la0>ePHL6?b+s%2tr>s92ogtx8lhxe#yX!fB@Uu`HmW!y5SYTW|UiN&J_lI_@!r` zAoIhM{fjg;SI20~5W)BOM@FlM=6$=$l(%3;_KDe%wW%g*8F6mJ$Q%8hyD;NYu4%>A zk8_1D&sxjxw>*bu(a2Q`*F74;8{x9dW|P&9x=Zl`{CgiiBwxPvOVznlz7-2{;d5RS zH4@Ke@LYVG6nSdkX?)!vOA1AXJYVn7H@&pc=~H*=jZf3I{S;H2kcWE5>AzGJ z{4b*nBg6l41YjHt>?w0e|Fkk)DpNKa^swDWl=qfUmM8r4 zI?f+NPe8N)MoEEppn~@ge^VvT)%{%1>qHBmk z6L4p#UkvV1;Im1z)xrgvgaOyVAxAwRm%G=A>dT{`>qagKavk8JiaK zGOuQ`aGxgW^apBBeuQDlW^e+ADP=d#$HR-g zRtvOO5Dj@JwQ!N^GL2k1uASN6#6(0_-K|$D-29=i4!>cc6QW;NGKtU61P_^=BUGgH zjrB>GR@u#o-=;j0WzK`l2D7RLQz7@67Y3Ys7an5sLRr1*v3cult zs#L)j2{Jw3u!1ya^t+_&75!Bpt0j*dd%j<5WFJ{qY;DUYKv8;p89(yYul== zMubh!(bKTViQzfMsbH8QA{kASU0EnCG|*ibm~Q?`U4S?>sps02QuWJ=t&T(OQ7A}4 z^@*Wqf;18bHh2KZ*Zo4C*y0+!WLAnOd5$Kh8meF&;VNz z+7HbJF-c*pS%k)G4~^M_(#7zZxc2byJap!X@JG%@vS*n*-X!+7&9YfSw05?|De*5j z2{&;pv(3Lt2?`1}+PV3gxAYIMUoqhld!33H{Q-F!_erUDnHJ+$v?iMxso?&{+F2)xjdyeFmGiau8;u{Y|hHeV8)>DW(K;g{+g#*%^M7 zC1KD<^C|$^MBQJXm&3A><%Jvi`SfEh^4$05sESv61t`UZEsMSLw_B!|mnd z@=m;hw&Hf@I(fWiMrOnIy*)oT2 zC%?vX%}%`dO1*3a2=e$toZekN+o-x(H^2U~^m1k=)w0CxOv`w;+(C7RkncL`uR|g3 zX&R4sZl#mcZSEdTln!5$rKh%37i8zbdn-^0w|dk8?7eXt&e#>B`lF) ze=gvg3x=c=bSj=EZo8)L(Bg;D(BQpkpd>C5&@;=ltAVUv-%O2=`5k=JMQ&#u%Kr*t zyAn#yXE^D($T=$+ri57OeS}xk;OQ9Fs&&TvYh+Hq6!OrX9_cwxw%$tnjpgt zDudrqa@OP&T*m^PyPJXH#3!lJ3j<#Dq|BhzT*l62PULz1VBhH5gd%_vh965Cky8z! zniRjPBfdAZ4LZXgce976qO=@?wh>|1*Q_mfyA)TSD@1PxtG?3NRYN%TvY9PUiJ?u@ zryDQWxlzqBYm{0$3Rm|!X2j-Kmg`x9=@2&~1?!=B9}zO;{u-+^&}q}L4g(UlmzHcq z8?aSNDjv{W&C1$7SW%gIM{x(ZRV5xrpBguuqD$!(P!Y)FVJ3YIn%r##QuoI#OQv}u z7kz;t@=Qv0SS5zQZ7TMi+n&?x&1Kk6Fel=WJCo__Ek-;Em#ibuDWs;?MsXQSV3My1!)U%t&5|Lexxa zgyej^9-KYsBwHPuvkJMbO*%m%3$+Z-BC7;TrGYNRvNlItK4TqQk<=wY{6q9_?erQA!j5ELBe=4Ga{!-nnt@-`eDR{j z#Q%cr{!(Z=N-09sHD`pJHgA3V%RJFT7OQGNSHk<4G0eoGLsc9nD(^Sp7Ir~pmiwe9 z{*_i)g7GOuhGFc;`QtrvCzUsOx@5;}DNAy?)pqiCyyJt}5S6K-7?(S`gz_eu`wK4m zCD!(8&f*`{dSAxDj{wTl(&c2*dML9XtKV{UFR5aXVY2>LJ<*#P27;8OKa3ccZiTY*aH6!OngTw3S@2 zyNCJ$SNE~B`gKgtXg&;wJXC`b4erqMTj(;^1V8c3XAWrx>45+oRCaEBw_HvoIgQpp zDF`|G6XH$ZW!eSgsv)}Y#Y{V0(sibgN7973sK*rKA{0V4tC;*a;*S}goR!~&LAOH{ z+~4YWg5A3KmD51D-E3X`7a_XBKTv@)T7b>c9Jz539TqPSQ_}Qcr3yUh!YE%m@PWt$KN_2 ziIqX3U>-wr>dI=@1~dD?y}YYX5^uF759?OEWNQnok?Xhea<|C+8%tb)BJ{hA#C-Eq zqLpF#BG8zc&e`GwTo?BOv3|t?y8%dNV1cvecFK2S&j7zd>}p{yWFKnyw;ZwI z87@D>SYZaJZn1f~qx~Rtq9Dibp@R0o-kys&XsKhNM^WFXn~^;AE(qtHU|T0cukz{B zO!TN^LS1+K@{Npcc(Tu4_QeG$3aC1Y!74E-9 zOcEt8WaK6f8G8|s){2dQs5Ni1-w7mHbBu`dP+7^$$rbH!i%e7geLfQSC#$gkJIPFn zE!cz=qJwr5Zt9L3=`%;@$FSVyUKxOAA@p3}y${rAW$TM1E!5cWAA)oD&pRjPsMjMh zyek!RApZ*;V_rPe&m-dT^*-C1CSJ;)ND~|gXd5)F#k(>tj4@i)*6bE5xUKvNrnAbX zfOsoCFY!X&lpn%+U&N3`1bKGhnact03qC$f-Q^TtZ_hWq=GR+Z8JI_fS=@!XG*LrU zD*@{EH9JW`-O*Q;)(~kC%-}%~8s7=VaC0y#pcbYa?oNErYhGy&79%@V^+F!Y$Z)%I zpH5yXn%;{AWfa*07Swj2n0j+H;o$o>9ly4&wi)GK(UmoCqwl^2u*bhl(hYI1P2-Z< zmn&n3F(p3SsalHVlc87~?tc!1$5x`{RReZiU5B75^Do1u0a~ZpR*944sQvOWtCvZQj9J!@5gtM zOR3w0I?bv-x84!z`Gjut`?=%)2Fw5b*Af3cmcRP0wJc>mg0cBNB}0Dk?5Z3=Wg5`h zjZ)V|H>m@Y3%Qmn>uO0nri$$3BJ#9KEo$ZUx3P_!%w}KWe@j||;SIp&46hNH1>K2G z6cv};+0EuVUk`yMt3?rCrGa{bU?z%oAdmK#j-qaz2vHt>y5U|9MJ6E0z$&ySXaMUv zX+(3r@*TTIyo7$Oh_|?g-c<>MxdVO~8~ya zuyC^=*QjmR96WKSpHZU{WBh^2jVWI@5#bp~*ySjcr@tD7WRH&H6~XI!fBc!0cQW7< zmS3jRMq8X0ORPn}bn`__JfSJ%f>C!A0Zd0-uXvrrY?!oKF=x+o^qcGQ2QSBrd4(Mc|;lBju zis*)&JY&|ICQ(nk+NdJ6es=F)wYssW694P;w^cjuds zv@#`4hvqr@v20SqUG%52DZNnH9`euFHLN-LgSH6On`38?j#xAUpFTb}LDw?{Mr@Ar+UYq18pX3D2paM@r{D`&$ZU9x z)^KCbQ6ahRT$H{)>HT8+WGhwJmMu9?0>k0+<7whgHRoe zZeCrV?ud&lH^nu|v(EwK$CP3cg^#yZ#Q1{V7)a*mevRXo5Ri6+s@lhRs6S8cJvcS! zG{0sQ{Z$eNrklwP3v>gW@1`!lv@Jr<2Tq=@e)W8+==}4Yyd4xaqh6fUJf1uyz!dp1 zJMZ<#e!0DDyaED(BFYpHTH8r9sl+n_Itcq^%SamO1l7`8s}|Q4}dAD?BDw^e7>O}eQjXd)AUYIz@hAuG{1it!{oJaZrykIeEIw^DP}BZR4@}> zuzzUz1n2~OIzfV-9wA7+<#O_whwP=mpn%EBfE>|&ApF`RMoEMjsrpl)?blO`5L_X* zYzRN0ta5{SkYaCT)_O?mo#NRW1G@)x>cC@y-A$&j#KaiYOMH7iIaVN}on9RsSwF2^ zS;-_E{*N4`)F~~v{?$ybQE~VFf;odBnrv*-3>;K`IKJ|!jpmhsI0)oMdP<%63Pz6pc{z&%n+CKuR=mtkn%DRkA7HAhf*b>;}Nf*Y=DeoCmg4 z^2@1gn<)gdW{(wqQkmg_iYqgLQR*0?0}QQuL3(Nn3;Uv)*NaU*)W+ZM5?tpG#QzL_ zpA!}u_|prw|MwagZDHwH(?-fvvRzhTg39Fz3WzROW|Ksb}6j{zkBL%I~|c96k}q^{sU zWQ#TbB7HLiHh}IM^=EVaFuaBe8b-jwX{T-gG7po|79q=Ceqr!q1 ziIj(C!Q~Hu>o64ptF#M89SJHO;-D8GCYg7x_HOvjS5O0QRA^WHWu+~~)~lr+K7ipn zFbqrTY&>`s{h=}Wwv_5pZ=U$f7bB=wFCNW1L-Jc>(HL`Mu$)4#fpldVnKzxz!9i)~ zj7Fg$@si&`f1^TaV$sdL8*q-&hBL|;SGoEDio!*rk#bSB15&ejSf=lR?}aS#r^){3F1s9h3r^^Qhi6fhaxOzwK+JmUDl zO((5mw$#F7XF67e_hvoOK-V#iE3sQIf+2NFfatk(A-#`kTy@4+@UBx$z*@7Q?%3ak z$u5aQrX8L7HxX*0lsfuw`yvdeWYAD6GPbS#W|LLB${nz}5!`twTEW>Q{me+c(eU=l zGrzBz2NJ0Ca1!W!4bo^|8cHcvtn_!&s*U-)C=-&uU{2= zimPw!G{ZY!o_grh^TMe|4kL{?{ZqmbQHV6`Y%OsocCCY@%}P#{lHE2IJ=-xD>8K)i4|Y=UK3hf_`lCXIanOe9#wQ6mXuk>AJ22@???3 z1~)07^bwYirzgseXV+idAYsmGp!8vtT%Qw2VTT~nuW{Vb%Lc?k7O_0Mb3z>LO^44w zsOw0#&+Z5d(o3`HZAwdNj_S|ktqrub<5*;?fWhY^Uta&iU~?|b|D$arOxvRF__+CV z(6TcTz&CMpBz-}-egUvQj2!xJNBISPZTon?9Q}q|rb0!S)`W6@Dkk;@D->q>7B>ZL z@(=t26AW1~0_ctoo0l`&Gb6!WII9>ngn#n*lL7A3rlQ6Lng_p#!3JpHkj`zCxiR~> zzFJ`WHawa-&NaX>xWHo=-De+h%&`uBL@wvU=&Mko{8pkjF=vR>e0!xG9vmRRMF>H~ z!&Ss((mJEFu`?c4W|X4KXQ(~{NeMwg2?~{d2%W^-0+dNI4Ho|@Rl6B{{*c1w^)nAw zc8I07#@1^W+osiz-|v)sFvEAffj}4-l4@T#iG^$46_KFb%>SqBqiBZ)%H^&DGV*NzNKg{nD1 zk(p1>9;^~i8SQ0}&RT704c)q6RmM;IWaD5MDP&4GMQgR!ot{(t9;onYN;v7alW`$G z6~Gr;zenVYr~KBFhhZfl{6a2`zvc9&+NNscbua!dd~AClX@j-!_*My>{wX*lDGh)l zMH=r%e5h$W-*#5OU_pAHC`>jKo(OITX}(uEyf~zm78S?{!TVDb4hYa@z@lnn%Ct26 zP2B?4O9s~?79-jKFSs8TF{34%l$$H z{>kaGh_q~QBXI=RsQdyPb%~@TJg|-^m2G!iyGsKV=mrJOiy=~16`Mhxg(=^`qe&h` z@o*Ul`->9kDAT7zDdlA&by)4{JS*xMhaua|r1ODH$m~pXW?K0v^F(8H0ltij#iHe6 z_DNGXOEG6G%Zt~n?SwE&xag&5LXnKnq;8TVjR|_HOm&=abt?Rpy~y?7i(M9;Q%nk& zswN(7gGztvx!)}IHCHl35iD(+JZCkL)Kpf^Wpkoki>9i@LtM^kZtNQUB50-1RuQtz z!ms&yNzk1&<8lZYtlwp&-WJV+_snMlH8>nP^Vs zj6#!PS|1kPOI6i|xa6yywtxuF4`iw0v*I9OlW1v|OFu=t%^#+_k*GLpSOzR%*vahz z?E9kw+N9~5%&;lKJ(WuXL^vsT!TwL$9WAtx+fPDE6tcAa7%Q`gGMeDuTLDZLOVeOZ&L!5` z?53g3y3Kk6!&K7?Ih4CiYS|oG73KsMLeIO3)xtGL^NNDtnJdpIRNFA->F!}FYvW*P zSa>nfy=5E5pj}UA4S)}FKWYZle=x9UT9GIhhwPT?(I6gkTEO;-Y{7BxdynM;va@W< zOB>Fjx*^pRLWW#h^UKGrvP!wOSUP0BVnp<}m4G;YNX7Pcx7Cv!Xs(Fmm? zd=V(!QVXCJQ1q^v#Nqkp^eE_RQBb=X$Mjg;vcfVX=4t#ZQ^MxdHHLL@ysM`kyT;Z* z44yyqbqM1Z!-x8|a(kYPlTvq!cfQqH;&H#YLHiwi#sZ@h42mFw9gct9lNOhLpMjiP zUT}sweKW;fNb?#uPk1y*j+W{|!Di~dELnN$1l!W|B&Upk*Y!S19`aB;2^P}b&{RBx zTHWiLm_sz^bY6Jze#aZw_hDPX$@*hs2u!(M;0@4sa%3({)+OWCjEwN-Qhjj4o+nJ| zR-_~!Apoc;7P`oAtK6zB+s(CiO^)gx$$71n?(JH&@#3Z|wyb^Zo-|cS%bPE}D>dI4 zU@h3EEMtCgDyX?PVKMI2?exLZ@K`@;K(|C)Mh@t+IrfwCLmrhWC9R5Z<=u5eBxqVH@lmQ1}$mtpQg3X zBttS3n>W!$o%?;?TM2RHS_ZoNxq2+3f{zot1>OqVVqsEheP#zc zk&JkG2CP7;zw{uxp@ zzWLdiw()ZZ_7ix2e~eM3V6$vCcNmP-_Q*0vV!MN=K0rr^eL8A}$!w28uOu@oKj0}jG zvTVNK-D0**W!uyp!KGykqDal~S%OJT&wBNIIn#w*#vUKv*UOZQDYE1$kju`Qfc8~A z|Mc0N+SJsYhUWSG`E?sUx!6w3ihOb1(sezb!D>h@^qzT7K%s`ATH(hmA9Hed01l}C zhG4wZhT?v*L+g-L3uv~>(NiJkFCj1NS(DyB&TV-3yLoTj@*i#|=1vZ}v68H}7slq9#2-c$LOqC0bky2Cc5z}0 zX9AheZ?iL;7}i<~Fe^p#Xsrl5LvYOoV?xd4xq8-RPz<)+-?Fr}RM8(=fRjB}N;jhT?* z_j$fvM@N$5!n}(xC(5AXY))*9bTA`^1CWFgVp^QaW)(J{n|=LAsZobFUNgQ2{e7f8 ze=18OoA=+LDma?XKvE#5F?wVXk>8dUw3k${$?_>9&Uuo4WPj%smmR4Toi+4+82#E* zYnmhv{|^*3^AuA%JCTCvI4^EGTM|T1X}f1f|9EX4k;-Kd zD;b&MZfFEg%1L_d!oM%~3?Ph2Z?TXcOTL_%C7xhL#lDrp?iq_Z(7XL!96~?luzFbi zR__cZ?)0IK3FsR(L%Hky`$-&4t2c|*;#4?13{*9*>;Tiw)eV=47lY#&3lXv7Z_Is> z+7Po2r})%dI3|=QG^2`W-p3ecV2rN(g!lY;z9wFlPzgfE0)0dCJCE8;OB4<6E+qAp zKd2XQ9oD7B=9zoI3_=@RKlf1$w26;m-_diuY+_450FuurNQxu6;|_#Fu)mw0V@uAX z6nPPUY;wMZ45JQE2FavRf{d=~x2Qxq+@!>p%Q)6shgspwl$xf2kb|KW%VrSn5mP)6 zdeX=aUG)eudl7-`iP)@LuyEVLHj`hrFc4Z}UY9NE+$_-A+{X{f!IS3maA><_=npp% z;0eIH0%Ou{k&Pfv(#KjsTmxv`!Jc`$7@5Ab-|<|d6qzQMZR}i8Glv+M4}4n%Y9%;? zdTVnOo+gH*l3_rxu?3NjVNVmB`h#JUN$8VX;~0^%#z&*P+Sv1Q+zb1qt)0*+byw%n zv7(pa-Sh~i){#VbZyj@sSgPOiHlCgpksqrkyoJj@V|wuP9wF@? zK#}DK7hU_qFdGG#=}RE>Cl0}W2HuXmWbs8KN7%_ZA#M{op?)+~Dn&ZlEohkh6|N7@nmnRgWEy{o^g`fS!(IkHnsAbN zvXi4UTd}K?#nl#W=<5nd%HttiEP_XI_LnlfU&&^k>H^1U6V>?~GfZz5zbkTiOcMKv zv&v>qdL5fO;TEG|HyCwE(UVnTraurFb5zDP0Y-u0 zgP)Zjpa+?QaEJpYzu&p3+2LkStiJ5qu?F)y$Jgf-*>YzW1!vp$yLibjz^BBAsTf(N z$Y~ZB>j4B}`eu`*+!rH6zmElmL)Q+?Zy1+~(t3%6>&S@__l}?&h(Zknl|KMk%4ib?>kJBQ?!VtB{C?o>;muZorz#N;gUkP zB%y|&LnCS#6l$)EVbtK6h;!j_t_|25uVGHy(d)}EX+g}>=8m_hh2He?=?Edj_Rx4N}u;(?nL}~EmN6FO=xN?fycBs#5dbp9|-x0LOsQ+$!O4d#|f6ZcULH+mQPIs{^7>lem@S*QI9PM zCK(RBAo-cfVWz0Q{s;v&f!pxD(dMA<9^xryu`EKMzN!3jyZ4vZ=2}8JeR|z#$saX7 z=&g`y%OUopt5+a6r0f`}s_AYhj}(lpF;AEadu>)dZ1o*}j?wuN4^rduXEyLRC_C)a zy`BF?^REfyk!QCH7&o{mV70yQVui(4sv>J|b3!gcQZpn|#7AL1s#R#Fy)urZN5?L6nyS@LtiRvyzHzRJiL`ma+RRr>*BU)I-tVx;o zsK4av{TNWbkR_HwQvF5~aELZrbmn+!ko^SnOe}d^qW_WxcxB!54ZUUU6iDb`@myy# ze0Gi-4|pYd1fqYvQ;UYO=xG;SIn67#S?v~)A7!H-Hh)0ee6V3pXd0!O%1kio+FvhS zklwJ&96(@HV&nee`K#{~KHyb2V-rQdlQxc4-ds?{_%n`LGm`%8jf3Et7w<3i3_n6* zdic7IvvOQ0FqtPuPGajcOP9;ni~nJbZ0- z@lJO6t*SFQDx-;BVT=XM8ew0gO0|CpoDtvu?D=F#I5VZ!S1vF+mrZi~2zM?uMu=x! z9W>ryV9Ul0tcm3U3&VoCDkz*hDu_BZe`Cf=~5LN>NCrn{ow* zaB;JQkEbB*m04Q$ej-BIFuUlmQBV_;QzcBaXs4)%8$RKYPEmPceoNiq61@~%$;zsb zWd|L`w~UrS9X_b{ETM=<;;Em)_{`n@L*%rvZztO?E)1x1AH7}4c02H(gE#D$+v#kT z?Mm_j1n>ysnmfK;K>-xbrw{x671j@$Hly|P4s3(fVg|Smoq@_6cP!3Kw|b__RD!E$qz8cEHG_nvPMr zN%EIVKgGUwwdeagj$u^Hy(Mp@lJd90r;xLNn8mqo*0fzj@md30V5=|X{bDiN{JGnf z$_j(-+b>FOr@VhFo7q#3DyN(uXII{^)N!?nGb*pkn9epF4kk+*-Q);IGh)sPOW0JW zqkpO6+p3HnkI_HZ|4gyDKt3K*(LVgh=v}&9*G#MFa;HY_;*m8@RyF>Dub#di9Z0qY zUOER?X}p|wtv1ttfdZXPnfbMkc>Z|4`gex=tefy; zmO~YglofBicM_rLn1s+KJ|j9!bdU*#28?@Z z$A+qP=}UBi&2fBjNM~>B7%7oM@(dVuI z@2plWnvb9CCkSS*YT>&^da|Xbf?~wl7GadI6Qkob=Af)g(t2!}Vl9s9E`~qPI`K0? z-N%P%;O5!u?PTw9=ymA@YT}4*+p1e1y@RCX6)}10yyC{?<-PmegK4GtrgS98-S8 zDbS{vyveOY*;5`4{A4#?@z#v-w*7c}EQpUs^<-0t z=i2FvHQs#6i-f$ikt>_NuG=J=)v8wU!^G|%LU1#Ptc+O@myL0<9N3rIS2+gGTnzd6 z8XgE%XC;c4z!n>g1K(rCYO0eS*xB0t9jXXz1xlFAmUMXr_*C`Ie>+W<>Gp%@WYaT9 z{EbjEH$4G`)R77087M)AGn_%H@>O!EFP5aYqnw0 zdXt4N&`zEU97~`18F;kTU7mR_IPk+!HkL-duB=h-yix2WbskMuM{1ar!h?m&Frc=J zE-#5^-N{LNZVMYp71sEa*i^4qe!K)~z!}-U8qW@i6IxBII?u)_S#gLs{!vMxpe#su z83ch25K|QGgznvO2iKt{wghM9mb~$FxD_hqX4f1AoHQO7!C9ro{eAA$#iF~;cIg<< zgWTt^r_|!~(8$wAyjRvHTLG<-L$+~uw~6 z&M^g0z$AAmiwlbAcYT#gobQix`JtALE9sz;8qK%tmrFJcaQnV z*B{i&X;DS0>=!{;-g#?2D_<&%M@cy>C9lW>Dmta4&Rqr_S@1$Ws$dYEc@dxD0 z#GELT0oDQ{$+`f$nHW|RZ-E|eiGN@28@NZgY9$dN5&K+G`a!c_R6$fKv?61-QEMw; zN@dYmhr}O(Zli`FoW-bw(RA<_2C+G7vOGWK z)@|0*3KGE4dHJ}=R01Dr(Sq7nhL^@X7_87gT7L-HK3`Ah(3My@@6<(l=A%wle`Pon z{J6J_pvdS2p)Pp6ATmWbh6~0sI{!EwC4+#Cg&C+BrlUzkwD^OFtE3Rv?mZ>Su<8Fv z{~!+MmH-jfwHBgs@R+Z>cJwS*F|XQ@avbTs#`=hBi8(_* z0$+0@Uk|3~;8^9j7HgnoDb#vKevXB=?rv<`Q!C)10`cxKcn-^K{`?r1X(N`+4 zueSG#i(eNJ@i3B4=7blPup6ZYG~L$vc%*@0mr!P(imU)V3g$RNie*7>U&xpEF71k2 z?^REgS@G4a_B)fNC)1Uf95Uy>e1zMo^#yR|KPhi&5s8RQ*L6a1dM@4d%NN6{noVDM z>W|I4gEF88Hj|b!2{pa$yfY6<*s(PnbU(tRTrYH+^xnyv$&y#bYeij(cQT{-3==(_ zUYEnP6#%$e`da@D)7{Upk)q!m#c|F`tlrA}3D=>ocM@)<2fG@A#|Giavp+0zkieBB zdpYv2&?yDlO3U%)rn46Bh8-q~b;J0QYi8hfQ&C6x+beB!@_Bn7-?UHP`3OEggTkh_ zj9x4l(}~4nb7aQ1qv8(?dp`H2n|57fo6P=~i2z`m`4&m}Y-Tk*Z|?BE@2dHuF(se3 z4#Z@uYFs%qCFWEvH@8mL%Tc&E*$W%~h3aO$#)?k0*+}{3SJvk7FKMAzhb;7Da$cj3 zB+!=kXTcjKtM7V!fK7RM3ya-re`sM$Ehw;C@Y|kCHB0cm76w#sAtSlQRqZzcQ%9GN zrw2}-)+bCUZ4=smnH5tQM!=}R7#Y|SZW7Ufe}XhN*seF~Q^-oOMR;b|){-rd!{osh z0U2nW=*=$B6>}h)K;gTGU+tckGsEe{_0 zxsyz6V-mEL44GbzVlq4Y)~RZ~8YxLKUG5L}d;{-&IToBraJt(T z3qqNs=%xwZ;dk4Z(^y0Xk%1}8gT43tOY|idm;IV2ayl#I!%3AGOHcV-h8RoCcs<;K z9mYzb5I9%3;J=fJ^O&xnEhxb5mZBPyXqtD*f%Spg zvp|S)NUv~zMV$evtol#RJFeD7v@j)ZUTA*|p`cULFe8Tg3JxXRyi1Rc-1aEhxFGu) zP^`(Ga6J$-L2UvXr$}#~Q{KNa9ZYlWXJctc-r@tDmaMRoeT2UkXNeAlp{|mvT0g<# zF(*pPw)RKRBv6nwLadRZ%fT)f8B0Vs%7O6+@knk7m#l|~#0lNOlU6Vyf7|UQ+vlCs z5E@z-)P;C?vv_suB}kR>5)ETpczt6hxn`wjy?oH0RQrPPpDGi>D}fSbavTTQ942M( z7V1^ii8@GaII`052VF1^20v=r8Fz}wCJ=_mD+r91{AIVx+cx^8T|#$u;)cZS?0}#3 z|8R3T3ez3NjDt=RH?`d~p%T_W8pd`Fh<%^X>*a#Ys7Wqkmpe6sobj4)`XydV*-kg- zqR*R97PPiN6livT%_+Qb+8t7*a0-Gd!C_4YJu&0w|KQ9r&&m?$@OHp+u_!efoS1=u zr}G+5$%+6**9yi;DqHfFT8{4Vy8(CT))0>-Ljeg{cx_SY2nD8drFaQhJdrm;EiRI- z(U!}ygnYpk8j0{#mB;c;DCsn!nvyLOx$PzYG#7H+C>KF!Ex@Ji{p|@Z-D#3B(9H55 z^7eF{=G1Cdo%4A}*VM|NF`T<(gL01k)Y?DdP2CSKuvuv)t+yNx?N$BhPOk8Ovy$ z<8jf6;9=eorD-&0ad;{daX=!hEJR0p&^7S4H0Ygf{5`#4^~Ikv*s}5}8S;YTrOvXA&i)U$4@z{Gt zKAR=MK%8l*aa>0V(Eud@v~b2E92$EVK60S6C1a%|ps~7))e6==MO|!jSkQx{qqJ<8 z__f&ymO1mZ5R*(gUWn;`FBTUlO|hG}!Kc?wWHhI-aVvt&VSEcdrZm(7$&~vH#Y)`V z7Qw|W`?Eq9Vng%gox&A$(v=29!$+i5EST>^@Aa$C={3}E;DLV=t{R&RcjBtxPmI)% z=wxXqJtF%Ebcz2+_%)dg$jc#&aaFEvU#@H)~3@}wJb%OAscAbDTJddt?BaJ!J zr3|9%^bKV=q5^tr6=2U<&8jzJ+bzbVaElN5So_UHMdEH#GcmqPuiff z5vRd*CMPFcY0B>UyO2SE>0jg@#XnK4VoxBSDsjq_%K=FN!SAF?8aB%X{(>4W{{El@ zyg1m2Edw+9tuw~vv1*ke5HB{9NEeYzLV#g|8S~IT$&q|hk#NYvLi7a}Wpx5;jo3KC zGZlS}m+k~_wSZ`4w+kzq>E;@gD4)0XUT~Ok5VK9YkyBQs#t+n7iGJ1_lIt}6o?N!S zZk2x_DuBZZZUlouI4}Bum1#3df`6~`cqU&+6Q$37SWaT7k`rq#%4onDg~`>?tJjl< z&y_XT;09rb?O0LST5fI{kj{PFU;2gDlL8qBoLiFjWb$^q;RFq7tlO6K{F#<<5ioV7 zCfh=Z-G8fv!9L0uz~Pgv)72p{hcU0&gS8^L2lBe-3I=c zVRCK%@R-rYb??cqbkmirQWk)gq*jYX$CfhPRISe^l_(^v6kl;zLpVY50_yLI; zzoZ;^scJtuduhi<`cvo+h&697o zjRcAY=Jmd6h!O3Bh-6Pu&NX6CDV|L$7rvY}63Vn_s}_3rl!_au#+fSblbc0^JI9{~ z!JS$+Y+$E-Gri}f97QSHa}a#lZoF1_>Y{6Eqfw*Okcl7TYO{}1GIQgbr;pcSrr zT4@2MQSB?q_hPog=0l$F<%Y<@>1E%L zw&#m}e){4lX8V_K*Glbk?8f0ykMGKBzxH|2Gs&R)UsE0F+LZs;p6**lzA=+(8uhCJ zL)QO{KjgU$7dG)$@(lpHKD#z~JEhco9S0QXLB+6NB^u&8KJJdT+r@i$43kEvE6Tq3 zb6>5H(oV0X#@f0csg)>Q`hHSt9)?x(*6BT*WHoy;hb8#8{0gF!`GI-(i~PrA#OJ@U zBw6+J=jv=`wY09ofwI8_WMCI%vn>D<-$G<(QSgQoB}4^M(f#nZSs~p zvS4^96ajf8Jv>Mj9&*%3>LN=ZgEP^ci)l}RSBK2|+w5gP5gZi6S_LLrrVNh8On+sx z^LVrW?e;2c25lFx&iOa{Q{P3iO_Ss?* z^%Ry+Bepu&6{-{?oduo?B9}5&7h|@V1BErfdUd3n3LS$*#>ODky0%zr(pz@IxJ+)g zx30EyWlg2DRj++bR|{SvUl9j+B1#rp9;T_rca=l9;A8lYv3W-0YH=~|AkP|Ny1f~k zi$dfCbmSSZ`(x8qIL0$fgHfBhE}|boel%8AAZ4dnx~?i!N;kgd3*OMC++gKpV?SNR z+$3?U$6O)xHHXhUs^@DfrD2JNBhiCLYF21N7O!R~jn3qw8iXMoMYm^nG&Mvh>%$vq zbhp>xL8`t29T9c;=C=#;JLp({XvQp&^10EKr5y;6#R>f`GGj$FLa$cyIguR1v`lsB zIR|zVu5yR1nbmimLjmdAa97CK;{K<+uMMBiE}IM8+jZ1Hqq%Gx0T!OkP0fmf<~10w znj4fIS{V7>Po~E|p+4@^T(hW_zz-2*FbvZf3|v$gv6?hpd%0-EF>HvsSPNT>+isa+ zuZaw39u|!S^_HhNE3JQ8=TgejIU_DclKT6_=uRse>(;q`pjDXkVQ3dbVuuV%NVI?S z5AMRaoRPIHY{gOjDEUw8qZ3r!O=fJ1JtUANhYR3yu?}z45K};q!tMkXac#PyX8xJ= zFV-RTQEf;BEnxBEc3i7_lu2aZ^)F#ihig2bdY9dvpN|@n1FSk#=KzZv?@Xentu4)g z*KH)fJxqPS_6p1s6oBw1kE%%9aO|#(O#ZR~eV^3*_6#}mtkJY_34+J}UU?)TgW+RZ zEi3~Tv#)_D@Yzxq74*kF4L%5DjuB)eG$&iU5AV-E?L1qpt%(~-{9xUZ31cct zCf#-8(p6-ZS=EAJ1X46=Sd*yTQMT#w4CeY2tlDVG8a)G@_BLvA=$IL<|WaI>7t#-dV;M;MTZ&oPtvBEP(!R)EM8`o1=U2a^U zj{Knmna+Oa%;KaktU5c!Bv>aotWGb@B&7JoWNI$+L-I@60;+EOEo}M(HNFkqkx<)= zm$E!k%dv?Sb%!?p-h*RQni}RcSY{fi2}EN9jOzPqxyz z0G5MVn!rWPZZwOC&q?Xz@&VUDNtH#wYTad-cN~O3=6ez^t)p9Cmr!{Ww^2MBbw(ROH<$i` zjxZ>s`H~@SsA8HXomDU9d1nbHKHR&{!>)K|UF1tEy>nF^Bk!y+(xB_^V`Qca6Tp0q zNzG@l@wfba9}fcyB9NN<=gIPzk(}*8kTNBUu#%9JwiypD&N?nKP+LaVTUP3Zr)Obl z4we;+gf$@M15P?pRiBkHx5b`^%K z1~(=x>s|p>Mx%hSwD8p0FSQB{sz5LXF=(!A+N3%JxGJNWpje%}5gb;X_v7Jp}b^l zWp8m~YGp$ihOD!1Y*WAukM61PrGzWn>x#YXT}6A$mG>iIg>x@nYahADDc|8Qm2IFk z2UY!u44CD(Pt1j2HR!7jn!Ki;c9@miyYh$Ch}!1OR*7>v?v~e@6`6C=_*w;O+%BK8 z>^hDR^|Gxp(8AJ3ip#fiw!_rpH2nfvWefIxX#%|Nv9M8U=&_}w@b}r7cjkOK4zlt* z7P8b@=o&7O6Uz&5CKO#`>c={Uf$)|`ZQWNO;S zs~d~Fj07{&8S>gR#-p2z@rh6&0Zyv!T`oN{{5FE2EflZbQ1mFfsuPRblb+^Pd$0qi@paiW#p)g%TG^cEwFwii zgBc!ffZUat7g&+y-Q?k2YD=h3KUMHTvXA@Y=PRr4)+Q1cIF46;J|)vVdP>+;+}BTp z@+fFTmZz6q&9r*XB2k`1{4-U}@c;hfj>%7XuB)rOC}nR`!8-=r2U%BVSwCvZ#bZln zY*&me^dSsYKQLOXiC}PupxGzsa>yVx8lP8W)eZfk12+m$rs23wSfC=sO7T2 zsc1RQe|z;dDOs+nP$`bn7vNRfHjWoRu*)V#`4dBXVzFq?kRsa9u`WM?s$J@hBKbg_ zNYgs*+ z8iL{A@n7SH(t>yf8czx2e<+n`$mU=G$Oh91EF18?%weD5L~E)r9^*&`Uu5(h{dMUb z&1(*+U&2j`y;jN-6+@u~M$m|Usw2msb)5fIa@j2qeK{2pq( z)Jj(_Z~ht&GjteoJ5pB{8Z$2@MG6@};8V(O{~$1q4|gXALZQHL+8`P+Kl=SSwT!hk z1h>S@!QTo83Eucn1*vi&lv-z(8htfw5LA994VpytC^} zJ-*KWA>i%o>isH7^>2f$WR+N`#ZU~dfmPV{MUSzviov;k?>b+!i=>a5&p$c zP}(tgcBb`3_E*dZN=9V}2VLO)lIu#uD!i2(r*z|eHAg!~R`V5K%rLOy-`|5%Z+91K zr=Wy?CI3Y3?o-z=jIJgglkjVxUy>+!1)%*}*yu6fcz3_OsYj}lR^qIDGdr_W2YWKQ zdAe|4fv>&Bjf4>#IC`2G=p(a^I_h+lQZv6E(8vqZJHeFlPvK0+0>516ZHqnLuMIlH z@1CtW(<3tq{{2rw<#(&zY-TOQ<6p9{l1mylO~HE;*y(T$3mdfKMPjKt%jGUzQkN&KnO zG*ZKy-pXZ3GPR{q3`MD&&hWCyJ-9}a<;|Qobr7a=7y^Z;B|UX>HI!d9cv*(QD@X`J z7g!T&g2?q8YggyI%vQ2IfUcj-b1|^P080%T4&JR{0O?oJG3=)IXav2b{Sxq%nu}1o z{qqp$Mo+&XEt)9YL@51(6wqw6%dmm8S1pseKntzOL}2l=rB0CB4Wg}gcs6+f@`UH4 zc_DPSYsh>Zv$A^r#dW{PWYXPYZOOf?8i+|J#gaTMWeTg+%wYAM9rLi~V(b4(ey&FC zGSPn$;bS8Cotk{3OnhwBX7?J#4gX4!Y~dj#r6j>yUnG5s)^-Z6s5mT6sNVu zVe~`dSYQuvk_$5xZ7&Up+;Xmfrp#!}Pr7T-m|8|?+54Uu1^3>G$)L<#j!9sH0f@rs zRX5jH8FlA6oAb{myX6pQ$?K}dSDHCJG=_Rgz>9pNdP#2osd}(d=+l@}N$-9ljD-bT zC?tl}0p=0OYM&XJHm6o|T3h?WpB4;@~bAK(qxG7Ku;?IF@wN!3t{!@1+&?#15^pjnlH)%V~!m#F*~oW6G5z~IDA%6us&(mg}O z--+v-`OrQOs>D7gqo#wwL+ck)*y!ciLqcNU5W~?N2c7B zExUcm+L~*-Po{|s1oL~-;mX9WLzt<{yHp)0|P!Q)zE2M%K>$gk!3p))~B2(6dQ}5D;qm6^K}(>@2oBKmw5gS8<{* zmj{ETh*LDNEeJAB98wi@t%eVBthYRbQ@z^*f~ELFm|Yk&MS8xlUC}cd49H3&zjK=` zG$&7=7_M>|{qFkt8WQQ|`1=#5a*8OBPaGp+g+1DwfE7lB;VJ-yor*`^z7*`5IbjdB zDc{ND$p06y^^9viDOIDrI8rIU>@xC}M@%OrUUgJxpt;vOGw({i2;9qgbau45yO=N( zZ;rmqb#U!4o8>zfm8F)|6i~iDWO8X@?=;;U!M+T8Z;ZZC@?D7v8^PSvdqREWp6@D* zX(t75&D6(ioK#d zc7K~bKo-U~@f*t(nhdk~GFr7`?q4Tau*w?fbJn6eZoWvHjlsYN&E2d&@b4 zxGcJ`EENBvm~`h~tdUJEiIp={EK)q=V(r#s&A|7q9DIXWw4s$mv>7$2<$Mag6d*?G zXOz(K%I9I?17e=U>7IADog^Zh^Fu=?VD0m|lVO5yAS53-9W(xr#G$@~ys3;)g7}Ll zc=k)q(@rRNr7XDfL!UXOu)m$yV!N)L$`@k39!W_p#JA)YFFo)&{chV#H{!L3xX=L1 z3$-&?yV-h=e~Hk7{`UrewCIg6UPz~*F4&B>nyUx*9>@%3_m^1WsYVI27c}I4K+D8q z48`AYi1wc1zKk2$V*6d{jrAc+V$9xA)MvcR4%__BM&AD<0Yl24uqjND?dt7}c-s?kDC zjNcb0;-?Zj9knQzL~5#^Sw8^Mm|wB09Tb zBP%Fy3|4sv*WVvn?v(K77ya-YQCN2jf%yGrnc_;KtWt7 zRqcu%ELo|ECIiS3eHp1>mmFLDW9B84spLU0$t5xpI6r$Trs<8_t%>TU-~3>OjTXL1 z0<@fII>vhNZYW{IRjOsG_@jXG{H%jvdIGvt^T2$iLKBU0TD@UZt;RyG!K91`q&dVc zKGiyQ8M5vcA-+587Sa{#(`6S>piQ+ugZTWH4^>c1TW)N5<_B-7b+|2W{;iYaliMX; zV#$#4Xqy?=KQUT!bfi+h1s@s}mux!y1i}Y{aeHOwo#koQoDD0m2jE%!K`J5gx`!VW zk`ADX?jBUL9Q9GHzg2z;z77H@BB7Q~&!&=4iE<`Cv@cPT)2!+1RayYyH{dXDcVR!P zkE3rhgggn{&cxenEDP zHS$MhUZnAs!)Dg$g&Wdl{Ak(Zy@4pZix0AC3~F`ZQ=2j3r5Ks`f;gE6P{p4947LLH zJ*Hjn$OpFr1$romFlJMLrJ>=`h?2ZZ+1?pWL~slU;dwvBY{3LZ^Pz)Q4{|+bkOChM zAMVX@Vz@lXCf6$e?zlS@T>rw%DVIAD2K!?See(z)iqnk0`5d=a!$Fhn^Z?J<-^t1^ z9bk#*Wf$p-mk8%UzHWs8leG-GMH;xtP9n7x=d!;3nj&pZl*BE!C_$yC5ce8{wgz0ZI(e235M8GVnP zp?*z!Bd^-E`A;CXQF6 zefr3$NxCvHo9`bRFgqYWPEQo7ykvkUX;&fY*05YG;FQK2a-;@21fpyvQxEgq^4!E~ z#o85Hqf2l}5Ml;oU}DJd*V(EPHc!8aLD1LpPzoYlM+3l1~ zcVWpo5alFN@pl^aA3H~(rH!NJQ%O=WQ3(+V?a!NzPFs9v9|(We!>S^kb}^Q-jwP4= zvJdfzAO^N{U_wq4_i?#h`qKd?Km9mNmFU8~MnzBc=g;8%vuwRhe}+032E6?YI|8Fo zs;4QCOQ>D%P0MKP-F_=odIH>T4trdle+{OLONDJNUP1OIBJ9+KnrIojYcV(ZfXUe! zRQN5!iFwo3GIgZx>ZR*st29Uq*YihIH5?Bw>+Y{pHV)lgDm0_)pwT6}33Q=O<(46G zx`vbzK`o!;I|ZF_*2pV~cqzFxzg4W?zqADi7@ zZw46qYcc;!?V4z3N3>X782Yxlzuex==2Lf~B(&bBhYy^;b|d^3>5Mj(vMsIUENpsI zUV+lHsp^+sFQ5B?qn}@P#a~BPudErT@s`;&chheOT7ITkJ%jjE(LJp>^>dgFD&6O@ zFWnwVNk^UWk>0DRvMaQ5EW5=y`G<4{Jl8~2Z4vWN!d)IcWE0M>a=TW%i+10755k{F zBcIj?J}QN9gkc|Lhjh!gYK7I*^yY1Wo4{x<)=B(umzDjy2_>~tL_Zye%*xWAF6~tT zKi78GhLW|)9iSFnmUT}Oz2cSm+4@RArgj7oe+rkdAiM#pcAfI%>|adb9X+5~+FBLV z1!eChdH5ee%KHocqcD&vqwa>_Na>A6>Zc71$L8znM>$=ON3|{TlMy9uF#;+FN50Up7vIb@N3N%Ir1agot=BUT(>mFxBiC9 zgHRCOdb!mz(bMGXs-x68Wg?_K{=kskJW*Rkf(wqMI$yYEO5fUwNMR8(wcDCrPSB$n zZvWe2s4jV&42=*j&E4cyJM2f51Jt4Rbwr?7c)JfLN$ds}ybgAOB%YaHmK8PjGhsNX zL2Pxp6x6Kes)Y~XcPl-=+)>t_5dK2MzWp8H2lEHM#sj{}`WTA|GRI9fLcq$N?DFXP zP6IbIdv{{ihyA_%F0x_3#x$jEFhn0it$PXzy6`1dj*VF&Y z02f?uh^BoyXHOa~V$vab58F)haT^e21E$G|p6j6qn``5yE0rrmzoXOjBXkETzReXe z;-?r?@DvnrE_PqeimLN+XF--wu7KPL+XNT%)sF|)FNrKA)?;4dmEhXUJ{w~#56}u| zipDKgZ3>DMuzY939In?xY~a3l=O22xe5gml>Ns)@chr?Mg_6S*)bLa|^8U$Ha+5!$ zH_l4xcA~6L1fgxsUCj)R-Ah z9(OC{@d;{^c6<2thNoZ8em{G#Nd6O??-o`?QT2%Ac+O-ORZ3SH&-S0oLY|*luCb1; zBh#hxD`=vo(%iM#Pe}$WjzpRf6D)bqSu*vo*)yVLfmSEH9zy*fsaj}thCv5-W}yXr z7M>vAbR)2*A9jeVp+IDp$}K7VDzD${$;XW|pt>YHamF9IDn0t-E-$VpHW!Lk&V5P8 zR|Sm}Z~C|kTYwqc##K5^a3n*!t>|*H0MJmi;XvrTfpY%3C$mkmYEiLaoVTqjesD`g zsYEzSQYhU;Ud4U$qt})>f~+t-_4)e#2Jx^Sl8a1jL1M?r2;fpd6VA%C9I~#MURuEP z(vTP%=(X-#i6q2xk)vv_F=YFWyaN_U2-QHW@}VmGG5KLuaKXIH*eIii)KiXO0>A@IyO@yUO!bIk{^4!s1FRvFcpd zxHwN#d6QR((C2Mwg&bTt=+j#2k-xDPHkHHD+M5OS!Q8Dr-I$ZA4d}C8G8jm1%ol=% zTpN4A{SAxyFzeu50}hQBq?58oSW7JC*Iw=! zI5R(0y;Ia^M6r{1Z9`9-ugA)|h6)OG{MQm2;1;c%l9hd6Uuj*_uLE_ZC5G9_BMuCG zc|&;f04^QRw==)B-DiU@4^YO^lIFTZnn;A6sbhHeZ9V=q>YipQDSr5ethBjX-_zvX zOgtYPSJvNTHXl4tGjeAY^DFB4?X&YYZYVNnjoIDUpQnV#E-%Yv+_%?sKfHNx-3Ks{ zOk|MxhrKV@=X@wBahIjX^?~u7eq@=|-Cxrl--_(>KK)XjHW1S{0m#!b+zV>$;MVC` z)_u)AX1@WxkwA&rvsRMY!98$a9#zzC&ua4e;XYr!Mf-X=>rgmC!XvEPJ-WveG7gj6 z#&Us#>x}?HQtX>>X|;yffxlo~q?WUD7zOrfIBrUpW`6HHbdei{rfD29Y ztn@$!QFq&q4{)sm_ox^*YZRLTu9>=8MRhx-R^xAcXx zKwcPIMpWf;zf2|k&)5Y)kbJ13Mh(GyNM^F;tZejS^4v1e& zmI8rZ_Hv-PjfbP9Ckn)G>Mk5KK)OUx^}g`X1hzmlD|4NFw#Q_NuJaa?khRDO4GCuxE31%{r4XHBK0i*T;@j>@Ok8~?GFn`C}Kh>?p^_zhkT1X!6y;ARc zc;5EWO+D(KZ889l?E?H`dat?(spP z4w3)197yGM@>HQ5r!Y#!+=C-^afmcRgqK6`^#Q(r8z)LB{@?i*69@DE&(bqx-5G!f z!kU6~3iu76XJ`E06HTFW`9@fUBbUS{N;;Nju5jv0oFi)>>i@8ByRDgC1|Xs~#e*j} zi!>vEK?G)Je>0!6Ke3esNVpYL$pXAKWb)a0)`h(qtE4KS!fqLF=Jr;}2*fvp2!*{3 zkn@;D*|Q;uUXpHUk)~koLxmk>;L^m)bn-zx8$q;=kS%DQ*>l84ZpMJ%%#ol%(W3ZM z_>b6wx_L-o>}SFd;D&aye&S<7#RzGKLCMkz@IP@kg~9xQ;zh(|O%TJpnOc21e(8|8<0krr zBy@Fb#zb`9I%9GgQ0&^ekk(|_A`(mpxrT~C2Pnf*FjnFasmxw4IEBpIU!xa=PqA1$ z#+p@vdzJVhybjsUBaWW%X0tnoCNswIgI03h?t3)vB2+WEFmS*$Fj=@tA3rUfxR(G1 z;Yt)t*mf%R#6K@A-sE7~jl-pWXiMqd@qu9>j^-SD#8eWzXbTfr?Nx2f3E&(Uh~tCF?pk|yf|B#m9sGOM*5+^>2xuIj z;ye9~p&F$f^dN!QAXj1$2Z3)e0@1m<;pTni*pKM_f*$OMP1H@jBJOC_zM*e%WHo!f zxRLteR9yo%-+`K@rXCx?%G6^c&x0>;QiZeyJj-` z$-n=SaZb0|vdZS}$7mOSu2OfL@LQbjpS}SV>b|Z| znL|V>ND=X~!%doy{9+lmgv@o=!q9fid6k;8nL1oX4b{zhwz8pp#pQ;OhOljlb_Zk> z6*KxrAzT1KMXrAMW-*^!T5vM8{A&U6Ji7;}kX36H4+ZAAD_8Q9itf{kOkHGjoz|gG zJJlSpdYg9Jw4=DI!(vS&jyB>g-;(t#|FXf`BWD=~tj#&DMP$~M!hEiLwa5gnD&n40 zFGWSe*@#(g-A2Xvgz@U`tdoki&YrNAe-*EpHp^L(pckL#RcPf0PO`^=y&{LO7Bnm= z1mvK-X%OEcWMJlyJ$?dKv(6H_PFxFn__hx)4H{l;d$G`aS?J}nl1T`QG>$)9U&Ecu zSA0-Drs!j_2@8X7=(Oa_G{bi75S&}teG-xl|H}|Ltp#$Wh`s%5`;@yRVF-@H=<%cH zGb%G1iQj3M$eZk;xjdfjH<@bO3bZkaxDlU#4Yll#o%B@A=d^`k{0Rb!EU~J%RgQb$ zbm3Au7MmI}KWB)u{;#_Fa0$MQbYw6)Ges$D_W&8DvLc~)DpN(6l2nj#u43QW1s`M_ zKUzzE_*EFezmC>`q3F?rlX+sVRn~}Q@HXJ%?F)=``d%KNJx=q?_&fTtT`$&6L!isg zAAvseELuU!w1f*Q4^M^~|Mb`pcEq%RQHqj3oxeg-8!=@#X85t5OfWz`@nHlgN<>GQ z1PkZ|%jo}eETtFcUuWLp_vSMK>ZUjB+~q;3 z^CiO48rb!<4U9;wxfsyqESA7JIbm3gNN6oG^F{0McAI9O+4|0j_U_RN-y{|U3tb!4 z7ZUtRCU$D5enHi80S_F43p0|6URG3PM3r=kBL5B_?%*Z}L**V>$%O|Gy+WT)KU{MS zW&7_w2fRWU$mu%uNw7r?0BBy(sS~W$k!mcvn}(Q)LZ`#NH-y#fMS!~BU7rru z54G{hT!cssmJBe{tzP-+sN>A=d}yOb>xa%n63kFn9g3@&eqODea*`!!Q3INr+)~jk zb=O>^zhZM{^2vG;ehMVjVbO_9j-c71Q#HvF+evFSaZ6Dw%7=)zL1MEiuhhkdCr}dj zmHVx%8HM>Pt&I0okOn|zG@j@ufc*nVM*C&DN0t^SU1#3%V2EZTK=My|RJ!%N*}}Gh zu*%U>!}-Lfj`oaExKgyONwJ?VWr}2Ia z$}N`z1k)D4zFVJp@!j_k*cC8Krmz~i(Bxl zHdRzg(oSt-1z|M|nNHGDSix{QRP9HuO_Pv`B?)bd7l%irIg%uVVQns^#mxHJ=3@US$Rk z;uI79fU-Iwgv0j)YlCDU`!s zcAq9uu#y8obM$wKvMU({mC5MHu5CmVYE~-Dv zp~E-7`Gq#kUQq&Z#xTfUC*?%Cs*iuusx;ZM{-Fc7N*n2Lf zQ5D%l%I+(5*84fo-EDE_Qt2@?9j%jPN2|1qGcJibfO`$I9QnCZDAQ~By4`e&Jbv!Z zuIzq)eH9C%;vW4;lt@?JXnCqdx(PnE`G!2Le!*KXc-LpylJttCtoVq*U*s0{v@$YjjSPsv?Hbo+^as1e7j1mDM3{w5BcmhTtpl4_%gcJo?uWLi)Dn z(IBq(1ev8bw@&A)JhgY^dyvUjFwUaG*?#YAaS}gog7nti={Z^m?B5$Y;cif0%JG+%}8D;r`;cVpdBm2ikUH!U|s2Id$^dB*ga5*d? zKGxxg{ix6Rn`Dmks_cn|v{%*nQKIef7_@CSw)vd_)RVyq_rZUN%kaM3se5LieLoYu zeJ2QW2u}ODhw3gb>Ht7vlI@7m9%^TpU&+!40*64hO*glcyc7V+e~_32$^h~daen|G z2vZ9G9)Q1P)E{8^6NLG{%*+htgegY=_&}LB{u9U7vJe57gZf{#X3JC( z;0WY@Z%Wxr0kD8Fv;0q(QYs(`6qNaYTCiG>G63NqDadXh>@8A~j|L;*Kj{zMZ02Vfu7Tp(s1PC|-3nK$dO64oy z*Z-bQzXHqv%8PtAN28Dj}5g|_?UCoX(QWKKTicxz}|MMk=~q~JrrJ$(mzBDkZ181 zIuDm+W^<>_KV0cr@Kn&tj=qKrYp+uT3JmuLKLcfv$q4K`A=bp;y7*84HCuV0>wi5 z2q1B18fSxVAZpudaev^>YJmi9a7JRcn1Qx6X~F~@!dT&Yhyo}eqp?Aj(U-J#1XdYD zglB^HBQwE)9+1*T###7gZ4{Y_utH=oC8K;(GxReC!f+x)HPB|{ERhAfVIus^qb0Ze z#mY+j^s>TW^*<9(Ke@@Bi!+4>@H4Le!Vs1X5kb+L+=%yqn+R}$kDz8gC5nLGx)Z~~ zMMLbM4fk`g7sX;7Q zHMK-&ET(_GlUDWRd_*xkhJ03?Y{!2t-q+*ap3V4w z0%a1K?Q%MKW8O#C^_1iLOd(@E}I8KKCNJm@0X=R^su4-@5}PFc*H9i-+aMc6UpeZNy@BJ})tS0Ls%kmm#9u#0v6 zY>_bTH#6s{wQFbk!ZZ`Z5{s(iY0*$0S>(tlS(Wyv}J} zjZdn@M5%uq8SrO$HCUy3xjV9Po6QP6^RsQxjJ(?Nh+&yu`g7k`}pzZ6M!$2HpPX& z@|Sh>3`o{P)>#y$!W28+*@tBSTU5_&%a^~a~Lsv7qq++%R_;DB|ws>3Y_f&KUOY%^c3 z7ce@91IzmJ#q;9wak;M={gZ(=9?who0L6cONV^VJ5$fyBoAnaFRY5EGyIg_SB#9o~ z!}egDXuL!C{|Wobl!?!KB<{!c^`;=x=)K(6yU)dgNAt~9bzWS)+|8>U#r~j;{hYko zZdcVU4F-ePXcFC)f|+HayXRZlP;KW@i$EnCh3IZ)z(M+Ag*4$Mq+_zI^;7rQP4fbtEHlUDNRf z_OBV&-B)p4PjKB2xbD6c*G~pM!ga+*xUSe&bNzJEhg`QE*Uk6fI(NiC!}+}l^ZKXf zzdn7|`flzZ3>_+A=&|`X>Whn`J3)UK&h2T&D4jD#sc*%YlaY@x#*Q(@d^KZEXMMmJ zB@<(m_#TX5p&(HaYU*S-U1X?aR0<@OLCajBLwS>=A?P%VHCm{fR0pjPg~GtlBl@#+ z4yl{7O@l(H!kJbJ+SFDpi*Ps>k*w1Z(TLU!O-F)I!;&EwMKPr=ghD5625f&50nrko z2a6ok+pdHb?tA`{GXd85@JG)Z2+zVxqCsw!E}rVMrE^I-IlJq%gBLyO0mEyDhbm(8 zDq^Rq2s`!Kk*AA@=6Mm(>{}I)(~*x<5iLimh!$U6MJCcdEF-=vBkubxvSjl+NGu%2 z%TTY|DnTd7^aDX4oC@oH`pqC{G+PEdZTdGg0;>f&b|(@2S>xj4rbd6zsWKh-3rtNd zBW|D$tK7h-J^D^n-+ah?qXZxl}R~CwIE>JZ%Y2~%d1z<(wc;eyQU;&`vtuR zI90hXo09J2l;jgr(hh&7r1@4;@^s{5Q__!3N%z%Law6@6DXDdul3IO_DcQ_H-R*gp zNfRwJjSM{FX=kHZ+SxQsn-s!!Mroh6GhwUHP?dfXy)P{v|M%a&JpND0kG+eQjNTNE ze*eT);mg*W=rvADH}R%0gUz`4W}ETTk&n>QjP1tF*YCzB(msEnrRlcg<~wZ1JJ0`B zE~-6UcJ!QnOl^#94m7mz{vXRZ<)Rso7`u&^X4bmq&NX+wL7;tgxN;G%?b`jT%yZEs zgo_9t4=H#&8{u&a%j5E#$M-9blbS~capV7x8#YGeqInnZeZ2S4oNvqv*4zuu;w5PqSp) z>C3o(lW|aFq_(vBi`@{(3l*)axR)PAX}jsJUv!Q>HV9W6b0p%`H$f;d3j6X3XZl z-qELjR-Zbb23WGg;!{}MXIP_tv4q9lB$mmr#0XX_7XeF8U@4dR6}=o_sSZnISgCjP zi;eZ>sWCTsrpF@Tvm(J| zzerrTfk|gOllc@#M&cA0p_+24>n)~w+bnoz#<05JtsBRRy~jMUT^x3DDjS|*`7Cv< zn;~_rn{Ddk&}0|99R%MRiNUimhtrH##!vM&{TY9+>{#%|by#K+OJ!JNei_|i$#JZv zx3OcrMc1264TE->Cpm*#H=ALwIfK=}r0z#jCp9)#pWglRua(>+0dIy1pov`=wyQY_`?qYT!M&Gt`DwqrB`r75S{!alV8~UqC|6h2dUkUj&SY~f zxIEu|p1t3bTeoEmCB=t+-)vU@YfM%<(%XNRyOQ*VVC(V&nbfdWT`%kXdHC^pRL9J) z%lf$4y{~rX;V0`{i$-I&In=?P*?cNj+nZog4-~f>>{LHj>jf5f^iA%*F6#?+&YqstFB>awn9qkKdN- z{l{t-rndD*dWSHv%lh|#6Q5r%7Z}BdvvM0gui@z0a(#=94hiBi{~c;H!l{P^E9t#E zDr?OX#Kv2GG!iKDcGwz;Ac^ZskTo7ZnTQfa64RG}EWE9Rjl_^7^WQ?EGaY|Nk_^Y6 z(+{6L3EVM#4M8!)w|JCL3~?b6B@~0}dn_mhm+4RpKGUHXd}a$V2-f#lSdwHqh(YiS z$CIR=K^V9FJ`ypAEbEd6SM>dph(Tmomy#qI4q^~l)}x zOfjUbdyj32N1fgYvcWgkH>-arXxDdNQP930v=T)6zH16WOVLk*qW7H4kf~D13V@B0f^eE&Jk0@eqm zNzhLH__f`iXd=j#Gf7C}G8Z8pn@NADLbD8sDsWNqJgi*(GsiP?!CxEtm`W1bQ__W$XP1}**L7tR#x!Nq+lg@NbnocOY zJx!pywrTIUsD9eatIIe0syohniIwuB5MGhCVgCmxrbd|xWp0wrPTNholRTp(r z7p&Qu*Il(*-aI)ksx@zl+dALJo6r87-}#^a`Jdh7{ zmlD_kJE0Z!!aj&y`U{K`H%{C*KdXZqFkKuf=WFLVtN}hZ=hO0b=;kOJU?XgT&Cmeb zU<=eqDxHh7Z{++Md9|Jc^uk%V0#{ks1x=Ewn&TbYVJqy0W`9W=-;aeB=zvaWlhiU6 z3wxj)x?sPguJKqn0EgiS9E0P~EvfJG*rf-K!XY>ZC*c$fzzIp?OJnCgI1Rlp1ZQDT z($wYHWf;!Dc^Hv2JrE0{Fb-pqUVJCEUEt!xIbVj0l3uwN3zr}+&V;1b2V>h6n1V@3 zw|B?3YcLJhC4ar!9ou4-QS6MQ4>Gn{68Z3_ctyZQu~s7K?tE;cXCrT;U87hlmvleo zVWe!7ZM18&Ycy$OYn76|Scv`T+348_-PXZ+sFCz-OYBk$G;UmRqamx8^zg@6-N4nL zFry~hBpgJ28dH&_tMs-HJHX?HK zR%|0UyMF*wXVhe5X6Hp-tB74L!Wc{fHQ5xu%SAv#HX-tMZR~sHR+=|M3x9Br&6`BNn~VdrLK}3zR_KBvhKFsj z3%BO{0K{kP75Sk*77oH8*ayd;2adxjI3ii;v+HsKh#@};r=d@>s%MAlhchq)BQPM@ zy0>C=7zW`yaBIm&C2RaWR>$BRT!PCm3FDHre-^t;z(u$Y@ie$5S@+Xey#chL0wutF zm48%}nJZ5XB?e`*%HWkTYK(D4vfe*AloEqh0%KGXP!dpTQaVsNP#RKV& zKxs&MJZX~br8nZp#Gu5W#GnjbsYEG4n}Fddl_&`)(^tl$3`*-HyA)+BF(@%8(^npW zN&?DrP^n3YK}kjT0`t{Qpev;ar7I;1KYzk2($jcF2TB6U%$3KY5`*>uqt!0Sp8F-v zg~2ObDMcu4=l~FsGIJ#hB?~1VB?jd&tKBfl88BZZ0VNP80VNBKrx01_IFNvY#y0lz zScn7g*mmBbSAm8+j&cnM#5ugukP?l@M^cl{@MoM$F|sG0@ZFr6obT%-6Wp{4>VLoU*+R7^5>^CwXVI9uG8fbOz-#=``sLlD+wFaSd)w7Q9y$ z7lK5cM18Advr;U~zEE%{e%(&2PGVQSQ!@LgSg=0|?tUMu#OlQAt&+{%EEeXT7R-MW zt7PkB>m8CkC@U5oyjt+Frozen snapshot an Scoring code SHA-256 prefix -baaf2506800b +e75a972a2f98 US run label @@ -299,7 +299,7 @@

Frozen snapshot an -
+
@@ -1096,15 +1096,15 @@

Uncertainty

Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 6 of the other 49 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.

Cost and latency

Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.001 (Claude Haiku 5.5) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.2% exact, sits mid-pack at $0.027 per household and 23 s.

-
-
+
+
+
-Table 7: Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: reconstructed from token counts at the list price configured at request time, or the provider-reported charge where no reconstruction was available. List-price overrides apply at request time, not retroactively to recorded costs. Models without per-call costs use the frozen release-metadata cost. Latency is the median across households of each household’s summed request time. A dash marks usage or wall-clock latency that was not measured per call. +Table 7: Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: reconstructed from token counts at the list price configured at request time, or the provider-reported charge where no reconstruction was available. List-price overrides apply at request time, not retroactively to recorded costs. Claude Fable 5’s cost and latency come from its original run’s per-call rows, which its board rows do not carry. Latency is the median across households of each household’s summed request time. A cost dash marks a provider-reported charge of zero on every call (GLM-5.3-Flash (preview)). A latency dash marks rows that record no request time (GPT-5.5, through OpenAI’s batch API) or a median household answered from the harness’s response cache rather than by the provider (GLM-5.2, Kimi K2.6, MiniMax M3, and Qwen3.7-max; see text).
-
- +
@@ -1417,11 +1417,12 @@

Cost and latency

Model
-
-

Cost preserves each call’s recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Rows without per-call costs use the frozen release-metadata cost. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes: Claude-family models issue chunked serial calls that lengthen wall-clock time, several models ran through batch or asynchronous dispatch whose logged times reflect submission rather than generation (shown as a dash), and Claude Fable 5’s usage and latency are recorded at the run level rather than per call. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.

+
+

Cost preserves each call’s recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5’s board rows carry no usage or timing fields, because they were rebuilt from release dashboard-data-20260701 without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (Table 2) sends a household’s calls one after another, and its household time is their sum. GPT-5.5 ran through OpenAI’s batch API, which records no per-call request time, so it shows a dash.

+

The harness also caches provider responses on disk, keyed on the request and the prompt contract version. A request identical to an earlier one is answered from the cache in hundredths of a second, and the timer records that as if it were a call. A replayed response keeps the cost recorded when it was generated, so a household’s cost counts each of its responses once, whether or not the board run paid for it. GLM-5.2, Kimi K2.6, MiniMax M3, and Qwen3.7-max were finished over several passes, each of which replayed the earlier passes’ successful responses. Each of them records a replay on 53 to 93 of its 100 households, so its median falls under a second and it shows a dash as well. In 11 other models, 29 households sum to under a second. The audit traces 27 of them to an earlier request on the same household, such as an onboarding smoke or a restarted chunk; the other two come from runs that predate per-call timestamps and left no cache or ledger behind. Dropping them would raise no model’s median by more than 2.5 seconds. A further 33 households replay their first recorded call and then reach the provider, so their times leave out that first call. The benchmark card summarizes the replays, and the repository’s latency replay audit lists each household. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.

Simple baselines

Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 69.7% on the headline exact-match rate — below the 85% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 25.5 points, and the within-1% companion shows the same separation with near-miss tolerance.

diff --git a/app/src/paperSnapshot.json b/app/src/paperSnapshot.json index 79d714ae..a93e1655 100644 --- a/app/src/paperSnapshot.json +++ b/app/src/paperSnapshot.json @@ -1,6 +1,6 @@ { - "pdfVersion": "d0b236367fb3", + "pdfVersion": "3cb8f835f260", "responseWindow": "June 12 and October 13, 2026", "snapshotDate": "2026-10-14", - "webVersion": "533a0cb79b6a" + "webVersion": "031238989dc0" } diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index 09732563..86a6636f 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -420,23 +420,61 @@ the site's types and leaderboard already filter on `condition`. Each frozen row uses its recorded per-call cost: reconstructed from token counts at the list price configured at request time, or the provider-reported charge -where no reconstruction was available. List-price overrides apply at request time, not retroactively to -recorded costs. Models without per-call costs use the frozen release-metadata -cost. Published model costs retain these recorded totals rather than repricing -past calls at today's rates. +where no reconstruction was available. List-price overrides apply at request +time, not retroactively to recorded costs. Published model costs retain these +recorded totals rather than repricing past calls at today's rates. GPT-5.5 ran +through OpenAI's batch API, and its costs are reconstructed at standard +synchronous rates. Claude Fable 5's board rows carry no usage fields: they were +rebuilt from release dashboard-data-20260701 without them. Its published cost, +tokens and latency come from the per-call rows of its original run, which that +release publishes as `claude-fable-5-predictions.csv.gz`. The harness caches provider responses on disk, keyed on the request and the -prompt contract version (`policybench/cache.py`). A board request identical to -one sent earlier, such as an onboarding smoke on the same household, returns -the cached response: the row records that response's answers, tokens and cost, -and a latency of about zero. Each prediction row records its call's latency -divided among the call's outputs, so a household's latency is the sum over its -rows. The Muse Spark 1.3 row replays its four smoke households (scenario_000, -scenario_001, scenario_099 and scenario_119), each with the smoke's provider -response id and a household latency of 0.03 seconds or less. In the frozen -predictions other rows also have households under 0.05 seconds by that sum, -among them scenario_000 and scenario_001 for Muse Wonder and Claude Opus 5.5 -and scenario_000 for Claude Haiku 5.5. +prompt contract version (`policybench/cache.py`). A request identical to an +earlier one returns the cached response, and its row records that response's +answers, tokens and cost. A replayed response therefore keeps the cost recorded +when it was generated. Each household's cost counts its responses once, +whether or not the board run paid for them, and leaves out spend on attempts +that did not reach the board. + +## Latency basis + +Each prediction row records its household's request time divided among the +household's rows, so a household's latency is the sum over its rows +(`analysis.household_latency_seconds`). A household that needs several calls +(one or three outputs per request, a repair, or a re-request with a larger +completion budget) makes them one after another and records their sum. Published `latencySeconds` is the median +across households. A household with no recorded time is left out rather than +counted as zero, so GPT-5.5, whose batch rows record none, has no latency. + +A cached response is answered in hundredths of a second, and its row records +that lookup as the call's latency. In the frozen predictions, 330 households +sum to under a second, all under 0.12 s; the fastest live household takes +1.75 s. The audit (`docs/latency_replay_audit.md`, with every household in +`docs/latency_replay_audit/households.csv`) traces 328 of them to an earlier +request on the same household. The other two (Grok 4.3's scenario_000 and +Claude Haiku 4.5's scenario_009) come from runs that predate per-call +timestamps and left no cache or ledger. Another 33 households replay their +first recorded call and then reach the provider, so their times leave out that +call. + +- Qwen3.7-max (93 households), GLM-5.2 (80), Kimi K2.6 (75) and MiniMax M3 + (53) were finished over several passes, each of which replayed the earlier + passes' successes. Their published medians, 0.003 to 0.033 seconds, are + cache lookups, not latency, and the paper shows them as a dash. +- Claude Sonnet 5 (9 households) replays its smoke and two restarted chunks. + Muse Spark 1.3 (4) and Muse Wonder, Claude Opus 5.5 and Claude Haiku 5.5 + (2 each) replay their onboarding smokes. DeepSeek V4 Pro and GPT-5.6 Sol, + Luna and Terra (2 each) replay requests sent minutes to an hour before + their runs. Replays can only lower a median. Dropping these households would + raise no published median by more than 2.5 seconds (DeepSeek V4 Pro, 169.9 + to 172.4 s). +- The 33 partial replays are in GLM-5.3 (18), GLM-5.3-Flash (preview) (4), + Kimi K2.6 (4), Claude Sonnet 5 (2), Kimi K3 (2), and GLM-5.2, MiniMax M3 and + Qwen3.7-max (1 each). + +The leaderboard does not display latency. Whether published medians should +leave out replayed households is an open methodology decision. ## Country data paths diff --git a/docs/gpt61sol/design.md b/docs/gpt61sol/design.md index 72607196..5ffe1740 100644 --- a/docs/gpt61sol/design.md +++ b/docs/gpt61sol/design.md @@ -101,9 +101,10 @@ and `test_every_committed_adjudication_decides_a_case_the_seed_judged` run anywhere, and `prepare`, `bind-seed` and every later step check the seed and the stage's binding against it. -The exporter still needs the Fable 5 usage carry-over. Fable 5 ran through the -Anthropic batch adapter, and its committed rows carry no cost, token or latency -fields, so `export_full_run` reports `costUsd` $0 and omits the rest. The live +The exporter still needs the Fable 5 usage carry-over. Fable 5's committed rows +were rebuilt from release 20260701 without their cost, token and latency +fields (an earlier version of this note said it ran through the Anthropic +batch adapter; it ran synchronously, see `docs/latency_replay_audit.md`), so `export_full_run` reports `costUsd` $0 and omits the rest. The live payload has $54.11, 3,850,174 tokens and 97.68 s. The driver copies those four values from the base, and the drift gate then checks every field and the key order. Ox Alpha's `costUsd` of $0.0 comes out of the exporter unchanged, so it diff --git a/docs/latency_replay_audit.md b/docs/latency_replay_audit.md index 2152644b..185cdc8a 100644 --- a/docs/latency_replay_audit.md +++ b/docs/latency_replay_audit.md @@ -11,9 +11,10 @@ The household-level evidence is `latency_replay_audit/households.csv`, written b ## How latency is recorded The harness times each provider call. A household that needs several calls -(one or three outputs per request, a repair round, or a retry round) makes them -one after another in `run_single_no_tools`. Their times are summed, and the sum -is divided equally among the household's rows (`elapsed_seconds / batch_size` +(one or three outputs per request, a repair after an incomplete answer, or a +re-request with a larger completion budget) makes them one after another, as +`run_single_no_tools` does for chunks. Their times are summed, and the sum is +divided equally among the household's rows (`elapsed_seconds / batch_size` in `policybench/eval_no_tools.py`; all three lines date from 2026-05-01, before every board run). A household's latency is therefore the sum over its rows. `analysis.household_latency_seconds` computes that sum. @@ -64,11 +65,12 @@ before the household's request began. The evidence is any of the following: than a minute before the request (`.policybench_cache` in the main checkout, 121,648 entries). - **OpenRouter id.** OpenRouter ids are `gen--…`, and the time - is more than a minute before the request. On live households it falls - within 4 seconds of `request_started_at`. + is more than a minute before the request. On all 962 live households that + carry one, it falls within 23 seconds of `request_started_at`, and within 10 + seconds on 99% of them. - **OpenAI id.** An OpenAI Responses id embeds its creation time in hex, after - litellm's base64 wrapper is removed. On live households it falls within 4 - seconds. + litellm's base64 wrapper is removed. On all 694 live households that carry + one, it falls within 4 seconds. - **Spend ledger.** The board run's spend ledger records `cache_hit: true` for the call. - **Smoke.** The same response id was served live in the model's onboarding @@ -79,8 +81,10 @@ the cache. Every full replay in the run sums to under 0.12 s, while the fastest live household takes 1.75 s (Gemini 3.5 Flash-Lite), so the one-second line (`analysis.REPLAYED_HOUSEHOLD_SECONDS`) separates them cleanly. A partial replay sums to a second or more. Its first recorded call was a replay, and a later -call (another chunk, a repair, or a retry round's new request) went to the -provider. Its time leaves out the replayed call's original duration. +call (another chunk, a repair, or a re-request with a larger completion budget) +went to the provider. Its time leaves out the replayed call's original +duration. In GLM-5.3's scenario_007, for example, the ledger shows the replayed +16,384-token call, then live calls at 32,768 and 65,536 tokens. | Model | Full | Partial | Where the replayed responses came from | Published median (s) | Without sub-second households (s) | |---|---|---|---|---|---| @@ -115,16 +119,17 @@ the last one. ### The four open-weight models: a multi-pass run -Kimi K2.6, GLM-5.2, MiniMax M3 and Qwen3.7-max were added on 2026-07-05 to 07 -(`results/local/openweight_run_20260705.md`). OpenRouter's connection -allowance collapsed under concurrent long-lived requests, so whole households -errored and wrote no rows. `results/local/rerun_openweight.sh` and -`results/local/grind_model.sh` then re-ran each model's full range several -times. Each pass deleted the part files and re-requested every household, and -the cache answered the households earlier passes had completed. The board rows -are each model's final pass. MiniMax M3's first pass completed 53 of its 100 -households, and 53 of its households are full replays. Qwen3.7-max reached -93/100 in the round before its last, and 93 are full replays. Kimi K2.6's +Kimi K2.6, GLM-5.2, MiniMax M3 and Qwen3.7-max were added on 2026-07-05 to 07. +The run notes (`results/local/openweight_run_20260705.md`) record that +concurrent long-lived requests exhausted the OpenRouter key's connections, so +whole households errored and wrote no rows. `results/local/rerun_openweight.sh` +and `results/local/grind_model.sh` then re-ran each model's full range several +times. Each pass deletes the part files and re-requests every household, and the +cache answered the households earlier passes had completed. The board rows are +each model's final pass. The notes record MiniMax M3's first pass at 53 of 100 +households, and 53 of its households are full replays. They record Qwen3.7-max +at 93/100 in the round before its last, and 93 of its households are full +replays. Kimi K2.6's scenario_109 shows a partial replay at work. In earlier rounds its answer call succeeded and was cached, but its explanation repair timed out at 300 s (`part_84_100.grind*.log`). In the final pass the answer was replayed and only @@ -132,11 +137,11 @@ the repair reached the provider, so the household records 777 s and the replayed answer's id. For these four models the households left after removing replays are the ones -earlier passes failed on, which are the slow ones. Their medians (10.7 to -1,194 s) are not estimates of the model's latency. For OpenRouter responses, -the cache's store time minus the id's creation time approximates a replayed -call's original duration. It agrees with the recorded time to a median of -0.5 s on single-call live households. Applied to the replays it gives medians +earlier passes failed on, many of them after timeouts and retries. Their +medians (10.7 to 1,194 s) are not estimates of the model's latency. For +OpenRouter responses, the cache's store time minus the id's creation time +approximates a call's duration. It agrees with the recorded time to a median of +0.5 s on live households, and it misses badly where a household retried. Applied to the replays it gives medians of about 133 s (Qwen3.7-max), 117 s (GLM-5.2), 294 s (Kimi K2.6) and 10 s (MiniMax M3). It measures only the call that succeeded, not the retries the harness timer would include, so it is a different quantity, and nothing @@ -149,9 +154,10 @@ publishes it. median of household sums with replays counted as calls. Every value reproduces from the frozen predictions except Fable 5's, which is carried from its original rows. For the four open-weight models it publishes 0.003 to - 0.033 seconds, which is cache-lookup time and not latency. For the twelve - other models with full replays it publishes medians lowered by up to 2.5 s - (DeepSeek V4 Pro); partial replays lower them further by an unknown amount. + 0.033 seconds, which is cache-lookup time and not latency. For the eleven + other models with sub-second households it publishes medians lowered by up + to 2.5 s (DeepSeek V4 Pro). Partial replays lower medians further, by an + unknown amount. GPT-5.5 has no `latencySeconds`. - **The leaderboard** shows no latency: the column was dropped on 2026-07-05 (#100), and `latencySeconds` is not rendered anywhere in the app. @@ -163,14 +169,16 @@ publishes it. generation". That fits no model in the board: GPT-5.5's batch rows log no time at all, and the four sub-second models ran synchronously. The paragraph and caption now give the real causes, computed from the frozen data - (`PaperResults.latency_dash_models` and its neighbours). The rounded table - values are unchanged; without sub-second households Claude Sonnet 5 would - read 65 s rather than 64 s and DeepSeek V4 Pro 172 s rather than 170 s. + (`PaperResults.latency_dash_models` and its neighbours). The table values + are unchanged. Without sub-second households, Claude Sonnet 5 would read 65 s + rather than 64 s, DeepSeek V4 Pro 172 s rather than 170 s and GPT-5.6 Luna + 15 s rather than 14 s. - **`costUsd`.** A replayed response keeps the cost recorded when it was generated. Spend ledgers record each cache hit's spend as $0 and the replayed - cost as `cached_response_cost_usd`, and the board row carries the latter - (163 hits). On 43 single-call MiniMax M3 replays the row cost equals the - cost embedded in the cached response exactly. A household's published cost + cost as `cached_response_cost_usd`. On all ten ledger-era full replays (Muse + Spark 1.3, Muse Wonder, Claude Opus 5.5 and Claude Haiku 5.5), the board + household's cost equals that field exactly. On 43 of MiniMax M3's full + replays, the row cost equals the cost embedded in the cached response. A household's published cost therefore counts each of its responses once, whether or not the board run paid for it, and leaves out spend on attempts that did not make the board, such as the open-weight models' failed passes. That matches the paper's @@ -187,10 +195,32 @@ publishes it. ## Changing the published numbers -Excluding replayed households from `latencySeconds`, or publishing none for the -four open-weight models, changes published numbers, so it is a methodology -choice for Max (decision queued with `cos add-decision`). This audit changes -only disclosures, comments and the tests that tie them to the data. +Changing `latencySeconds` changes published numbers, so it is a methodology +choice for Max, queued as a decision. This audit changes only disclosures, +comments, and the tests that tie them to the data. The options: + +- **A. Keep the numbers.** This PR discloses them. The dashboard data keeps + 0.003 to 0.033 s for the four open-weight models. +- **B. Leave replays out of the median (recommended).** `model_cost_latency` + would take the median over households that sum to a second or more. It would + publish no latency when more than half a model's households are replays, + which is the rule the paper's dash already applies (the property test + `test_a_median_under_a_second_means_most_households_replayed`). The four + open-weight models would lose `latencySeconds`. Eleven other medians would + rise by 0.02 to 2.5 s. In the paper table, Claude Sonnet 5 would read 65 s + instead of 64 s, DeepSeek V4 Pro 172 s instead of 170 s and GPT-5.6 Luna 15 s + instead of 14 s, and nothing else would change after rounding. Partial + replays would stay as disclosed lower bounds, because finding them needs + response-id times or the caches, not just household times. +- **C. Reconstruct the replayed calls' durations** from cache store time minus + OpenRouter creation time. That gives about 133, 117, 294 and 10 s for the four + models, but it times only the call that succeeded, not the retries the + harness timer includes. +- **D. Re-time the four models.** Re-send their replayed households live with + the cache off, for latency only, leaving their predictions alone. The full + replays' recorded cost suggests about $47 (Kimi K2.6 $28.38, Qwen3.7-max + $11.99, GLM-5.2 $6.23, MiniMax M3 $0.36). The timings would come + from a later date than the board's answers. ## Reproducing diff --git a/paper/index.qmd b/paper/index.qmd index cdc29b1d..f29b8381 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -1197,7 +1197,7 @@ def cost_latency_caption() -> str: md_table(cost_latency, caption=cost_latency_caption(), label="tbl-us-cost-latency") ``` -Cost preserves each call's recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5's board rows carry no usage or timing fields, because they were rebuilt from the July 1, 2026 release without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (@tbl-model-runs) sends a household's calls one after another, and its household time is their sum. `{python} r.batch_latency_models_fmt` ran through OpenAI's batch API, which records no per-call request time, so it shows a dash. +Cost preserves each call's recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5's board rows carry no usage or timing fields, because they were rebuilt from release dashboard-data-20260701 without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (@tbl-model-runs) sends a household's calls one after another, and its household time is their sum. `{python} r.batch_latency_models_fmt` ran through OpenAI's batch API, which records no per-call request time, so it shows a dash. The harness also caches provider responses on disk, keyed on the request and the prompt contract version. A request identical to an earlier one is answered from the cache in hundredths of a second, and the timer records that as if it were a call. A replayed response keeps the cost recorded when it was generated, so a household's cost counts each of its responses once, whether or not the board run paid for it. `{python} r.replayed_latency_models_fmt` were finished over several passes, each of which replayed the earlier passes' successful responses. Each of them records a replay on `{python} r.replayed_latency_range_fmt` of its `{python} r.n_households_fmt` households, so its median falls under a second and it shows a dash as well. In `{python} r.other_subsecond_model_count_fmt` other models, `{python} r.other_subsecond_household_count` households sum to under a second. The audit traces `{python} r.traced_other_subsecond_count` of them to an earlier request on the same household, such as an onboarding smoke or a restarted chunk; the other `{python} r.untraced_subsecond_count_fmt` come from runs that predate per-call timestamps and left no cache or ledger behind. Dropping them would raise no model's median by more than `{python} r.subsecond_median_shift_fmt` seconds. A further `{python} r.partial_replay_count` households replay their first recorded call and then reach the provider, so their times leave out that first call. The benchmark card summarizes the replays, and the repository's latency replay audit lists each household. Both cost and latency are properties of this run's providers, prompts, and serving configurations at evaluation time rather than fixed model constants. diff --git a/paper/snapshot/20260501/manifest.json b/paper/snapshot/20260501/manifest.json index dbdc04e3..3a0dfcce 100644 --- a/paper/snapshot/20260501/manifest.json +++ b/paper/snapshot/20260501/manifest.json @@ -164,12 +164,12 @@ "rendered_paper_artifacts": { "pdf": { "path": "app/public/paper/policybench.pdf", - "sha256": "d0b236367fb3f30340e4a9ba3d0d25d253b33096a8a789f3d3b00ccd8b4c34ae" + "sha256": "3cb8f835f26018803584989e976c360c6492ad7b4df32d8829179d41ae21c219" }, "web": { "files": { "figures/positive_zero_scatter.png": "47a28fba15b58e86aa00cb52c991a09c5f393c40b9311c24001c201741610062", - "index.html": "533a0cb79b6a54353d73ace8e3ac2d2bf0ef41eff5001b4422876ad8f9a864a0", + "index.html": "031238989dc07e5624cbb38b0d2b8fc399aaddea9e2a724eb4da81291ccdfcc7", "pe-tokens.css": "8f24d8da26f583c8ffddffcdcd172b6d52cbecfec20eda55bd39d7aa829f41d8", "policybench-theme.css": "0e12c5fd615558259e5bce0167a38424e54f9ceb280666c4afd660d759cd1cb9", "site_libs/clipboard/clipboard.min.js": "e17a1d816e13c0826e0ed7febfabc3277f45571234bde0bf9120829a7169edc9", @@ -193,7 +193,7 @@ "The top-level scenario, reference-output, and impact-summary CSVs are byte-identical to the corresponding compact source-run artifacts copied under paper/snapshot/20260501/runs/.", "Model responses were collected in waves between June 12 and October 13, 2026, as models were added to the board; each model's full 100-household run is a single consistent wave. PolicyBench computes each scored reference output with policyengine_us.Simulation from policyengine-us 2.41.0, using the household's own listed inputs, the nine publication conventions, the Maryland output-scope adapter and the scenario builder's stated-hours alias, as the reference sidecar's engine_upgrade revision pins them (fix_modules, builder); policyengine.py 6.1.2 is recorded for provenance only. The households were drawn from the certified PolicyEngine US populace dataset (populace-us-2024-5da5a95-20260611, populace_us_2024), as scenarios.csv.meta.json records (household_dataset); the reference_output_refresh dataset fields name the reference runtime's default dataset, which reference computation does not read.", "Canonical prediction files include parser recovery. Later waves ran under the resumable supervised runner, which retries failed or timed-out scenarios in bounded rounds; every model's canonical file covers all 100 households.", - "Raw provider responses are retained in the compressed source-run predictions.csv.gz file wherever the transport exposed them; rows served through the Anthropic batch adapter (Claude Fable 5) and 64 Kimi K3 parse failures carry no raw payload. The separate LiteLLM cache remains local-only because it is a generated request cache, not the canonical snapshot artifact.", + "Raw provider responses are retained in the compressed source-run predictions.csv.gz file wherever the transport exposed them; Claude Fable 5's rows, rebuilt from release dashboard-data-20260701 without their raw payloads (that release's claude-fable-5-predictions.csv.gz asset keeps them), and 64 Kimi K3 parse failures carry none. The separate LiteLLM cache remains local-only because it is a generated request cache, not the canonical snapshot artifact.", "The frozen scenarios.csv source_dataset column carries a stale enhanced_cps_2024 label from the pre-#77 scenario generator; the run metadata (scenarios.csv.meta.json) records the populace_us_2024 build actually loaded.", "Model APIs and upstream model aliases may change after the recorded 2026-06-12 to 2026-10-13 response window, so exact reruns can diverge even with the committed household inputs, reference outputs, parsed dashboard export, and analysis summaries." ], diff --git a/scripts/audit_latency_replays.py b/scripts/audit_latency_replays.py index 26bb1ecd..932340a7 100644 --- a/scripts/audit_latency_replays.py +++ b/scripts/audit_latency_replays.py @@ -21,7 +21,7 @@ Two kinds of evidence need only the frozen predictions. OpenRouter ids (``gen--...``) and OpenAI Responses ids (``resp_`` + 16 hex + ``00`` + 8 hex seconds) embed their creation time. On live households that -time falls within a few seconds of the row's ``request_started_at``. A +time falls within 23 seconds of the row's ``request_started_at``. A creation time more than a minute before the request began means the response already existed when the request was sent. @@ -61,7 +61,8 @@ ) OUTPUT = ROOT / "docs" / "latency_replay_audit" / "households.csv" # A response that existed this long before the request began was not generated -# for it. Live households sit within 4 seconds of their request start. +# for it. On live households an id's creation time sits within 23 seconds of +# the request start (OpenRouter; OpenAI ids within 4). MARGIN_SECONDS = 60.0 # Every replayed household in the frozen run sums to under 0.12 s; the fastest # live household (Gemini 3.5 Flash-Lite) takes 1.75 s. @@ -134,10 +135,26 @@ def cache_store_times(cache_dirs: list[Path]) -> dict[str, float]: return stored +# Trees that hold no run artifacts, skipped so a walk over a large review +# folder stays quick. +SKIP_DIRS = frozenset( + {".git", ".venv", "venv", "node_modules", "site-packages", "__pycache__"} +) + + +def walk_files(root: Path, suffix: str): + """Files under ``root`` ending in ``suffix``, skipping SKIP_DIRS.""" + for directory, subdirs, names in os.walk(root): + subdirs[:] = [name for name in subdirs if name not in SKIP_DIRS] + for name in names: + if name.endswith(suffix): + yield Path(directory) / name + + def ledger_records(roots: list[Path]) -> pd.DataFrame: records = [] for root in roots: - for path in root.rglob("*.spend.jsonl"): + for path in walk_files(root, ".spend.jsonl"): for line in path.read_text().splitlines(): try: record = json.loads(line) @@ -151,7 +168,7 @@ def ledger_records(roots: list[Path]) -> pd.DataFrame: def smoke_file_ids(roots: list[Path]) -> set[str]: ids: set[str] = set() for root in roots: - for path in root.rglob("*.csv"): + for path in walk_files(root, ".csv"): if "smoke" not in str(path): continue try: diff --git a/scripts/finish_argon.py b/scripts/finish_argon.py index 3c5027df..c426ebbb 100644 --- a/scripts/finish_argon.py +++ b/scripts/finish_argon.py @@ -212,8 +212,10 @@ INPUT_PINS = ROOT / INPUT_PINS_PATH PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: -# Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# Fable 5's board rows were rebuilt from release 20260701 without their cost, +# token and latency fields (its original per-call rows are that release's +# claude-fable-5-predictions.csv.gz), so export_full_run reports $0 and +# omits the rest. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_gpt61sol.py b/scripts/finish_gpt61sol.py index 982ae6f6..1d312c59 100644 --- a/scripts/finish_gpt61sol.py +++ b/scripts/finish_gpt61sol.py @@ -174,8 +174,10 @@ INPUT_PINS = ROOT / INPUT_PINS_PATH PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: -# Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# Fable 5's board rows were rebuilt from release 20260701 without their cost, +# token and latency fields (its original per-call rows are that release's +# claude-fable-5-predictions.csv.gz), so export_full_run reports $0 and +# omits the rest. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_haiku55.py b/scripts/finish_haiku55.py index 233f09ec..b1c081ae 100644 --- a/scripts/finish_haiku55.py +++ b/scripts/finish_haiku55.py @@ -192,8 +192,10 @@ INPUT_PINS = ROOT / INPUT_PINS_PATH PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: -# Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# Fable 5's board rows were rebuilt from release 20260701 without their cost, +# token and latency fields (its original per-call rows are that release's +# claude-fable-5-predictions.csv.gz), so export_full_run reports $0 and +# omits the rest. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/finish_musewonder.py b/scripts/finish_musewonder.py index 16fe4e99..8db8d063 100644 --- a/scripts/finish_musewonder.py +++ b/scripts/finish_musewonder.py @@ -202,8 +202,10 @@ INPUT_PINS = ROOT / INPUT_PINS_PATH PINNED_INPUTS = ("predictions.csv", "run_state.json") # Incumbent usage the exporter cannot recompute from committed predictions: -# Fable 5 ran through the Anthropic batch adapter, and its rows carry no cost, -# token or latency fields, so export_full_run reports $0 and omits the rest. +# Fable 5's board rows were rebuilt from release 20260701 without their cost, +# token and latency fields (its original per-call rows are that release's +# claude-fable-5-predictions.csv.gz), so export_full_run reports $0 and +# omits the rest. # The released values are carried over; the drift gate then checks every field. CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/scripts/freeze_snapshot.py b/scripts/freeze_snapshot.py index 329caefa..320c9a03 100644 --- a/scripts/freeze_snapshot.py +++ b/scripts/freeze_snapshot.py @@ -1823,9 +1823,11 @@ def build_manifest( "failed or timed-out scenarios in bounded rounds; every model's " "canonical file covers all 100 households.", "Raw provider responses are retained in the compressed source-run " - "predictions.csv.gz file wherever the transport exposed them; rows " - "served through the Anthropic batch adapter (Claude Fable 5) and " - "64 Kimi K3 parse failures carry no raw payload. The separate " + "predictions.csv.gz file wherever the transport exposed them; " + "Claude Fable 5's rows, rebuilt from release dashboard-data-20260701 " + "without their raw payloads (that release's " + "claude-fable-5-predictions.csv.gz asset keeps them), and 64 Kimi " + "K3 parse failures carry none. The separate " "LiteLLM cache remains " "local-only because it is a generated request cache, not the " "canonical snapshot artifact.", diff --git a/scripts/release_20261006.py b/scripts/release_20261006.py index 42b4ab94..a7f8e078 100644 --- a/scripts/release_20261006.py +++ b/scripts/release_20261006.py @@ -86,8 +86,8 @@ STAGE_ROOT = Path("results/local") BOARD_MODELS = 46 TOTAL_OUTPUTS = 1984 -# Fable 5 ran through the Anthropic batch adapter, whose rows carry no cost, -# token or latency fields; release 20260930 carried its released values, and +# Fable 5's board rows were rebuilt from release 20260701 without their cost, +# token and latency fields; release 20260930 carried its released values, and # so does this release (scripts/finish_gpt61sol.py, CARRIED_USAGE). CARRIED_USAGE = { "claude-fable-5": ("costUsd", "costPerHousehold", "totalTokens", "latencySeconds") diff --git a/tests/test_cost_latency.py b/tests/test_cost_latency.py index df3b8dfb..cad952a4 100644 --- a/tests/test_cost_latency.py +++ b/tests/test_cost_latency.py @@ -92,3 +92,120 @@ def test_all_null_latency_is_omitted_not_zero(): def test_empty_predictions_returns_empty(): assert model_cost_latency(pd.DataFrame()) == {} + + +# ----- invariants of household latency (docs/latency_replay_audit.md) ------- + +from hypothesis import given, settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +from policybench.analysis import ( # noqa: E402 + REPLAYED_HOUSEHOLD_SECONDS, + household_latency_seconds, +) + +# A household's calls: each call's latency and how many rows it answered. +CALLS = st.lists( + st.tuples( + st.floats(min_value=0.0, max_value=2_000.0, allow_nan=False), + st.integers(min_value=1, max_value=8), + ), + min_size=1, + max_size=6, +) +HOUSEHOLDS = st.lists(CALLS, min_size=1, max_size=15) + + +def _split_rows(households): + """Rows the way the harness writes them: each call's latency divided + equally among the rows it answered.""" + rows = [] + for index, calls in enumerate(households): + for seconds, n_rows in calls: + rows += [_row("m", f"s{index:03d}", 0.01, seconds / n_rows)] * n_rows + return pd.DataFrame(rows) + + +@given(HOUSEHOLDS) +@settings(max_examples=200, deadline=None) +def test_household_latency_is_the_sum_of_its_calls(households): + """Splitting a call's latency across its rows and summing the rows gives + back the household's total request time.""" + latency = household_latency_seconds(_split_rows(households)) + for index, calls in enumerate(households): + expected = sum(seconds for seconds, _ in calls) + assert latency[("m", f"s{index:03d}")] == pytest.approx(expected, abs=1e-9) + + +@given(HOUSEHOLDS) +@settings(max_examples=200, deadline=None) +def test_published_latency_is_the_median_household(households): + preds = _split_rows(households) + expected = float(np.median([sum(s for s, _ in calls) for calls in households])) + out = model_cost_latency(preds) + assert out["m"]["latencySeconds"] == pytest.approx(expected, abs=1e-6) + # Cost is additive the same way: the run total is every row's cost. + assert out["m"]["costUsd"] == pytest.approx(0.01 * len(preds)) + + +@given( + st.lists( + st.floats(min_value=0.0, max_value=2_000.0, allow_nan=False), + min_size=1, + max_size=20, + ), + st.data(), +) +@settings(max_examples=200, deadline=None) +def test_a_replay_can_only_lower_the_median(seconds, data): + """Replacing a household's time with a cache lookup's (smaller) time + never raises the published median, so replays bias it down, never up.""" + index = data.draw(st.integers(min_value=0, max_value=len(seconds) - 1)) + lookup = data.draw(st.floats(min_value=0.0, max_value=seconds[index])) + before = model_cost_latency( + pd.DataFrame([_row("m", f"s{i}", 0.01, t) for i, t in enumerate(seconds)]) + )["m"]["latencySeconds"] + replayed = list(seconds) + replayed[index] = lookup + after = model_cost_latency( + pd.DataFrame([_row("m", f"s{i}", 0.01, t) for i, t in enumerate(replayed)]) + )["m"]["latencySeconds"] + assert after <= before + 1e-9 + + +@given( + st.integers(min_value=0, max_value=12), + st.integers(min_value=0, max_value=12), +) +def test_a_median_under_a_second_means_most_households_replayed(n_fast, n_slow): + """With replays under the line and calls above it, the median is under the + line when more than half the households replayed and at or above it when + fewer than half did: the paper's dash rule is a majority-replay rule.""" + if n_fast + n_slow == 0: + return + seconds = [0.02] * n_fast + [30.0] * n_slow + median = model_cost_latency( + pd.DataFrame([_row("m", f"s{i}", 0.01, t) for i, t in enumerate(seconds)]) + )["m"]["latencySeconds"] + if 2 * n_fast > n_fast + n_slow: + assert median < REPLAYED_HOUSEHOLD_SECONDS + if 2 * n_fast < n_fast + n_slow: + assert median >= REPLAYED_HOUSEHOLD_SECONDS + + +@given( + st.lists(st.floats(min_value=1.0, max_value=500.0), min_size=1, max_size=10), + st.integers(min_value=1, max_value=10), +) +def test_untimed_households_are_left_out_not_counted_as_zero(seconds, n_untimed): + timed = [_row("m", f"t{i}", 0.01, t) for i, t in enumerate(seconds)] + untimed = [_row("m", f"u{i}", 0.01, np.nan) for i in range(n_untimed)] + out = model_cost_latency(pd.DataFrame(timed + untimed)) + assert out["m"]["latencySeconds"] == pytest.approx(float(np.median(seconds))) + latency = household_latency_seconds(pd.DataFrame(untimed)) + assert latency.isna().all() + + +def test_household_latency_of_nothing_is_empty(): + assert household_latency_seconds(pd.DataFrame()).empty + assert household_latency_seconds(pd.DataFrame([{"model": "m"}])).empty diff --git a/tests/test_disclosures.py b/tests/test_disclosures.py index 37fe51e9..560082c2 100644 --- a/tests/test_disclosures.py +++ b/tests/test_disclosures.py @@ -66,6 +66,15 @@ ) PAPER_FALSE_CLAIM_PATTERNS = ( + # No board model logged submission times: GPT-5.5's batch rows log none, + # and the sub-second medians are cache replays of synchronous calls. + # Claude Fable 5 ran synchronously, and its usage was recorded per call + # (docs/latency_replay_audit.md). + r"batch\s+or\s+asynchronous\s+dispatch", + r"reflect\s+submission\s+rather\s+than\s+generation", + r"recorded\s+at\s+the\s+run\s+level", + r"Anthropic\s+batch\s+adapter", + r"not\s+measured\s+per\s+call", # Exclusion covers what the audits found, not every defect or unstated # input: an output a fix or reading not run on the reference engine would # move can still be scored (the paper's limitations say so). @@ -1165,65 +1174,54 @@ def names(model: str, sentence: str) -> bool: assert sorted(m for m in serving if names(m, note_choice)) == sorted(card_choice) -# The benchmark card's cache-replay examples ("Cost basis"): household -# latency is the sum of its rows' elapsed_seconds, since each row records its -# call's latency divided among the call's outputs. -REPLAYED_SMOKE_HOUSEHOLDS = { - ("muse-spark-1.3", "scenario_000"), - ("muse-spark-1.3", "scenario_001"), - ("muse-spark-1.3", "scenario_099"), - ("muse-spark-1.3", "scenario_119"), -} -NEAR_ZERO_EXAMPLES = { - ("muse-wonder", "scenario_000"), - ("muse-wonder", "scenario_001"), - ("claude-opus-5.5", "scenario_000"), - ("claude-opus-5.5", "scenario_001"), - ("claude-haiku-5.5", "scenario_000"), -} - +# The benchmark card's cache-replay accounting ("Latency basis"), checked +# against the frozen predictions; tests/test_latency_replay_audit.py checks the +# audit's household list itself. +def test_card_latency_basis_matches_the_frozen_households(): + """Every count the card states is the frozen data's: the sub-second + households, the traced and untraced ones, the per-model replays, and the + largest median shift.""" + from policybench.analysis import REPLAYED_HOUSEHOLD_SECONDS -def _household_latency() -> dict[tuple[str, str], tuple[float, int]]: - """(model, scenario_id) -> (summed row latency in seconds, row count), - from the frozen predictions.""" - import pandas as pd - - manifest = json.loads(SNAPSHOT_MANIFEST.read_text()) - run_dir = ( - ROOT - / manifest["source_run_artifacts"][manifest["source_run_labels"]["us"]]["path"] - ) - predictions = pd.read_csv( - run_dir / "predictions.csv.gz", - usecols=["model", "scenario_id", "elapsed_seconds"], - ) - grouped = predictions.groupby(["model", "scenario_id"])["elapsed_seconds"] - sums, counts = grouped.sum(min_count=1), grouped.size() - return {key: (float(sums[key]), int(counts[key])) for key in sums.index} - - -def test_cache_replay_examples_match_the_frozen_household_latencies(): - """The card's replayed smoke households take 0.03 seconds or less, and - each other example it names sums to under 0.05 seconds; every example is - a household the card names, with recorded (not missing) latency.""" text = re.sub(r"\s+", " ", BENCHMARK_CARD.read_text()) + latency = r.household_latencies + subsecond = latency[latency < REPLAYED_HOUSEHOLD_SECONDS] + audit = r.latency_replay_audit + full = audit[audit["replay"] == "full"].groupby("model").size() + partial = audit[audit["replay"] == "partial"].groupby("model").size() + assert "a household's latency is the sum over its rows" in text + assert f"{len(subsecond)} households sum to under a second" in text + assert f"traces {int(full.sum())} of them" in text + assert f"Another {int(partial.sum())} households replay their first" in text + untraced = audit[audit["replay"] == "untraced"] + assert sorted(zip(untraced["model"], untraced["scenario_id"])) == [ + ("claude-haiku-4.5", "scenario_009"), + ("grok-4.3", "scenario_000"), + ] + assert "(Grok 4.3's scenario_000 and Claude Haiku 4.5's scenario_009)" in text + for model, label in ( + ("qwen-3.7-max", "Qwen3.7-max"), + ("glm-5.2", "GLM-5.2"), + ("kimi-k2.6", "Kimi K2.6"), + ("minimax-m3", "MiniMax M3"), + ("claude-sonnet-5", "Claude Sonnet 5"), + ("muse-spark-1.3", "Muse Spark 1.3"), + ): + assert f"{label} ({int(full[model])}" in text, model + for model in ("muse-wonder", "claude-opus-5.5", "claude-haiku-5.5"): + assert int(full[model]) == 2, model + for model in ("deepseek-v4-pro", "gpt-5.6-sol", "gpt-5.6-luna", "gpt-5.6-terra"): + assert int(full[model]) == 2, model + for model, label in ( + ("glm-5.3", "GLM-5.3"), + ("ox-alpha", "GLM-5.3-Flash (preview)"), + ): + assert f"{label} ({int(partial[model])})" in text, model assert ( - "The Muse Spark 1.3 row replays its four smoke households (scenario_000, " - "scenario_001, scenario_099 and scenario_119)" - ) in text - assert "a household latency of 0.03 seconds or less" in text - assert ( - "among them scenario_000 and scenario_001 for Muse Wonder and Claude " - "Opus 5.5 and scenario_000 for Claude Haiku 5.5." - ) in text - latency = _household_latency() - for key in REPLAYED_SMOKE_HOUSEHOLDS: - seconds, rows = latency[key] - assert rows > 1 and 0 < seconds <= 0.03, (key, seconds) - for key in NEAR_ZERO_EXAMPLES: - seconds, rows = latency[key] - assert rows > 1 and 0 < seconds < 0.05, (key, seconds) - # Round 1's wrong example: the household is slower once its rows are - # summed, so the card must not name it. - assert latency[("claude-haiku-5.5", "scenario_001")][0] >= 0.05 + f"no published median by more than {r.subsecond_median_shift_fmt} seconds" + in text + ) + published = {row["model"]: row.get("latencySeconds") for row in r.model_stats} + replayed = [published[m] for m in r.replayed_latency_models] + assert f"{min(replayed):.3f} to {max(replayed):.3f} seconds" in text diff --git a/tests/test_latency_replay_audit.py b/tests/test_latency_replay_audit.py new file mode 100644 index 00000000..16e8eda1 --- /dev/null +++ b/tests/test_latency_replay_audit.py @@ -0,0 +1,314 @@ +"""The latency replay audit agrees with the frozen predictions. + +``docs/latency_replay_audit/households.csv`` was built from artifacts outside +the repository (disk caches, spend ledgers, smoke files). These tests check +everything about it that the frozen predictions alone can check: that each +row is the household it names, that the sub-second households are exactly the +full replays plus the untraced ones, and that the id-time evidence recomputes. +They also pin the facts the paper and card state about the untimed models. +""" + +from __future__ import annotations + +import gzip +import json +import re +from pathlib import Path + +import pandas as pd +import pytest + +from policybench.analysis import ( + REPLAYED_HOUSEHOLD_SECONDS, + household_latency_seconds, + model_cost_latency, +) +from policybench.config import PRICE_OVERRIDES_PER_1M +from policybench.paper_results import LATENCY_REPLAY_AUDIT, r +from scripts.audit_latency_replays import ( + MARGIN_SECONDS, + id_created_at, + provider_response_id, +) + +ROOT = Path(__file__).resolve().parents[1] +RUN_DIR = ( + ROOT + / "paper" + / "snapshot" + / "20260501" + / "runs" + / "us_full_run_20260612_policyengine_4_16_1_populace" +) +AUDIT_DOC = ROOT / "docs" / "latency_replay_audit.md" +MANIFEST = ROOT / "paper" / "snapshot" / "20260501" / "manifest.json" +# Fable 5's usage, carried from its original run's per-call rows (release +# dashboard-data-20260701, claude-fable-5-predictions.csv.gz). +FABLE_CARRIED = { + "costUsd": 54.109099999999955, + "costPerHousehold": 0.5410909999999995, + "totalTokens": 3850174, + "latencySeconds": 97.6780539804895, +} + + +@pytest.fixture(scope="module") +def predictions() -> pd.DataFrame: + return pd.read_csv(RUN_DIR / "predictions.csv.gz", low_memory=False) + + +@pytest.fixture(scope="module") +def households(predictions) -> pd.DataFrame: + grouped = predictions.groupby(["model", "scenario_id"], sort=True) + return pd.DataFrame( + { + "latency": household_latency_seconds(predictions), + "first_response_id": grouped["provider_response_id"].first(), + "started": grouped["request_started_at"].min(), + } + ) + + +@pytest.fixture(scope="module") +def audit() -> pd.DataFrame: + # An untraced household cites no evidence: an empty field, not NaN. + return pd.read_csv(LATENCY_REPLAY_AUDIT).fillna({"evidence": ""}) + + +@pytest.fixture(scope="module") +def model_stats() -> dict[str, dict]: + payload = json.loads(gzip.open(RUN_DIR / "data.json.gz").read()) + return {row["model"]: row for row in payload["modelStats"]} + + +def test_each_audit_row_is_the_household_it_names(audit, households): + assert not audit.duplicated(["model", "scenario_id"]).any() + for row in audit.itertuples(): + household = households.loc[(row.model, row.scenario_id)] + assert row.household_latency_seconds == pytest.approx( + household["latency"], abs=1e-6 + ), row + assert row.first_response_id == household["first_response_id"], row + + +def test_subsecond_households_are_the_full_replays_and_the_untraced(audit, households): + """A full replay sums to under a second and a partial one does not, and no + sub-second household is missing from the audit.""" + subsecond = set( + households.index[households["latency"] < REPLAYED_HOUSEHOLD_SECONDS] + ) + audited = { + (m, s): kind for m, s, kind in audit[["model", "scenario_id", "replay"]].values + } + assert subsecond == {key for key, kind in audited.items() if kind != "partial"} + for (model, scenario), kind in audited.items(): + seconds = households.loc[(model, scenario), "latency"] + if kind == "partial": + assert seconds >= REPLAYED_HOUSEHOLD_SECONDS, (model, scenario) + else: + assert seconds < REPLAYED_HOUSEHOLD_SECONDS, (model, scenario) + + +def test_replays_and_live_households_are_far_apart(households, audit): + """Every full replay sums to under 0.12 s and the fastest household the + audit does not list takes over 1.5 s, so the one-second line is not close + to any household.""" + full = audit[audit["replay"] == "full"] + assert full["household_latency_seconds"].max() < 0.12 + listed = set(zip(audit["model"], audit["scenario_id"])) + unlisted = households[[key not in listed for key in households.index]] + assert unlisted["latency"].dropna().min() > 1.5 + + +def test_id_time_evidence_recomputes_and_is_complete(audit, households): + """Where an id embeds its creation time, the audit cites it exactly when + the response predates its request by more than the margin; on every other + household that time sits within the margin of the request.""" + cited = { + (m, s) + for m, s, evidence in audit[["model", "scenario_id", "evidence"]].values + if re.search(r"openrouter_id|openai_id", evidence) + } + checked = 0 + for (model, scenario), household in households.iterrows(): + created = id_created_at(provider_response_id(household["first_response_id"])) + if created is None or pd.isna(household["started"]): + continue + checked += 1 + early = created < household["started"] - MARGIN_SECONDS + assert early == ((model, scenario) in cited), (model, scenario) + if not early: + assert abs(household["started"] - created) <= MARGIN_SECONDS + assert checked > 1000 + + +def test_the_untraced_households_predate_per_call_timestamps(audit, predictions): + untraced = audit[audit["replay"] == "untraced"] + assert sorted(untraced["model"]) == ["claude-haiku-4.5", "grok-4.3"] + for model in untraced["model"]: + rows = predictions[predictions["model"] == model] + assert rows["request_started_at"].isna().all(), model + + +def test_the_audit_doc_counts_are_the_csvs(audit): + text = re.sub(r"\s+", " ", AUDIT_DOC.read_text()) + full = int((audit["replay"] == "full").sum()) + partial = int((audit["replay"] == "partial").sum()) + replayed_models = audit.loc[audit["replay"] != "untraced", "model"].nunique() + assert ( + f"That is {full} full and {partial} partial replays across " + f"{replayed_models} models." + ) in text + for line in AUDIT_DOC.read_text().splitlines(): + cells = [cell.strip() for cell in line.strip("|").split("|")] + if len(cells) != 6 or not cells[1].isdigit(): + continue + name, n_full, n_partial = cells[0], int(cells[1]), int(cells[2]) + model = next(m for m in audit["model"].unique() if r.model_name(m) == name) + rows = audit[audit["model"] == model] + assert n_full == int((rows["replay"] == "full").sum()), name + assert n_partial == int((rows["replay"] == "partial").sum()), name + + +def test_the_audit_doc_medians_are_the_frozen_ones(households, model_stats): + """The table's published median is modelStats', and its other median + drops the model's sub-second households.""" + for line in AUDIT_DOC.read_text().splitlines(): + cells = [cell.strip() for cell in line.strip("|").split("|")] + if len(cells) != 6 or not cells[1].isdigit(): + continue + model = next(m for m in model_stats if r.model_name(m) == cells[0]) + published = float(cells[4].replace(",", "")) + assert published == pytest.approx( + model_stats[model]["latencySeconds"], abs=0.051 + ), model + latency = households.loc[model, "latency"].dropna() + kept = latency[latency >= REPLAYED_HOUSEHOLD_SECONDS] + without = float(cells[5].split(" ")[0].replace(",", "")) + assert without == pytest.approx(kept.median(), abs=0.051), model + if "households" in cells[5]: + assert f"({len(kept)} households)" in cells[5], model + + +def test_published_cost_and_latency_reproduce_except_fables_carried_usage( + predictions, model_stats +): + """modelStats' cost and latency are model_cost_latency's over the frozen + rows; the only exceptions are Claude Fable 5's four carried fields.""" + computed = model_cost_latency(predictions, PRICE_OVERRIDES_PER_1M) + for model, row in model_stats.items(): + for field in ("costUsd", "costPerHousehold", "latencySeconds", "totalTokens"): + if model == "claude-fable-5" and field in FABLE_CARRIED: + assert row[field] == FABLE_CARRIED[field] + continue + expected = computed.get(model, {}).get(field) + if expected is None: + assert row.get(field) is None, (model, field) + else: + assert row[field] == pytest.approx(expected, rel=1e-12), (model, field) + + +def test_the_untimed_models_are_the_batch_run_and_fables_rebuilt_rows(predictions): + """GPT-5.5's batch rows have costs but no times; Fable 5's rebuilt rows + have neither, nor raw payloads, but keep their explanations.""" + assert r.untimed_latency_models == ["claude-fable-5", "gpt-5.5"] + gpt = predictions[predictions["model"] == "gpt-5.5"] + assert gpt["request_started_at"].isna().all() + assert gpt["total_cost_usd"].notna().all() + fable = predictions[predictions["model"] == "claude-fable-5"] + for column in ("elapsed_seconds", "total_cost_usd", "total_tokens", "raw_response"): + assert fable[column].isna().all(), column + assert fable["explanation"].notna().all() + + +def test_the_paper_dashes_exactly_the_untimed_and_replayed_medians(model_stats): + assert r.latency_dash_models == [ + "glm-5.2", + "gpt-5.5", + "kimi-k2.6", + "minimax-m3", + "qwen-3.7-max", + ] + assert r.replayed_latency_models == [ + "glm-5.2", + "kimi-k2.6", + "minimax-m3", + "qwen-3.7-max", + ] + for model in r.replayed_latency_models: + assert model_stats[model]["latencySeconds"] < 0.05, model + assert r.zero_cost_models == ["ox-alpha"] + + +def test_the_manifest_no_longer_says_fable_ran_through_the_batch_adapter(): + notes = " ".join(json.loads(MANIFEST.read_text())["reproducibility_notes"]) + assert "Anthropic batch adapter" not in notes + assert "rebuilt from release dashboard-data-20260701" in notes + + +PAPER_HTML = ROOT / "app" / "public" / "paper" / "web" / "index.html" + + +def _cost_latency_rows() -> list[list[str]]: + """The rendered cost-and-latency table's rows, as visible cell text.""" + from html import unescape + + page = PAPER_HTML.read_text() + header = page.index("Latency (median)") + table = page[page.rfind("", header)] + rows = [] + for row in re.findall(r"]*>(.*?)", table, re.S): + cells = re.findall(r"]*>(.*?)", row, re.S) + rows.append( + [ + unescape(re.sub(r"<[^>]+>", "", cell)).replace("​", "").strip() + for cell in cells + ] + ) + return rows + + +def _visible_text() -> str: + from html import unescape + + text = re.sub(r"<[^>]+>", " ", PAPER_HTML.read_text()) + text = re.sub(r"\s+", " ", unescape(text)).replace("​", "") + # Quarto renders apostrophes as typographic quotes. + return text.replace("’", "'").replace("‘", "'") + + +def test_every_dash_in_the_rendered_table_is_one_the_text_explains(): + header, *rows = _cost_latency_rows() + assert header == ["Model", "Exact (%)", "Cost / household", "Latency (median)"] + assert len(rows) == len(r.model_stats) + latency_dash = sorted(row[0] for row in rows if row[3] == "—") + cost_dash = sorted(row[0] for row in rows if row[2] == "—") + assert latency_dash == sorted(r.model_name(m) for m in r.latency_dash_models) + assert cost_dash == sorted(r.model_name(m) for m in r.zero_cost_models) + # Fable 5's untimed rows are not dashed: its carried per-call median shows. + fable = next(row for row in rows if row[0] == r.model_name("claude-fable-5")) + assert fable[3] == f"{round(FABLE_CARRIED['latencySeconds'])} s" + + +def test_the_rendered_paper_states_why_each_latency_is_missing(): + text = _visible_text() + assert ( + f"{r.batch_latency_models_fmt} ran through OpenAI's batch API, which " + "records no per-call request time, so it shows a dash." in text + ) + assert ( + f"{r.replayed_latency_models_fmt} were finished over several passes, " + "each of which replayed the earlier passes' successful responses." in text + ) + assert ( + f"records a replay on {r.replayed_latency_range_fmt} of its " + f"{r.n_households_fmt} households" in text + ) + assert ( + f"In {r.other_subsecond_model_count_fmt} other models, " + f"{r.other_subsecond_household_count} households sum to under a second." in text + ) + assert f"The audit traces {r.traced_other_subsecond_count} of them" in text + assert f"more than {r.subsecond_median_shift_fmt} seconds" in text + assert f"A further {r.partial_replay_count} households replay" in text + assert "Claude Fable 5's board rows carry no usage or timing fields" in text From 803b49edeae1f2ea4a0e7473517d37fd44d00977 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 07:52:41 -0400 Subject: [PATCH 3/4] Keep the cost-basis caption's phrases contiguous in the paper source tests/test_paper_results.py reads the qmd source for the cost-basis wording, so the caption's static text stays one literal; the rendered caption is unchanged. Co-Authored-By: Claude Opus 5.5 --- paper/index.qmd | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/paper/index.qmd b/paper/index.qmd index f29b8381..1fc8f703 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -1178,19 +1178,10 @@ def cost_latency_caption() -> str: assert sorted(latency_dash) == r.latency_dash_models, latency_dash zero_cost = r.zero_cost_models_fmt return ( - "Estimated cost and median latency per household for the frozen US snapshot, " - "alongside the headline exact-match rate. Each row uses its recorded per-call " - "cost: reconstructed from token counts at the list price configured at request " - "time, or the provider-reported charge where no reconstruction was available. " - "List-price overrides apply at request time, not retroactively to recorded " - "costs. Claude Fable 5's cost and latency come from its original run's per-call " - "rows, which its board rows do not carry. Latency is the median across " - "households of each household's summed request time. A cost dash marks a " - f"provider-reported charge of zero on every call ({zero_cost}). A latency dash " - f"marks rows that record no request time ({r.batch_latency_models_fmt}, through " - "OpenAI's batch API) or a median household answered from the harness's " - f"response cache rather than by the provider ({r.replayed_latency_models_fmt}; " - "see text)." + "Estimated cost and median latency per household for the frozen US snapshot, alongside the headline exact-match rate. Each row uses its recorded per-call cost: reconstructed from token counts at the list price configured at request time, or the provider-reported charge where no reconstruction was available. List-price overrides apply at request time, not retroactively to recorded costs. Claude Fable 5's cost and latency come from its original run's per-call rows, which its board rows do not carry. Latency is the median across households of each household's summed request time. " + f"A cost dash marks a provider-reported charge of zero on every call ({zero_cost}). " + f"A latency dash marks rows that record no request time ({r.batch_latency_models_fmt}, through OpenAI's batch API) " + f"or a median household answered from the harness's response cache rather than by the provider ({r.replayed_latency_models_fmt}; see text)." ) From f1caa705ed61ec9898cff079bc9b09d1d5790410 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 14 Oct 2026 09:19:05 -0400 Subject: [PATCH 4/4] Answer review round 1: request-specific evidence, cost and retry wording - The audit script ties each kind of evidence to the frozen request it explains. A ledger cache hit must be on the same model and household at the request's own start time; smoke ledgers, smoke files and the cache must predate the request; smoke files are read by column name; "smoke" is judged below each root. Unit tests cover each rule. The regenerated CSV is unchanged except that Claude Haiku 5.5's two smoke replays add smoke_ledger evidence. - A replayed row's cost is computed when the row is written from the replayed response itself, not "kept from when it was generated"; the ledger agreement is no longer cited as independent evidence. - The harness restarts its timer on each retry and times only the attempt that succeeded; the paper, card and audit doc say so, and the old phrases are now false-claim patterns. - The majority-replay property is qualified at exact ties, with a tie test; the median-shift bound rounds up. Paper re-rendered and re-pinned; only the two corrected sentences change. Co-Authored-By: Claude Opus 5.5 --- app/public/paper/policybench.pdf | Bin 262801 -> 263075 bytes app/public/paper/web/index.html | 8 +- app/src/paperSnapshot.json | 4 +- docs/benchmark_card.md | 13 +- docs/latency_replay_audit.md | 51 ++++--- docs/latency_replay_audit/households.csv | 4 +- paper/index.qmd | 4 +- paper/snapshot/20260501/manifest.json | 4 +- policybench/paper_results.py | 6 +- scripts/audit_latency_replays.py | 164 +++++++++++++++-------- tests/test_cost_latency.py | 22 ++- tests/test_disclosures.py | 4 + tests/test_latency_replay_audit.py | 125 +++++++++++++++++ 13 files changed, 315 insertions(+), 94 deletions(-) diff --git a/app/public/paper/policybench.pdf b/app/public/paper/policybench.pdf index a61214297baefa9d81bac525f3b932365762f08e..d577f1f87b7116a075fcee8a60f041e07c20be6a 100644 GIT binary patch delta 6927 zcmV+q8}Q_jg%G2M5P-A+e4+z0HZ+kz6n~u?$&MSj_3p3G517M!0fYv+EcXC82EGNk zW=?a-{~uCZ-bS({IWXE%6}QK?Z_>BFZu<7;H$6U&U*G=t3gu?BD*F6)`}Lo1=hy7c zQGGMF?354TmUL0to7=CyZ~wY$?Y^`QM(g-E|Lpl}>7CD7+pmxB&gFBa`EkB^z<--^ zzKLbLjUTo?*R}q^=SR!u)_t9Yaewp`_I&Ft><{znlRaldJ>avjTV8gn^TRowO8tX3 zuW#1hhy3x~2UB;=^RD%r3LC&0H9}BVdcq3bwC(>8yXd|;937grQ~fIwuKxY&AK#49 zCP#DvkR1^YP-`1mY(K$1_?tHW==0-) zh}JH8z|29y4nrhJ<;=wH&@NtJJ-n&+3I5}Ie`y(p1wa*tbWIrDZ(V_J;N||;sNQg& z44`xHpItai09O(1ITbtIPEjj%tm~4Y?*v1lGY8eH_8Z6rMQSFZ2CuS5qRu)Y5}4z1=_j#| z16$#%t?)^X04lnmt@y9w4dn&i)Y(l$53(c`vCZ;1U}TtmZ4UtjKz|re{-kiZ4RXh1 z(ZsFH!Fj6^vMWzu$h<$WYkcG)yP84GWk~P_0!CDuMbR5&OAw;?7_8>$d#7H!Po|?w zZG2^h`@&Gg>?7OzLWZU#8dU%NYk>F9KLxbbL{&_g!HvgQ+r2cg!dm5%SuPk6V#}E8 zi(j$HL;LoIn`6Ertba|HY#(QujdQxh0h<#xL4rF|lhUeK?!H5soyN8fbUU5;h6{S3 zqis^jNA4F9YoEL8!o^T8>J{$OM}{>+rR%GWbD!sv>&n+x{%Nj5{ls;eDPC~~it3|_ zUDIZ!NrGmI7zk{tWEw^+-OUH-8KD++GdD*E?_AfB-qu{I(toxF!~pb9BN#B3AzcN# z2|%0QqGulBxVVt8CfgJFj8~?lJ^8>ifTPikin}xOiesHwK>4o@l^ZIod)Z@t26bZz zs&n)fic>j})5fiCNzeVjF#`~Ta2GCil_IbYx_>3U>&6$Z%A#f34x~47zp~H>0`dh? zQA4+mXm7&32Y+&)$xlOuI$73`lfG?+3FI6ixshC$lJoXL~;l!HK1C0%7o$OAWw){*q?#(|7_usWDn+@q&gK zLSMK2NZav{JzDZmx&PK^(AG5_j~K_>a?0pDF>$%`(SO7}M?F+dzx{(*FikEi>Pfp1 z#YWi}RZ7C}yL8e-gGcC5b903TXEnY?=LvYu>)C^+H*=`(jeempld0E_Y7r@5s2X}g@e|?{Gt{$3_bF_n3G5Xv<}2|HeIF1F(!T=8`T6Y>8S}rbh57SqkGtcv%^(lFeQ0T?TYa z*;3vCG5nZRnB_?p3NP0@1s8)fT#triJ-g_jRJe&=gQnn$iV3|e(9NLF&{QMZFXH4r z{?f#)-s*>)&8|Cm_N#}=eO8OMiPfiM1j`;t+JAGJaz~Z`N1sSUQykZkrFI}K0!jtV zCc>6H$32U~diE1p{x{tF3ngl!GB(ZPCyny#z(cSP>dv7nG4p5Q{8hyDJjVRMo$R*TP+#ryvxA>JpAkR# z8L}i$=Y&t`X*9}xj`Mtn*B^Xmd*AscIPxHfBBLrh5gdQv&DBxEd;^}Nz=$owDVTn- zoXU*obpik26(OQwxv)+*6(vJBBCmWA%r=t5h0()D|?M%F>NAfw89bY8a>QK-bcuV z5JwjCg4j((Gas^Y8SB%;xaOsbGk-5@t%j#C`h1~A!DO^k5B~NYBH9PvqpMMteZx?c zvpa_c(iZ93-I!AQc$hyZsOYw0&ALnZ;<_6O`D;DM+{GnCJYRE1(OpTK_bR)6%{uri zAq%~}hk@6?0F3b1k#<^Q#o&cQHV_`dE`sc^c$t}?!VavPN(vWA(6B3Sa(@J4(2Kwe zb=i)B0%lLU!#s>X5D{xGQJ@ylNd-{|oZ9F`wC9~sHlvyp@f0uRgF5N)iv39>=*9b8 zd%&U&uojwFT?e~PFcQb2s=bcLt~4`?CZ9H)K&`fe6#0A+1T8HFtaN*@+JMbZ^zbg& z!>q@H$>EnER}BrO)(~%!N`H>B)t}&qxZg26vZ;XiONfk~Yu^=5deQztDO6Udzwp*% zd47>1=eMDSIQ=o7F*)bU8f=PS35dK;Kx``y{^|W!na9?L4DeBy(u*l*0B4eFuT1u9 zx$`m$);kau66mY87jL%h1fo;H3+qKeTho0J1?HGyVJr?qDNU@aj(;z2I9i(ore1}a z6PcrB%fcnEcH^B7tjh#g^W2_k3rLV%6!}3OC$r~@R?&sIR zhAg$IP`Z?q&P(c3mVc$@RKZ4bJ?qZs)yXG0Tfi|dP4NmRuZE6PM~J)%`Pl8jHKZt- z_qXg^;h+hvO5Up*3aDl^WqRgGAIW@i;V6mJHzmhL7sVf6uH4KS&xK%8ZFHRp=TF5- zUJO?4PecXXl!BpagK$#U0~Xyai!CiL_fF7EZMUmC?2J=MQ-7<%)$!54cl?;?SO^+g z5EdxIO^ud=_qdze!#h8?)6Q;Al=UkX15NI#mB7T$siM5oSU0s>ys>?&d2%v#E0XxMw(woYaZC zY_G4nw*C*mQGNL(eD~Y%y}jizBH)g8d9wqlI%`Ri&gQ71#?(ANoaB1W3`l9?p|)&( z-RCIYO&EGDb&P0$7^ksBy~QRa(X!x&HR_I|6^(Ov~P!@Y&mg+Ef z5X$PXHG|=JN*3T64%(`RZF1_o3#XS%M%JjASb|l6Lx+B7)j78l40lz6y{dx+E?>M6 z+#awFUG@jeXS#9ToHrr$6oP%#LmhsHfkb8>V!7tWKRC=X`Jbse+!VaLRvv8lkS^rZ zc?0qmWq)^;4MhcAw%u9O>XCUvwu~LN%F~vYR+q;%v;4AHCiG$el)FOWX*g=CUbBv5 z9qES|>*-KACRvsxwWUkYmf4$w6AktHO52;~)`xw^`}u$OtHc#g_P-E3A5@8N_3#Qu zpZwc>M6qH7=FUC{?Jb)G4hq6ZouzGsP1GBnDt}A}$VBGnpsuhu<<-$mv#=6LDk2@~ z&4FwEQR7!ci5EQ0^z4H2O3auPJEeyj4^`*>8$nzGT=92sTGz{|4RY92| zuzzlRJ9)rMT{UL0if1R%F|{yv^a|~5=f;k!=-8U9aQ(dS=c?4#z@Pjh!N#Aui`v|n&BcDx5HPb~>dO)MdA*0-eM z87{?5#A`}Wnf=zxFj?2EN9MtS`dlOG0bUK2f)BT1RR|Q{_iFlX=byj6{SO@234@mrO#u^!tF8gJtF8j7odGnH zfz}m&)mqNvZUvx|vcQoJI?>)bx2%#Lb8TbvYj5?xLe zkJV+3XvJyNNm6!N1;f?VP|!wK6Q)v38zaVlE1_c(mIOwL&=D7jz((F4z%4ni4>WRYDyZ z93u2Wn3W;~ByApbqP7ZbS}K93pjF#Qa@hn3{u%O{5LA(4v{?!|f_ald$Hp6q&P28v zK}YQqMlqi3ZI);qjLHd|z$9ne6~RP*w5ACWSk?9^!$`1V7>!0VC`_)YNt4jAK^&Ym zR-BwdO85j~R{I2E!R!PrX=a)bV^p?{Gpw$d_!-0%%7yioK=vY|0~;Y2Oli9USus;1 zwvOo)5-ZRQ)`V8$gkao7AUXnlkpc$LddX%HSYyH3(NR|lRzevT9&EMAEZzox6<1-< z_#&`jGnW`1*d>CW84hmHElV!m{( zL#F7}s3%Q1eIK3y>$m>Hhpu~ewYYdOU4lpV?58KLdpZ5MT+jCM(}yWOync)Sk1_Lf zzPyG*tcG{br`L;{tBdJ%^XB${>gVa3+3Oz{A2FX{M25*hqnHqrTwyf)tKZ3^`FvrM zY3SE7dDXDP9t=CIJ=<7Bqj%kloAYIR{`YMDr|W)PT)mlIy|OvNS@*B*Y4`Y5Y!9}) z3)>e%UJChTEuJ9-uF;#}=Nddi2WQX=TGhL zyLZdwhwC4@?%CpUcJb*S^S874)W4lA-`$-1*`oWE|KIyJ{@wEZ z<@Z?o?fRP?<-B1>so4>KWaDAW+h(*&yfQikG#^GO5I0=x1m>x-W1c|Mq2{R@=Be>Z zW_g4?q{LnJYMjJyME&%P1o*Bjo^( zHz}019Hel*T587;24C-9#`Vql^|aZyCm+Ya+gQ1=|Lk8Oc}vMo;$I~lfwx=4eUO*x z1;OhOZ`?%GulLN#JDyq7p`KZF+cRs~@XVsUC-$CMde5w31ay@GAKB(sMwAp@LprS(j|sAweWeW$ z&;)tXX+WUT5g@Qdyf^8(gHm$#R@faz%`6`E^O2!`6KDp43x)ju;9`~RFmSQLcnvN! z_F%FC6)Ue-ZS31LVr3x+Y;55^`V0mPqf1u1!dVz)+#V-?jbW710wqjHX{)rzN$-u9 zG9kLfHOR2~h*J5UVRb~hC3dI)wc8m+ESCS^xus!ytzeXdhmeKZLB*{YE$vfO(5JvU zJMuah>$Zt+%f{-%od|XpH7AJp;JJY@KAiyVQrJPeScalq;x*dY8DB%S*rQuaeS7o#DWFUW>1M@=c( zF}iQ(qXDo^UpSfi@9gc()zn|S{?Ll1Q4mB>K&G`u#oAMA^cVolNbbryj*y0voTv;= zR%`XBQrGJbu#0ey)~^M{Nm(b`C8J0=O|oqlA%_Qlpe^E6kg#Ln8stD~L7ikV|1fG| zETrn8)PR9lzg+v@9i%n`+2DXxb^=z)M+U6T%_TiTCtJk5l;I$D$MW}~+jjixFXw-l zf!e31?^}MfUmBo-03QFHRA}d1*>0f*hH=_B5wRPv7M^<*5{+1-m@`&Jkb$$VZWR$+ zU^kL~$lJZ%u=I4mI>65km6+5ARN{#M648?W3LWoHlWCMR8O~>Sd7n{M#MgWpkRm?9 zA-0Hnkg_w)Tn?lrh@5Kv_J@eXao1()R{#lz;nx*|_pl3woQJx-Y@AitZFF;Y<}nUF zEBv{xPrP!lLy2_X(sDsg%=U8wPCf%J45sXVmfO5lYCiZ%Wo0hM#2vmP?x0{)krR}R zvU4wH@o%Wn$)?fCc(Enc2%C%R`>#-ZsI>q7Yvaa;@6eeXabvPgywX`J>ND$+U^i4x zdAGEe!KHmp8+z)03SmDR8Bbp*)kDs!pFp&qZ6N|86$pfoyf*^#D3=PT%j_!X^Ld{s zDpKte)d_M;ogll}ISX=97|^#5s(JVDJP$rR&zpH@UZKv}UvN1#<+lt}1DmD=IWaad zFqf?Q10DrAF*Y$Ux6k?mssTncF+(&mG(t4w6s9Wt1YxZDYUfZ(c0JT$N__8;$xzTm?%+QyqIcKkZ_T6Xy-`V@7l+u5NlnP1GjdJ2jKn50p#b602 zf~BAWRDxw-IamQ!f+|oAR)N)^2CRPpYe6ko2R4I^lG2*^@%3OsJ{#`=2S5jC1I^$F zT?$|~*aKR@e$WXHfm1n*i5n9)=4W+a9+dN;ioABQ2x>t+XaR>n52I`WTR}Z&0NcP$ zupQJ%TJnC(zLEJwVG{$K0%PC`n4{1Pc1Wt}h$nV|CeQ|2ByD{?3MJ4Ey1;*4Nu{YM zbbx)J8yt|-JspLEAQtB+I1c(ikEFrRqf0M11`dM@;3OCTXF$KCi#5@C5S#`lz%V!u zh9u3-MwfHoEEoeLlFB1d7zG!>xTLEuN81F86H~haE=hXo!zf$^Pk>2D&y7ag6qp6m zlAiC3wi!?cbCRz0Mq6YV#x8$IdZmIkOCs0r#}xt_##%wrn_os7J{x))?i$8gg``{U zQAK6LY{Ol{UBgL3TdR`v*`w%(&xX&2=(YiD0=1IvM6|XM;JC5khC{Yl(l_5nbqlKz z1#Gf3a|{U1~p&}SO=;^J{paFYe6;G3O0f|5F5&7umfxZ z>qTyV8;@aAv4Mr{<%$io*aUWhU0^rZ2UX1)^I(#Tma|6B$$w_^l=m}gNtAq%z$~2hul*cUBZZ5A5m_# zz-?Kw{-5JkxLtpc?CA$_ixG+f$|0h7pp}56mCUsY5U3KoazZHQjAoKe?2l29L6Jd8 zUpa~tEfm*uJs?m;F2x8%E=3DP3q=dXI>iIU9Pi@_vz0&j${fDDQZieZWZ%E_sCpd5eTiVTWyiZ6;5dK6%oB7>r` zB7+jVa%d{)>kuGrC1358?CMNR10xiN^c=t;C2mCvMGHkK#RKL1)mV)U}O=Iof`{H~*vvs-vf zh>RSOoVR~;Y;tUJChjQZc;LK@BbOtW^OlYgj%LoAJ6brxIjiAV=V;+9ljEP`nzMe6 zaE@@!5Iam=eE|v$bgI+?_-N>6 z`06>$#s7bBlEK)>%&L+hw7M#jOEA_K5o~k&>t=3$elz3u{_EQxU$Ne-*{V7I-G2S& zo864{$6Ir=x8k=D<1L%InP6|f{=WU|ZjApjCR$_CcvZg{k}WsFZwj&19DdJ=Y+;f|-l;renncZ^78(C5Z#-kh!$kN3H@ zAAIuqWbizekMAM$Rm-w!`1sxr(^GeM`k|HF%U{9va!DbZ`!|wk-?Ffq$Qn#N4(F zk_fVnIux+TgSWX_SqJw5v*cLyrC`<^GaUrEkA)%p{e zxWR;GVW{Q6Oe-QDT9_9eRv5XR)Fg(j1Ujk;n_wnR3bVc&gvsz(wSU!$Y$$@%fCmZ~ ziP|1r@w-7Z2Kd84M-Snzxp`mO+xA-;7CYj4=71hxcI>{Q8A@93EMeO$(+X(?89myf z%?}pxgB?fgK;AJA;z(b9_MJufPBa7bKV4#=UER1Ri3Pj48c|{(^0t(J8v^v9&yCNY zfCM;?S8>tbWpu@Npnnb{;*+>N*@+)63{ck?Pp-^OamXEdn+XFd+*9*VNc7b~o`go= z>51lBXd=ORw3-cQyW7h93GDq(5XxsYVsBwK|l(t13My^qkjO}C^5j^52d^a3XzWh zel3PYQ(m2~kB|bBGz2G218E@qWXfIm5rcG*q78USYGBIju={%C?ZkRO;Q^4;X-Teq zUU%}uaXPJ(On)MkxZdU4ERm!cE(cEk5FyMkge5=eP_k7^HdeotOpvhA=5sUXNT!`c zm}JSr`rts)5`SmXf>hIGjuM|c%xN`n)N)rI)1{Z?Hy9c3XR=vOIfre~&Vt(MlLGxP z=9rE76NMV!K|FIJb#Wp@u5bppK_VhoW)iF16T>~?=;j>*r3*GA(XM0>+c2MYr3iMy zH)JOYz?DXm-#OMIqa}hzm2p+(Vz6ry1FZc!cnfAsynmRVKEO>NaDaJPn7@qR@CFS* zo?XtI1xqRH936N$&g>);B(pWhJA(=_r7hq86W@77saujlKUXxt=-;epg5iaWtU~T_U2Q`rvu4qLf;$W(2Immy z%p!DYvo6o&6OCvX6U=BvqlxR3XhIY!C6cQ%YEgbjA&-vtNW5_fCDP2bFo5GON=s5c z9e>O;r~4Bpy(eD@2_c!67zL!AE@hO6J@E;w9CoPjOv@ajbz98kG;<_4o*p6r>q$dB zNm3|%gSRc&2>6X`!_+e?rl*irRFziLmuW?PndDSy*Jg2}27^N1b(c@ykztp=jmnLR z>aS7Vlate7>X)zages4oL(@>};byC}$A5roKv=qIa1=lXzICX$B*!hf>+W8Px{-)nDLH3+Y(l+*z@lV2>GFIEZ=!m_<& zk3Dq=lvoxdA6veNSXA@&A^SXQy~du3z+To}{2bt6tQYn0@X)nhOxe2UOS9}a)wNSI zLd|Qqx6CZiGO|z8Ac?Y8EtytBv>GNEhOiay=E99`H9L`ZTkNOHUR1gxZGRVCR8^!p zx2brAssLyfc6yBM(?x9svVwOko=b1e$sl}D{W^v1@#X2Mgs2LFvNXQ5U(iZeJrV;( z6y_Woz1>`4k6JlM!4g3O0U^QgrZ-5cn{!^=m!np0g;zapub|BXjM?w(?UB#2z|W^( zM-cB+&WNg)^Eg|BXj@v3VSm0SvZO&v1nhGf-8qnlKPj^Xo44#qU=+2BZq#GnYuD8T zLQ%3k!qo+AQ~&Yh>&+gaHb$HKZ7EH};lekr%u`qNU|}>ps90tQnl4V zvu+}0y}C6e&_0Ik_#rL6+CGK%wjU$C$WD7DHAQZ7IBJ0nt&|YBYkyNhy>&AN>TfV~ z$%d1x-Z+{}34d2gi#rz+HlVb}Dv-%bSZ) z!bO<_Qs(uL$Vpq9#5P%=6909MpWdw|2BUq9YA`P)`QYdhh-wLk5vKrCI0Y+Z7tStd zVC|~dC-=4ln&rSN;X_^VT8|fg?bpZ)nX{=`zh#qDex^#1$A4;(RbqH5dK!jP!nzM( zvuI`I3$Eszxt;!Z*%~%S#?zjrv=yMpK3ne@B5-)gLpxj`AOp zX5APle4C&<_&8tday!=n&VUcM66Af_R>M48{AOc5tzmM$P-Ksl{m`ZJ1Z8|0b>)TN z*?vWa9idZzt$(nXWs)ay+9f06=s1{vx_j4cWy`wVw%o#Ub-52bcC=h|(6P5nihq%2 zAO8ccQb!y(el6S&vX95dl7FZ}!8x1QQ+)6I@GhkEj;;UtOOwgwaJ-;pM7X$Jqk%?|2n9-lhOJca|zrfJR?dvO+D-8=mnI*19c+ zGOuTMUBY37Z2`U?{)ebFIdHNgF32lBNge9y_Z^y2PvFmpkMP=l_tKg9|QYbu2E4 zxR}@_#SkfmrM*hRj%q?%TrTW#bdo9_s}qfAv(u=Pr0le2OjlP!K^t97SW2;MjF_*4 ze~v9!5)dUqC%Z@lG>Q>YgDEcNU?bH<;W$YpxTwW|*~JPZ8DJrTfCA(J*i*&Z$oQ#1 z+sH1!l8okp5ys3^jAENrgU!Ukpp`5F-vXg1NO)9QNvPs!=e;>GEGZhS z1S+R6hcjE`3bSEf{Z5|D=L=g*Lr=@(P4gZ0V7|lJvkf8|yz5@woG;t+FSGf7UH8M{ z>fQ9}jja*Rx}Uq}-P1R*Js5cxwl4<16#Nq{p1}pK(VOB+4W1!`v+jv&9DC)upBAqd z#_{i#Z_h8MUcWaUZSkTff8##9@Hj56@xnBK7gkiNSLj<*_67C|Yx1UcXOp|#w9Rx+ z9zAMEx+fRQ*<#+k>i+fex&43lVY&Qt{kN`rvACRF{Q39!``LWz-_MpGZqEH|(S2&e z=hOMchi*21H~sAY`1sC$Sbn_x9<<-m-}FdiLyuC^Bk;z<*0$AXe;0VAbTGaHAv4$w z6FY%=YTi*#_^+YrsT=C4@k?rXgg&IiK3gp7^&wI65s~L4`k;K6KCI<7uB-~}$g0Gl zvMPB~Rx2VcHCVNE)s`@xYe5|L0lFo^UM#MP&j?wf>hpfjlD_C!1~@oTFU0~?^buj3 zHT#4}M?yYu+($+ke~}C%Ixu)k_^Kzej81&D%no{GiQ&CuT;H5uPn&vs@^K7zi~Pe| zl5uBVoIK+2Nvx}+BfRYv@XCjBNZxXrLu_%=pMJYtR^GA8nhv$gs@ry1%Z6PR?d`C) z%hKCr4I}v0ToBWd_T9zWQhY#~_g72*Tm1@Bgbd5g5tgl{e+SqO99T}(Gcx%-7G1}% zAqM-1KvgD*}JjMsWA6Htd0fGVeNvClFm5y)%Tflo$u6rP}h>xfj4})gT z9@X=afqv7`40J9O4ULSrs~n$IT=(&zAYM&P~Ypp?t|t879YGee2h;g z@OCNec)M7Jdb`AHZ)Ycb&8x-UyT#PEx8vNm(ziN7e~P66`nFa#jZima`47!%1+@}n zYe&+y6*~grN-rrnJ{l(i)t!5g< zgJjf?X)V!g?WrYti~|%TcV!(%NW)3aQwAriwfYp+VI1iZ+6P3x78ECCoop9Dk#d@3 z+pa+le-A)gz^fo(hr%`F1EDkOB#ZfnK@)?Js)s_u4MbGVUkX_F2dT|KHaK9Foq(0( z$bhxExui$PWDB_GG7Q9SSiT;*ZO6a<^ZY-ip!Vwd$Ce-M=LVp_fTv&M3hkUK+a1)v zFismMB6k1P!gDV|q7jP}GsemYGB8##%DKqcV0GUU*$ATiueeF*aGgIR7l>g)enOvu$-#=^@oVWao1$( zH#iaw!!Iia?_t*&vJ5r*ym3}xchSw2OjOZ zD}~}iB`4V~*6FJkK75bN@)0*C+rTTCrJ`O~k92lJ0qyoXN3HL@|9_3O2Fm;(-1>K$ZsUp=rQJs*FsT1;UcFx%@IqM06 zs_Y(~%izOv*~~*_<#o>f2UYbpHkZ+L0~5D3RRf5oC^s@OFflDKG$$}HCn+E=O>bmG zVRU66C`39kFfukVFflYRH!?9WF}LOW1F8W=LNqrvF)}bQHbp~3K{iA)Lp4D+IX6Q^ zLq;<}L`F0}(AYGF|eIJ)` z#Q_(W3;qKPf0S1#EfflcmQtYfK`FF_mRBFpLMgORT4*gT?Z~KuWsEw}1dWO*#w;U2 z5j8Q6j4>uoa14>eNSrk$4qTcLW#Guf#7Ls&Taz#6?6uFn`|STad*75&`mc~uAxXM1 zm$(v;fhAxmSO$t5B7mB&?%{FItm9tEY4AI9Q1;2NdsR;mmY8o90upX32+je27Qvo z%cJuEI0gE_FgOQ>B+bl3m$TpuxBx~Z&5cB16pVv0Nzc6*Z5LUbnA&A9A?f)~qi_j4 z1tulEJQ{6NUqb`fBmVUwYmT@ZO`e{FP`0ApYpV3W=8T`mDQWRoJV) zlAS4ujbbS%0R@qFUyrtBU^&RZDo_U2fE6OQCHk!fE5SNY4k|z;C>8l^H2SRvYr!^9 z1vZ1&P^v*a*bX*|eDy^2zt z+5r1*@c?KS`LRDbcK~+f;t{Y{>_ zN>4h&wvpy3MRl9I4fDrGtuQdI0q)dMai13N8u6}2h-pRxC-)+ z5B(Wk!iZcSQEs)s?VM!2_v2Q$f1Q`?g@5?aug|AD6Z*7K%k0TiV=!jiWZ6%iWZ7>iU*20-pdtcD}gG~C`Kqo zC^9I)E5R#go|3+D%qo^BYp*1zwUW(N#!N9nF+$q`GAJ@AhA9dtC#T|pe{z5;GAP0+ zz9?GgQGj8J42sH%3`+3Ip{b;=Lx8xIe6>fiX9r^%7@;_%X8{f=aVuIVS|~~>9w_Ip z#%ja?MCOVDiaCx4iWbTd?GZfE=oGg=qm6zZg%|)!oF{Y^;E=~C{H@S=Ze#c)+2cpd z*)hlYT}LTrxA2${895?3e{bp7K$zQH* z(ZX3K$3Mq4XZ;-E9O0ZLcC2%(bC%t)&f|X=E9qG02@w8=xksLdHk7NrCd12wbe;hYIk1C!zo_dR9 zx4y{>AC%>|{d-iA%W>D+B)jurUii2&$GxAUinfln-XYn8*1YiGqa43QJ7zkRYk{SV{Frozen snapshot an
-
+
@@ -1096,7 +1096,7 @@

Uncertainty

Sampling uncertainty is substantial at this sample size. In the frozen snapshot, 6 of the other 49 models have an exact-match 95% interval that overlaps the leader’s. Small differences in the leaderboard ordering should therefore be read as ties rather than as a strict ranking, and the rank-range column makes the set of plausible positions explicit. These intervals are a manuscript artifact: the public leaderboard and app report point estimates only, because the benchmark sample, not run-to-run model variation, is the dominant source of uncertainty here.

Cost and latency

Accuracy is one axis; the price and speed of a no-tools answer are another. Table 7 reports each model’s estimated cost to compute one household’s full output vector and its median per-household latency, next to the headline exact rate. Per-household cost spans $0.001 (Claude Haiku 5.5) to $0.541 (Claude Fable 5), and median per-household latency 3 s (Gemini 3.5 Flash-Lite) to 1023 s (Kimi K3). Neither dimension cleanly tracks accuracy: the top-scoring model, GPT-6 Sol at 95.2% exact, sits mid-pack at $0.027 per household and 23 s.

-
+
@@ -1421,8 +1421,8 @@

Cost and latency

-

Cost preserves each call’s recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5’s board rows carry no usage or timing fields, because they were rebuilt from release dashboard-data-20260701 without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (Table 2) sends a household’s calls one after another, and its household time is their sum. GPT-5.5 ran through OpenAI’s batch API, which records no per-call request time, so it shows a dash.

-

The harness also caches provider responses on disk, keyed on the request and the prompt contract version. A request identical to an earlier one is answered from the cache in hundredths of a second, and the timer records that as if it were a call. A replayed response keeps the cost recorded when it was generated, so a household’s cost counts each of its responses once, whether or not the board run paid for it. GLM-5.2, Kimi K2.6, MiniMax M3, and Qwen3.7-max were finished over several passes, each of which replayed the earlier passes’ successful responses. Each of them records a replay on 53 to 93 of its 100 households, so its median falls under a second and it shows a dash as well. In 11 other models, 29 households sum to under a second. The audit traces 27 of them to an earlier request on the same household, such as an onboarding smoke or a restarted chunk; the other two come from runs that predate per-call timestamps and left no cache or ledger behind. Dropping them would raise no model’s median by more than 2.5 seconds. A further 33 households replay their first recorded call and then reach the provider, so their times leave out that first call. The benchmark card summarizes the replays, and the repository’s latency replay audit lists each household. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.

+

Cost preserves each call’s recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5’s board rows carry no usage or timing fields, because they were rebuilt from release dashboard-data-20260701 without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the 100 households. The timer spans one request as the provider handles it, including anything that happens inside that call. When the harness retries a failed request, only the attempt that succeeded is timed, without the failed attempts or the wait between them. The median is reported in preference to the mean, which a few slow calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (Table 2) sends a household’s calls one after another, and its household time is their sum. GPT-5.5 ran through OpenAI’s batch API, which records no per-call request time, so it shows a dash.

+

The harness also caches provider responses on disk, keyed on the request and the prompt contract version. A request identical to an earlier one is answered from the cache in hundredths of a second, and the timer records that as if it were a call. A replayed response’s cost is computed from that response when its row is written: its token counts at the configured list price, or the provider’s charge embedded in it where no reconstruction exists. A household’s cost therefore counts each of its responses once, whether or not the board run paid for it. GLM-5.2, Kimi K2.6, MiniMax M3, and Qwen3.7-max were finished over several passes, each of which replayed the earlier passes’ successful responses. Each of them records a replay on 53 to 93 of its 100 households, so its median falls under a second and it shows a dash as well. In 11 other models, 29 households sum to under a second. The audit traces 27 of them to an earlier request on the same household, such as an onboarding smoke or a restarted chunk; the other two come from runs that predate per-call timestamps and left no cache or ledger behind. Dropping them would raise no model’s median by more than 2.5 seconds. A further 33 households replay their first recorded call and then reach the provider, so their times leave out that first call. The benchmark card summarizes the replays, and the repository’s latency replay audit lists each household. Both cost and latency are properties of this run’s providers, prompts, and serving configurations at evaluation time rather than fixed model constants.

Simple baselines

Simple baselines help interpret score levels in a zero-heavy benchmark. Table 8 reports an always-zero response and a median-reference-by-output response on the same frozen household sample, scored with the household-impact weighting. These are not model competitors; they show how much of each metric can be earned without household-specific policy calculation. The always-zero baseline scores 69.7% on the headline exact-match rate — below the 85% of outputs that are exact zeros because the weighting concentrates mass in high-dollar output groups that are rarely zero (Section 3.1). The top model clears the baseline by 25.5 points, and the within-1% companion shows the same separation with near-miss tolerance.

diff --git a/app/src/paperSnapshot.json b/app/src/paperSnapshot.json index a93e1655..aa918c8a 100644 --- a/app/src/paperSnapshot.json +++ b/app/src/paperSnapshot.json @@ -1,6 +1,6 @@ { - "pdfVersion": "3cb8f835f260", + "pdfVersion": "d16eb03ce07d", "responseWindow": "June 12 and October 13, 2026", "snapshotDate": "2026-10-14", - "webVersion": "031238989dc0" + "webVersion": "6cb2fe1ad2b3" } diff --git a/docs/benchmark_card.md b/docs/benchmark_card.md index 86a6636f..29b1cc02 100644 --- a/docs/benchmark_card.md +++ b/docs/benchmark_card.md @@ -432,10 +432,11 @@ release publishes as `claude-fable-5-predictions.csv.gz`. The harness caches provider responses on disk, keyed on the request and the prompt contract version (`policybench/cache.py`). A request identical to an earlier one returns the cached response, and its row records that response's -answers, tokens and cost. A replayed response therefore keeps the cost recorded -when it was generated. Each household's cost counts its responses once, -whether or not the board run paid for them, and leaves out spend on attempts -that did not reach the board. +answers and tokens. Its cost is computed from the replayed response when the +row is written: its token counts at the configured list price, or the +provider's charge embedded in it where no reconstruction exists. Each +household's cost counts its responses once, whether or not the board run paid +for them, and leaves out spend on attempts that did not reach the board. ## Latency basis @@ -443,7 +444,9 @@ Each prediction row records its household's request time divided among the household's rows, so a household's latency is the sum over its rows (`analysis.household_latency_seconds`). A household that needs several calls (one or three outputs per request, a repair, or a re-request with a larger -completion budget) makes them one after another and records their sum. Published `latencySeconds` is the median +completion budget) makes them one after another and records their sum. A +request the harness retries after a provider error is timed only on the +attempt that succeeded. Published `latencySeconds` is the median across households. A household with no recorded time is left out rather than counted as zero, so GPT-5.5, whose batch rows record none, has no latency. diff --git a/docs/latency_replay_audit.md b/docs/latency_replay_audit.md index 185cdc8a..0cf4859d 100644 --- a/docs/latency_replay_audit.md +++ b/docs/latency_replay_audit.md @@ -16,7 +16,12 @@ re-request with a larger completion budget) makes them one after another, as `run_single_no_tools` does for chunks. Their times are summed, and the sum is divided equally among the household's rows (`elapsed_seconds / batch_size` in `policybench/eval_no_tools.py`; all three lines date from 2026-05-01, before -every board run). A household's latency is therefore the sum over its rows. +every board run). A household's latency is therefore the sum over its rows. The +timer wraps the provider call, so whatever happens inside that call counts. +When the harness itself retries a failed request, it restarts the timer and +keeps only the attempt that succeeded; the failed attempts and the backoff +between them are not counted (the retry loop in `_request_predictions_once`, the +same in the July code). `analysis.household_latency_seconds` computes that sum. `analysis.model_cost_latency` publishes the median across households as `latencySeconds`. A household whose rows carry no time is left out of the @@ -130,21 +135,24 @@ each model's final pass. The notes record MiniMax M3's first pass at 53 of 100 households, and 53 of its households are full replays. They record Qwen3.7-max at 93/100 in the round before its last, and 93 of its households are full replays. Kimi K2.6's -scenario_109 shows a partial replay at work. In earlier rounds its answer call -succeeded and was cached, but its explanation repair timed out at 300 s -(`part_84_100.grind*.log`). In the final pass the answer was replayed and only -the repair reached the provider, so the household records 777 s and the -replayed answer's id. +scenario_109 shows a partial replay at work. In earlier rounds its first call +succeeded and was cached, and the household then failed on an explanation +repair that timed out at 300 s (`part_84_100.grind*.log`). In the final pass +its first call was replayed and its later calls reached the provider, so the +household records 777 s next to the replayed first call's id. For these four models the households left after removing replays are the ones earlier passes failed on, many of them after timeouts and retries. Their medians (10.7 to 1,194 s) are not estimates of the model's latency. For OpenRouter responses, the cache's store time minus the id's creation time approximates a call's duration. It agrees with the recorded time to a median of -0.5 s on live households, and it misses badly where a household retried. Applied to the replays it gives medians +0.5 s on live households. It times only the first call, though, the one whose +id the rows keep, so it falls far short wherever a household made several +calls. Applied to the replays it gives medians of about 133 s (Qwen3.7-max), 117 s (GLM-5.2), 294 s (Kimi K2.6) and 10 s -(MiniMax M3). It measures only the call that succeeded, not the retries the -harness timer would include, so it is a different quantity, and nothing +(MiniMax M3). Qwen3.7-max, GLM-5.2 and Kimi K2.6 were served three outputs +per request, so their households make several calls and the first one is only +part of each household's time. It is a different quantity, and nothing publishes it. ## What the published figures do with them @@ -173,12 +181,18 @@ publishes it. are unchanged. Without sub-second households, Claude Sonnet 5 would read 65 s rather than 64 s, DeepSeek V4 Pro 172 s rather than 170 s and GPT-5.6 Luna 15 s rather than 14 s. -- **`costUsd`.** A replayed response keeps the cost recorded when it was - generated. Spend ledgers record each cache hit's spend as $0 and the replayed - cost as `cached_response_cost_usd`. On all ten ledger-era full replays (Muse - Spark 1.3, Muse Wonder, Claude Opus 5.5 and Claude Haiku 5.5), the board - household's cost equals that field exactly. On 43 of MiniMax M3's full - replays, the row cost equals the cost embedded in the cached response. A household's published cost +- **`costUsd`.** A replayed row's cost is computed when the row is written, + from the replayed response itself (`_extract_usage_metadata`). Where a token + reconstruction exists, its token counts are priced at the list prices + configured then; otherwise it is the provider's charge embedded in the + response. Spend ledgers record each cache hit's spend as $0 and that computed + cost as `cached_response_cost_usd`, and on all ten ledger-era full replays + (Muse Spark 1.3, Muse Wonder, Claude Opus 5.5 and Claude Haiku 5.5) the board + household's cost equals it. That shows the row carries the computed value, + not that it matches what the original call was billed. For OpenRouter models + priced by the provider's charge, the cached response carries the original + charge: on 43 of MiniMax M3's full replays the row cost equals the cost + embedded in the cached response. A household's published cost therefore counts each of its responses once, whether or not the board run paid for it, and leaves out spend on attempts that did not make the board, such as the open-weight models' failed passes. That matches the paper's @@ -203,9 +217,10 @@ comments, and the tests that tie them to the data. The options: 0.003 to 0.033 s for the four open-weight models. - **B. Leave replays out of the median (recommended).** `model_cost_latency` would take the median over households that sum to a second or more. It would - publish no latency when more than half a model's households are replays, - which is the rule the paper's dash already applies (the property test - `test_a_median_under_a_second_means_most_households_replayed`). The four + publish no latency when more than half a model's households are replays. + Away from an exact tie, that is the rule the paper's dash already applies + (`test_a_median_under_a_second_means_most_households_replayed`); at exactly + half, the paper's median can fall on either side of a second. The four open-weight models would lose `latencySeconds`. Eleven other medians would rise by 0.02 to 2.5 s. In the paper table, Claude Sonnet 5 would read 65 s instead of 64 s, DeepSeek V4 Pro 172 s instead of 170 s and GPT-5.6 Luna 15 s diff --git a/docs/latency_replay_audit/households.csv b/docs/latency_replay_audit/households.csv index fc9066b0..f61108ea 100644 --- a/docs/latency_replay_audit/households.csv +++ b/docs/latency_replay_audit/households.csv @@ -1,7 +1,7 @@ model,scenario_id,household_latency_seconds,replay,evidence,source,first_response_id,request_started_at,response_existed_at claude-haiku-4.5,scenario_009,0.026461,untraced,,untraced,chatcmpl-9c7cea35-2052-496a-bf4e-3602ca12515c,, -claude-haiku-5.5,scenario_000,0.017322,full,smoke_file+spend_ledger,onboarding smoke,chatcmpl-992b7cba-d5e8-42da-9449-c06899393c28,1791495684.702,1791495644.564 -claude-haiku-5.5,scenario_001,0.117501,full,smoke_file+spend_ledger,onboarding smoke,chatcmpl-308e4847-b356-4499-a13a-3ab9f3208450,1791495684.777,1791495651.476 +claude-haiku-5.5,scenario_000,0.017322,full,smoke_file+smoke_ledger+spend_ledger,onboarding smoke,chatcmpl-992b7cba-d5e8-42da-9449-c06899393c28,1791495684.702,1791495644.564 +claude-haiku-5.5,scenario_001,0.117501,full,smoke_file+smoke_ledger+spend_ledger,onboarding smoke,chatcmpl-308e4847-b356-4499-a13a-3ab9f3208450,1791495684.777,1791495651.476 claude-opus-5.5,scenario_000,0.029634,full,smoke_file+smoke_ledger+spend_ledger,onboarding smoke,chatcmpl-e65b82db-643d-47f9-b87a-d6143e484ca6,1790095878.103,1790095745.462 claude-opus-5.5,scenario_001,0.029532,full,smoke_file+smoke_ledger+spend_ledger,onboarding smoke,chatcmpl-f5a49889-34ba-4ef9-8e78-3cd1cbd21ef7,1790095878.103,1790095771.107 claude-sonnet-5,scenario_000,0.031374,full,cache+smoke_file,onboarding smoke,chatcmpl-d78ec31e-c962-43fd-a568-a6f2f8fc0214,1782978630.541,1782978530.271 diff --git a/paper/index.qmd b/paper/index.qmd index 1fc8f703..0ecec41e 100644 --- a/paper/index.qmd +++ b/paper/index.qmd @@ -1188,9 +1188,9 @@ def cost_latency_caption() -> str: md_table(cost_latency, caption=cost_latency_caption(), label="tbl-us-cost-latency") ``` -Cost preserves each call's recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5's board rows carry no usage or timing fields, because they were rebuilt from release dashboard-data-20260701 without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The per-call timer spans any provider-side retry or rate-limit backoff, so the median is reported in preference to the mean, which a few throttled calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (@tbl-model-runs) sends a household's calls one after another, and its household time is their sum. `{python} r.batch_latency_models_fmt` ran through OpenAI's batch API, which records no per-call request time, so it shows a dash. +Cost preserves each call's recorded total: a request-time reconstruction from token counts where one was available, and the provider-reported charge otherwise. Configured list-price overrides do not reprice recorded calls. Claude Fable 5's board rows carry no usage or timing fields, because they were rebuilt from release dashboard-data-20260701 without those columns; its cost, tokens and latency come from the per-call rows of its original run, which that release publishes. Latency is wall-clock request time as observed by the harness, summed within each household and taken as the median across the `{python} r.n_households_fmt` households. The timer spans one request as the provider handles it, including anything that happens inside that call. When the harness retries a failed request, only the attempt that succeeded is timed, without the failed attempts or the wait between them. The median is reported in preference to the mean, which a few slow calls inflate. Latency is not comparable across serving regimes. A model served one or three outputs per request (@tbl-model-runs) sends a household's calls one after another, and its household time is their sum. `{python} r.batch_latency_models_fmt` ran through OpenAI's batch API, which records no per-call request time, so it shows a dash. -The harness also caches provider responses on disk, keyed on the request and the prompt contract version. A request identical to an earlier one is answered from the cache in hundredths of a second, and the timer records that as if it were a call. A replayed response keeps the cost recorded when it was generated, so a household's cost counts each of its responses once, whether or not the board run paid for it. `{python} r.replayed_latency_models_fmt` were finished over several passes, each of which replayed the earlier passes' successful responses. Each of them records a replay on `{python} r.replayed_latency_range_fmt` of its `{python} r.n_households_fmt` households, so its median falls under a second and it shows a dash as well. In `{python} r.other_subsecond_model_count_fmt` other models, `{python} r.other_subsecond_household_count` households sum to under a second. The audit traces `{python} r.traced_other_subsecond_count` of them to an earlier request on the same household, such as an onboarding smoke or a restarted chunk; the other `{python} r.untraced_subsecond_count_fmt` come from runs that predate per-call timestamps and left no cache or ledger behind. Dropping them would raise no model's median by more than `{python} r.subsecond_median_shift_fmt` seconds. A further `{python} r.partial_replay_count` households replay their first recorded call and then reach the provider, so their times leave out that first call. The benchmark card summarizes the replays, and the repository's latency replay audit lists each household. Both cost and latency are properties of this run's providers, prompts, and serving configurations at evaluation time rather than fixed model constants. +The harness also caches provider responses on disk, keyed on the request and the prompt contract version. A request identical to an earlier one is answered from the cache in hundredths of a second, and the timer records that as if it were a call. A replayed response's cost is computed from that response when its row is written: its token counts at the configured list price, or the provider's charge embedded in it where no reconstruction exists. A household's cost therefore counts each of its responses once, whether or not the board run paid for it. `{python} r.replayed_latency_models_fmt` were finished over several passes, each of which replayed the earlier passes' successful responses. Each of them records a replay on `{python} r.replayed_latency_range_fmt` of its `{python} r.n_households_fmt` households, so its median falls under a second and it shows a dash as well. In `{python} r.other_subsecond_model_count_fmt` other models, `{python} r.other_subsecond_household_count` households sum to under a second. The audit traces `{python} r.traced_other_subsecond_count` of them to an earlier request on the same household, such as an onboarding smoke or a restarted chunk; the other `{python} r.untraced_subsecond_count_fmt` come from runs that predate per-call timestamps and left no cache or ledger behind. Dropping them would raise no model's median by more than `{python} r.subsecond_median_shift_fmt` seconds. A further `{python} r.partial_replay_count` households replay their first recorded call and then reach the provider, so their times leave out that first call. The benchmark card summarizes the replays, and the repository's latency replay audit lists each household. Both cost and latency are properties of this run's providers, prompts, and serving configurations at evaluation time rather than fixed model constants. ### Simple baselines diff --git a/paper/snapshot/20260501/manifest.json b/paper/snapshot/20260501/manifest.json index 3a0dfcce..20e4975b 100644 --- a/paper/snapshot/20260501/manifest.json +++ b/paper/snapshot/20260501/manifest.json @@ -164,12 +164,12 @@ "rendered_paper_artifacts": { "pdf": { "path": "app/public/paper/policybench.pdf", - "sha256": "3cb8f835f26018803584989e976c360c6492ad7b4df32d8829179d41ae21c219" + "sha256": "d16eb03ce07dc57c4b8e7c3a3edb70ac8ccbaef8b830ce9cc38d9da892989481" }, "web": { "files": { "figures/positive_zero_scatter.png": "47a28fba15b58e86aa00cb52c991a09c5f393c40b9311c24001c201741610062", - "index.html": "031238989dc07e5624cbb38b0d2b8fc399aaddea9e2a724eb4da81291ccdfcc7", + "index.html": "6cb2fe1ad2b32c4c76984c150dbeffec2f623951e8710d038bc282feb15cd238", "pe-tokens.css": "8f24d8da26f583c8ffddffcdcd172b6d52cbecfec20eda55bd39d7aa829f41d8", "policybench-theme.css": "0e12c5fd615558259e5bce0167a38424e54f9ceb280666c4afd660d759cd1cb9", "site_libs/clipboard/clipboard.min.js": "e17a1d816e13c0826e0ed7febfabc3277f45571234bde0bf9120829a7169edc9", diff --git a/policybench/paper_results.py b/policybench/paper_results.py index 328bfd3b..6373337a 100644 --- a/policybench/paper_results.py +++ b/policybench/paper_results.py @@ -1434,7 +1434,11 @@ def subsecond_median_shift_seconds(self) -> float: @property def subsecond_median_shift_fmt(self) -> str: - return f"{self.subsecond_median_shift_seconds:.1f}" + """The shift rounded up to a tenth, so the prose's "no more than" + bound holds for the unrounded value.""" + import math + + return f"{math.ceil(self.subsecond_median_shift_seconds * 10) / 10:.1f}" @cached_property def latency_replay_audit(self) -> pd.DataFrame: diff --git a/scripts/audit_latency_replays.py b/scripts/audit_latency_replays.py index 932340a7..9adb2731 100644 --- a/scripts/audit_latency_replays.py +++ b/scripts/audit_latency_replays.py @@ -16,19 +16,30 @@ ``--ledger-root``, repeatable), which record ``cache_hit`` per call. A ledger under a path containing ``smoke`` belongs to an onboarding smoke. - smoke prediction files (CSV under a path containing ``smoke``) below the - ledger roots, which keep each smoke response's id. - -Two kinds of evidence need only the frozen predictions. OpenRouter ids -(``gen--...``) and OpenAI Responses ids (``resp_`` + 16 hex + -``00`` + 8 hex seconds) embed their creation time. On live households that -time falls within 23 seconds of the row's ``request_started_at``. A -creation time more than a minute before the request began means the response -already existed when the request was sent. - -A household counts as replayed when any evidence holds for its first recorded -response id. A full replay sums to under a second: every call came from the -cache. A partial replay sums to a second or more, so a later call in the -household (a repair, a retry round or another chunk) went to the provider. + ledger roots, which keep each smoke response's id and request times. + +Each kind of evidence is tied to the frozen request it explains, the +household's first call (the one whose id its rows keep): + +- ``cache``: an entry holding the response id was stored before the request + began. Both times are this machine's clock. +- ``spend_ledger``: a ledger outside the smokes records ``cache_hit`` for that + response id on the same model and household, at the frozen request's own + start time. A later run that replayed a response the board fetched live + therefore does not count. +- ``smoke_ledger`` / ``smoke_file``: an onboarding smoke served the same + response id and finished before the request began. +- ``openrouter_id`` / ``openai_id``: the frozen predictions alone suffice. + OpenRouter ids (``gen--...``) and OpenAI Responses ids + (``resp_`` + 16 hex + ``00`` + 8 hex seconds) embed their creation time, on + the provider's clock. On live households that time falls within 23 seconds + of the row's ``request_started_at``, so a creation time more than a minute + before the request began means the response already existed. + +A household counts as replayed when any evidence holds. A full replay sums to +under a second: every call came from the cache. A partial replay sums to a +second or more, so a later call in the household (another chunk, a repair, or +a re-request with a larger completion budget) went to the provider. """ from __future__ import annotations @@ -67,6 +78,9 @@ # Every replayed household in the frozen run sums to under 0.12 s; the fastest # live household (Gemini 3.5 Flash-Lite) takes 1.75 s. FULL_REPLAY_SECONDS = 1.0 +# A ledger record belongs to the frozen request when it starts at the same +# moment; the board's own ledger records it to the microsecond. +SAME_REQUEST_SECONDS = 0.5 RESPONSE_ID = re.compile( rb"(gen-\d{10}-[A-Za-z0-9]{10,40}" @@ -151,6 +165,12 @@ def walk_files(root: Path, suffix: str): yield Path(directory) / name +def _under_smoke(path: Path, root: Path) -> bool: + """Whether a file sits in a smoke run, judged below its root so a root + whose own path mentions a smoke does not make every file a smoke's.""" + return "smoke" in str(path.relative_to(root)) + + def ledger_records(roots: list[Path]) -> pd.DataFrame: records = [] for root in roots: @@ -160,54 +180,79 @@ def ledger_records(roots: list[Path]) -> pd.DataFrame: record = json.loads(line) except json.JSONDecodeError: continue - record["_smoke"] = "smoke" in str(path) + record["_smoke"] = _under_smoke(path, root) records.append(record) return pd.DataFrame(records) -def smoke_file_ids(roots: list[Path]) -> set[str]: - ids: set[str] = set() +def smoke_file_times(roots: list[Path]) -> dict[str, float]: + """When each smoke response was first served: the earliest completion time + of a smoke row carrying its id. Rows without a time cannot show order and + are skipped.""" + finished: dict[str, float] = {} + wanted = {"provider_response_id", "request_completed_at"} for root in roots: for path in walk_files(root, ".csv"): - if "smoke" not in str(path): + if not _under_smoke(path, root): continue try: frame = pd.read_csv( - path, - usecols=lambda column: column == "provider_response_id", - low_memory=False, + path, usecols=lambda column: column in wanted, low_memory=False ) except (ValueError, pd.errors.ParserError, UnicodeDecodeError): continue - if "provider_response_id" in frame: - ids.update(frame["provider_response_id"].dropna().astype(str)) - return ids + if not wanted <= set(frame.columns): + continue + completed = pd.to_numeric(frame["request_completed_at"], errors="coerce") + pairs = pd.DataFrame( + {"id": frame["provider_response_id"], "completed": completed} + ).dropna() + for response_id, when in zip(pairs["id"], pairs["completed"]): + key = str(response_id) + if key not in finished or when < finished[key]: + finished[key] = float(when) + return finished + + +def _ledger_hits(ledgers: pd.DataFrame) -> dict[str, list[tuple[str, str, float]]]: + """Cache hits outside the smokes: response id -> (model, household, start).""" + hits: dict[str, list[tuple[str, str, float]]] = {} + if ledgers.empty or "cache_hit" not in ledgers: + return hits + board = ledgers[~ledgers["_smoke"] & (ledgers["cache_hit"] == True)] # noqa: E712 + board = board.dropna(subset=["provider_response_id", "request_started_at"]) + columns = ["provider_response_id", "model", "scenario_id", "request_started_at"] + for response_id, model, scenario, started in board[columns].itertuples(index=False): + hits.setdefault(response_id, []).append((model, scenario, float(started))) + return hits + + +def _smoke_ledger_times(ledgers: pd.DataFrame) -> dict[str, float]: + """When a smoke first served each response id live.""" + if ledgers.empty or "cache_hit" not in ledgers: + return {} + smoke = ledgers[ledgers["_smoke"] & (ledgers["cache_hit"] != True)] # noqa: E712 + return ( + smoke.dropna(subset=["provider_response_id", "request_completed_at"]) + .groupby("provider_response_id")["request_completed_at"] + .min() + .to_dict() + ) def build( predictions: pd.DataFrame, stored: dict[str, float], ledgers: pd.DataFrame, - smoke_ids: set[str], + smoke_times: dict[str, float], ) -> list[dict]: latency = household_latency_seconds(predictions) grouped = predictions.groupby(["model", "scenario_id"], sort=True) first_id = grouped["provider_response_id"].first() started = grouped["request_started_at"].min() board_start = predictions.groupby("model")["request_started_at"].min() - - ledger_hit: set[str] = set() - smoke_live: dict[str, float] = {} - if not ledgers.empty: - board = ledgers[~ledgers["_smoke"] & (ledgers["cache_hit"] == True)] # noqa: E712 - ledger_hit = set(board["provider_response_id"].dropna()) - smoke = ledgers[ledgers["_smoke"] & (ledgers["cache_hit"] != True)] # noqa: E712 - smoke_live = ( - smoke.dropna(subset=["provider_response_id"]) - .groupby("provider_response_id")["request_completed_at"] - .min() - .to_dict() - ) + ledger_hits = _ledger_hits(ledgers) + smoke_ledger = _smoke_ledger_times(ledgers) rows = [] for (model, scenario), seconds in latency.items(): @@ -216,25 +261,28 @@ def build( recorded = first_id.get((model, scenario)) inner = provider_response_id(recorded) start = started.get((model, scenario)) - existed: dict[str, float] = {} + timed = pd.notna(start) + # When the response is known to have existed, by source and clock. + local: dict[str, float] = {} if isinstance(inner, str) and inner in stored: - existed["cache"] = stored[inner] + local["cache"] = stored[inner] + if recorded in smoke_ledger: + local["smoke_ledger"] = float(smoke_ledger[recorded]) + if recorded in smoke_times: + local["smoke_file"] = smoke_times[recorded] + existed = {name: when for name, when in local.items() if timed and when < start} created = id_created_at(inner) - if created is not None: + if created is not None and timed and created < start - MARGIN_SECONDS: name = "openrouter_id" if inner.startswith("gen-") else "openai_id" existed[name] = created - if recorded in smoke_live: - existed["smoke_ledger"] = float(smoke_live[recorded]) - evidence = sorted( - name - for name, when in existed.items() - if pd.notna(start) and when < start - MARGIN_SECONDS - ) - if recorded in ledger_hit: - evidence.append("spend_ledger") - if recorded in smoke_ids: - evidence.append("smoke_file") - evidence = sorted(set(evidence)) + evidence = set(existed) + if timed and any( + hit_model == model + and hit_scenario == scenario + and abs(hit_start - start) <= SAME_REQUEST_SECONDS + for hit_model, hit_scenario, hit_start in ledger_hits.get(recorded, ()) + ): + evidence.add("spend_ledger") if not evidence and seconds >= FULL_REPLAY_SECONDS: continue if not evidence: @@ -242,9 +290,11 @@ def build( else: replay = "full" if seconds < FULL_REPLAY_SECONDS else "partial" earliest = min(existed.values()) if existed else None - if "smoke_ledger" in evidence or "smoke_file" in evidence: + if {"smoke_ledger", "smoke_file"} & evidence: source = "onboarding smoke" - elif earliest is not None and earliest >= board_start[model]: + elif earliest is None: + source = "unknown" + elif earliest >= board_start[model]: source = "same run" else: source = "before the board rows" @@ -254,10 +304,10 @@ def build( "scenario_id": scenario, "household_latency_seconds": f"{seconds:.6f}", "replay": replay, - "evidence": "+".join(evidence), + "evidence": "+".join(sorted(evidence)), "source": source, "first_response_id": recorded, - "request_started_at": "" if pd.isna(start) else f"{start:.3f}", + "request_started_at": f"{start:.3f}" if timed else "", "response_existed_at": ( f"{min(existed.values()):.3f}" if existed else "" ), @@ -279,7 +329,7 @@ def main() -> None: predictions, cache_store_times(args.cache_dir), ledger_records(args.ledger_root), - smoke_file_ids(args.ledger_root), + smoke_file_times(args.ledger_root), ) args.output.parent.mkdir(parents=True, exist_ok=True) with open(args.output, "w", newline="") as handle: diff --git a/tests/test_cost_latency.py b/tests/test_cost_latency.py index cad952a4..a43555d0 100644 --- a/tests/test_cost_latency.py +++ b/tests/test_cost_latency.py @@ -180,7 +180,8 @@ def test_a_replay_can_only_lower_the_median(seconds, data): def test_a_median_under_a_second_means_most_households_replayed(n_fast, n_slow): """With replays under the line and calls above it, the median is under the line when more than half the households replayed and at or above it when - fewer than half did: the paper's dash rule is a majority-replay rule.""" + fewer than half did. At exactly half it can fall either side (the tie test + below), so the paper's dash rule is a majority-replay rule only off ties.""" if n_fast + n_slow == 0: return seconds = [0.02] * n_fast + [30.0] * n_slow @@ -193,6 +194,25 @@ def test_a_median_under_a_second_means_most_households_replayed(n_fast, n_slow): assert median >= REPLAYED_HOUSEHOLD_SECONDS +@given( + st.integers(min_value=1, max_value=12), + st.floats(min_value=0.0, max_value=0.12), + st.floats(min_value=1.0, max_value=3.0), +) +def test_at_exactly_half_replays_the_median_is_the_middle_pairs_mean(n, replay, call): + """Half replays and half calls: the median is the mean of the slowest + replay and the fastest call, which is under a second only when the calls + are fast enough (50 replays at 0.02 s and 50 calls at 1.75 s give 0.885 s).""" + seconds = [replay] * n + [call] * n + median = model_cost_latency( + pd.DataFrame([_row("m", f"s{i}", 0.01, t) for i, t in enumerate(seconds)]) + )["m"]["latencySeconds"] + assert median == pytest.approx((replay + call) / 2) + assert (median < REPLAYED_HOUSEHOLD_SECONDS) == ( + replay + call < 2 * REPLAYED_HOUSEHOLD_SECONDS + ) + + @given( st.lists(st.floats(min_value=1.0, max_value=500.0), min_size=1, max_size=10), st.integers(min_value=1, max_value=10), diff --git a/tests/test_disclosures.py b/tests/test_disclosures.py index 560082c2..6cd217d4 100644 --- a/tests/test_disclosures.py +++ b/tests/test_disclosures.py @@ -75,6 +75,10 @@ r"recorded\s+at\s+the\s+run\s+level", r"Anthropic\s+batch\s+adapter", r"not\s+measured\s+per\s+call", + # The harness restarts its timer on each retry and times only the attempt + # that succeeded; a replayed row's cost is recomputed when it is written. + r"spans\s+any\s+provider-side\s+retry", + r"keeps\s+the\s+cost\s+recorded\s+when\s+it\s+was\s+generated", # Exclusion covers what the audits found, not every defect or unstated # input: an output a fix or reading not run on the reference engine would # move can still be scored (the paper's limitations say so). diff --git a/tests/test_latency_replay_audit.py b/tests/test_latency_replay_audit.py index 16e8eda1..0367891f 100644 --- a/tests/test_latency_replay_audit.py +++ b/tests/test_latency_replay_audit.py @@ -312,3 +312,128 @@ def test_the_rendered_paper_states_why_each_latency_is_missing(): assert f"more than {r.subsecond_median_shift_fmt} seconds" in text assert f"A further {r.partial_replay_count} households replay" in text assert "Claude Fable 5's board rows carry no usage or timing fields" in text + + +# ----- the builder's evidence rules (scripts/audit_latency_replays.py) ------- + + +def _frozen(rows): + """Frozen-style rows: (model, scenario, response id, start, household s).""" + records = [] + for model, scenario, response_id, start, seconds in rows: + for variable in ("a", "b"): + records.append( + { + "model": model, + "scenario_id": scenario, + "variable": variable, + "provider_response_id": response_id, + "request_started_at": start, + "elapsed_seconds": seconds / 2, + } + ) + return pd.DataFrame(records) + + +def _ledger(rows): + """Ledger records: (response id, model, scenario, start, cache hit, smoke).""" + return pd.DataFrame( + [ + { + "provider_response_id": response_id, + "model": model, + "scenario_id": scenario, + "request_started_at": start, + "request_completed_at": start + 30.0, + "cache_hit": hit, + "_smoke": smoke, + } + for response_id, model, scenario, start, hit, smoke in rows + ] + ) + + +def _built(predictions, stored=None, ledgers=None, smoke_times=None): + from scripts.audit_latency_replays import build + + rows = build( + predictions, + stored or {}, + ledgers if ledgers is not None else pd.DataFrame(), + smoke_times or {}, + ) + return {(row["model"], row["scenario_id"]): row for row in rows} + + +def test_a_later_replay_of_a_live_board_response_is_not_evidence(): + """A run after the board that replays a response the board fetched live + records a cache hit for the same id, but not at the board request.""" + board = _frozen([("m", "s1", "chatcmpl-live", 1_000.0, 30.0)]) + later = _ledger([("chatcmpl-live", "m", "s1", 5_000.0, True, False)]) + assert _built(board, ledgers=later) == {} + same = _ledger([("chatcmpl-live", "m", "s1", 1_000.0, True, False)]) + row = _built(board, ledgers=same)[("m", "s1")] + assert (row["replay"], row["evidence"]) == ("partial", "spend_ledger") + other = _ledger([("chatcmpl-live", "m", "s2", 1_000.0, True, False)]) + assert _built(board, ledgers=other) == {} + + +def test_a_smoke_counts_only_if_it_finished_before_the_request(): + board = _frozen([("m", "s1", "chatcmpl-x", 1_000.0, 0.02)]) + after = _built(board, smoke_times={"chatcmpl-x": 1_500.0})[("m", "s1")] + assert (after["replay"], after["evidence"]) == ("untraced", "") + before = _built(board, smoke_times={"chatcmpl-x": 900.0})[("m", "s1")] + assert (before["replay"], before["evidence"], before["source"]) == ( + "full", + "smoke_file", + "onboarding smoke", + ) + ledger = _ledger([("chatcmpl-x", "m", "s1", 1_500.0, False, True)]) + late = _built(board, ledgers=ledger)[("m", "s1")] + assert late["evidence"] == "" + + +def test_a_cached_response_counts_only_if_stored_before_the_request(): + board = _frozen([("m", "s1", "chatcmpl-x", 1_000.0, 45.0)]) + assert _built(board, stored={"chatcmpl-x": 1_010.0}) == {} + row = _built(board, stored={"chatcmpl-x": 990.0})[("m", "s1")] + assert (row["replay"], row["evidence"]) == ("partial", "cache") + + +def test_id_times_need_the_margin_and_live_households_are_not_listed(): + created = 1_783_264_675 + early = f"gen-{created}-abcdefghijABCDEFGHIJ" + board = _frozen( + [ + ("m", "s1", early, created + 120.0, 0.03), + ("m", "s2", early.replace("abc", "xyz"), created + 10.0, 40.0), + ("m", "s3", "chatcmpl-fast", created + 5.0, 0.04), + ] + ) + built = _built(board) + assert built[("m", "s1")]["evidence"] == "openrouter_id" + assert built[("m", "s1")]["replay"] == "full" + assert ("m", "s2") not in built + assert built[("m", "s3")]["replay"] == "untraced" + + +def test_smoke_file_times_read_columns_by_name(tmp_path): + """Smoke CSVs order their columns differently; a row without a completion + time cannot show order and is skipped, and files outside a smoke path are + not read.""" + from scripts.audit_latency_replays import smoke_file_times + + smoke = tmp_path / "x_smoke" / "run" + smoke.mkdir(parents=True) + pd.DataFrame( + { + "request_completed_at": [20.0, 10.0, None], + "provider_response_id": ["gen-a", "gen-a", "gen-b"], + } + ).to_csv(smoke / "predictions.csv", index=False) + board = tmp_path / "board" + board.mkdir() + pd.DataFrame( + {"provider_response_id": ["gen-c"], "request_completed_at": [5.0]} + ).to_csv(board / "predictions.csv", index=False) + assert smoke_file_times([tmp_path]) == {"gen-a": 10.0}