Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 23 additions & 13 deletions bench/site_facts.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@

RECORDED. Seconds, bytes, milliseconds. These are a machine, not a language,
and README.md is scrupulous about saying so: it names the exact box and it
publishes a tired session reading 0.22 / 1.81 / 0.27 where a good one read
0.13 / 1.17 / 0.16. So a recorded number is never emitted on its own. It is
publishes other sessions reading 0.16 / 1.29 / 0.19 and 0.22 / 1.81 / 0.27
where the table's read 0.13 / 1.17 / 0.16. So a recorded number is never emitted on its own. It is
emitted as a group carrying the machine that produced it, the command that
prints it, and every session README.md publishes, so the site can draw the
spread and lead with the ratio. The ratio is the claim. The second is a
Expand Down Expand Up @@ -306,14 +306,14 @@ def context_scope() -> dict:
("machine", r"\(([^()]*Ryzen[^()]*)\)", ("name",)),
(
"edit_loop",
r"\| \*\*rebuild after one edit\*\* \| \*\*([\d.]+)\*\* \| ([\d.]+) \| ([\d.]+) \|",
r"\| \*\*one module's body\*\* \| \*\*([\d.]+)\*\* \| ([\d.]+) \| ([\d.]+) \|",
("iyi", "crystal", "go"),
),
(
"edit_loop_sessions",
r"read (\d+\.\d+ / \d+\.\d+ / \d+\.\d+) and (\d+\.\d+ / \d+\.\d+ / \d+\.\d+) "
r"on the first row, and a tired session reads (\d+\.\d+ / \d+\.\d+ / \d+\.\d+)",
("a", "b", "tired"),
r"other sessions on the same box read (\d+\.\d+ / \d+\.\d+ / \d+\.\d+) and\s+"
r"(\d+\.\d+ / \d+\.\d+ / \d+\.\d+) on the first row",
("a", "b"),
),
("edit_loop_ratio", r"costs \*\*([\d.]+)x less\*\*", ("times",)),
(
Expand Down Expand Up @@ -369,16 +369,23 @@ def context_scope() -> dict:
),
(
"context_tokens",
r"\*\*(\d+)–(\d+)% fewer prompt tokens\*\* over ([a-z]+(?:-[a-z]+)*) measured runs",
("low", "high", "runs"),
r"\*\*(\d+)–(\d+)% fewer prompt tokens\*\*",
("low", "high"),
),
# The other half of the same measurement, and the half that does not go
# iyi's way: the model's rounds to green. README.md publishes it in the
# same sentence as the token saving, so the site parses it from there and
# states it beside the saving rather than leaving a reader to assume.
# README.md no longer counts the runs; it names where they are written
# up, and the site says that rather than a count nobody maintains.
(
"context_runs",
r"over the runs written up in ([A-Z_]+\.md §\d+)",
("where",),
),
(
"context_rounds",
r"rounds-to-green are the model's, tied ([a-z]+) times and lost ([a-z]+),",
r"rounds to green tied ([a-z]+) times and lost\s+([a-z]+)\.",
("tied", "lost"),
),
]
Expand All @@ -400,6 +407,7 @@ def context_scope() -> dict:
"runtime_arithmetic": "python3 bench/runtime.py",
"context_pack": "python3 bench/context_pack.py",
"context_tokens": "python3 bench/context_pack.py",
"context_runs": "python3 bench/context_pack.py",
"context_rounds": "python3 bench/context_pack.py",
}

Expand Down Expand Up @@ -442,8 +450,10 @@ def sessions(raw: dict) -> list[dict[str, float]]:
column is parsed, because a drift in the row is still a drift, and then
dropped.
"""
rows = []
for slot in ("a", "b", "tired"):
# The table's own row is the first reading; the sentence under it gives
# the other two sessions.
rows = [{"iyi": float(raw["table"]["iyi"]), "crystal": float(raw["table"]["crystal"])}]
for slot in ("a", "b"):
iyi, crystal, _ = (float(v) for v in raw[slot].split(" / "))
rows.append({"iyi": iyi, "crystal": crystal})
return rows
Expand All @@ -461,7 +471,7 @@ def build() -> tuple[dict, list[str]]:
return {}, missing

default_machine = machine(raw["machine"]["name"])
loop = sessions(raw["edit_loop_sessions"])
loop = sessions({**raw["edit_loop_sessions"], "table": raw["edit_loop"]})
ratios = [round(s["crystal"] / s["iyi"], 2) for s in loop]
tree = measured()
# Counted here rather than by `measured()`, which measures that project's
Expand Down Expand Up @@ -562,7 +572,7 @@ def build() -> tuple[dict, list[str]]:
"size_high": int(raw["context_pack"]["size_high"]),
"tokens_low": int(raw["context_tokens"]["low"]),
"tokens_high": int(raw["context_tokens"]["high"]),
"runs": raw["context_tokens"]["runs"],
"runs_where": raw["context_runs"]["where"],
"rounds_tied": raw["context_rounds"]["tied"],
"rounds_lost": raw["context_rounds"]["lost"],
"unit": "%",
Expand Down
12 changes: 6 additions & 6 deletions records/agent.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"recorded": {
"compiler": "iyi 0.17.0 (built on Crystal 1.22.0-dev [fe3db8354] (2026-10-14))",
"commit": "fe3db8354f56ce84d411611d48e6dad235172275",
"machine": "Linux 7.0.0-31-generic x86_64, QEMU Virtual CPU version 2.5+",
"compiler": "iyi 0.20.0 (built on Crystal 1.22.0-dev [224ed74fc] (2026-11-02))",
"commit": "224ed74fc7ba9d78345dfd10f21ece4b499702fa",
"machine": "Linux 7.0.0-38-generic x86_64, QEMU Virtual CPU version 2.5+",
"command": "npm run record:agent",
"when": "2026-10-14T11:26:22.139Z"
"when": "2026-11-02T07:19:31.432Z"
},
"project": [
{
Expand Down Expand Up @@ -104,7 +104,7 @@
"command": "iyi test --json .",
"exitCode": 0,
"payload": "stdout",
"stdout": "{\"tests\":[{\"file\":\"./app_test.iyi\",\"status\":\"pass\",\"seconds\":0.267}],\"passed\":1,\"failed\":0,\"skipped\":0}\n",
"stdout": "{\"tests\":[{\"file\":\"./app_test.iyi\",\"status\":\"pass\",\"seconds\":0.394}],\"passed\":1,\"failed\":0,\"skipped\":0}\n",
"stderr": ""
},
{
Expand Down Expand Up @@ -167,7 +167,7 @@
},
"serverInfo": {
"name": "iyi",
"version": "0.17.0"
"version": "0.20.0"
}
},
"tools": [
Expand Down
21 changes: 21 additions & 0 deletions records/break/share_reach.iyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# Breaks III.4.4's reach check: the block names nothing unshareable, but
# `greeting`, which it calls, reads `GREETING`, an `Array` any worker could
# reach while another changes it. The compiler walks the calls from the
# block and warns at the block, by the shortest path. A warning, so the
# build is asked to fail on one: `--error-on-warnings`, as CI would.
# `@[SafeReach("...")]` on the constant, saying why it is safe, quiets it.
module diagnostics/share_reach

import std/http::{Response, Server}
import std/socket::{IyiSocket}

GREETING = ["hello", "from", "every", "core"]

def greeting : String
GREETING.join(" ")
end

listener = IyiSocket.listen("127.0.0.1", 0)
Server.serve(listener) do |req|
Response.new(200, greeting)
end
17 changes: 17 additions & 0 deletions records/break/share_serve_capture.iyi
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
# Breaks III.4.4 as III.4.13 applies it: `Server.serve` runs each connection
# on the pool's workers, so its block is checked as `spawn_parallel`'s is,
# and every value it captures must be `Share`. An `Array` is not: two
# workers appending at once would race its size. Everything else here is
# legal, and the refusal names both ways out - the array behind a `Mutex`,
# or `workers: 1`, which keeps the server on one thread.
module diagnostics/share_serve_capture

import std/http::{Response, Server}
import std/socket::{IyiSocket}

seen = [] of String
listener = IyiSocket.listen("127.0.0.1", 0)
Server.serve(listener) do |req|
seen << req.path
Response.new(200, "ok")
end
26 changes: 22 additions & 4 deletions records/diagnostics.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"recorded": {
"compiler": "iyi 0.17.0 (built on Crystal 1.22.0-dev [fe3db8354] (2026-10-14))",
"commit": "fe3db8354f56ce84d411611d48e6dad235172275",
"machine": "Linux 7.0.0-31-generic x86_64, QEMU Virtual CPU version 2.5+",
"compiler": "iyi 0.20.0 (built on Crystal 1.22.0-dev [224ed74fc] (2026-11-02))",
"commit": "224ed74fc7ba9d78345dfd10f21ece4b499702fa",
"machine": "Linux 7.0.0-38-generic x86_64, QEMU Virtual CPU version 2.5+",
"command": "npm run record:diagnostics",
"when": "2026-10-14T11:26:14.391Z"
"when": "2026-11-02T07:19:23.048Z"
},
"cases": [
{
Expand Down Expand Up @@ -105,6 +105,24 @@
"command": "IYI_PATH=<iyi>/src iyi build --no-codegen --no-color records/break/errors_non_exhaustive.iyi",
"exitCode": 1,
"stderr": "Showing last frame. Use --error-trace for full trace.\n\nIn records/break/errors_non_exhaustive.iyi:28:1\n\n 28 | case read(\"missing\")\n ^\nError: case is not exhaustive.\n\nMissing types:\n - Diagnostics::ErrorsNonExhaustive::IOError\n"
},
{
"id": "share-serve-capture",
"rule": "III.4",
"title": "a server's block that captures an Array",
"path": "records/break/share_serve_capture.iyi",
"command": "IYI_PATH=<iyi>/src iyi build --no-codegen --no-color records/break/share_serve_capture.iyi",
"exitCode": 1,
"stderr": "Showing last frame. Use --error-trace for full trace.\n\nIn records/break/share_serve_capture.iyi:15:3\n\n 15 | seen << req.path\n ^---\nError: the block Server.serve keeps, which may run on other threads captures `seen : Array(String)`, which is not Share: Array(String)'s field @size is assigned in `append`. Put it behind a lock, a `Mutex(Array(String))` (std/mutex), or pass `workers: 1` so the block runs on one thread (SPEC.md III.4.4)\n"
},
{
"id": "share-reach",
"rule": "III.4",
"title": "a server's block that reaches module state through a call",
"path": "records/break/share_reach.iyi",
"command": "IYI_PATH=<iyi>/src iyi build --no-codegen --no-color --error-on-warnings records/break/share_reach.iyi",
"exitCode": 1,
"stderr": "In records/break/share_reach.iyi:19:24\n\n 19 | Server.serve(listener) do |req|\n ^\nWarning: the block Server.serve keeps, which may run on other threads reaches `GREETING : Array(String)` (not Share) through `greeting` → `GREETING.join`: Array(String)'s field @size is assigned in `append`. If it is only read, make it a `List(String)` (std/list), which is Share. Keep the value behind a `Mutex` (std/mutex), or, when every thread may reach it safely - it is set once and then only read, or guarded some other way - say why on the constant: `@[SafeReach(\"...\")]` (SPEC.md III.4.4)\n\nA total of 1 warnings were found.\n"
}
]
}
Loading
Loading