Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 65 additions & 54 deletions bench/site_facts.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,20 +22,23 @@
stated flatly.

RECORDED. Seconds, bytes, milliseconds. These are a machine, not a language,
and README.md is scrupulous about saying so: it names the exact box and it
publishes other sessions reading 0.16 / 1.29 / 0.19 and 0.22 / 1.81 / 0.27
where the table's read 0.13 / 1.17 / 0.16. So a recorded number is never emitted on its own. It is
emitted as a group carrying the machine that produced it, the command that
prints it, and every session README.md publishes, so the site can draw the
spread and lead with the ratio. The ratio is the claim. The second is a
circumstance.

README.md is the source for the recorded class because that is where this
project publishes these measurements. Parsing it means the site restates the
repository rather than remembering it. When a pattern below stops matching,
that is a failure and not a default: a sentence this was written to cover was
reworded, so the site would be quoting something nobody is maintaining. Same
rule `doc_numbers.py` already applies to itself.
and the repository is scrupulous about saying so: BENCHMARKS.md names the
exact box and publishes other sessions reading 0.16 / 1.29 / 0.19 and
0.22 / 1.81 / 0.27 where the table's read 0.13 / 1.17 / 0.16. So a recorded
number is never emitted on its own. It is emitted as a group carrying the
machine that produced it, the command that prints it, and every session
BENCHMARKS.md publishes, so the site can draw the spread and lead with the
ratio. The ratio is the claim. The second is a circumstance.

README.md and BENCHMARKS.md are the source for the recorded class because
that is where this project publishes these measurements: the claims a reader
meets first in README.md, and since 0.20.1 the build, run-time and server
tables behind them in BENCHMARKS.md. Each pattern below names the file it
reads. Parsing them means the site restates the repository rather than
remembering it. When a pattern stops matching, that is a failure and not a
default: a sentence this was written to cover was reworded or moved, so the
site would be quoting something nobody is maintaining. Same rule
`doc_numbers.py` already applies to itself.
"""

from __future__ import annotations
Expand Down Expand Up @@ -64,7 +67,8 @@


def flat(text: str) -> str:
"""README.md wraps at about 80 columns, so match against one long line."""
"""The repository's prose wraps at about 80 columns, so match against one
long line."""
return re.sub(r"\s+", " ", text)


Expand Down Expand Up @@ -301,74 +305,85 @@ def context_scope() -> dict:
}


# (key, pattern, group names). A miss is an error, never a default.
RECORDED_PATTERNS: list[tuple[str, str, tuple[str, ...]]] = [
("machine", r"\(([^()]*Ryzen[^()]*)\)", ("name",)),
# The two files the recorded class is published in.
README = "README.md"
BENCHMARKS = "BENCHMARKS.md"

# (key, file, pattern, group names). A miss is an error, never a default.
RECORDED_PATTERNS: list[tuple[str, str, str, tuple[str, ...]]] = [
("machine", BENCHMARKS, r"\(([^()]*Ryzen[^()]*)\)", ("name",)),
(
"edit_loop",
BENCHMARKS,
r"\| \*\*one module's body\*\* \| \*\*([\d.]+)\*\* \| ([\d.]+) \| ([\d.]+) \|",
("iyi", "crystal", "go"),
),
(
"edit_loop_sessions",
BENCHMARKS,
r"other sessions on the same box read (\d+\.\d+ / \d+\.\d+ / \d+\.\d+) and\s+"
r"(\d+\.\d+ / \d+\.\d+ / \d+\.\d+) on the first row",
("a", "b"),
),
("edit_loop_ratio", r"costs \*\*([\d.]+)x less\*\*", ("times",)),
("edit_loop_ratio", README, r"costs \*\*([\d.]+)x less\*\*", ("times",)),
(
"hello_binary",
README,
r"is a (?:\*\*)?([\d,]+) KB(?:\*\*)? binary that starts in (?:\*\*)?([\d.]+) ms",
("kb", "ms"),
),
(
"crystal_binary",
README,
r"library is ([\d,]+) KB and ([\d.]+) ms",
("kb", "ms"),
),
(
"full_build",
BENCHMARKS,
r"\| generated pair, ([\d,]+) lines \| ([\d.]+) s \| \*\*([\d.]+) s\*\* \|",
("lines", "iyi", "go"),
),
(
"hello_build",
BENCHMARKS,
r"\| `hello` \(([\d,]+) lines\) \| \*\*([\d.]+) s\*\* \| ([\d.]+) s \|",
("lines", "iyi", "go"),
),
("front_end", r"answers `hello` in \*\*([\d.]+) s\*\*", ("seconds",)),
("front_end", README, r"answers `hello` in \*\*([\d.]+) s\*\*", ("seconds",)),
(
"runtime_string",
BENCHMARKS,
r"\| string building \| ([\d.]+)x \| \*\*([\d.]+)x\*\* \|",
("as_it_runs", "gc_off"),
),
(
"runtime_hash",
BENCHMARKS,
r"\| hash insert and read \| ([\d.]+)x \| ([\d.]+)x \|",
("as_it_runs", "gc_off"),
),
(
"runtime_array",
BENCHMARKS,
r"\| array append and read \| ([\d.]+)x \| ([\d.]+)x \|",
("as_it_runs", "gc_off"),
),
(
"runtime_arithmetic",
BENCHMARKS,
r"\| arithmetic \| ([\d.]+)x \| ([\d.]+)x \|",
("as_it_runs", "gc_off"),
),
(
"prelude_vs_crystal",
r"([\d,]+) lines of Crystal's standard library instead of ([\d,]+)",
("crystal", "iyi"),
),
(
"context_pack",
README,
r"at \*\*(\d+)–(\d+)%\*\* of the sources' size",
("size_low", "size_high"),
),
(
"context_tokens",
README,
r"\*\*(\d+)–(\d+)% fewer prompt tokens\*\*",
("low", "high"),
),
Expand All @@ -380,17 +395,19 @@ def context_scope() -> dict:
# up, and the site says that rather than a count nobody maintains.
(
"context_runs",
README,
r"over the runs written up in ([A-Z_]+\.md §\d+)",
("where",),
),
(
"context_rounds",
README,
r"rounds to green tied ([a-z]+) times and lost\s+([a-z]+)\.",
("tied", "lost"),
),
]

# Which command prints each recorded group, quoted from README.md's own
# Which command prints each recorded group, quoted from the repository's own
# attribution of it. The site shows this beside the number.
COMMANDS: dict[str, str] = {
"edit_loop": "python3 bench/incremental.py",
Expand Down Expand Up @@ -421,16 +438,16 @@ def context_scope() -> dict:
}


def recorded(readme: str) -> tuple[dict, list[str]]:
text = flat(readme)
def recorded() -> tuple[dict, list[str]]:
texts = {name: flat((REPO / name).read_text()) for name in (README, BENCHMARKS)}
out: dict = {}
missing: list[str] = []
for key, pattern, names in RECORDED_PATTERNS:
m = re.search(pattern, text)
for key, source, pattern, names in RECORDED_PATTERNS:
m = re.search(pattern, texts[source])
if not m:
missing.append(
f"{key}: /{pattern}/ no longer matches README.md. The sentence "
f"this quoted was reworded or removed, so the site would be "
f"{key}: /{pattern}/ no longer matches {source}. The sentence "
f"this quoted was reworded or moved, so the site would be "
f"publishing a number nobody is maintaining"
)
continue
Expand All @@ -439,16 +456,18 @@ def recorded(readme: str) -> tuple[dict, list[str]]:


def sessions(raw: dict) -> list[dict[str, float]]:
"""Every session README.md publishes for the edit loop, not just the best.

The spread is the point. README.md prints three readings and says to read
the columns against each other because they pay the same machine together,
so the site draws all of them and leads with the ratio they agree on.

README.md prints a third column, another compiler's, on the same row. The
site publishes iyi against the compiler it forked and nothing else, so that
column is parsed, because a drift in the row is still a drift, and then
dropped.
"""Every session BENCHMARKS.md publishes for the edit loop, not just the
best.

The spread is the point. BENCHMARKS.md prints three readings and says to
read the columns against each other because they pay the same machine
together, so the site draws all of them and leads with the ratio they
agree on.

BENCHMARKS.md prints a third column, another compiler's, on the same row.
The site publishes iyi against the compiler it forked and nothing else, so
that column is parsed, because a drift in the row is still a drift, and
then dropped.
"""
# The table's own row is the first reading; the sentence under it gives
# the other two sessions.
Expand All @@ -465,8 +484,8 @@ def machine(name: str) -> str:


def build() -> tuple[dict, list[str]]:
readme = (REPO / "README.md").read_text()
raw, missing = recorded(readme)
readme = (REPO / README).read_text()
raw, missing = recorded()
if missing:
return {}, missing

Expand All @@ -487,7 +506,7 @@ def build() -> tuple[dict, list[str]]:
"recorded": {
"edit_loop": {
# The claim, which is machine-independent and held across every
# session README.md publishes.
# session BENCHMARKS.md publishes.
"claim": {
"kind": "ratio",
"value": float(raw["edit_loop_ratio"]["times"]),
Expand Down Expand Up @@ -547,14 +566,6 @@ def build() -> tuple[dict, list[str]]:
"command": COMMANDS["runtime_string"],
"subject": "the same program under both libraries",
},
"library_size": {
"iyi": int(raw["prelude_vs_crystal"]["iyi"].replace(",", "")),
"crystal": int(raw["prelude_vs_crystal"]["crystal"].replace(",", "")),
"unit": "lines",
"machine": None,
"command": "python3 bench/doc_numbers.py",
"subject": "what a program has",
},
"front_end": {
"seconds": float(raw["front_end"]["seconds"]),
"unit": "s",
Expand Down Expand Up @@ -586,7 +597,7 @@ def build() -> tuple[dict, list[str]]:
"bench/doc_numbers.py measured(), the bench's own project "
"generator, and git"
),
"recorded_source": "README.md, parsed",
"recorded_source": "README.md and BENCHMARKS.md, parsed",
"generator": "bench/site_facts.py",
},
}
Expand Down
17 changes: 9 additions & 8 deletions doc/STACK.md
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,9 @@ site/
reads the iyi tree at it, and refuses a recording
taken from anything else
facts.mjs runs bench/site_facts.py, writes generated/facts.json
targets.mjs reads the platforms out of iyi's CI workflow and the
run set out of README.md, writes generated/targets.json
targets.mjs reads the platforms out of iyi's CI workflow, the
run set out of README.md and the object audit out
of INSTALL.md, writes generated/targets.json
releases.mjs reads CHANGELOG.md's release headings, writes
generated/releases.json
samples.mjs transcludes samples/iyi, samples/tour, records/break
Expand Down Expand Up @@ -76,12 +77,12 @@ This is the load-bearing part, and it is the reason the site cannot drift.
repository through `bench/site_facts.py`, which has two sources. Structural
numbers come from `bench/doc_numbers.py`'s `measured()`, the same function that
already gates `README.md` in CI, so the page and the gate cannot disagree.
Recorded numbers (seconds, bytes, milliseconds) are parsed out of `README.md`,
because that is where this project publishes them, and they are emitted only as
groups carrying the machine that produced them, the command that prints them,
and every session the README publishes. If any pattern stops matching, the
script fails rather than defaulting, which is the same rule
`bench/doc_numbers.py` applies to itself.
Recorded numbers (seconds, bytes, milliseconds) are parsed out of `README.md`
and `BENCHMARKS.md`, because that is where this project publishes them, and
they are emitted only as groups carrying the machine that produced them, the
command that prints them, and every session BENCHMARKS.md publishes. If any
pattern stops matching, the script fails rather than defaulting, which is the
same rule `bench/doc_numbers.py` applies to itself.

The build chain runs `bench/site_facts.py` and writes
`src/generated/facts.json`. The components read that file. `Measure`
Expand Down
10 changes: 5 additions & 5 deletions records/agent.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"recorded": {
"compiler": "iyi 0.20.0 (built on Crystal 1.22.0-dev [224ed74fc] (2026-11-02))",
"commit": "224ed74fc7ba9d78345dfd10f21ece4b499702fa",
"compiler": "iyi 0.20.1 (built on Crystal 1.22.0-dev [91d012b1e] (2026-11-04))",
"commit": "91d012b1ed978c7df5cc033624ec161a2daf01b6",
"machine": "Linux 7.0.0-38-generic x86_64, QEMU Virtual CPU version 2.5+",
"command": "npm run record:agent",
"when": "2026-11-02T07:19:31.432Z"
"when": "2026-11-04T04:32:53.960Z"
},
"project": [
{
Expand Down Expand Up @@ -104,7 +104,7 @@
"command": "iyi test --json .",
"exitCode": 0,
"payload": "stdout",
"stdout": "{\"tests\":[{\"file\":\"./app_test.iyi\",\"status\":\"pass\",\"seconds\":0.394}],\"passed\":1,\"failed\":0,\"skipped\":0}\n",
"stdout": "{\"tests\":[{\"file\":\"./app_test.iyi\",\"status\":\"pass\",\"seconds\":0.281}],\"passed\":1,\"failed\":0,\"skipped\":0}\n",
"stderr": ""
},
{
Expand Down Expand Up @@ -167,7 +167,7 @@
},
"serverInfo": {
"name": "iyi",
"version": "0.20.0"
"version": "0.20.1"
}
},
"tools": [
Expand Down
6 changes: 3 additions & 3 deletions records/diagnostics.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"recorded": {
"compiler": "iyi 0.20.0 (built on Crystal 1.22.0-dev [224ed74fc] (2026-11-02))",
"commit": "224ed74fc7ba9d78345dfd10f21ece4b499702fa",
"compiler": "iyi 0.20.1 (built on Crystal 1.22.0-dev [91d012b1e] (2026-11-04))",
"commit": "91d012b1ed978c7df5cc033624ec161a2daf01b6",
"machine": "Linux 7.0.0-38-generic x86_64, QEMU Virtual CPU version 2.5+",
"command": "npm run record:diagnostics",
"when": "2026-11-02T07:19:23.048Z"
"when": "2026-11-04T04:32:47.944Z"
},
"cases": [
{
Expand Down
Loading
Loading