SEO: titles and descriptions that fit, and a standard library reference

README.md

@@ -45,9 +45,10 @@ cite exists, in this tree or in iyi's.
 
 `records/` holds what only a machine with the compiler can produce: the wasm
 modules the playground runs, the diagnostics the break-this-rule exercises
-show, the agent loop the agents page is a transcript of, and the syntax
-highlighting of every listing. They are committed, and they are evidence about
-**a release**, not about somebody's checkout.
+show, the agent loop the agents page is a transcript of, the syntax
+highlighting of every listing, and the standard library's surface the
+reference under `/reference/` is cut from. They are committed, and they are
+evidence about **a release**, not about somebody's checkout.
 
 That is what makes them checkable without making the build brittle. Every
 recorder refuses to run against anything but the tag CHANGELOG.md states, and
@@ -75,6 +76,16 @@ target README.md publishes, and wasi-sdk renamed that sysroot to
 `wasm32-wasip1` and dropped the old name afterwards. `record:wasm` refuses one
 without that sysroot by name rather than failing at the linker.
 
+`records/stdlib.json` is what `iyi doc` printed for every module directly
+under `src/std/` and every type `iyi doc prelude` lists, with the digest of
+each source. `scripts/stdlib.mjs` refuses it on the build when the release
+ships a module it does not cover or a file it was not printed from, then reads
+each module's header comment out of the tag (no compiler output carries it)
+and writes `src/generated/stdlib.json`. `record:stdlib` needs only the
+compiler, so the release's own binary will do: point `IYI_BUILD` at the `bin/`
+of an unpacked release, `iyi` or `iyi.exe`, and `IYI_REPO` at a worktree at
+its tag.
+
 `records/agent/` is the second thing this repository authors in iyi: four
 modules, left broken on purpose, that `record:agent` takes to green with the
 compiler's own verbs and keeps every frame of. That recorder is also the gate

astro.config.mjs

@@ -2,6 +2,7 @@ import { defineConfig } from "astro/config";
 import mdx from "@astrojs/mdx";
 import sitemap from "@astrojs/sitemap";
 import { unified } from "@astrojs/markdown-remark";
+import { readFileSync } from "node:fs";
 
 // GitHub Pages serves this repository at `iyi.dev` (the CNAME under public/,
 // which the build copies into dist/ - so the domain is this file's and a
@@ -10,12 +11,26 @@ import { unified } from "@astrojs/markdown-remark";
 const site = process.env.SITE_ORIGIN ?? "https://iyi.dev";
 const base = process.env.SITE_BASE ?? "/";
 
+// The project documents scripts/project.mjs marks as not this project's to
+// rank for. Their pages carry a robots noindex, and a sitemap that listed them
+// would be asking a crawler to fetch a page in order to be told to drop it.
+// Read from the generated record, which `npm run generate` writes before any
+// build, so the list is the script's and is not typed a second time here.
+const project = JSON.parse(
+  readFileSync(new URL("./src/generated/project.json", import.meta.url), "utf8"),
+);
+const unlisted = new Set(
+  project.documents
+    .filter((document) => !document.indexed)
+    .map((document) => new URL(`${base.replace(/\/$/, "")}/project/${document.id}/`, site).href),
+);
+
 export default defineConfig({
   site,
   base,
   output: "static",
   trailingSlash: "always",
-  integrations: [mdx(), sitemap()],
+  integrations: [mdx(), sitemap({ filter: (page) => !unlisted.has(page) })],
 
   // A lesson's URL is its file name, and a published URL outlives a rename:
   // the old one answers with a page that sends the reader to the new one.

doc/ART-DIRECTION.md

@@ -7,8 +7,11 @@ the other way, the reason it did not is written down. Rendered proof is in
 ## The concept
 
 **A plain, friendly site with a mascot, and every number on it checkable.**
-The one line above the fold says what iyi is for: a friendly, fast language for
-people and their agents. Beside it stands the mascot, a red heart with an `i`
+The one line above the fold says what iyi is for: a friendly, fast programming
+language for people and their agents. "Programming language" is in it on
+purpose: it is the category a reader searches for, and a line that leaves it
+out leaves a search engine guessing what the site is about. Beside it stands
+the mascot, a red heart with an `i`
 on each cheek, waving. Under it a real program, read out of the repository at
 build time, and the bytes the compiler printed when it ran that program. The
 site's job is to be readable in one pass by someone who has never heard of iyi,

package.json

@@ -3,13 +3,14 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "generate": "node scripts/fonts.mjs && node scripts/facts.mjs && node scripts/targets.mjs && node scripts/proofs.mjs && node scripts/releases.mjs && node scripts/install-targets.mjs && node scripts/samples.mjs && node scripts/thesis.mjs && node scripts/project.mjs && node scripts/download.mjs && node scripts/records.mjs && node scripts/contrast.mjs && node scripts/no-transcription.mjs && node scripts/blocker.mjs",
+    "generate": "node scripts/fonts.mjs && node scripts/facts.mjs && node scripts/targets.mjs && node scripts/proofs.mjs && node scripts/releases.mjs && node scripts/install-targets.mjs && node scripts/samples.mjs && node scripts/thesis.mjs && node scripts/project.mjs && node scripts/download.mjs && node scripts/records.mjs && node scripts/stdlib.mjs && node scripts/contrast.mjs && node scripts/no-transcription.mjs && node scripts/blocker.mjs",
     "generate:install-targets": "node scripts/install-targets.mjs",
     "record:wasm": "node scripts/record-wasm.mjs",
     "record:diagnostics": "node scripts/record-diagnostics.mjs",
     "record:highlight": "node scripts/record-highlight.mjs",
     "record:agent": "node scripts/record-agent.mjs",
-    "record": "npm run record:wasm && npm run record:diagnostics && npm run record:agent && npm run record:highlight",
+    "record:stdlib": "node scripts/record-stdlib.mjs",
+    "record": "npm run record:wasm && npm run record:diagnostics && npm run record:agent && npm run record:highlight && npm run record:stdlib",
     "predev": "npm run generate",
     "dev": "astro dev",
     "prebuild": "npm run generate",

scripts/budget.json

@@ -36,7 +36,10 @@
     { "match": "/learn/*/", "ceiling": 245760 },
     { "match": "/playground/", "ceiling": 229376 },
     { "match": "/playground/iyi/*/", "ceiling": 262144 },
-    { "match": "/playground/*/", "ceiling": 294912 }
+    { "match": "/playground/*/", "ceiling": 294912 },
+    { "match": "/reference/", "ceiling": 245760 },
+    { "match": "/reference/std/*/", "ceiling": 344064 },
+    { "match": "/reference/prelude/*/", "ceiling": 229376 }
   ],
   "assets": 638976
 }

scripts/project.mjs

@@ -43,11 +43,23 @@ const die = (message) => {
 // report a hole, how people are expected to behave, whose code this is.
 // `blurb` is the one sentence of ours on the index, and it says what the
 // document is, never what it says.
+//
+// `name` is the page's title in a results list and a browser tab, which the
+// document's own first heading is a poor one for: NOTICE.md opens with
+// "Crystal Programming Language", the right heading for a notice about
+// Crystal's code and the wrong title for a page on this site. The heading
+// stays the page's heading; the document is published whole.
+//
+// `indexed` is false for the two documents whose text is not this project's
+// to rank for: the Contributor Covenant is published on thousands of sites
+// word for word, and the notice is Crystal's copyright and a list of other
+// people's libraries. Both stay published and linked; neither is put forward
+// as a search result, and astro.config.mjs keeps them out of the sitemap.
 const DOCUMENTS = [
-  { id: "contributing", file: "CONTRIBUTING.md", blurb: "What a change to this repository is expected to come with." },
-  { id: "security", file: "SECURITY.md", blurb: "Where a vulnerability goes, and which of the two trackers is the right one." },
-  { id: "conduct", file: "CODE_OF_CONDUCT.md", blurb: "The Contributor Covenant, and the address that enforces it here." },
-  { id: "notice", file: "NOTICE.md", blurb: "Crystal's copyright, the license, and every library the compiler links." },
+  { id: "contributing", file: "CONTRIBUTING.md", name: "Contributing to iyi", indexed: true, blurb: "What a change to this repository is expected to come with." },
+  { id: "security", file: "SECURITY.md", name: "Reporting a security vulnerability in iyi", indexed: true, blurb: "Where a vulnerability goes, and which of the two trackers is the right one." },
+  { id: "conduct", file: "CODE_OF_CONDUCT.md", name: "iyi code of conduct", indexed: false, blurb: "The Contributor Covenant, and the address that enforces it here." },
+  { id: "notice", file: "NOTICE.md", name: "iyi license and notices", indexed: false, blurb: "Crystal's copyright, the license, and every library the compiler links." },
 ];
 
 const commit = execFileSync("git", ["-C", repo, "rev-parse", "HEAD"], {

scripts/prose.mjs

@@ -42,8 +42,17 @@ const INLINE = "a|code|strong|em|b|i|span|abbr|kbd|dfn";
 // Blocks whose insides are a token stream rather than prose. Matched by class
 // because that is what the site's own CSS keys on, so a new block type that
 // wants the exemption has to say so in the same vocabulary.
+//
+// `doc-comment` is the one block here that is prose, and it is exempt for the
+// other half of this gate's reasoning: the defect is a template's line break
+// that Astro drops, and a doc comment on a reference page reaches the HTML as
+// one string out of the repository, with no template line between its words.
+// What it does contain is the vocabulary of a standard library, `RFC 6265bis`,
+// `0xc0`, `1xx`, which the figure check reads as a number welded to a word.
+// Each one wraps exactly one paragraph or list, never a listing, so the
+// first closing tag after it is its own.
 const LISTINGS =
-  /<(pre|figure|div)[^>]*class="[^"]*(?:source|console|diagnostic|editor-ink|editor|cases|case|stamped|recording)[^"]*"[\s\S]*?<\/\1>/g;
+  /<(pre|figure|div)[^>]*class="[^"]*(?:source|console|diagnostic|editor-ink|editor|cases|case|stamped|recording|doc-comment)[^"]*"[\s\S]*?<\/\1>/g;
 
 function htmlFiles(dir) {
   const found = [];

scripts/record-stdlib.mjs

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+// Records the standard library's exported surface, as the compiler prints it.
+//
+// The reference under /reference/ is this record. Every module under
+// src/std/ and every type the prelude declares is put through `iyi doc`, the
+// verb that prints a module's exported surface with its doc comments, and
+// what it printed is kept byte for byte with the digest of the source it was
+// printed from.
+//
+// WHY THE COMPILER AND NOT A PARSER HERE. What a module exports is the
+// compiler's answer: `pub`, a reopened prelude type whose methods export
+// without it, a macro, a `forall`. A second reading of iyi's grammar in this
+// repository would be a second answer, and the first time the two disagreed
+// the reference would document a function nobody can call. `iyi doc` is also
+// the surface an agent is given over MCP, so the page and the tool say the
+// same thing.
+//
+// WHY `iyi doc` AND NOT `iyi mod dump --json`. The JSON leaves out a reopened
+// type's methods and a module's constants, and most of std/int, std/text and
+// std/set is exactly that. The text surface is the complete one.
+//
+// Regenerate with: npm run record:stdlib
+
+import { createHash } from "node:crypto";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
+import { arch, cpus, release as osRelease, type as osType } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import {
+  blobAt,
+  releaseRef,
+  requireReleaseCompiler,
+  requireReleaseTree,
+  treeAt,
+} from "./release-ref.mjs";
+
+const here = dirname(fileURLToPath(import.meta.url));
+const site = resolve(here, "..");
+const repo = process.env.IYI_REPO ? resolve(process.env.IYI_REPO) : resolve(site, "..", "iyi");
+const out = resolve(site, "records", "stdlib.json");
+
+// ---------------------------------------------------------------------------
+// Toolchain and provenance
+// ---------------------------------------------------------------------------
+
+/* The binary is `iyi` on a Unix build and `iyi.exe` on Windows, where the
+ * release ships a zip; IYI_BUILD may name either a build directory or the
+ * `bin/` of an unpacked release. */
+const buildDir = process.env.IYI_BUILD ? resolve(process.env.IYI_BUILD) : resolve(repo, ".build");
+const iyi = ["iyi", "iyi.exe"].map((name) => join(buildDir, name)).find(existsSync);
+if (iyi === undefined) {
+  throw new Error(
+    `no iyi compiler in ${buildDir}. Build it, or set IYI_BUILD to the ` +
+      `directory holding the iyi binary.`,
+  );
+}
+
+// Evidence about a release, taken from the release: the tree is the tag and
+// the compiler was built there. scripts/release-ref.mjs says why.
+const release = releaseRef(repo);
+requireReleaseTree(repo, release, "record:stdlib");
+const version = requireReleaseCompiler(iyi, release, "record:stdlib");
+
+const recorded = {
+  compiler: version,
+  commit: release.commit,
+  machine: `${osType()} ${osRelease()} ${arch()}, ${cpus()[0]?.model.trim() ?? "unknown cpu"}`,
+  command: "npm run record:stdlib",
+  when: new Date().toISOString(),
+};
+for (const [key, value] of Object.entries(recorded)) {
+  if (!value) throw new Error(`provenance field "${key}" came out empty`);
+}
+
+const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
+
+/* One `iyi doc` call. The std imports resolve through the tree being
+ * recorded, not whatever library sits beside the binary. */
+const env = { ...process.env, IYI_PATH: resolve(repo, "src") };
+function doc(target) {
+  const run = spawnSync(iyi, ["doc", target], { cwd: repo, env, encoding: "utf8" });
+  if (run.status !== 0) {
+    throw new Error(
+      `iyi doc ${target} exited ${run.status}:\n${run.stderr || run.stdout}`,
+    );
+  }
+  return run.stdout.replace(/\r\n/g, "\n");
+}
+
+// ---------------------------------------------------------------------------
+// The modules: every file directly under src/std, as the release ships it
+// ---------------------------------------------------------------------------
+
+/* A file one directory down (src/std/eiy/process.iyi) is a module's private
+ * helper, declared under a name of its own and not imported as std/anything,
+ * so the modules are the files directly under src/std. Each one declares
+ * `module std/<file name>`, and one that does not is refused rather than
+ * filed under a name it does not answer to. */
+const paths = treeAt(repo, release.tag, "src/std").filter((path) =>
+  /^src\/std\/[^/]+\.iyi$/.test(path),
+);
+if (paths.length === 0) {
+  throw new Error(`${release.tag} has no src/std/*.iyi, so there is no library to record`);
+}
+
+const modules = paths.map((path) => {
+  const bytes = blobAt(repo, release.tag, path);
+  const name = `std/${path.slice("src/std/".length, -".iyi".length)}`;
+  const declared = /^module\s+(\S+)/m.exec(bytes.toString("utf8"))?.[1];
+  if (declared !== name) {
+    throw new Error(`${path} declares module ${declared ?? "nothing"}, not ${name}`);
+  }
+  const surface = doc(path);
+  if (!surface.startsWith(`module ${name}\n`)) {
+    throw new Error(`iyi doc ${path} did not open with "module ${name}":\n${surface.slice(0, 200)}`);
+  }
+  return { name, path, sourceSha256: sha256(bytes), surface };
+});
+
+// ---------------------------------------------------------------------------
+// The prelude: every type `iyi doc prelude` lists, one surface each
+// ---------------------------------------------------------------------------
+
+/* `iyi doc prelude` prints one line per type, `class String   # Text ...`,
+ * and `iyi doc String` prints that type's surface. The list is the
+ * compiler's, so a type the prelude gains reaches the reference without an
+ * edit here. */
+const index = doc("prelude");
+const typeNames = [
+  ...new Set(
+    index
+      .split("\n")
+      .map((line) => /^[a-z]+\s+([A-Z]\w*)/.exec(line)?.[1])
+      .filter((name) => name !== undefined),
+  ),
+];
+if (typeNames.length === 0) {
+  throw new Error(`iyi doc prelude listed no types:\n${index.slice(0, 400)}`);
+}
+const types = typeNames.map((name) => ({ name, surface: doc(name) }));
+
+/* What the prelude was printed from, so the build can tell a record of an
+ * older prelude from this one. */
+const preludeSources = treeAt(repo, release.tag, "src/iyi")
+  .filter((path) => path.endsWith(".iyi"))
+  .map((path) => ({ path, sha256: sha256(blobAt(repo, release.tag, path)) }));
+
+mkdirSync(dirname(out), { recursive: true });
+writeFileSync(
+  out,
+  `${JSON.stringify({ recorded, modules, prelude: { sources: preludeSources, types } }, null, 2)}\n`,
+  "utf8",
+);
+
+console.log(
+  `stdlib: ${modules.length} modules and ${types.length} prelude types, ` +
+    `at ${release.commit.slice(0, 9)}`,
+);

scripts/release-ref.mjs

@@ -168,10 +168,17 @@ export function requireReleaseTree(repo, release, verb) {
  * from a developer's branch is caught here rather than in a recording that
  * names a release and was produced by something else. The provenance line on
  * every page carries this string.
+ *
+ * A build in a worktree prints nine hex digits; the release's own binary, the
+ * one install.sh unpacks, prints seven (`iyi 0.18.0 (built on Crystal ...
+ * [2a96530] ...)`). Both name the commit, so a bracketed hash the tag's commit
+ * starts with is accepted beside the nine-digit spelling.
  */
 export function requireReleaseCompiler(iyi, release, verb) {
   const version = execFileSync(iyi, ["--version"], { encoding: "utf8" }).split("\n")[0].trim();
   if (version.includes(release.commit.slice(0, 9))) return version;
+  const named = /\[([0-9a-f]{7,40})\]/.exec(version)?.[1];
+  if (named !== undefined && release.commit.startsWith(named)) return version;
   throw new Error(
     `${iyi} prints "${version}", which does not name ${release.commit.slice(0, 9)}, ` +
       `the commit ${release.tag} is. ${verb} would then stamp a release onto a ` +