Publish search.json for the family's search

AGENTS.md

@@ -18,6 +18,7 @@ it depends on abap2UI5/docs or abap2UI5/playground at build time.
 | `src/components/` | `Stats` (the headline tiles under the hero), `Icon` (inline marks) |
 | `src/styles/global.css` | The one stylesheet; light and dark follow the system |
 | `public/` | Font (Inter, self-hosted), logo, icons, `apps/` (one screenshot per example app; the hero shows one of them) |
+| `scripts/search-feed.mjs` | Runs after `astro build` and writes `dist/search.json`: one entry per section with an id and per FAQ question, read from the built pages. abap2UI5/docs fetches it at build time into the family's one search index (the home page and the playground read it); the shape `{ site, url, entries: [{ title, text, headings, terms, url }] }` is the same on every site of the family. A page that yields nothing fails the build |
 | `test/site.test.mjs` | `node:test` over the built `dist/index.html` |
 | `.github/workflows/pages.yml` | Build + test on every PR; deploy on main |
 

package.json

@@ -10,7 +10,7 @@
   },
   "scripts": {
     "dev": "astro dev",
-    "build": "astro build",
+    "build": "astro build && node scripts/search-feed.mjs",
     "preview": "astro preview",
     "test": "node --test test/*.test.mjs",
     "check": "npm run build && npm test"

scripts/search-feed.mjs

@@ -0,0 +1,73 @@
+#!/usr/bin/env node
+// What the family's search finds of this site, written into dist/ after the
+// build: dist/search.json. abap2UI5/docs fetches it at build time into the one
+// search index the home page (https://abap2ui5.github.io/) and the playground
+// read, as the group "Why abap2UI5". The shape is the one every site of the
+// family publishes: { site, url, entries: [{ title, text, headings, terms, url }] }.
+//
+// Read from the BUILT pages, not from the .astro files: the text of this site
+// is data inside its pages, and the HTML is the one place where it is all
+// written out. One entry per section with an id (the ids are addresses, see
+// AGENTS.md, so a hit lands on the section), and one per question of the FAQ.
+// Nothing here may fail the build quietly: a page that yields no entry is an
+// error, because it means the markup changed under this reader.
+import fs from "node:fs";
+import path from "node:path";
+
+const DIST = path.join(import.meta.dirname, "..", "dist");
+const ROOT = "https://abap2ui5.github.io/why/";
+
+const decode = (s) =>
+  s.replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;|&#x27;/g, "'").replace(/&nbsp;/g, " ");
+const text = (html) => decode(html.replace(/<[^>]+>/g, " ")).replace(/\s+/g, " ").trim();
+const sentence = (s, max = 220) => {
+  const first = s.match(/^.{20,}?[.!?](\s|$)/)?.[0].trim() ?? s;
+  return first.length > max ? `${first.slice(0, max - 1).trimEnd()}…` : first;
+};
+const words = (s) =>
+  [...new Set(s.toLowerCase().split(/[^\p{L}\p{N}_]+/u).filter((w) => w.length >= 4 && !/^\d+$/.test(w)))].join(" ");
+
+function sections(file, url) {
+  const html = fs.readFileSync(path.join(DIST, file), "utf8");
+  const out = [];
+  for (const m of html.matchAll(/<section\b[^>]*\sid="([^"]+)"[^>]*>([\s\S]*?)<\/section>/g)) {
+    const [, id, body] = m;
+    const h2 = body.match(/<h2\b[^>]*>([\s\S]*?)<\/h2>/);
+    if (!h2) continue;
+    /* the first paragraph that is a sentence: the short line above a heading
+     * (the kicker, "What it costs") is a label, not what the section says */
+    const p = [...body.matchAll(/<p\b[^>]*>([\s\S]*?)<\/p>/g)].map((x) => text(x[1])).find((t) => t.length >= 40);
+    const headings = [...body.matchAll(/<h3\b[^>]*>([\s\S]*?)<\/h3>/g)].map((h) => text(h[1])).filter(Boolean);
+    out.push({
+      title: text(h2[1]),
+      text: p ? sentence(p) : "",
+      /* the h3 of a section have no id of their own, and the entry's url
+       * already carries the section's: a heading here would send a reader to
+       * `#id#id` in a box that appends its anchor. Their words are matched. */
+      headings: [],
+      terms: words(`${headings.join(" ")} ${text(body)}`),
+      url: `${url}#${id}`,
+    });
+  }
+  if (!out.length) throw new Error(`search-feed: no section with an id and a heading in ${file}`);
+  return out;
+}
+
+function questions(file, url) {
+  const html = fs.readFileSync(path.join(DIST, file), "utf8");
+  const out = [];
+  for (const m of html.matchAll(/<details\b[^>]*>\s*<summary>([\s\S]*?)<\/summary>([\s\S]*?)<\/details>/g)) {
+    const answer = text(m[2]);
+    out.push({ title: text(m[1]), text: sentence(answer), headings: [], terms: words(answer), url });
+  }
+  if (!out.length) throw new Error(`search-feed: no question in ${file}`);
+  return out;
+}
+
+const entries = [
+  ...sections("index.html", ROOT),
+  ...questions("faq/index.html", `${ROOT}faq/`),
+  ...sections("community/index.html", `${ROOT}community/`),
+];
+fs.writeFileSync(path.join(DIST, "search.json"), JSON.stringify({ site: "Why abap2UI5", url: ROOT, entries }));
+console.log(`search-feed: ${entries.length} entries -> dist/search.json`);

test/site.test.mjs

@@ -151,3 +151,20 @@ test("the release the page names is the newest release of the framework", async
   const latest = tag.replace(/-\w+$/, "").replace(/\.\d+$/, "");
   assert.equal(named[1], latest, `the page says Release ${named[1]}, the newest release is ${tag}`);
 });
+
+test("the search feed: every section and every question, at an address that exists", () => {
+  const feed = JSON.parse(fs.readFileSync(path.join(DIST, "search.json"), "utf8"));
+  assert.equal(feed.site, "Why abap2UI5");
+  assert.equal(feed.url, "https://abap2ui5.github.io/why/");
+  assert.ok(feed.entries.length >= 20, `${feed.entries.length} entries`);
+  for (const e of feed.entries) {
+    assert.ok(e.title, "a title");
+    assert.match(e.url, /^https:\/\/abap2ui5\.github\.io\/why\//);
+    const [page, id] = e.url.replace("https://abap2ui5.github.io/why/", "").split("#");
+    const html = fs.readFileSync(path.join(DIST, page, "index.html"), "utf8");
+    if (id) assert.match(html, new RegExp(`id="${id}"`), e.url);
+  }
+  for (const id of ["what-it-costs", "what-you-keep", "when-to-use-it-and-when-not"]) {
+    assert.ok(feed.entries.some((e) => e.url.endsWith(`#${id}`)), `#${id} is findable`);
+  }
+});