feat(docs): give every docs page its own URL and prerendered HTML

frontend/docs/.gitignore

@@ -9,3 +9,4 @@ dist/
 /public/llms-full.txt
 /test-results/
 /playwright-report/
+/dist-ssr/

frontend/docs/e2e/docs.spec.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "@playwright/test"
-import { aliases, headings, slug } from "../src/navigation"
+import { aliases, documentTitle, headings, slug } from "../src/navigation"
 import { readFileSync } from "node:fs"
 
 const config = JSON.parse(readFileSync(new URL("../src/content/openscience/docs.json", import.meta.url), "utf8")) as {
@@ -13,9 +13,9 @@ test("all pages render with working section links and page metadata", async ({ p
   for (const name of pages) {
     const source = readFileSync(new URL("../src/content/openscience/" + name + ".mdx", import.meta.url), "utf8")
     const title = source.match(/^title: "(.+)"$/m)![1]
-    await page.goto("#/openscience/" + name)
+    await page.goto(name === "index" ? "./" : name)
     await expect(page.getByRole("heading", { level: 1 })).toHaveText(title)
-    await expect(page).toHaveTitle(title + " · OpenScience Docs")
+    await expect(page).toHaveTitle(documentTitle(title, name))
     for (const heading of headings(source, 3)) await expect(page.locator('[id="' + slug(heading) + '"]')).toHaveCount(1)
     const overflow = await page.evaluate(() => document.documentElement.scrollWidth > window.innerWidth)
     expect(overflow, name + " must fit the viewport").toBe(false)
@@ -24,7 +24,7 @@ test("all pages render with working section links and page metadata", async ({ p
 })
 
 test("section navigation, old links, and back navigation keep the right page", async ({ page }) => {
-  await page.goto("#/openscience/sessions")
+  await page.goto("sessions")
   await page.getByRole("complementary", { name: "on this page" }).getByRole("link", { name: "Exit codes" }).click()
   await expect(page).toHaveURL(/sessions#exit-codes$/)
   await expect(page.getByRole("heading", { level: 1 })).toHaveText("Sessions and terminal runs")
@@ -33,15 +33,15 @@ test("section navigation, old links, and back navigation keep the right page", a
   await expect(page).toHaveURL(/sessions#exit-codes$/)
   for (const [old, current] of Object.entries(aliases)) {
     await page.goto("#/openscience/" + old)
-    await expect(page).toHaveURL(new RegExp("/openscience/" + current + "$"))
+    await expect(page).toHaveURL(new RegExp("/docs/" + current + "$"))
     await expect(page.getByRole("heading", { level: 1 })).not.toHaveText("Page not found")
   }
-  await page.goto("#/openscience/missing")
+  await page.goto("missing")
   await expect(page.getByRole("heading", { level: 1 })).toHaveText("Page not found")
 })
 
 test("keyboard search selects a result and escape dismisses it", async ({ page }) => {
-  await page.goto("#/openscience/index")
+  await page.goto("./")
   const search = page.getByRole("combobox", { name: "Search documentation" })
   await search.fill("pricing")
   await search.press("Enter")
@@ -54,7 +54,7 @@ test("keyboard search selects a result and escape dismisses it", async ({ page }
 
 test("mobile navigation, search, and wide tables remain usable", async ({ page }) => {
   await page.setViewportSize({ width: 390, height: 844 })
-  await page.goto("#/openscience/index")
+  await page.goto("./")
   await expect(page.getByRole("combobox", { name: "Search documentation" })).toBeVisible()
   const menu = page.getByRole("button", { name: "Browse documentation" })
   await expect(menu).toHaveAttribute("aria-expanded", "false")
@@ -68,7 +68,7 @@ test("mobile navigation, search, and wide tables remain usable", async ({ page }
 
 test("code copy reports success and exports are served", async ({ page, context }) => {
   await context.grantPermissions(["clipboard-read", "clipboard-write"])
-  await page.goto("#/openscience/quickstart")
+  await page.goto("quickstart")
   await page.getByRole("button", { name: "Copy code", exact: true }).first().click()
   await expect(page.getByRole("button", { name: "Copy code", exact: true }).first()).toContainText("Copied")
   expect(await page.evaluate(() => navigator.clipboard.readText())).toContain("npm install -g @synsci/openscience")
@@ -78,12 +78,12 @@ test("code copy reports success and exports are served", async ({ page, context
   const full = await page.request.get("llms-full.txt")
   expect(full.ok()).toBe(true)
   expect(await full.text()).toContain("## Ace terms")
-  await page.goto("#/openscience/index")
+  await page.goto("./")
   await page.screenshot({ path: "test-results/docs-desktop.png", fullPage: false })
 })
 
 test("tools and skills tabs expose complete catalogs with usable deep links", async ({ page }) => {
-  await page.goto("#/openscience/index")
+  await page.goto("./")
   const sections = page.getByRole("navigation", { name: "documentation sections" })
   await expect(sections.getByRole("link")).toHaveCount(5)
   await sections.getByRole("link", { name: "Explore tools", exact: true }).click()

frontend/docs/index.html

@@ -9,6 +9,34 @@
       name="description"
       content="Documentation for OpenScience, the open-source AI workbench for scientific research."
     />
+    <link rel="canonical" href="https://www.openscience.sh/docs" />
+    <meta name="robots" content="index, follow, max-image-preview:large, max-snippet:-1" />
+    <meta property="og:type" content="article" />
+    <meta property="og:site_name" content="OpenScience" />
+    <meta property="og:url" content="https://www.openscience.sh/docs" />
+    <meta property="og:title" content="OpenScience Docs" />
+    <meta
+      property="og:description"
+      content="Documentation for OpenScience, the open-source AI workbench for scientific research."
+    />
+    <meta property="og:image" content="https://www.openscience.sh/og.jpg" />
+    <meta name="twitter:card" content="summary_large_image" />
+    <meta name="twitter:site" content="@SynScience" />
+    <meta name="twitter:title" content="OpenScience Docs" />
+    <meta
+      name="twitter:description"
+      content="Documentation for OpenScience, the open-source AI workbench for scientific research."
+    />
+    <meta name="twitter:image" content="https://www.openscience.sh/og.jpg" />
+    <script>
+      // Applies the stored theme before the prerendered page paints.
+      try {
+        if (localStorage.getItem("docs-theme") === "dark") {
+          document.documentElement.classList.add("dark")
+          document.documentElement.style.colorScheme = "dark"
+        }
+      } catch {}
+    </script>
     <link rel="icon" href="/synthetic-sciences.svg" type="image/svg+xml" />
     <link rel="preload" href="/fonts/cmc/cmunorm.woff2" as="font" type="font/woff2" crossorigin />
     <link rel="preconnect" href="https://fonts.googleapis.com" />

frontend/docs/package.json

@@ -6,7 +6,7 @@
   "license": "Apache-2.0",
   "scripts": {
     "dev": "vite",
-    "build": "bun run export && vite build",
+    "build": "bun run export && vite build && vite build --ssr src/entry-server.tsx --outDir dist-ssr && bun script/prerender.ts && rm -rf dist-ssr",
     "preview": "vite preview",
     "typecheck": "tsc --noEmit",
     "catalog": "bun --no-env-file script/catalog.ts",

frontend/docs/script/export.ts

@@ -1,5 +1,6 @@
 import path from "node:path"
 import config from "../src/content/openscience/docs.json"
+import { pageUrl } from "../src/navigation"
 
 const output = path.resolve(import.meta.dir, "../public")
 const directory = path.resolve(import.meta.dir, "../src/content/openscience")
@@ -8,23 +9,23 @@ const index = [
   "",
   "> Guides for installation, model access, Ace pricing, scientific research, and customization.",
   "",
-  "Full text: https://openscience.sh/docs/llms-full.txt",
+  "Full text: https://www.openscience.sh/docs/llms-full.txt",
   "",
 ]
-const full = ["# OpenScience documentation", "", "Source: https://openscience.sh/docs/", ""]
+const full = ["# OpenScience documentation", "", "Source: https://www.openscience.sh/docs", ""]
 for (const tab of config.navigation.tabs) {
   for (const group of tab.groups) {
     index.push("## " + group.group, "")
     for (const name of group.pages) {
       const raw = await Bun.file(path.join(directory, name + ".mdx")).text()
       const title = raw.match(/^title: "(.+)"$/m)?.[1] ?? name
       const description = raw.match(/^description: "(.+)"$/m)?.[1] ?? ""
-      const url = "https://openscience.sh/docs/#/openscience/" + name
+      const url = pageUrl(name)
       const body = raw
         .replace(/^---\n[\s\S]*?\n---\n?/, "")
         .replace(/<Card\s+title="([^"]+)"\s+href="([^"]+)">\s*([\s\S]*?)\s*<\/Card>/g, "[$1]($2): $3")
         .replace(/<\/?(?:Columns|CardGroup)\b[^>]*>/g, "")
-        .replace(/\]\(\/openscience\/([^)]*)\)/g, "](https://openscience.sh/docs/#/openscience/$1)")
+        .replace(/\]\(\/openscience\/([^)]*)\)/g, "](https://www.openscience.sh/docs/$1)")
       index.push("- [" + title + "](" + url + "): " + description)
       full.push("# " + title, "", description, "", "URL: " + url, "", body.trim(), "")
     }

frontend/docs/script/prerender.ts

@@ -0,0 +1,123 @@
+// Runs after the client and SSR builds. Writes every page to dist/<page>.html
+// with its own title, description, canonical URL and JSON-LD, plus 404.html
+// for unknown paths and sitemap.xml, so each page can be indexed on its own.
+import path from "node:path"
+import { BASE, SITE, documentTitle, pageUrl } from "../src/navigation"
+
+type Page = { path: string; title: string; description: string }
+type Group = { group: string; pages: Array<string | Group> }
+
+const dist = path.resolve(import.meta.dir, "../dist")
+const ssr = await import(path.resolve(import.meta.dir, "../dist-ssr/entry-server.js"))
+const render: (path: string) => string = ssr.render
+const pages: Record<string, Page> = ssr.DOC_PAGES
+const config: { navigation: { tabs: { groups: Group[] }[] } } = ssr.DOC_CONFIG
+
+const flatten = (items: Array<string | Group>): string[] =>
+  items.flatMap((item) => (typeof item === "string" ? [item] : flatten(item.pages)))
+const order = config.navigation.tabs.flatMap((tab) => tab.groups.flatMap((group) => flatten(group.pages)))
+
+const esc = (value: string) =>
+  value.replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;").replace(/>/g, "&gt;")
+
+// Attributes may be split across lines in index.html, hence \s+. Replacements go
+// through functions so `$` in a value isn't read as a group reference.
+const TAGS: Array<[RegExp, "title" | "description" | "url"]> = [
+  [/(<title>)[^<]*(<\/title>)/, "title"],
+  [/(<meta\s+name="description"\s+content=")[^"]*(")/, "description"],
+  [/(<link\s+rel="canonical"\s+href=")[^"]*(")/, "url"],
+  [/(<meta\s+property="og:url"\s+content=")[^"]*(")/, "url"],
+  [/(<meta\s+property="og:title"\s+content=")[^"]*(")/, "title"],
+  [/(<meta\s+property="og:description"\s+content=")[^"]*(")/, "description"],
+  [/(<meta\s+name="twitter:title"\s+content=")[^"]*(")/, "title"],
+  [/(<meta\s+name="twitter:description"\s+content=")[^"]*(")/, "description"],
+]
+
+function withHead(html: string, values: Record<"title" | "description" | "url", string>): string {
+  return TAGS.reduce((out, [pattern, key]) => {
+    if (!pattern.test(out)) throw new Error(`prerender: ${pattern} not found in index.html`)
+    return out.replace(pattern, (_match, before: string, after: string) => before + esc(values[key]) + after)
+  }, html)
+}
+
+function structuredData(page: Page, title: string): string {
+  const url = pageUrl(page.path)
+  const crumbs = [
+    { name: "OpenScience", item: SITE + "/" },
+    { name: "Docs", item: SITE + BASE },
+    ...(page.path === "index" ? [] : [{ name: page.title, item: url }]),
+  ]
+  const data = {
+    "@context": "https://schema.org",
+    "@graph": [
+      {
+        "@type": page.path === "index" ? "CollectionPage" : "TechArticle",
+        "@id": url + "#page",
+        headline: page.title,
+        name: title,
+        description: page.description,
+        url,
+        inLanguage: "en",
+        isPartOf: { "@id": SITE + "/#website" },
+        about: { "@type": "SoftwareApplication", name: "OpenScience", url: SITE + "/" },
+        publisher: { "@type": "Organization", name: "Synthetic Sciences", url: "https://syntheticsciences.ai/" },
+        breadcrumb: { "@id": url + "#breadcrumb" },
+      },
+      {
+        "@type": "BreadcrumbList",
+        "@id": url + "#breadcrumb",
+        itemListElement: crumbs.map((crumb, index) => ({ "@type": "ListItem", position: index + 1, ...crumb })),
+      },
+    ],
+  }
+  return `<script type="application/ld+json">${JSON.stringify(data).replace(/</g, "\\u003c")}</script>`
+}
+
+// Fails the build rather than shipping a page that breaks the on-page rules.
+function check(name: string, html: string, title: string, description: string) {
+  const h1s = (html.slice(html.indexOf("<body")).match(/<h1[\s>]/g) ?? []).length
+  const problems = [
+    title.length > 60 && `title is ${title.length} characters (max 60)`,
+    (!description || description.length > 155) && `description is ${description.length} characters (max 155)`,
+    h1s !== 1 && `${h1s} h1 elements (want 1)`,
+  ].filter(Boolean)
+  if (problems.length) throw new Error(`prerender: ${name}: ${problems.join("; ")}`)
+}
+
+const template = await Bun.file(path.join(dist, "index.html")).text()
+const slot = '<div id="root"></div>'
+if (!template.includes(slot)) throw new Error(`prerender: ${slot} not found in dist/index.html`)
+
+for (const name of order) {
+  const page = pages[name]
+  if (!page) throw new Error(`prerender: docs.json lists ${name}, which has no page`)
+  const title = documentTitle(page.title, name)
+  const html = withHead(template, { title, description: page.description, url: pageUrl(name) })
+    .replace("</head>", () => structuredData(page, title) + "\n  </head>")
+    .replace(slot, () => `<div id="root">${render(BASE + "/" + name)}</div>`)
+  check(name, html, title, page.description)
+  await Bun.write(path.join(dist, name === "index" ? "index.html" : name + ".html"), html)
+}
+
+// Unknown /docs paths are rewritten here by the site's vercel.json.
+const missing = withHead(template, {
+  title: documentTitle("Page not found"),
+  description: "This documentation page does not exist or has moved.",
+  url: SITE + BASE,
+})
+  .replace(
+    /(<meta\s+name="robots"\s+content=")[^"]*(")/,
+    (_match, before: string, after: string) => before + "noindex" + after,
+  )
+  .replace(slot, () => `<div id="root">${render(BASE + "/404")}</div>`)
+await Bun.write(path.join(dist, "404.html"), missing)
+
+await Bun.write(
+  path.join(dist, "sitemap.xml"),
+  `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${order.map((name) => `  <url>\n    <loc>${pageUrl(name)}</loc>\n  </url>`).join("\n")}
+</urlset>
+`,
+)
+console.log(`Prerendered ${order.length} documentation pages, 404.html and sitemap.xml.`)

frontend/docs/src/App.tsx

@@ -1,5 +1,5 @@
 import { DocumentationPage } from "./DocsApp"
 
-export function App() {
-  return <DocumentationPage />
+export function App({ initialPath }: { initialPath?: string } = {}) {
+  return <DocumentationPage initialPath={initialPath} />
 }