Fix import-source stripping whole Wikipedia pages (#195 regression)

supabase/functions/import-source/html-text.ts

@@ -106,23 +106,36 @@ export function getTitle(html: string, fallback: string): string {
 
 // Article roots we would rather keep. Everything else that looks like Wikipedia
 // chrome (edit links, the table of contents, navboxes, language lists) is dropped
-// before its text can reach the importer.
-const ARTICLE_ROOT = /(?:^|[\s_-])(?:mw-content-text|mw-parser-output)(?:$|[\s_-])/i;
-const WIKI_CHROME = /(?:^|[\s_-])(?:mw-editsection|mw-editsection-like|toc|navbox|interlanguage|vector-language|mw-portlet-lang|p-lang|sistersitebox|noprint|mw-jump-link|vector-page-toolbar|vector-sticky-header|vector-toc)(?:$|[\s_-])/i;
+// before its text can reach the importer. Both lists match whole id or class
+// tokens only: Wikipedia puts flags such as `vector-toc-available` and
+// `vector-feature-toc-pinned-clientpref-1` on <html>, so a hyphen-bounded match
+// for `toc` would strip the entire page.
+const ARTICLE_ROOT = new Set(["mw-content-text", "mw-parser-output"]);
+const WIKI_CHROME = new Set([
+  "mw-editsection", "mw-editsection-like", "toc", "navbox", "interlanguage", "vector-language", "mw-portlet-lang",
+  "p-lang", "sistersitebox", "noprint", "mw-jump-link", "vector-page-toolbar", "vector-sticky-header", "vector-toc",
+]);
+// Outer document elements are never chrome, whatever flags their classes carry.
+const NEVER_CHROME = new Set(["html", "body"]);
 
 function idClassOf(tag: string): string {
   return `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`;
 }
 
+function hasToken(idClass: string, tokens: Set<string>): boolean {
+  return idClass.toLowerCase().split(/\s+/).some((token) => tokens.has(token));
+}
+
 function isArticleRoot(name: string, idClass: string): boolean {
-  return name === "main" || name === "article" || ARTICLE_ROOT.test(idClass);
+  return name === "main" || name === "article" || hasToken(idClass, ARTICLE_ROOT);
 }
 
 function shouldStrip(name: string, tag: string): boolean {
   if (STRIP_TAGS.has(name)) return true;
   const idClass = idClassOf(tag);
   if (isArticleRoot(name, idClass)) return false;
-  if (WIKI_CHROME.test(idClass)) return true;
+  if (NEVER_CHROME.has(name)) return false;
+  if (hasToken(idClass, WIKI_CHROME)) return true;
   if (name !== "div" && name !== "section") return false;
   const role = readAttr(tag, "role")?.toLowerCase();
   if (role && ROLE_STRIP.has(role)) return true;
@@ -225,7 +238,7 @@ function collectContent(html: string): { cleaned: string; candidates: ContentCan
         const idClass = idClassOf(tag);
         candidates.push({
           html: html.slice(end + 1, close),
-          preferred: parsed.name === "main" || parsed.name === "article" || ARTICLE_ROOT.test(idClass),
+          preferred: parsed.name === "main" || parsed.name === "article" || hasToken(idClass, ARTICLE_ROOT),
         });
       }
     }

supabase/functions/import-source/html-text_test.ts

@@ -107,3 +107,29 @@ Deno.test("text after the last > is kept, with no tag parsing", () => {
   const html = `<p>${"word ".repeat(25)}</p> 3 < 5 and more words here`;
   assertEquals(stripHtml(html).includes("3 < 5 and more words here"), true);
 });
+
+Deno.test("a real Wikipedia page keeps its article text and drops the chrome", async () => {
+  // en.wikipedia.org/wiki/Free_public_transport as served on 2026-10-22 (CC BY-SA 4.0), gzipped.
+  // Its <html> carries `vector-toc-available` and `vector-feature-toc-pinned-*`, which once
+  // matched the `toc` chrome rule and stripped the whole page.
+  const gz = await Deno.readFile(new URL("./testdata/wikipedia-free-public-transport.html.gz", import.meta.url));
+  const html = await new Response(new Blob([gz]).stream().pipeThrough(new DecompressionStream("gzip"))).text();
+  const text = stripHtml(html);
+  const words = text.split(/\s+/).filter(Boolean).length;
+  if (words < 1_000) throw new Error(`expected the article body, got ${words} words`);
+  assertEquals(/free public transport/i.test(text.slice(0, 600)), true);
+  // Each of these leaked into imports before #195 (language menu, edit links).
+  assertEquals(text.includes("18 languages"), false);
+  assertEquals(text.includes("Čeština"), false);
+  assertEquals(/\[\s*edit\s*\]/.test(text), false);
+});
+
+Deno.test("chrome rules match whole class tokens, not fragments of page flags", () => {
+  const body = "Free fares raised ridership in Tallinn and cut car trips in the city centre. ".repeat(6);
+  const html = `<html class="vector-toc-available vector-feature-toc-pinned-clientpref-1 noprint-flag">` +
+    `<body class="skin-vector-2022"><div id="toc" class="toc">Contents 1 History</div>` +
+    `<div class="mw-body vector-toc-landmark"><p>${body}</p></div></body></html>`;
+  const text = stripHtml(html);
+  assertEquals(text.includes("Free fares raised ridership"), true);
+  assertEquals(text.includes("Contents 1 History"), false);
+});