diff --git a/supabase/functions/import-source/html-text.ts b/supabase/functions/import-source/html-text.ts
index 6b93986c4..77d084296 100644
--- a/supabase/functions/import-source/html-text.ts
+++ b/supabase/functions/import-source/html-text.ts
@@ -106,23 +106,36 @@ export function getTitle(html: string, fallback: string): string {
// Article roots we would rather keep. Everything else that looks like Wikipedia
// chrome (edit links, the table of contents, navboxes, language lists) is dropped
-// before its text can reach the importer.
-const ARTICLE_ROOT = /(?:^|[\s_-])(?:mw-content-text|mw-parser-output)(?:$|[\s_-])/i;
-const WIKI_CHROME = /(?:^|[\s_-])(?:mw-editsection|mw-editsection-like|toc|navbox|interlanguage|vector-language|mw-portlet-lang|p-lang|sistersitebox|noprint|mw-jump-link|vector-page-toolbar|vector-sticky-header|vector-toc)(?:$|[\s_-])/i;
+// before its text can reach the importer. Both lists match whole id or class
+// tokens only: Wikipedia puts flags such as `vector-toc-available` and
+// `vector-feature-toc-pinned-clientpref-1` on , so a hyphen-bounded match
+// for `toc` would strip the entire page.
+const ARTICLE_ROOT = new Set(["mw-content-text", "mw-parser-output"]);
+const WIKI_CHROME = new Set([
+ "mw-editsection", "mw-editsection-like", "toc", "navbox", "interlanguage", "vector-language", "mw-portlet-lang",
+ "p-lang", "sistersitebox", "noprint", "mw-jump-link", "vector-page-toolbar", "vector-sticky-header", "vector-toc",
+]);
+// Outer document elements are never chrome, whatever flags their classes carry.
+const NEVER_CHROME = new Set(["html", "body"]);
function idClassOf(tag: string): string {
return `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`;
}
+function hasToken(idClass: string, tokens: Set): boolean {
+ return idClass.toLowerCase().split(/\s+/).some((token) => tokens.has(token));
+}
+
function isArticleRoot(name: string, idClass: string): boolean {
- return name === "main" || name === "article" || ARTICLE_ROOT.test(idClass);
+ return name === "main" || name === "article" || hasToken(idClass, ARTICLE_ROOT);
}
function shouldStrip(name: string, tag: string): boolean {
if (STRIP_TAGS.has(name)) return true;
const idClass = idClassOf(tag);
if (isArticleRoot(name, idClass)) return false;
- if (WIKI_CHROME.test(idClass)) return true;
+ if (NEVER_CHROME.has(name)) return false;
+ if (hasToken(idClass, WIKI_CHROME)) return true;
if (name !== "div" && name !== "section") return false;
const role = readAttr(tag, "role")?.toLowerCase();
if (role && ROLE_STRIP.has(role)) return true;
@@ -225,7 +238,7 @@ function collectContent(html: string): { cleaned: string; candidates: ContentCan
const idClass = idClassOf(tag);
candidates.push({
html: html.slice(end + 1, close),
- preferred: parsed.name === "main" || parsed.name === "article" || ARTICLE_ROOT.test(idClass),
+ preferred: parsed.name === "main" || parsed.name === "article" || hasToken(idClass, ARTICLE_ROOT),
});
}
}
diff --git a/supabase/functions/import-source/html-text_test.ts b/supabase/functions/import-source/html-text_test.ts
index a1f77ee89..a7f897948 100644
--- a/supabase/functions/import-source/html-text_test.ts
+++ b/supabase/functions/import-source/html-text_test.ts
@@ -107,3 +107,29 @@ Deno.test("text after the last > is kept, with no tag parsing", () => {
const html = `${"word ".repeat(25)}
3 < 5 and more words here`;
assertEquals(stripHtml(html).includes("3 < 5 and more words here"), true);
});
+
+Deno.test("a real Wikipedia page keeps its article text and drops the chrome", async () => {
+ // en.wikipedia.org/wiki/Free_public_transport as served on 2026-10-22 (CC BY-SA 4.0), gzipped.
+ // Its carries `vector-toc-available` and `vector-feature-toc-pinned-*`, which once
+ // matched the `toc` chrome rule and stripped the whole page.
+ const gz = await Deno.readFile(new URL("./testdata/wikipedia-free-public-transport.html.gz", import.meta.url));
+ const html = await new Response(new Blob([gz]).stream().pipeThrough(new DecompressionStream("gzip"))).text();
+ const text = stripHtml(html);
+ const words = text.split(/\s+/).filter(Boolean).length;
+ if (words < 1_000) throw new Error(`expected the article body, got ${words} words`);
+ assertEquals(/free public transport/i.test(text.slice(0, 600)), true);
+ // Each of these leaked into imports before #195 (language menu, edit links).
+ assertEquals(text.includes("18 languages"), false);
+ assertEquals(text.includes("Čeština"), false);
+ assertEquals(/\[\s*edit\s*\]/.test(text), false);
+});
+
+Deno.test("chrome rules match whole class tokens, not fragments of page flags", () => {
+ const body = "Free fares raised ridership in Tallinn and cut car trips in the city centre. ".repeat(6);
+ const html = `` +
+ `Contents 1 History
` +
+ ``;
+ const text = stripHtml(html);
+ assertEquals(text.includes("Free fares raised ridership"), true);
+ assertEquals(text.includes("Contents 1 History"), false);
+});
diff --git a/supabase/functions/import-source/testdata/wikipedia-free-public-transport.html.gz b/supabase/functions/import-source/testdata/wikipedia-free-public-transport.html.gz
new file mode 100644
index 000000000..d400b25f2
Binary files /dev/null and b/supabase/functions/import-source/testdata/wikipedia-free-public-transport.html.gz differ