diff --git a/supabase/functions/import-source/html-text.ts b/supabase/functions/import-source/html-text.ts index 6b93986c4..77d084296 100644 --- a/supabase/functions/import-source/html-text.ts +++ b/supabase/functions/import-source/html-text.ts @@ -106,23 +106,36 @@ export function getTitle(html: string, fallback: string): string { // Article roots we would rather keep. Everything else that looks like Wikipedia // chrome (edit links, the table of contents, navboxes, language lists) is dropped -// before its text can reach the importer. -const ARTICLE_ROOT = /(?:^|[\s_-])(?:mw-content-text|mw-parser-output)(?:$|[\s_-])/i; -const WIKI_CHROME = /(?:^|[\s_-])(?:mw-editsection|mw-editsection-like|toc|navbox|interlanguage|vector-language|mw-portlet-lang|p-lang|sistersitebox|noprint|mw-jump-link|vector-page-toolbar|vector-sticky-header|vector-toc)(?:$|[\s_-])/i; +// before its text can reach the importer. Both lists match whole id or class +// tokens only: Wikipedia puts flags such as `vector-toc-available` and +// `vector-feature-toc-pinned-clientpref-1` on , so a hyphen-bounded match +// for `toc` would strip the entire page. +const ARTICLE_ROOT = new Set(["mw-content-text", "mw-parser-output"]); +const WIKI_CHROME = new Set([ + "mw-editsection", "mw-editsection-like", "toc", "navbox", "interlanguage", "vector-language", "mw-portlet-lang", + "p-lang", "sistersitebox", "noprint", "mw-jump-link", "vector-page-toolbar", "vector-sticky-header", "vector-toc", +]); +// Outer document elements are never chrome, whatever flags their classes carry. +const NEVER_CHROME = new Set(["html", "body"]); function idClassOf(tag: string): string { return `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`; } +function hasToken(idClass: string, tokens: Set): boolean { + return idClass.toLowerCase().split(/\s+/).some((token) => tokens.has(token)); +} + function isArticleRoot(name: string, idClass: string): boolean { - return name === "main" || name === "article" || ARTICLE_ROOT.test(idClass); + return name === "main" || name === "article" || hasToken(idClass, ARTICLE_ROOT); } function shouldStrip(name: string, tag: string): boolean { if (STRIP_TAGS.has(name)) return true; const idClass = idClassOf(tag); if (isArticleRoot(name, idClass)) return false; - if (WIKI_CHROME.test(idClass)) return true; + if (NEVER_CHROME.has(name)) return false; + if (hasToken(idClass, WIKI_CHROME)) return true; if (name !== "div" && name !== "section") return false; const role = readAttr(tag, "role")?.toLowerCase(); if (role && ROLE_STRIP.has(role)) return true; @@ -225,7 +238,7 @@ function collectContent(html: string): { cleaned: string; candidates: ContentCan const idClass = idClassOf(tag); candidates.push({ html: html.slice(end + 1, close), - preferred: parsed.name === "main" || parsed.name === "article" || ARTICLE_ROOT.test(idClass), + preferred: parsed.name === "main" || parsed.name === "article" || hasToken(idClass, ARTICLE_ROOT), }); } } diff --git a/supabase/functions/import-source/html-text_test.ts b/supabase/functions/import-source/html-text_test.ts index a1f77ee89..a7f897948 100644 --- a/supabase/functions/import-source/html-text_test.ts +++ b/supabase/functions/import-source/html-text_test.ts @@ -107,3 +107,29 @@ Deno.test("text after the last > is kept, with no tag parsing", () => { const html = `

${"word ".repeat(25)}

3 < 5 and more words here`; assertEquals(stripHtml(html).includes("3 < 5 and more words here"), true); }); + +Deno.test("a real Wikipedia page keeps its article text and drops the chrome", async () => { + // en.wikipedia.org/wiki/Free_public_transport as served on 2026-10-22 (CC BY-SA 4.0), gzipped. + // Its carries `vector-toc-available` and `vector-feature-toc-pinned-*`, which once + // matched the `toc` chrome rule and stripped the whole page. + const gz = await Deno.readFile(new URL("./testdata/wikipedia-free-public-transport.html.gz", import.meta.url)); + const html = await new Response(new Blob([gz]).stream().pipeThrough(new DecompressionStream("gzip"))).text(); + const text = stripHtml(html); + const words = text.split(/\s+/).filter(Boolean).length; + if (words < 1_000) throw new Error(`expected the article body, got ${words} words`); + assertEquals(/free public transport/i.test(text.slice(0, 600)), true); + // Each of these leaked into imports before #195 (language menu, edit links). + assertEquals(text.includes("18 languages"), false); + assertEquals(text.includes("Čeština"), false); + assertEquals(/\[\s*edit\s*\]/.test(text), false); +}); + +Deno.test("chrome rules match whole class tokens, not fragments of page flags", () => { + const body = "Free fares raised ridership in Tallinn and cut car trips in the city centre. ".repeat(6); + const html = `` + + `
Contents 1 History
` + + `

${body}

`; + const text = stripHtml(html); + assertEquals(text.includes("Free fares raised ridership"), true); + assertEquals(text.includes("Contents 1 History"), false); +}); diff --git a/supabase/functions/import-source/testdata/wikipedia-free-public-transport.html.gz b/supabase/functions/import-source/testdata/wikipedia-free-public-transport.html.gz new file mode 100644 index 000000000..d400b25f2 Binary files /dev/null and b/supabase/functions/import-source/testdata/wikipedia-free-public-transport.html.gz differ