diff --git a/supabase/functions/import-source/html-text.ts b/supabase/functions/import-source/html-text.ts new file mode 100644 index 000000000..53c596d91 --- /dev/null +++ b/supabase/functions/import-source/html-text.ts @@ -0,0 +1,230 @@ +// HTML cleanup for URL import. Tag removal is a single forward scan: the old +// regular expressions retried unclosed ``) stay linear in the page size. + +/** indexOf for a forward scan: repeated calls with growing `from` cost O(n) in total. */ +function forwardFinder(haystack: string, needle: string): (from: number) => number { + let found = -2; // -2: not searched yet; -1: absent from `searchedFrom` on + let searchedFrom = 0; + return (from: number) => { + if (found === -1 && from >= searchedFrom) return -1; + if (found >= from && from >= searchedFrom) return found; + searchedFrom = from; + found = haystack.indexOf(needle, from); + return found; + }; +} + +const STRIP_TAGS = new Set([ + "script", "style", "template", "noscript", "nav", "footer", "header", "aside", "form", "svg", "iframe", "button", +]); +const ROLE_STRIP = new Set(["banner", "navigation", "complementary", "search", "contentinfo"]); +const CLASS_HINT = /(?:^|[\s_-])(?:cookie|consent|banner|navbar|navigation|nav|menu|sidebar|header|footer|subscribe|newsletter|social|share|paywall)(?:$|[\s_-])/i; +const CONTENT_HINT = /(?:^|[\s_-])(?:article|content|story|post|entry|main|body|reader)(?:$|[\s_-])/i; +const CONTENT_TAGS = new Set(["article", "main", "section", "div"]); + +export function decodeEntities(text: string): string { + return text + .replace(/ /gi, " ") + .replace(/&/gi, "&") + .replace(/"/gi, "\"") + .replace(/'/gi, "'") + .replace(/</gi, "<") + .replace(/>/gi, ">") + .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => fromCodePoint(hex, 16)) + .replace(/&#(\d+);/g, (_, dec: string) => fromCodePoint(dec, 10)); +} + +function fromCodePoint(raw: string, radix: number): string { + const code = parseInt(raw, radix); + // Surrogates and values past Unicode's last code point throw from fromCodePoint. + if (!Number.isInteger(code) || code < 0 || code > 0x10FFFF || (code >= 0xD800 && code <= 0xDFFF)) return ""; + return String.fromCodePoint(code); +} + +function readAttr(tag: string, attr: string): string | undefined { + const match = tag.match(new RegExp(`\\b${attr}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, "i")); + return match?.[1] ?? match?.[2]; +} + +function parseOpenTag(tag: string): { name: string; closing: boolean; selfClosing: boolean } | null { + const match = /^<\s*(\/)?\s*([a-zA-Z][\w:-]*)/.exec(tag); + if (!match) return null; + return { + name: match[2].toLowerCase(), + closing: Boolean(match[1]), + selfClosing: /\/\s*>$/.test(tag), + }; +} + +export function getMeta(html: string, names: string[]): string | undefined { + const wanted = new Set(names.map((name) => name.toLowerCase())); + const lower = html.toLowerCase(); + const nextMeta = forwardFinder(lower, ""); + let i = 0; + while (i < html.length) { + const start = nextMeta(i); + if (start === -1) return undefined; + const next = lower[start + 5]; + if (next && /[a-z0-9]/.test(next)) { + i = start + 5; + continue; + } + const end = nextGt(start); + // No `>` left means no complete tag left. + if (end === -1) return undefined; + // An overlong tag is skipped in one step. Searching it with a regex retried the whole page. + if (end - start > 4000) { + i = start + 5; + continue; + } + const tag = html.slice(start, end + 1); + const name = (readAttr(tag, "property") || readAttr(tag, "name") || "").toLowerCase(); + const content = readAttr(tag, "content"); + if (name && content !== undefined && wanted.has(name)) return decodeEntities(content.trim()); + i = end + 1; + } + return undefined; +} + +export function getTitle(html: string, fallback: string): string { + const meta = getMeta(html, ["og:title", "twitter:title"]); + if (meta) return meta; + const lower = html.toLowerCase(); + const start = lower.indexOf("", start); + const close = openEnd === -1 ? -1 : lower.indexOf("", openEnd); + if (openEnd === -1 || close === -1 || close - openEnd > 2000) return fallback; + const text = decodeEntities(html.slice(openEnd + 1, close).replace(/\s+/g, " ").trim()); + return text || fallback; +} + +function shouldStrip(name: string, tag: string): boolean { + if (STRIP_TAGS.has(name)) return true; + if (name !== "div" && name !== "section") return false; + const role = readAttr(tag, "role")?.toLowerCase(); + if (role && ROLE_STRIP.has(role)) return true; + const idClass = `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`; + return CLASS_HINT.test(idClass); +} + +function isContentBlock(name: string, tag: string): boolean { + if (!CONTENT_TAGS.has(name)) return false; + if (name === "article" || name === "main") return true; + const idClass = `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`; + return CONTENT_HINT.test(idClass); +} + +/** Index just after the matching close tag, or -1 when this element is never closed. */ +function matchingClose(html: string, lower: string, tag: string, from: number, absentAfter: Map): number { + const known = absentAfter.get(tag); + if (known !== undefined && from >= known) return -1; + const openNeedle = `<${tag}`; + const closeNeedle = ` 0) { + const nextOpen = findOpen(i); + const nextClose = findClose(i); + if (nextClose === -1) { + absentAfter.set(tag, from); + return -1; + } + const openIsTag = nextOpen !== -1 && !/[a-z0-9]/i.test(lower[nextOpen + openNeedle.length] || ""); + if (openIsTag && nextOpen < nextClose) { + depth += 1; + i = nextOpen + openNeedle.length; + continue; + } + depth -= 1; + i = nextClose + closeNeedle.length; + } + const end = html.indexOf(">", i - closeNeedle.length); + return end === -1 ? html.length : end + 1; +} + +export function stripHtml(html: string): string { + const lower = html.toLowerCase(); + const absentAfter = new Map(); + const candidates: string[] = []; + const nextGt = forwardFinder(html, ">"); + let cleaned = ""; + let i = 0; + while (i < html.length) { + const start = html.indexOf("<", i); + if (start === -1) { + cleaned += html.slice(i); + break; + } + cleaned += html.slice(i, start); + if (lower.startsWith("", start + 4); + cleaned += " "; + i = end === -1 ? html.length : end + 3; + continue; + } + const end = nextGt(start + 1); + if (end === -1) { + // No `>` left, so nothing after this point can be a tag. + cleaned += html.slice(start); + break; + } + if (end - start > 8000) { + cleaned += "<"; + i = start + 1; + continue; + } + const tag = html.slice(start, end + 1); + const parsed = parseOpenTag(tag); + if (!parsed || parsed.closing || parsed.selfClosing) { + cleaned += tag; + i = end + 1; + continue; + } + if (shouldStrip(parsed.name, tag)) { + const close = matchingClose(html, lower, parsed.name, end + 1, absentAfter); + cleaned += " "; + i = close === -1 ? end + 1 : close; + continue; + } + if (isContentBlock(parsed.name, tag) && candidates.length < 8) { + const close = matchingClose(html, lower, parsed.name, end + 1, absentAfter); + if (close !== -1) candidates.push(html.slice(end + 1, close)); + } + cleaned += tag; + i = end + 1; + } + + const textCandidates = [...candidates, cleaned] + .map((candidate) => htmlToText(candidate)) + .filter((text) => text.split(/\s+/).filter(Boolean).length >= 20); + return textCandidates.sort((a, b) => scoreReadableText(b) - scoreReadableText(a))[0] || htmlToText(cleaned); +} + +function htmlToText(html: string): string { + return decodeEntities(html + .replace(/<\/(p|div|section|article|main|h[1-6]|li|blockquote|tr)>/gi, "\n") + .replace(//gi, "\n") + // `[^<>]` keeps each attempt from rescanning to the end of a page full of stray `<`. + .replace(/<[^<>]+>/g, " ") + .replace(/[ \t]+/g, " ") + .replace(/\n{3,}/g, "\n\n") + .trim()); +} + +function scoreReadableText(text: string): number { + const words = text.split(/\s+/).filter(Boolean).length; + const paragraphs = text.split(/\n{2,}/).filter((part) => part.trim().split(/\s+/).length >= 12).length; + const punctuation = (text.match(/[.!?]/g) || []).length; + const navPenalty = (text.match(/\b(cookie|subscribe|sign in|advertisement|privacy policy|all rights reserved)\b/gi) || []).length * 30; + return words + paragraphs * 80 + punctuation * 4 - navPenalty; +} diff --git a/supabase/functions/import-source/html-text_test.ts b/supabase/functions/import-source/html-text_test.ts new file mode 100644 index 000000000..e8d70b1b5 --- /dev/null +++ b/supabase/functions/import-source/html-text_test.ts @@ -0,0 +1,75 @@ +import { assertEquals } from "jsr:@std/assert@1.0.19"; +import { decodeEntities, getMeta, getTitle, stripHtml } from "./html-text.ts"; + +Deno.test("out-of-range numeric entities do not crash the import", () => { + const html = `

${"word ".repeat(30)}� � � & more.

`; + const text = stripHtml(html); + assertEquals(text.includes("Invalid code point"), false); + assertEquals(text.includes("&"), true); + assertEquals(decodeEntities("�"), ""); + assertEquals(decodeEntities("&"), "&"); + assertEquals(decodeEntities("café"), "cafĂ©"); +}); + +Deno.test("article text is kept and a closed nav is dropped", () => { + const words = "Public transport is cheaper than driving for most students in cities."; + const html = `

${words} ${words}

`; + const text = stripHtml(html); + assertEquals(text.includes("Public transport"), true); + assertEquals(text.includes("Home"), false); +}); + +Deno.test("meta titles are read without retrying an unclosed tag", () => { + const hostile = "Voting age`, "fallback"), "Voting age"); + const closed = ``; + assertEquals(getMeta(closed, ["og:title"]), "Free buses & trains"); + if (Date.now() - started > 500) throw new Error("unclosed meta tags took too long"); +}); + +Deno.test("a long run of unclosed nav tags stays fast", () => { + const html = "