From eea3ba7c3cfb3bd86ad241e4a9c90352df40d257 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 22 Oct 2026 10:50:29 +0000 Subject: [PATCH 1/2] Keep URL import working on hostile HTML. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A numeric character past Unicode's range crashed the import, and unclosed meta or nav tags made the old regular expressions retry until the request died. Parsing now walks the page once and skips characters it cannot decode. Co-authored-by: Gáborjáni-Szabó Ádám --- supabase/functions/import-source/html-text.ts | 199 ++++++++++++++++++ .../functions/import-source/html-text_test.ts | 38 ++++ supabase/functions/import-source/index.ts | 81 +------ 3 files changed, 238 insertions(+), 80 deletions(-) create mode 100644 supabase/functions/import-source/html-text.ts create mode 100644 supabase/functions/import-source/html-text_test.ts diff --git a/supabase/functions/import-source/html-text.ts b/supabase/functions/import-source/html-text.ts new file mode 100644 index 000000000..c40acd931 --- /dev/null +++ b/supabase/functions/import-source/html-text.ts @@ -0,0 +1,199 @@ +// HTML cleanup for URL import. Tag removal is a single forward scan: the old +// regular expressions retried unclosed `") + .replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => fromCodePoint(hex, 16)) + .replace(/&#(\d+);/g, (_, dec: string) => fromCodePoint(dec, 10)); +} + +function fromCodePoint(raw: string, radix: number): string { + const code = parseInt(raw, radix); + // Surrogates and values past Unicode's last code point throw from fromCodePoint. + if (!Number.isInteger(code) || code < 0 || code > 0x10FFFF || (code >= 0xD800 && code <= 0xDFFF)) return ""; + return String.fromCodePoint(code); +} + +function readAttr(tag: string, attr: string): string | undefined { + const match = tag.match(new RegExp(`\\b${attr}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, "i")); + return match?.[1] ?? match?.[2]; +} + +function parseOpenTag(tag: string): { name: string; closing: boolean; selfClosing: boolean } | null { + const match = /^<\s*(\/)?\s*([a-zA-Z][\w:-]*)/.exec(tag); + if (!match) return null; + return { + name: match[2].toLowerCase(), + closing: Boolean(match[1]), + selfClosing: /\/\s*>$/.test(tag), + }; +} + +export function getMeta(html: string, names: string[]): string | undefined { + const wanted = new Set(names.map((name) => name.toLowerCase())); + const lower = html.toLowerCase(); + let i = 0; + while (i < html.length) { + const start = lower.indexOf("", start); + // An unclosed tag is skipped in one step. Searching it with a regex retried the whole page. + if (end === -1 || end - start > 4000) { + i = start + 5; + continue; + } + const tag = html.slice(start, end + 1); + const name = (readAttr(tag, "property") || readAttr(tag, "name") || "").toLowerCase(); + const content = readAttr(tag, "content"); + if (name && content !== undefined && wanted.has(name)) return decodeEntities(content.trim()); + i = end + 1; + } + return undefined; +} + +export function getTitle(html: string, fallback: string): string { + const meta = getMeta(html, ["og:title", "twitter:title"]); + if (meta) return meta; + const lower = html.toLowerCase(); + const start = lower.indexOf("", start); + const close = openEnd === -1 ? -1 : lower.indexOf("", openEnd); + if (openEnd === -1 || close === -1 || close - openEnd > 2000) return fallback; + const text = decodeEntities(html.slice(openEnd + 1, close).replace(/\s+/g, " ").trim()); + return text || fallback; +} + +function shouldStrip(name: string, tag: string): boolean { + if (STRIP_TAGS.has(name)) return true; + if (name !== "div" && name !== "section") return false; + const role = readAttr(tag, "role")?.toLowerCase(); + if (role && ROLE_STRIP.has(role)) return true; + const idClass = `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`; + return CLASS_HINT.test(idClass); +} + +function isContentBlock(name: string, tag: string): boolean { + if (!CONTENT_TAGS.has(name)) return false; + if (name === "article" || name === "main") return true; + const idClass = `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`; + return CONTENT_HINT.test(idClass); +} + +/** Index just after the matching close tag, or -1 when this element is never closed. */ +function matchingClose(html: string, lower: string, tag: string, from: number, absentAfter: Map): number { + const known = absentAfter.get(tag); + if (known !== undefined && from >= known) return -1; + const openNeedle = `<${tag}`; + const closeNeedle = ` 0) { + const nextOpen = lower.indexOf(openNeedle, i); + const nextClose = lower.indexOf(closeNeedle, i); + if (nextClose === -1) { + absentAfter.set(tag, from); + return -1; + } + const openIsTag = nextOpen !== -1 && !/[a-z0-9]/i.test(lower[nextOpen + openNeedle.length] || ""); + if (openIsTag && nextOpen < nextClose) { + depth += 1; + i = nextOpen + openNeedle.length; + continue; + } + depth -= 1; + i = nextClose + closeNeedle.length; + } + const end = html.indexOf(">", i - closeNeedle.length); + return end === -1 ? html.length : end + 1; +} + +export function stripHtml(html: string): string { + const lower = html.toLowerCase(); + const absentAfter = new Map(); + const candidates: string[] = []; + let cleaned = ""; + let i = 0; + while (i < html.length) { + const start = html.indexOf("<", i); + if (start === -1) { + cleaned += html.slice(i); + break; + } + cleaned += html.slice(i, start); + if (lower.startsWith("", start + 4); + cleaned += " "; + i = end === -1 ? html.length : end + 3; + continue; + } + const end = html.indexOf(">", start + 1); + if (end === -1 || end - start > 8000) { + cleaned += "<"; + i = start + 1; + continue; + } + const tag = html.slice(start, end + 1); + const parsed = parseOpenTag(tag); + if (!parsed || parsed.closing || parsed.selfClosing) { + cleaned += tag; + i = end + 1; + continue; + } + if (shouldStrip(parsed.name, tag)) { + const close = matchingClose(html, lower, parsed.name, end + 1, absentAfter); + cleaned += " "; + i = close === -1 ? end + 1 : close; + continue; + } + if (isContentBlock(parsed.name, tag) && candidates.length < 8) { + const close = matchingClose(html, lower, parsed.name, end + 1, absentAfter); + if (close !== -1) candidates.push(html.slice(end + 1, close)); + } + cleaned += tag; + i = end + 1; + } + + const textCandidates = [...candidates, cleaned] + .map((candidate) => htmlToText(candidate)) + .filter((text) => text.split(/\s+/).filter(Boolean).length >= 20); + return textCandidates.sort((a, b) => scoreReadableText(b) - scoreReadableText(a))[0] || htmlToText(cleaned); +} + +function htmlToText(html: string): string { + return decodeEntities(html + .replace(/<\/(p|div|section|article|main|h[1-6]|li|blockquote|tr)>/gi, "\n") + .replace(//gi, "\n") + .replace(/<[^>]+>/g, " ") + .replace(/[ \t]+/g, " ") + .replace(/\n{3,}/g, "\n\n") + .trim()); +} + +function scoreReadableText(text: string): number { + const words = text.split(/\s+/).filter(Boolean).length; + const paragraphs = text.split(/\n{2,}/).filter((part) => part.trim().split(/\s+/).length >= 12).length; + const punctuation = (text.match(/[.!?]/g) || []).length; + const navPenalty = (text.match(/\b(cookie|subscribe|sign in|advertisement|privacy policy|all rights reserved)\b/gi) || []).length * 30; + return words + paragraphs * 80 + punctuation * 4 - navPenalty; +} diff --git a/supabase/functions/import-source/html-text_test.ts b/supabase/functions/import-source/html-text_test.ts new file mode 100644 index 000000000..40fe3de3f --- /dev/null +++ b/supabase/functions/import-source/html-text_test.ts @@ -0,0 +1,38 @@ +import { assertEquals } from "jsr:@std/assert@1.0.19"; +import { decodeEntities, getMeta, getTitle, stripHtml } from "./html-text.ts"; + +Deno.test("out-of-range numeric entities do not crash the import", () => { + const html = `

${"word ".repeat(30)}� � � & more.

`; + const text = stripHtml(html); + assertEquals(text.includes("Invalid code point"), false); + assertEquals(text.includes("&"), true); + assertEquals(decodeEntities("�"), ""); + assertEquals(decodeEntities("&"), "&"); + assertEquals(decodeEntities("café"), "café"); +}); + +Deno.test("article text is kept and a closed nav is dropped", () => { + const words = "Public transport is cheaper than driving for most students in cities."; + const html = `

${words} ${words}

`; + const text = stripHtml(html); + assertEquals(text.includes("Public transport"), true); + assertEquals(text.includes("Home"), false); +}); + +Deno.test("meta titles are read without retrying an unclosed tag", () => { + const hostile = "Voting age`, "fallback"), "Voting age"); + const closed = ``; + assertEquals(getMeta(closed, ["og:title"]), "Free buses & trains"); + if (Date.now() - started > 500) throw new Error("unclosed meta tags took too long"); +}); + +Deno.test("a long run of unclosed nav tags stays fast", () => { + const html = "