Skip to content

Commit eea3ba7

Browse files
cursoragentProgramcode
cursoragent
and
Programcode
committed
Keep URL import working on hostile HTML.
A numeric character past Unicode's range crashed the import, and unclosed meta or nav tags made the old regular expressions retry until the request died. Parsing now walks the page once and skips characters it cannot decode. Co-authored-by: Gáborjáni-Szabó Ádám <Programcode@users.noreply.github.com>
1 parent 4f9e0a0 commit eea3ba7

3 files changed

Lines changed: 238 additions & 80 deletions

File tree

‎supabase/functions/import-source/html-text.ts‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
// HTML cleanup for URL import. Tag removal is a single forward scan: the old
2+
// regular expressions retried unclosed `<meta` and `<nav` until the import
3+
// timed out, and an out-of-range numeric entity threw before any text was saved.
4+
5+
const STRIP_TAGS = new Set([
6+
"script", "style", "template", "noscript", "nav", "footer", "header", "aside", "form", "svg", "iframe", "button",
7+
]);
8+
const ROLE_STRIP = new Set(["banner", "navigation", "complementary", "search", "contentinfo"]);
9+
const CLASS_HINT = /(?:^|[\s_-])(?:cookie|consent|banner|navbar|navigation|nav|menu|sidebar|header|footer|subscribe|newsletter|social|share|paywall)(?:$|[\s_-])/i;
10+
const CONTENT_HINT = /(?:^|[\s_-])(?:article|content|story|post|entry|main|body|reader)(?:$|[\s_-])/i;
11+
const CONTENT_TAGS = new Set(["article", "main", "section", "div"]);
12+
13+
export function decodeEntities(text: string): string {
14+
return text
15+
.replace(/&nbsp;/gi, " ")
16+
.replace(/&amp;/gi, "&")
17+
.replace(/&quot;/gi, "\"")
18+
.replace(/&apos;/gi, "'")
19+
.replace(/&lt;/gi, "<")
20+
.replace(/&gt;/gi, ">")
21+
.replace(/&#x([0-9a-f]+);/gi, (_, hex: string) => fromCodePoint(hex, 16))
22+
.replace(/&#(\d+);/g, (_, dec: string) => fromCodePoint(dec, 10));
23+
}
24+
25+
function fromCodePoint(raw: string, radix: number): string {
26+
const code = parseInt(raw, radix);
27+
// Surrogates and values past Unicode's last code point throw from fromCodePoint.
28+
if (!Number.isInteger(code) || code < 0 || code > 0x10FFFF || (code >= 0xD800 && code <= 0xDFFF)) return "";
29+
return String.fromCodePoint(code);
30+
}
31+
32+
function readAttr(tag: string, attr: string): string | undefined {
33+
const match = tag.match(new RegExp(`\\b${attr}\\s*=\\s*(?:"([^"]*)"|'([^']*)')`, "i"));
34+
return match?.[1] ?? match?.[2];
35+
}
36+
37+
function parseOpenTag(tag: string): { name: string; closing: boolean; selfClosing: boolean } | null {
38+
const match = /^<\s*(\/)?\s*([a-zA-Z][\w:-]*)/.exec(tag);
39+
if (!match) return null;
40+
return {
41+
name: match[2].toLowerCase(),
42+
closing: Boolean(match[1]),
43+
selfClosing: /\/\s*>$/.test(tag),
44+
};
45+
}
46+
47+
export function getMeta(html: string, names: string[]): string | undefined {
48+
const wanted = new Set(names.map((name) => name.toLowerCase()));
49+
const lower = html.toLowerCase();
50+
let i = 0;
51+
while (i < html.length) {
52+
const start = lower.indexOf("<meta", i);
53+
if (start === -1) return undefined;
54+
const next = lower[start + 5];
55+
if (next && /[a-z0-9]/.test(next)) {
56+
i = start + 5;
57+
continue;
58+
}
59+
const end = html.indexOf(">", start);
60+
// An unclosed tag is skipped in one step. Searching it with a regex retried the whole page.
61+
if (end === -1 || end - start > 4000) {
62+
i = start + 5;
63+
continue;
64+
}
65+
const tag = html.slice(start, end + 1);
66+
const name = (readAttr(tag, "property") || readAttr(tag, "name") || "").toLowerCase();
67+
const content = readAttr(tag, "content");
68+
if (name && content !== undefined && wanted.has(name)) return decodeEntities(content.trim());
69+
i = end + 1;
70+
}
71+
return undefined;
72+
}
73+
74+
export function getTitle(html: string, fallback: string): string {
75+
const meta = getMeta(html, ["og:title", "twitter:title"]);
76+
if (meta) return meta;
77+
const lower = html.toLowerCase();
78+
const start = lower.indexOf("<title");
79+
if (start === -1) return fallback;
80+
const openEnd = html.indexOf(">", start);
81+
const close = openEnd === -1 ? -1 : lower.indexOf("</title>", openEnd);
82+
if (openEnd === -1 || close === -1 || close - openEnd > 2000) return fallback;
83+
const text = decodeEntities(html.slice(openEnd + 1, close).replace(/\s+/g, " ").trim());
84+
return text || fallback;
85+
}
86+
87+
function shouldStrip(name: string, tag: string): boolean {
88+
if (STRIP_TAGS.has(name)) return true;
89+
if (name !== "div" && name !== "section") return false;
90+
const role = readAttr(tag, "role")?.toLowerCase();
91+
if (role && ROLE_STRIP.has(role)) return true;
92+
const idClass = `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`;
93+
return CLASS_HINT.test(idClass);
94+
}
95+
96+
function isContentBlock(name: string, tag: string): boolean {
97+
if (!CONTENT_TAGS.has(name)) return false;
98+
if (name === "article" || name === "main") return true;
99+
const idClass = `${readAttr(tag, "id") || ""} ${readAttr(tag, "class") || ""}`;
100+
return CONTENT_HINT.test(idClass);
101+
}
102+
103+
/** Index just after the matching close tag, or -1 when this element is never closed. */
104+
function matchingClose(html: string, lower: string, tag: string, from: number, absentAfter: Map<string, number>): number {
105+
const known = absentAfter.get(tag);
106+
if (known !== undefined && from >= known) return -1;
107+
const openNeedle = `<${tag}`;
108+
const closeNeedle = `</${tag}`;
109+
let depth = 1;
110+
let i = from;
111+
while (depth > 0) {
112+
const nextOpen = lower.indexOf(openNeedle, i);
113+
const nextClose = lower.indexOf(closeNeedle, i);
114+
if (nextClose === -1) {
115+
absentAfter.set(tag, from);
116+
return -1;
117+
}
118+
const openIsTag = nextOpen !== -1 && !/[a-z0-9]/i.test(lower[nextOpen + openNeedle.length] || "");
119+
if (openIsTag && nextOpen < nextClose) {
120+
depth += 1;
121+
i = nextOpen + openNeedle.length;
122+
continue;
123+
}
124+
depth -= 1;
125+
i = nextClose + closeNeedle.length;
126+
}
127+
const end = html.indexOf(">", i - closeNeedle.length);
128+
return end === -1 ? html.length : end + 1;
129+
}
130+
131+
export function stripHtml(html: string): string {
132+
const lower = html.toLowerCase();
133+
const absentAfter = new Map<string, number>();
134+
const candidates: string[] = [];
135+
let cleaned = "";
136+
let i = 0;
137+
while (i < html.length) {
138+
const start = html.indexOf("<", i);
139+
if (start === -1) {
140+
cleaned += html.slice(i);
141+
break;
142+
}
143+
cleaned += html.slice(i, start);
144+
if (lower.startsWith("<!--", start)) {
145+
const end = html.indexOf("-->", start + 4);
146+
cleaned += " ";
147+
i = end === -1 ? html.length : end + 3;
148+
continue;
149+
}
150+
const end = html.indexOf(">", start + 1);
151+
if (end === -1 || end - start > 8000) {
152+
cleaned += "<";
153+
i = start + 1;
154+
continue;
155+
}
156+
const tag = html.slice(start, end + 1);
157+
const parsed = parseOpenTag(tag);
158+
if (!parsed || parsed.closing || parsed.selfClosing) {
159+
cleaned += tag;
160+
i = end + 1;
161+
continue;
162+
}
163+
if (shouldStrip(parsed.name, tag)) {
164+
const close = matchingClose(html, lower, parsed.name, end + 1, absentAfter);
165+
cleaned += " ";
166+
i = close === -1 ? end + 1 : close;
167+
continue;
168+
}
169+
if (isContentBlock(parsed.name, tag) && candidates.length < 8) {
170+
const close = matchingClose(html, lower, parsed.name, end + 1, absentAfter);
171+
if (close !== -1) candidates.push(html.slice(end + 1, close));
172+
}
173+
cleaned += tag;
174+
i = end + 1;
175+
}
176+
177+
const textCandidates = [...candidates, cleaned]
178+
.map((candidate) => htmlToText(candidate))
179+
.filter((text) => text.split(/\s+/).filter(Boolean).length >= 20);
180+
return textCandidates.sort((a, b) => scoreReadableText(b) - scoreReadableText(a))[0] || htmlToText(cleaned);
181+
}
182+
183+
function htmlToText(html: string): string {
184+
return decodeEntities(html
185+
.replace(/<\/(p|div|section|article|main|h[1-6]|li|blockquote|tr)>/gi, "\n")
186+
.replace(/<br\s*\/?>/gi, "\n")
187+
.replace(/<[^>]+>/g, " ")
188+
.replace(/[ \t]+/g, " ")
189+
.replace(/\n{3,}/g, "\n\n")
190+
.trim());
191+
}
192+
193+
function scoreReadableText(text: string): number {
194+
const words = text.split(/\s+/).filter(Boolean).length;
195+
const paragraphs = text.split(/\n{2,}/).filter((part) => part.trim().split(/\s+/).length >= 12).length;
196+
const punctuation = (text.match(/[.!?]/g) || []).length;
197+
const navPenalty = (text.match(/\b(cookie|subscribe|sign in|advertisement|privacy policy|all rights reserved)\b/gi) || []).length * 30;
198+
return words + paragraphs * 80 + punctuation * 4 - navPenalty;
199+
}

‎supabase/functions/import-source/html-text_test.ts‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
import { assertEquals } from "jsr:@std/assert@1.0.19";
2+
import { decodeEntities, getMeta, getTitle, stripHtml } from "./html-text.ts";
3+
4+
Deno.test("out-of-range numeric entities do not crash the import", () => {
5+
const html = `<article><p>${"word ".repeat(30)}&#99999999; &#x110000; &#xD800; &#38; more.</p></article>`;
6+
const text = stripHtml(html);
7+
assertEquals(text.includes("Invalid code point"), false);
8+
assertEquals(text.includes("&"), true);
9+
assertEquals(decodeEntities("&#99999999;"), "");
10+
assertEquals(decodeEntities("&#38;"), "&");
11+
assertEquals(decodeEntities("caf&#233;"), "café");
12+
});
13+
14+
Deno.test("article text is kept and a closed nav is dropped", () => {
15+
const words = "Public transport is cheaper than driving for most students in cities.";
16+
const html = `<nav><a>Home</a><a>Sign in</a></nav><article><p>${words} ${words}</p></article>`;
17+
const text = stripHtml(html);
18+
assertEquals(text.includes("Public transport"), true);
19+
assertEquals(text.includes("Home"), false);
20+
});
21+
22+
Deno.test("meta titles are read without retrying an unclosed tag", () => {
23+
const hostile = "<meta name='og:title' ".repeat(8000);
24+
const started = Date.now();
25+
assertEquals(getMeta(hostile, ["og:title"]), undefined);
26+
assertEquals(getTitle(`${hostile}<title>Voting age</title>`, "fallback"), "Voting age");
27+
const closed = `<meta property="og:title" content="Free buses &amp; trains">`;
28+
assertEquals(getMeta(closed, ["og:title"]), "Free buses & trains");
29+
if (Date.now() - started > 500) throw new Error("unclosed meta tags took too long");
30+
});
31+
32+
Deno.test("a long run of unclosed nav tags stays fast", () => {
33+
const html = "<nav class='menu'>".repeat(4000) + `<article><p>${"evidence ".repeat(40)}</p></article>`;
34+
const started = Date.now();
35+
const text = stripHtml(html);
36+
if (Date.now() - started > 500) throw new Error("unclosed nav tags took too long");
37+
assertEquals(text.includes("evidence"), true);
38+
});

‎supabase/functions/import-source/index.ts‎

Lines changed: 1 addition & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { createClient, type User } from "https://esm.sh/@supabase/supabase-js@2.114.0";
22
import { isPrivateIpv4, isPrivateIpv6, validatePublicHttpUrl, validateResolvedPublicHost } from "./url-validation.ts";
33
import { readBoundedText } from "./resource-limits.ts";
4+
import { getMeta, getTitle, stripHtml } from "./html-text.ts";
45
import { BodyTooLargeError, discardRequestBody, readBoundedBody, untilAborted } from "../_shared/bounded-body.ts";
56
import { HttpError, jsonError, requireUser } from "../_shared/auth.ts";
67
import { getCorsHeaders } from "../_shared/cors.ts";
@@ -102,85 +103,6 @@ async function fetchWithRedirects(initialUrl: URL, signal: AbortSignal) {
102103
throw new Error("Too many redirects.");
103104
}
104105

105-
function decodeEntities(text: string) {
106-
return text
107-
.replace(/&nbsp;/gi, " ")
108-
.replace(/&amp;/gi, "&")
109-
.replace(/&quot;/gi, "\"")
110-
.replace(/&apos;/gi, "'")
111-
.replace(/&lt;/gi, "<")
112-
.replace(/&gt;/gi, ">")
113-
.replace(/&#x([0-9a-f]+);/gi, (_, hex) => {
114-
const code = parseInt(hex, 16);
115-
return Number.isFinite(code) ? String.fromCodePoint(code) : "";
116-
})
117-
.replace(/&#(\d+);/g, (_, dec) => {
118-
const code = parseInt(dec, 10);
119-
return Number.isFinite(code) ? String.fromCodePoint(code) : "";
120-
});
121-
}
122-
123-
function getMeta(html: string, names: string[]) {
124-
for (const name of names) {
125-
const pattern = new RegExp(`<meta\\s+[^>]*(?:name|property)=["']${name}["'][^>]*content=["']([^"']+)["'][^>]*>`, "i");
126-
const reversePattern = new RegExp(`<meta\\s+[^>]*content=["']([^"']+)["'][^>]*(?:name|property)=["']${name}["'][^>]*>`, "i");
127-
const match = html.match(pattern) || html.match(reversePattern);
128-
if (match?.[1]) return decodeEntities(match[1].trim());
129-
}
130-
return undefined;
131-
}
132-
133-
function getTitle(html: string, fallback: string) {
134-
return getMeta(html, ["og:title", "twitter:title"]) ||
135-
decodeEntities(html.match(/<title[^>]*>([\s\S]*?)<\/title>/i)?.[1]?.replace(/\s+/g, " ").trim() || "") ||
136-
fallback;
137-
}
138-
139-
function stripHtml(html: string) {
140-
const cleaned = html
141-
.replace(/<script[\s\S]*?<\/script>/gi, " ")
142-
.replace(/<style[\s\S]*?<\/style>/gi, " ")
143-
.replace(/<template[\s\S]*?<\/template>/gi, " ")
144-
.replace(/<noscript[\s\S]*?<\/noscript>/gi, " ")
145-
.replace(/<!--[\s\S]*?-->/g, " ")
146-
.replace(/<(nav|footer|header|aside|form|svg|iframe|button)[\s\S]*?<\/\1>/gi, " ")
147-
.replace(/<(div|section)[^>]*role=["'](banner|navigation|complementary|search|contentinfo)["'][\s\S]*?<\/\1>/gi, " ")
148-
.replace(/<(div|section)[^>]*(?:id|class)=["'][^"']*(?:cookie|consent|banner|nav(?:bar|igation)?|menu|sidebar|header|footer|subscribe|newsletter|social|share|paywall)[^"']*["'][\s\S]*?<\/\1>/gi, " ");
149-
150-
const candidates: string[] = [];
151-
const blockPattern = /<(article|main|section|div)[^>]*(?:id|class)=["'][^"']*(?:article|content|story|post|entry|main|body|reader)[^"']*["'][^>]*>([\s\S]*?)<\/\1>/gi;
152-
for (const match of cleaned.matchAll(blockPattern)) {
153-
if (match[2]) candidates.push(match[2]);
154-
}
155-
const semanticMatch = cleaned.match(/<(article|main)[^>]*>([\s\S]*?)<\/\1>/i);
156-
if (semanticMatch?.[2]) candidates.push(semanticMatch[2]);
157-
candidates.push(cleaned);
158-
159-
const textCandidates = candidates
160-
.map((candidate) => htmlToText(candidate))
161-
.filter((text) => text.split(/\s+/).filter(Boolean).length >= 20);
162-
163-
return textCandidates.sort((a, b) => scoreReadableText(b) - scoreReadableText(a))[0] || htmlToText(cleaned);
164-
}
165-
166-
function htmlToText(html: string) {
167-
return decodeEntities(html
168-
.replace(/<\/(p|div|section|article|main|h[1-6]|li|blockquote|tr)>/gi, "\n")
169-
.replace(/<br\s*\/?>/gi, "\n")
170-
.replace(/<[^>]+>/g, " ")
171-
.replace(/[ \t]+/g, " ")
172-
.replace(/\n{3,}/g, "\n\n")
173-
.trim());
174-
}
175-
176-
function scoreReadableText(text: string) {
177-
const words = text.split(/\s+/).filter(Boolean).length;
178-
const paragraphs = text.split(/\n{2,}/).filter((part) => part.trim().split(/\s+/).length >= 12).length;
179-
const punctuation = (text.match(/[.!?]/g) || []).length;
180-
const navPenalty = (text.match(/\b(cookie|subscribe|sign in|advertisement|privacy policy|all rights reserved)\b/gi) || []).length * 30;
181-
return words + paragraphs * 80 + punctuation * 4 - navPenalty;
182-
}
183-
184106
const UI_PHRASE_PATTERNS = [
185107
/^skip to (?:main )?content\b/i,
186108
/^skip to navigation\b/i,
@@ -313,7 +235,6 @@ export async function handleRequest(req: Request): Promise<Response> {
313235
}
314236

315237
const raw = await readBoundedText(response, deadline.signal);
316-
clearTimeout(deadlineTimer);
317238
const isHtml = contentType.includes("html") || /<html|<article|<main|<p[\s>]/i.test(raw);
318239
const content = normalizeText(isHtml ? stripHtml(raw) : raw);
319240

0 commit comments

Comments
 (0)