diff --git a/.github/scripts/issue-quality-core.cjs b/.github/scripts/issue-quality-core.cjs index 40fadaca6d..8b3c1efb9b 100644 --- a/.github/scripts/issue-quality-core.cjs +++ b/.github/scripts/issue-quality-core.cjs @@ -31,7 +31,7 @@ function unwrapSingleEnclosingFence(text) { */ function normalizeRawSectionValue(raw) { if (typeof raw !== "string") return null; - let value = raw.replace(//g, "").trim(); + let value = raw.replace(/|$)/g, "").trim(); if (!value) return null; // A lone fenced block whose entire body is a stand-in is still a stand-in @@ -179,7 +179,7 @@ function stripHtmlMedia(text) { if (typeof text !== "string") return ""; let s = text .replace(/]*>/gi, " ") - .replace(//g, " "); + .replace(/|$)/g, " "); // Whole media blocks: replace only when the inner content is not // substantive text (no word characters outside tags). @@ -419,12 +419,129 @@ function isMediaOnly(text) { return stripped.replace(/\s+/g, "").length === 0; } +/** + * Strip HTML comments, placeholder-only values, and trim whitespace. + */ +function stripHtmlCommentsOutsideCode(raw) { + const text = String(raw); + // Linear scanner, deliberately not a regex. The previous masker combined a + // variable-length delimiter capture, a lazy whole-input scan and a + // backreference, which backtracks catastrophically on adversarial input: a + // 60k-character issue body took ~10.5s, inside an automation trust boundary + // that anyone can post to. This walks the string once. + // + // It also fixes two GFM cases the regex got wrong: a closing fence may be + // LONGER than its opener, and a code span may contain a line ending. + let out = ""; + let i = 0; + let atLineStart = true; + + const lineEnd = (from) => { + const nl = text.indexOf("\n", from); + return nl === -1 ? text.length : nl; + }; + + while (i < text.length) { + const ch = text[i]; + + // Fenced block: at most three leading spaces, then >= 3 backticks or tildes. + if (atLineStart && (ch === "`" || ch === "~" || ch === " " || ch === "\t")) { + let j = i; + let indent = 0; + while (j < text.length && (text[j] === " " || text[j] === "\t") && indent < 4) { j++; indent++; } + const marker = text[j]; + if (indent < 4 && (marker === "`" || marker === "~")) { + let run = 0; + while (text[j + run] === marker) run++; + if (run >= 3) { + // Copy the opening line verbatim, then everything up to a closing + // fence of the SAME character and AT LEAST the same length. + // `cursor` is the index of the newline ending the current line, so the + // next line starts at cursor + 1. + let cursor = lineEnd(j + run); + let closed = false; + while (cursor < text.length) { + const start = cursor + 1; + let p = start; + let ind = 0; + while (p < text.length && (text[p] === " " || text[p] === "\t") && ind < 4) { p++; ind++; } + let closeRun = 0; + while (text[p + closeRun] === marker) closeRun++; + const after = p + closeRun; + const rest = text.slice(after, lineEnd(after)); + if (ind < 4 && closeRun >= run && rest.trim() === "") { + const end = lineEnd(after); + out += text.slice(i, end); + i = end; + closed = true; + break; + } + const next = lineEnd(start); + if (next <= cursor) break; + cursor = next; + } + if (closed) { atLineStart = true; continue; } + // Unclosed fence runs to end of input, per GFM. + out += text.slice(i); + return out; + } + } + } + + // Inline code span: a run of N backticks closed by the next run of exactly N. + if (ch === "`") { + let run = 0; + while (text[i + run] === "`") run++; + let p = i + run; + let close = -1; + while (p < text.length) { + if (text[p] === "`") { + let r = 0; + while (text[p + r] === "`") r++; + if (r === run) { close = p + r; break; } + p += r; + continue; + } + p++; + } + if (close !== -1) { + out += text.slice(i, close); + atLineStart = false; + i = close; + continue; + } + } + + // HTML comment outside any code region. An unclosed one runs to EOF. + if (ch === "<" && text.startsWith("", i + 4); + if (end === -1) return out; + i = end + 3; + continue; + } + + out += ch; + atLineStart = ch === "\n"; + i++; + } + return out; +} + +/** + * Strip HTML comments, placeholder-only values, and trim whitespace. + */ /** * Strip HTML comments, placeholder-only values, and trim whitespace. */ function clean(raw) { if (typeof raw !== "string") return ""; - let s = raw.replace(//g, ""); + // Comment stripping must not reach inside fenced code. GFM treats fence + // contents as literal text, so a ` world"), "Hello world"); + assert.equal(clean("\nVisible text"), "Visible text"); }); it("normalises punctuation and capitalisation", () => { @@ -2092,3 +2094,50 @@ describe("detectAreaLabels", () => { assert.ok(labels.includes("streaming"), `got ${labels.join(",")}`); }); }); + +describe("clean() respects fenced code (regression)", () => { + it("keeps section text that follows a comment-like literal in a fence", () => { + // A `").trim(), ""); + }); +}); + +describe("code-region scanning is GFM-correct and linear (regression)", () => { + it("honors a closing fence longer than its opener", () => { + // GFM allows the closing fence to be longer. Requiring an exact-length + // match left the block unterminated, so the comment inside it ran to EOF + // and swallowed the visible section below. + const goal = ["```html", "/g; +/** HTML comments, which GitHub never renders. An unclosed comment runs through EOF. */ +const HTML_COMMENT_RE = /|$)/g; /** Fenced code blocks (``` or ~~~) whose content GitHub does not render. */ const FENCED_CODE_RE = /(?:^|\n)[ \t]*(`{3,}|~{3,})[^\n]*\n[\s\S]*?^[ \t]*\1[ \t]*(?=\n|$)/gm; /** Embedded markdown image (`![alt](url)`), as GitHub renders for dropped images. */ @@ -147,7 +147,7 @@ function assessPrDescription(body) { const withoutTemplate = stripPrTemplateBoilerplate(withoutReadiness); const cleaned = clean(withoutTemplate); if (!cleaned) { - const strippedComments = withoutTemplate.replace(//g, "").trim(); + const strippedComments = withoutTemplate.replace(HTML_COMMENT_RE, "").trim(); if (!strippedComments) return { ok: false, reason: "empty" }; if (isPlaceholderOnlyValue(strippedComments)) { return { ok: false, reason: "placeholder" }; @@ -245,7 +245,13 @@ function hasGuiOverride({ comments = [] }) { * fenced code blocks. Image syntax there is literal text, not evidence. */ function stripNonRenderedRegions(body) { - return body.replace(HTML_COMMENT_RE, "").replace(FENCED_CODE_RE, ""); + // Fenced code MUST be removed first. GFM treats fence contents as literal + // text, so a `\n\n").reason, "empty", ); + assert.equal( + assessPrDescription("'), false, ); + assert.equal( + hasScreenshotEvidence(""].join("\n"); + assert.equal(hasScreenshotEvidence(body), false); + }); +}); diff --git a/scripts/release-notes.ts b/scripts/release-notes.ts index 98aa308731..1508fe7167 100644 --- a/scripts/release-notes.ts +++ b/scripts/release-notes.ts @@ -161,8 +161,8 @@ export function stripCarriedReleaseNotes(body: string): string { export function isEmptyGeneratedNotes(body: string): boolean { const withoutComment = body .replace(/\r\n/g, "\n") + .replace(/|$)/g, "") .split("\n") - .filter(line => !/^$/.test(line.trim())) .filter(line => !/^\*\*Full Changelog\*\*:/.test(line)) .join("\n"); return !hasNonWhitespace(withoutComment); diff --git a/src/codex/inject.ts b/src/codex/inject.ts index 7d5b516abe..0a02e3dd10 100644 --- a/src/codex/inject.ts +++ b/src/codex/inject.ts @@ -449,7 +449,7 @@ function stripRootRoutedModel(content: string): string { .filter((line, i) => { const isRoot = firstTable === -1 || i < firstTable; if (!isRoot) return true; - const m = line.match(/^\s*model\s*=\s*("(?:\\.|[^"])*"|'[^']*')\s*$/); + const m = line.match(/^\s*model\s*=\s*("(?:\\.|[^"\\])*"|'[^']*')\s*$/); if (!m) return true; const model = parseTomlString(m[1]); return !model?.includes("/"); @@ -485,7 +485,7 @@ function setRootModelCatalogPath(content: string, catalogPath: string): string { const rootEnd = firstTable === -1 ? lines.length : firstTable; for (let i = 0; i < rootEnd; i++) { const m = lines[i].match( - /^\s*model_catalog_json\s*=\s*("(?:\\.|[^"])*"|'[^']*')\s*$/, + /^\s*model_catalog_json\s*=\s*("(?:\\.|[^"\\])*"|'[^']*')\s*$/, ); if (!m) continue; const existing = parseTomlString(m[1]); @@ -580,7 +580,7 @@ function stripOpencodexCatalogPath(content: string): string { .split("\n") .filter((line) => { const m = line.match( - /^\s*model_catalog_json\s*=\s*("(?:\\.|[^"])*"|'[^']*')\s*$/, + /^\s*model_catalog_json\s*=\s*("(?:\\.|[^"\\])*"|'[^']*')\s*$/, ); return !m || !isOpencodexCatalogPath(parseTomlString(m[1])); }) diff --git a/src/codex/plugins-doctor.ts b/src/codex/plugins-doctor.ts index db6657f927..3a58b47c3d 100644 --- a/src/codex/plugins-doctor.ts +++ b/src/codex/plugins-doctor.ts @@ -59,7 +59,7 @@ function readMarketplaceTable(configText: string, name: string): Record { expect(stripped).toContain('model_verbosity = "high"'); }); + test("malformed quoted root values cannot wedge restore transforms", () => { + const slashRun = "\\".repeat(64); + const stripped = stripOpencodexConfig([ + 'model_provider = "opencodex"', + `model = "${slashRun}`, + `model_catalog_json = "${slashRun}`, + "", + ].join("\n")); + + expect(stripped).toContain(`model = "${slashRun}`); + expect(stripped).toContain(`model_catalog_json = "${slashRun}`); + }, 2_000); + test("preserves non-opencodex routed model names during fallback restore", () => { const stripped = stripOpencodexConfig([ 'model_provider = "proxy"', diff --git a/tests/codex-plugins-doctor.test.ts b/tests/codex-plugins-doctor.test.ts index af7ff029c6..1c19d4558e 100644 --- a/tests/codex-plugins-doctor.test.ts +++ b/tests/codex-plugins-doctor.test.ts @@ -120,6 +120,64 @@ describe("diagnoseCodexBundledPlugins (direct, platform-injected)", () => { } }); + test("malformed quoted marketplace values cannot wedge diagnosis", () => { + const { dir, configPath } = makeConfig( + `[marketplaces.openai-bundled]\nsource_type = "local"\nsource = "${"\\".repeat(64)}\n`, + ); + try { + const result = diagnoseCodexBundledPlugins({ + platform: "win32", + configPath, + locateCurrent: () => null, + }); + expect(result.applicable).toBe(true); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 2_000); + + test("malformed quoted values with long trailing whitespace are not parsed as bare values", () => { + const { dir, configPath } = makeConfig( + `[marketplaces.openai-bundled]\nsource_type = "local"\nsource = "${" ".repeat(20_000)}x\n`, + ); + try { + const result = diagnoseCodexBundledPlugins({ + platform: "win32", + configPath, + locateCurrent: () => null, + }); + expect(result.applicable).toBe(true); + if (result.applicable) { + expect(result.marketplace.sourceType).toBe("local"); + expect(result.marketplace.source).toBeNull(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }, 2_000); + + test("unterminated quoted values are rejected instead of parsed as bare values", () => { + const { dir, configPath } = makeConfig( + `[marketplaces.openai-bundled]\nsource_type = "local"\nsource = "unterminated\n`, + ); + try { + const result = diagnoseCodexBundledPlugins({ + platform: "win32", + configPath, + locateCurrent: () => null, + }); + expect(result.applicable).toBe(true); + if (result.applicable) { + expect(result.marketplace.sourceType).toBe("local"); + expect(result.marketplace.source).toBeNull(); + expect(result.stale).toBe(false); + expect(result.suggestedRepair).toBeNull(); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + test("parses a table header with a trailing inline comment", () => { const { dir, configPath } = makeConfig( `[marketplaces.openai-bundled] # bundled\nsource_type = "local"\nsource = "X:\\\\gone"\n`, diff --git a/tests/project-config-warnings.test.ts b/tests/project-config-warnings.test.ts index c5609f9c56..eb079760af 100644 --- a/tests/project-config-warnings.test.ts +++ b/tests/project-config-warnings.test.ts @@ -9,6 +9,7 @@ import { explainProjectConfigBypass, isGlobalOpencodexRoutingActive, invalidateProjectConfigDiagnosticsCache, + parseTomlDocument, parseTrustedProjectPathsFromCodexConfig, relPath, resolveEffectiveProjectModelProvider, @@ -119,7 +120,17 @@ base_url = "http://127.0.0.1:10100/v1" }); }); -describe("parseTrustedProjectPathsFromCodexConfig", () => { +describe("parseTomlDocument", () => { + test("malformed basic strings cannot wedge parsing and escaped strings still parse", () => { + const malformed = parseTomlDocument('model_provider = "' + "\\".repeat(64)); + expect(typeof malformed.root.model_provider).toBe("string"); + + const valid = parseTomlDocument('model_provider = "provider\\\\name"'); + expect(valid.root.model_provider).toBe("provider\\name"); + }, 2_000); + }); + + describe("parseTrustedProjectPathsFromCodexConfig", () => { test("collects only trusted project paths", () => { const text = ` [projects.'C:\\repo-a'] diff --git a/tests/release-notes.test.ts b/tests/release-notes.test.ts index e9a0562c82..aa6bc7d296 100644 --- a/tests/release-notes.test.ts +++ b/tests/release-notes.test.ts @@ -172,7 +172,15 @@ describe("stripCarriedReleaseNotes", () => { }); }); -describe("joinCarriedPreviewNotes", () => { +describe("hasMeaningfulCarriedNotes", () => { + test("HTML comments stay non-meaningful through a closing marker or EOF", () => { + expect(hasMeaningfulCarriedNotes("")).toBe(false); + expect(hasMeaningfulCarriedNotes("\n## What's Changed\n* visible fix")).toBe(true); + }); + }); + + describe("joinCarriedPreviewNotes", () => { test("aggregates multiple incremental preview bodies in order", () => { const joined = joinCarriedPreviewNotes([ "## What's Changed\n* fix A",