diff --git a/products/maka/workbench/app/BotReply.tsx b/products/maka/workbench/app/BotReply.tsx index c595536270..be1552765f 100644 --- a/products/maka/workbench/app/BotReply.tsx +++ b/products/maka/workbench/app/BotReply.tsx @@ -22,6 +22,7 @@ import { Button, TextView } from 'nsh/base'; import { MermaidView, mermaidExtensions } from './MermaidView'; import type { BotTranscriptMessage } from './botTranscript'; import { tr } from './locale'; +import { closeCjkEmphasis } from './markdownEmphasis'; import { Text } from 'nsh'; import { Icon } from './Icon'; import { C, bodyTextStyle, captionTextStyle } from './ui'; @@ -45,7 +46,7 @@ export function BotReply(props: { String(message.sequence)}> {(message) => ( node.text ?? ''} />} sourceId={`bot/${message().sequence}`} diff --git a/products/maka/workbench/app/markdownEmphasis.ts b/products/maka/workbench/app/markdownEmphasis.ts new file mode 100644 index 0000000000..86ab49ec39 --- /dev/null +++ b/products/maka/workbench/app/markdownEmphasis.ts @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * CommonMark closes `**` only if the character before it is not punctuation, or the one + * after it is whitespace or punctuation (and mirrored for opening). Chinese is written + * without spaces, so `**回升能否守住:**如果…` never closes and its asterisks show as text. + * + * Where a `**…**` pair fails for that reason, the punctuation at the edge of the bold text + * moves outside the markers (`**回升能否守住**:如果…`): the visible text is the same and + * the pair closes. A balanced wrapper that surrounds the bold text moves out as a pair + * (`他说**“不行”**然后` → `他说“**不行**”然后`), so nothing is half bold. + * + * This is a text pass, not a parser, so it only rewrites what it can recognise as plain + * prose and leaves every other case exactly as written: fenced and indented code, code + * spans (including ones that run over lines), escapes, link destinations, reference + * definitions, autolinks, raw URLs and HTML tags. A pair it cannot move without splitting + * bold text (`**营收(同比 +12%)**增长`) is left as written too. + */ + +/** Punctuation that ends a clause or label and is safe to move out of the bold text. */ +const PEEL = new Set([':', ':', ',', ',', '。', '.', ';', ';', '、', '!', '!', '?', '?']); +const OPENERS = '“‘《【「『(([〈«{'; +const CLOSERS = '”’》】」』))]〉»}'; +const isPunctuation = (char: string) => /[\p{P}\p{S}]/u.test(char); +const isBlank = (char: string) => /\s/u.test(char); +const closesOn = (next: string | undefined) => + next === undefined || isBlank(next) || isPunctuation(next); +const opensAfter = (previous: string | undefined) => + previous === undefined || isBlank(previous) || isPunctuation(previous); +const wraps = (open: string, close: string) => + OPENERS.includes(open) && OPENERS.indexOf(open) === CLOSERS.indexOf(close); + +type Range = readonly [number, number]; + +/** The parts of a paragraph that are not plain prose: they are never read or rewritten. */ +function protectedRanges(text: string): Range[] { + const found: Range[] = []; + // Reference definitions are whole lines. + for (const match of text.matchAll(/^ {0,3}\[[^\]\n]+\]:.*$/gmu)) + found.push([match.index, match.index + match[0].length]); + let index = 0; + while (index < text.length) { + const char = text[index]; + if (char === '\\') { + found.push([index, Math.min(index + 2, text.length)]); + index += 2; + } else if (char === '`') { + let end = index; + while (text[end] === '`') end += 1; + const length = end - index; + // A span closes at the next run of exactly the same length, even over lines. + let close = -1; + for (let at = end; at < text.length; ) { + if (text[at] !== '`') { + at += 1; + continue; + } + let stop = at; + while (text[stop] === '`') stop += 1; + if (stop - at === length) { + close = stop; + break; + } + at = stop; + } + if (close === -1) { + found.push([index, end]); + index = end; + } else { + found.push([index, close]); + index = close; + } + } else if (char === '<') { + const tag = + /^<(?:[A-Za-z][A-Za-z0-9+.-]{1,31}:[^\s<>]*|[^\s@<>]+@[^\s<>]+|\/?[A-Za-z][A-Za-z0-9-]*(?:\s[^<>]*)?\/?|!--[\s\S]*?--)>/u.exec( + text.slice(index), + ); + if (tag) { + found.push([index, index + tag[0].length]); + index += tag[0].length; + } else index += 1; + } else if (char === ']' && text[index + 1] === '(') { + // A link or image destination, with balanced parentheses. + let depth = 0; + let end = index + 1; + for (; end < text.length && text[end] !== '\n'; end += 1) { + if (text[end] === '(') depth += 1; + else if (text[end] === ')' && --depth === 0) break; + } + found.push([index + 1, Math.min(end + 1, text.length)]); + index = Math.min(end + 1, text.length); + } else { + const url = /^(?:https?:\/\/|www\.)[^\s<]*/iu.exec(text.slice(index)); + if (url) { + found.push([index, index + url[0].length]); + index += url[0].length; + } else index += 1; + } + } + return found; +} + +function wrapsAll(chars: readonly string[]): boolean { + if (chars.length <= 2) return false; + const open = chars[0]; + const close = chars.at(-1)!; + if (!wraps(open, close)) return false; + let depth = 0; + for (let index = 0; index < chars.length - 1; index += 1) { + if (chars[index] === open) depth += 1; + else if (chars[index] === close && --depth === 0) return false; + } + return depth === 1; +} + +const inside = (ranges: readonly Range[], at: number) => + ranges.some(([start, end]) => at >= start && at < end); + +type Edit = { start: number; end: number; text: string }; + +/** The rewritten `**body**` for one pair, or null when it should stay as written. */ +function repair(paragraph: string, open: number, close: number, ranges: readonly Range[]) { + const before = open > 0 ? [...paragraph.slice(Math.max(0, open - 2), open)].at(-1) : undefined; + const after = [...paragraph.slice(close + 2, close + 4)][0]; + const chars = [...paragraph.slice(open + 2, close)]; + if (chars.length === 0 || isBlank(chars[0]) || isBlank(chars.at(-1)!)) return null; + const blockedClose = isPunctuation(chars.at(-1)!) && !closesOn(after); + const blockedOpen = isPunctuation(chars[0]) && !opensAfter(before); + if (!blockedClose && !blockedOpen) return null; + + let bodyStart = open + 2; + let bodyEnd = close; + let leading = ''; + let trailing = ''; + + // Peel trailing clause/label punctuation (`PEEL`) that is plain prose. + while (chars.length > 1 && PEEL.has(chars.at(-1)!)) { + const char = chars.at(-1)!; + if (inside(ranges, bodyEnd - char.length)) break; + chars.pop(); + bodyEnd -= char.length; + trailing = char + trailing; + } + + // A balanced wrapper around the remaining bold text moves out as a pair when an edge is blocked. + const firstAfter = [...trailing][0] ?? after; + const stillBlockedClose = isPunctuation(chars.at(-1)!) && !closesOn(firstAfter); + const stillBlockedOpen = isPunctuation(chars[0]) && !opensAfter(before); + if ( + (stillBlockedOpen || stillBlockedClose) && + wrapsAll(chars) && + !inside(ranges, bodyStart) && + !inside(ranges, bodyEnd - chars.at(-1)!.length) + ) { + const openWrap = chars.shift()!; + const closeWrap = chars.pop()!; + bodyStart += openWrap.length; + bodyEnd -= closeWrap.length; + leading = openWrap; + trailing = closeWrap + trailing; + } + + if ( + chars.length === 0 || + isBlank(chars[0]) || + isBlank(chars.at(-1)!) || + chars.every((char) => isPunctuation(char) || isBlank(char)) + ) { + return null; + } + // The pair must now be able to open and to close under CommonMark flank rules. + const openBefore = leading ? [...leading].at(-1) : before; + const closeAfter = trailing ? [...trailing][0] : after; + if (isPunctuation(chars[0]) && !opensAfter(openBefore)) return null; + if (isPunctuation(chars.at(-1)!) && !closesOn(closeAfter)) return null; + return `${leading}**${chars.join('')}**${trailing}`; +} + +/** One paragraph (no blank line, no fence): pair its `**` runs on each line and repair them. */ +function fixParagraph(paragraph: string): string { + const ranges = protectedRanges(paragraph); + const edits: Edit[] = []; + let pending: number | null = null; + for (let index = 0; index < paragraph.length; ) { + if (paragraph[index] === '\n') { + pending = null; + index += 1; + continue; + } + if (paragraph[index] !== '*') { + index += 1; + continue; + } + let end = index; + while (paragraph[end] === '*') end += 1; + const length = end - index; + const at = index; + index = end; + if (inside(ranges, at)) continue; + // Only a plain `**`; longer or shorter runs are left alone and end any open pair. + if (length !== 2) { + pending = null; + continue; + } + const next = [...paragraph.slice(at + 2, at + 4)][0]; + const previous = at > 0 ? [...paragraph.slice(Math.max(0, at - 2), at)].at(-1) : undefined; + const canClose = previous !== undefined && !isBlank(previous); + const canOpen = + next !== undefined && + next !== '\n' && + !isBlank(next) && + (!isPunctuation(next) || opensAfter(previous) || OPENERS.includes(next)); + if (canClose && pending !== null) { + const text = repair(paragraph, pending, at, ranges); + if (text !== null) edits.push({ start: pending, end: at + 2, text }); + pending = null; + } else if (canOpen) pending = at; + } + let result = paragraph; + for (const edit of edits.reverse()) + result = result.slice(0, edit.start) + edit.text + result.slice(edit.end); + return result; +} + +/** Strips the container prefixes (`>`, list markers) a fence line may sit behind. */ +const FENCE_PREFIX = String.raw`(?:[ \t]*>)*[ \t]*(?:(?:[-*+]|\d+[.)])[ \t]+)?`; +const FENCE_OPEN = new RegExp(`^${FENCE_PREFIX}(\`{3,}|~{3,})(.*)$`, 'u'); +const THEMATIC_BREAK = /^([-*_])(?:[ \t]*\1){2,}[ \t]*$/u; +const LIST_ITEM = /^([-*+]|\d{1,9}[.)])(?:([ \t]+)(.*))?$/u; + +function advanceCols(cols: number, text: string): number { + let current = cols; + for (let index = 0; index < text.length; index += 1) { + if (text[index] === ' ') current += 1; + else if (text[index] === '\t') current += 4 - (current % 4); + else break; + } + return current; +} + +function lineIndent(line: string): { cols: number; offset: number } { + let cols = 0; + let offset = 0; + while (offset < line.length) { + if (line[offset] === ' ') { + cols += 1; + offset += 1; + } else if (line[offset] === '\t') { + cols += 4 - (cols % 4); + offset += 1; + } else { + break; + } + } + return { cols, offset }; +} + +/** + * `text` with bold pairs that CommonMark would leave open made to close. Fenced and + * indented code blocks are skipped; the rest is rewritten paragraph by paragraph. + */ +export function closeCjkEmphasis(text: string): string { + if (!text.includes('**')) return text; + const lines = text.split('\n'); + const out: string[] = []; + let fence: string | null = null; + let indentedCodeCols: number | null = null; + const listStack: number[] = []; + let paragraph: string[] = []; + const flush = () => { + if (paragraph.length) out.push(...fixParagraph(paragraph.join('\n')).split('\n')); + paragraph = []; + }; + for (const line of lines) { + if (fence !== null) { + const closing = new RegExp( + `^${FENCE_PREFIX}(\`{${fence.length},}|~{${fence.length},})[ \\t]*$`, + 'u', + ).exec(line); + if (closing && closing[1][0] === fence[0]) fence = null; + out.push(line); + continue; + } + if (indentedCodeCols !== null) { + if (line.trim() === '' || lineIndent(line).cols >= indentedCodeCols) { + out.push(line); + continue; + } + indentedCodeCols = null; + } + if (line.trim() === '') { + flush(); + out.push(line); + continue; + } + const { cols, offset } = lineIndent(line); + while (listStack.length > 0 && listStack.at(-1)! > cols) listStack.pop(); + const baseIndent = listStack.at(-1) ?? 0; + if (paragraph.length === 0 && cols >= baseIndent + 4) { + indentedCodeCols = baseIndent + 4; + out.push(line); + continue; + } + const opening = FENCE_OPEN.exec(line); + // A backtick fence's info string cannot contain a backtick: that is an inline span. + if (opening && !(opening[1][0] === '`' && opening[2].includes('`'))) { + flush(); + fence = opening[1]; + out.push(line); + continue; + } + const rest = line.slice(offset); + if (!THEMATIC_BREAK.test(rest)) { + const item = LIST_ITEM.exec(rest); + if (item && (item[2] !== undefined || rest.length === item[1].length)) { + const markerEnd = cols + item[1].length; + const contentCols = item[2] ? advanceCols(markerEnd, item[2]) : markerEnd + 1; + const indent = !item[3] || contentCols - markerEnd >= 5 ? markerEnd + 1 : contentCols; + listStack.push(indent); + } + } + paragraph.push(line); + } + flush(); + return out.join('\n'); +} diff --git a/products/maka/workbench/tests/markdown-emphasis.test.mjs b/products/maka/workbench/tests/markdown-emphasis.test.mjs new file mode 100644 index 0000000000..21b4f7ca9f --- /dev/null +++ b/products/maka/workbench/tests/markdown-emphasis.test.mjs @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { closeCjkEmphasis } from '../app/markdownEmphasis.ts'; + +test('punctuation inside the end marker moves out when a character follows', () => { + assert.equal( + closeCjkEmphasis('- **回升能否守住:**如果很快跌回'), + '- **回升能否守住**:如果很快跌回', + ); + assert.equal(closeCjkEmphasis('**Note:**Then text'), '**Note**:Then text'); + assert.equal(closeCjkEmphasis('**重点,**说明'), '**重点**,说明'); +}); + +test('punctuation inside the start marker moves out when a character precedes it', () => { + assert.equal(closeCjkEmphasis('他说**“不行”**然后'), '他说“**不行**”然后'); +}); + +test('text that already works is left exactly as written', () => { + for (const text of [ + '**回升能否守住**:如果很快跌回', + '**回升能否守住:** 如果很快跌回', + '**重点**说明', + '**Note:** Then text', + '**done.**', + '段落末尾 **加粗:**', + '**a**b**c**', + 'no bold at all', + ]) + assert.equal(closeCjkEmphasis(text), text, text); +}); + +test('code, links, URLs, HTML and escapes stay byte-identical while the prose next to them is fixed', () => { + const cases = [ + ['`**回升:**如果` 与 **回升:**如果', '`**回升:**如果` 与 **回升**:如果'], + ['`代码` **丙:**丁', '`代码` **丙**:丁'], + ['**甲:**乙 丙', '**甲**:乙 丙'], + ['**甲:**乙 \\* 丙', '**甲**:乙 \\* 丙'], + ['![图](a.png) **甲:**乙', '![图](a.png) **甲**:乙'], + [ + '[查看](https://example.test/**path:**next) **甲:**乙', + '[查看](https://example.test/**path:**next) **甲**:乙', + ], + [ + '见 https://example.test/**path:**next 这里 **甲:**乙', + '见 https://example.test/**path:**next 这里 **甲**:乙', + ], + ]; + for (const [input, expected] of cases) assert.equal(closeCjkEmphasis(input), expected, input); + // Nothing to fix: the protected construct is all there is. + for (const text of [ + '[查看](https://example.test/**path:**next)', + '', + '[ref]: https://example.test/**x:**y', + '`line one\n**原文:**继续\nline three`', + '``a`**原文:**继续``', + ]) + assert.equal(closeCjkEmphasis(text), text, text); +}); + +test('code blocks stay as written, including multiline indented and nested fences', () => { + for (const text of [ + '```md\n**回升能否守住:**如果\n```', + '~~~\n**甲:**乙\n~~~', + ' **原文:**继续', + '示例代码:\n\n const a = 1;\n const b = "**原文:**继续";', + ' const a = 1;\n\n const b = 2;\n const c = "**原文:**继续";', + // Six spaces after a blank line under `- ` is an indented code block inside the list item. + '- 列表项\n\n const a = 1;\n const b = "**原文:**继续";', + // Thematic break followed by four-space indented code block. + '- - -\n\n const a = 1;\n const b = "**原文:**继续";', + // An inner fence with an info string does not close the outer one. + '````markdown\n```python\n**重点:**这是代码示例\n```\n````', + // A fence line with text after it does not close the block. + '```md\nalpha\n``` not a closing fence\n**原文:**继续\n```', + // A closing fence needs a run at least as long as the opening one. + '````\n```\n**甲:**乙\n````', + ]) + assert.equal(closeCjkEmphasis(text), text, text); + assert.equal( + closeCjkEmphasis('```md\n**甲:**乙\n```\n**丙:**丁'), + '```md\n**甲:**乙\n```\n**丙**:丁', + ); + assert.equal( + closeCjkEmphasis(' const a = 1;\n const b = "**甲:**乙";\n\n**丙:**丁'), + ' const a = 1;\n const b = "**甲:**乙";\n\n**丙**:丁', + ); + // Triple backticks that are an inline span, not a fence. + assert.equal( + closeCjkEmphasis('```foo``` 是内联代码\n**甲:**乙'), + '```foo``` 是内联代码\n**甲**:乙', + ); +}); + +test('lists, quotes and tables with code, links or loose continuations are fixed item by item', () => { + const cases = [ + [ + '- **回升能否守住:**如果很快跌回 `3200` 下方则偏弱\n- **量能配合:**成交额需持续放大', + '- **回升能否守住**:如果很快跌回 `3200` 下方则偏弱\n- **量能配合**:成交额需持续放大', + ], + [ + '- **回升能否守住:**如果很快跌回\n- **参考来源:**详见 [日报](https://example.com)', + '- **回升能否守住**:如果很快跌回\n- **参考来源**:详见 [日报](https://example.com)', + ], + [ + '- **主线方向:**关注半导体\n - **细分环节:**设备与材料优先', + '- **主线方向**:关注半导体\n - **细分环节**:设备与材料优先', + ], + [ + '- 项目一\n\n 这是列表项内的缩进续行 **重点:**不是代码块\n 第二行 **结论:**也不是代码块', + '- 项目一\n\n 这是列表项内的缩进续行 **重点**:不是代码块\n 第二行 **结论**:也不是代码块', + ], + ['> **结论:**短线观望为主', '> **结论**:短线观望为主'], + ['- [x] **第一步:**已完成配置', '- [x] **第一步**:已完成配置'], + ['**回撤控制:**若单日跌幅 >5% 则减仓', '**回撤控制**:若单日跌幅 >5% 则减仓'], + ]; + for (const [input, expected] of cases) assert.equal(closeCjkEmphasis(input), expected, input); +}); + +test('labels ending in brackets, quotes, units, code, links or symbols before punctuation close', () => { + const cases = [ + ['- **回升(短期):**如果很快跌回 3200 下方', '- **回升(短期)**:如果很快跌回 3200 下方'], + ['- **营收增速 (YoY):**同比增长 12%', '- **营收增速 (YoY)**:同比增长 12%'], + ['1. **方案 A(推荐):**直接升级依赖', '1. **方案 A(推荐)**:直接升级依赖'], + ['- **涨幅 +12%:**创阶段新高', '- **涨幅 +12%**:创阶段新高'], + ['- **所谓的“核心资产”:**需要结合估值看', '- **所谓的“核心资产”**:需要结合估值看'], + ['- **检查 `config.toml`:**确认端口配置正确', '- **检查 `config.toml`**:确认端口配置正确'], + [ + '- **详见 [官方文档](https://example.com):**按步骤操作', + '- **详见 [官方文档](https://example.com)**:按步骤操作', + ], + ['- **风险提示🔥:**仓位不宜过重', '- **风险提示🔥**:仓位不宜过重'], + ['请参考**《操作指南》:**按步骤执行', '请参考《**操作指南**》:按步骤执行'], + ['这是**“核心指标”:**需要重点关注', '这是“**核心指标**”:需要重点关注'], + ]; + for (const [input, expected] of cases) assert.equal(closeCjkEmphasis(input), expected, input); +}); + +test('wrappers move out as a pair and nothing is half bold', () => { + assert.equal( + closeCjkEmphasis('他说**“不行”**。详见**《操作指南》**,当前状态**【正常】**'), + '他说“**不行**”。详见《**操作指南**》,当前状态【**正常**】', + ); + assert.equal( + closeCjkEmphasis('这是 **“核心指标”**需要重点关注'), + '这是 “**核心指标**”需要重点关注', + ); + assert.equal(closeCjkEmphasis('他说**“没问题,”**随后离开'), '他说“**没问题,**”随后离开'); + // Two separate quoted spans do not form a single outer wrapper: unchanged rather than split. + assert.equal(closeCjkEmphasis('他说**“甲”和“乙”**都是重点'), '他说**“甲”和“乙”**都是重点'); + // A bracket that closes inside the bold text cannot be separated from its opener: unchanged. + assert.equal(closeCjkEmphasis('**营收(同比 +12%)**增长明显'), '**营收(同比 +12%)**增长明显'); + // A unit is not clause punctuation and is never peeled out. + assert.equal(closeCjkEmphasis('**增长12%**明显'), '**增长12%**明显'); +}); + +test('a literal ** is not paired with a real opener', () => { + assert.equal( + closeCjkEmphasis('计算 2 ** 3 之后,**结论:**成立'), + '计算 2 ** 3 之后,**结论**:成立', + ); + assert.equal(closeCjkEmphasis('指针类型 T**,**结论:**成立'), '指针类型 T**,**结论**:成立'); +}); + +test('bold made only of punctuation or spaces is not rewritten', () => { + assert.equal(closeCjkEmphasis('字**::**字'), '字**::**字'); + assert.equal(closeCjkEmphasis('a** **b'), 'a** **b'); +}); + +test('every pair on every line is handled, and the result is stable', () => { + const input = '**甲:**乙\n- **丙,**丁\n他说**“戊”**己'; + const once = closeCjkEmphasis(input); + assert.equal(once, '**甲**:乙\n- **丙**,丁\n他说“**戊**”己'); + assert.equal(closeCjkEmphasis(once), once); +});