diff --git a/rust/DESIGN.md b/rust/DESIGN.md index b7e2e3b2904..16fbe6c4c95 100644 --- a/rust/DESIGN.md +++ b/rust/DESIGN.md @@ -184,7 +184,7 @@ Each milestone ends with green CI and a report to the Space. | M2 | `rich-text` (headings, quotes, and the command handlers a text surface dispatches: typing, Enter, Backspace, formatting) and `history` (undo), which the playgrounds need first; then the node packages `list`, `link`, `code`, `overflow`, `hashtag`, with `selection` and `utils` | `serialization/*`, `property/documents` (JSON and text), `entities/hashtag-*`, `scripts/history` | done (every vector it names passes): `history` is done (`scripts/history` passes); `rich-text`'s nodes and `registerRichText`'s model command handlers are done (`scripts/rich-text` and `scripts/rich-text-commands` pass, with the heading and quote cases of `property/documents`); `plain-text`'s model command handlers are done (`scripts/plain-text` passes); `list`'s nodes, `registerList` and the model part of `registerCheckList` are done (`scripts/list` and `serialization/list` pass, with the list cases of `property/documents`); `link`'s nodes, `$toggleLink` and `registerLink`'s model handlers are done (`scripts/link` and `serialization/link` pass, with the link cases of `property/documents` that need no other package), `registerAutoLink` and its matchers are done behind `autolink` (`scripts/autolink` and `entities/autolink-match` pass); `selection`'s `$setBlocksType` is done (`scripts/set-blocks-type` passes); `code`'s nodes, `FlatStructureUtils.ts`, `CodeExtension`'s transform and `registerCodeIndentation`'s model handlers are done (`scripts/code` and `serialization/code` pass, with the code cases of `property/documents` that need no other package); `overflow`'s `OverflowNode` is done (`scripts/overflow` and `serialization/overflow` pass, with the overflow cases of `property/documents` that need no other package); `hashtag`'s `HashtagNode`, its matcher (generated character tables, no `regex`) and `registerLexicalHashtag`, with `text`'s `registerLexicalTextEntity`, are done (`entities/hashtag-match`, `entities/hashtag-documents` and `entities/hashtag-typing` pass, with the last waiting cases of `property/documents`), so M2's node packages are done | | M3 | The commands `property/sequences` dispatches | `property/sequences` | | | M4 | `html`: a DOM abstraction over an HTML parser, CSS declarations, export and import | `html/*`, `css/*`, `property/documents` (HTML) | | -| M5 | `markdown` | `markdown/import-export`, `property/documents` (Markdown) | | +| M5 | `markdown` | `markdown/import-export`, `property/documents` (Markdown) | import, export and the transformers are done (`markdown/import-export` and the Markdown of `property/documents` pass); `registerMarkdownShortcuts` is next | | M6 | `yjs`: a port of `@lexical/yjs` on `yrs` (behind the feature), with new JS vectors for the binding (sevki/lexical-rs's `lexical-yjs` and jpcamara/yrby's `lexical-yjs-html` only as cross-checks) | the new collab vectors | | | M7 | Releases: the mirror, changesets, a name tmm settles | | | diff --git a/rust/PARITY.md b/rust/PARITY.md index f47f1bbef7d..79d3f83572f 100644 --- a/rust/PARITY.md +++ b/rust/PARITY.md @@ -46,6 +46,7 @@ updated with each milestone. | `@lexical/extension`'s decorators (the `extension` feature) | `packages/lexical-extension/src/HorizontalRuleExtension.ts` (`HorizontalRuleNode`: a block decorator whose text is `\n`; `$createHorizontalRuleNode`, `$isHorizontalRuleNode`), `DecoratorTextExtension.ts` (`DecoratorTextNode`: inline, `__isInlineFormattable`, `getFormat`, `hasFormat`, `setFormat`, `toggleFormat`, its `format` as a flat NodeState; `$isDecoratorTextNode`), with `LexicalNode`'s generated JSON | `extension` | `scripts/decorator` | | `registerLexicalTextEntity` (the `text` feature) | `packages/lexical-text/src/registerLexicalTextEntity.ts`: `EntityMatch`, `registerLexicalTextEntity` (the `TextNode` transform and `$reverseNodeTransform`, `$replaceWithSimpleText`, `$wasTurnedIntoText`, `getFirstMatchAfter`, `isNextTextEntity`, `isTextAfterOutsideSource`, `isEntityAfter`, `$checkNextSibling`, `$moveSelectionIntoEntity`, `$wouldMerge`, `MAX_MATCHES_PASSED`, `MAX_TURNED_INTO_TEXT`, the per-update `markedInUpdate` and `turnedIntoTextInUpdate`); from the core `INTERNAL_getActiveEditorState` | `text`, `updates` | `entities/hashtag-documents`, `entities/hashtag-typing` | | `$setBlocksType` (the `selection` feature) | `packages/lexical-selection/src/range-selection.ts` (`$setBlocksType`, with its default `$afterCreateElement`, `$copyBlockFormatIndent`, and `$isPointAtBlockEdge`), through `LexicalNode.replace` and `ListItemNode.replace` | `selection` | `scripts/set-blocks-type`, the block-decorator case of `scripts/decorator` | +| Markdown (the `markdown` feature) | `packages/lexical-markdown/src`: `MarkdownTransformers.ts` (the four transformer types; `HEADING`, `QUOTE`, `CODE`, `UNORDERED_LIST`, `ORDERED_LIST`, `CHECK_LIST`, the text formats, `LINK`, `TRANSFORMERS` and the groups by type; the list marker, code fence, code meta and hard line break NodeStates; `normalizeMarkdown`, `isTableRowDivider`), `MarkdownImport.ts` (`createMarkdownImport`, `$importMultiline`, `$importBlocks`), `importTextTransformers.ts`, `importTextFormatTransformer.ts`, `importTextMatchTransformer.ts`, `MarkdownExport.ts` (`createMarkdownExport`, with selection), `utils.ts`; `$convertFromMarkdownString`, `$convertToMarkdownString` and `$convertSelectionToMarkdownString` (`index.ts`) | `markdown` | `markdown/import-export`, `property/documents` | | UTF-16 offsets | `String#length`, `String#slice` | `utf16` | through the text methods; unit tests against Node | | Upstream version | `meta.json` | `UPSTREAM_VERSION` | `meta` | @@ -238,6 +239,20 @@ touches: `false` where JS passes `editor.isComposing()`, as a headless JS editor does; the `composition-start` and `composition-end` tags work as in JS. +- Of `@lexical/markdown` (the `markdown` feature): `registerMarkdownShortcuts`, + which comes next; `$generateNodesFromMarkdownString`, which imports into + an `ArtificialNode__DO_NOT_USE` the core doesn't have; and the mdast + converter, which Swift doesn't port either. `markdown/import-export`'s + export cases read the state JS built from each case's HTML from its + `exportState` JSON, as `$generateNodesFromDOM` comes with `html`, and its + `importHtml` (what `$generateHtmlFromNodes` renders of the import) is + checked once `html` is ported; the imported state, the absence of a + selection after importing and the exported markdown are checked in every + case. The transformers are `Rc`s of structs with public fields, so a + variant of a built-in is a copy with fields replaced, where JS spreads + the object; `replace` returns `false` for JS's `false` and `true` for + its `true` and `undefined`, and a text match transformer's `export` is + `None` for an unset one. How `getHashtagMatch`'s regular expression runs without one (the `hashtag` feature): JS builds it with `i` and without `u`, so it matches UTF-16 code units, and the matcher works on the text's code units too. @@ -285,6 +300,28 @@ How `registerAutoLink`'s regular expressions run on the `regex` crate (the exception below), or on the string `undefined` past either end of a text, as JS does. +How `@lexical/markdown`'s regular expressions run (the `markdown` +feature): its transformers are JS `RegExp`s whose sources and flags are +public API (custom transformers bring their own, and import rebuilds them +with `new RegExp(regExp.source, flags)`), and the `regex` crate has no +lookaround or backreferences, so `markdown::RegExp` is the module's own +backtracking engine. It follows ECMA-262's pattern semantics on UTF-16 +code units: offsets are code units, a pattern without `u` reads a +character outside the BMP as two, and one with `u` as one code point. It +has groups (capturing, non-capturing and named), lookahead and lookbehind, +backreferences, greedy and lazy quantifiers with the spec's capture reset +and empty-iteration check, classes, `\p{..}` and `\P{..}` for the general +categories `rust/scripts/generate-reg-exp-tables.mjs` reads from Node +(Unicode 17), and the flags `d g i m s u y`; not the `v` flag, nor Annex +B's legacy octal escapes and `\c` in classes, which fail to compile. Two +approximations, which no upstream pattern reaches: with both `u` and `i` +characters fold with the `Canonicalize` of `i` without `u` (JS uses +Unicode simple case folding there), and a lookbehind is matched forwards +from each earlier position, so it matches when JS's does but may capture a +different substring inside it. `split` leaves out a separator's groups +that did not participate, which JS splices in as `undefined`. Swift's port +runs the same patterns on `NSRegularExpression` (`JSRegExp.swift`). + ## Documented exceptions Cases that match JS only under one of these are counted apart in the diff --git a/rust/conformance/Cargo.toml b/rust/conformance/Cargo.toml index c2ed107710f..63c0c12d659 100644 --- a/rust/conformance/Cargo.toml +++ b/rust/conformance/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true repository.workspace = true [dependencies] -lexical = { workspace = true, features = ["autolink", "code", "extension", "hashtag", "history", "link", "list", "overflow", "plain-text", "rich-text"] } +lexical = { workspace = true, features = ["autolink", "code", "extension", "hashtag", "history", "link", "list", "markdown", "overflow", "plain-text", "rich-text"] } [lints] workspace = true diff --git a/rust/conformance/src/suites/markdown.rs b/rust/conformance/src/suites/markdown.rs new file mode 100644 index 00000000000..0bc1de3e050 --- /dev/null +++ b/rust/conformance/src/suites/markdown.rs @@ -0,0 +1,343 @@ +use crate::case::{Case, Outcome, same}; +use lexical::json::Value; +use lexical::link::{LinkAttributes, create_link_node}; +use lexical::markdown::{ + MultilineElementTransformer, MultilineImportArgs, MultilineImportResult, RegExp, RegExpMatch, + TextMatchTransformer, Transformer, check_list, code, convert_from_markdown_string, + convert_to_markdown_string, link, transformers, +}; +use lexical::{ + CreateEditorArgs, Editor, LexicalError, NodeRef, UpdateOptions, create_editor, + create_text_node, get_selection, utf16, +}; +use std::rc::Rc; + +/// `markdown/import-export.json`, upstream LexicalMarkdown.test.ts's +/// `IMPORT_AND_EXPORT` table. Each `md` imports, in a discrete update, to +/// the editor state JS made from it, without selecting ("should not select +/// when importing"). Each `exportState`, the state JS built from the +/// table's `html`, exports to the markdown JS wrote from it. +/// +/// JS built that state with `$generateNodesFromDOM`, and checks the +/// `importHtml` it rendered with `$generateHtmlFromNodes`; both wait for +/// `@lexical/html` (rust/PARITY.md), so this reads the state from its JSON +/// and leaves `importHtml` unchecked. +pub fn check(case: &Value, _file: &Value) -> Result { + let custom = custom_transformers(case)?; + let flag = |key: &str| case.get(key).and_then(Value::as_bool).unwrap_or(false); + let should_preserve_new_lines = flag("shouldPreserveNewLines"); + let should_merge_adjacent_lines = flag("shouldMergeAdjacentLines"); + + if let Some(import_state) = case.get("importState").and_then(Value::as_str) { + let md = case.str("md")?.to_owned(); + let editor = create_test_editor(); + let mut all = custom.clone(); + all.extend(transformers()); + all.push(Transformer::TextMatch(highlight_text_match_import())); + editor + .update( + move || { + convert_from_markdown_string( + &md, + &all, + None, + should_preserve_new_lines, + should_merge_adjacent_lines, + ) + }, + UpdateOptions { + discrete: true, + ..UpdateOptions::default() + }, + ) + .map_err(|e| format!("importing: {e}"))?; + same( + "imported state", + editor + .get_editor_state() + .to_json(false) + .to_string() + .as_str(), + import_state, + )?; + let selected = editor + .read(|| get_selection().is_some()) + .map_err(|e| format!("reading the selection: {e}"))?; + same("a selection after importing", selected, false)?; + } + + if let Some(export_state) = case.get("exportState").and_then(Value::as_str) { + let editor = create_test_editor(); + let state = editor + .parse_editor_state(export_state) + .map_err(|e| format!("parsing exportState: {e}"))?; + editor + .set_editor_state(&state, None) + .map_err(|e| format!("setting exportState: {e}"))?; + let mut all = custom; + all.extend(transformers()); + let md = editor + .read(|| convert_to_markdown_string(&all, None, should_preserve_new_lines)) + .and_then(|r| r) + .map_err(|e| format!("exporting: {e}"))?; + same("exported markdown", md.as_str(), case.str("exportMd")?)?; + } + Ok(Outcome::Pass) +} + +/// `createTestEditor()`: heading, list, list item, quote, code and link. +fn create_test_editor() -> Editor { + create_editor(CreateEditorArgs { + nodes: [ + lexical::rich_text::rich_text_nodes(), + lexical::list::list_nodes(), + lexical::code::code_nodes(), + lexical::link::link_nodes(), + ] + .concat(), + }) +} + +/// The custom transformers a case names, ported from upstream +/// LexicalMarkdown.test.ts (copied into packages/lexical-conformance's +/// markdown.test.ts). +fn custom_transformers(case: &Value) -> Result, String> { + let Some(names) = case.get("customTransformers") else { + return Ok(Vec::new()); + }; + names + .as_array() + .ok_or("customTransformers is not an array")? + .iter() + .map(|name| { + Ok(match name.as_str() { + Some("CHECK_LIST") => Transformer::Element(check_list()), + Some("CODE_TAG_COUNTER_EXAMPLE") => { + Transformer::MultilineElement(code_tag_counter_example()) + } + Some("HIGHLIGHT_TEXT_MATCH_IMPORT") => { + Transformer::TextMatch(highlight_text_match_import()) + } + Some("MDX_HTML_TRANSFORMER") => { + Transformer::MultilineElement(mdx_html_transformer()) + } + Some("SIMPLE_INLINE_JSX_MATCHER") => { + Transformer::TextMatch(simple_inline_jsx_matcher()) + } + _ => return Err(format!("unknown custom transformer {name}")), + }) + }) + .collect() +} + +/// `HIGHLIGHT_TEXT_MATCH_IMPORT`: `LINK` importing `$text$` as highlighted +/// text. +fn highlight_text_match_import() -> Rc { + Rc::new(TextMatchTransformer { + import_reg_exp: Some(RegExp::of(r"\$([^$]+?)\$", "")), + replace: Some(Rc::new(|text_node: &NodeRef, _m: &RegExpMatch| { + text_node.toggle_format("highlight"); + Ok(None) + })), + ..(*link()).clone() + }) +} + +const MY_TAG_END: &str = ""; + +/// `SIMPLE_INLINE_JSX_MATCHER`: `text` imports as a link +/// to `simple-jsx`, with nested tags counted to find the closing one. +fn simple_inline_jsx_matcher() -> Rc { + Rc::new(TextMatchTransformer { + dependencies: vec!["link"], + export: None, + get_end_index: Some(Rc::new(|node: &NodeRef, m: &RegExpMatch| { + let mut opened_sub_start_matches = 0; + let start = m.index + m.match_len(); + let mut end_index = start; + let line: Vec = node.get_text_content().encode_utf16().collect(); + let lt = u16::from(b'<'); + let slash = u16::from(b'/'); + for i in start..line.len() { + if line[i] == lt { + if line.get(i + 1) == Some(&slash) { + if opened_sub_start_matches == 0 { + end_index = i + utf16::len(MY_TAG_END); + break; + } + opened_sub_start_matches -= 1; + } else { + opened_sub_start_matches += 1; + } + } + } + Ok(Some(end_index)) + })), + import_reg_exp: Some(RegExp::of(r"<(MyTag)\s*>", "")), + reg_exp: RegExp::of("__ignore__", ""), + replace: Some(Rc::new(|text_node: &NodeRef, m: &RegExpMatch| { + let link_node = create_link_node("simple-jsx", &LinkAttributes::default()); + let text_start = m.match_len() + m.index; + let text_end = (m.index + utf16::len(&text_node.get_text_content())) + .saturating_sub(utf16::len(MY_TAG_END)); + let text = if text_start < text_end { + utf16::slice(&m.input, text_start, text_end) + } else { + String::new() + }; + let link_text_node = create_text_node(&text); + link_text_node.set_format(text_node.text_data().get_format()); + link_node.append(&[link_text_node])?; + text_node.replace(&link_node, false)?; + Ok(None) + })), + trigger: None, + }) +} + +/// `MDX_HTML_TRANSFORMER`: `` to its closing tag imports as a +/// `MyComponent` code block of `From HTML: ` and the lines between, and +/// exports back. +fn mdx_html_transformer() -> Rc { + Rc::new(MultilineElementTransformer { + handle_import_after_start_match: None, + dependencies: vec!["code"], + export: Some(Rc::new(|node, _export_children, _selection| { + let text = node.get_text_content(); + Ok(text.starts_with("From HTML:").then(|| { + format!( + "{}", + text.replacen("From HTML: ", "", 1) + ) + })) + })), + reg_exp_end: Some(lexical::markdown::RegExpEnd { + optional: false, + reg_exp: RegExp::of(r"<\/(\w+)\s*>", ""), + }), + reg_exp_start: RegExp::of(r"<(\w+)[^>]*>", ""), + replace: Rc::new( + |root_node, _children, start_match, _end_match, lines_in_between, _is_import| { + let Some(lines) = lines_in_between else { + return Ok(false); + }; + if start_match.get(1) == Some("MyComponent") { + let code_block_node = lexical::code::create_code_node(start_match.get(1), None); + let text_node = create_text_node(&format!("From HTML: {}", lines.join("\n"))); + code_block_node.append(&[text_node])?; + root_node.append(&[code_block_node])?; + return Ok(true); + } + Ok(false) + }, + ), + }) +} + +/// `CODE_TAG_COUNTER_EXAMPLE`: `CODE`, with nested fences counted to find +/// the closing one. +fn code_tag_counter_example() -> Rc { + let code = code(); + Rc::new(MultilineElementTransformer { + handle_import_after_start_match: Some(Rc::new(code_tag_counter_import)), + dependencies: code.dependencies.clone(), + export: code.export.clone(), + reg_exp_start: code.reg_exp_start.clone(), + reg_exp_end: None, + replace: code.replace.clone(), + }) +} + +/// `text.slice(start, -end)`. +fn slice_both(text: &str, start: usize, end: usize) -> String { + let len = utf16::len(text); + utf16::slice(text, start, len.saturating_sub(end)) +} + +/// `CODE_TAG_COUNTER_EXAMPLE.handleImportAfterStartMatch`. +fn code_tag_counter_import( + args: &MultilineImportArgs<'_>, +) -> Result { + let MultilineImportArgs { + lines, + root_node, + start_line_index, + start_match, + .. + } = *args; + let regexp_end_regex = RegExp::of(r"[ \t]*```$", ""); + let sub_start_regex = RegExp::of(r"^[ \t]*```(\w+)?", ""); + let is_end_optional = false; + let mut end_line_index = start_line_index; + let lines_length = lines.len(); + let mut opened_sub_start_matches: i64 = 0; + + while end_line_index < lines_length { + let potential_sub_start_match = sub_start_regex.match_first(&lines[end_line_index]); + let end_match = regexp_end_regex.match_first(&lines[end_line_index]); + if let Some(sub) = &potential_sub_start_match { + match &end_match { + Some(end) => { + if sub.index < end.index { + opened_sub_start_matches += 1; + } + } + None => opened_sub_start_matches += 1, + } + } + if end_match.is_some() { + opened_sub_start_matches -= 1; + } + if (end_match.is_none() || opened_sub_start_matches > 0) + && (!is_end_optional || end_line_index < lines_length - 1) + { + end_line_index += 1; + continue; + } + if let Some(end) = &end_match + && start_line_index == end_line_index + && end.index == start_match.index + { + end_line_index += 1; + continue; + } + let mut lines_in_between = Vec::new(); + match &end_match { + Some(end) if start_line_index == end_line_index => { + lines_in_between.push(slice_both( + &lines[start_line_index], + start_match.match_len(), + end.match_len(), + )); + } + _ => { + for (i, line) in lines + .iter() + .enumerate() + .take(end_line_index + 1) + .skip(start_line_index) + { + if i == start_line_index { + lines_in_between.push(utf16::slice_from(line, start_match.match_len())); + } else if let Some(end) = end_match.as_ref().filter(|_| i == end_line_index) { + lines_in_between.push(slice_both(line, 0, end.match_len())); + } else { + lines_in_between.push(line.clone()); + } + } + } + } + if (code().replace)( + root_node, + None, + start_match, + end_match.as_ref(), + Some(lines_in_between), + true, + )? { + return Ok(MultilineImportResult::Handled(true, end_line_index)); + } + break; + } + Ok(MultilineImportResult::Handled(false, start_line_index)) +} diff --git a/rust/conformance/src/suites/mod.rs b/rust/conformance/src/suites/mod.rs index 9654c3c4621..a89bf62a6ed 100644 --- a/rust/conformance/src/suites/mod.rs +++ b/rust/conformance/src/suites/mod.rs @@ -2,6 +2,7 @@ //! reported as not implemented. mod entities; +mod markdown; mod meta; mod property; mod serialization; @@ -155,9 +156,14 @@ pub fn all() -> Vec { variant: None, check: crate::scripts::check, }, + Suite { + file: "markdown/import-export", + variant: None, + check: markdown::check, + }, Suite { file: "property/documents", - variant: Some("JSON and text"), + variant: Some("JSON, text and Markdown"), check: property::check_document, }, ] diff --git a/rust/conformance/src/suites/property.rs b/rust/conformance/src/suites/property.rs index 353f2d5d4cb..b725faa2e31 100644 --- a/rust/conformance/src/suites/property.rs +++ b/rust/conformance/src/suites/property.rs @@ -1,6 +1,7 @@ use super::package_of; use crate::case::{Case, Outcome, same}; use lexical::json::{self, Value}; +use lexical::markdown::transformers; use lexical::{CreateEditorArgs, create_editor, get_root}; /// The first node type in a serialized document that needs a package not @@ -20,8 +21,9 @@ fn unported_package(node: &Value) -> Option<&'static str> { /// `property/documents.json`: a random document parses (or fails to, when /// JS's did) to the same state, written in both forms, a second parse of -/// each form gives what JS's gave, and the text content is the same. The -/// Markdown and HTML JS recorded are checked once those packages are ported. +/// each form gives what JS's gave, and the text content and the markdown +/// export are the same. The HTML JS recorded is checked once `@lexical/html` +/// is ported. pub fn check_document(case: &Value, _file: &Value) -> Result { let input = case.str("input")?; let parsed_input = json::parse(input).map_err(|e| format!("input: {e}"))?; @@ -104,5 +106,15 @@ pub fn check_document(case: &Value, _file: &Value) -> Result { .read(|| get_root().get_text_content()) .map_err(|e| format!("reading the text: {e}"))?; same("text", text.as_str(), case.str("text")?)?; + + // `$convertToMarkdownString(TRANSFORMERS)`, null when it threw. + let markdown = editor + .read(|| lexical::markdown::convert_to_markdown_string(&transformers(), None, false)) + .map_err(|e| format!("reading the markdown: {e}"))?; + match (case.field("markdown")?.as_str(), markdown) { + (Some(want), Ok(got)) => same("markdown", got.as_str(), want)?, + (None, Err(_)) => {} + (want, got) => return Err(format!("markdown: Rust {got:?}, JS {want:?}")), + } Ok(Outcome::Pass) } diff --git a/rust/lexical-editor/Cargo.toml b/rust/lexical-editor/Cargo.toml index 707a43f8018..765abbd3163 100644 --- a/rust/lexical-editor/Cargo.toml +++ b/rust/lexical-editor/Cargo.toml @@ -58,6 +58,9 @@ hashtag = ["text"] # DecoratorTextNode (an inline one). The rest of the package, the extension # API, isn't ported. extension = [] +# @lexical/markdown: import, export and the transformers, with a JS RegExp +# engine of its own (no `regex`). +markdown = ["rich-text", "list", "link", "code"] [dependencies] # Without `perf-literal`, which would add aho-corasick and memchr. diff --git a/rust/lexical-editor/src/lib.rs b/rust/lexical-editor/src/lib.rs index ebe6f9e1f4e..c78c82b6102 100644 --- a/rust/lexical-editor/src/lib.rs +++ b/rust/lexical-editor/src/lib.rs @@ -69,6 +69,8 @@ mod lexical_utils; pub mod link; #[cfg(feature = "list")] pub mod list; +#[cfg(feature = "markdown")] +pub mod markdown; mod node; mod node_state; pub mod nodes; diff --git a/rust/lexical-editor/src/markdown/import_text_format_transformer.rs b/rust/lexical-editor/src/markdown/import_text_format_transformer.rs new file mode 100644 index 00000000000..2ad6c9f04b6 --- /dev/null +++ b/rust/lexical-editor/src/markdown/import_text_format_transformer.rs @@ -0,0 +1,575 @@ +//! `importTextFormatTransformer.ts`: code spans and CommonMark emphasis +//! (delimiter runs, flanking, the rule of 3), found once per text and read +//! again for what is left of it after each match. Offsets are UTF-16 code +//! units. + +use super::markdown_import::TextFormatTransformersIndex; +use super::markdown_transformers::TextFormatTransformer; +use super::reg_exp::RegExpMatch; +use super::utils::{ + character_at, character_before, get_delimiter_run_edge, get_emphasis_sides, is_escaped, string, + unit_at, +}; +use crate::{LexicalError, NodeRef}; +use std::rc::Rc; + +const BACKTICK: u16 = b'`' as u16; + +#[derive(Clone, Debug)] +struct Delimiter { + index: usize, + char: u16, + length: usize, + can_open: bool, + can_close: bool, + active: bool, +} + +/// `FormatMatch`: a code span or emphasis, from `start_index` to +/// `end_index`. +#[derive(Clone, Debug)] +pub(crate) struct FormatMatch { + pub start_index: usize, + pub end_index: usize, + pub tag: String, + pub content: String, +} + +/// `FormatScan`: the code spans and outermost emphasis of a text, read +/// again for what is left of it after each match, at `offset`. +#[derive(Clone, Debug)] +pub(crate) struct FormatScan { + pub text: Vec, + /// The characters that start an emphasis tag. + pub delimiters: Vec, + pub code_spans: Vec, + pub emphasis: Vec, + pub offset: usize, + pub next_code_span: usize, + pub next_emphasis: usize, + /// An emphasis that starts at `offset`, found after the scan. + pub pending: Option, +} + +impl FormatScan { + pub(crate) fn has_delimiter(&self, c: Option) -> bool { + c.is_some_and(|c| self.delimiters.iter().any(|d| u32::from(*d) == c)) + } +} + +/// `scanTextFormats(text, textFormatTransformersIndex, before, after)`. +pub(crate) fn scan_text_formats( + text: &[u16], + index: &TextFormatTransformersIndex, + before: Option, + after: Option, +) -> FormatScan { + // Code spans first: emphasis delimiters inside them are not read. + let code_spans: Vec = if index.transformer("`").is_some() { + scan_code_spans(text) + .into_iter() + .map(|span| FormatMatch { + start_index: span.start_index, + end_index: span.end_index, + tag: "`".to_owned(), + content: span.content, + }) + .collect() + } else { + Vec::new() + }; + let mut delimiter_chars: Vec = Vec::new(); + for (tag, _) in &index.transformers_by_tag { + let first = tag.encode_utf16().next(); + if let Some(first) = first.filter(|f| *f != BACKTICK) + && !delimiter_chars.contains(&first) + { + delimiter_chars.push(first); + } + } + let mut delimiters = scan_delimiters(text, &delimiter_chars, &code_spans, before, after); + let emphasis = if delimiters.is_empty() { + Vec::new() + } else { + process_emphasis(text, &mut delimiters, index) + }; + FormatScan { + text: text.to_vec(), + delimiters: delimiter_chars, + code_spans, + emphasis, + offset: 0, + next_code_span: 0, + next_emphasis: 0, + pending: None, + } +} + +/// `advanceFormatScan(scan, offset, isFormatEnd)`: moves the scan on to the +/// text after `offset`, where a match ends; false when what it found does +/// not hold for that text on its own. +pub(crate) fn advance_format_scan( + scan: &mut FormatScan, + offset: usize, + is_format_end: bool, +) -> bool { + scan.offset = offset; + let pending = scan.pending.take(); + if pending.as_ref().is_some_and(|p| p.end_index > offset) { + return false; + } + while scan.next_code_span < scan.code_spans.len() + && scan.code_spans[scan.next_code_span].start_index < offset + { + let span = &scan.code_spans[scan.next_code_span]; + scan.next_code_span += 1; + if span.end_index > offset { + return false; + } + } + while scan.next_emphasis < scan.emphasis.len() + && scan.emphasis[scan.next_emphasis].start_index < offset + { + let e = &scan.emphasis[scan.next_emphasis]; + scan.next_emphasis += 1; + if e.end_index > offset { + return false; + } + } + let cut = unit_at(&scan.text, offset as isize - 1); + let next = unit_at(&scan.text, offset as isize); + cut != Some(u32::from(b'\\')) + && ((is_format_end && pending.is_none()) + || cut != next + || (cut != Some(u32::from(BACKTICK)) && !scan.has_delimiter(cut))) +} + +/// `findPairAtEdge(scan, rest, textFormatTransformersIndex)`: lets the +/// scan read the delimiter run that starts `rest` beside the edge of a +/// line; false when the rest has to be scanned again. +pub(crate) fn find_pair_at_edge( + scan: &mut FormatScan, + rest: &[u16], + index: &TextFormatTransformersIndex, +) -> bool { + let offset = scan.offset; + let mut span = scan.next_code_span; + let next = scan.emphasis.get(scan.next_emphasis); + let end = if span < scan.code_spans.len() + && next.is_none_or(|n| scan.code_spans[span].start_index < n.start_index) + { + scan.code_spans[span].start_index - offset + } else { + rest.len() + }; + let mut length = 64; + loop { + // Cut neither a code span nor a run of a delimiter or backtick. + while span < scan.code_spans.len() && scan.code_spans[span].end_index <= offset + length { + span += 1; + } + if span < scan.code_spans.len() && scan.code_spans[span].start_index < offset + length { + length = scan.code_spans[span].end_index - offset; + } + let last = unit_at(rest, length as isize - 1); + if length < end && (last == Some(u32::from(BACKTICK)) || scan.has_delimiter(last)) { + length = get_delimiter_run_edge(rest, length, 1); + } + if length >= end { + if end == rest.len() { + return false; + } + length = end; + } + let pair = scan_text_formats( + &rest[..length], + index, + None, + character_at(rest, length as isize), + ) + .emphasis + .into_iter() + .next(); + if let Some(pair) = pair.filter(|p| p.start_index == 0) { + scan.pending = Some(FormatMatch { + end_index: pair.end_index + offset, + start_index: offset, + ..pair + }); + return true; + } + if length == end { + return true; + } + length *= 2; + } +} + +/// `partlyHoldsCodeSpan(scan, startIndex, endIndex)`. +pub(crate) fn partly_holds_code_span( + scan: &FormatScan, + start_index: usize, + end_index: usize, +) -> bool { + let offset = scan.offset as isize; + for span in &scan.code_spans[scan.next_code_span.min(scan.code_spans.len())..] { + let span_start = span.start_index as isize - offset; + let span_end = span.end_index as isize - offset; + if span_start >= end_index as isize { + break; + } + if span_end > start_index as isize + && (span_start < start_index as isize || span_end > end_index as isize) + { + return true; + } + } + false +} + +/// What `findOutermostTextFormatTransformer` returns. +pub(crate) struct FoundTextFormat { + pub start_index: usize, + pub end_index: usize, + pub transformer: Rc, + pub m: RegExpMatch, + /// Whether the outermost format is a code span, whose content is never + /// read again. + pub is_code_span: bool, +} + +/// `findOutermostTextFormatTransformer(text, index, scan)`: the outermost +/// format match in `text`, what is left after the scan's offset of the +/// text it scanned. +pub(crate) fn find_outermost_text_format_transformer( + text: &[u16], + index: &TextFormatTransformersIndex, + scan: &FormatScan, +) -> Option { + let code_match = scan.code_spans.get(scan.next_code_span); + let emphasis_match = scan + .pending + .as_ref() + .or_else(|| scan.emphasis.get(scan.next_emphasis)); + let (result, is_code_span) = match (emphasis_match, code_match) { + (Some(e), Some(c)) if e.start_index <= c.start_index => (e, false), + (Some(e), None) => (e, false), + (_, Some(c)) => (c, true), + (None, None) => return None, + }; + let transformer = index.transformer(&result.tag)?; + let start_index = result.start_index - scan.offset; + let end_index = result.end_index - scan.offset; + let m = RegExpMatch::new( + vec![ + Some(string(&text[start_index..end_index])), + Some(result.tag.clone()), + Some(result.content.clone()), + ], + start_index, + &string(text), + ); + Some(FoundTextFormat { + start_index, + end_index, + transformer, + m, + is_code_span, + }) +} + +/// A code span that `scanCodeSpans` finds. +#[derive(Clone, Debug)] +pub(crate) struct CodeSpan { + pub start_index: usize, + pub end_index: usize, + pub content: String, +} + +/// `scanCodeSpans(text)`: every inline code span, left to right and +/// non-overlapping. A run opens a span and the next run of equal length +/// closes it; an escaped backtick is literal and the rest of its run may +/// open one. +pub(crate) fn scan_code_spans(text: &[u16]) -> Vec { + // Maximal backtick runs, as (index, length). + let mut runs: Vec<(usize, usize)> = Vec::new(); + let mut i = 0; + while i < text.len() { + if text[i] == BACKTICK { + let mut length = 1; + while i + length < text.len() && text[i + length] == BACKTICK { + length += 1; + } + runs.push((i, length)); + i += length; + } else { + i += 1; + } + } + // The runs of each length, and how many of them the openers passed. + let mut runs_by_length: Vec<(usize, Vec, usize)> = Vec::new(); + for (index, &(_, length)) in runs.iter().enumerate() { + match runs_by_length.iter_mut().find(|(l, _, _)| *l == length) { + Some((_, indexes, _)) => indexes.push(index), + None => runs_by_length.push((length, vec![index], 0)), + } + } + let mut find_closer = |index: usize, length: usize| -> Option { + let (_, indexes, passed) = runs_by_length.iter_mut().find(|(l, _, _)| *l == length)?; + let mut next = *passed; + while next < indexes.len() && indexes[next] <= index { + next += 1; + } + *passed = next; + indexes.get(next).copied() + }; + let mut spans = Vec::new(); + let mut open_idx = 0; + while open_idx < runs.len() { + let mut opener = runs[open_idx]; + if is_escaped(text, opener.0) { + if opener.1 == 1 { + open_idx += 1; + continue; + } + opener = (opener.0 + 1, opener.1 - 1); + } + let Some(close_idx) = find_closer(open_idx, opener.1) else { + open_idx += 1; + continue; + }; + let closer = runs[close_idx]; + let mut content = &text[opener.0 + opener.1..closer.0]; + let space = u16::from(b' '); + if content.len() >= 2 + && content[0] == space + && content[content.len() - 1] == space + && content.iter().any(|c| *c != space) + { + content = &content[1..content.len() - 1]; + } + spans.push(CodeSpan { + content: string(content), + end_index: closer.0 + closer.1, + start_index: opener.0, + }); + open_idx = close_idx + 1; + } + spans +} + +/// `scanDelimiters(text, delimiterChars, codeSpans, before, after)`. +fn scan_delimiters( + text: &[u16], + delimiter_chars: &[u16], + code_spans: &[FormatMatch], + before: Option, + after: Option, +) -> Vec { + let mut delimiters = Vec::new(); + let mut span = 0; + let mut is_in_code_span = |index: usize| -> bool { + while span < code_spans.len() && code_spans[span].end_index <= index { + span += 1; + } + span < code_spans.len() && index >= code_spans[span].start_index + }; + let mut i = 0; + while i < text.len() { + let c = text[i]; + if !delimiter_chars.contains(&c) || is_escaped(text, i) || is_in_code_span(i) { + i += 1; + continue; + } + let len = get_delimiter_run_edge(text, i + 1, 1) - i; + let sides = get_emphasis_sides( + u32::from(c), + if i == 0 { + before + } else { + character_before(text, i as isize) + }, + if i + len == text.len() { + after + } else { + character_at(text, (i + len) as isize) + }, + ); + if sides.can_open || sides.can_close { + delimiters.push(Delimiter { + active: true, + can_close: sides.can_close, + can_open: sides.can_open, + char: c, + index: i, + length: len, + }); + } + i += len; + } + delimiters +} + +/// `processEmphasis(text, delimiters, transformersIndex)`: the outermost +/// emphasis matches, in order. +fn process_emphasis( + text: &[u16], + delimiters: &mut [Delimiter], + index: &TextFormatTransformersIndex, +) -> Vec { + // Keyed by marker, whether the closer can open, and its length mod 3. + let mut openers_bottom: Vec<((u16, bool, usize), isize)> = Vec::new(); + let mut current_pos = 0; + let mut matches: Vec = Vec::new(); + while current_pos < delimiters.len() { + let closer = delimiters[current_pos].clone(); + if !closer.active || !closer.can_close || closer.length == 0 { + current_pos += 1; + continue; + } + let bottom_key = (closer.char, closer.can_open, closer.length % 3); + let bottom = openers_bottom + .iter() + .find(|(k, _)| *k == bottom_key) + .map_or(-1, |(_, b)| *b); + let mut found_opener = false; + let mut open_idx = current_pos as isize - 1; + while open_idx > bottom { + let oi = open_idx as usize; + open_idx -= 1; + let opener = delimiters[oi].clone(); + if !opener.active + || !opener.can_open + || opener.length == 0 + || opener.char != closer.char + { + continue; + } + // Rule of 3, on the remaining lengths. + if opener.can_close || closer.can_open { + let sum = opener.length + closer.length; + if sum.is_multiple_of(3) + && !opener.length.is_multiple_of(3) + && !closer.length.is_multiple_of(3) + { + continue; + } + } + let max_len = opener.length.min(closer.length); + // The longest tag of this marker that fits, the first of equal + // ones (a stable sort). + let mut matched_tag: Option<&String> = None; + for (tag, _) in &index.transformers_by_tag { + let units: Vec = tag.encode_utf16().collect(); + if units.first() == Some(&opener.char) + && units.len() <= max_len + && matched_tag.is_none_or(|m| units.len() > m.encode_utf16().count()) + { + matched_tag = Some(tag); + } + } + let Some(matched_tag) = matched_tag else { + continue; + }; + found_opener = true; + let match_len = matched_tag.encode_utf16().count(); + matches.push(FormatMatch { + content: string(&text[opener.index + opener.length..closer.index]), + end_index: closer.index + match_len, + start_index: opener.index + (opener.length - match_len), + tag: matched_tag.clone(), + }); + for d in &mut delimiters[oi + 1..current_pos] { + d.active = false; + } + let o = &mut delimiters[oi]; + o.length -= match_len; + o.active = o.length > 0; + let c = &mut delimiters[current_pos]; + c.length -= match_len; + if c.length > 0 { + c.index += match_len; + } else { + c.active = false; + current_pos += 1; + } + break; + } + if !found_opener { + let value = current_pos as isize - 1; + match openers_bottom.iter_mut().find(|(k, _)| *k == bottom_key) { + Some((_, b)) => *b = value, + None => openers_bottom.push((bottom_key, value)), + } + if !closer.can_open { + delimiters[current_pos].active = false; + } + current_pos += 1; + } + } + // Pairs nest, so the outermost ones are those no earlier one holds. + matches.sort_by(|a, b| { + a.start_index + .cmp(&b.start_index) + .then(b.end_index.cmp(&a.end_index)) + }); + let mut outermost: Vec = Vec::new(); + for m in matches { + if outermost + .last() + .is_none_or(|last| m.start_index >= last.end_index) + { + outermost.push(m); + } + } + outermost +} + +/// What `importTextFormatTransformer` returns. +pub(crate) struct ImportedFormat { + pub transformed_node: NodeRef, + pub node_before: Option, + pub node_after: Option, +} + +/// `importTextFormatTransformer(textNode, startIndex, endIndex, +/// transformer, match)`. +pub(crate) fn import_text_format_transformer( + text_node: &NodeRef, + start_index: usize, + end_index: usize, + transformer: &TextFormatTransformer, + m: &RegExpMatch, +) -> Result { + let text_content = text_node.get_text_content(); + let (transformed_node, node_before, node_after); + // Matching the whole content reuses the node. + if m.text() == text_content { + transformed_node = text_node.clone(); + node_before = None; + node_after = None; + } else if start_index == 0 { + let mut parts = text_node.split_text(&[end_index as i64])?.into_iter(); + transformed_node = parts.next().expect("a part"); + node_after = parts.next(); + node_before = None; + } else { + let mut parts = text_node + .split_text(&[start_index as i64, end_index as i64])? + .into_iter(); + node_before = parts.next(); + transformed_node = parts.next().expect("a part"); + node_after = parts.next(); + } + transformed_node.set_text_content(m.str(2)); + for format in &transformer.format { + if !transformed_node.has_format(format) { + transformed_node.toggle_format(format); + } + } + Ok(ImportedFormat { + transformed_node, + node_before, + node_after, + }) +} diff --git a/rust/lexical-editor/src/markdown/import_text_match_transformer.rs b/rust/lexical-editor/src/markdown/import_text_match_transformer.rs new file mode 100644 index 00000000000..0562a67d9e5 --- /dev/null +++ b/rust/lexical-editor/src/markdown/import_text_match_transformer.rs @@ -0,0 +1,311 @@ +//! `importTextMatchTransformer.ts`: the outermost text match (a link, or a +//! custom transformer's) in a text, and importing it. + +use super::import_text_format_transformer::{FormatMatch, scan_code_spans}; +use super::markdown_transformers::{ + LINK_IMPORT_REG_EXP, TextMatchTransformer, get_link_label_opener, +}; +use super::reg_exp::{RegExp, RegExpMatch}; +use super::utils::{index_of, is_escaped, last_index_of, units}; +use crate::{LexicalError, NodeRef}; +use std::rc::Rc; + +/// `moveMatch(match, text, distance)`: a copy of `match` for `text`, what +/// is left of the text it was found in after `distance` code units. +fn move_match(m: &RegExpMatch, text: &str, distance: usize) -> RegExpMatch { + let mut moved = m.clone(); + moved.index = m.index - distance; + moved.input = text.to_owned(); + moved.ranges = m + .ranges + .iter() + .map(|r| r.map(|(s, e)| (s - distance, e - distance))) + .collect(); + moved +} + +/// `CodeSpans`: code spans in order, as (start, end). +pub(crate) type CodeSpans<'a> = &'a [FormatMatch]; + +/// `matchLink(text, regExp, codeSpans, offset)`: the first link in `text`, +/// starting at the `[` that really opens its text, with none of its syntax +/// inside a code span. +fn match_link( + text: &str, + reg_exp: &RegExp, + code_spans: Option>, + offset: usize, +) -> Option { + let flags: String = reg_exp + .flags() + .chars() + .filter(|f| *f != 'g' && *f != 'y') + .chain(std::iter::once('g')) + .collect(); + let search = RegExp::of(reg_exp.source(), &flags); + let u = units(text); + // Code spans bind tighter than link brackets. + let own_spans: Vec<(usize, usize)>; + let (spans, span_offset): (Vec<(usize, usize)>, usize) = match code_spans { + Some(spans) => ( + spans.iter().map(|s| (s.start_index, s.end_index)).collect(), + offset, + ), + None => { + own_spans = if text.contains('`') { + scan_code_spans(&u) + .iter() + .map(|s| (s.start_index, s.end_index)) + .collect() + } else { + Vec::new() + }; + (own_spans, 0) + } + }; + let code_span_at = |index: usize| -> Option { + let at = index + span_offset; + let (mut low, mut high) = (0isize, spans.len() as isize - 1); + while low <= high { + let middle = ((low + high) >> 1) as usize; + let (start, end) = spans[middle]; + if at < start { + high = middle as isize - 1; + } else if at >= end { + low = middle as isize + 1; + } else { + return Some(middle); + } + } + None + }; + // No link starts after the last `](`. + let last_closer = last_index_of(&u, &units("](")).map_or(-1, |i| i as isize); + let open = u16::from(b'['); + while (search.last_index() as isize) < last_closer { + let m = search.exec(text)?; + if let Some(opener_span) = code_span_at(m.index) { + // No link can open inside this code span. + search.set_last_index(spans[opener_span].1 - span_offset); + continue; + } + let label_end = m.index + 1 + crate::utf16::len(m.str(1)); + if code_span_at(label_end).is_some() || code_span_at(m.index + m.match_len() - 1).is_some() + { + search.set_last_index(label_end - 1); + continue; + } + // A `[` the label holds outside a code span is the link's real + // opener: search on from there. + let opener = get_link_label_opener(m.str(1)); + if let Some(o) = opener.filter(|o| *o >= 0) { + search.set_last_index(m.index + 1 + o as usize); + continue; + } + if !is_escaped(&u, m.index) { + return Some(m); + } + // Each escaped `[` up to the next unescaped one finds what the last + // of them finds. + let mut next = m.index; + loop { + match index_of(&u, &[open], next + 1) { + Some(n) if is_escaped(&u, n) => next = n, + Some(n) => { + next = n; + break; + } + None => return None, + } + } + let last = u[..next].iter().rposition(|c| *c == open); + search.set_last_index(match last { + Some(l) if l > m.index => l, + _ => next, + }); + } + None +} + +/// `looksOnlyAhead(regExp)`: whether a match depends only on the text from +/// where it starts: no `^`, word boundary or lookbehind outside a class, +/// and no flag that keeps state. +fn looks_only_ahead(reg_exp: &RegExp) -> bool { + if reg_exp.flags().contains(['g', 'y', 'v']) { + return false; + } + let source: Vec = reg_exp.source().chars().collect(); + let mut in_class = false; + let mut i = 0; + while i < source.len() { + let c = source[i]; + if c == '\\' { + i += 1; + if !in_class && matches!(source.get(i), Some('b' | 'B')) { + return false; + } + } else if in_class { + in_class = c != ']'; + } else if c == '[' { + in_class = true; + } else { + let rest: String = source[i..source.len().min(i + 4)].iter().collect(); + let is_lookbehind_or_named = rest.starts_with("(?<"); + let is_named = + is_lookbehind_or_named && rest.chars().nth(3).is_some_and(|c| c != '=' && c != '!'); + if (c == '^' || is_lookbehind_or_named) && !is_named { + return false; + } + } + i += 1; + } + true +} + +/// `TextMatches`: the match each text match transformer found in a text and +/// the offset of the text it was found in, or `None` when it found none. +#[derive(Default)] +pub(crate) struct TextMatches(Vec<(Rc, Option)>); + +/// A match and the offset of the text it was found in. +type FoundMatch = (RegExpMatch, usize); + +impl TextMatches { + fn get(&self, t: &Rc) -> Option<&Option<(RegExpMatch, usize)>> { + self.0 + .iter() + .find(|(k, _)| Rc::ptr_eq(k, t)) + .map(|(_, v)| v) + } + + fn set(&mut self, t: &Rc, value: Option<(RegExpMatch, usize)>) { + match self.0.iter_mut().find(|(k, _)| Rc::ptr_eq(k, t)) { + Some((_, v)) => *v = value, + None => self.0.push((t.clone(), value)), + } + } +} + +/// What `findOutermostTextMatchTransformer` returns. +pub(crate) struct FoundTextMatch { + pub start_index: usize, + pub end_index: usize, + pub transformer: Rc, + pub m: RegExpMatch, +} + +/// `findOutermostTextMatchTransformer(textNode, textMatchTransformers, +/// matches, offset, codeSpans)`. +pub(crate) fn find_outermost_text_match_transformer( + text_node: &NodeRef, + text_match_transformers: &[Rc], + matches: &mut TextMatches, + offset: usize, + code_spans: Option>, +) -> Result, LexicalError> { + let mut found: Option = None; + for transformer in text_match_transformers { + let (Some(_), Some(import_reg_exp)) = (&transformer.replace, &transformer.import_reg_exp) + else { + continue; + }; + let text = text_node.get_text_content(); + // Transformers sharing the built-in link syntax need the same + // candidate rules. + let is_link = import_reg_exp.source() == LINK_IMPORT_REG_EXP; + let m = if is_link || looks_only_ahead(import_reg_exp) { + match matches.get(transformer) { + Some(Some((m, found_offset))) if found_offset + m.index >= offset => { + Some(move_match(m, &text, offset - found_offset)) + } + Some(None) => None, + _ => { + let result = if is_link { + match_link(&text, import_reg_exp, code_spans, offset) + } else { + import_reg_exp.exec(&text) + }; + matches.set(transformer, result.clone().map(|m| (m, offset))); + result + } + } + } else { + import_reg_exp.string_match(&text) + }; + let Some(m) = m else { + continue; + }; + let start_index = m.index; + let end_index = match &transformer.get_end_index { + Some(get_end_index) => match get_end_index(text_node, &m)? { + Some(end) => end, + None => continue, + }, + None => start_index + m.match_len(), + }; + let replaces = match &found { + None => true, + // Wraps the previous match or is strictly before it. + Some(f) => { + start_index < f.start_index + && (end_index > f.end_index || end_index <= f.start_index) + } + }; + if replaces { + found = Some(FoundTextMatch { + start_index, + end_index, + transformer: transformer.clone(), + m, + }); + } + } + Ok(found) +} + +/// What `importFoundTextMatchTransformer` returns. +pub(crate) struct ImportedTextMatch { + pub matched_node: NodeRef, + pub transformed_node: Option, + pub node_before: Option, + pub node_after: Option, + /// The text of `node_after` before the transformer ran. + pub text_after: Option, +} + +/// `importFoundTextMatchTransformer(textNode, startIndex, endIndex, +/// transformer, match)`. +pub(crate) fn import_found_text_match_transformer( + text_node: &NodeRef, + start_index: usize, + end_index: usize, + transformer: &TextMatchTransformer, + m: &RegExpMatch, +) -> Result, LexicalError> { + let (transformed_node, node_before, node_after); + if start_index == 0 { + let mut parts = text_node.split_text(&[end_index as i64])?.into_iter(); + transformed_node = parts.next().expect("a part"); + node_after = parts.next(); + node_before = None; + } else { + let mut parts = text_node + .split_text(&[start_index as i64, end_index as i64])? + .into_iter(); + node_before = parts.next(); + transformed_node = parts.next().expect("a part"); + node_after = parts.next(); + } + let Some(replace) = &transformer.replace else { + return Ok(None); + }; + let text_after = node_after.as_ref().map(NodeRef::get_text_content); + let potential = replace(&transformed_node, m)?; + Ok(Some(ImportedTextMatch { + matched_node: transformed_node, + node_after, + node_before, + text_after, + transformed_node: potential, + })) +} diff --git a/rust/lexical-editor/src/markdown/import_text_transformers.rs b/rust/lexical-editor/src/markdown/import_text_transformers.rs new file mode 100644 index 00000000000..84448c6bc24 --- /dev/null +++ b/rust/lexical-editor/src/markdown/import_text_transformers.rs @@ -0,0 +1,372 @@ +//! `importTextTransformers.ts`: applies the text format and text match +//! transformers to a line's text, outermost match first. + +use super::import_text_format_transformer::{ + FormatScan, advance_format_scan, find_outermost_text_format_transformer, find_pair_at_edge, + import_text_format_transformer, partly_holds_code_span, scan_text_formats, +}; +use super::import_text_match_transformer::{ + TextMatches, find_outermost_text_match_transformer, import_found_text_match_transformer, +}; +use super::markdown_import::TextFormatTransformersIndex; +use super::markdown_transformers::TextMatchTransformer; +use super::utils::{ + character_at, character_before, get_delimiter_run_edge, get_emphasis_sides, unescape_text, + unit_at, units, +}; +use crate::{LexicalError, NodeRef}; +use std::rc::Rc; + +/// `canContainTransformableMarkdown(node)`: a text node without the code +/// format. +pub(crate) fn can_contain_transformable_markdown(node: Option<&NodeRef>) -> bool { + node.is_some_and(|n| n.is_text() && !n.has_format("code")) +} + +/// `readsOtherwiseBesideEdge(text, before, outside)`. +fn reads_otherwise_beside_edge(text: &[u16], before: Option, outside: Option) -> bool { + let edge = get_delimiter_run_edge(text, 1, 1); + let after = if edge < text.len() { + character_at(text, edge as isize) + } else { + outside + }; + let marker = u32::from(text[0]); + let at_edge = get_emphasis_sides(marker, None, after); + let beside_before = get_emphasis_sides(marker, before, after); + at_edge != beside_before +} + +/// `SPEC_MARKERS`: the markers whose runs CommonMark and GFM define. +const SPEC_MARKERS: &[u16] = &[b'*' as u16, b'_' as u16, b'~' as u16]; + +/// `characterBeyondRun(text, index, step, outside, delimiters, +/// isBesideTextMatch)`. +fn character_beyond_run( + text: &[u16], + index: usize, + step: isize, + outside: Option, + scan: &FormatScan, + is_beside_text_match: bool, +) -> Option { + let beside = if step == 1 { + unit_at(text, index as isize - 1) + } else { + unit_at(text, index as isize) + }; + let edge = if beside == Some(u32::from(b'`')) || scan.has_delimiter(beside) { + get_delimiter_run_edge(text, index, step) + } else { + index + }; + if is_beside_text_match && edge == index { + return None; + } + if step == 1 { + if edge < text.len() { + character_at(text, edge as isize) + } else { + outside + } + } else if edge > 0 { + character_before(text, edge as isize) + } else { + outside + } +} + +/// `TextPart`: text to import, with the characters around it. +struct TextPart { + node: Option, + before: Option, + after: Option, +} + +/// `Deferred`. +struct Deferred { + outside: TextPart, + inside: TextPart, + matched: Option, +} + +/// `TextScan`: the formats and text matches found in a text, for what is +/// left of it, `rest`. +struct TextScan { + formats: FormatScan, + matches: TextMatches, + rest: String, +} + +/// `TextRest`. +struct TextRest { + node: NodeRef, + before: Option, + after: Option, + scan: Option, +} + +/// `importTextTransformers(textNode, textFormatTransformersIndex, +/// textMatchTransformers, before, after)`: finds the outermost text format +/// or text match and applies it, goes on with the text after it, then, +/// from the last match back to the first, imports the text before and +/// inside each. +pub(crate) fn import_text_transformers( + text_node: &NodeRef, + index: &TextFormatTransformersIndex, + text_match_transformers: &[Rc], + before: Option, + after: Option, +) -> Result<(), LexicalError> { + let mut deferred: Vec = Vec::new(); + let mut rest = Some(TextRest { + after, + before, + node: text_node.clone(), + scan: None, + }); + while let Some(r) = rest { + rest = import_outermost(r, index, text_match_transformers, &mut deferred)?; + } + for d in deferred.into_iter().rev() { + import_text_part(&d.outside, index, text_match_transformers)?; + import_text_part(&d.inside, index, text_match_transformers)?; + if let Some(matched) = d.matched.as_ref() + && !can_contain_transformable_markdown(d.inside.node.as_ref()) + && can_contain_transformable_markdown(Some(matched)) + && matched.get_parent().is_some() + { + // A text match that returned no node: read its escapes once. + matched.set_text_content(&unescape_text(&matched.get_text_content())); + } + } + Ok(()) +} + +fn import_text_part( + part: &TextPart, + index: &TextFormatTransformersIndex, + text_match_transformers: &[Rc], +) -> Result<(), LexicalError> { + if let Some(node) = part.node.as_ref() + && can_contain_transformable_markdown(Some(node)) + { + import_text_transformers( + node, + index, + text_match_transformers, + part.before, + part.after, + )?; + } + Ok(()) +} + +/// `importOutermost(rest, ...)`: applies the outermost transformer, defers +/// the import of the text before and inside its match, and returns the +/// text after it. +fn import_outermost( + rest_in: TextRest, + index: &TextFormatTransformersIndex, + text_match_transformers: &[Rc], + deferred: &mut Vec, +) -> Result, LexicalError> { + let TextRest { + node: text_node, + before, + after, + scan: previous_scan, + } = rest_in; + let text_string = text_node.get_text_content(); + let text = units(&text_string); + let mut scan = match previous_scan { + Some(s) if s.rest == text_string => s, + _ => TextScan { + formats: scan_text_formats(&text, index, before, after), + matches: TextMatches::default(), + rest: text_string.clone(), + }, + }; + let mut found_text_format = find_outermost_text_format_transformer(&text, index, &scan.formats); + let code_spans = if index.transformer("`").is_some() { + Some(scan.formats.code_spans.clone()) + } else { + None + }; + let mut found_text_match = find_outermost_text_match_transformer( + &text_node, + text_match_transformers, + &mut scan.matches, + scan.formats.offset, + code_spans.as_deref(), + )?; + + if let (Some(format), Some(text_match)) = (&found_text_format, &found_text_match) { + // Code spans bind tighter than text matches: a match goes only when + // it wraps the whole span. + let (start_index, end_index) = (text_match.start_index, text_match.end_index); + if partly_holds_code_span(&scan.formats, start_index, end_index) { + found_text_match = None; + } else if format.is_code_span { + if (start_index <= format.start_index && end_index >= format.end_index) + || end_index <= format.start_index + { + found_text_format = None; + } else { + found_text_match = None; + } + } else if (format.start_index <= start_index && format.end_index >= end_index) + || start_index > format.end_index + { + found_text_match = None; + } else { + found_text_format = None; + } + } + + // `importAround`. + let import_around = |mut scan: TextScan, + deferred: &mut Vec, + node_before: Option, + node_after: Option, + text_after: Option>, + start_index: usize, + end_index: usize, + inside: TextPart, + matched: Option| + -> Option { + let is_text_match = text_after.is_some(); + deferred.push(Deferred { + inside, + matched, + outside: TextPart { + after: character_beyond_run( + &text, + start_index, + 1, + after, + &scan.formats, + is_text_match, + ), + before, + node: node_before, + }, + }); + let node_after = node_after.filter(|n| can_contain_transformable_markdown(Some(n)))?; + let rest = node_after.get_text_content(); + if let Some(text_after) = &text_after + && text_after.as_deref() != Some(rest.as_str()) + { + return Some(TextRest { + after: None, + before: None, + node: node_after, + scan: None, + }); + } + let rest_before = + character_beyond_run(&text, end_index, -1, before, &scan.formats, is_text_match); + // The scan read a run at the start of the rest beside the end of + // the match; read beside a line's edge it can only change the pair + // it opens. + let offset = scan.formats.offset + end_index; + let mut is_reusable = advance_format_scan(&mut scan.formats, offset, !is_text_match); + let rest_units = units(&rest); + if is_reusable + && is_text_match + && rest_before.is_none() + && scan + .formats + .has_delimiter(rest_units.first().map(|u| u32::from(*u))) + && reads_otherwise_beside_edge( + &rest_units, + character_before(&text, end_index as isize), + after, + ) + { + is_reusable = find_pair_at_edge(&mut scan.formats, &rest_units, index); + } + scan.rest = rest; + Some(TextRest { + after, + before: rest_before, + node: node_after, + scan: is_reusable.then_some(scan), + }) + }; + + if let Some(format) = found_text_format { + let result = import_text_format_transformer( + &text_node, + format.start_index, + format.end_index, + &format.transformer, + &format.m, + )?; + let (start_index, end_index, is_code_span) = + (format.start_index, format.end_index, format.is_code_span); + let tag = units(format.m.str(1)); + let tag_len = tag.len(); + // The content reads a run at its edge beside the tag when it is of + // a spec marker the tag does not cut; otherwise beside the line's + // edge. A code span's content is not read for emphasis. + let beside = |c: Option| -> Option { + let c = c?; + (!is_code_span + && SPEC_MARKERS.iter().any(|m| u32::from(*m) == c) + && c != u32::from(tag[0])) + .then_some(u32::from(tag[0])) + }; + let inside = TextPart { + after: beside(unit_at(&text, end_index as isize - tag_len as isize - 1)), + before: beside(unit_at(&text, (start_index + tag_len) as isize)), + node: Some(result.transformed_node), + }; + return Ok(import_around( + scan, + deferred, + result.node_before, + result.node_after, + None, + start_index, + end_index, + inside, + None, + )); + } else if let Some(text_match) = found_text_match { + let Some(result) = import_found_text_match_transformer( + &text_node, + text_match.start_index, + text_match.end_index, + &text_match.transformer, + &text_match.m, + )? + else { + return Ok(None); + }; + let matched = if result.transformed_node.is_none() { + Some(result.matched_node.clone()) + } else { + None + }; + return Ok(import_around( + scan, + deferred, + result.node_before, + result.node_after, + Some(result.text_after), + text_match.start_index, + text_match.end_index, + TextPart { + after: None, + before: None, + node: result.transformed_node, + }, + matched, + )); + } + + // Escapes, for a node no transformer matched. + text_node.set_text_content(&unescape_text(&text_string)); + Ok(None) +} diff --git a/rust/lexical-editor/src/markdown/markdown_export.rs b/rust/lexical-editor/src/markdown/markdown_export.rs new file mode 100644 index 00000000000..dab0067b973 --- /dev/null +++ b/rust/lexical-editor/src/markdown/markdown_export.rs @@ -0,0 +1,1866 @@ +//! `MarkdownExport.ts`: `createMarkdownExport` and +//! `createSelectionMarkdownExport`, with the flanking, escaping and +//! bracket rules that make the output read back as the document. + +use super::markdown_transformers::{ + ElementExport, TextFormatTransformer, TextMatchTransformer, Transformer, + get_code_span_delimiter, hard_line_break_state, +}; +use super::utils::{ + char_units, character_at, character_before, get_portable_emphasis_sides, has_non_space, + is_empty_paragraph, is_escaped, is_letter_for_some_reader, is_line_break_node, + is_portably_flanked_by, is_space, re, string, to_character_reference, transformers_by_type, + trim_end, trim_start, units, +}; +use crate::link::{is_auto_link_node, is_link_node}; +use crate::{ + LexicalError, NodeKey, NodeRef, PointType, SelectionRef, caret_range_from_selection, get_root, + get_selection, get_state, get_text_point_caret_slice_for_node, +}; +use std::cell::RefCell; +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; + +/// `LexicalNode.isSelected(selection)`: whether the selection (the +/// current one when `None`) holds the node. +pub(crate) fn is_selected( + node: &NodeRef, + selection: Option<&SelectionRef>, +) -> Result { + let target = match selection { + Some(s) => s.clone(), + None => match get_selection() { + Some(s) => s, + None => return Ok(false), + }, + }; + let key = node.get_key(); + let is_selected = target.get_nodes()?.iter().any(|n| n.get_key() == key); + if !is_selected || node.is_text() { + return Ok(is_selected); + } + if let Some(range) = target.range() + && range.anchor.point_type == PointType::Element + && range.focus.point_type == PointType::Element + { + if range.is_collapsed() { + return Ok(false); + } + if let Some(parent) = node.get_parent() + && node.is_decorator() + && node.is_inline() + { + let first_point = if target.is_backward()? { + &range.focus + } else { + &range.anchor + }; + if parent.get_key() == first_point.key + && first_point.offset == parent.get_children_size() + && parent.get_last_child().as_ref() == Some(node) + { + return Ok(false); + } + } + } + Ok(is_selected) +} + +/// The text `$sliceSelectedTextNodeContent(selection, node, 'clone')` +/// leaves the node with. +fn slice_selected_text(node: &NodeRef, selection: &SelectionRef) -> Result { + let text = node.get_text_content(); + let Some(range) = target_points(selection) else { + return Ok(text); + }; + let key = node.get_key(); + if (range.0.key != key && range.1.key != key) + || !is_selected(node, Some(selection))? + || node.is_token() + || node.is_segmented() + { + return Ok(text); + } + let (start, end) = if selection.is_backward()? { + (range.1, range.0) + } else { + (range.0, range.1) + }; + let u = units(&text); + let (start_offset, end_offset) = if selection.is_range() + && (start.point_type == PointType::Element || end.point_type == PointType::Element) + { + let range = selection.range().expect("a range selection"); + let slices = caret_range_from_selection(&range)?.get_text_slices(); + match get_text_point_caret_slice_for_node(&slices, node) { + Some(slice) => { + let (a, b) = slice.get_slice_indices(); + (a, Some(b)) + } + None => ( + if start.key == key { start.offset } else { 0 }, + (end.key == key).then_some(end.offset), + ), + } + } else { + ( + if start.key == key { start.offset } else { 0 }, + (end.key == key).then_some(end.offset), + ) + }; + let end_offset = end_offset.unwrap_or(u.len()).min(u.len()); + let start_offset = start_offset.min(u.len()); + Ok(if end_offset > start_offset { + string(&u[start_offset..end_offset]) + } else { + String::new() + }) +} + +/// `selection.getStartEndPoints()`. +fn target_points(selection: &SelectionRef) -> Option<(crate::Point, crate::Point)> { + let range = selection.range()?; + Some((range.anchor.clone(), range.focus.clone())) +} + +/// The exported text of `node`: its selected slice, if any +/// (`$exportedText`). +fn exported_text(node: &NodeRef, selection: Option<&SelectionRef>) -> Result { + match selection { + Some(s) => slice_selected_text(node, s), + None => Ok(node.get_text_content()), + } +} + +/// What export uses of the transformers. +struct Exporter { + element_exports: Vec, + text_formats: Vec>, + text_matches: Vec>, + should_preserve_new_lines: bool, +} + +impl Exporter { + fn new(transformers: &[Transformer], should_preserve_new_lines: bool) -> Self { + let by_type = transformers_by_type(transformers); + let mut element_exports: Vec = by_type + .multiline_element + .iter() + .filter_map(|t| t.export.clone()) + .collect(); + element_exports.extend(by_type.element.iter().map(|t| t.export.clone())); + // Formats of a single format only, with code last, so `**`code`**` + // gets its bold. + let mut text_formats: Vec> = by_type + .text_format + .into_iter() + .filter(|t| t.format.len() == 1) + .collect(); + text_formats.sort_by_key(|t| t.format.contains(&"code")); + Self { + element_exports, + text_formats, + text_matches: by_type.text_match, + should_preserve_new_lines, + } + } +} + +/// `createMarkdownExport(transformers, shouldPreserveNewLines)(node)`. +pub(crate) fn export_markdown( + transformers: &[Transformer], + should_preserve_new_lines: bool, + node: Option<&NodeRef>, +) -> Result { + let exporter = Exporter::new(transformers, should_preserve_new_lines); + let is_newline_delimited = !should_preserve_new_lines; + with_bracket_context(exporter.text_matches.clone(), || { + let mut output: Vec = Vec::new(); + let children = node.cloned().unwrap_or_else(get_root).get_children(); + for (i, child) in children.iter().enumerate() { + if let Some(result) = export_top_level_elements(child, &exporter)? { + output.push( + if is_newline_delimited + && i > 0 + && !is_empty_paragraph(child) + && !is_empty_paragraph(&children[i - 1]) + { + format!("\n{result}") + } else { + result + }, + ); + } + } + Ok(output.join("\n")) + }) +} + +/// `createSelectionMarkdownExport(transformers, +/// shouldPreserveNewLines)(selection)`. +pub(crate) fn export_selection_markdown( + transformers: &[Transformer], + should_preserve_new_lines: bool, + selection: &SelectionRef, +) -> Result { + let exporter = Exporter::new(transformers, should_preserve_new_lines); + let is_newline_delimited = !should_preserve_new_lines; + with_bracket_context(exporter.text_matches.clone(), || { + // A copy that caches its nodes, which every node asks about. + let nodes = selection.get_nodes()?; + let selection = selection.clone_selection(); + selection.set_cached_nodes(Some(nodes)); + let mut output: Vec = Vec::new(); + let mut previous_exported: Option = None; + for child in get_root().get_children() { + let (should_include, markdown) = + process_node_for_selection(&child, &selection, &exporter)?; + if should_include && let Some(markdown) = markdown { + output.push( + if is_newline_delimited + && previous_exported + .as_ref() + .is_some_and(|p| !is_empty_paragraph(&child) && !is_empty_paragraph(p)) + { + format!("\n{markdown}") + } else { + markdown + }, + ); + previous_exported = Some(child); + } + } + Ok(output.join("\n")) + }) +} + +/// `$processNodeForSelection`. +fn process_node_for_selection( + node: &NodeRef, + selection: &SelectionRef, + exporter: &Exporter, +) -> Result<(bool, Option), LexicalError> { + let mut should_include = is_selected(node, Some(selection))?; + for export in &exporter.element_exports { + let export_children = |n: &NodeRef| -> Result { + let tags = RefCell::new(Vec::new()); + let mut pieces = Pieces::default(); + export_children_for_selection(n, selection, exporter, &tags, &[], &mut pieces)?; + Ok(join_pieces(&pieces)) + }; + if let Some(result) = export(node, &export_children, Some(selection))? { + if !should_include && node.is_element() { + let tags = RefCell::new(Vec::new()); + let mut pieces = Pieces::default(); + if export_children_for_selection( + node, + selection, + exporter, + &tags, + &[], + &mut pieces, + )? { + should_include = true; + } + } + return Ok((should_include, Some(result))); + } + } + if node.is_element() { + let tags = RefCell::new(Vec::new()); + let mut pieces = Pieces::default(); + let included = + export_children_for_selection(node, selection, exporter, &tags, &[], &mut pieces)?; + Ok((should_include || included, Some(join_pieces(&pieces)))) + } else if node.is_decorator() { + Ok((should_include, Some(node.get_text_content()))) + } else { + Ok((should_include, None)) + } +} + +/// An open format tag (`OpenTag`), shared by the lists that name it, with +/// `$continuesTag`'s answers so far. +struct OpenTag { + format: &'static str, + tag: String, + answers: RefCell>, +} + +type Tags = RefCell>>; + +/// `$exportChildrenForSelection`: whether any child was included. +fn export_children_for_selection( + node: &NodeRef, + selection: &SelectionRef, + exporter: &Exporter, + unclosed_tags: &Tags, + unclosable_tags: &[Rc], + output: &mut Pieces, +) -> Result { + let mut any_child_included = false; + 'main: for child in node.get_children() { + let mut child_included = is_selected(&child, Some(selection))?; + for transformer in &exporter.text_matches { + let Some(export) = &transformer.export else { + continue; + }; + let unclosable: Vec> = unclosable_tags + .iter() + .chain(unclosed_tags.borrow().iter()) + .cloned() + .collect(); + let export_children = |n: &NodeRef| -> Result { + let mut pieces = Pieces::default(); + export_children_for_selection( + n, + selection, + exporter, + unclosed_tags, + &unclosable, + &mut pieces, + )?; + Ok(join_pieces(&pieces)) + }; + let export_format = |n: &NodeRef, _text: &str| -> Result { + let mut pieces = Pieces::default(); + export_text_format( + n, + &exported_text(n, Some(selection))?, + exporter, + unclosed_tags, + unclosable_tags, + Some(selection), + &mut pieces, + )?; + Ok(join_pieces(&pieces)) + }; + if let Some(result) = export(&child, &export_children, &export_format)? { + if !child_included && child.is_element() { + let mut any = false; + for c in child.get_children() { + if is_selected(&c, Some(selection))? { + any = true; + break; + } + } + if any && child.extract_with_child(&child, Some(selection), "html")? { + child_included = true; + } + } + if child_included { + output.push(Kind::Raw, &result); + any_child_included = true; + } + continue 'main; + } + } + if is_line_break_node(&child) { + if child_included { + output.push(Kind::Raw, &export_line_break(&child)?); + any_child_included = true; + } + } else if child.is_text() { + if child_included { + export_text_format( + &child, + &exported_text(&child, Some(selection))?, + exporter, + unclosed_tags, + unclosable_tags, + Some(selection), + output, + )?; + any_child_included = true; + } + } else if child.is_element() { + let included = export_children_for_selection( + &child, + selection, + exporter, + unclosed_tags, + unclosable_tags, + output, + )?; + if !child_included + && included + && child.extract_with_child(&child, Some(selection), "html")? + { + child_included = true; + } + if child_included || included { + any_child_included = true; + } + } else if child.is_decorator() && child_included { + output.push(Kind::Text, &child.get_text_content()); + any_child_included = true; + } + } + Ok(any_child_included) +} + +/// `$exportTopLevelElements`. +fn export_top_level_elements( + node: &NodeRef, + exporter: &Exporter, +) -> Result, LexicalError> { + for export in &exporter.element_exports { + let export_children = |n: &NodeRef| export_children(n, exporter, None, &[]); + if let Some(result) = export(node, &export_children, None)? { + return Ok(Some(result)); + } + } + if node.is_element() { + export_children(node, exporter, None, &[]).map(Some) + } else if node.is_decorator() { + Ok(Some(node.get_text_content())) + } else { + Ok(None) + } +} + +/// `$exportChildren`. +fn export_children( + node: &NodeRef, + exporter: &Exporter, + unclosed_tags: Option<&Tags>, + unclosable_tags: &[Rc], +) -> Result { + let own = RefCell::new(Vec::new()); + let unclosed_tags = unclosed_tags.unwrap_or(&own); + let mut pieces = Pieces::default(); + export_child_pieces(node, exporter, unclosed_tags, unclosable_tags, &mut pieces)?; + Ok(join_pieces(&pieces)) +} + +/// `$exportChildPieces`. +fn export_child_pieces( + node: &NodeRef, + exporter: &Exporter, + unclosed_tags: &Tags, + unclosable_tags: &[Rc], + output: &mut Pieces, +) -> Result<(), LexicalError> { + 'main: for child in node.get_children() { + for transformer in &exporter.text_matches { + let Some(export) = &transformer.export else { + continue; + }; + // Tags open now can't be closed by nested text match output: + // `**text [text**](url)` is invalid markdown. + let unclosable: Vec> = unclosable_tags + .iter() + .chain(unclosed_tags.borrow().iter()) + .cloned() + .collect(); + let export_children_fn = + |n: &NodeRef| export_children(n, exporter, Some(unclosed_tags), &unclosable); + let export_format = |n: &NodeRef, text: &str| -> Result { + let mut pieces = Pieces::default(); + export_text_format( + n, + text, + exporter, + unclosed_tags, + unclosable_tags, + None, + &mut pieces, + )?; + Ok(join_pieces(&pieces)) + }; + if let Some(result) = export(&child, &export_children_fn, &export_format)? { + output.push(Kind::Raw, &result); + continue 'main; + } + } + if is_line_break_node(&child) { + output.push(Kind::Raw, &export_line_break(&child)?); + } else if child.is_text() { + export_text_format( + &child, + &child.get_text_content(), + exporter, + unclosed_tags, + unclosable_tags, + None, + output, + )?; + } else if child.is_element() { + export_child_pieces(&child, exporter, unclosed_tags, unclosable_tags, output)?; + } else if child.is_decorator() { + output.push(Kind::Text, &child.get_text_content()); + } + } + Ok(()) +} + +/// `$exportLineBreak(node)`. +fn export_line_break(node: &NodeRef) -> Result { + Ok(get_state(node, &hard_line_break_state())? + "\n") +} + +/// `$exportTextFormat`: a text node's text with its format tags, the tags +/// placed inside its whitespace (` **foo** `). +fn export_text_format( + node: &NodeRef, + text_content: &str, + exporter: &Exporter, + unclosed_tags: &Tags, + unclosable_tags: &[Rc], + selection: Option<&SelectionRef>, + pieces: &mut Pieces, +) -> Result<(), LexicalError> { + let is_code = node.has_format("code"); + let mut output = text_content.to_owned(); + if !is_code { + if exporter.should_preserve_new_lines { + // A whole run of backslashes is matched, so one before no + // marker is not tried again from each of its backslashes. + output = re(r"\\+|[*_`~]", "g").replace_with(&output, |m| { + let escaped = m.text(); + if !escaped.starts_with('\\') { + return format!("\\{escaped}"); + } + let input = units(&m.input); + let after = input.get(m.index + m.match_len()).copied(); + if after.is_some_and(|c| matches!(c, 0x2A | 0x5F | 0x60 | 0x7E)) { + format!("{escaped}{escaped}") + } else { + escaped.to_owned() + } + }); + } else { + let replaced = re(r"([*_`~\\]|<(?=\S|$)|&(?=#|$))", "g").replace_with(&output, |m| { + let c = m.text(); + if matches!(c, "*" | "_" | "`" | "~" | "\\") { + format!("\\{c}") + } else { + to_character_reference(c.chars().next().map_or(0, |c| c as u32)) + } + }); + output = escape_literal_brackets(node, &replaced, selection)?; + } + } + + let holds_tag = can_hold_tag(text_content); + let mut opening_tags = String::new(); + let mut closing_tags_before = String::new(); + let mut closing_tags_after = String::new(); + let prev_node = get_text_sibling(node, true); + let mut applied: Vec<&'static str> = Vec::new(); + for transformer in &exporter.text_formats { + let format = transformer.format[0]; + let tag = &transformer.tag; + // Inline code uses a fence of its own, below. + if format == "code" { + continue; + } + if holds_tag && node.has_format(format) && !applied.contains(&format) { + applied.push(format); + // Open the tag unless the previous node has the format and the + // tag is open. + let is_open = unclosed_tags.borrow().iter().any(|t| t.tag == *tag); + if !has_format(prev_node.as_ref(), format) || !is_open { + unclosed_tags.borrow_mut().push(Rc::new(OpenTag { + answers: RefCell::new(HashMap::new()), + format, + tag: tag.clone(), + })); + opening_tags += tag; + } + } + } + + // Close the tags in the order they were applied, where needed. + let count = unclosed_tags.borrow().len(); + for i in 0..count { + let tag_i = unclosed_tags.borrow()[i].clone(); + let node_has_format = has_format(Some(node), tag_i.format); + if node_has_format && continues_tag(node, &tag_i, selection)? { + continue; + } + let mut unhandled: Vec> = unclosed_tags.borrow().clone(); + while unhandled.len() > i { + let unclosed_tag = unhandled.pop().expect("a tag"); + // An unclosable tag stays open until it no longer is. + if unclosable_tags.iter().any(|t| t.tag == unclosed_tag.tag) { + continue; + } + if node_has_format { + closing_tags_after += &unclosed_tag.tag; + } else { + closing_tags_before += &unclosed_tag.tag; + } + unclosed_tags.borrow_mut().pop(); + } + break; + } + + if is_code && !text_content.is_empty() { + // Inline code is an atomic span with a fence from its content. + let delimiter = get_code_span_delimiter(text_content); + output = format!("{}{}{}", delimiter.fence, delimiter.padded, delimiter.fence); + } else if !holds_tag { + pieces.push(Kind::Closer, &(closing_tags_before + &closing_tags_after)); + pieces.push(Kind::Text, &output); + return Ok(()); + } + + // Tags go inside the whitespace, as CommonMark's flanking needs. + let trimmed_start = trim_start(&output); + let trimmed_output = trim_end(trimmed_start); + let leading_space = &output[..output.len() - trimmed_start.len()]; + let trailing_space = &trimmed_start[trimmed_output.len()..]; + pieces.push(Kind::Closer, &closing_tags_before); + pieces.push(Kind::Text, leading_space); + pieces.push(Kind::Opener, &opening_tags); + // Inline code is a literal span: a character reference would show. + pieces.push(if is_code { Kind::Raw } else { Kind::Text }, trimmed_output); + pieces.push(Kind::Closer, &closing_tags_after); + pieces.push(Kind::Text, trailing_space); + Ok(()) +} + +/// A piece's kind (`PieceKind`): text and decorator content, which a +/// character reference can stand in for; anything else (`RAW`); or format +/// markers. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Text, + Raw, + Opener, + Closer, +} + +fn is_marker(kind: Option) -> bool { + matches!(kind, Some(Kind::Opener | Kind::Closer)) +} + +/// `MarkdownPieces`: exported inline markdown, with the markers kept apart +/// from the text until `joinPieces` sees both sides of each run. +#[derive(Default)] +struct Pieces { + kinds: Vec, + values: Vec>, +} + +impl Pieces { + /// `pushPiece`: adjacent text (and adjacent raw output) is merged. + fn push(&mut self, kind: Kind, value: &str) { + if value.is_empty() { + return; + } + if let Some(last) = self.kinds.last() + && *last == kind + && !is_marker(Some(kind)) + { + self.values + .last_mut() + .expect("a value") + .extend(value.encode_utf16()); + } else { + self.kinds.push(kind); + self.values.push(units(value)); + } + } +} + +/// `LINK_END`: `/[\s\p{Cc}<]/u` on one code unit. +fn is_link_end(c: u32) -> bool { + is_space(c) || is_control(c) || c == u32::from(b'<') +} + +fn is_control(c: u32) -> bool { + c < 0x20 || (0x7F..=0x9F).contains(&c) +} + +/// `isLinkCharacter(character)`. +fn is_link_character(c: u16) -> bool { + !is_link_end(u32::from(c)) +} + +/// `linkRunStart(text)`: where the run of URL characters that ends `text` +/// starts. +fn link_run_start(text: &[u16]) -> usize { + let mut start = text.len(); + while start > 0 && is_link_character(text[start - 1]) { + start -= 1; + } + start +} + +/// `LINK_PREFIX`'s source. +const LINK_PREFIX: &str = r"(?:https?|ftp):\/\/(?![\s\p{Cc}!-,./:-@[-^`{-~])|www\."; + +/// `startsLink(prefix, before)`; `before` is `None` for `''`. +fn starts_link(prefix: &str, before: Option) -> bool { + let is_letter = |c: u16| (c as u8 as u16 == c) && (c as u8).is_ascii_alphabetic(); + if prefix.starts_with(['w', 'W']) { + before.is_none_or(|b| matches!(b, 0x2A | 0x5F | 0x7E | 0x28)) + } else { + !before.is_some_and(is_letter) + } +} + +/// `ADDRESS_CHARACTER`: `/[\w.+@\\-]/`. +fn is_address_character(c: u16) -> bool { + c < 0x80 && { + let b = c as u8; + b.is_ascii_alphanumeric() || matches!(b, b'_' | b'.' | b'+' | b'@' | b'\\' | b'-') + } +} + +/// `LINK_TRAIL`: what GFM readers leave out at the end of a URL. +fn is_link_trail(c: u16) -> bool { + matches!(c, 0x3F | 0x21 | 0x2E | 0x2C | 0x3A | 0x2A | 0x5F | 0x7E) +} + +/// `UNREFERENCEABLE`: control characters, surrogates and noncharacters. +fn is_unreferenceable(c: u32) -> bool { + is_control(c) + || (0xD800..=0xDFFF).contains(&c) + || (0xFDD0..=0xFDEF).contains(&c) + || (c & 0xFFFE) == 0xFFFE +} + +/// `isReferenceable(character)`. +fn is_referenceable(character: Option) -> bool { + character.is_some_and(|c| is_letter_for_some_reader(c) && !is_unreferenceable(c)) +} + +/// `ESCAPABLE`: `/^[!-/:-@[-`{-~\n\r]/`. +fn is_escapable(value: &[u16]) -> bool { + value.first().is_some_and( + |c| matches!(c, 0x21..=0x2F | 0x3A..=0x40 | 0x5B..=0x60 | 0x7B..=0x7E | 0x0A | 0x0D), + ) +} + +/// `joinPieces(pieces)`: the markdown. A word character beside a marker +/// run that could not open or close otherwise goes out as a numeric +/// reference, whose `&` or `;` is punctuation. +fn join_pieces(pieces: &Pieces) -> String { + let Pieces { kinds, values } = pieces; + let text: Vec = values.concat(); + if kinds.iter().all(|k| *k == Kind::Text) { + return string(&text); + } + let (encode_start, encode_end) = decide_references(kinds, values); + let mut is_reference = vec![false; text.len()]; + let mut base = 0; + for (index, value) in values.iter().enumerate() { + let start = if encode_start[index] { + char_units(character_at(value, 0).expect("a character")) + } else { + 0 + }; + let mut end = value.len(); + if encode_end[index] && end > start { + end -= char_units(character_before(value, end as isize).expect("a character")); + } + // A backslash left unescaped would escape the reference, marker or + // raw punctuation after it. + let next_kind = kinds.get(index + 1).copied(); + if kinds[index] == Kind::Text + && (is_marker(next_kind) + || (next_kind == Some(Kind::Raw) && is_escapable(&values[index + 1]))) + && end > start + && is_escaped(value, end) + { + end -= 1; + } + for r in &mut is_reference[base..base + start] { + *r = true; + } + for r in &mut is_reference[base + end..base + value.len()] { + *r = true; + } + base += value.len(); + } + spoil_links(kinds, values, &mut is_reference); + let mut markdown: Vec = Vec::with_capacity(text.len()); + let mut at = 0; + while at < text.len() { + if is_reference[at] { + let character = character_at(&text, at as isize).expect("a character"); + markdown.extend(to_character_reference(character).encode_utf16()); + at += char_units(character); + } else { + markdown.push(text[at]); + at += 1; + } + } + string(&markdown) +} + +/// `maskRaw(value)`: raw output as the bare-URL scan sees it. +fn mask_raw(value: &[u16]) -> Vec { + value + .iter() + .map(|&c| { + if c < 0x80 && (c as u8).is_ascii_alphabetic() { + u16::from(b'x') + } else if is_space(u32::from(c)) || matches!(c, 0x3C | 0x2A | 0x5F | 0x7E | 0x28) { + c + } else { + u16::from(b'#') + } + }) + .collect() +} + +/// `LINKISH`: `/www\.|:\/\/|@/i`. +fn is_linkish(value: &[u16]) -> bool { + re(r"www\.|:\/\/|@", "i").test(&string(value)) +} + +/// `spoilLinks(kinds, values, isReference)`: writes the `.` of `www.` or +/// the `:` after a scheme as a reference where a GFM reader would link a +/// URL that a reference lands in or that takes in the markers or raw +/// output after it. +fn spoil_links(kinds: &[Kind], values: &[Vec], is_reference: &mut [bool]) { + let has_references = is_reference.iter().any(|r| *r); + if !values.iter().enumerate().any(|(i, value)| { + kinds[i] == Kind::Text + && is_linkish(value) + && (has_references || is_linkish(&value[link_run_start(value)..])) + }) { + return; + } + let mut text: Vec = Vec::new(); + let mut is_format: Vec = Vec::new(); + for (i, value) in values.iter().enumerate() { + if kinds[i] == Kind::Raw { + text.extend(mask_raw(value)); + } else { + text.extend_from_slice(value); + } + is_format.extend(std::iter::repeat_n(kinds[i] != Kind::Text, value.len())); + } + let mut references = vec![0u32; text.len() + 1]; + let mut formats = vec![0u32; text.len() + 1]; + for i in 0..text.len() { + references[i + 1] = references[i] + u32::from(is_reference[i]); + formats[i + 1] = formats[i] + u32::from(is_format[i]); + } + let has = |sums: &[u32], from: usize, to: usize| from < to && sums[to] > sums[from]; + let address = re(r"[\w.+-]@[\w-]+\.[\w-]", ""); + // A reference in an email would cut a micromark reader's link short, + // so each `@` of one that holds a reference is spoiled. + let spoil_emails = |is_reference: &mut [bool], from: usize, to: usize| { + let mut start = from; + while start < to { + let mut end = start; + while end < to && is_address_character(text[end]) { + end += 1; + } + if has(&references, start, end) { + let plain: Vec = text[start..end] + .iter() + .copied() + .filter(|c| *c != u16::from(b'\\')) + .collect(); + if address.test(&string(&plain)) { + for i in start..end { + if text[i] == u16::from(b'@') + && !is_reference[i] + && !is_format[i] + && !is_escaped(&text, i) + { + is_reference[i] = true; + } + } + } + } + start = end.max(start + 1); + } + }; + let link_prefix = re(LINK_PREFIX, "giu"); + let mut start = 0; + while start < text.len() { + let mut end = start; + while end < text.len() && (is_link_character(text[end]) || is_reference[end]) { + end += 1; + } + if end == start { + start += 1; + continue; + } + // A URL runs to the end of the run, less the punctuation at its end. + let mut trimmed = end; + while trimmed > start && is_link_trail(text[trimmed - 1]) { + trimmed -= 1; + } + if is_escaped(&text, trimmed) && has(&formats, trimmed, end) { + trimmed = end; + } + let mut rest = start; + for prefix in link_prefix.match_all(&string(&text[start..end])) { + let at = start + prefix.index; + if at < rest { + continue; + } + let before = (at > start).then(|| text[at - 1]); + let was_link = at == start || starts_link(prefix.text(), before); + if at > start + && !starts_link( + prefix.text(), + if is_reference[at - 1] { + Some(u16::from(b';')) + } else { + before + }, + ) + { + continue; + } + spoil_emails(is_reference, rest, at); + let prefix_len = prefix.match_len(); + if was_link && !has(&references, at, end) && !has(&formats, at, trimmed) { + rest = end; + break; + } + // A reference in the prefix spoils it already; a `.` or `:` + // that is a custom format's marker is left. + let spoil = at + + prefix + .text() + .encode_utf16() + .position(|c| c == u16::from(b'.') || c == u16::from(b':')) + .unwrap_or(0); + if !has(&references, at, at + prefix_len) { + if is_format[spoil] { + rest = end; + break; + } + is_reference[spoil] = true; + } + rest = at + prefix_len; + } + spoil_emails(is_reference, rest, end); + start = end; + } +} + +/// `MarkerRun`: the pieces before and after a run of markers, whether its +/// first delimiter run opens, the marker after that run within the markers, +/// whether its last closes, and the marker before that. +#[derive(Clone, Copy)] +struct MarkerRun { + before: isize, + after: usize, + opens: bool, + after_first: Option, + closes: bool, + before_last: Option, +} + +/// `scanDelimiterRun(kinds, values, from, to, step, kind)`. +fn scan_delimiter_run( + kinds: &[Kind], + values: &[Vec], + from: usize, + to: usize, + step: isize, + kind: Kind, +) -> (bool, Option) { + let edge = &values[from]; + let delimiter = if step == 1 { + edge[0] + } else { + edge[edge.len() - 1] + }; + let mut has_kind = false; + let mut i = from as isize; + loop { + let value = &values[i as usize]; + let indexes: Box> = if step == 1 { + Box::new(0..value.len()) + } else { + Box::new((0..value.len()).rev()) + }; + for c in indexes { + if value[c] != delimiter { + return (has_kind, Some(value[c])); + } + has_kind |= kinds[i as usize] == kind; + } + if i == to as isize { + break; + } + i += step; + } + (has_kind, None) +} + +/// `decideReferences(kinds, values)`: which text pieces' first and last +/// characters go out as references for the marker run beside them to open +/// or close. +fn decide_references(kinds: &[Kind], values: &[Vec]) -> (Vec, Vec) { + let count = kinds.len(); + let mut encode_start = vec![false; count]; + let mut encode_end = vec![false; count]; + let mut runs: Vec = Vec::new(); + let mut run_after: Vec> = vec![None; count]; + let mut run_before: Vec> = vec![None; count]; + let mut start = 0; + while start < count { + if is_marker(Some(kinds[start])) { + let mut end = start; + while is_marker(kinds.get(end + 1).copied()) { + end += 1; + } + if start > 0 { + run_after[start - 1] = Some(runs.len()); + } + if end + 1 < count { + run_before[end + 1] = Some(runs.len()); + } + let (opens, after_first) = + scan_delimiter_run(kinds, values, start, end, 1, Kind::Opener); + let (closes, before_last) = + scan_delimiter_run(kinds, values, end, start, -1, Kind::Closer); + runs.push(MarkerRun { + before: start as isize - 1, + after: end + 1, + opens, + after_first, + closes, + before_last, + }); + start = end; + } + start += 1; + } + let first_char = |i: usize| character_at(&values[i], 0).expect("a character"); + let last_char = + |i: usize| character_before(&values[i], values[i].len() as isize).expect("a character"); + let is_one_character = |i: usize| char_units(first_char(i)) == values[i].len(); + let amp = u32::from(b'&'); + let semi = u32::from(b';'); + + // The state the closures read lives in cells, as they are rewritten + // while runs are decided. + let encode_start_c = RefCell::new(std::mem::take(&mut encode_start)); + let encode_end_c = RefCell::new(std::mem::take(&mut encode_end)); + let is_encoded = + |i: usize| (encode_start_c.borrow()[i] || encode_end_c.borrow()[i]) && is_one_character(i); + let start_char = |i: usize| -> Option { + if i >= count { + None + } else if encode_start_c.borrow()[i] || is_encoded(i) { + Some(amp) + } else { + Some(first_char(i)) + } + }; + let end_char = |i: isize| -> Option { + if i < 0 { + None + } else if encode_end_c.borrow()[i as usize] || is_encoded(i as usize) { + Some(semi) + } else { + Some(last_char(i as usize)) + } + }; + let can_reference = |i: isize, at_end: bool| -> bool { + i >= 0 + && (i as usize) < count + && kinds[i as usize] == Kind::Text + && is_referenceable(if at_end { + end_char(i) + } else { + start_char(i as usize) + }) + }; + // 0 while unknown, 1 when it spreads and 2 when it is blocked. + let spreads = [ + RefCell::new(vec![0u8; count]), + RefCell::new(vec![0u8; count]), + ]; + let can_spread = |index: usize, leftward: bool| -> bool { + let known = &spreads[usize::from(leftward)]; + let mut visited: Vec = Vec::new(); + let mut result = true; + let mut piece = index; + loop { + let k = known.borrow()[piece]; + if k != 0 { + result = k == 1; + break; + } + visited.push(piece); + let run_index = if leftward { + run_before[piece] + } else { + run_after[piece] + }; + let Some(run_index) = run_index else { + break; + }; + let run = runs[run_index]; + let next: isize = if leftward { + run.before + } else { + run.after as isize + }; + let touches_both = if leftward { + run.opens && run.after_first.is_none() + } else { + run.closes && run.before_last.is_none() + }; + if !touches_both + || is_portably_flanked_by( + Some(if leftward { amp } else { semi }), + if leftward { + end_char(next) + } else { + start_char(next as usize) + }, + ) + { + break; + } + if !can_reference(next, leftward) { + result = false; + break; + } + if !is_one_character(next as usize) { + break; + } + piece = next as usize; + } + let mut known = known.borrow_mut(); + for p in visited { + known[p] = if result { 1 } else { 2 }; + } + result + }; + let is_helped_by = |run: &MarkerRun, opening: bool| -> bool { + let piece: isize = if opening { + run.before + } else { + run.after as isize + }; + if !(if opening { run.opens } else { run.closes }) || !can_reference(piece, opening) { + return false; + } + let marker = if opening { + u32::from(values[(run.before + 1) as usize][0]) + } else { + let v = &values[run.after - 1]; + u32::from(v[v.len() - 1]) + }; + let previous = (if opening { + None + } else { + run.before_last.map(u32::from) + }) + .or_else(|| end_char(run.before)); + let next = (if opening { + run.after_first.map(u32::from) + } else { + None + }) + .or_else(|| start_char(run.after)); + let side = |s: super::utils::EmphasisSides| if opening { s.can_open } else { s.can_close }; + !side(get_portable_emphasis_sides(marker, previous, next)) + && side(get_portable_emphasis_sides( + marker, + if opening { Some(semi) } else { previous }, + if opening { next } else { Some(amp) }, + )) + && (!is_one_character(piece as usize) || can_spread(piece as usize, opening)) + }; + // A tag character text doesn't escape (the `=` of `==`) beside a + // marker of the same character would join its run. + for i in 0..count { + let value = &values[i]; + if kinds[i] == Kind::Text { + if is_marker(kinds.get(i + 1).copied()) + && value[value.len() - 1] == values[i + 1][0] + && !is_escaped(value, value.len() - 1) + { + encode_end_c.borrow_mut()[i] = true; + } + if i > 0 && is_marker(Some(kinds[i - 1])) { + let prev = &values[i - 1]; + if value[0] == prev[prev.len() - 1] { + encode_start_c.borrow_mut()[i] = true; + } + } + } + } + let mut pending: Vec = (0..runs.len()).collect(); + while let Some(index) = pending.pop() { + let run = runs[index]; + if is_helped_by(&run, true) { + let before = run.before as usize; + encode_end_c.borrow_mut()[before] = true; + if is_encoded(before) + && let Some(r) = run_before[before] + { + pending.push(r); + } + } + if is_helped_by(&run, false) { + encode_start_c.borrow_mut()[run.after] = true; + // An opener that sees the same character, of a run that both + // closes and opens, may now need its own reference. + if run.opens && run.after_first.is_none() { + pending.push(index); + } + if is_encoded(run.after) + && let Some(r) = run_after[run.after] + { + pending.push(r); + } + } + } + (encode_start_c.into_inner(), encode_end_c.into_inner()) +} + +// --- Brackets ------------------------------------------------------------- + +/// `BARRIER`: exported text that cannot be part of a label (a link). +const BARRIER: u32 = 0x01; +/// `UNKNOWN`: exported text this pass cannot see. +const UNKNOWN: u32 = 0x02; +/// `MAX_LABEL_DISTANCE`. +const MAX_LABEL_DISTANCE: isize = 1000; + +/// `asLiteral` of one character. +fn as_literal_char(c: u32) -> u32 { + if c == BARRIER || c == UNKNOWN { + 0xFFFD + } else { + c + } +} + +/// `asLiteral(text)`. +fn as_literal(text: &str) -> String { + text.replace(['\u{1}', '\u{2}'], "\u{FFFD}") +} + +/// `LinePhase`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum LinePhase { + Start, + Bullet, + Digits, + Text, +} + +/// `LineState`. A missing character (JS `''`) is `None`. +#[derive(Clone, Copy, Debug)] +struct LineState { + phase: LinePhase, + opener_distance: isize, + opener_char: Option, + opener_next: Option, + opener_at_line_start: bool, +} + +const BLOCK_START: LineState = LineState { + opener_at_line_start: false, + opener_char: None, + opener_distance: -1, + opener_next: None, + phase: LinePhase::Start, +}; + +fn is_digit(c: u32) -> bool { + (u32::from(b'0')..=u32::from(b'9')).contains(&c) +} + +/// `nextLinePhase(phase, character)`. +fn next_line_phase(phase: LinePhase, character: u32) -> LinePhase { + if character == UNKNOWN { + return LinePhase::Start; + } + let space = is_space(character); + match phase { + LinePhase::Start => { + if space || character == u32::from(b'>') { + LinePhase::Start + } else if matches!(character, 0x2D | 0x2A | 0x2B) { + LinePhase::Bullet + } else if is_digit(character) { + LinePhase::Digits + } else { + LinePhase::Text + } + } + LinePhase::Bullet => { + if space { + LinePhase::Start + } else { + LinePhase::Text + } + } + LinePhase::Digits => { + if is_digit(character) { + LinePhase::Digits + } else if matches!(character, 0x2E | 0x29) { + LinePhase::Bullet + } else { + LinePhase::Text + } + } + LinePhase::Text => LinePhase::Text, + } +} + +/// `advanceLineState(state, character)`. +fn advance_line_state(state: &mut LineState, character: u32) { + if state.opener_distance != -1 { + state.opener_distance += 1; + if state.opener_distance == 1 { + state.opener_next = Some(character); + } else if state.opener_distance > MAX_LABEL_DISTANCE { + state.opener_distance = -1; + } + } + if character == u32::from(b'\n') { + state.phase = LinePhase::Start; + return; + } + if character == u32::from(b'[') || character == BARRIER || character == UNKNOWN { + let keeps_line_start = character == UNKNOWN + && state.opener_distance != -1 + && state.opener_char != Some(BARRIER) + && state.opener_at_line_start; + state.opener_distance = 0; + state.opener_char = Some(character); + state.opener_next = None; + state.opener_at_line_start = state.phase == LinePhase::Start || keeps_line_start; + } + state.phase = next_line_phase(state.phase, character); +} + +/// `linkTextBracketsToEscape(text, nextChar)`. +fn link_text_brackets_to_escape(text: &[u16], next_char: Option) -> HashSet { + let mut escaped = HashSet::new(); + let mut openers: Vec = Vec::new(); + for (index, &c) in text.iter().enumerate() { + if c == u16::from(b'[') { + openers.push(index); + } else if c == u16::from(b']') { + let opener = openers.pop(); + let after = text.get(index + 1).map(|u| u32::from(*u)).or(next_char); + match opener { + None => { + escaped.insert(index); + } + Some(o) if after == Some(u32::from(b'(')) || after == Some(UNKNOWN) => { + escaped.insert(o); + escaped.insert(index); + } + Some(_) => {} + } + } + } + escaped.extend(openers); + escaped +} + +/// `$literalBracketsToEscape(before, text, nextChar, $labelClosesAfter)`. +fn literal_brackets_to_escape( + before: LineState, + text: &[u16], + next_char: Option, + label_closes_after: &dyn Fn(isize) -> Result, +) -> Result, LexicalError> { + let mut escaped = HashSet::new(); + let mut line_start_openers: Vec = Vec::new(); + let mut state = before; + for index in 0..text.len() { + let character = as_literal_char(u32::from(text[index])); + if character == u32::from(b']') { + let after = text.get(index + 1).map(|u| u32::from(*u)).or(next_char); + let distance = if state.opener_distance == -1 { + -1 + } else { + state.opener_distance + 1 + }; + let opens_line = + distance != -1 && state.opener_char != Some(BARRIER) && state.opener_at_line_start; + let is_unknown = state.opener_char == Some(UNKNOWN); + let after_is_space = after.is_some_and(is_space); + if after == Some(u32::from(b'(')) + || after == Some(UNKNOWN) + || (opens_line && after == Some(u32::from(b':'))) + || (opens_line + && after_is_space + && (is_unknown + || distance == 1 + || (distance == 2 + && state.opener_next.is_some_and(|n| { + is_space(n) || n == u32::from(b'x') || n == u32::from(b'X') + })))) + { + escaped.insert(index); + } + } else if character == u32::from(b'[') && state.phase == LinePhase::Start { + line_start_openers.push(index); + } + advance_line_state(&mut state, character); + } + for index in line_start_openers { + if label_closes_in_code(text, index, &escaped, label_closes_after)? { + // The `]` that would close it is in inline code, where it can't + // be escaped, so the `[` is. + escaped.insert(index); + } + } + Ok(escaped) +} + +/// `$labelClosesInCode(text, index, escaped, $rest)`. +fn label_closes_in_code( + text: &[u16], + index: usize, + escaped: &HashSet, + rest: &dyn Fn(isize) -> Result, +) -> Result { + for (next, &unit) in text.iter().enumerate().skip(index + 1) { + if unit == u16::from(b'[') || (unit == u16::from(b']') && !escaped.contains(&next)) { + return Ok(false); + } + } + let budget = MAX_LABEL_DISTANCE - (text.len() - index) as isize; + Ok(budget > 0 && rest(budget)?) +} + +/// `$labelClosesInCodeAfter(node, selection, budget)`. +fn label_closes_in_code_after( + node: &NodeRef, + selection: Option<&SelectionRef>, + budget: isize, +) -> Result { + let mut remaining = budget; + let mut leaf = next_leaf(node, selection)?; + while let Some(l) = leaf { + if remaining <= 0 { + break; + } + match export_kind(&l) { + ExportKind::Link => return Ok(false), + ExportKind::Unknown => return Ok(true), + _ => {} + } + if l.is_text() { + let text = units(&as_literal(&exported_text(&l, selection)?)); + if l.has_format("code") { + if let Some(bracket) = text + .iter() + .position(|c| *c == u16::from(b'[') || *c == u16::from(b']')) + { + return Ok(text[bracket] == u16::from(b']') + && text.get(bracket + 1) == Some(&u16::from(b':'))); + } + } else if text.contains(&u16::from(b'[')) { + return Ok(false); + } + remaining -= text.len() as isize; + } else if is_line_break_node(&l) { + remaining -= 1; + } + leaf = next_leaf(&l, selection)?; + } + Ok(false) +} + +/// `ExportKind`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ExportKind { + Link, + Text, + Transparent, + Unknown, +} + +/// `BracketContext`: the export in progress's text match transformers, the +/// export kind of each node asked about, and the line state after each +/// leaf read. +struct BracketContext { + text_match_transformers: Vec>, + export_kinds: HashMap, + line_states: HashMap, +} + +thread_local! { + static BRACKET_CONTEXT: RefCell> = const { RefCell::new(None) }; +} + +/// `withBracketContext(textMatchTransformers, callback)`. +fn with_bracket_context( + text_match_transformers: Vec>, + callback: impl FnOnce() -> T, +) -> T { + let previous = BRACKET_CONTEXT.with(|c| { + c.replace(Some(BracketContext { + export_kinds: HashMap::new(), + line_states: HashMap::new(), + text_match_transformers, + })) + }); + struct Restore(Option>); + impl Drop for Restore { + fn drop(&mut self) { + let previous = self.0.take().expect("a context to restore"); + BRACKET_CONTEXT.with(|c| *c.borrow_mut() = previous); + } + } + let _restore = Restore(Some(previous)); + callback() +} + +/// `$exportKind(node)`. +fn export_kind(node: &NodeRef) -> ExportKind { + if is_line_break_node(node) || (node.is_element() && !node.is_inline()) { + return ExportKind::Text; + } else if !node.is_text() && !node.is_element() { + return ExportKind::Unknown; + } + let key = node.get_key().to_owned(); + let (has_context, cached, transformers) = BRACKET_CONTEXT.with(|c| match &*c.borrow() { + Some(context) => ( + true, + context.export_kinds.get(&key).copied(), + context.text_match_transformers.clone(), + ), + None => (false, None, Vec::new()), + }); + if let Some(cached) = cached { + return cached; + } + let unknown = char::from_u32(UNKNOWN).expect("a char").to_string(); + let mut written: Option = None; + if has_context { + for transformer in &transformers { + let Some(export) = &transformer.export else { + continue; + }; + let empty_children = |_: &NodeRef| Ok(String::new()); + let empty_format = |_: &NodeRef, _: &str| Ok(String::new()); + written = export(node, &empty_children, &empty_format).unwrap_or(Some(unknown.clone())); + if written.is_some() { + break; + } + } + } else if is_link_node(node) && !is_auto_link_node(node) { + written = Some("[]()".to_owned()); + } + let kind = match written { + None if node.is_text() => ExportKind::Text, + None => ExportKind::Transparent, + Some(w) if node.is_element() && w.starts_with('[') => ExportKind::Link, + Some(_) => ExportKind::Unknown, + }; + BRACKET_CONTEXT.with(|c| { + if let Some(context) = c.borrow_mut().as_mut() { + context.export_kinds.insert(key, kind); + } + }); + kind +} + +/// `$isInsideLink(node)`. +fn is_inside_link(node: &NodeRef) -> bool { + let mut parent = node.get_parent(); + while let Some(p) = parent { + if export_kind(&p) == ExportKind::Link { + return true; + } + if !p.is_inline() { + return false; + } + parent = p.get_parent(); + } + false +} + +/// `$isInlineElement(node)`. +fn is_inline_element(node: Option<&NodeRef>) -> bool { + node.is_some_and(|n| n.is_element() && n.is_inline()) +} + +/// `$isTransparentInline(node)`. +fn is_transparent_inline(node: Option<&NodeRef>) -> bool { + node.is_some_and(|n| { + n.is_element() && n.is_inline() && export_kind(n) == ExportKind::Transparent + }) +} + +/// `$nextExportedChar(node, selection)`: the first character of the +/// exported text after `node` in its block (`None` for JS `''`). +fn next_exported_char( + node: &NodeRef, + selection: Option<&SelectionRef>, +) -> Result, LexicalError> { + let mut current = node.clone(); + loop { + let mut next = current.get_next_sibling(); + while next.is_none() { + let parent = current.get_parent(); + if let Some(p) = &parent + && export_kind(p) == ExportKind::Link + { + return Ok(Some(u32::from(b']'))); + } + if !is_transparent_inline(parent.as_ref()) { + return Ok(is_inline_element(parent.as_ref()).then_some(UNKNOWN)); + } + current = parent.expect("a parent"); + next = current.get_next_sibling(); + } + let mut next = next.expect("a sibling"); + while is_transparent_inline(Some(&next)) + && let Some(first) = next.get_first_child() + { + next = first; + } + if let Some(selection) = selection + && !is_selected(&next, Some(selection))? + { + return Ok(None); + } + let kind = export_kind(&next); + if kind == ExportKind::Link { + return Ok(Some(u32::from(b'['))); + } else if kind == ExportKind::Unknown { + return Ok(Some(UNKNOWN)); + } else if next.is_text() { + let exported = units(&exported_text(&next, selection)?); + if let Some(first) = exported.first() { + return Ok(Some(if next.has_format("code") { + u32::from(b'`') + } else { + as_literal_char(u32::from(*first)) + })); + } + } else if is_line_break_node(&next) { + return Ok(Some(u32::from(b'\n'))); + } + current = next; + } +} + +/// `$nextLeaf(node, selection)`. +fn next_leaf( + node: &NodeRef, + selection: Option<&SelectionRef>, +) -> Result, LexicalError> { + let mut current = node.clone(); + let mut next = current.get_next_sibling(); + while next.is_none() { + let parent = current.get_parent(); + if !is_transparent_inline(parent.as_ref()) { + return Ok(parent.filter(|p| is_inline_element(Some(p)))); + } + current = parent.expect("a parent"); + next = current.get_next_sibling(); + } + let mut next = next.expect("a sibling"); + while is_transparent_inline(Some(&next)) + && let Some(first) = next.get_first_child() + { + next = first; + } + if let Some(selection) = selection + && !is_selected(&next, Some(selection))? + { + return Ok(None); + } + Ok(Some(next)) +} + +/// `$previousLeaf(node, selection)`. +fn previous_leaf( + node: &NodeRef, + selection: Option<&SelectionRef>, +) -> Result, LexicalError> { + let mut current = node.clone(); + let mut previous = current.get_previous_sibling(); + while previous.is_none() { + let parent = current.get_parent(); + if !is_transparent_inline(parent.as_ref()) { + return Ok(parent.filter(|p| is_inline_element(Some(p)))); + } + current = parent.expect("a parent"); + previous = current.get_previous_sibling(); + } + let mut previous = previous.expect("a sibling"); + while is_transparent_inline(Some(&previous)) + && let Some(last) = previous.get_last_child() + { + previous = last; + } + if let Some(selection) = selection + && !is_selected(&previous, Some(selection))? + { + return Ok(None); + } + Ok(Some(previous)) +} + +/// `$leafText(leaf, selection)`, as the characters LineState reads (code +/// points, as JS's `for...of` iterates). +fn leaf_text(leaf: &NodeRef, selection: Option<&SelectionRef>) -> Result, LexicalError> { + Ok(match export_kind(leaf) { + ExportKind::Link => vec![BARRIER], + ExportKind::Unknown => vec![UNKNOWN], + _ if leaf.is_text() => { + let text = as_literal(&exported_text(leaf, selection)?); + let is_code = leaf.has_format("code"); + text.chars() + .map(|c| { + if is_code && matches!(c, '[' | ']' | ' ' | '\t') { + u32::from(b'`') + } else { + c as u32 + } + }) + .collect() + } + _ if is_line_break_node(leaf) => vec![u32::from(b'\n')], + _ => Vec::new(), + }) +} + +/// `$lineStateBefore(node, selection)`. +fn line_state_before( + node: &NodeRef, + selection: Option<&SelectionRef>, +) -> Result { + let mut pending: Vec = Vec::new(); + let mut state = BLOCK_START; + let mut leaf = previous_leaf(node, selection)?; + while let Some(l) = leaf { + let cached = BRACKET_CONTEXT.with(|c| { + c.borrow() + .as_ref() + .and_then(|context| context.line_states.get(l.get_key()).copied()) + }); + if let Some(cached) = cached { + state = cached; + break; + } + leaf = previous_leaf(&l, selection)?; + pending.push(l); + } + for l in pending.iter().rev() { + let mut next = state; + for character in leaf_text(l, selection)? { + advance_line_state(&mut next, character); + } + BRACKET_CONTEXT.with(|c| { + if let Some(context) = c.borrow_mut().as_mut() { + context.line_states.insert(l.get_key().to_owned(), next); + } + }); + state = next; + } + Ok(state) +} + +/// `$escapeLiteralBrackets(node, text, selection)`: escapes only the +/// literal brackets that would otherwise turn into markdown syntax. +fn escape_literal_brackets( + node: &NodeRef, + text: &str, + selection: Option<&SelectionRef>, +) -> Result { + if !text.contains(['[', ']']) { + return Ok(text.to_owned()); + } + let u = units(text); + let next_char = next_exported_char(node, selection)?; + let escaped = if is_inside_link(node) { + link_text_brackets_to_escape(&u, next_char) + } else { + let before = line_state_before(node, selection)?; + literal_brackets_to_escape(before, &u, next_char, &|budget| { + label_closes_in_code_after(node, selection, budget) + })? + }; + if escaped.is_empty() { + return Ok(text.to_owned()); + } + let mut output: Vec = Vec::with_capacity(u.len()); + for (index, c) in u.iter().enumerate() { + if escaped.contains(&index) { + output.extend(format!("&#{c};").encode_utf16()); + } else { + output.push(*c); + } + } + Ok(string(&output)) +} + +/// `getTextSibling(node, backward)`. +fn get_text_sibling(node: &NodeRef, backward: bool) -> Option { + let sibling = if backward { + node.get_previous_sibling() + } else { + node.get_next_sibling() + }; + sibling.filter(NodeRef::is_text) +} + +/// `$continuesTag(node, tag, selection)`: whether `tag` carries on past +/// `node` to exported text that can hold it. +fn continues_tag( + node: &NodeRef, + tag: &OpenTag, + selection: Option<&SelectionRef>, +) -> Result { + if let Some(known) = tag.answers.borrow().get(node.get_key()) { + return Ok(*known); + } + let mut passed: Vec = Vec::new(); + let mut next = get_text_sibling(node, false); + let mut result = false; + while let Some(n) = next { + if !n.has_format(tag.format) { + break; + } + if let Some(selection) = selection + && !is_selected(&n, Some(selection))? + { + break; + } + let text = exported_text(&n, selection)?; + if can_hold_tag(&text) { + result = true; + break; + } + // Markdown import matches a tag within one line. + if text.contains('\n') { + break; + } + if let Some(answer) = tag.answers.borrow().get(n.get_key()) { + result = *answer; + break; + } + next = get_text_sibling(&n, false); + passed.push(n); + } + let mut answers = tag.answers.borrow_mut(); + answers.insert(node.get_key().to_owned(), result); + for p in passed { + answers.insert(p.get_key().to_owned(), result); + } + Ok(result) +} + +/// `hasFormat(node, format)`. +fn has_format(node: Option<&NodeRef>, format: &str) -> bool { + node.is_some_and(|n| n.is_text() && n.has_format(format)) +} + +/// `canHoldTag(text)`: CommonMark keeps tags off whitespace. +fn can_hold_tag(text: &str) -> bool { + has_non_space(text) +} diff --git a/rust/lexical-editor/src/markdown/markdown_import.rs b/rust/lexical-editor/src/markdown/markdown_import.rs new file mode 100644 index 00000000000..439a0ad4182 --- /dev/null +++ b/rust/lexical-editor/src/markdown/markdown_import.rs @@ -0,0 +1,327 @@ +//! `MarkdownImport.ts`: `$importMarkdownNodes`, which parses markdown line +//! by line into a container. + +use super::import_text_transformers::import_text_transformers; +use super::markdown_transformers::{ + ElementTransformer, MultilineElementTransformer, MultilineImportArgs, MultilineImportResult, + TextFormatTransformer, TextMatchTransformer, Transformer, create_markdown_line_break_node, + with_list_indent_columns, +}; +use super::reg_exp::RegExpMatch; +use super::utils::{ + index_of, is_empty_paragraph, is_paragraph_node, is_tab_node, string, transformers_by_type, + units, +}; +use crate::list::{is_list_item_node, is_list_node}; +use crate::nodes::tab::create_tab_node; +use crate::rich_text::is_quote_node; +use crate::utils::find_matching_parent; +use crate::{LexicalError, NodeRef, create_paragraph_node, create_text_node, utf16}; +use std::rc::Rc; + +/// `TextFormatTransformersIndex`: the text format transformers by tag, in +/// the order their tags were first given (a later one with the same tag +/// replaces an earlier one, as JS's object does). +#[derive(Clone, Default)] +pub(crate) struct TextFormatTransformersIndex { + pub transformers_by_tag: Vec<(String, Rc)>, +} + +impl TextFormatTransformersIndex { + pub(crate) fn transformer(&self, tag: &str) -> Option> { + self.transformers_by_tag + .iter() + .find(|(t, _)| t == tag) + .map(|(_, t)| t.clone()) + } +} + +/// `$importMarkdownNodes(markdownString, container, transformers, +/// shouldPreserveNewLines)`: parses markdown and appends the nodes to +/// `container`, without clearing it or touching the selection. +pub fn import_markdown_nodes( + markdown_string: &str, + container: &NodeRef, + transformers: &[Transformer], + should_preserve_new_lines: bool, +) -> Result<(), LexicalError> { + let by_type = transformers_by_type(transformers); + let index = create_text_format_transformers_index(&by_type.text_format); + let lines: Vec = markdown_string.split('\n').map(str::to_owned).collect(); + // Blank lines between list lines make the list loose rather than + // ending it, unless they are preserved. + with_list_indent_columns( + !should_preserve_new_lines, + || -> Result<(), LexicalError> { + let mut i = 0; + while i < lines.len() { + let (imported, shifted_index) = + import_multiline(&lines, i, &by_type.multiline_element, container)?; + if imported { + i = shifted_index + 1; + continue; + } + import_blocks( + &lines[i], + container, + &by_type.element, + &index, + &by_type.text_match, + should_preserve_new_lines, + )?; + i += 1; + } + Ok(()) + }, + )?; + for child in container.get_children() { + if !should_preserve_new_lines + && is_empty_paragraph(&child) + && container.get_children_size() > 1 + { + child.remove(false)?; + continue; + } + if child.is_element() { + for text_node in get_all_text_nodes(&child) { + normalize_markdown_text_node(&text_node)?; + } + } + } + Ok(()) +} + +/// `ElementNode.getAllTextNodes()`. +fn get_all_text_nodes(node: &NodeRef) -> Vec { + let mut text_nodes = Vec::new(); + for child in node.get_children() { + if child.is_text() { + text_nodes.push(child); + } else if child.is_element() { + text_nodes.extend(get_all_text_nodes(&child)); + } + } + text_nodes +} + +/// `$importMultiline(lines, startLineIndex, multilineElementTransformers, +/// rootNode)`: whether a multiline element was imported, and the index of +/// the last line processed. +fn import_multiline( + lines: &[String], + start_line_index: usize, + transformers: &[Rc], + root_node: &NodeRef, +) -> Result<(bool, usize), LexicalError> { + 'transformers: for transformer in transformers { + let Some(start_match) = transformer + .reg_exp_start + .string_match(&lines[start_line_index]) + else { + continue; + }; + if let Some(handle) = &transformer.handle_import_after_start_match { + match handle(&MultilineImportArgs { + lines, + root_node, + start_line_index, + start_match: &start_match, + transformer, + })? { + MultilineImportResult::Skip => continue, + MultilineImportResult::Handled(imported, last) => return Ok((imported, last)), + MultilineImportResult::Default => {} + } + } + let regexp_end = transformer.reg_exp_end.as_ref().map(|e| &e.reg_exp); + let is_end_optional = match &transformer.reg_exp_end { + Some(end) => end.optional, + None => true, + }; + let mut end_line_index = start_line_index; + let lines_length = lines.len(); + while end_line_index < lines_length { + let end_match: Option = + regexp_end.and_then(|r| r.string_match(&lines[end_line_index])); + if end_match.is_none() && (!is_end_optional || end_line_index < lines_length - 1) { + end_line_index += 1; + continue; + } + // A closing match at the opening one's place doesn't count. + if let Some(end) = &end_match + && start_line_index == end_line_index + && end.index == start_match.index + { + end_line_index += 1; + continue; + } + let mut lines_in_between = Vec::new(); + let start_len = start_match.match_len(); + if let Some(end) = end_match + .as_ref() + .filter(|_| start_line_index == end_line_index) + { + let line = units(&lines[start_line_index]); + lines_in_between.push(slice(&line, start_len, slice_end(&line, end))); + } else { + for (i, text) in lines + .iter() + .enumerate() + .take(end_line_index + 1) + .skip(start_line_index) + { + let line = units(text); + if i == start_line_index { + lines_in_between.push(slice(&line, start_len, line.len())); + } else if let Some(end) = end_match.as_ref().filter(|_| i == end_line_index) { + lines_in_between.push(slice(&line, 0, slice_end(&line, end))); + } else { + lines_in_between.push(text.clone()); + } + } + } + if (transformer.replace)( + root_node, + None, + &start_match, + end_match.as_ref(), + Some(lines_in_between), + true, + )? { + return Ok((true, end_line_index)); + } + // The transformer declined: let the rest try from the start. + continue 'transformers; + } + } + Ok((false, start_line_index)) +} + +/// `-endMatch[0].length` as `slice`'s end: `-0` is 0, not the end. +fn slice_end(line: &[u16], end: &RegExpMatch) -> usize { + match end.match_len() { + 0 => 0, + n => line.len().saturating_sub(n), + } +} + +/// `text.slice(start, end)` with JS's clamping (an end before the start is +/// empty). +fn slice(text: &[u16], start: usize, end: usize) -> String { + let start = start.min(text.len()); + let end = end.min(text.len()); + if end <= start { + String::new() + } else { + string(&text[start..end]) + } +} + +/// `$importBlocks(lineText, rootNode, elementTransformers, +/// textFormatTransformersIndex, textMatchTransformers, +/// shouldPreserveNewLines)`. +fn import_blocks( + line_text: &str, + root_node: &NodeRef, + element_transformers: &[Rc], + index: &TextFormatTransformersIndex, + text_match_transformers: &[Rc], + should_preserve_new_lines: bool, +) -> Result<(), LexicalError> { + let text_node = create_text_node(line_text); + let element_node = create_paragraph_node(); + element_node.append(std::slice::from_ref(&text_node))?; + root_node.append(std::slice::from_ref(&element_node))?; + for transformer in element_transformers { + if let Some(m) = transformer.reg_exp.string_match(line_text) { + text_node.set_text_content(&utf16::slice_from(line_text, m.match_len())); + if (transformer.replace)(&element_node, std::slice::from_ref(&text_node), &m, true)? { + break; + } + } + } + import_text_transformers(&text_node, index, text_match_transformers, None, None)?; + + // A paragraph no transformer replaced joins the paragraph, quote or + // list before it. + if element_node.get_parent().is_some() && !line_text.is_empty() { + let previous_node = element_node.get_previous_sibling(); + if !should_preserve_new_lines + && let Some(previous) = previous_node + .filter(|p| is_paragraph_node(p) || is_quote_node(p) || is_list_node(p)) + { + let target_node = if is_list_node(&previous) { + previous + .get_last_descendant() + .and_then(|d| find_matching_parent(&d, is_list_item_node)) + } else { + Some(previous) + }; + if let Some(target) = target_node.filter(|t| t.get_text_content_size() > 0) { + let mut nodes = vec![create_markdown_line_break_node(&target)?]; + nodes.extend(element_node.get_children()); + target.splice(target.get_children_size(), 0, &nodes)?; + element_node.remove(false)?; + } + } + } + Ok(()) +} + +/// `$normalizeMarkdownTextNode(textNode)`: a tab node for each `\t`. +fn normalize_markdown_text_node(text_node: &NodeRef) -> Result<(), LexicalError> { + if is_tab_node(text_node) { + return Ok(()); + } + let text = text_node.get_text_content(); + if !text.contains('\t') { + return Ok(()); + } + let u = units(&text); + let data = text_node.text_data(); + let format = data.get_format(); + let style = data.get_style().to_owned(); + let mut nodes = Vec::new(); + let text_part = |s: &[u16]| { + let node = create_text_node(&string(s)); + node.set_format(format); + node.set_style(&style); + node + }; + let tab = [u16::from(b'\t')]; + let mut start = 0; + let mut found = index_of(&u, &tab, 0); + while let Some(index) = found { + if index > start { + nodes.push(text_part(&u[start..index])); + } + nodes.push(create_tab_node()); + start = index + 1; + found = index_of(&u, &tab, index + 1); + } + if start < u.len() { + nodes.push(text_part(&u[start..])); + } + let index = text_node.get_index_within_parent().unwrap_or(0); + text_node.get_parent_or_throw()?.splice(index, 1, &nodes) +} + +/// `createTextFormatTransformersIndex(textTransformers)`. +fn create_text_format_transformers_index( + text_transformers: &[Rc], +) -> TextFormatTransformersIndex { + let mut index = TextFormatTransformersIndex::default(); + for transformer in text_transformers { + match index + .transformers_by_tag + .iter_mut() + .find(|(t, _)| *t == transformer.tag) + { + Some((_, t)) => *t = transformer.clone(), + None => index + .transformers_by_tag + .push((transformer.tag.clone(), transformer.clone())), + } + } + index +} diff --git a/rust/lexical-editor/src/markdown/markdown_transformers.rs b/rust/lexical-editor/src/markdown/markdown_transformers.rs new file mode 100644 index 00000000000..1e349b798f7 --- /dev/null +++ b/rust/lexical-editor/src/markdown/markdown_transformers.rs @@ -0,0 +1,1452 @@ +//! `MarkdownTransformers.ts`: the transformer types, the transformers +//! upstream exports (`HEADING`, `QUOTE`, `CODE`, the lists, the text +//! formats and `LINK`), the markdown NodeStates, `normalizeMarkdown` and +//! `isTableRowDivider`. +//! +//! JS's transformers are objects compared by identity; here each is an +//! `Rc` of a struct with public fields, so a variant is made from a copy +//! (`{...LINK, importRegExp}` is `TextMatchTransformer { import_reg_exp, +//! ..(*link()).clone() }`), and the constants are functions that return the +//! same `Rc` on every call on a thread. + +use super::import_text_format_transformer::scan_code_spans; +use super::markdown_export::is_selected; +use super::reg_exp::{RegExp, RegExpMatch}; +use super::utils::{ + has_non_space, is_empty_paragraph, is_escaped, is_line_break_node, last_index_of, re, string, + text_format, to_character_reference, trim, trim_start, unescape_text, units, +}; +use crate::code::{create_code_node, is_code_node}; +use crate::json::Value; +use crate::link::{LinkAttributes, create_link_node, is_auto_link_node, is_link_node}; +use crate::list::{ + ListType, create_list_item_node, create_list_node, is_list_item_node, is_list_node, +}; +use crate::nodes::line_break::create_line_break_node; +use crate::rich_text::{ + HeadingTagType, create_heading_node, create_quote_node, is_heading_node, is_quote_node, +}; +use crate::utils::find_matching_parent; +use crate::{ + LexicalError, NodeRef, SelectionRef, StateConfig, StateValueConfig, create_state_with, + create_text_node, get_state, set_state, utf16, +}; +use std::cell::RefCell; +use std::rc::Rc; + +/// `Transformer`: one of the four kinds. +#[derive(Clone)] +pub enum Transformer { + Element(Rc), + MultilineElement(Rc), + TextFormat(Rc), + TextMatch(Rc), +} + +impl Transformer { + /// `transformer.type`. + pub fn transformer_type(&self) -> &'static str { + match self { + Transformer::Element(_) => "element", + Transformer::MultilineElement(_) => "multiline-element", + Transformer::TextFormat(_) => "text-format", + Transformer::TextMatch(_) => "text-match", + } + } + + /// `transformer.dependencies`: the node types it needs, for those that + /// have them. + pub fn dependencies(&self) -> &[&'static str] { + match self { + Transformer::Element(t) => &t.dependencies, + Transformer::MultilineElement(t) => &t.dependencies, + Transformer::TextFormat(_) => &[], + Transformer::TextMatch(t) => &t.dependencies, + } + } +} + +impl From> for Transformer { + fn from(t: Rc) -> Self { + Transformer::Element(t) + } +} + +impl From> for Transformer { + fn from(t: Rc) -> Self { + Transformer::MultilineElement(t) + } +} + +impl From> for Transformer { + fn from(t: Rc) -> Self { + Transformer::TextFormat(t) + } +} + +impl From> for Transformer { + fn from(t: Rc) -> Self { + Transformer::TextMatch(t) + } +} + +/// `(node: ElementNode) => string`: exports an element's children. +pub type ExportChildren<'a> = &'a dyn Fn(&NodeRef) -> Result; + +/// `(node: TextNode, textContent: string) => string`: exports a text node's +/// formats around `textContent`. +pub type ExportFormat<'a> = &'a dyn Fn(&NodeRef, &str) -> Result; + +/// An element transformer's `export`: the markdown of `node`, or `None` +/// (JS `null`) to let the next transformer try. With a selection, only the +/// selected content is exported. +pub type ElementExport = Rc< + dyn Fn( + &NodeRef, + ExportChildren<'_>, + Option<&SelectionRef>, + ) -> Result, LexicalError>, +>; + +/// `ElementTransformer.replace(parentNode, children, match, isImport)`: +/// `false` cancels the transform (the next transformer is tried); JS's +/// `void` and `true` are both `true`. +pub type ElementReplace = + Rc Result>; + +/// `ElementTransformer`: a block that a line's leading markdown makes, +/// such as a heading or a list item. +#[derive(Clone)] +pub struct ElementTransformer { + pub dependencies: Vec<&'static str>, + pub export: ElementExport, + pub reg_exp: RegExp, + pub replace: ElementReplace, + /// `triggerOnEnter`: whether `registerMarkdownShortcuts` may run this + /// transformer on Enter at the end of the line, without the space. + pub trigger_on_enter: bool, +} + +/// The arguments of `handleImportAfterStartMatch`. +pub struct MultilineImportArgs<'a> { + pub lines: &'a [String], + pub root_node: &'a NodeRef, + pub start_line_index: usize, + pub start_match: &'a RegExpMatch, + pub transformer: &'a Rc, +} + +/// What `handleImportAfterStartMatch` returns. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum MultilineImportResult { + /// `[imported, lastLineIndex]`. + Handled(bool, usize), + /// `null`: try the next multiline element transformer. + Skip, + /// `undefined`: the default behavior. + Default, +} + +/// `handleImportAfterStartMatch`. +pub type HandleImportAfterStartMatch = + Rc) -> Result>; + +/// `MultilineElementTransformer.replace(rootNode, children, startMatch, +/// endMatch, linesInBetween, isImport)`: `children` is given (and +/// `linesInBetween` is not) for a markdown shortcut; `false` cancels. +pub type MultilineReplace = Rc< + dyn Fn( + &NodeRef, + Option<&[NodeRef]>, + &RegExpMatch, + Option<&RegExpMatch>, + Option>, + bool, + ) -> Result, +>; + +/// `regExpEnd`: a regex, or `{optional: true, regExp}`. +#[derive(Clone, Debug)] +pub struct RegExpEnd { + /// Whether the end match is optional: when none is found, the text from + /// the start match to the end of the document matches. + pub optional: bool, + pub reg_exp: RegExp, +} + +/// `MultilineElementTransformer`: a block that spans lines between a start +/// and an end match, such as a fenced code block. +#[derive(Clone)] +pub struct MultilineElementTransformer { + pub handle_import_after_start_match: Option, + pub dependencies: Vec<&'static str>, + pub export: Option, + pub reg_exp_start: RegExp, + pub reg_exp_end: Option, + pub replace: MultilineReplace, +} + +/// `TextFormatTransformer`: a text format between a pair of tags, such as +/// `**bold**`. +#[derive(Clone, Debug)] +pub struct TextFormatTransformer { + pub format: Vec<&'static str>, + pub tag: String, + /// `intraword`; `Some(false)` keeps the tag from opening or closing + /// inside a word. + pub intraword: Option, +} + +/// A text match transformer's `export`. +pub type TextMatchExport = Rc< + dyn Fn(&NodeRef, ExportChildren<'_>, ExportFormat<'_>) -> Result, LexicalError>, +>; + +/// `TextMatchTransformer.replace(node, match)`: may return a text node that +/// text formats are then imported in. +pub type TextMatchReplace = + Rc Result, LexicalError>>; + +/// `getEndIndex(node, match)`: `None` is JS `false`. +pub type GetEndIndex = Rc Result, LexicalError>>; + +/// `TextMatchTransformer`: inline text a regex matches, such as a link. +#[derive(Clone)] +pub struct TextMatchTransformer { + pub dependencies: Vec<&'static str>, + /// How a node exports, or `None` when this transformer does not export + /// it. Export may call it with placeholder callbacks only to learn + /// whether it writes the node, so it should have no side effects. + pub export: Option, + /// What text is matched on import. + pub import_reg_exp: Option, + /// What text is matched for a markdown shortcut while typing. + pub reg_exp: RegExp, + pub replace: Option, + pub get_end_index: Option, + /// The character whose typing triggers the transformer. + pub trigger: Option, +} + +const ORDERED_LIST_REGEX: &str = r"^(\s*)(\d{1,})\.\s"; +const UNORDERED_LIST_REGEX: &str = r"^(\s*)[-*+]\s"; +const CHECK_LIST_REGEX: &str = r"^(\s*)(?:[-*+]\s)?\s?(\[(\s|x)?\])\s"; +const HEADING_REGEX: &str = r"^(#{1,6})\s"; +const QUOTE_REGEX: &str = r"^>\s"; +const CODE_START_REGEX: &str = r"^([ \t]*`{3,})([\w-]+)?[ \t]?"; +const CODE_END_REGEX: &str = r"^[ \t]*`{3,}$"; +const CODE_SINGLE_LINE_REGEX: &str = r"^[ \t]*```[^`]+(?:(?:`{1,2}|`{4,})[^`]+)*```(?:[^`]|$)"; +const TABLE_ROW_REG_EXP: &str = r"^(?:\|)(.+)(?:\|)\s?$"; +const TAG_START_REGEX: &str = r"^<[a-z_][\w-]*(?:\s[^<>]*)?\/?>"; +const TAG_END_REGEX: &str = r"^<\/[a-z_][\w-]*\s*>"; + +/// `isTableRowDivider(line)`: whether `line` is a table delimiter row such +/// as `| --- | :--: |`, the linear-time form of `/^(\| ?:?-*:? ?)+\|\s?$/`. +pub fn is_table_row_divider(line: &str) -> bool { + let line = units(line); + let at = |j: usize| line.get(j).copied(); + let is = |j: usize, c: u8| at(j) == Some(u16::from(c)); + if !is(0, b'|') { + return false; + } + let length = line.len(); + let mut i = 1; + let mut cells = 0; + while i < length { + let mut j = i; + if is(j, b' ') { + j += 1; + } + if is(j, b':') { + j += 1; + } + while is(j, b'-') { + j += 1; + } + if is(j, b':') { + j += 1; + } + if is(j, b' ') { + j += 1; + } + if !is(j, b'|') { + break; + } + cells += 1; + i = j + 1; + } + cells > 0 && (i == length || (i == length - 1 && super::utils::is_space(u32::from(line[i])))) +} + +/// `ENDS_WITH(regex)`: `/(?:source)$/flags`. +fn ends_with(source: &str, flags: &str) -> RegExp { + RegExp::of(&format!("(?:{source})$"), flags) +} + +/// A string state with a `parse` and `resetOnCopyNode: true`. +fn string_state(key: &str, parse: impl Fn(&Value) -> String + 'static) -> StateConfig { + let mut config = StateValueConfig::new(parse); + config.reset_on_copy_node = true; + create_state_with(key, config) +} + +thread_local! { + static LIST_MARKER_STATE: StateConfig = string_state("mdListMarker", |v| { + match v.as_str() { + Some(s) if re(r"^[-*+]$", "").test(s) => s.to_owned(), + _ => "-".to_owned(), + } + }); + static CODE_FENCE_STATE: StateConfig = string_state("mdCodeFence", |v| { + match v.as_str() { + Some(s) if re(r"^`{3,}$", "").test(s) => s.to_owned(), + _ => "```".to_owned(), + } + }); + static CODE_META_STATE: StateConfig = + string_state("mdCodeMeta", |v| v.as_str().unwrap_or("").to_owned()); + static HARD_LINE_BREAK_STATE: StateConfig = string_state("mdHardLineBreak", |v| { + match v.as_str() { + Some(s) if re(r"^(\\| {2,})$", "").test(s) => s.to_owned(), + _ => String::new(), + } + }); +} + +/// `listMarkerState`: the bullet (`-`, `*` or `+`) a list was written with. +pub fn list_marker_state() -> StateConfig { + LIST_MARKER_STATE.with(Clone::clone) +} + +/// `codeFenceState`: the fence (three or more backticks) a code block was +/// written with. +pub fn code_fence_state() -> StateConfig { + CODE_FENCE_STATE.with(Clone::clone) +} + +/// `codeMetaState`: the info string's tail after a fenced code block's +/// language (`title="x"` in ```` ```js title="x" ````), which `CodeNode` +/// does not model. +pub fn code_meta_state() -> StateConfig { + CODE_META_STATE.with(Clone::clone) +} + +/// `hardLineBreakState`: the marker (`\` or two or more spaces) of a hard +/// line break, on the line break node. +pub fn hard_line_break_state() -> StateConfig { + HARD_LINE_BREAK_STATE.with(Clone::clone) +} + +/// `parseMarkdownHardLineBreak(line)`: the line without its hard line +/// break marker, and the marker, when it ends with one. +pub fn parse_markdown_hard_line_break(line: &str) -> Option<(String, String)> { + if let Some(rest) = line.strip_suffix('\\') { + // A trailing backslash is a hard line break only when it is not + // itself escaped, so only an odd-length run counts. + let backslashes = line.chars().rev().take_while(|c| *c == '\\').count(); + return (backslashes % 2 == 1).then(|| (rest.to_owned(), "\\".to_owned())); + } + let spaces = re(r"^(.*?\S)( {2,})$", "").exec(line)?; + Some((spaces.str(1).to_owned(), spaces.str(2).to_owned())) +} + +/// `hasNonWhitespaceContentOnLine(children, endIndex)`. +fn has_non_whitespace_content_on_line(children: &[NodeRef], end_index: usize) -> bool { + for child in children[..end_index].iter().rev() { + if is_line_break_node(child) { + return false; + } + if has_non_space(&child.get_text_content()) { + return true; + } + } + false +} + +/// `$extractMarkdownHardLineBreakMarker(previousNode)`. +fn extract_markdown_hard_line_break_marker(previous_node: &NodeRef) -> Option { + let children = previous_node.get_children(); + let last_child_index = children.len().checked_sub(1)?; + let last_child = &children[last_child_index]; + if !last_child.is_text() { + return None; + } + let last_text = last_child.get_text_content(); + if let Some((text, marker)) = parse_markdown_hard_line_break(&last_text) { + last_child.set_text_content(&text); + return Some(marker); + } + if re(r"^ {2,}$", "").test(&last_text) + && has_non_whitespace_content_on_line(&children, last_child_index) + { + last_child.set_text_content(""); + return Some(last_text); + } + None +} + +/// `$createMarkdownLineBreakNode(previousNode)`: a line break that keeps +/// the hard line break marker the end of `previous_node` holds. +pub fn create_markdown_line_break_node(previous_node: &NodeRef) -> Result { + let line_break = create_line_break_node(); + if let Some(marker) = extract_markdown_hard_line_break_marker(previous_node) { + set_state(&line_break, &hard_line_break_state(), marker)?; + } + Ok(line_break) +} + +/// `$isUnreplaceableBlock(parentNode)`: a quote holds inline content, so a +/// block shortcut typed in it would lose it. +fn is_unreplaceable_block(parent_node: &NodeRef) -> bool { + is_quote_node(parent_node) +} + +/// `stripFenceIndent(line, indent)`: removes up to `indent` spaces or tabs. +fn strip_fence_indent(line: &str, indent: usize) -> String { + let u = units(line); + let mut index = 0; + while index < indent && matches!(u.get(index), Some(0x20 | 0x09)) { + index += 1; + } + string(&u[index..]) +} + +/// `$setCodeMeta(parentNode, meta)`. +fn set_code_meta(parent_node: &NodeRef, meta: &str) -> Result<(), LexicalError> { + if let Some(code_node) = parent_node.get_last_child() + && !meta.is_empty() + && is_code_node(&code_node) + { + set_state(&code_node, &code_meta_state(), meta.to_owned())?; + } + Ok(()) +} + +/// `createBlockNode(createNode)`'s replace. +fn create_block_node( + node: NodeRef, + parent_node: &NodeRef, + children: &[NodeRef], + is_import: bool, +) -> Result { + node.append(children)?; + parent_node.replace(&node, false)?; + if !is_import { + node.select(Some(0), Some(0))?; + } + Ok(true) +} + +/// `LIST_INDENT_SIZE`: the spaces of one indentation level. +const LIST_INDENT_SIZE: usize = 4; + +/// `getIndent(whitespaces)`. +fn get_indent(whitespaces: &str) -> usize { + let tabs = whitespaces.chars().filter(|c| *c == '\t').count(); + let spaces = whitespaces.chars().filter(|c| *c == ' ').count(); + tabs + spaces / LIST_INDENT_SIZE +} + +/// `getColumn(whitespaces)`: the column after `whitespaces`, with tabs +/// expanded to the next multiple of `LIST_INDENT_SIZE`. +fn get_column(whitespaces: &str) -> usize { + let mut column = 0; + for c in whitespaces.chars() { + column += if c == '\t' { + LIST_INDENT_SIZE - column % LIST_INDENT_SIZE + } else { + 1 + }; + } + column +} + +thread_local! { + /// `importListColumns`: the content columns of the list levels the + /// lines placed so far have left open, or `None` outside an import. + static IMPORT_LIST_COLUMNS: RefCell>> = const { RefCell::new(None) }; + /// `importJoinsLooseLists`. + static IMPORT_JOINS_LOOSE_LISTS: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// `withListIndentColumns(joinLooseLists, fn)`: runs `f` with the columns +/// of one markdown import tracked across the lines it places, restoring +/// what was tracked around it. +pub(crate) fn with_list_indent_columns(join_loose_lists: bool, f: impl FnOnce() -> T) -> T { + let previous_columns = IMPORT_LIST_COLUMNS.with(|c| c.replace(Some(Vec::new()))); + let previous_joins = IMPORT_JOINS_LOOSE_LISTS.with(|c| c.replace(join_loose_lists)); + struct Restore(Option>>, bool); + impl Drop for Restore { + fn drop(&mut self) { + let columns = self.0.take().expect("columns to restore"); + IMPORT_LIST_COLUMNS.with(|c| *c.borrow_mut() = columns); + IMPORT_JOINS_LOOSE_LISTS.with(|c| c.set(self.1)); + } + } + let _restore = Restore(Some(previous_columns), previous_joins); + f() +} + +/// `getColumnIndent(columns, whitespaces)`. +fn get_column_indent(columns: &[usize], whitespaces: &str) -> usize { + let column = get_column(whitespaces); + for i in (0..columns.len()).rev() { + if column >= columns[i] { + return i + 1; + } + } + 0 +} + +/// `getContentColumn(match, listType)`. +fn get_content_column(m: &RegExpMatch, list_type: ListType) -> usize { + let mut prefix = m.str(0).to_owned(); + if list_type == ListType::Check { + let rest = utf16::slice_from(&prefix, utf16::len(m.str(1))); + if let Some(bullet) = re(r"^[-*+]\s", "").exec(&rest) { + prefix = m.str(1).to_owned() + bullet.str(0); + } + } + get_column(&prefix).min(get_column(m.str(1)) + LIST_INDENT_SIZE) +} + +/// `setOpenColumn(columns, indent, match, listType)`. +fn set_open_column(columns: &mut Vec, indent: usize, m: &RegExpMatch, list_type: ListType) { + for i in columns.len()..indent { + columns.push((i + 1) * LIST_INDENT_SIZE); + } + columns.truncate(indent); + columns.push(get_content_column(m, list_type)); +} + +/// `Number(match[2])`. +fn number(text: &str) -> f64 { + text.parse().unwrap_or(f64::NAN) +} + +/// `listReplace(listType)`'s replace. +fn list_replace( + list_type: ListType, + parent_node: &NodeRef, + children: &[NodeRef], + m: &RegExpMatch, + is_import: bool, +) -> Result { + if is_heading_node(parent_node) || (!is_import && is_unreplaceable_block(parent_node)) { + return Ok(false); + } + let previous_node = parent_node.get_previous_sibling(); + let next_node = parent_node.get_next_sibling(); + let list_item = create_list_item_node(if list_type == ListType::Check { + Some(re("^x$", "i").test(m.str(3))) + } else { + None + }); + let first_match_char = trim(m.str(0)).chars().next().map(String::from); + let list_marker = match &first_match_char { + Some(c) + if matches!(list_type, ListType::Bullet | ListType::Check) + && matches!(c.as_str(), "-" | "*" | "+") => + { + Some(c.clone()) + } + _ => None, + }; + // A block of another kind closes every level above this line; blank + // lines only make the list loose (see upstream's comment). + let mut columns = IMPORT_LIST_COLUMNS.with(|c| c.borrow_mut().take()); + let mut preceding_block = previous_node.clone(); + if let Some(columns) = columns.as_mut() { + if IMPORT_JOINS_LOOSE_LISTS.with(std::cell::Cell::get) { + while let Some(block) = preceding_block.clone().filter(is_empty_paragraph) { + preceding_block = block.get_previous_sibling(); + } + } + if !preceding_block.as_ref().is_some_and(is_list_node) { + columns.clear(); + } + } + let result = (|| -> Result<(), LexicalError> { + let indent = match &columns { + Some(c) if !c.is_empty() => get_column_indent(c, m.str(1)), + _ => get_indent(m.str(1)), + }; + if let Some(next) = next_node + .as_ref() + .filter(|n| is_list_node(n) && n.get_list_type() == Some(list_type)) + { + match next.get_first_child() { + Some(first) => { + first.insert_before(&list_item, true)?; + } + None => next.append(std::slice::from_ref(&list_item))?, + } + // The new item lands at index 0, so the typed number becomes + // the list's start (#8677), unless it is indented. + if list_type == ListType::Number && indent == 0 { + next.set_list_start(number(m.str(2))); + } + parent_node.remove(false)?; + } else if let Some(preceding) = preceding_block + .as_ref() + .filter(|p| is_list_node(p) && (p.get_list_type() == Some(list_type) || indent > 0)) + { + preceding.append(std::slice::from_ref(&list_item))?; + parent_node.remove(false)?; + } else { + let list = create_list_node( + list_type, + if list_type == ListType::Number { + number(m.str(2)) + } else { + 1.0 + }, + ); + list.append(std::slice::from_ref(&list_item))?; + parent_node.replace(&list, false)?; + } + list_item.append(children)?; + if !is_import { + list_item.select(Some(0), Some(0))?; + } + if indent > 0 { + list_item.set_indent(indent as f64)?; + retype_nested_list(&list_item, list_type, m)?; + } + // The marker belongs to the list the item ends up in. + if let (Some(marker), Some(list_node)) = (&list_marker, list_item.get_parent()) + && is_list_node(&list_node) + { + set_state(&list_node, &list_marker_state(), marker.clone())?; + } + if let Some(columns) = columns.as_mut() { + set_open_column(columns, indent, m, list_type); + } + Ok(()) + })(); + IMPORT_LIST_COLUMNS.with(|c| *c.borrow_mut() = columns); + result.map(|()| true) +} + +/// `$retypeNestedList(listItem, listType, match)`: moves an item nested by +/// `setIndent` into a list of its own type. +fn retype_nested_list( + list_item: &NodeRef, + list_type: ListType, + m: &RegExpMatch, +) -> Result<(), LexicalError> { + let Some(nested_list) = list_item.get_parent() else { + return Ok(()); + }; + if !is_list_node(&nested_list) || nested_list.get_list_type() == Some(list_type) { + return Ok(()); + } + let Some(wrapper) = nested_list.get_parent().filter(is_list_item_node) else { + return Ok(()); + }; + let retyped_list = create_list_node( + list_type, + if list_type == ListType::Number { + number(m.str(2)) + } else { + 1.0 + }, + ); + let is_first = list_item.get_previous_sibling().is_none(); + retyped_list.append(std::slice::from_ref(list_item))?; + let retyped_wrapper = create_list_item_node(None); + retyped_wrapper.append(std::slice::from_ref(&retyped_list))?; + if is_first { + wrapper.insert_before(&retyped_wrapper, true)?; + } else { + wrapper.insert_after(&retyped_wrapper, true)?; + } + if nested_list.get_children_size() == 0 { + wrapper.remove(false)?; + } + Ok(()) +} + +/// `$listExport(listNode, exportChildren, depth, selection)`. +fn list_export( + list_node: &NodeRef, + export_children: ExportChildren<'_>, + depth: usize, + selection: Option<&SelectionRef>, +) -> Result { + let mut output = Vec::new(); + let mut index = 0.0; + for list_item_node in list_node.get_children() { + if !is_list_item_node(&list_item_node) { + continue; + } + if list_item_node.get_children_size() == 1 + && let Some(first_child) = list_item_node.get_first_child().filter(is_list_node) + { + let nested = list_export(&first_child, export_children, depth + 1, selection)?; + if !nested.is_empty() { + output.push(nested); + } + continue; + } + if let Some(selection) = selection { + let mut any = false; + for child in list_item_node.get_children() { + if is_selected(&child, Some(selection))? { + any = true; + break; + } + } + if !any { + continue; + } + } + let indent = " ".repeat(depth * LIST_INDENT_SIZE); + let list_type = list_node.get_list_type().unwrap_or(ListType::Bullet); + let list_marker = get_state(list_node, &list_marker_state())?; + let prefix = match list_type { + ListType::Number => format!( + "{}. ", + crate::json::number_to_string(list_node.get_list_start().unwrap_or(1.0) + index) + ), + ListType::Check => format!( + "{list_marker} [{}] ", + if list_item_node.get_checked() == Some(true) { + 'x' + } else { + ' ' + } + ), + ListType::Bullet => format!("{list_marker} "), + }; + let mut children_text = export_children(&list_item_node)?; + if list_type != ListType::Number { + children_text = re(r"^(\s{0,3}\d+)(\.\s)", "").replace(&children_text, "$1\\$2"); + } + output.push(indent + &prefix + &children_text); + index += 1.0; + } + Ok(output.join("\n")) +} + +fn list_transformer(list_type: ListType, reg_exp: &'static str) -> ElementTransformer { + ElementTransformer { + dependencies: vec!["list", "listitem"], + export: Rc::new(|node, export_children, selection| { + if is_list_node(node) { + list_export(node, export_children, 0, selection).map(Some) + } else { + Ok(None) + } + }), + reg_exp: re( + reg_exp, + if list_type == ListType::Check { + "i" + } else { + "" + }, + ), + replace: Rc::new(move |parent, children, m, is_import| { + list_replace(list_type, parent, children, m, is_import) + }), + trigger_on_enter: true, + } +} + +/// `CODE.replace`. +fn code_replace( + root_node: &NodeRef, + children: Option<&[NodeRef]>, + start_match: &RegExpMatch, + end_match: Option<&RegExpMatch>, + lines_in_between: Option>, + is_import: bool, +) -> Result { + let fence = match start_match.get(1).filter(|f| !f.is_empty()) { + Some(f) => trim(f).to_owned(), + None => "```".to_owned(), + }; + let language = start_match.get(2).filter(|l| !l.is_empty()); + let fence_indent = start_match + .get(1) + .map_or(0, |f| utf16::len(f) - utf16::len(trim_start(f))); + match (children, lines_in_between) { + (None, Some(mut lines)) => { + let code_block_node = create_code_node(language, None); + let code = if lines.len() == 1 { + if end_match.is_some() { + strip_fence_indent(&lines[0], fence_indent) + } else { + lines[0].strip_prefix(' ').unwrap_or(&lines[0]).to_owned() + } + } else { + if !lines.is_empty() { + if trim(&lines[0]).is_empty() { + lines.remove(0); + } else if let Some(rest) = lines[0].strip_prefix(' ') { + lines[0] = rest.to_owned(); + } + } + while lines.last().is_some_and(String::is_empty) { + lines.pop(); + } + lines + .iter() + .map(|line| strip_fence_indent(line, fence_indent)) + .collect::>() + .join("\n") + }; + set_state(&code_block_node, &code_fence_state(), fence)?; + let text_node = create_text_node(&code); + code_block_node.append(&[text_node])?; + root_node.append(&[code_block_node])?; + Ok(true) + } + (Some(children), _) => { + if !is_import && is_unreplaceable_block(root_node) { + return Ok(false); + } + create_block_node( + create_code_node(start_match.get(2), None), + root_node, + children, + is_import, + ) + } + (None, None) => Ok(true), + } +} + +/// `CODE.handleImportAfterStartMatch`. +fn code_handle_import_after_start_match( + args: &MultilineImportArgs<'_>, +) -> Result { + let MultilineImportArgs { + lines, + root_node, + start_line_index, + start_match, + .. + } = *args; + let fence = start_match.str(1); + let fence_length = utf16::len(trim(fence)); + let current_line = &lines[start_line_index]; + let after_fence_index = start_match.index + utf16::len(fence); + let after_fence = utf16::slice_from(current_line, after_fence_index); + + let single_line_end_regex = RegExp::of(&format!("`{{{fence_length},}}$"), ""); + if let Some(end_match) = single_line_end_regex.exec(&after_fence) { + let after = units(&after_fence); + let end = last_index_of(&after, &units(end_match.text())).unwrap_or(0); + let content = string(&after[..end]); + let mut fake_start_match = start_match.clone(); + if fake_start_match.groups.len() > 2 { + fake_start_match.groups[2] = Some(String::new()); + } + code_replace( + root_node, + None, + &fake_start_match, + Some(&end_match), + Some(vec![content]), + true, + )?; + return Ok(MultilineImportResult::Handled(true, start_line_index)); + } + + let multiline_end_regex = RegExp::of(&format!("^[ \\t]*`{{{fence_length},}}$"), ""); + let after_start = utf16::slice_from(current_line, start_match.match_len()); + for i in start_line_index + 1..lines.len() { + let line = &lines[i]; + if let Some(end_match) = multiline_end_regex.exec(line) { + let mut lines_in_between: Vec = lines[start_line_index + 1..i].to_vec(); + // Only the info string's first word is the language; what follows + // it is metadata. With no language, the remainder is content. + let meta = if start_match.get(2).is_some_and(|l| !l.is_empty()) { + trim(&after_start).to_owned() + } else { + String::new() + }; + lines_in_between.insert( + 0, + if meta.is_empty() { + after_start.clone() + } else { + String::new() + }, + ); + code_replace( + root_node, + None, + start_match, + Some(&end_match), + Some(lines_in_between), + true, + )?; + set_code_meta(root_node, &meta)?; + return Ok(MultilineImportResult::Handled(true, i)); + } + } + + let mut lines_in_between: Vec = lines[start_line_index + 1..].to_vec(); + lines_in_between.insert(0, after_start); + code_replace( + root_node, + None, + start_match, + None, + Some(lines_in_between), + true, + )?; + Ok(MultilineImportResult::Handled(true, lines.len() - 1)) +} + +/// `CODE.export`. +fn code_export(node: &NodeRef) -> Result, LexicalError> { + if !is_code_node(node) { + return Ok(None); + } + let text_content = node.get_text_content(); + let mut fence = get_state(node, &code_fence_state())?; + if text_content.contains(&fence) { + let backticks = re("`{3,}", "g").match_all(&text_content); + if !backticks.is_empty() { + let max_length = backticks + .iter() + .map(RegExpMatch::match_len) + .max() + .unwrap_or(0); + fence = "`".repeat(max_length + 1); + } + } + let language = node.get_language().unwrap_or_default(); + let meta = if language.is_empty() { + String::new() + } else { + get_state(node, &code_meta_state())? + }; + Ok(Some(format!( + "{fence}{language}{}{}\n{fence}", + if meta.is_empty() { + String::new() + } else { + format!(" {meta}") + }, + if text_content.is_empty() { + String::new() + } else { + format!("\n{text_content}") + }, + ))) +} + +/// What `getCodeSpanDelimiter` returns. +pub struct CodeSpanDelimiter { + pub fence: String, + pub padded: String, +} + +/// `getCodeSpanDelimiter(content)`: a CommonMark fence and padded content +/// for an inline code span. +pub fn get_code_span_delimiter(content: &str) -> CodeSpanDelimiter { + let mut longest_run = 0; + let mut run = 0; + for c in content.chars() { + if c == '`' { + run += 1; + longest_run = longest_run.max(run); + } else { + run = 0; + } + } + let fence = "`".repeat(longest_run + 1); + let first = content.chars().next(); + let last = content.chars().last(); + let is_space = |c: Option| c.is_some_and(|c| super::utils::is_space(c as u32)); + let needs_padding = content.is_empty() + || content.contains('`') + || (is_space(first) && is_space(last) && content.chars().any(|c| c != ' ')); + CodeSpanDelimiter { + fence, + padded: if needs_padding { + format!(" {content} ") + } else { + content.to_owned() + }, + } +} + +/// `NOT_RAW_DESTINATION`: a URL with any of these goes in the pointy form. +const NOT_RAW_DESTINATION: &str = r"[\s\0-\x1f\x7f]"; + +/// `escapeLinkPart(value, syntax)`. +fn escape_link_part(value: &str, syntax: &RegExp) -> String { + let value = + re(r"&(?=#\d+;|#[Xx][\dA-Fa-f]+;|[A-Za-z][\dA-Za-z]*;)", "g").replace(value, "&"); + let value = syntax.replace(&value, "\\$&"); + let value = value.replace('`', "\\`"); + re(r"[\r\n]", "g").replace_with(&value, |m| { + to_character_reference(m.text().chars().next().map_or(0, |c| c as u32)) + }) +} + +/// `getLinkLabelOpener(text)`: the inner opener of a link label after a +/// literal prefix, `Some(-1)` for a balanced label, or `None` when +/// unmatched closers leave no recoverable opener. +pub fn get_link_label_opener(text: &str) -> Option { + let u = units(text); + let code_spans = scan_code_spans(&u); + let mut code_index = 0; + let mut brackets = u.clone(); + let mut i = 0; + while i < u.len() { + let c = u[i]; + let part_len = if c == u16::from(b'\\') + && u.get(i + 1) + .is_some_and(|n| !matches!(n, 0x0A | 0x0D | 0x2028 | 0x2029)) + { + 2 + } else if c == u16::from(b'[') || c == u16::from(b']') { + 1 + } else { + i += 1; + continue; + }; + while code_index < code_spans.len() && i >= code_spans[code_index].end_index { + code_index += 1; + } + let in_code = code_index < code_spans.len() && i >= code_spans[code_index].start_index; + if in_code || c == u16::from(b'\\') { + for b in &mut brackets[i..i + part_len] { + *b = u16::from(b' '); + } + } + i += part_len; + } + let mut openers: Vec = Vec::new(); + let mut unmatched_closer = false; + for (index, c) in brackets.iter().enumerate() { + if *c == u16::from(b'[') { + openers.push(index); + } else if *c == u16::from(b']') && openers.pop().is_none() { + unmatched_closer = true; + } + } + match openers.pop() { + Some(o) => Some(o as isize), + None if unmatched_closer => None, + None => Some(-1), + } +} + +/// `unescapeLinkPrefix(text)`. +fn unescape_link_prefix(text: &str) -> String { + let u = units(text); + let mut output = String::new(); + let mut previous = 0; + for span in scan_code_spans(&u) { + output += &unescape_text(&string(&u[previous..span.start_index])); + output += &string(&u[span.start_index..span.end_index]); + previous = span.end_index; + } + output + &unescape_text(&string(&u[previous..])) +} + +/// `LINK.export`. +fn link_export( + node: &NodeRef, + export_children: ExportChildren<'_>, +) -> Result, LexicalError> { + if !is_link_node(node) || is_auto_link_node(node) { + return Ok(None); + } + let text_content = export_children(node)?; + let title = node.get_title(); + let raw_url = node.get_url().unwrap_or_default(); + let url = if raw_url.is_empty() || re(NOT_RAW_DESTINATION, "").test(&raw_url) { + format!("<{}>", escape_link_part(&raw_url, &re(r"[\\<>]", "g"))) + } else { + escape_link_part(&raw_url, &re(r"[\\()]|^<", "g")) + }; + Ok(Some(match title.filter(|t| !t.is_empty()) { + Some(title) => format!( + "[{text_content}]({url} \"{}\")", + escape_link_part(&title, &re(r#"[\\"]"#, "g")) + ), + None => format!("[{text_content}]({url})"), + })) +} + +/// `LINK.replace`. +fn link_replace(text_node: &NodeRef, m: &RegExpMatch) -> Result, LexicalError> { + let source = units(&m.input); + let start_index = m.index; + if find_matching_parent(text_node, is_link_node).is_some() + || is_escaped(&source, start_index) + || is_escaped(&source, start_index + utf16::len(m.str(1)) + 1) + { + return Ok(None); + } + let link_text = m.str(1); + // At most one destination shape matched, and either may be empty. + let link_url = unescape_text(m.get(2).or(m.get(3)).unwrap_or("")); + let raw_link_title = m.get(4).or(m.get(5)).or(m.get(6)); + let link_title = raw_link_title.map(unescape_text); + let link_node = create_link_node( + &link_url, + &LinkAttributes { + title: Some(link_title), + ..LinkAttributes::default() + }, + ); + let Some(open_bracket_index) = get_link_label_opener(link_text) else { + return Ok(None); + }; + let mut parsed_link_text = link_text.to_owned(); + let mut outside_link_text = String::new(); + if open_bracket_index != -1 { + let o = open_bracket_index as usize; + outside_link_text = "[".to_owned() + &unescape_link_prefix(&utf16::slice(link_text, 0, o)); + parsed_link_text = utf16::slice_from(link_text, o + 1); + } + // The text left outside the link carries the replaced node's format too. + let format = text_format(text_node); + let link_text_node = create_text_node(&parsed_link_text); + link_text_node.set_format(format); + link_node.append(std::slice::from_ref(&link_text_node))?; + text_node.replace(&link_node, false)?; + if !outside_link_text.is_empty() { + let outside = create_text_node(&outside_link_text); + outside.set_format(format); + link_node.insert_before(&outside, true)?; + } + Ok(Some(link_text_node)) +} + +/// `LINK.importRegExp`'s source. +pub(crate) const LINK_IMPORT_REG_EXP: &str = r#"(?:\[(.+?)\])(?:\(\s*(?:(?:<((?:\\[^\n\r]|[^<>\n\r\\])*)>|((?!<)(?:\\[^\s]|\\(?=\s)|\((?:\\[^\s]|[^\s()\\]|\((?:\\[^\s]|[^\s()\\]|\((?:\\[^\s]|[^\s()\\])*\))*\))*\)|[^\s()\\])+))(?:\s+(?:"((?:\\[\s\S]|[^"\\])*)"|'((?:\\[\s\S]|[^'\\])*)'|\(((?:\\[\s\S]|[^()\\])*)\)))?\s*)?\))"#; +const LINK_REG_EXP: &str = r#"(?:\[([^[\]]*(?:\[[^[\]]*\][^[\]]*)*)\])(?:\(\s*(?:(?:<((?:\\[^\n\r]|[^<>\n\r\\])*)>|((?!<)(?:\\[^\s]|\\(?=\s)|\((?:\\[^\s]|[^\s()\\]|\((?:\\[^\s]|[^\s()\\]|\((?:\\[^\s]|[^\s()\\])*\))*\))*\)|[^\s()\\])+))(?:\s+(?:"((?:\\[\s\S]|[^"\\])*)"|'((?:\\[\s\S]|[^'\\])*)'|\(((?:\\[\s\S]|[^()\\])*)\)))?\s*)?\))$"#; + +fn format_transformer( + format: &[&'static str], + tag: &str, + intraword: Option, +) -> Rc { + Rc::new(TextFormatTransformer { + format: format.to_vec(), + tag: tag.to_owned(), + intraword, + }) +} + +struct Builtins { + heading: Rc, + quote: Rc, + code: Rc, + unordered_list: Rc, + check_list: Rc, + ordered_list: Rc, + inline_code: Rc, + highlight: Rc, + bold_italic_star: Rc, + bold_italic_underscore: Rc, + bold_star: Rc, + bold_underscore: Rc, + strikethrough: Rc, + italic_star: Rc, + italic_underscore: Rc, + link: Rc, +} + +fn builtins() -> Builtins { + Builtins { + heading: Rc::new(ElementTransformer { + dependencies: vec!["heading"], + export: Rc::new(|node, export_children, _| { + if !is_heading_node(node) { + return Ok(None); + } + let level = node + .get_heading_tag() + .map_or(1, |t| t.as_str()[1..].parse::().unwrap_or(1)); + Ok(Some("#".repeat(level) + " " + &export_children(node)?)) + }), + reg_exp: re(HEADING_REGEX, ""), + replace: Rc::new(|parent, children, m, is_import| { + if !is_import && is_unreplaceable_block(parent) { + return Ok(false); + } + let tag = HeadingTagType::parse(&format!("h{}", utf16::len(m.str(1)))) + .expect("one to six #"); + create_block_node(create_heading_node(tag), parent, children, is_import) + }), + trigger_on_enter: true, + }), + quote: Rc::new(ElementTransformer { + dependencies: vec!["quote"], + export: Rc::new(|node, export_children, _| { + if !is_quote_node(node) { + return Ok(None); + } + let text = export_children(node)?; + Ok(Some( + text.split('\n') + .map(|line| format!("> {line}")) + .collect::>() + .join("\n"), + )) + }), + reg_exp: re(QUOTE_REGEX, ""), + replace: Rc::new(|parent, children, _m, is_import| { + if is_import + && let Some(previous) = parent.get_previous_sibling().filter(is_quote_node) + { + let mut nodes = vec![create_markdown_line_break_node(&previous)?]; + nodes.extend_from_slice(children); + previous.splice(previous.get_children_size(), 0, &nodes)?; + parent.remove(false)?; + return Ok(true); + } + create_block_node(create_quote_node(false), parent, children, is_import) + }), + trigger_on_enter: true, + }), + code: Rc::new(MultilineElementTransformer { + handle_import_after_start_match: Some(Rc::new(code_handle_import_after_start_match)), + dependencies: vec!["code"], + export: Some(Rc::new(|node, _, _| code_export(node))), + reg_exp_start: re(CODE_START_REGEX, ""), + reg_exp_end: Some(RegExpEnd { + optional: true, + reg_exp: re(CODE_END_REGEX, ""), + }), + replace: Rc::new(code_replace), + }), + unordered_list: Rc::new(list_transformer(ListType::Bullet, UNORDERED_LIST_REGEX)), + check_list: Rc::new(list_transformer(ListType::Check, CHECK_LIST_REGEX)), + ordered_list: Rc::new(list_transformer(ListType::Number, ORDERED_LIST_REGEX)), + inline_code: format_transformer(&["code"], "`", None), + highlight: format_transformer(&["highlight"], "==", None), + bold_italic_star: format_transformer(&["bold", "italic"], "***", None), + bold_italic_underscore: format_transformer(&["bold", "italic"], "___", Some(false)), + bold_star: format_transformer(&["bold"], "**", None), + bold_underscore: format_transformer(&["bold"], "__", Some(false)), + strikethrough: format_transformer(&["strikethrough"], "~~", None), + italic_star: format_transformer(&["italic"], "*", None), + italic_underscore: format_transformer(&["italic"], "_", Some(false)), + link: Rc::new(TextMatchTransformer { + dependencies: vec!["link"], + export: Some(Rc::new(|node, export_children, _| { + link_export(node, export_children) + })), + import_reg_exp: Some(re(LINK_IMPORT_REG_EXP, "")), + reg_exp: re(LINK_REG_EXP, ""), + replace: Some(Rc::new(link_replace)), + get_end_index: None, + trigger: Some(")".to_owned()), + }), + } +} + +thread_local! { + static BUILTINS: Builtins = builtins(); +} + +macro_rules! builtin { + ($(#[$doc:meta])* $name:ident: $ty:ty) => { + $(#[$doc])* + pub fn $name() -> Rc<$ty> { + BUILTINS.with(|b| b.$name.clone()) + } + }; +} + +builtin!( + /// `HEADING`: `# ` to `###### ` make a heading. + heading: ElementTransformer +); +builtin!( + /// `QUOTE`: `> ` makes a quote. + quote: ElementTransformer +); +builtin!( + /// `CODE`: a fenced code block. + code: MultilineElementTransformer +); +builtin!( + /// `UNORDERED_LIST`: `- `, `* ` or `+ `. + unordered_list: ElementTransformer +); +builtin!( + /// `CHECK_LIST`: `- [ ] ` or `- [x] `. Not in `TRANSFORMERS`. + check_list: ElementTransformer +); +builtin!( + /// `ORDERED_LIST`: `1. `. + ordered_list: ElementTransformer +); +builtin!( + /// `INLINE_CODE`: `` `code` ``. + inline_code: TextFormatTransformer +); +builtin!( + /// `HIGHLIGHT`: `==highlight==`. + highlight: TextFormatTransformer +); +builtin!( + /// `BOLD_ITALIC_STAR`: `***bold italic***`. + bold_italic_star: TextFormatTransformer +); +builtin!( + /// `BOLD_ITALIC_UNDERSCORE`: `___bold italic___`. + bold_italic_underscore: TextFormatTransformer +); +builtin!( + /// `BOLD_STAR`: `**bold**`. + bold_star: TextFormatTransformer +); +builtin!( + /// `BOLD_UNDERSCORE`: `__bold__`. + bold_underscore: TextFormatTransformer +); +builtin!( + /// `STRIKETHROUGH`: `~~strikethrough~~`. + strikethrough: TextFormatTransformer +); +builtin!( + /// `ITALIC_STAR`: `*italic*`. + italic_star: TextFormatTransformer +); +builtin!( + /// `ITALIC_UNDERSCORE`: `_italic_`. + italic_underscore: TextFormatTransformer +); +builtin!( + /// `LINK`: `[text](url "title")`. + link: TextMatchTransformer +); + +/// `ELEMENT_TRANSFORMERS`. +pub fn element_transformers() -> Vec> { + vec![heading(), quote(), unordered_list(), ordered_list()] +} + +/// `MULTILINE_ELEMENT_TRANSFORMERS`. +pub fn multiline_element_transformers() -> Vec> { + vec![code()] +} + +/// `TEXT_FORMAT_TRANSFORMERS`: code first, as it prevents any formats +/// inside, then longer tags before shorter ones. +pub fn text_format_transformers() -> Vec> { + vec![ + inline_code(), + bold_italic_star(), + bold_italic_underscore(), + bold_star(), + bold_underscore(), + highlight(), + italic_star(), + italic_underscore(), + strikethrough(), + ] +} + +/// `TEXT_MATCH_TRANSFORMERS`. +pub fn text_match_transformers() -> Vec> { + vec![link()] +} + +/// `TRANSFORMERS`: every transformer above but `CHECK_LIST`. +pub fn transformers() -> Vec { + let mut all: Vec = element_transformers().into_iter().map(Into::into).collect(); + all.extend(multiline_element_transformers().into_iter().map(Into::into)); + all.extend(text_format_transformers().into_iter().map(Into::into)); + all.extend(text_match_transformers().into_iter().map(Into::into)); + all +} + +/// `normalizeMarkdown(input, shouldMergeAdjacentLines)`. +pub fn normalize_markdown(input: &str, should_merge_adjacent_lines: bool) -> String { + let lines: Vec<&str> = input.split('\n').collect(); + let mut code_block_fence_length = 0; + let mut sanitized_lines: Vec = Vec::new(); + let test = |source: &'static str, flags: &'static str, text: &str| re(source, flags).test(text); + + for (i, raw_line) in lines.iter().enumerate() { + let raw_line = *raw_line; + let line = super::utils::trim_end(raw_line); + let last_line = sanitized_lines.last().cloned(); + let hard_line_break = if i < lines.len() - 1 { + parse_markdown_hard_line_break(raw_line) + } else { + None + }; + let last_line_has_hard_line_break = last_line + .as_deref() + .is_some_and(|l| parse_markdown_hard_line_break(l).is_some()); + + // Code blocks of ```single line``` don't toggle the code block. + if test(CODE_SINGLE_LINE_REGEX, "", line) { + sanitized_lines.push(line.to_owned()); + continue; + } + if code_block_fence_length == 0 { + if let Some(open_match) = re(CODE_START_REGEX, "").exec(line) { + code_block_fence_length = utf16::len(trim(open_match.str(1))); + sanitized_lines.push(line.to_owned()); + continue; + } + } else { + // Only a bare fence at least as long as the opening one closes. + if test(CODE_END_REGEX, "", line) && utf16::len(trim(line)) >= code_block_fence_length { + code_block_fence_length = 0; + sanitized_lines.push(line.to_owned()); + continue; + } + sanitized_lines.push(raw_line.to_owned()); + continue; + } + + let last = last_line.as_deref().unwrap_or(""); + if line.is_empty() + || last_line.as_deref().is_none_or(str::is_empty) + || test(HEADING_REGEX, "", last) + || test(HEADING_REGEX, "", line) + || test(QUOTE_REGEX, "", line) + || test(ORDERED_LIST_REGEX, "", line) + || test(UNORDERED_LIST_REGEX, "", line) + || test(CHECK_LIST_REGEX, "i", line) + || test(TABLE_ROW_REG_EXP, "", line) + || is_table_row_divider(line) + || last_line_has_hard_line_break + || !should_merge_adjacent_lines + || test(TAG_START_REGEX, "i", line) + || test(TAG_END_REGEX, "i", line) + || ends_with(TAG_END_REGEX, "i").test(last) + || ends_with(TAG_START_REGEX, "i").test(last) + || test(CODE_END_REGEX, "", last) + { + // When not merging, keep trailing whitespace (hard line break + // markers); whitespace-only lines still collapse to ''. + sanitized_lines.push( + if (!should_merge_adjacent_lines && !line.is_empty()) || hard_line_break.is_some() { + raw_line.to_owned() + } else { + line.to_owned() + }, + ); + } else { + let merged = last.to_owned() + + " " + + trim_start(if hard_line_break.is_none() { + line + } else { + raw_line + }); + *sanitized_lines.last_mut().expect("a last line") = merged; + } + } + sanitized_lines.join("\n") +} diff --git a/rust/lexical-editor/src/markdown/mod.rs b/rust/lexical-editor/src/markdown/mod.rs new file mode 100644 index 00000000000..6183b961c35 --- /dev/null +++ b/rust/lexical-editor/src/markdown/mod.rs @@ -0,0 +1,100 @@ +//! `@lexical/markdown` (the `markdown` feature), from +//! `packages/lexical-markdown/src`: markdown import +//! (`$convertFromMarkdownString`), export (`$convertToMarkdownString`, +//! `$convertSelectionToMarkdownString`), the transformers +//! (`MarkdownTransformers.ts`). +//! +//! Upstream's transformers are JS regular expressions, and custom +//! transformers bring their own; [`RegExp`] is a small JS regular +//! expression engine of this module's, so the feature needs no +//! third-party crate (rust/PARITY.md). +//! +//! Not ported: `$generateNodesFromMarkdownString`, which imports into an +//! `ArtificialNode__DO_NOT_USE` the core doesn't have yet. + +mod import_text_format_transformer; +mod import_text_match_transformer; +mod import_text_transformers; +mod markdown_export; +mod markdown_import; +mod markdown_transformers; +mod reg_exp; +mod reg_exp_tables; +mod utils; + +pub use markdown_import::import_markdown_nodes; +pub use markdown_transformers::{ + CodeSpanDelimiter, ElementExport, ElementReplace, ElementTransformer, ExportChildren, + ExportFormat, GetEndIndex, HandleImportAfterStartMatch, MultilineElementTransformer, + MultilineImportArgs, MultilineImportResult, MultilineReplace, RegExpEnd, TextFormatTransformer, + TextMatchExport, TextMatchReplace, TextMatchTransformer, Transformer, bold_italic_star, + bold_italic_underscore, bold_star, bold_underscore, check_list, code, code_fence_state, + code_meta_state, create_markdown_line_break_node, element_transformers, + get_code_span_delimiter, get_link_label_opener, hard_line_break_state, heading, highlight, + inline_code, is_table_row_divider, italic_star, italic_underscore, link, list_marker_state, + multiline_element_transformers, normalize_markdown, ordered_list, + parse_markdown_hard_line_break, quote, strikethrough, text_format_transformers, + text_match_transformers, transformers, unordered_list, +}; +pub use reg_exp::{RegExp, RegExpMatch}; + +use crate::{LexicalError, NodeRef, SelectionRef, get_root, get_selection}; + +/// `$convertFromMarkdownString(markdown, transformers, node, +/// shouldPreserveNewLines, shouldMergeAdjacentLines)`: replaces the +/// children of `node` (the root by default) with the markdown's nodes, and +/// moves a selection, if there is one, to the start. +/// `should_merge_adjacent_lines` merges adjacent non-empty lines as +/// CommonMark does, unless `should_preserve_new_lines`. +pub fn convert_from_markdown_string( + markdown: &str, + transformers: &[Transformer], + node: Option<&NodeRef>, + should_preserve_new_lines: bool, + should_merge_adjacent_lines: bool, +) -> Result<(), LexicalError> { + let sanitized_markdown = if should_preserve_new_lines { + markdown.to_owned() + } else { + normalize_markdown(markdown, should_merge_adjacent_lines) + }; + let root = node.cloned().unwrap_or_else(get_root); + root.clear()?; + import_markdown_nodes( + &sanitized_markdown, + &root, + transformers, + should_preserve_new_lines, + )?; + if get_selection().is_some() { + root.select_start()?; + } + Ok(()) +} + +/// `$convertToMarkdownString(transformers, node, shouldPreserveNewLines)`: +/// the markdown of `node`'s children (the root's by default). +pub fn convert_to_markdown_string( + transformers: &[Transformer], + node: Option<&NodeRef>, + should_preserve_new_lines: bool, +) -> Result { + markdown_export::export_markdown(transformers, should_preserve_new_lines, node) +} + +/// `$convertSelectionToMarkdownString(transformers, selection, +/// shouldPreserveNewLines)`: the markdown of the selected content; empty +/// for no selection or a collapsed range. +pub fn convert_selection_to_markdown_string( + transformers: &[Transformer], + selection: Option<&SelectionRef>, + should_preserve_new_lines: bool, +) -> Result { + let Some(selection) = selection else { + return Ok(String::new()); + }; + if selection.range().is_some_and(|r| r.is_collapsed()) { + return Ok(String::new()); + } + markdown_export::export_selection_markdown(transformers, should_preserve_new_lines, selection) +} diff --git a/rust/lexical-editor/src/markdown/reg_exp.rs b/rust/lexical-editor/src/markdown/reg_exp.rs new file mode 100644 index 00000000000..7697a48af86 --- /dev/null +++ b/rust/lexical-editor/src/markdown/reg_exp.rs @@ -0,0 +1,1633 @@ +//! JS regular expressions (`RegExp`), for the markdown feature. +//! +//! Upstream's transformers are written as JS `RegExp`s, and their sources +//! and flags are public API: custom transformers pass their own, and import +//! rebuilds them with `new RegExp(regExp.source, flags)`. The default build +//! has no third-party dependencies, so this is a small backtracking engine +//! that follows ECMA-262's pattern semantics (22.2.2) on the UTF-16 code +//! units of the text, as JS does: offsets are code units, a pattern without +//! the `u` flag reads a character outside the BMP as two, and one with it as +//! one code point. +//! +//! It implements the syntax the markdown package and its tests use, and the +//! rest of the common syntax: alternation, capturing, non-capturing and +//! named groups, lookahead and lookbehind, backreferences (numbered and +//! named), greedy and lazy quantifiers (with the spec's capture reset and +//! empty-iteration check), `^`, `$`, `\b`, `\B`, `.`, classes with ranges, +//! `\d \D \w \W \s \S`, `\p{..}` and `\P{..}` with `u` for the general +//! categories in `reg_exp_tables.rs` (generated from Node), and the flags +//! `d g i m s u y`. Not implemented: the `v` flag, and Annex B's legacy +//! octal escapes and `\c` inside classes. Two documented approximations +//! (rust/PARITY.md): with `u` and `i`, characters are folded with the +//! `Canonicalize` of `i` without `u` (`toUpperCase`), not Unicode simple +//! case folding; and a lookbehind is matched forwards from each earlier +//! position, which agrees with JS on whether it matches but may capture a +//! different substring inside it. + +use super::reg_exp_tables::{CANONICALIZE, LETTER, NUMBER, PUNCTUATION, SPACE_SEPARATOR, SYMBOL}; +use std::cell::Cell; +use std::fmt; +use std::rc::Rc; + +/// A `RegExp`: `/source/flags`. +#[derive(Clone)] +pub struct RegExp { + source: String, + flags: String, + program: Rc, + /// `regExp.lastIndex`, in UTF-16 code units, for `g` and `y`. + last_index: Cell, +} + +struct Program { + root: Node, + /// The number of capturing groups, without the whole match. + groups: usize, + names: Vec<(String, usize)>, + ignore_case: bool, + multiline: bool, + dot_all: bool, + unicode: bool, + global: bool, + sticky: bool, +} + +impl fmt::Debug for RegExp { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "/{}/{}", self.source, self.flags) + } +} + +impl PartialEq for RegExp { + /// The same source and flags. + fn eq(&self, other: &Self) -> bool { + self.source == other.source && self.flags == other.flags + } +} + +/// A `RegExpExecArray`: `[0]` the match, then each group (`None` for one +/// that did not participate, JS `undefined`), with `index` in UTF-16 code +/// units. Code that builds a match array by hand, as upstream does, builds +/// one of these. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct RegExpMatch { + pub groups: Vec>, + /// `match.indices`: the UTF-16 range of each group, when known. + pub ranges: Vec>, + /// The named groups, by name, with their group number. + pub names: Vec<(String, usize)>, + /// `match.index`. + pub index: usize, + /// `match.input`. + pub input: String, +} + +impl RegExpMatch { + /// A match array of `groups`, as `[a, b, c]` with `index` and `input` + /// set by hand. + pub fn new(groups: Vec>, index: usize, input: &str) -> Self { + Self { + groups, + ranges: Vec::new(), + names: Vec::new(), + index, + input: input.to_owned(), + } + } + + /// `match[i]`: `None` for `undefined`. + pub fn get(&self, i: usize) -> Option<&str> { + self.groups.get(i).and_then(|g| g.as_deref()) + } + + /// `match[i]`, with `undefined` as `''` (JS's `match[i] || ''`). + pub fn str(&self, i: usize) -> &str { + self.get(i).unwrap_or("") + } + + /// `match[0]`. + pub fn text(&self) -> &str { + self.str(0) + } + + /// `match.length`. + pub fn len(&self) -> usize { + self.groups.len() + } + + pub fn is_empty(&self) -> bool { + self.groups.is_empty() + } + + /// `match.groups[name]`. + pub fn named(&self, name: &str) -> Option<&str> { + let (_, i) = self.names.iter().find(|(n, _)| n == name)?; + self.get(*i) + } + + /// `match[0].length`, in UTF-16 code units. + pub fn match_len(&self) -> usize { + crate::utf16::len(self.text()) + } +} + +impl std::ops::Index for RegExpMatch { + type Output = str; + fn index(&self, i: usize) -> &str { + self.str(i) + } +} + +impl RegExp { + /// `new RegExp(source, flags)`, or the `SyntaxError` JS throws. + pub fn new(source: &str, flags: &str) -> Result { + let mut sorted: Vec = flags.chars().collect(); + sorted.sort_unstable(); + for (i, f) in sorted.iter().enumerate() { + if !"dgimsuy".contains(*f) || (i > 0 && sorted[i - 1] == *f) { + return Err(format!( + "Invalid flags supplied to RegExp constructor '{flags}'" + )); + } + } + let sorted_flags: String = sorted.iter().collect(); + let has = |c| sorted.contains(&c); + let unicode = has('u'); + let units: Vec = if unicode { + source.chars().map(|c| c as u32).collect() + } else { + source.encode_utf16().map(u32::from).collect() + }; + let mut parser = Parser { + src: &units, + pos: 0, + unicode, + groups: 0, + names: Vec::new(), + total_groups: count_groups(&units), + named_refs: Vec::new(), + }; + let root = parser.alternation()?; + if parser.pos < units.len() { + return Err(format!( + "Invalid regular expression: /{source}/: Unmatched ')'" + )); + } + let names = parser.names; + let mut root = root; + if !parser.named_refs.is_empty() { + resolve_named(&mut root, &names)?; + } + Ok(Self { + source: source.to_owned(), + flags: sorted_flags, + program: Rc::new(Program { + root, + groups: parser.groups, + names, + ignore_case: has('i'), + multiline: has('m'), + dot_all: has('s'), + unicode, + global: has('g'), + sticky: has('y'), + }), + last_index: Cell::new(0), + }) + } + + /// `/source/flags`, for a pattern known to be valid; panics otherwise. + pub fn of(source: &str, flags: &str) -> Self { + Self::new(source, flags) + .unwrap_or_else(|e| panic!("invalid regular expression /{source}/{flags}: {e}")) + } + + /// `regExp.source`. + pub fn source(&self) -> &str { + &self.source + } + + /// `regExp.flags`, in JS's order (`dgimsuy`). + pub fn flags(&self) -> &str { + &self.flags + } + + pub fn global(&self) -> bool { + self.program.global + } + + pub fn sticky(&self) -> bool { + self.program.sticky + } + + /// `regExp.lastIndex`. + pub fn last_index(&self) -> usize { + self.last_index.get() + } + + pub fn set_last_index(&self, index: usize) { + self.last_index.set(index); + } + + /// `regExp.exec(text)`: from `lastIndex` for a `g` or `y` regex, which + /// it then moves past the match (or back to 0). + pub fn exec(&self, text: &str) -> Option { + let units: Vec = text.encode_utf16().collect(); + self.exec_units(&units, text) + } + + fn exec_units(&self, units: &[u16], text: &str) -> Option { + let p = &self.program; + let uses_last_index = p.global || p.sticky; + let start = if uses_last_index { + self.last_index.get() + } else { + 0 + }; + let found = self.find_at(units, start); + match found { + None => { + if uses_last_index { + self.last_index.set(0); + } + None + } + Some(caps) => { + let (s, e) = caps[0].expect("the whole match"); + if uses_last_index { + self.last_index.set(e); + } + Some(self.to_match(units, text, caps, s)) + } + } + } + + fn to_match( + &self, + units: &[u16], + text: &str, + caps: Vec>, + index: usize, + ) -> RegExpMatch { + RegExpMatch { + groups: caps + .iter() + .map(|c| c.map(|(s, e)| String::from_utf16_lossy(&units[s..e]))) + .collect(), + ranges: caps, + names: self.program.names.clone(), + index, + input: text.to_owned(), + } + } + + /// The captures of the first match at or after `start` (only at it, for + /// a sticky regex), without touching `lastIndex`. + fn find_at(&self, units: &[u16], start: usize) -> Option>> { + let p = &self.program; + if start > units.len() { + return None; + } + let mut ctx = Ctx { + text: units, + p, + caps: vec![None; p.groups + 1], + }; + let mut s = start; + loop { + let mut end = None; + if m(&p.root, s, &mut ctx, &mut |e, _| { + end = Some(e); + true + }) { + ctx.caps[0] = Some((s, end.expect("an end"))); + return Some(ctx.caps); + } + if p.sticky || s >= units.len() { + return None; + } + s = advance(units, s, p.unicode); + for c in ctx.caps.iter_mut() { + *c = None; + } + } + } + + /// `regExp.test(text)`. + pub fn test(&self, text: &str) -> bool { + self.exec(text).is_some() + } + + /// The UTF-16 offset of the first match, as `text.search(regExp)` + /// (`None` for -1). + pub fn search(&self, text: &str) -> Option { + let units: Vec = text.encode_utf16().collect(); + self.find_at(&units, 0).map(|c| c[0].expect("a match").0) + } + + /// `text.match(regExp)` for a regex without `g`: the first match. + pub fn match_first(&self, text: &str) -> Option { + if self.program.global { + let units: Vec = text.encode_utf16().collect(); + return self.find_at(&units, 0).map(|c| { + let s = c[0].expect("a match").0; + self.to_match(&units, text, c, s) + }); + } + self.exec(text) + } + + /// `text.match(regExp)`: the first match (from `lastIndex` for a sticky + /// regex) without `g`; with it, every match's `[0]` as the groups of + /// one match array at index 0, as JS's array of strings reads. + pub fn string_match(&self, text: &str) -> Option { + if !self.program.global { + return self.exec(text); + } + let all = self.match_all(text); + self.last_index.set(0); + if all.is_empty() { + return None; + } + Some(RegExpMatch::new( + all.into_iter() + .map(|m| m.groups.into_iter().next().flatten()) + .collect(), + 0, + text, + )) + } + + /// Every match, as `text.matchAll(regExp)` or a `g` regex's + /// `text.match(regExp)` (whose strings are each match's `[0]`). + pub fn match_all(&self, text: &str) -> Vec { + let units: Vec = text.encode_utf16().collect(); + let mut out = Vec::new(); + let mut at = 0; + while at <= units.len() { + let Some(caps) = self.find_at(&units, at) else { + break; + }; + let (s, e) = caps[0].expect("a match"); + out.push(self.to_match(&units, text, caps, s)); + at = if e == s { + advance(&units, e, self.program.unicode) + } else { + e + }; + if self.program.sticky && out.last().is_some_and(|m| m.index != s) { + break; + } + } + out + } + + /// `text.replace(regExp, fn)`: every match for a `g` regex, else the + /// first, replaced by what `replacer` returns for it. + pub fn replace_with( + &self, + text: &str, + mut replacer: impl FnMut(&RegExpMatch) -> String, + ) -> String { + let units: Vec = text.encode_utf16().collect(); + let mut out: Vec = Vec::with_capacity(units.len()); + let mut from = 0; + let mut at = if self.program.global { + 0 + } else { + self.last_index_for_replace() + }; + while at <= units.len() { + let Some(caps) = self.find_at(&units, at) else { + break; + }; + let (s, e) = caps[0].expect("a match"); + let found = self.to_match(&units, text, caps, s); + out.extend_from_slice(&units[from..s]); + out.extend(replacer(&found).encode_utf16()); + from = e; + if !self.program.global { + break; + } + at = if e == s { + advance(&units, e, self.program.unicode) + } else { + e + }; + } + out.extend_from_slice(&units[from..]); + String::from_utf16_lossy(&out) + } + + fn last_index_for_replace(&self) -> usize { + if self.program.sticky { + self.last_index.get() + } else { + 0 + } + } + + /// `text.replace(regExp, template)`, with `$$`, `$&`, `` $` ``, `$'`, + /// `$n`, `$nn` and `$` (`GetSubstitution`). + pub fn replace(&self, text: &str, template: &str) -> String { + self.replace_with(text, |m| substitute(m, template)) + } + + /// `text.split(regExp)`, without a limit. A group that did not + /// participate in a separator's match, which JS splices in as + /// `undefined`, is left out. + pub fn split(&self, text: &str) -> Vec { + let units: Vec = text.encode_utf16().collect(); + if units.is_empty() { + return if self.find_at(&units, 0).is_some() { + Vec::new() + } else { + vec![String::new()] + }; + } + let mut out = Vec::new(); + let mut p = 0; + let mut q = p; + while q < units.len() { + let sticky = RegExp { + program: self.program.clone(), + source: self.source.clone(), + flags: self.flags.clone(), + last_index: Cell::new(0), + }; + match sticky.match_sticky(&units, q) { + Some(caps) if caps[0].expect("a match").1 != p => { + let e = caps[0].expect("a match").1; + out.push(String::from_utf16_lossy(&units[p..q])); + for (s, e) in caps.iter().skip(1).flatten() { + out.push(String::from_utf16_lossy(&units[*s..*e])); + } + p = e; + q = p; + } + _ => q = advance(&units, q, self.program.unicode), + } + } + out.push(String::from_utf16_lossy(&units[p..])); + out + } + + fn match_sticky(&self, units: &[u16], at: usize) -> Option>> { + let p = &self.program; + let mut ctx = Ctx { + text: units, + p, + caps: vec![None; p.groups + 1], + }; + let mut end = None; + if m(&p.root, at, &mut ctx, &mut |e, _| { + end = Some(e); + true + }) { + ctx.caps[0] = Some((at, end.expect("an end"))); + Some(ctx.caps) + } else { + None + } + } +} + +/// `GetSubstitution`. +fn substitute(m: &RegExpMatch, template: &str) -> String { + let t: Vec = template.chars().collect(); + let mut out = String::new(); + let mut i = 0; + let groups = m.len().saturating_sub(1); + while i < t.len() { + if t[i] != '$' || i + 1 == t.len() { + out.push(t[i]); + i += 1; + continue; + } + let c = t[i + 1]; + match c { + '$' => { + out.push('$'); + i += 2; + } + '&' => { + out.push_str(m.text()); + i += 2; + } + '`' => { + out.push_str(&crate::utf16::slice(&m.input, 0, m.index)); + i += 2; + } + '\'' => { + out.push_str(&crate::utf16::slice_from(&m.input, m.index + m.match_len())); + i += 2; + } + '0'..='9' => { + let d1 = c as usize - '0' as usize; + let two = t + .get(i + 2) + .filter(|d| d.is_ascii_digit()) + .map(|d| d1 * 10 + (*d as usize - '0' as usize)); + if let Some(n) = two.filter(|n| *n >= 1 && *n <= groups) { + out.push_str(m.str(n)); + i += 3; + } else if d1 >= 1 && d1 <= groups { + out.push_str(m.str(d1)); + i += 2; + } else { + out.push('$'); + i += 1; + } + } + '<' if !m.names.is_empty() => match t[i + 2..].iter().position(|c| *c == '>') { + Some(close) => { + let name: String = t[i + 2..i + 2 + close].iter().collect(); + out.push_str(m.named(&name).unwrap_or("")); + i += close + 3; + } + None => { + out.push('$'); + i += 1; + } + }, + _ => { + out.push('$'); + i += 1; + } + } + } + out +} + +/// `AdvanceStringIndex`. +fn advance(units: &[u16], i: usize, unicode: bool) -> usize { + if unicode && i + 1 < units.len() && is_high(units[i]) && is_low(units[i + 1]) { + i + 2 + } else { + i + 1 + } +} + +fn is_high(u: u16) -> bool { + (0xD800..0xDC00).contains(&u) +} + +fn is_low(u: u16) -> bool { + (0xDC00..0xE000).contains(&u) +} + +// --- Syntax ------------------------------------------------------------- + +#[derive(Debug)] +enum Node { + Empty, + Char(u32), + Any, + Class(Class), + Start, + End, + WordBoundary(bool), + Group(Box, Option), + Look { + node: Box, + ahead: bool, + negate: bool, + }, + BackRef(usize), + NamedBackRef(String), + Concat(Vec), + Alt(Vec), + Repeat { + node: Box, + min: u32, + max: Option, + greedy: bool, + /// The capturing groups inside, `first..last`. + groups: (usize, usize), + }, +} + +#[derive(Debug, Clone)] +enum Item { + Range(u32, u32), + Digit(bool), + Word(bool), + Space(bool), + Prop(Prop, bool), +} + +#[derive(Debug, Clone, Copy)] +enum Prop { + Table(&'static [(u32, u32)]), + Control, + Surrogate, + Noncharacter, + Any, + Ascii, +} + +#[derive(Debug)] +struct Class { + negate: bool, + items: Vec, +} + +struct Parser<'a> { + src: &'a [u32], + pos: usize, + unicode: bool, + groups: usize, + names: Vec<(String, usize)>, + total_groups: usize, + named_refs: Vec, +} + +/// The number of capturing groups in a pattern, for reading `\N`. +fn count_groups(src: &[u32]) -> usize { + let mut n = 0; + let mut i = 0; + let mut in_class = false; + while i < src.len() { + let c = src[i]; + if c == '\\' as u32 { + i += 2; + continue; + } + if in_class { + in_class = c != ']' as u32; + } else if c == '[' as u32 { + in_class = true; + } else if c == '(' as u32 { + let next = src.get(i + 1).copied(); + let after = src.get(i + 2).copied(); + if next != Some('?' as u32) + || (after == Some('<' as u32) + && !matches!( + src.get(i + 3).map(|c| char::from_u32(*c)), + Some(Some('=' | '!')) + )) + { + n += 1; + } + } + i += 1; + } + n +} + +fn resolve_named(node: &mut Node, names: &[(String, usize)]) -> Result<(), String> { + match node { + Node::NamedBackRef(name) => { + let (_, i) = names + .iter() + .find(|(n, _)| n == name) + .ok_or_else(|| "Invalid named capture referenced".to_owned())?; + *node = Node::BackRef(*i); + } + Node::Group(n, _) | Node::Look { node: n, .. } | Node::Repeat { node: n, .. } => { + resolve_named(n, names)? + } + Node::Concat(ns) | Node::Alt(ns) => { + for n in ns { + resolve_named(n, names)?; + } + } + _ => {} + } + Ok(()) +} + +fn err(message: &str) -> Result { + Err(format!("Invalid regular expression: {message}")) +} + +impl Parser<'_> { + fn peek(&self) -> Option { + self.src.get(self.pos).copied() + } + + fn peek_is(&self, c: char) -> bool { + self.peek() == Some(c as u32) + } + + fn eat(&mut self, c: char) -> bool { + if self.peek_is(c) { + self.pos += 1; + true + } else { + false + } + } + + fn alternation(&mut self) -> Result { + let mut alts = vec![self.sequence()?]; + while self.eat('|') { + alts.push(self.sequence()?); + } + Ok(if alts.len() == 1 { + alts.pop().expect("one") + } else { + Node::Alt(alts) + }) + } + + fn sequence(&mut self) -> Result { + let mut items = Vec::new(); + while let Some(c) = self.peek() { + if c == '|' as u32 || c == ')' as u32 { + break; + } + let groups_before = self.groups; + let atom = self.atom()?; + let quantifiable = !matches!( + atom, + Node::Start | Node::End | Node::WordBoundary(_) | Node::Look { ahead: false, .. } + ) && !(self.unicode && matches!(atom, Node::Look { .. })); + let atom = self.quantifier(atom, groups_before, quantifiable)?; + items.push(atom); + } + Ok(match items.len() { + 0 => Node::Empty, + 1 => items.pop().expect("one"), + _ => Node::Concat(items), + }) + } + + /// `{n}`, `{n,}` or `{n,m}` at the current position, if there is one. + fn braces(&mut self) -> Option<(u32, Option)> { + let start = self.pos; + if !self.eat('{') { + return None; + } + let number = |p: &mut Self| -> Option { + let s = p.pos; + let mut v: u64 = 0; + while let Some(d) = p.peek().filter(|d| (b'0' as u32..=b'9' as u32).contains(d)) { + v = (v * 10 + u64::from(d - b'0' as u32)).min(u64::from(u32::MAX)); + p.pos += 1; + } + (p.pos > s).then_some(v as u32) + }; + let Some(min) = number(self) else { + self.pos = start; + return None; + }; + let max = if self.eat(',') { + if self.peek_is('}') { + None + } else { + match number(self) { + Some(m) => Some(m), + None => { + self.pos = start; + return None; + } + } + } + } else { + Some(min) + }; + if !self.eat('}') { + self.pos = start; + return None; + } + Some((min, max)) + } + + fn quantifier( + &mut self, + atom: Node, + groups_before: usize, + quantifiable: bool, + ) -> Result { + let (min, max) = match self.peek().and_then(char::from_u32) { + Some('*') => { + self.pos += 1; + (0, None) + } + Some('+') => { + self.pos += 1; + (1, None) + } + Some('?') => { + self.pos += 1; + (0, Some(1)) + } + Some('{') => match self.braces() { + Some(q) => q, + None if self.unicode => return err("Incomplete quantifier"), + None => return Ok(atom), + }, + _ => return Ok(atom), + }; + if !quantifiable { + return err("Nothing to repeat"); + } + if max.is_some_and(|m| m < min) { + return err("numbers out of order in {} quantifier"); + } + let greedy = !self.eat('?'); + Ok(Node::Repeat { + node: Box::new(atom), + min, + max, + greedy, + groups: (groups_before + 1, self.groups + 1), + }) + } + + fn atom(&mut self) -> Result { + let c = self.peek().expect("an atom"); + self.pos += 1; + Ok(match char::from_u32(c) { + Some('^') => Node::Start, + Some('$') => Node::End, + Some('.') => Node::Any, + Some('*' | '+' | '?') => return err("Nothing to repeat"), + Some('{') => { + self.pos -= 1; + if self.unicode || self.braces().is_some() { + return err("Nothing to repeat"); + } + self.pos += 1; + Node::Char(c) + } + Some('}' | ']') if self.unicode => return err("Lone quantifier brackets"), + Some('(') => self.group()?, + Some('[') => Node::Class(self.class()?), + Some('\\') => self.atom_escape()?, + _ => { + if !self.unicode && is_high(c as u16) && false { + unreachable!() + } + Node::Char(c) + } + }) + } + + fn group(&mut self) -> Result { + let node = if self.eat('?') { + if self.eat(':') { + Node::Group(Box::new(self.alternation()?), None) + } else if self.eat('=') { + look(self.alternation()?, true, false) + } else if self.eat('!') { + look(self.alternation()?, true, true) + } else if self.eat('<') { + if self.eat('=') { + look(self.alternation()?, false, false) + } else if self.eat('!') { + look(self.alternation()?, false, true) + } else { + let name = self.group_name()?; + if self.names.iter().any(|(n, _)| *n == name) { + return err("Duplicate capture group name"); + } + self.groups += 1; + let index = self.groups; + self.names.push((name, index)); + Node::Group(Box::new(self.alternation()?), Some(index)) + } + } else { + return err("Invalid group"); + } + } else { + self.groups += 1; + let index = self.groups; + Node::Group(Box::new(self.alternation()?), Some(index)) + }; + if !self.eat(')') { + return err("Unterminated group"); + } + Ok(node) + } + + fn group_name(&mut self) -> Result { + let mut name = String::new(); + loop { + match self.peek().and_then(char::from_u32) { + Some('>') => { + self.pos += 1; + break; + } + Some(c) if c == '$' || c == '_' || c.is_alphanumeric() => { + name.push(c); + self.pos += 1; + } + _ => return err("Invalid capture group name"), + } + } + if name.is_empty() { + return err("Invalid capture group name"); + } + Ok(name) + } + + fn atom_escape(&mut self) -> Result { + let Some(c) = self.peek() else { + return err("\\ at end of pattern"); + }; + match char::from_u32(c) { + Some('b') => { + self.pos += 1; + Ok(Node::WordBoundary(true)) + } + Some('B') => { + self.pos += 1; + Ok(Node::WordBoundary(false)) + } + Some('1'..='9') => { + let start = self.pos; + let mut n = 0usize; + while let Some(d) = self + .peek() + .filter(|d| (b'0' as u32..=b'9' as u32).contains(d)) + { + n = n + .saturating_mul(10) + .saturating_add((d - b'0' as u32) as usize); + self.pos += 1; + } + if n <= self.total_groups { + Ok(Node::BackRef(n)) + } else if self.unicode { + err("Invalid escape") + } else { + // Annex B: an identity escape of the first digit. + self.pos = start + 1; + Ok(Node::Char(c)) + } + } + Some('k') if self.unicode || !self.names_absent() => { + self.pos += 1; + if !self.eat('<') { + return err("Invalid named reference"); + } + let name = self.group_name()?; + self.named_refs.push(name.clone()); + Ok(Node::NamedBackRef(name)) + } + _ => match self.class_escape()? { + Item::Range(a, b) if a == b => Ok(Node::Char(a)), + item => Ok(Node::Class(Class { + negate: false, + items: vec![item], + })), + }, + } + } + + /// Whether the pattern has no named groups, so that `\k` is an identity + /// escape (Annex B). + fn names_absent(&self) -> bool { + let mut i = 0; + while i + 2 < self.src.len() { + if self.src[i] == '(' as u32 + && self.src[i + 1] == '?' as u32 + && self.src[i + 2] == '<' as u32 + && !matches!( + self.src.get(i + 3).and_then(|c| char::from_u32(*c)), + Some('=' | '!') + ) + { + return false; + } + i += 1; + } + true + } + + /// An escape after `\` that stands for characters, inside or outside a + /// class. + fn class_escape(&mut self) -> Result { + let Some(c) = self.peek() else { + return err("\\ at end of pattern"); + }; + self.pos += 1; + let ch = |v: u32| Ok(Item::Range(v, v)); + match char::from_u32(c) { + Some('d') => Ok(Item::Digit(false)), + Some('D') => Ok(Item::Digit(true)), + Some('w') => Ok(Item::Word(false)), + Some('W') => Ok(Item::Word(true)), + Some('s') => Ok(Item::Space(false)), + Some('S') => Ok(Item::Space(true)), + Some('t') => ch(9), + Some('n') => ch(10), + Some('v') => ch(11), + Some('f') => ch(12), + Some('r') => ch(13), + Some('0') + if !self + .peek() + .is_some_and(|d| (b'0' as u32..=b'9' as u32).contains(&d)) => + { + ch(0) + } + Some('c') => match self.peek().and_then(char::from_u32) { + Some(l) if l.is_ascii_alphabetic() => { + self.pos += 1; + ch(l as u32 % 32) + } + _ if self.unicode => err("Invalid unicode escape"), + _ => { + self.pos -= 1; + ch('\\' as u32) + } + }, + Some('x') => match self.hex(2) { + Some(v) => ch(v), + None if self.unicode => err("Invalid escape"), + None => ch('x' as u32), + }, + Some('u') => { + if self.unicode && self.eat('{') { + let start = self.pos; + let mut v: u32 = 0; + while let Some(d) = self + .peek() + .and_then(char::from_u32) + .and_then(|c| c.to_digit(16)) + { + v = v.saturating_mul(16).saturating_add(d); + self.pos += 1; + } + if self.pos == start || !self.eat('}') || v > 0x10FFFF { + return err("Invalid Unicode escape"); + } + return ch(v); + } + match self.hex(4) { + Some(v) => { + if self.unicode && is_high(v as u16) && v < 0x10000 { + let save = self.pos; + if self.eat('\\') + && self.eat('u') + && let Some(lo) = self.hex(4) + && is_low(lo as u16) + { + return ch(0x10000 + ((v - 0xD800) << 10) + (lo - 0xDC00)); + } + self.pos = save; + } + ch(v) + } + None if self.unicode => err("Invalid Unicode escape"), + None => ch('u' as u32), + } + } + Some(p @ ('p' | 'P')) if self.unicode => { + if !self.eat('{') { + return err("Invalid property name"); + } + let mut name = String::new(); + while let Some(c) = self.peek().and_then(char::from_u32) { + self.pos += 1; + if c == '}' { + let prop = property(&name).ok_or_else(|| { + format!( + "Invalid regular expression: Invalid property name \\{p}{{{name}}}" + ) + })?; + return Ok(Item::Prop(prop, p == 'P')); + } + name.push(c); + } + err("Invalid property name") + } + Some(c) if self.unicode && !"^$\\.*+?()[]{}|/-".contains(c) => err("Invalid escape"), + _ => ch(c), + } + } + + fn hex(&mut self, n: usize) -> Option { + let mut v = 0; + for i in 0..n { + let d = self + .src + .get(self.pos + i) + .and_then(|c| char::from_u32(*c)) + .and_then(|c| c.to_digit(16))?; + v = v * 16 + d; + } + self.pos += n; + Some(v) + } + + fn class(&mut self) -> Result { + let negate = self.eat('^'); + let mut items = Vec::new(); + loop { + let Some(c) = self.peek() else { + return err("Unterminated character class"); + }; + if c == ']' as u32 { + self.pos += 1; + return Ok(Class { negate, items }); + } + let first = self.class_atom()?; + if self.peek_is('-') + && self.src.get(self.pos + 1) != Some(&(']' as u32)) + && self.src.get(self.pos + 1).is_some() + { + self.pos += 1; + let second = self.class_atom()?; + match (&first, &second) { + (Item::Range(a, _), Item::Range(b, _)) => { + if a > b { + return err("Range out of order in character class"); + } + items.push(Item::Range(*a, *b)); + } + _ if self.unicode => return err("Invalid character class"), + _ => { + items.push(first); + items.push(Item::Range('-' as u32, '-' as u32)); + items.push(second); + } + } + } else { + items.push(first); + } + } + } + + fn class_atom(&mut self) -> Result { + let c = self.peek().expect("a class atom"); + self.pos += 1; + if c != '\\' as u32 { + return Ok(Item::Range(c, c)); + } + match self.peek().and_then(char::from_u32) { + Some('b') => { + self.pos += 1; + Ok(Item::Range(8, 8)) + } + Some('-') if self.unicode => { + self.pos += 1; + Ok(Item::Range('-' as u32, '-' as u32)) + } + Some(d @ '1'..='9') if !self.unicode => { + self.pos += 1; + Ok(Item::Range(d as u32, d as u32)) + } + _ => self.class_escape(), + } + } +} + +fn look(node: Node, ahead: bool, negate: bool) -> Node { + Node::Look { + node: Box::new(node), + ahead, + negate, + } +} + +/// The general categories and binary properties `\p{..}` knows. +fn property(name: &str) -> Option { + let name = name + .strip_prefix("General_Category=") + .or_else(|| name.strip_prefix("gc=")) + .unwrap_or(name); + Some(match name { + "L" | "Letter" => Prop::Table(LETTER), + "N" | "Number" => Prop::Table(NUMBER), + "P" | "Punctuation" | "punct" => Prop::Table(PUNCTUATION), + "S" | "Symbol" => Prop::Table(SYMBOL), + "Zs" | "Space_Separator" => Prop::Table(SPACE_SEPARATOR), + "Cc" | "Control" | "cntrl" => Prop::Control, + "Cs" | "Surrogate" => Prop::Surrogate, + "Noncharacter_Code_Point" | "NChar" => Prop::Noncharacter, + "Any" => Prop::Any, + "ASCII" => Prop::Ascii, + _ => return None, + }) +} + +// --- Matching ----------------------------------------------------------- + +struct Ctx<'a> { + text: &'a [u16], + p: &'a Program, + caps: Vec>, +} + +type Cont<'k> = dyn FnMut(usize, &mut Ctx) -> bool + 'k; + +/// The character at `pos` and its length in code units: a code point for a +/// `u` regex, else a code unit. +fn char_at(ctx: &Ctx, pos: usize) -> Option<(u32, usize)> { + let t = ctx.text; + let u = *t.get(pos)?; + if ctx.p.unicode && is_high(u) && pos + 1 < t.len() && is_low(t[pos + 1]) { + let lo = t[pos + 1]; + return Some(( + 0x10000 + ((u32::from(u) - 0xD800) << 10) + (u32::from(lo) - 0xDC00), + 2, + )); + } + Some((u32::from(u), 1)) +} + +fn is_line_terminator(c: u32) -> bool { + matches!(c, 0x0A | 0x0D | 0x2028 | 0x2029) +} + +/// JS's `\s`: WhiteSpace and LineTerminator. +pub(crate) fn is_js_space(c: u32) -> bool { + matches!( + c, + 0x09..=0x0D | 0x20 | 0xA0 | 0x1680 | 0x2000..=0x200A | 0x2028 | 0x2029 | 0x202F | 0x205F | 0x3000 | 0xFEFF + ) +} + +fn is_word(c: u32) -> bool { + matches!(c, 0x30..=0x39 | 0x41..=0x5A | 0x61..=0x7A | 0x5F) +} + +pub(crate) fn in_table(table: &[(u32, u32)], c: u32) -> bool { + table + .binary_search_by(|&(a, b)| { + if c < a { + std::cmp::Ordering::Greater + } else if c > b { + std::cmp::Ordering::Less + } else { + std::cmp::Ordering::Equal + } + }) + .is_ok() +} + +fn prop_has(p: Prop, c: u32) -> bool { + match p { + Prop::Table(t) => in_table(t, c), + Prop::Control => c < 0x20 || (0x7F..=0x9F).contains(&c), + Prop::Surrogate => (0xD800..=0xDFFF).contains(&c), + Prop::Noncharacter => (0xFDD0..=0xFDEF).contains(&c) || (c & 0xFFFE) == 0xFFFE, + Prop::Any => true, + Prop::Ascii => c < 0x80, + } +} + +/// `Canonicalize` without `u`: `toUpperCase` when it is one code unit and +/// does not map a non-ASCII unit to ASCII. +fn canonicalize(c: u32) -> u32 { + if c > 0xFFFF { + return c; + } + match CANONICALIZE.binary_search_by_key(&(c as u16), |&(a, _)| a) { + Ok(i) => u32::from(CANONICALIZE[i].1), + Err(_) => c, + } +} + +/// The characters that canonicalize as `c` does, `c` among them. +fn case_variants(c: u32) -> Vec { + let canon = canonicalize(c); + let mut out = vec![canon]; + for &(a, b) in CANONICALIZE { + if u32::from(b) == canon && u32::from(a) != canon { + out.push(u32::from(a)); + } + } + out +} + +fn item_has(item: &Item, c: u32) -> bool { + match item { + Item::Range(a, b) => (*a..=*b).contains(&c), + Item::Digit(neg) => (0x30..=0x39).contains(&c) != *neg, + Item::Word(neg) => is_word(c) != *neg, + Item::Space(neg) => is_js_space(c) != *neg, + Item::Prop(p, neg) => prop_has(*p, c) != *neg, + } +} + +fn class_has(class: &Class, c: u32, ignore_case: bool) -> bool { + let found = if ignore_case { + case_variants(c) + .into_iter() + .chain(std::iter::once(c)) + .any(|v| class.items.iter().any(|i| item_has(i, v))) + } else { + class.items.iter().any(|i| item_has(i, c)) + }; + found != class.negate +} + +/// Whether the character `c` matches the single-character `node`. +fn single_matches(node: &Node, c: u32, p: &Program) -> bool { + match node { + Node::Char(x) => *x == c || (p.ignore_case && canonicalize(*x) == canonicalize(c)), + Node::Any => p.dot_all || !is_line_terminator(c), + Node::Class(class) => class_has(class, c, p.ignore_case), + _ => unreachable!("not a single character"), + } +} + +fn is_single(node: &Node) -> bool { + matches!(node, Node::Char(_) | Node::Any | Node::Class(_)) +} + +fn word_at(ctx: &Ctx, pos: isize) -> bool { + if pos < 0 || pos as usize >= ctx.text.len() { + return false; + } + let c = u32::from(ctx.text[pos as usize]); + is_word(c) || (ctx.p.ignore_case && ctx.p.unicode && matches!(c, 0x017F | 0x212A)) +} + +/// Matches `node` at `pos`, then calls `k` with where it ends; backtracks +/// into `node` while `k` fails. Restores the captures it set when it fails. +fn m(node: &Node, pos: usize, ctx: &mut Ctx, k: &mut Cont) -> bool { + match node { + Node::Empty => k(pos, ctx), + Node::Char(_) | Node::Any | Node::Class(_) => match char_at(ctx, pos) { + Some((c, len)) if single_matches(node, c, ctx.p) => k(pos + len, ctx), + _ => false, + }, + Node::Start => { + (pos == 0 || (ctx.p.multiline && is_line_terminator(u32::from(ctx.text[pos - 1])))) + && k(pos, ctx) + } + Node::End => { + (pos == ctx.text.len() + || (ctx.p.multiline && is_line_terminator(u32::from(ctx.text[pos])))) + && k(pos, ctx) + } + Node::WordBoundary(is) => { + let a = word_at(ctx, pos as isize - 1); + let b = word_at(ctx, pos as isize); + ((a != b) == *is) && k(pos, ctx) + } + Node::Group(inner, index) => match index { + None => m(inner, pos, ctx, k), + Some(g) => { + let g = *g; + m(inner, pos, ctx, &mut |e, c: &mut Ctx| { + let old = c.caps[g]; + c.caps[g] = Some((pos, e)); + if k(e, c) { + true + } else { + c.caps[g] = old; + false + } + }) + } + }, + Node::Look { + node: inner, + ahead, + negate, + } => { + let saved = ctx.caps.clone(); + let found = if *ahead { + m(inner, pos, ctx, &mut |_, _| true) + } else { + (0..=pos) + .rev() + .any(|start| m(inner, start, ctx, &mut |e, _| e == pos)) + }; + if *negate { + ctx.caps = saved; + !found && k(pos, ctx) + } else if !found { + ctx.caps = saved; + false + } else if k(pos, ctx) { + true + } else { + ctx.caps = saved; + false + } + } + Node::BackRef(g) => { + let Some((s, e)) = ctx.caps.get(*g).copied().flatten() else { + return k(pos, ctx); + }; + let len = e - s; + if pos + len > ctx.text.len() { + return false; + } + let same = (0..len).all(|i| { + let a = u32::from(ctx.text[s + i]); + let b = u32::from(ctx.text[pos + i]); + a == b || (ctx.p.ignore_case && canonicalize(a) == canonicalize(b)) + }); + same && k(pos + len, ctx) + } + Node::NamedBackRef(_) => unreachable!("resolved when compiled"), + Node::Concat(nodes) => seq(nodes, pos, ctx, k), + Node::Alt(alts) => alts.iter().any(|alt| m(alt, pos, ctx, &mut *k)), + Node::Repeat { + node: inner, + min, + max, + greedy, + groups, + } => { + if is_single(inner) { + repeat_single(inner, *min, *max, *greedy, pos, ctx, k) + } else { + repeat(inner, *min, *max, *greedy, *groups, pos, ctx, k) + } + } + } +} + +fn seq(nodes: &[Node], pos: usize, ctx: &mut Ctx, k: &mut Cont) -> bool { + match nodes { + [] => k(pos, ctx), + [last] => m(last, pos, ctx, k), + [first, rest @ ..] => m(first, pos, ctx, &mut |e, c: &mut Ctx| { + seq(rest, e, c, &mut *k) + }), + } +} + +/// A repeated single character, without recursion per iteration. +fn repeat_single( + node: &Node, + min: u32, + max: Option, + greedy: bool, + pos: usize, + ctx: &mut Ctx, + k: &mut Cont, +) -> bool { + let max = max.map_or(usize::MAX, |m| m as usize); + let min = min as usize; + let mut ends = vec![pos]; + let mut at = pos; + while ends.len() - 1 < max { + if !greedy && ends.len() > min && k(at, ctx) { + return true; + } + match char_at(ctx, at) { + Some((c, len)) if single_matches(node, c, ctx.p) => { + at += len; + ends.push(at); + } + _ => break, + } + } + if !greedy { + return ends.len() - 1 == max && ends.len() > min && k(at, ctx); + } + if ends.len() - 1 < min { + return false; + } + for &e in ends[min..].iter().rev() { + if k(e, ctx) { + return true; + } + } + false +} + +/// `RepeatMatcher` (ECMA-262 22.2.2.3.1). +#[allow(clippy::too_many_arguments)] +fn repeat( + node: &Node, + min: u32, + max: Option, + greedy: bool, + groups: (usize, usize), + pos: usize, + ctx: &mut Ctx, + k: &mut Cont, +) -> bool { + if max == Some(0) { + return k(pos, ctx); + } + let saved: Vec> = ctx.caps[groups.0..groups.1].to_vec(); + let once = |ctx: &mut Ctx, k: &mut Cont| -> bool { + for c in &mut ctx.caps[groups.0..groups.1] { + *c = None; + } + let found = m(node, pos, ctx, &mut |e, c: &mut Ctx| { + if min == 0 && e == pos { + return false; + } + repeat( + node, + min.saturating_sub(1), + max.map(|m| m - 1), + greedy, + groups, + e, + c, + &mut *k, + ) + }); + if !found { + ctx.caps[groups.0..groups.1].copy_from_slice(&saved); + } + found + }; + if min > 0 { + return once(ctx, k); + } + if !greedy { + if k(pos, ctx) { + return true; + } + return once(ctx, k); + } + if once(ctx, &mut *k) { + return true; + } + k(pos, ctx) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn exec(source: &str, flags: &str, text: &str) -> Option>> { + RegExp::of(source, flags).exec(text).map(|m| m.groups) + } + + fn s(v: &[Option<&str>]) -> Option>> { + Some(v.iter().map(|g| g.map(str::to_owned)).collect()) + } + + // Expected values are what Node 24 prints for each `exec`. + #[test] + fn matches_as_js_does() { + assert_eq!( + exec("^(#{1,6})\\s", "", "## a"), + s(&[Some("## "), Some("##")]) + ); + assert_eq!(exec("(a)|b", "", "b"), s(&[Some("b"), None])); + assert_eq!( + exec("(z)((a+)?(b+)?(c))*", "", "zaacbbbcac"), + s(&[ + Some("zaacbbbcac"), + Some("z"), + Some("ac"), + Some("a"), + None, + Some("c") + ]) + ); + assert_eq!(exec("(a*)*", "", "b"), s(&[Some(""), None])); + assert_eq!(exec("(a*)b\\1+", "", "baaaac"), s(&[Some("b"), Some("")])); + assert_eq!( + exec("(?=(a+))a*b\\1", "", "baaabac"), + s(&[Some("aba"), Some("a")]) + ); + assert_eq!( + exec("(.*?)a(?!(a+)b\\2c)\\2(.*)", "", "baaabaac"), + s(&[Some("baaabaac"), Some("ba"), None, Some("abaac")]) + ); + assert_eq!( + exec("\\$([^$]+?)\\$", "", "x $a$ $b$"), + s(&[Some("$a$"), Some("a")]) + ); + assert_eq!( + exec( + "^(\\s*)(?:[-*+]\\s)?\\s?(\\[(\\s|x)?\\])\\s", + "i", + "- [X] a" + ), + s(&[Some("- [X] "), Some(""), Some("[X]"), Some("X")]) + ); + assert_eq!(exec("a{,2}", "", "a{,2}"), s(&[Some("a{,2}")])); + assert_eq!(exec("\\bfoo\\b", "", "a foo."), s(&[Some("foo")])); + assert_eq!(exec("(?<=\\$)\\d+", "", "a$12"), s(&[Some("12")])); + assert_eq!( + exec("(?\\d{4})-\\k", "", "2020-2020"), + s(&[Some("2020-2020"), Some("2020")]) + ); + assert_eq!(exec("[\\p{P}\\p{S}]", "u", "a€"), s(&[Some("€")])); + assert_eq!(exec("^.$", "", "😀"), None); + assert_eq!(exec("^.$", "u", "😀"), s(&[Some("😀")])); + assert_eq!(exec("ſ", "i", "s"), None); + assert_eq!(exec("[a-z]+", "i", "ABC"), s(&[Some("ABC")])); + } + + #[test] + fn global_flags_and_strings() { + let re = RegExp::of("a", "g"); + assert!(re.test("aa")); + assert_eq!(re.last_index(), 1); + assert!(re.test("aa")); + assert!(!re.test("aa")); + assert_eq!(re.last_index(), 0); + assert_eq!( + RegExp::of("(\\*|\\^|\\+)", "g").replace("**^x", "\\$1"), + "\\*\\*\\^x" + ); + assert_eq!(RegExp::of("x*", "g").replace("abc", "-"), "-a-b-c-"); + assert_eq!(RegExp::of("😀", "u").search("a😀"), Some(1)); + assert_eq!(RegExp::of(",", "").split("a,b,,c"), ["a", "b", "", "c"]); + assert_eq!(RegExp::of("\\s+", "").split(""), [""]); + assert!(RegExp::new("(", "").is_err()); + assert!(RegExp::new("a", "gg").is_err()); + assert_eq!(RegExp::of("a", "yig").flags(), "giy"); + } +} diff --git a/rust/lexical-editor/src/markdown/reg_exp_tables.rs b/rust/lexical-editor/src/markdown/reg_exp_tables.rs new file mode 100644 index 00000000000..73bb3dea502 --- /dev/null +++ b/rust/lexical-editor/src/markdown/reg_exp_tables.rs @@ -0,0 +1,2475 @@ +//! @generated by `rust/scripts/generate-reg-exp-tables.mjs` from Node +//! v24.21.0's regular expressions and `toUpperCase`. Do not edit +//! by hand; run the script again with the Node in `.nvmrc`. + +/// `\p{P}`: General_Category Punctuation. +pub(super) const PUNCTUATION: &[(u32, u32)] = &[ + (0x0021, 0x0023), + (0x0025, 0x002A), + (0x002C, 0x002F), + (0x003A, 0x003B), + (0x003F, 0x0040), + (0x005B, 0x005D), + (0x005F, 0x005F), + (0x007B, 0x007B), + (0x007D, 0x007D), + (0x00A1, 0x00A1), + (0x00A7, 0x00A7), + (0x00AB, 0x00AB), + (0x00B6, 0x00B7), + (0x00BB, 0x00BB), + (0x00BF, 0x00BF), + (0x037E, 0x037E), + (0x0387, 0x0387), + (0x055A, 0x055F), + (0x0589, 0x058A), + (0x05BE, 0x05BE), + (0x05C0, 0x05C0), + (0x05C3, 0x05C3), + (0x05C6, 0x05C6), + (0x05F3, 0x05F4), + (0x0609, 0x060A), + (0x060C, 0x060D), + (0x061B, 0x061B), + (0x061D, 0x061F), + (0x066A, 0x066D), + (0x06D4, 0x06D4), + (0x0700, 0x070D), + (0x07F7, 0x07F9), + (0x0830, 0x083E), + (0x085E, 0x085E), + (0x0964, 0x0965), + (0x0970, 0x0970), + (0x09FD, 0x09FD), + (0x0A76, 0x0A76), + (0x0AF0, 0x0AF0), + (0x0C77, 0x0C77), + (0x0C84, 0x0C84), + (0x0DF4, 0x0DF4), + (0x0E4F, 0x0E4F), + (0x0E5A, 0x0E5B), + (0x0F04, 0x0F12), + (0x0F14, 0x0F14), + (0x0F3A, 0x0F3D), + (0x0F85, 0x0F85), + (0x0FD0, 0x0FD4), + (0x0FD9, 0x0FDA), + (0x104A, 0x104F), + (0x10FB, 0x10FB), + (0x1360, 0x1368), + (0x1400, 0x1400), + (0x166E, 0x166E), + (0x169B, 0x169C), + (0x16EB, 0x16ED), + (0x1735, 0x1736), + (0x17D4, 0x17D6), + (0x17D8, 0x17DA), + (0x1800, 0x180A), + (0x1944, 0x1945), + (0x1A1E, 0x1A1F), + (0x1AA0, 0x1AA6), + (0x1AA8, 0x1AAD), + (0x1B4E, 0x1B4F), + (0x1B5A, 0x1B60), + (0x1B7D, 0x1B7F), + (0x1BFC, 0x1BFF), + (0x1C3B, 0x1C3F), + (0x1C7E, 0x1C7F), + (0x1CC0, 0x1CC7), + (0x1CD3, 0x1CD3), + (0x2010, 0x2027), + (0x2030, 0x2043), + (0x2045, 0x2051), + (0x2053, 0x205E), + (0x207D, 0x207E), + (0x208D, 0x208E), + (0x2308, 0x230B), + (0x2329, 0x232A), + (0x2768, 0x2775), + (0x27C5, 0x27C6), + (0x27E6, 0x27EF), + (0x2983, 0x2998), + (0x29D8, 0x29DB), + (0x29FC, 0x29FD), + (0x2CF9, 0x2CFC), + (0x2CFE, 0x2CFF), + (0x2D70, 0x2D70), + (0x2E00, 0x2E2E), + (0x2E30, 0x2E4F), + (0x2E52, 0x2E5D), + (0x3001, 0x3003), + (0x3008, 0x3011), + (0x3014, 0x301F), + (0x3030, 0x3030), + (0x303D, 0x303D), + (0x30A0, 0x30A0), + (0x30FB, 0x30FB), + (0xA4FE, 0xA4FF), + (0xA60D, 0xA60F), + (0xA673, 0xA673), + (0xA67E, 0xA67E), + (0xA6F2, 0xA6F7), + (0xA874, 0xA877), + (0xA8CE, 0xA8CF), + (0xA8F8, 0xA8FA), + (0xA8FC, 0xA8FC), + (0xA92E, 0xA92F), + (0xA95F, 0xA95F), + (0xA9C1, 0xA9CD), + (0xA9DE, 0xA9DF), + (0xAA5C, 0xAA5F), + (0xAADE, 0xAADF), + (0xAAF0, 0xAAF1), + (0xABEB, 0xABEB), + (0xFD3E, 0xFD3F), + (0xFE10, 0xFE19), + (0xFE30, 0xFE52), + (0xFE54, 0xFE61), + (0xFE63, 0xFE63), + (0xFE68, 0xFE68), + (0xFE6A, 0xFE6B), + (0xFF01, 0xFF03), + (0xFF05, 0xFF0A), + (0xFF0C, 0xFF0F), + (0xFF1A, 0xFF1B), + (0xFF1F, 0xFF20), + (0xFF3B, 0xFF3D), + (0xFF3F, 0xFF3F), + (0xFF5B, 0xFF5B), + (0xFF5D, 0xFF5D), + (0xFF5F, 0xFF65), + (0x10100, 0x10102), + (0x1039F, 0x1039F), + (0x103D0, 0x103D0), + (0x1056F, 0x1056F), + (0x10857, 0x10857), + (0x1091F, 0x1091F), + (0x1093F, 0x1093F), + (0x10A50, 0x10A58), + (0x10A7F, 0x10A7F), + (0x10AF0, 0x10AF6), + (0x10B39, 0x10B3F), + (0x10B99, 0x10B9C), + (0x10D6E, 0x10D6E), + (0x10EAD, 0x10EAD), + (0x10ED0, 0x10ED0), + (0x10F55, 0x10F59), + (0x10F86, 0x10F89), + (0x11047, 0x1104D), + (0x110BB, 0x110BC), + (0x110BE, 0x110C1), + (0x11140, 0x11143), + (0x11174, 0x11175), + (0x111C5, 0x111C8), + (0x111CD, 0x111CD), + (0x111DB, 0x111DB), + (0x111DD, 0x111DF), + (0x11238, 0x1123D), + (0x112A9, 0x112A9), + (0x113D4, 0x113D5), + (0x113D7, 0x113D8), + (0x1144B, 0x1144F), + (0x1145A, 0x1145B), + (0x1145D, 0x1145D), + (0x114C6, 0x114C6), + (0x115C1, 0x115D7), + (0x11641, 0x11643), + (0x11660, 0x1166C), + (0x116B9, 0x116B9), + (0x1173C, 0x1173E), + (0x1183B, 0x1183B), + (0x11944, 0x11946), + (0x119E2, 0x119E2), + (0x11A3F, 0x11A46), + (0x11A9A, 0x11A9C), + (0x11A9E, 0x11AA2), + (0x11B00, 0x11B09), + (0x11BE1, 0x11BE1), + (0x11C41, 0x11C45), + (0x11C70, 0x11C71), + (0x11EF7, 0x11EF8), + (0x11F43, 0x11F4F), + (0x11FFF, 0x11FFF), + (0x12470, 0x12474), + (0x12FF1, 0x12FF2), + (0x16A6E, 0x16A6F), + (0x16AF5, 0x16AF5), + (0x16B37, 0x16B3B), + (0x16B44, 0x16B44), + (0x16D6D, 0x16D6F), + (0x16E97, 0x16E9A), + (0x16FE2, 0x16FE2), + (0x1BC9F, 0x1BC9F), + (0x1DA87, 0x1DA8B), + (0x1E5FF, 0x1E5FF), + (0x1E95E, 0x1E95F), +]; + +/// `\p{S}`: General_Category Symbol. +pub(super) const SYMBOL: &[(u32, u32)] = &[ + (0x0024, 0x0024), + (0x002B, 0x002B), + (0x003C, 0x003E), + (0x005E, 0x005E), + (0x0060, 0x0060), + (0x007C, 0x007C), + (0x007E, 0x007E), + (0x00A2, 0x00A6), + (0x00A8, 0x00A9), + (0x00AC, 0x00AC), + (0x00AE, 0x00B1), + (0x00B4, 0x00B4), + (0x00B8, 0x00B8), + (0x00D7, 0x00D7), + (0x00F7, 0x00F7), + (0x02C2, 0x02C5), + (0x02D2, 0x02DF), + (0x02E5, 0x02EB), + (0x02ED, 0x02ED), + (0x02EF, 0x02FF), + (0x0375, 0x0375), + (0x0384, 0x0385), + (0x03F6, 0x03F6), + (0x0482, 0x0482), + (0x058D, 0x058F), + (0x0606, 0x0608), + (0x060B, 0x060B), + (0x060E, 0x060F), + (0x06DE, 0x06DE), + (0x06E9, 0x06E9), + (0x06FD, 0x06FE), + (0x07F6, 0x07F6), + (0x07FE, 0x07FF), + (0x0888, 0x0888), + (0x09F2, 0x09F3), + (0x09FA, 0x09FB), + (0x0AF1, 0x0AF1), + (0x0B70, 0x0B70), + (0x0BF3, 0x0BFA), + (0x0C7F, 0x0C7F), + (0x0D4F, 0x0D4F), + (0x0D79, 0x0D79), + (0x0E3F, 0x0E3F), + (0x0F01, 0x0F03), + (0x0F13, 0x0F13), + (0x0F15, 0x0F17), + (0x0F1A, 0x0F1F), + (0x0F34, 0x0F34), + (0x0F36, 0x0F36), + (0x0F38, 0x0F38), + (0x0FBE, 0x0FC5), + (0x0FC7, 0x0FCC), + (0x0FCE, 0x0FCF), + (0x0FD5, 0x0FD8), + (0x109E, 0x109F), + (0x1390, 0x1399), + (0x166D, 0x166D), + (0x17DB, 0x17DB), + (0x1940, 0x1940), + (0x19DE, 0x19FF), + (0x1B61, 0x1B6A), + (0x1B74, 0x1B7C), + (0x1FBD, 0x1FBD), + (0x1FBF, 0x1FC1), + (0x1FCD, 0x1FCF), + (0x1FDD, 0x1FDF), + (0x1FED, 0x1FEF), + (0x1FFD, 0x1FFE), + (0x2044, 0x2044), + (0x2052, 0x2052), + (0x207A, 0x207C), + (0x208A, 0x208C), + (0x20A0, 0x20C1), + (0x2100, 0x2101), + (0x2103, 0x2106), + (0x2108, 0x2109), + (0x2114, 0x2114), + (0x2116, 0x2118), + (0x211E, 0x2123), + (0x2125, 0x2125), + (0x2127, 0x2127), + (0x2129, 0x2129), + (0x212E, 0x212E), + (0x213A, 0x213B), + (0x2140, 0x2144), + (0x214A, 0x214D), + (0x214F, 0x214F), + (0x218A, 0x218B), + (0x2190, 0x2307), + (0x230C, 0x2328), + (0x232B, 0x2429), + (0x2440, 0x244A), + (0x249C, 0x24E9), + (0x2500, 0x2767), + (0x2794, 0x27C4), + (0x27C7, 0x27E5), + (0x27F0, 0x2982), + (0x2999, 0x29D7), + (0x29DC, 0x29FB), + (0x29FE, 0x2B73), + (0x2B76, 0x2BFF), + (0x2CE5, 0x2CEA), + (0x2E50, 0x2E51), + (0x2E80, 0x2E99), + (0x2E9B, 0x2EF3), + (0x2F00, 0x2FD5), + (0x2FF0, 0x2FFF), + (0x3004, 0x3004), + (0x3012, 0x3013), + (0x3020, 0x3020), + (0x3036, 0x3037), + (0x303E, 0x303F), + (0x309B, 0x309C), + (0x3190, 0x3191), + (0x3196, 0x319F), + (0x31C0, 0x31E5), + (0x31EF, 0x31EF), + (0x3200, 0x321E), + (0x322A, 0x3247), + (0x3250, 0x3250), + (0x3260, 0x327F), + (0x328A, 0x32B0), + (0x32C0, 0x33FF), + (0x4DC0, 0x4DFF), + (0xA490, 0xA4C6), + (0xA700, 0xA716), + (0xA720, 0xA721), + (0xA789, 0xA78A), + (0xA828, 0xA82B), + (0xA836, 0xA839), + (0xAA77, 0xAA79), + (0xAB5B, 0xAB5B), + (0xAB6A, 0xAB6B), + (0xFB29, 0xFB29), + (0xFBB2, 0xFBD2), + (0xFD40, 0xFD4F), + (0xFD90, 0xFD91), + (0xFDC8, 0xFDCF), + (0xFDFC, 0xFDFF), + (0xFE62, 0xFE62), + (0xFE64, 0xFE66), + (0xFE69, 0xFE69), + (0xFF04, 0xFF04), + (0xFF0B, 0xFF0B), + (0xFF1C, 0xFF1E), + (0xFF3E, 0xFF3E), + (0xFF40, 0xFF40), + (0xFF5C, 0xFF5C), + (0xFF5E, 0xFF5E), + (0xFFE0, 0xFFE6), + (0xFFE8, 0xFFEE), + (0xFFFC, 0xFFFD), + (0x10137, 0x1013F), + (0x10179, 0x10189), + (0x1018C, 0x1018E), + (0x10190, 0x1019C), + (0x101A0, 0x101A0), + (0x101D0, 0x101FC), + (0x10877, 0x10878), + (0x10AC8, 0x10AC8), + (0x10D8E, 0x10D8F), + (0x10ED1, 0x10ED8), + (0x1173F, 0x1173F), + (0x11FD5, 0x11FF1), + (0x16B3C, 0x16B3F), + (0x16B45, 0x16B45), + (0x1BC9C, 0x1BC9C), + (0x1CC00, 0x1CCEF), + (0x1CCFA, 0x1CCFC), + (0x1CD00, 0x1CEB3), + (0x1CEBA, 0x1CED0), + (0x1CEE0, 0x1CEF0), + (0x1CF50, 0x1CFC3), + (0x1D000, 0x1D0F5), + (0x1D100, 0x1D126), + (0x1D129, 0x1D164), + (0x1D16A, 0x1D16C), + (0x1D183, 0x1D184), + (0x1D18C, 0x1D1A9), + (0x1D1AE, 0x1D1EA), + (0x1D200, 0x1D241), + (0x1D245, 0x1D245), + (0x1D300, 0x1D356), + (0x1D6C1, 0x1D6C1), + (0x1D6DB, 0x1D6DB), + (0x1D6FB, 0x1D6FB), + (0x1D715, 0x1D715), + (0x1D735, 0x1D735), + (0x1D74F, 0x1D74F), + (0x1D76F, 0x1D76F), + (0x1D789, 0x1D789), + (0x1D7A9, 0x1D7A9), + (0x1D7C3, 0x1D7C3), + (0x1D800, 0x1D9FF), + (0x1DA37, 0x1DA3A), + (0x1DA6D, 0x1DA74), + (0x1DA76, 0x1DA83), + (0x1DA85, 0x1DA86), + (0x1E14F, 0x1E14F), + (0x1E2FF, 0x1E2FF), + (0x1ECAC, 0x1ECAC), + (0x1ECB0, 0x1ECB0), + (0x1ED2E, 0x1ED2E), + (0x1EEF0, 0x1EEF1), + (0x1F000, 0x1F02B), + (0x1F030, 0x1F093), + (0x1F0A0, 0x1F0AE), + (0x1F0B1, 0x1F0BF), + (0x1F0C1, 0x1F0CF), + (0x1F0D1, 0x1F0F5), + (0x1F10D, 0x1F1AD), + (0x1F1E6, 0x1F202), + (0x1F210, 0x1F23B), + (0x1F240, 0x1F248), + (0x1F250, 0x1F251), + (0x1F260, 0x1F265), + (0x1F300, 0x1F6D8), + (0x1F6DC, 0x1F6EC), + (0x1F6F0, 0x1F6FC), + (0x1F700, 0x1F7D9), + (0x1F7E0, 0x1F7EB), + (0x1F7F0, 0x1F7F0), + (0x1F800, 0x1F80B), + (0x1F810, 0x1F847), + (0x1F850, 0x1F859), + (0x1F860, 0x1F887), + (0x1F890, 0x1F8AD), + (0x1F8B0, 0x1F8BB), + (0x1F8C0, 0x1F8C1), + (0x1F8D0, 0x1F8D8), + (0x1F900, 0x1FA57), + (0x1FA60, 0x1FA6D), + (0x1FA70, 0x1FA7C), + (0x1FA80, 0x1FA8A), + (0x1FA8E, 0x1FAC6), + (0x1FAC8, 0x1FAC8), + (0x1FACD, 0x1FADC), + (0x1FADF, 0x1FAEA), + (0x1FAEF, 0x1FAF8), + (0x1FB00, 0x1FB92), + (0x1FB94, 0x1FBEF), + (0x1FBFA, 0x1FBFA), +]; + +/// `\p{L}`: General_Category Letter. +pub(super) const LETTER: &[(u32, u32)] = &[ + (0x0041, 0x005A), + (0x0061, 0x007A), + (0x00AA, 0x00AA), + (0x00B5, 0x00B5), + (0x00BA, 0x00BA), + (0x00C0, 0x00D6), + (0x00D8, 0x00F6), + (0x00F8, 0x02C1), + (0x02C6, 0x02D1), + (0x02E0, 0x02E4), + (0x02EC, 0x02EC), + (0x02EE, 0x02EE), + (0x0370, 0x0374), + (0x0376, 0x0377), + (0x037A, 0x037D), + (0x037F, 0x037F), + (0x0386, 0x0386), + (0x0388, 0x038A), + (0x038C, 0x038C), + (0x038E, 0x03A1), + (0x03A3, 0x03F5), + (0x03F7, 0x0481), + (0x048A, 0x052F), + (0x0531, 0x0556), + (0x0559, 0x0559), + (0x0560, 0x0588), + (0x05D0, 0x05EA), + (0x05EF, 0x05F2), + (0x0620, 0x064A), + (0x066E, 0x066F), + (0x0671, 0x06D3), + (0x06D5, 0x06D5), + (0x06E5, 0x06E6), + (0x06EE, 0x06EF), + (0x06FA, 0x06FC), + (0x06FF, 0x06FF), + (0x0710, 0x0710), + (0x0712, 0x072F), + (0x074D, 0x07A5), + (0x07B1, 0x07B1), + (0x07CA, 0x07EA), + (0x07F4, 0x07F5), + (0x07FA, 0x07FA), + (0x0800, 0x0815), + (0x081A, 0x081A), + (0x0824, 0x0824), + (0x0828, 0x0828), + (0x0840, 0x0858), + (0x0860, 0x086A), + (0x0870, 0x0887), + (0x0889, 0x088F), + (0x08A0, 0x08C9), + (0x0904, 0x0939), + (0x093D, 0x093D), + (0x0950, 0x0950), + (0x0958, 0x0961), + (0x0971, 0x0980), + (0x0985, 0x098C), + (0x098F, 0x0990), + (0x0993, 0x09A8), + (0x09AA, 0x09B0), + (0x09B2, 0x09B2), + (0x09B6, 0x09B9), + (0x09BD, 0x09BD), + (0x09CE, 0x09CE), + (0x09DC, 0x09DD), + (0x09DF, 0x09E1), + (0x09F0, 0x09F1), + (0x09FC, 0x09FC), + (0x0A05, 0x0A0A), + (0x0A0F, 0x0A10), + (0x0A13, 0x0A28), + (0x0A2A, 0x0A30), + (0x0A32, 0x0A33), + (0x0A35, 0x0A36), + (0x0A38, 0x0A39), + (0x0A59, 0x0A5C), + (0x0A5E, 0x0A5E), + (0x0A72, 0x0A74), + (0x0A85, 0x0A8D), + (0x0A8F, 0x0A91), + (0x0A93, 0x0AA8), + (0x0AAA, 0x0AB0), + (0x0AB2, 0x0AB3), + (0x0AB5, 0x0AB9), + (0x0ABD, 0x0ABD), + (0x0AD0, 0x0AD0), + (0x0AE0, 0x0AE1), + (0x0AF9, 0x0AF9), + (0x0B05, 0x0B0C), + (0x0B0F, 0x0B10), + (0x0B13, 0x0B28), + (0x0B2A, 0x0B30), + (0x0B32, 0x0B33), + (0x0B35, 0x0B39), + (0x0B3D, 0x0B3D), + (0x0B5C, 0x0B5D), + (0x0B5F, 0x0B61), + (0x0B71, 0x0B71), + (0x0B83, 0x0B83), + (0x0B85, 0x0B8A), + (0x0B8E, 0x0B90), + (0x0B92, 0x0B95), + (0x0B99, 0x0B9A), + (0x0B9C, 0x0B9C), + (0x0B9E, 0x0B9F), + (0x0BA3, 0x0BA4), + (0x0BA8, 0x0BAA), + (0x0BAE, 0x0BB9), + (0x0BD0, 0x0BD0), + (0x0C05, 0x0C0C), + (0x0C0E, 0x0C10), + (0x0C12, 0x0C28), + (0x0C2A, 0x0C39), + (0x0C3D, 0x0C3D), + (0x0C58, 0x0C5A), + (0x0C5C, 0x0C5D), + (0x0C60, 0x0C61), + (0x0C80, 0x0C80), + (0x0C85, 0x0C8C), + (0x0C8E, 0x0C90), + (0x0C92, 0x0CA8), + (0x0CAA, 0x0CB3), + (0x0CB5, 0x0CB9), + (0x0CBD, 0x0CBD), + (0x0CDC, 0x0CDE), + (0x0CE0, 0x0CE1), + (0x0CF1, 0x0CF2), + (0x0D04, 0x0D0C), + (0x0D0E, 0x0D10), + (0x0D12, 0x0D3A), + (0x0D3D, 0x0D3D), + (0x0D4E, 0x0D4E), + (0x0D54, 0x0D56), + (0x0D5F, 0x0D61), + (0x0D7A, 0x0D7F), + (0x0D85, 0x0D96), + (0x0D9A, 0x0DB1), + (0x0DB3, 0x0DBB), + (0x0DBD, 0x0DBD), + (0x0DC0, 0x0DC6), + (0x0E01, 0x0E30), + (0x0E32, 0x0E33), + (0x0E40, 0x0E46), + (0x0E81, 0x0E82), + (0x0E84, 0x0E84), + (0x0E86, 0x0E8A), + (0x0E8C, 0x0EA3), + (0x0EA5, 0x0EA5), + (0x0EA7, 0x0EB0), + (0x0EB2, 0x0EB3), + (0x0EBD, 0x0EBD), + (0x0EC0, 0x0EC4), + (0x0EC6, 0x0EC6), + (0x0EDC, 0x0EDF), + (0x0F00, 0x0F00), + (0x0F40, 0x0F47), + (0x0F49, 0x0F6C), + (0x0F88, 0x0F8C), + (0x1000, 0x102A), + (0x103F, 0x103F), + (0x1050, 0x1055), + (0x105A, 0x105D), + (0x1061, 0x1061), + (0x1065, 0x1066), + (0x106E, 0x1070), + (0x1075, 0x1081), + (0x108E, 0x108E), + (0x10A0, 0x10C5), + (0x10C7, 0x10C7), + (0x10CD, 0x10CD), + (0x10D0, 0x10FA), + (0x10FC, 0x1248), + (0x124A, 0x124D), + (0x1250, 0x1256), + (0x1258, 0x1258), + (0x125A, 0x125D), + (0x1260, 0x1288), + (0x128A, 0x128D), + (0x1290, 0x12B0), + (0x12B2, 0x12B5), + (0x12B8, 0x12BE), + (0x12C0, 0x12C0), + (0x12C2, 0x12C5), + (0x12C8, 0x12D6), + (0x12D8, 0x1310), + (0x1312, 0x1315), + (0x1318, 0x135A), + (0x1380, 0x138F), + (0x13A0, 0x13F5), + (0x13F8, 0x13FD), + (0x1401, 0x166C), + (0x166F, 0x167F), + (0x1681, 0x169A), + (0x16A0, 0x16EA), + (0x16F1, 0x16F8), + (0x1700, 0x1711), + (0x171F, 0x1731), + (0x1740, 0x1751), + (0x1760, 0x176C), + (0x176E, 0x1770), + (0x1780, 0x17B3), + (0x17D7, 0x17D7), + (0x17DC, 0x17DC), + (0x1820, 0x1878), + (0x1880, 0x1884), + (0x1887, 0x18A8), + (0x18AA, 0x18AA), + (0x18B0, 0x18F5), + (0x1900, 0x191E), + (0x1950, 0x196D), + (0x1970, 0x1974), + (0x1980, 0x19AB), + (0x19B0, 0x19C9), + (0x1A00, 0x1A16), + (0x1A20, 0x1A54), + (0x1AA7, 0x1AA7), + (0x1B05, 0x1B33), + (0x1B45, 0x1B4C), + (0x1B83, 0x1BA0), + (0x1BAE, 0x1BAF), + (0x1BBA, 0x1BE5), + (0x1C00, 0x1C23), + (0x1C4D, 0x1C4F), + (0x1C5A, 0x1C7D), + (0x1C80, 0x1C8A), + (0x1C90, 0x1CBA), + (0x1CBD, 0x1CBF), + (0x1CE9, 0x1CEC), + (0x1CEE, 0x1CF3), + (0x1CF5, 0x1CF6), + (0x1CFA, 0x1CFA), + (0x1D00, 0x1DBF), + (0x1E00, 0x1F15), + (0x1F18, 0x1F1D), + (0x1F20, 0x1F45), + (0x1F48, 0x1F4D), + (0x1F50, 0x1F57), + (0x1F59, 0x1F59), + (0x1F5B, 0x1F5B), + (0x1F5D, 0x1F5D), + (0x1F5F, 0x1F7D), + (0x1F80, 0x1FB4), + (0x1FB6, 0x1FBC), + (0x1FBE, 0x1FBE), + (0x1FC2, 0x1FC4), + (0x1FC6, 0x1FCC), + (0x1FD0, 0x1FD3), + (0x1FD6, 0x1FDB), + (0x1FE0, 0x1FEC), + (0x1FF2, 0x1FF4), + (0x1FF6, 0x1FFC), + (0x2071, 0x2071), + (0x207F, 0x207F), + (0x2090, 0x209C), + (0x2102, 0x2102), + (0x2107, 0x2107), + (0x210A, 0x2113), + (0x2115, 0x2115), + (0x2119, 0x211D), + (0x2124, 0x2124), + (0x2126, 0x2126), + (0x2128, 0x2128), + (0x212A, 0x212D), + (0x212F, 0x2139), + (0x213C, 0x213F), + (0x2145, 0x2149), + (0x214E, 0x214E), + (0x2183, 0x2184), + (0x2C00, 0x2CE4), + (0x2CEB, 0x2CEE), + (0x2CF2, 0x2CF3), + (0x2D00, 0x2D25), + (0x2D27, 0x2D27), + (0x2D2D, 0x2D2D), + (0x2D30, 0x2D67), + (0x2D6F, 0x2D6F), + (0x2D80, 0x2D96), + (0x2DA0, 0x2DA6), + (0x2DA8, 0x2DAE), + (0x2DB0, 0x2DB6), + (0x2DB8, 0x2DBE), + (0x2DC0, 0x2DC6), + (0x2DC8, 0x2DCE), + (0x2DD0, 0x2DD6), + (0x2DD8, 0x2DDE), + (0x2E2F, 0x2E2F), + (0x3005, 0x3006), + (0x3031, 0x3035), + (0x303B, 0x303C), + (0x3041, 0x3096), + (0x309D, 0x309F), + (0x30A1, 0x30FA), + (0x30FC, 0x30FF), + (0x3105, 0x312F), + (0x3131, 0x318E), + (0x31A0, 0x31BF), + (0x31F0, 0x31FF), + (0x3400, 0x4DBF), + (0x4E00, 0xA48C), + (0xA4D0, 0xA4FD), + (0xA500, 0xA60C), + (0xA610, 0xA61F), + (0xA62A, 0xA62B), + (0xA640, 0xA66E), + (0xA67F, 0xA69D), + (0xA6A0, 0xA6E5), + (0xA717, 0xA71F), + (0xA722, 0xA788), + (0xA78B, 0xA7DC), + (0xA7F1, 0xA801), + (0xA803, 0xA805), + (0xA807, 0xA80A), + (0xA80C, 0xA822), + (0xA840, 0xA873), + (0xA882, 0xA8B3), + (0xA8F2, 0xA8F7), + (0xA8FB, 0xA8FB), + (0xA8FD, 0xA8FE), + (0xA90A, 0xA925), + (0xA930, 0xA946), + (0xA960, 0xA97C), + (0xA984, 0xA9B2), + (0xA9CF, 0xA9CF), + (0xA9E0, 0xA9E4), + (0xA9E6, 0xA9EF), + (0xA9FA, 0xA9FE), + (0xAA00, 0xAA28), + (0xAA40, 0xAA42), + (0xAA44, 0xAA4B), + (0xAA60, 0xAA76), + (0xAA7A, 0xAA7A), + (0xAA7E, 0xAAAF), + (0xAAB1, 0xAAB1), + (0xAAB5, 0xAAB6), + (0xAAB9, 0xAABD), + (0xAAC0, 0xAAC0), + (0xAAC2, 0xAAC2), + (0xAADB, 0xAADD), + (0xAAE0, 0xAAEA), + (0xAAF2, 0xAAF4), + (0xAB01, 0xAB06), + (0xAB09, 0xAB0E), + (0xAB11, 0xAB16), + (0xAB20, 0xAB26), + (0xAB28, 0xAB2E), + (0xAB30, 0xAB5A), + (0xAB5C, 0xAB69), + (0xAB70, 0xABE2), + (0xAC00, 0xD7A3), + (0xD7B0, 0xD7C6), + (0xD7CB, 0xD7FB), + (0xF900, 0xFA6D), + (0xFA70, 0xFAD9), + (0xFB00, 0xFB06), + (0xFB13, 0xFB17), + (0xFB1D, 0xFB1D), + (0xFB1F, 0xFB28), + (0xFB2A, 0xFB36), + (0xFB38, 0xFB3C), + (0xFB3E, 0xFB3E), + (0xFB40, 0xFB41), + (0xFB43, 0xFB44), + (0xFB46, 0xFBB1), + (0xFBD3, 0xFD3D), + (0xFD50, 0xFD8F), + (0xFD92, 0xFDC7), + (0xFDF0, 0xFDFB), + (0xFE70, 0xFE74), + (0xFE76, 0xFEFC), + (0xFF21, 0xFF3A), + (0xFF41, 0xFF5A), + (0xFF66, 0xFFBE), + (0xFFC2, 0xFFC7), + (0xFFCA, 0xFFCF), + (0xFFD2, 0xFFD7), + (0xFFDA, 0xFFDC), + (0x10000, 0x1000B), + (0x1000D, 0x10026), + (0x10028, 0x1003A), + (0x1003C, 0x1003D), + (0x1003F, 0x1004D), + (0x10050, 0x1005D), + (0x10080, 0x100FA), + (0x10280, 0x1029C), + (0x102A0, 0x102D0), + (0x10300, 0x1031F), + (0x1032D, 0x10340), + (0x10342, 0x10349), + (0x10350, 0x10375), + (0x10380, 0x1039D), + (0x103A0, 0x103C3), + (0x103C8, 0x103CF), + (0x10400, 0x1049D), + (0x104B0, 0x104D3), + (0x104D8, 0x104FB), + (0x10500, 0x10527), + (0x10530, 0x10563), + (0x10570, 0x1057A), + (0x1057C, 0x1058A), + (0x1058C, 0x10592), + (0x10594, 0x10595), + (0x10597, 0x105A1), + (0x105A3, 0x105B1), + (0x105B3, 0x105B9), + (0x105BB, 0x105BC), + (0x105C0, 0x105F3), + (0x10600, 0x10736), + (0x10740, 0x10755), + (0x10760, 0x10767), + (0x10780, 0x10785), + (0x10787, 0x107B0), + (0x107B2, 0x107BA), + (0x10800, 0x10805), + (0x10808, 0x10808), + (0x1080A, 0x10835), + (0x10837, 0x10838), + (0x1083C, 0x1083C), + (0x1083F, 0x10855), + (0x10860, 0x10876), + (0x10880, 0x1089E), + (0x108E0, 0x108F2), + (0x108F4, 0x108F5), + (0x10900, 0x10915), + (0x10920, 0x10939), + (0x10940, 0x10959), + (0x10980, 0x109B7), + (0x109BE, 0x109BF), + (0x10A00, 0x10A00), + (0x10A10, 0x10A13), + (0x10A15, 0x10A17), + (0x10A19, 0x10A35), + (0x10A60, 0x10A7C), + (0x10A80, 0x10A9C), + (0x10AC0, 0x10AC7), + (0x10AC9, 0x10AE4), + (0x10B00, 0x10B35), + (0x10B40, 0x10B55), + (0x10B60, 0x10B72), + (0x10B80, 0x10B91), + (0x10C00, 0x10C48), + (0x10C80, 0x10CB2), + (0x10CC0, 0x10CF2), + (0x10D00, 0x10D23), + (0x10D4A, 0x10D65), + (0x10D6F, 0x10D85), + (0x10E80, 0x10EA9), + (0x10EB0, 0x10EB1), + (0x10EC2, 0x10EC7), + (0x10F00, 0x10F1C), + (0x10F27, 0x10F27), + (0x10F30, 0x10F45), + (0x10F70, 0x10F81), + (0x10FB0, 0x10FC4), + (0x10FE0, 0x10FF6), + (0x11003, 0x11037), + (0x11071, 0x11072), + (0x11075, 0x11075), + (0x11083, 0x110AF), + (0x110D0, 0x110E8), + (0x11103, 0x11126), + (0x11144, 0x11144), + (0x11147, 0x11147), + (0x11150, 0x11172), + (0x11176, 0x11176), + (0x11183, 0x111B2), + (0x111C1, 0x111C4), + (0x111DA, 0x111DA), + (0x111DC, 0x111DC), + (0x11200, 0x11211), + (0x11213, 0x1122B), + (0x1123F, 0x11240), + (0x11280, 0x11286), + (0x11288, 0x11288), + (0x1128A, 0x1128D), + (0x1128F, 0x1129D), + (0x1129F, 0x112A8), + (0x112B0, 0x112DE), + (0x11305, 0x1130C), + (0x1130F, 0x11310), + (0x11313, 0x11328), + (0x1132A, 0x11330), + (0x11332, 0x11333), + (0x11335, 0x11339), + (0x1133D, 0x1133D), + (0x11350, 0x11350), + (0x1135D, 0x11361), + (0x11380, 0x11389), + (0x1138B, 0x1138B), + (0x1138E, 0x1138E), + (0x11390, 0x113B5), + (0x113B7, 0x113B7), + (0x113D1, 0x113D1), + (0x113D3, 0x113D3), + (0x11400, 0x11434), + (0x11447, 0x1144A), + (0x1145F, 0x11461), + (0x11480, 0x114AF), + (0x114C4, 0x114C5), + (0x114C7, 0x114C7), + (0x11580, 0x115AE), + (0x115D8, 0x115DB), + (0x11600, 0x1162F), + (0x11644, 0x11644), + (0x11680, 0x116AA), + (0x116B8, 0x116B8), + (0x11700, 0x1171A), + (0x11740, 0x11746), + (0x11800, 0x1182B), + (0x118A0, 0x118DF), + (0x118FF, 0x11906), + (0x11909, 0x11909), + (0x1190C, 0x11913), + (0x11915, 0x11916), + (0x11918, 0x1192F), + (0x1193F, 0x1193F), + (0x11941, 0x11941), + (0x119A0, 0x119A7), + (0x119AA, 0x119D0), + (0x119E1, 0x119E1), + (0x119E3, 0x119E3), + (0x11A00, 0x11A00), + (0x11A0B, 0x11A32), + (0x11A3A, 0x11A3A), + (0x11A50, 0x11A50), + (0x11A5C, 0x11A89), + (0x11A9D, 0x11A9D), + (0x11AB0, 0x11AF8), + (0x11BC0, 0x11BE0), + (0x11C00, 0x11C08), + (0x11C0A, 0x11C2E), + (0x11C40, 0x11C40), + (0x11C72, 0x11C8F), + (0x11D00, 0x11D06), + (0x11D08, 0x11D09), + (0x11D0B, 0x11D30), + (0x11D46, 0x11D46), + (0x11D60, 0x11D65), + (0x11D67, 0x11D68), + (0x11D6A, 0x11D89), + (0x11D98, 0x11D98), + (0x11DB0, 0x11DDB), + (0x11EE0, 0x11EF2), + (0x11F02, 0x11F02), + (0x11F04, 0x11F10), + (0x11F12, 0x11F33), + (0x11FB0, 0x11FB0), + (0x12000, 0x12399), + (0x12480, 0x12543), + (0x12F90, 0x12FF0), + (0x13000, 0x1342F), + (0x13441, 0x13446), + (0x13460, 0x143FA), + (0x14400, 0x14646), + (0x16100, 0x1611D), + (0x16800, 0x16A38), + (0x16A40, 0x16A5E), + (0x16A70, 0x16ABE), + (0x16AD0, 0x16AED), + (0x16B00, 0x16B2F), + (0x16B40, 0x16B43), + (0x16B63, 0x16B77), + (0x16B7D, 0x16B8F), + (0x16D40, 0x16D6C), + (0x16E40, 0x16E7F), + (0x16EA0, 0x16EB8), + (0x16EBB, 0x16ED3), + (0x16F00, 0x16F4A), + (0x16F50, 0x16F50), + (0x16F93, 0x16F9F), + (0x16FE0, 0x16FE1), + (0x16FE3, 0x16FE3), + (0x16FF2, 0x16FF3), + (0x17000, 0x18CD5), + (0x18CFF, 0x18D1E), + (0x18D80, 0x18DF2), + (0x1AFF0, 0x1AFF3), + (0x1AFF5, 0x1AFFB), + (0x1AFFD, 0x1AFFE), + (0x1B000, 0x1B122), + (0x1B132, 0x1B132), + (0x1B150, 0x1B152), + (0x1B155, 0x1B155), + (0x1B164, 0x1B167), + (0x1B170, 0x1B2FB), + (0x1BC00, 0x1BC6A), + (0x1BC70, 0x1BC7C), + (0x1BC80, 0x1BC88), + (0x1BC90, 0x1BC99), + (0x1D400, 0x1D454), + (0x1D456, 0x1D49C), + (0x1D49E, 0x1D49F), + (0x1D4A2, 0x1D4A2), + (0x1D4A5, 0x1D4A6), + (0x1D4A9, 0x1D4AC), + (0x1D4AE, 0x1D4B9), + (0x1D4BB, 0x1D4BB), + (0x1D4BD, 0x1D4C3), + (0x1D4C5, 0x1D505), + (0x1D507, 0x1D50A), + (0x1D50D, 0x1D514), + (0x1D516, 0x1D51C), + (0x1D51E, 0x1D539), + (0x1D53B, 0x1D53E), + (0x1D540, 0x1D544), + (0x1D546, 0x1D546), + (0x1D54A, 0x1D550), + (0x1D552, 0x1D6A5), + (0x1D6A8, 0x1D6C0), + (0x1D6C2, 0x1D6DA), + (0x1D6DC, 0x1D6FA), + (0x1D6FC, 0x1D714), + (0x1D716, 0x1D734), + (0x1D736, 0x1D74E), + (0x1D750, 0x1D76E), + (0x1D770, 0x1D788), + (0x1D78A, 0x1D7A8), + (0x1D7AA, 0x1D7C2), + (0x1D7C4, 0x1D7CB), + (0x1DF00, 0x1DF1E), + (0x1DF25, 0x1DF2A), + (0x1E030, 0x1E06D), + (0x1E100, 0x1E12C), + (0x1E137, 0x1E13D), + (0x1E14E, 0x1E14E), + (0x1E290, 0x1E2AD), + (0x1E2C0, 0x1E2EB), + (0x1E4D0, 0x1E4EB), + (0x1E5D0, 0x1E5ED), + (0x1E5F0, 0x1E5F0), + (0x1E6C0, 0x1E6DE), + (0x1E6E0, 0x1E6E2), + (0x1E6E4, 0x1E6E5), + (0x1E6E7, 0x1E6ED), + (0x1E6F0, 0x1E6F4), + (0x1E6FE, 0x1E6FF), + (0x1E7E0, 0x1E7E6), + (0x1E7E8, 0x1E7EB), + (0x1E7ED, 0x1E7EE), + (0x1E7F0, 0x1E7FE), + (0x1E800, 0x1E8C4), + (0x1E900, 0x1E943), + (0x1E94B, 0x1E94B), + (0x1EE00, 0x1EE03), + (0x1EE05, 0x1EE1F), + (0x1EE21, 0x1EE22), + (0x1EE24, 0x1EE24), + (0x1EE27, 0x1EE27), + (0x1EE29, 0x1EE32), + (0x1EE34, 0x1EE37), + (0x1EE39, 0x1EE39), + (0x1EE3B, 0x1EE3B), + (0x1EE42, 0x1EE42), + (0x1EE47, 0x1EE47), + (0x1EE49, 0x1EE49), + (0x1EE4B, 0x1EE4B), + (0x1EE4D, 0x1EE4F), + (0x1EE51, 0x1EE52), + (0x1EE54, 0x1EE54), + (0x1EE57, 0x1EE57), + (0x1EE59, 0x1EE59), + (0x1EE5B, 0x1EE5B), + (0x1EE5D, 0x1EE5D), + (0x1EE5F, 0x1EE5F), + (0x1EE61, 0x1EE62), + (0x1EE64, 0x1EE64), + (0x1EE67, 0x1EE6A), + (0x1EE6C, 0x1EE72), + (0x1EE74, 0x1EE77), + (0x1EE79, 0x1EE7C), + (0x1EE7E, 0x1EE7E), + (0x1EE80, 0x1EE89), + (0x1EE8B, 0x1EE9B), + (0x1EEA1, 0x1EEA3), + (0x1EEA5, 0x1EEA9), + (0x1EEAB, 0x1EEBB), + (0x20000, 0x2A6DF), + (0x2A700, 0x2B81D), + (0x2B820, 0x2CEAD), + (0x2CEB0, 0x2EBE0), + (0x2EBF0, 0x2EE5D), + (0x2F800, 0x2FA1D), + (0x30000, 0x3134A), + (0x31350, 0x33479), +]; + +/// `\p{N}`: General_Category Number. +pub(super) const NUMBER: &[(u32, u32)] = &[ + (0x0030, 0x0039), + (0x00B2, 0x00B3), + (0x00B9, 0x00B9), + (0x00BC, 0x00BE), + (0x0660, 0x0669), + (0x06F0, 0x06F9), + (0x07C0, 0x07C9), + (0x0966, 0x096F), + (0x09E6, 0x09EF), + (0x09F4, 0x09F9), + (0x0A66, 0x0A6F), + (0x0AE6, 0x0AEF), + (0x0B66, 0x0B6F), + (0x0B72, 0x0B77), + (0x0BE6, 0x0BF2), + (0x0C66, 0x0C6F), + (0x0C78, 0x0C7E), + (0x0CE6, 0x0CEF), + (0x0D58, 0x0D5E), + (0x0D66, 0x0D78), + (0x0DE6, 0x0DEF), + (0x0E50, 0x0E59), + (0x0ED0, 0x0ED9), + (0x0F20, 0x0F33), + (0x1040, 0x1049), + (0x1090, 0x1099), + (0x1369, 0x137C), + (0x16EE, 0x16F0), + (0x17E0, 0x17E9), + (0x17F0, 0x17F9), + (0x1810, 0x1819), + (0x1946, 0x194F), + (0x19D0, 0x19DA), + (0x1A80, 0x1A89), + (0x1A90, 0x1A99), + (0x1B50, 0x1B59), + (0x1BB0, 0x1BB9), + (0x1C40, 0x1C49), + (0x1C50, 0x1C59), + (0x2070, 0x2070), + (0x2074, 0x2079), + (0x2080, 0x2089), + (0x2150, 0x2182), + (0x2185, 0x2189), + (0x2460, 0x249B), + (0x24EA, 0x24FF), + (0x2776, 0x2793), + (0x2CFD, 0x2CFD), + (0x3007, 0x3007), + (0x3021, 0x3029), + (0x3038, 0x303A), + (0x3192, 0x3195), + (0x3220, 0x3229), + (0x3248, 0x324F), + (0x3251, 0x325F), + (0x3280, 0x3289), + (0x32B1, 0x32BF), + (0xA620, 0xA629), + (0xA6E6, 0xA6EF), + (0xA830, 0xA835), + (0xA8D0, 0xA8D9), + (0xA900, 0xA909), + (0xA9D0, 0xA9D9), + (0xA9F0, 0xA9F9), + (0xAA50, 0xAA59), + (0xABF0, 0xABF9), + (0xFF10, 0xFF19), + (0x10107, 0x10133), + (0x10140, 0x10178), + (0x1018A, 0x1018B), + (0x102E1, 0x102FB), + (0x10320, 0x10323), + (0x10341, 0x10341), + (0x1034A, 0x1034A), + (0x103D1, 0x103D5), + (0x104A0, 0x104A9), + (0x10858, 0x1085F), + (0x10879, 0x1087F), + (0x108A7, 0x108AF), + (0x108FB, 0x108FF), + (0x10916, 0x1091B), + (0x109BC, 0x109BD), + (0x109C0, 0x109CF), + (0x109D2, 0x109FF), + (0x10A40, 0x10A48), + (0x10A7D, 0x10A7E), + (0x10A9D, 0x10A9F), + (0x10AEB, 0x10AEF), + (0x10B58, 0x10B5F), + (0x10B78, 0x10B7F), + (0x10BA9, 0x10BAF), + (0x10CFA, 0x10CFF), + (0x10D30, 0x10D39), + (0x10D40, 0x10D49), + (0x10E60, 0x10E7E), + (0x10F1D, 0x10F26), + (0x10F51, 0x10F54), + (0x10FC5, 0x10FCB), + (0x11052, 0x1106F), + (0x110F0, 0x110F9), + (0x11136, 0x1113F), + (0x111D0, 0x111D9), + (0x111E1, 0x111F4), + (0x112F0, 0x112F9), + (0x11450, 0x11459), + (0x114D0, 0x114D9), + (0x11650, 0x11659), + (0x116C0, 0x116C9), + (0x116D0, 0x116E3), + (0x11730, 0x1173B), + (0x118E0, 0x118F2), + (0x11950, 0x11959), + (0x11BF0, 0x11BF9), + (0x11C50, 0x11C6C), + (0x11D50, 0x11D59), + (0x11DA0, 0x11DA9), + (0x11DE0, 0x11DE9), + (0x11F50, 0x11F59), + (0x11FC0, 0x11FD4), + (0x12400, 0x1246E), + (0x16130, 0x16139), + (0x16A60, 0x16A69), + (0x16AC0, 0x16AC9), + (0x16B50, 0x16B59), + (0x16B5B, 0x16B61), + (0x16D70, 0x16D79), + (0x16E80, 0x16E96), + (0x16FF4, 0x16FF6), + (0x1CCF0, 0x1CCF9), + (0x1D2C0, 0x1D2D3), + (0x1D2E0, 0x1D2F3), + (0x1D360, 0x1D378), + (0x1D7CE, 0x1D7FF), + (0x1E140, 0x1E149), + (0x1E2F0, 0x1E2F9), + (0x1E4F0, 0x1E4F9), + (0x1E5F1, 0x1E5FA), + (0x1E8C7, 0x1E8CF), + (0x1E950, 0x1E959), + (0x1EC71, 0x1ECAB), + (0x1ECAD, 0x1ECAF), + (0x1ECB1, 0x1ECB4), + (0x1ED01, 0x1ED2D), + (0x1ED2F, 0x1ED3D), + (0x1F100, 0x1F10C), + (0x1FBF0, 0x1FBF9), +]; + +/// `\p{Zs}`: General_Category Space_Separator. +pub(super) const SPACE_SEPARATOR: &[(u32, u32)] = &[ + (0x0020, 0x0020), + (0x00A0, 0x00A0), + (0x1680, 0x1680), + (0x2000, 0x200A), + (0x202F, 0x202F), + (0x205F, 0x205F), + (0x3000, 0x3000), +]; + +/// `Canonicalize` without the `u` flag: each BMP code unit that the `i` +/// flag reads as another, with the one it reads as, sorted. +pub(super) const CANONICALIZE: &[(u16, u16)] = &[ + (0x0061, 0x0041), + (0x0062, 0x0042), + (0x0063, 0x0043), + (0x0064, 0x0044), + (0x0065, 0x0045), + (0x0066, 0x0046), + (0x0067, 0x0047), + (0x0068, 0x0048), + (0x0069, 0x0049), + (0x006A, 0x004A), + (0x006B, 0x004B), + (0x006C, 0x004C), + (0x006D, 0x004D), + (0x006E, 0x004E), + (0x006F, 0x004F), + (0x0070, 0x0050), + (0x0071, 0x0051), + (0x0072, 0x0052), + (0x0073, 0x0053), + (0x0074, 0x0054), + (0x0075, 0x0055), + (0x0076, 0x0056), + (0x0077, 0x0057), + (0x0078, 0x0058), + (0x0079, 0x0059), + (0x007A, 0x005A), + (0x00B5, 0x039C), + (0x00E0, 0x00C0), + (0x00E1, 0x00C1), + (0x00E2, 0x00C2), + (0x00E3, 0x00C3), + (0x00E4, 0x00C4), + (0x00E5, 0x00C5), + (0x00E6, 0x00C6), + (0x00E7, 0x00C7), + (0x00E8, 0x00C8), + (0x00E9, 0x00C9), + (0x00EA, 0x00CA), + (0x00EB, 0x00CB), + (0x00EC, 0x00CC), + (0x00ED, 0x00CD), + (0x00EE, 0x00CE), + (0x00EF, 0x00CF), + (0x00F0, 0x00D0), + (0x00F1, 0x00D1), + (0x00F2, 0x00D2), + (0x00F3, 0x00D3), + (0x00F4, 0x00D4), + (0x00F5, 0x00D5), + (0x00F6, 0x00D6), + (0x00F8, 0x00D8), + (0x00F9, 0x00D9), + (0x00FA, 0x00DA), + (0x00FB, 0x00DB), + (0x00FC, 0x00DC), + (0x00FD, 0x00DD), + (0x00FE, 0x00DE), + (0x00FF, 0x0178), + (0x0101, 0x0100), + (0x0103, 0x0102), + (0x0105, 0x0104), + (0x0107, 0x0106), + (0x0109, 0x0108), + (0x010B, 0x010A), + (0x010D, 0x010C), + (0x010F, 0x010E), + (0x0111, 0x0110), + (0x0113, 0x0112), + (0x0115, 0x0114), + (0x0117, 0x0116), + (0x0119, 0x0118), + (0x011B, 0x011A), + (0x011D, 0x011C), + (0x011F, 0x011E), + (0x0121, 0x0120), + (0x0123, 0x0122), + (0x0125, 0x0124), + (0x0127, 0x0126), + (0x0129, 0x0128), + (0x012B, 0x012A), + (0x012D, 0x012C), + (0x012F, 0x012E), + (0x0133, 0x0132), + (0x0135, 0x0134), + (0x0137, 0x0136), + (0x013A, 0x0139), + (0x013C, 0x013B), + (0x013E, 0x013D), + (0x0140, 0x013F), + (0x0142, 0x0141), + (0x0144, 0x0143), + (0x0146, 0x0145), + (0x0148, 0x0147), + (0x014B, 0x014A), + (0x014D, 0x014C), + (0x014F, 0x014E), + (0x0151, 0x0150), + (0x0153, 0x0152), + (0x0155, 0x0154), + (0x0157, 0x0156), + (0x0159, 0x0158), + (0x015B, 0x015A), + (0x015D, 0x015C), + (0x015F, 0x015E), + (0x0161, 0x0160), + (0x0163, 0x0162), + (0x0165, 0x0164), + (0x0167, 0x0166), + (0x0169, 0x0168), + (0x016B, 0x016A), + (0x016D, 0x016C), + (0x016F, 0x016E), + (0x0171, 0x0170), + (0x0173, 0x0172), + (0x0175, 0x0174), + (0x0177, 0x0176), + (0x017A, 0x0179), + (0x017C, 0x017B), + (0x017E, 0x017D), + (0x0180, 0x0243), + (0x0183, 0x0182), + (0x0185, 0x0184), + (0x0188, 0x0187), + (0x018C, 0x018B), + (0x0192, 0x0191), + (0x0195, 0x01F6), + (0x0199, 0x0198), + (0x019A, 0x023D), + (0x019B, 0xA7DC), + (0x019E, 0x0220), + (0x01A1, 0x01A0), + (0x01A3, 0x01A2), + (0x01A5, 0x01A4), + (0x01A8, 0x01A7), + (0x01AD, 0x01AC), + (0x01B0, 0x01AF), + (0x01B4, 0x01B3), + (0x01B6, 0x01B5), + (0x01B9, 0x01B8), + (0x01BD, 0x01BC), + (0x01BF, 0x01F7), + (0x01C5, 0x01C4), + (0x01C6, 0x01C4), + (0x01C8, 0x01C7), + (0x01C9, 0x01C7), + (0x01CB, 0x01CA), + (0x01CC, 0x01CA), + (0x01CE, 0x01CD), + (0x01D0, 0x01CF), + (0x01D2, 0x01D1), + (0x01D4, 0x01D3), + (0x01D6, 0x01D5), + (0x01D8, 0x01D7), + (0x01DA, 0x01D9), + (0x01DC, 0x01DB), + (0x01DD, 0x018E), + (0x01DF, 0x01DE), + (0x01E1, 0x01E0), + (0x01E3, 0x01E2), + (0x01E5, 0x01E4), + (0x01E7, 0x01E6), + (0x01E9, 0x01E8), + (0x01EB, 0x01EA), + (0x01ED, 0x01EC), + (0x01EF, 0x01EE), + (0x01F2, 0x01F1), + (0x01F3, 0x01F1), + (0x01F5, 0x01F4), + (0x01F9, 0x01F8), + (0x01FB, 0x01FA), + (0x01FD, 0x01FC), + (0x01FF, 0x01FE), + (0x0201, 0x0200), + (0x0203, 0x0202), + (0x0205, 0x0204), + (0x0207, 0x0206), + (0x0209, 0x0208), + (0x020B, 0x020A), + (0x020D, 0x020C), + (0x020F, 0x020E), + (0x0211, 0x0210), + (0x0213, 0x0212), + (0x0215, 0x0214), + (0x0217, 0x0216), + (0x0219, 0x0218), + (0x021B, 0x021A), + (0x021D, 0x021C), + (0x021F, 0x021E), + (0x0223, 0x0222), + (0x0225, 0x0224), + (0x0227, 0x0226), + (0x0229, 0x0228), + (0x022B, 0x022A), + (0x022D, 0x022C), + (0x022F, 0x022E), + (0x0231, 0x0230), + (0x0233, 0x0232), + (0x023C, 0x023B), + (0x023F, 0x2C7E), + (0x0240, 0x2C7F), + (0x0242, 0x0241), + (0x0247, 0x0246), + (0x0249, 0x0248), + (0x024B, 0x024A), + (0x024D, 0x024C), + (0x024F, 0x024E), + (0x0250, 0x2C6F), + (0x0251, 0x2C6D), + (0x0252, 0x2C70), + (0x0253, 0x0181), + (0x0254, 0x0186), + (0x0256, 0x0189), + (0x0257, 0x018A), + (0x0259, 0x018F), + (0x025B, 0x0190), + (0x025C, 0xA7AB), + (0x0260, 0x0193), + (0x0261, 0xA7AC), + (0x0263, 0x0194), + (0x0264, 0xA7CB), + (0x0265, 0xA78D), + (0x0266, 0xA7AA), + (0x0268, 0x0197), + (0x0269, 0x0196), + (0x026A, 0xA7AE), + (0x026B, 0x2C62), + (0x026C, 0xA7AD), + (0x026F, 0x019C), + (0x0271, 0x2C6E), + (0x0272, 0x019D), + (0x0275, 0x019F), + (0x027D, 0x2C64), + (0x0280, 0x01A6), + (0x0282, 0xA7C5), + (0x0283, 0x01A9), + (0x0287, 0xA7B1), + (0x0288, 0x01AE), + (0x0289, 0x0244), + (0x028A, 0x01B1), + (0x028B, 0x01B2), + (0x028C, 0x0245), + (0x0292, 0x01B7), + (0x029D, 0xA7B2), + (0x029E, 0xA7B0), + (0x0345, 0x0399), + (0x0371, 0x0370), + (0x0373, 0x0372), + (0x0377, 0x0376), + (0x037B, 0x03FD), + (0x037C, 0x03FE), + (0x037D, 0x03FF), + (0x03AC, 0x0386), + (0x03AD, 0x0388), + (0x03AE, 0x0389), + (0x03AF, 0x038A), + (0x03B1, 0x0391), + (0x03B2, 0x0392), + (0x03B3, 0x0393), + (0x03B4, 0x0394), + (0x03B5, 0x0395), + (0x03B6, 0x0396), + (0x03B7, 0x0397), + (0x03B8, 0x0398), + (0x03B9, 0x0399), + (0x03BA, 0x039A), + (0x03BB, 0x039B), + (0x03BC, 0x039C), + (0x03BD, 0x039D), + (0x03BE, 0x039E), + (0x03BF, 0x039F), + (0x03C0, 0x03A0), + (0x03C1, 0x03A1), + (0x03C2, 0x03A3), + (0x03C3, 0x03A3), + (0x03C4, 0x03A4), + (0x03C5, 0x03A5), + (0x03C6, 0x03A6), + (0x03C7, 0x03A7), + (0x03C8, 0x03A8), + (0x03C9, 0x03A9), + (0x03CA, 0x03AA), + (0x03CB, 0x03AB), + (0x03CC, 0x038C), + (0x03CD, 0x038E), + (0x03CE, 0x038F), + (0x03D0, 0x0392), + (0x03D1, 0x0398), + (0x03D5, 0x03A6), + (0x03D6, 0x03A0), + (0x03D7, 0x03CF), + (0x03D9, 0x03D8), + (0x03DB, 0x03DA), + (0x03DD, 0x03DC), + (0x03DF, 0x03DE), + (0x03E1, 0x03E0), + (0x03E3, 0x03E2), + (0x03E5, 0x03E4), + (0x03E7, 0x03E6), + (0x03E9, 0x03E8), + (0x03EB, 0x03EA), + (0x03ED, 0x03EC), + (0x03EF, 0x03EE), + (0x03F0, 0x039A), + (0x03F1, 0x03A1), + (0x03F2, 0x03F9), + (0x03F3, 0x037F), + (0x03F5, 0x0395), + (0x03F8, 0x03F7), + (0x03FB, 0x03FA), + (0x0430, 0x0410), + (0x0431, 0x0411), + (0x0432, 0x0412), + (0x0433, 0x0413), + (0x0434, 0x0414), + (0x0435, 0x0415), + (0x0436, 0x0416), + (0x0437, 0x0417), + (0x0438, 0x0418), + (0x0439, 0x0419), + (0x043A, 0x041A), + (0x043B, 0x041B), + (0x043C, 0x041C), + (0x043D, 0x041D), + (0x043E, 0x041E), + (0x043F, 0x041F), + (0x0440, 0x0420), + (0x0441, 0x0421), + (0x0442, 0x0422), + (0x0443, 0x0423), + (0x0444, 0x0424), + (0x0445, 0x0425), + (0x0446, 0x0426), + (0x0447, 0x0427), + (0x0448, 0x0428), + (0x0449, 0x0429), + (0x044A, 0x042A), + (0x044B, 0x042B), + (0x044C, 0x042C), + (0x044D, 0x042D), + (0x044E, 0x042E), + (0x044F, 0x042F), + (0x0450, 0x0400), + (0x0451, 0x0401), + (0x0452, 0x0402), + (0x0453, 0x0403), + (0x0454, 0x0404), + (0x0455, 0x0405), + (0x0456, 0x0406), + (0x0457, 0x0407), + (0x0458, 0x0408), + (0x0459, 0x0409), + (0x045A, 0x040A), + (0x045B, 0x040B), + (0x045C, 0x040C), + (0x045D, 0x040D), + (0x045E, 0x040E), + (0x045F, 0x040F), + (0x0461, 0x0460), + (0x0463, 0x0462), + (0x0465, 0x0464), + (0x0467, 0x0466), + (0x0469, 0x0468), + (0x046B, 0x046A), + (0x046D, 0x046C), + (0x046F, 0x046E), + (0x0471, 0x0470), + (0x0473, 0x0472), + (0x0475, 0x0474), + (0x0477, 0x0476), + (0x0479, 0x0478), + (0x047B, 0x047A), + (0x047D, 0x047C), + (0x047F, 0x047E), + (0x0481, 0x0480), + (0x048B, 0x048A), + (0x048D, 0x048C), + (0x048F, 0x048E), + (0x0491, 0x0490), + (0x0493, 0x0492), + (0x0495, 0x0494), + (0x0497, 0x0496), + (0x0499, 0x0498), + (0x049B, 0x049A), + (0x049D, 0x049C), + (0x049F, 0x049E), + (0x04A1, 0x04A0), + (0x04A3, 0x04A2), + (0x04A5, 0x04A4), + (0x04A7, 0x04A6), + (0x04A9, 0x04A8), + (0x04AB, 0x04AA), + (0x04AD, 0x04AC), + (0x04AF, 0x04AE), + (0x04B1, 0x04B0), + (0x04B3, 0x04B2), + (0x04B5, 0x04B4), + (0x04B7, 0x04B6), + (0x04B9, 0x04B8), + (0x04BB, 0x04BA), + (0x04BD, 0x04BC), + (0x04BF, 0x04BE), + (0x04C2, 0x04C1), + (0x04C4, 0x04C3), + (0x04C6, 0x04C5), + (0x04C8, 0x04C7), + (0x04CA, 0x04C9), + (0x04CC, 0x04CB), + (0x04CE, 0x04CD), + (0x04CF, 0x04C0), + (0x04D1, 0x04D0), + (0x04D3, 0x04D2), + (0x04D5, 0x04D4), + (0x04D7, 0x04D6), + (0x04D9, 0x04D8), + (0x04DB, 0x04DA), + (0x04DD, 0x04DC), + (0x04DF, 0x04DE), + (0x04E1, 0x04E0), + (0x04E3, 0x04E2), + (0x04E5, 0x04E4), + (0x04E7, 0x04E6), + (0x04E9, 0x04E8), + (0x04EB, 0x04EA), + (0x04ED, 0x04EC), + (0x04EF, 0x04EE), + (0x04F1, 0x04F0), + (0x04F3, 0x04F2), + (0x04F5, 0x04F4), + (0x04F7, 0x04F6), + (0x04F9, 0x04F8), + (0x04FB, 0x04FA), + (0x04FD, 0x04FC), + (0x04FF, 0x04FE), + (0x0501, 0x0500), + (0x0503, 0x0502), + (0x0505, 0x0504), + (0x0507, 0x0506), + (0x0509, 0x0508), + (0x050B, 0x050A), + (0x050D, 0x050C), + (0x050F, 0x050E), + (0x0511, 0x0510), + (0x0513, 0x0512), + (0x0515, 0x0514), + (0x0517, 0x0516), + (0x0519, 0x0518), + (0x051B, 0x051A), + (0x051D, 0x051C), + (0x051F, 0x051E), + (0x0521, 0x0520), + (0x0523, 0x0522), + (0x0525, 0x0524), + (0x0527, 0x0526), + (0x0529, 0x0528), + (0x052B, 0x052A), + (0x052D, 0x052C), + (0x052F, 0x052E), + (0x0561, 0x0531), + (0x0562, 0x0532), + (0x0563, 0x0533), + (0x0564, 0x0534), + (0x0565, 0x0535), + (0x0566, 0x0536), + (0x0567, 0x0537), + (0x0568, 0x0538), + (0x0569, 0x0539), + (0x056A, 0x053A), + (0x056B, 0x053B), + (0x056C, 0x053C), + (0x056D, 0x053D), + (0x056E, 0x053E), + (0x056F, 0x053F), + (0x0570, 0x0540), + (0x0571, 0x0541), + (0x0572, 0x0542), + (0x0573, 0x0543), + (0x0574, 0x0544), + (0x0575, 0x0545), + (0x0576, 0x0546), + (0x0577, 0x0547), + (0x0578, 0x0548), + (0x0579, 0x0549), + (0x057A, 0x054A), + (0x057B, 0x054B), + (0x057C, 0x054C), + (0x057D, 0x054D), + (0x057E, 0x054E), + (0x057F, 0x054F), + (0x0580, 0x0550), + (0x0581, 0x0551), + (0x0582, 0x0552), + (0x0583, 0x0553), + (0x0584, 0x0554), + (0x0585, 0x0555), + (0x0586, 0x0556), + (0x10D0, 0x1C90), + (0x10D1, 0x1C91), + (0x10D2, 0x1C92), + (0x10D3, 0x1C93), + (0x10D4, 0x1C94), + (0x10D5, 0x1C95), + (0x10D6, 0x1C96), + (0x10D7, 0x1C97), + (0x10D8, 0x1C98), + (0x10D9, 0x1C99), + (0x10DA, 0x1C9A), + (0x10DB, 0x1C9B), + (0x10DC, 0x1C9C), + (0x10DD, 0x1C9D), + (0x10DE, 0x1C9E), + (0x10DF, 0x1C9F), + (0x10E0, 0x1CA0), + (0x10E1, 0x1CA1), + (0x10E2, 0x1CA2), + (0x10E3, 0x1CA3), + (0x10E4, 0x1CA4), + (0x10E5, 0x1CA5), + (0x10E6, 0x1CA6), + (0x10E7, 0x1CA7), + (0x10E8, 0x1CA8), + (0x10E9, 0x1CA9), + (0x10EA, 0x1CAA), + (0x10EB, 0x1CAB), + (0x10EC, 0x1CAC), + (0x10ED, 0x1CAD), + (0x10EE, 0x1CAE), + (0x10EF, 0x1CAF), + (0x10F0, 0x1CB0), + (0x10F1, 0x1CB1), + (0x10F2, 0x1CB2), + (0x10F3, 0x1CB3), + (0x10F4, 0x1CB4), + (0x10F5, 0x1CB5), + (0x10F6, 0x1CB6), + (0x10F7, 0x1CB7), + (0x10F8, 0x1CB8), + (0x10F9, 0x1CB9), + (0x10FA, 0x1CBA), + (0x10FD, 0x1CBD), + (0x10FE, 0x1CBE), + (0x10FF, 0x1CBF), + (0x13F8, 0x13F0), + (0x13F9, 0x13F1), + (0x13FA, 0x13F2), + (0x13FB, 0x13F3), + (0x13FC, 0x13F4), + (0x13FD, 0x13F5), + (0x1C80, 0x0412), + (0x1C81, 0x0414), + (0x1C82, 0x041E), + (0x1C83, 0x0421), + (0x1C84, 0x0422), + (0x1C85, 0x0422), + (0x1C86, 0x042A), + (0x1C87, 0x0462), + (0x1C88, 0xA64A), + (0x1C8A, 0x1C89), + (0x1D79, 0xA77D), + (0x1D7D, 0x2C63), + (0x1D8E, 0xA7C6), + (0x1E01, 0x1E00), + (0x1E03, 0x1E02), + (0x1E05, 0x1E04), + (0x1E07, 0x1E06), + (0x1E09, 0x1E08), + (0x1E0B, 0x1E0A), + (0x1E0D, 0x1E0C), + (0x1E0F, 0x1E0E), + (0x1E11, 0x1E10), + (0x1E13, 0x1E12), + (0x1E15, 0x1E14), + (0x1E17, 0x1E16), + (0x1E19, 0x1E18), + (0x1E1B, 0x1E1A), + (0x1E1D, 0x1E1C), + (0x1E1F, 0x1E1E), + (0x1E21, 0x1E20), + (0x1E23, 0x1E22), + (0x1E25, 0x1E24), + (0x1E27, 0x1E26), + (0x1E29, 0x1E28), + (0x1E2B, 0x1E2A), + (0x1E2D, 0x1E2C), + (0x1E2F, 0x1E2E), + (0x1E31, 0x1E30), + (0x1E33, 0x1E32), + (0x1E35, 0x1E34), + (0x1E37, 0x1E36), + (0x1E39, 0x1E38), + (0x1E3B, 0x1E3A), + (0x1E3D, 0x1E3C), + (0x1E3F, 0x1E3E), + (0x1E41, 0x1E40), + (0x1E43, 0x1E42), + (0x1E45, 0x1E44), + (0x1E47, 0x1E46), + (0x1E49, 0x1E48), + (0x1E4B, 0x1E4A), + (0x1E4D, 0x1E4C), + (0x1E4F, 0x1E4E), + (0x1E51, 0x1E50), + (0x1E53, 0x1E52), + (0x1E55, 0x1E54), + (0x1E57, 0x1E56), + (0x1E59, 0x1E58), + (0x1E5B, 0x1E5A), + (0x1E5D, 0x1E5C), + (0x1E5F, 0x1E5E), + (0x1E61, 0x1E60), + (0x1E63, 0x1E62), + (0x1E65, 0x1E64), + (0x1E67, 0x1E66), + (0x1E69, 0x1E68), + (0x1E6B, 0x1E6A), + (0x1E6D, 0x1E6C), + (0x1E6F, 0x1E6E), + (0x1E71, 0x1E70), + (0x1E73, 0x1E72), + (0x1E75, 0x1E74), + (0x1E77, 0x1E76), + (0x1E79, 0x1E78), + (0x1E7B, 0x1E7A), + (0x1E7D, 0x1E7C), + (0x1E7F, 0x1E7E), + (0x1E81, 0x1E80), + (0x1E83, 0x1E82), + (0x1E85, 0x1E84), + (0x1E87, 0x1E86), + (0x1E89, 0x1E88), + (0x1E8B, 0x1E8A), + (0x1E8D, 0x1E8C), + (0x1E8F, 0x1E8E), + (0x1E91, 0x1E90), + (0x1E93, 0x1E92), + (0x1E95, 0x1E94), + (0x1E9B, 0x1E60), + (0x1EA1, 0x1EA0), + (0x1EA3, 0x1EA2), + (0x1EA5, 0x1EA4), + (0x1EA7, 0x1EA6), + (0x1EA9, 0x1EA8), + (0x1EAB, 0x1EAA), + (0x1EAD, 0x1EAC), + (0x1EAF, 0x1EAE), + (0x1EB1, 0x1EB0), + (0x1EB3, 0x1EB2), + (0x1EB5, 0x1EB4), + (0x1EB7, 0x1EB6), + (0x1EB9, 0x1EB8), + (0x1EBB, 0x1EBA), + (0x1EBD, 0x1EBC), + (0x1EBF, 0x1EBE), + (0x1EC1, 0x1EC0), + (0x1EC3, 0x1EC2), + (0x1EC5, 0x1EC4), + (0x1EC7, 0x1EC6), + (0x1EC9, 0x1EC8), + (0x1ECB, 0x1ECA), + (0x1ECD, 0x1ECC), + (0x1ECF, 0x1ECE), + (0x1ED1, 0x1ED0), + (0x1ED3, 0x1ED2), + (0x1ED5, 0x1ED4), + (0x1ED7, 0x1ED6), + (0x1ED9, 0x1ED8), + (0x1EDB, 0x1EDA), + (0x1EDD, 0x1EDC), + (0x1EDF, 0x1EDE), + (0x1EE1, 0x1EE0), + (0x1EE3, 0x1EE2), + (0x1EE5, 0x1EE4), + (0x1EE7, 0x1EE6), + (0x1EE9, 0x1EE8), + (0x1EEB, 0x1EEA), + (0x1EED, 0x1EEC), + (0x1EEF, 0x1EEE), + (0x1EF1, 0x1EF0), + (0x1EF3, 0x1EF2), + (0x1EF5, 0x1EF4), + (0x1EF7, 0x1EF6), + (0x1EF9, 0x1EF8), + (0x1EFB, 0x1EFA), + (0x1EFD, 0x1EFC), + (0x1EFF, 0x1EFE), + (0x1F00, 0x1F08), + (0x1F01, 0x1F09), + (0x1F02, 0x1F0A), + (0x1F03, 0x1F0B), + (0x1F04, 0x1F0C), + (0x1F05, 0x1F0D), + (0x1F06, 0x1F0E), + (0x1F07, 0x1F0F), + (0x1F10, 0x1F18), + (0x1F11, 0x1F19), + (0x1F12, 0x1F1A), + (0x1F13, 0x1F1B), + (0x1F14, 0x1F1C), + (0x1F15, 0x1F1D), + (0x1F20, 0x1F28), + (0x1F21, 0x1F29), + (0x1F22, 0x1F2A), + (0x1F23, 0x1F2B), + (0x1F24, 0x1F2C), + (0x1F25, 0x1F2D), + (0x1F26, 0x1F2E), + (0x1F27, 0x1F2F), + (0x1F30, 0x1F38), + (0x1F31, 0x1F39), + (0x1F32, 0x1F3A), + (0x1F33, 0x1F3B), + (0x1F34, 0x1F3C), + (0x1F35, 0x1F3D), + (0x1F36, 0x1F3E), + (0x1F37, 0x1F3F), + (0x1F40, 0x1F48), + (0x1F41, 0x1F49), + (0x1F42, 0x1F4A), + (0x1F43, 0x1F4B), + (0x1F44, 0x1F4C), + (0x1F45, 0x1F4D), + (0x1F51, 0x1F59), + (0x1F53, 0x1F5B), + (0x1F55, 0x1F5D), + (0x1F57, 0x1F5F), + (0x1F60, 0x1F68), + (0x1F61, 0x1F69), + (0x1F62, 0x1F6A), + (0x1F63, 0x1F6B), + (0x1F64, 0x1F6C), + (0x1F65, 0x1F6D), + (0x1F66, 0x1F6E), + (0x1F67, 0x1F6F), + (0x1F70, 0x1FBA), + (0x1F71, 0x1FBB), + (0x1F72, 0x1FC8), + (0x1F73, 0x1FC9), + (0x1F74, 0x1FCA), + (0x1F75, 0x1FCB), + (0x1F76, 0x1FDA), + (0x1F77, 0x1FDB), + (0x1F78, 0x1FF8), + (0x1F79, 0x1FF9), + (0x1F7A, 0x1FEA), + (0x1F7B, 0x1FEB), + (0x1F7C, 0x1FFA), + (0x1F7D, 0x1FFB), + (0x1FB0, 0x1FB8), + (0x1FB1, 0x1FB9), + (0x1FBE, 0x0399), + (0x1FD0, 0x1FD8), + (0x1FD1, 0x1FD9), + (0x1FE0, 0x1FE8), + (0x1FE1, 0x1FE9), + (0x1FE5, 0x1FEC), + (0x214E, 0x2132), + (0x2170, 0x2160), + (0x2171, 0x2161), + (0x2172, 0x2162), + (0x2173, 0x2163), + (0x2174, 0x2164), + (0x2175, 0x2165), + (0x2176, 0x2166), + (0x2177, 0x2167), + (0x2178, 0x2168), + (0x2179, 0x2169), + (0x217A, 0x216A), + (0x217B, 0x216B), + (0x217C, 0x216C), + (0x217D, 0x216D), + (0x217E, 0x216E), + (0x217F, 0x216F), + (0x2184, 0x2183), + (0x24D0, 0x24B6), + (0x24D1, 0x24B7), + (0x24D2, 0x24B8), + (0x24D3, 0x24B9), + (0x24D4, 0x24BA), + (0x24D5, 0x24BB), + (0x24D6, 0x24BC), + (0x24D7, 0x24BD), + (0x24D8, 0x24BE), + (0x24D9, 0x24BF), + (0x24DA, 0x24C0), + (0x24DB, 0x24C1), + (0x24DC, 0x24C2), + (0x24DD, 0x24C3), + (0x24DE, 0x24C4), + (0x24DF, 0x24C5), + (0x24E0, 0x24C6), + (0x24E1, 0x24C7), + (0x24E2, 0x24C8), + (0x24E3, 0x24C9), + (0x24E4, 0x24CA), + (0x24E5, 0x24CB), + (0x24E6, 0x24CC), + (0x24E7, 0x24CD), + (0x24E8, 0x24CE), + (0x24E9, 0x24CF), + (0x2C30, 0x2C00), + (0x2C31, 0x2C01), + (0x2C32, 0x2C02), + (0x2C33, 0x2C03), + (0x2C34, 0x2C04), + (0x2C35, 0x2C05), + (0x2C36, 0x2C06), + (0x2C37, 0x2C07), + (0x2C38, 0x2C08), + (0x2C39, 0x2C09), + (0x2C3A, 0x2C0A), + (0x2C3B, 0x2C0B), + (0x2C3C, 0x2C0C), + (0x2C3D, 0x2C0D), + (0x2C3E, 0x2C0E), + (0x2C3F, 0x2C0F), + (0x2C40, 0x2C10), + (0x2C41, 0x2C11), + (0x2C42, 0x2C12), + (0x2C43, 0x2C13), + (0x2C44, 0x2C14), + (0x2C45, 0x2C15), + (0x2C46, 0x2C16), + (0x2C47, 0x2C17), + (0x2C48, 0x2C18), + (0x2C49, 0x2C19), + (0x2C4A, 0x2C1A), + (0x2C4B, 0x2C1B), + (0x2C4C, 0x2C1C), + (0x2C4D, 0x2C1D), + (0x2C4E, 0x2C1E), + (0x2C4F, 0x2C1F), + (0x2C50, 0x2C20), + (0x2C51, 0x2C21), + (0x2C52, 0x2C22), + (0x2C53, 0x2C23), + (0x2C54, 0x2C24), + (0x2C55, 0x2C25), + (0x2C56, 0x2C26), + (0x2C57, 0x2C27), + (0x2C58, 0x2C28), + (0x2C59, 0x2C29), + (0x2C5A, 0x2C2A), + (0x2C5B, 0x2C2B), + (0x2C5C, 0x2C2C), + (0x2C5D, 0x2C2D), + (0x2C5E, 0x2C2E), + (0x2C5F, 0x2C2F), + (0x2C61, 0x2C60), + (0x2C65, 0x023A), + (0x2C66, 0x023E), + (0x2C68, 0x2C67), + (0x2C6A, 0x2C69), + (0x2C6C, 0x2C6B), + (0x2C73, 0x2C72), + (0x2C76, 0x2C75), + (0x2C81, 0x2C80), + (0x2C83, 0x2C82), + (0x2C85, 0x2C84), + (0x2C87, 0x2C86), + (0x2C89, 0x2C88), + (0x2C8B, 0x2C8A), + (0x2C8D, 0x2C8C), + (0x2C8F, 0x2C8E), + (0x2C91, 0x2C90), + (0x2C93, 0x2C92), + (0x2C95, 0x2C94), + (0x2C97, 0x2C96), + (0x2C99, 0x2C98), + (0x2C9B, 0x2C9A), + (0x2C9D, 0x2C9C), + (0x2C9F, 0x2C9E), + (0x2CA1, 0x2CA0), + (0x2CA3, 0x2CA2), + (0x2CA5, 0x2CA4), + (0x2CA7, 0x2CA6), + (0x2CA9, 0x2CA8), + (0x2CAB, 0x2CAA), + (0x2CAD, 0x2CAC), + (0x2CAF, 0x2CAE), + (0x2CB1, 0x2CB0), + (0x2CB3, 0x2CB2), + (0x2CB5, 0x2CB4), + (0x2CB7, 0x2CB6), + (0x2CB9, 0x2CB8), + (0x2CBB, 0x2CBA), + (0x2CBD, 0x2CBC), + (0x2CBF, 0x2CBE), + (0x2CC1, 0x2CC0), + (0x2CC3, 0x2CC2), + (0x2CC5, 0x2CC4), + (0x2CC7, 0x2CC6), + (0x2CC9, 0x2CC8), + (0x2CCB, 0x2CCA), + (0x2CCD, 0x2CCC), + (0x2CCF, 0x2CCE), + (0x2CD1, 0x2CD0), + (0x2CD3, 0x2CD2), + (0x2CD5, 0x2CD4), + (0x2CD7, 0x2CD6), + (0x2CD9, 0x2CD8), + (0x2CDB, 0x2CDA), + (0x2CDD, 0x2CDC), + (0x2CDF, 0x2CDE), + (0x2CE1, 0x2CE0), + (0x2CE3, 0x2CE2), + (0x2CEC, 0x2CEB), + (0x2CEE, 0x2CED), + (0x2CF3, 0x2CF2), + (0x2D00, 0x10A0), + (0x2D01, 0x10A1), + (0x2D02, 0x10A2), + (0x2D03, 0x10A3), + (0x2D04, 0x10A4), + (0x2D05, 0x10A5), + (0x2D06, 0x10A6), + (0x2D07, 0x10A7), + (0x2D08, 0x10A8), + (0x2D09, 0x10A9), + (0x2D0A, 0x10AA), + (0x2D0B, 0x10AB), + (0x2D0C, 0x10AC), + (0x2D0D, 0x10AD), + (0x2D0E, 0x10AE), + (0x2D0F, 0x10AF), + (0x2D10, 0x10B0), + (0x2D11, 0x10B1), + (0x2D12, 0x10B2), + (0x2D13, 0x10B3), + (0x2D14, 0x10B4), + (0x2D15, 0x10B5), + (0x2D16, 0x10B6), + (0x2D17, 0x10B7), + (0x2D18, 0x10B8), + (0x2D19, 0x10B9), + (0x2D1A, 0x10BA), + (0x2D1B, 0x10BB), + (0x2D1C, 0x10BC), + (0x2D1D, 0x10BD), + (0x2D1E, 0x10BE), + (0x2D1F, 0x10BF), + (0x2D20, 0x10C0), + (0x2D21, 0x10C1), + (0x2D22, 0x10C2), + (0x2D23, 0x10C3), + (0x2D24, 0x10C4), + (0x2D25, 0x10C5), + (0x2D27, 0x10C7), + (0x2D2D, 0x10CD), + (0xA641, 0xA640), + (0xA643, 0xA642), + (0xA645, 0xA644), + (0xA647, 0xA646), + (0xA649, 0xA648), + (0xA64B, 0xA64A), + (0xA64D, 0xA64C), + (0xA64F, 0xA64E), + (0xA651, 0xA650), + (0xA653, 0xA652), + (0xA655, 0xA654), + (0xA657, 0xA656), + (0xA659, 0xA658), + (0xA65B, 0xA65A), + (0xA65D, 0xA65C), + (0xA65F, 0xA65E), + (0xA661, 0xA660), + (0xA663, 0xA662), + (0xA665, 0xA664), + (0xA667, 0xA666), + (0xA669, 0xA668), + (0xA66B, 0xA66A), + (0xA66D, 0xA66C), + (0xA681, 0xA680), + (0xA683, 0xA682), + (0xA685, 0xA684), + (0xA687, 0xA686), + (0xA689, 0xA688), + (0xA68B, 0xA68A), + (0xA68D, 0xA68C), + (0xA68F, 0xA68E), + (0xA691, 0xA690), + (0xA693, 0xA692), + (0xA695, 0xA694), + (0xA697, 0xA696), + (0xA699, 0xA698), + (0xA69B, 0xA69A), + (0xA723, 0xA722), + (0xA725, 0xA724), + (0xA727, 0xA726), + (0xA729, 0xA728), + (0xA72B, 0xA72A), + (0xA72D, 0xA72C), + (0xA72F, 0xA72E), + (0xA733, 0xA732), + (0xA735, 0xA734), + (0xA737, 0xA736), + (0xA739, 0xA738), + (0xA73B, 0xA73A), + (0xA73D, 0xA73C), + (0xA73F, 0xA73E), + (0xA741, 0xA740), + (0xA743, 0xA742), + (0xA745, 0xA744), + (0xA747, 0xA746), + (0xA749, 0xA748), + (0xA74B, 0xA74A), + (0xA74D, 0xA74C), + (0xA74F, 0xA74E), + (0xA751, 0xA750), + (0xA753, 0xA752), + (0xA755, 0xA754), + (0xA757, 0xA756), + (0xA759, 0xA758), + (0xA75B, 0xA75A), + (0xA75D, 0xA75C), + (0xA75F, 0xA75E), + (0xA761, 0xA760), + (0xA763, 0xA762), + (0xA765, 0xA764), + (0xA767, 0xA766), + (0xA769, 0xA768), + (0xA76B, 0xA76A), + (0xA76D, 0xA76C), + (0xA76F, 0xA76E), + (0xA77A, 0xA779), + (0xA77C, 0xA77B), + (0xA77F, 0xA77E), + (0xA781, 0xA780), + (0xA783, 0xA782), + (0xA785, 0xA784), + (0xA787, 0xA786), + (0xA78C, 0xA78B), + (0xA791, 0xA790), + (0xA793, 0xA792), + (0xA794, 0xA7C4), + (0xA797, 0xA796), + (0xA799, 0xA798), + (0xA79B, 0xA79A), + (0xA79D, 0xA79C), + (0xA79F, 0xA79E), + (0xA7A1, 0xA7A0), + (0xA7A3, 0xA7A2), + (0xA7A5, 0xA7A4), + (0xA7A7, 0xA7A6), + (0xA7A9, 0xA7A8), + (0xA7B5, 0xA7B4), + (0xA7B7, 0xA7B6), + (0xA7B9, 0xA7B8), + (0xA7BB, 0xA7BA), + (0xA7BD, 0xA7BC), + (0xA7BF, 0xA7BE), + (0xA7C1, 0xA7C0), + (0xA7C3, 0xA7C2), + (0xA7C8, 0xA7C7), + (0xA7CA, 0xA7C9), + (0xA7CD, 0xA7CC), + (0xA7CF, 0xA7CE), + (0xA7D1, 0xA7D0), + (0xA7D3, 0xA7D2), + (0xA7D5, 0xA7D4), + (0xA7D7, 0xA7D6), + (0xA7D9, 0xA7D8), + (0xA7DB, 0xA7DA), + (0xA7F6, 0xA7F5), + (0xAB53, 0xA7B3), + (0xAB70, 0x13A0), + (0xAB71, 0x13A1), + (0xAB72, 0x13A2), + (0xAB73, 0x13A3), + (0xAB74, 0x13A4), + (0xAB75, 0x13A5), + (0xAB76, 0x13A6), + (0xAB77, 0x13A7), + (0xAB78, 0x13A8), + (0xAB79, 0x13A9), + (0xAB7A, 0x13AA), + (0xAB7B, 0x13AB), + (0xAB7C, 0x13AC), + (0xAB7D, 0x13AD), + (0xAB7E, 0x13AE), + (0xAB7F, 0x13AF), + (0xAB80, 0x13B0), + (0xAB81, 0x13B1), + (0xAB82, 0x13B2), + (0xAB83, 0x13B3), + (0xAB84, 0x13B4), + (0xAB85, 0x13B5), + (0xAB86, 0x13B6), + (0xAB87, 0x13B7), + (0xAB88, 0x13B8), + (0xAB89, 0x13B9), + (0xAB8A, 0x13BA), + (0xAB8B, 0x13BB), + (0xAB8C, 0x13BC), + (0xAB8D, 0x13BD), + (0xAB8E, 0x13BE), + (0xAB8F, 0x13BF), + (0xAB90, 0x13C0), + (0xAB91, 0x13C1), + (0xAB92, 0x13C2), + (0xAB93, 0x13C3), + (0xAB94, 0x13C4), + (0xAB95, 0x13C5), + (0xAB96, 0x13C6), + (0xAB97, 0x13C7), + (0xAB98, 0x13C8), + (0xAB99, 0x13C9), + (0xAB9A, 0x13CA), + (0xAB9B, 0x13CB), + (0xAB9C, 0x13CC), + (0xAB9D, 0x13CD), + (0xAB9E, 0x13CE), + (0xAB9F, 0x13CF), + (0xABA0, 0x13D0), + (0xABA1, 0x13D1), + (0xABA2, 0x13D2), + (0xABA3, 0x13D3), + (0xABA4, 0x13D4), + (0xABA5, 0x13D5), + (0xABA6, 0x13D6), + (0xABA7, 0x13D7), + (0xABA8, 0x13D8), + (0xABA9, 0x13D9), + (0xABAA, 0x13DA), + (0xABAB, 0x13DB), + (0xABAC, 0x13DC), + (0xABAD, 0x13DD), + (0xABAE, 0x13DE), + (0xABAF, 0x13DF), + (0xABB0, 0x13E0), + (0xABB1, 0x13E1), + (0xABB2, 0x13E2), + (0xABB3, 0x13E3), + (0xABB4, 0x13E4), + (0xABB5, 0x13E5), + (0xABB6, 0x13E6), + (0xABB7, 0x13E7), + (0xABB8, 0x13E8), + (0xABB9, 0x13E9), + (0xABBA, 0x13EA), + (0xABBB, 0x13EB), + (0xABBC, 0x13EC), + (0xABBD, 0x13ED), + (0xABBE, 0x13EE), + (0xABBF, 0x13EF), + (0xFF41, 0xFF21), + (0xFF42, 0xFF22), + (0xFF43, 0xFF23), + (0xFF44, 0xFF24), + (0xFF45, 0xFF25), + (0xFF46, 0xFF26), + (0xFF47, 0xFF27), + (0xFF48, 0xFF28), + (0xFF49, 0xFF29), + (0xFF4A, 0xFF2A), + (0xFF4B, 0xFF2B), + (0xFF4C, 0xFF2C), + (0xFF4D, 0xFF2D), + (0xFF4E, 0xFF2E), + (0xFF4F, 0xFF2F), + (0xFF50, 0xFF30), + (0xFF51, 0xFF31), + (0xFF52, 0xFF32), + (0xFF53, 0xFF33), + (0xFF54, 0xFF34), + (0xFF55, 0xFF35), + (0xFF56, 0xFF36), + (0xFF57, 0xFF37), + (0xFF58, 0xFF38), + (0xFF59, 0xFF39), + (0xFF5A, 0xFF3A), +]; diff --git a/rust/lexical-editor/src/markdown/utils.rs b/rust/lexical-editor/src/markdown/utils.rs new file mode 100644 index 00000000000..80151c2aa4a --- /dev/null +++ b/rust/lexical-editor/src/markdown/utils.rs @@ -0,0 +1,393 @@ +//! `utils.ts`, and the JS string operations the markdown port needs on +//! UTF-16 code units. +//! +//! A JS string indexed by `text[i]` is a code unit, which can be half of a +//! surrogate pair; here such a "character" is a `u32` that keeps the +//! surrogate's value, so the flanking rules see what JS sees. `characterAt` +//! gives a whole code point. + +use super::markdown_transformers::{ + ElementTransformer, MultilineElementTransformer, TextFormatTransformer, TextMatchTransformer, + Transformer, +}; +use super::reg_exp::{RegExp, in_table, is_js_space}; +use super::reg_exp_tables::{PUNCTUATION, SPACE_SEPARATOR, SYMBOL}; +use crate::NodeRef; +use crate::nodes::line_break::LineBreakNode; +use crate::nodes::paragraph::ParagraphNode; +use crate::nodes::tab::TabNode; +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +/// A compiled JS regular expression for a pattern of upstream's, compiled +/// once per thread. Each call returns a copy with `lastIndex` 0. +pub(crate) fn re(source: &'static str, flags: &'static str) -> RegExp { + thread_local! { + static CACHE: RefCell> = + RefCell::new(HashMap::new()); + } + CACHE.with(|cache| { + cache + .borrow_mut() + .entry((source, flags)) + .or_insert_with(|| RegExp::of(source, flags)) + .clone() + }) +} + +/// The UTF-16 code units of `text`. +pub(crate) fn units(text: &str) -> Vec { + text.encode_utf16().collect() +} + +/// A string of UTF-16 code units (a lone surrogate as U+FFFD, under the +/// unpaired surrogate exception). +pub(crate) fn string(units: &[u16]) -> String { + String::from_utf16_lossy(units) +} + +/// `text[i]`, or `None` (`undefined`) past either end. +pub(crate) fn unit_at(text: &[u16], index: isize) -> Option { + if index < 0 { + return None; + } + text.get(index as usize).map(|u| u32::from(*u)) +} + +/// A character as a JS string, for the few places that write one out. +pub(crate) fn char_string(character: u32) -> String { + char::from_u32(character).map_or_else(|| "\u{FFFD}".to_owned(), String::from) +} + +/// How many code units a character takes. +pub(crate) fn char_units(character: u32) -> usize { + if character > 0xFFFF { 2 } else { 1 } +} + +/// Whether a code unit or code point is in JS's `\s`. +pub(crate) fn is_space(character: u32) -> bool { + is_js_space(character) +} + +/// `text.trimStart()`, with JS's whitespace. +pub(crate) fn trim_start(text: &str) -> &str { + text.trim_start_matches(|c: char| is_js_space(c as u32)) +} + +/// `text.trimEnd()`, with JS's whitespace. +pub(crate) fn trim_end(text: &str) -> &str { + text.trim_end_matches(|c: char| is_js_space(c as u32)) +} + +/// `text.trim()`, with JS's whitespace. +pub(crate) fn trim(text: &str) -> &str { + trim_end(trim_start(text)) +} + +/// `/\S/.test(text)`. +pub(crate) fn has_non_space(text: &str) -> bool { + text.chars().any(|c| !is_js_space(c as u32)) +} + +/// What `transformersByType` returns. +#[derive(Clone, Default)] +pub(crate) struct ByType { + pub element: Vec>, + pub multiline_element: Vec>, + pub text_format: Vec>, + pub text_match: Vec>, +} + +/// `transformersByType(transformers)`. +pub(crate) fn transformers_by_type(transformers: &[Transformer]) -> ByType { + let mut by_type = ByType::default(); + for transformer in transformers { + match transformer { + Transformer::Element(t) => by_type.element.push(t.clone()), + Transformer::MultilineElement(t) => by_type.multiline_element.push(t.clone()), + Transformer::TextFormat(t) => by_type.text_format.push(t.clone()), + Transformer::TextMatch(t) => by_type.text_match.push(t.clone()), + } + } + by_type +} + +// CommonMark counts every character in the Unicode P (punctuation) and S +// (symbol) general categories as punctuation, which includes all of the +// ASCII punctuation characters. Its Unicode whitespace is Zs plus tab, line +// feed, form feed and carriage return; JavaScript's `\s` adds others, such +// as U+FEFF. Delimiter flanking depends on both. + +/// `PUNCTUATION`: `/[\p{P}\p{S}]/u`. +fn is_punctuation(character: u32) -> bool { + in_table(PUNCTUATION, character) || in_table(SYMBOL, character) +} + +/// `WHITESPACE`: `/[\p{Zs}\t\n\f\r]/u`. +fn is_whitespace(character: u32) -> bool { + matches!(character, 0x09 | 0x0A | 0x0C | 0x0D) || in_table(SPACE_SEPARATOR, character) +} + +const WHITESPACE_CLASS: u8 = 0; +const PUNCTUATION_CLASS: u8 = 1; +const OTHER_CLASS: u8 = 2; + +/// `classify`: how flanking sees `character`; the edge of the line is +/// whitespace. +fn classify(character: Option) -> u8 { + match character { + None => WHITESPACE_CLASS, + Some(c) if is_whitespace(c) => WHITESPACE_CLASS, + Some(c) if is_punctuation(c) => PUNCTUATION_CLASS, + Some(_) => OTHER_CLASS, + } +} + +/// `isFlankedBy(primary, secondary)`: whether a delimiter run with +/// `primary` on the side it faces and `secondary` on the other side is +/// flanking. `None` is the edge of the line. +pub(crate) fn is_flanked_by(primary: Option, secondary: Option) -> bool { + is_flanked_by_class(classify(primary), classify(secondary)) +} + +fn is_flanked_by_class(primary: u8, secondary: u8) -> bool { + primary == OTHER_CLASS || (primary == PUNCTUATION_CLASS && secondary != OTHER_CLASS) +} + +/// What `getEmphasisSides` returns. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct EmphasisSides { + pub can_open: bool, + pub can_close: bool, +} + +/// `getEmphasisSides(marker, before, after)`: whether a delimiter run of +/// `marker` between `before` and `after` can open and close emphasis; `_` +/// opens or closes inside a word only beside punctuation. +pub(crate) fn get_emphasis_sides( + marker: u32, + before: Option, + after: Option, +) -> EmphasisSides { + let before_class = classify(before); + let after_class = classify(after); + let is_left = is_flanked_by_class(after_class, before_class); + let is_right = is_flanked_by_class(before_class, after_class); + if marker == u32::from(b'_') { + EmphasisSides { + can_close: is_right && (!is_left || after_class == PUNCTUATION_CLASS), + can_open: is_left && (!is_right || before_class == PUNCTUATION_CLASS), + } + } else { + EmphasisSides { + can_close: is_right, + can_open: is_left, + } + } +} + +/// `isPunctuationOrSpace(character)`. +pub(crate) fn is_punctuation_or_space(character: Option) -> bool { + classify(character) != OTHER_CLASS +} + +// Readers built on micromark, such as `@lexical/mdast`, read a character +// outside the Basic Multilingual Plane one UTF-16 unit at a time, so as a +// letter, and count JavaScript's whitespace, U+FEFF among it, as +// whitespace. Export has to open and close for those readers as well. + +/// `asMicromarkReads(character)`. +fn as_micromark_reads(character: Option) -> Option { + character.map(|c| { + if c > 0xFFFF { + u32::from(b'a') + } else if matches!(c, 0x0B | 0x2028 | 0x2029 | 0xFEFF) { + u32::from(b' ') + } else { + c + } + }) +} + +/// `isPortablyFlankedBy`: [`is_flanked_by`], both for CommonMark and for +/// micromark. +pub(crate) fn is_portably_flanked_by(primary: Option, secondary: Option) -> bool { + is_flanked_by(primary, secondary) + && is_flanked_by(as_micromark_reads(primary), as_micromark_reads(secondary)) +} + +/// `getPortableEmphasisSides`: [`get_emphasis_sides`], both for CommonMark +/// and for micromark. +pub(crate) fn get_portable_emphasis_sides( + marker: u32, + before: Option, + after: Option, +) -> EmphasisSides { + let sides = get_emphasis_sides(marker, before, after); + let micromark = get_emphasis_sides( + marker, + as_micromark_reads(before), + as_micromark_reads(after), + ); + EmphasisSides { + can_close: sides.can_close && micromark.can_close, + can_open: sides.can_open && micromark.can_open, + } +} + +/// `isLetterForSomeReader(character)`: whether CommonMark or micromark +/// reads `character` as a letter. +pub(crate) fn is_letter_for_some_reader(character: u32) -> bool { + !is_punctuation_or_space(Some(character)) + || !is_punctuation_or_space(as_micromark_reads(Some(character))) +} + +/// `characterAt(text, index)`: the character (code point) that starts at +/// `index`, if any; half of a pair when `index` is inside one. +pub(crate) fn character_at(text: &[u16], index: isize) -> Option { + let unit = unit_at(text, index)?; + if (0xD800..0xDC00).contains(&unit) + && let Some(low) = unit_at(text, index + 1).filter(|l| (0xDC00..0xE000).contains(l)) + { + return Some(0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)); + } + Some(unit) +} + +/// `characterBefore(text, index)`: the character (code point) that ends +/// just before `index`, if any. +pub(crate) fn character_before(text: &[u16], index: isize) -> Option { + match character_at(text, index - 2) { + Some(pair) if pair > 0xFFFF => Some(pair), + _ => unit_at(text, index - 1), + } +} + +/// `toCharacterReference(character)`: `&#N;`. +pub(crate) fn to_character_reference(character: u32) -> String { + format!("&#{character};") +} + +/// `isEmptyParagraph(node)`. +pub(crate) fn is_empty_paragraph(node: &NodeRef) -> bool { + if !node.get_latest().is::() { + return false; + } + match node.get_first_child() { + None => true, + Some(first) => { + node.get_children_size() == 1 + && first.is_text() + && re(r"^\s{0,3}$", "").test(&first.get_text_content()) + } + } +} + +/// `decodeCharacterReference(digits, hex)`: a numeric reference's +/// character, U+FFFD for U+0000, a surrogate or one past U+10FFFF. +fn decode_character_reference(digits: Option<&str>, hex: Option<&str>) -> String { + let code_point = match (digits, hex) { + (Some(d), _) => d.parse::().unwrap_or(u64::MAX), + (None, Some(h)) => u64::from_str_radix(h, 16).unwrap_or(u64::MAX), + (None, None) => 0, + }; + if code_point == 0 || (0xD800..=0xDFFF).contains(&code_point) || code_point > 0x10FFFF { + "\u{FFFD}".to_owned() + } else { + char_string(code_point as u32) + } +} + +/// `unescapeText(value)`: reads backslash escapes and numeric character +/// references. +pub(crate) fn unescape_text(value: &str) -> String { + if !value.contains('\\') && !value.contains('&') { + return value.to_owned(); + } + // Do not decode an escaped ampersand or revisit a decoded reference. + re( + r"\\([!-/:-@[-`{-~])|&#(?:(\d{1,7})|[xX]([\da-fA-F]{1,6}));", + "g", + ) + .replace_with(value, |m| match m.get(1) { + Some(escaped) => escaped.to_owned(), + None => decode_character_reference(m.get(2), m.get(3)), + }) +} + +/// `isEscaped(text, index)`: whether an odd run of backslashes comes just +/// before `index`. +pub(crate) fn is_escaped(text: &[u16], index: usize) -> bool { + let mut backslashes = 0; + let mut i = index; + while i > 0 && text.get(i - 1) == Some(&u16::from(b'\\')) { + backslashes += 1; + i -= 1; + } + backslashes % 2 == 1 +} + +/// `getDelimiterRunEdge(text, index, step)`: where the delimiter run that +/// `text` is cut in at `index` ends, walking by `step` over the delimiter +/// beside the cut. Walking back, an escaped delimiter is not part of the +/// run. +pub(crate) fn get_delimiter_run_edge(text: &[u16], index: usize, step: isize) -> usize { + let delimiter = if step == 1 { + unit_at(text, index as isize - 1) + } else { + unit_at(text, index as isize) + }; + let mut edge = index; + if step == 1 { + while edge < text.len() && Some(u32::from(text[edge])) == delimiter { + edge += 1; + } + } else { + while edge > 0 + && Some(u32::from(text[edge - 1])) == delimiter + && !is_escaped(text, edge - 1) + { + edge -= 1; + } + } + edge +} + +/// `$isLineBreakNode(node)`. +pub(crate) fn is_line_break_node(node: &NodeRef) -> bool { + node.get_latest().is::() +} + +/// `$isTabNode(node)`. +pub(crate) fn is_tab_node(node: &NodeRef) -> bool { + node.get_latest().is::() +} + +/// `$isParagraphNode(node)`. +pub(crate) fn is_paragraph_node(node: &NodeRef) -> bool { + node.get_latest().is::() +} + +/// A text node's `getFormat()`. +pub(crate) fn text_format(node: &NodeRef) -> f64 { + node.text_data().get_format() +} + +/// `text.indexOf(search, from)`, in UTF-16 code units. +pub(crate) fn index_of(text: &[u16], search: &[u16], from: usize) -> Option { + if search.len() > text.len() || from > text.len() - search.len() { + return None; + } + (from..=text.len() - search.len()).find(|&i| &text[i..i + search.len()] == search) +} + +/// `text.lastIndexOf(search)`, in UTF-16 code units. +pub(crate) fn last_index_of(text: &[u16], search: &[u16]) -> Option { + if search.len() > text.len() { + return None; + } + (0..=text.len() - search.len()) + .rev() + .find(|&i| &text[i..i + search.len()] == search) +} diff --git a/rust/scripts/generate-reg-exp-tables.mjs b/rust/scripts/generate-reg-exp-tables.mjs new file mode 100644 index 00000000000..f78373eefc1 --- /dev/null +++ b/rust/scripts/generate-reg-exp-tables.mjs @@ -0,0 +1,81 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ +// Generates rust/lexical-editor/src/markdown/reg_exp_tables.rs, the Unicode +// data of the markdown feature's JS regular expressions (`RegExp`), by +// asking the JS engine that records the vectors: +// +// - the code points `\p{P}`, `\p{S}`, `\p{L}`, `\p{N}` and `\p{Zs}` match +// (with the `u` flag), so the port's tables are Node's Unicode version; +// - `Canonicalize` for the `i` flag without `u` (ECMA-262 22.2.2.7.3): +// every BMP code unit whose `toUpperCase()` is one other code unit, unless +// that maps a non-ASCII unit to ASCII. +// +// Run from the repository root with the Node in .nvmrc: +// node rust/scripts/generate-reg-exp-tables.mjs +import {writeFileSync} from 'node:fs'; + +const OUT = 'rust/lexical-editor/src/markdown/reg_exp_tables.rs'; + +/** The ranges of code points `re` matches, as `[first, last]` pairs. */ +function ranges(re) { + const out = []; + let start = -1; + for (let cp = 0; cp <= 0x110000; cp++) { + const matches = cp < 0x110000 && re.test(String.fromCodePoint(cp)); + if (matches && start < 0) { + start = cp; + } else if (!matches && start >= 0) { + out.push([start, cp - 1]); + start = -1; + } + } + return out; +} + +const hex = n => '0x' + n.toString(16).toUpperCase().padStart(4, '0'); + +function table(name, doc, re) { + const rows = ranges(re) + .map(([a, b]) => ` (${hex(a)}, ${hex(b)}),`) + .join('\n'); + return `/// ${doc}\npub(super) const ${name}: &[(u32, u32)] = &[\n${rows}\n];\n`; +} + +const canonical = []; +for (let c = 0; c < 0x10000; c++) { + if (c >= 0xd800 && c <= 0xdfff) { + continue; + } + const upper = String.fromCharCode(c).toUpperCase(); + if (upper.length !== 1) { + continue; + } + const u = upper.charCodeAt(0); + if (u === c || (c >= 128 && u < 128)) { + continue; + } + canonical.push(` (${hex(c)}, ${hex(u)}),`); +} + +const out = `//! @generated by \`rust/scripts/generate-reg-exp-tables.mjs\` from Node +//! ${process.version}'s regular expressions and \`toUpperCase\`. Do not edit +//! by hand; run the script again with the Node in \`.nvmrc\`. + +${table('PUNCTUATION', '`\\p{P}`: General_Category Punctuation.', /\p{P}/u)} +${table('SYMBOL', '`\\p{S}`: General_Category Symbol.', /\p{S}/u)} +${table('LETTER', '`\\p{L}`: General_Category Letter.', /\p{L}/u)} +${table('NUMBER', '`\\p{N}`: General_Category Number.', /\p{N}/u)} +${table('SPACE_SEPARATOR', '`\\p{Zs}`: General_Category Space_Separator.', /\p{Zs}/u)} +/// \`Canonicalize\` without the \`u\` flag: each BMP code unit that the \`i\` +/// flag reads as another, with the one it reads as, sorted. +pub(super) const CANONICALIZE: &[(u16, u16)] = &[ +${canonical.join('\n')} +]; +`; + +writeFileSync(OUT, out);