diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a14e3d3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## 0.1.14 — 2026-10-18 + +- Execute bounded start-anchored literal runs directly from immutable program data, without DFA storage or evaluator scratch. +- Compare ASCII subjects as original bytes, retaining lossless UTF-16/WTF-8 cursor semantics for other storage and Unicode starts between surrogate halves. +- Expose `Program::literal_prefix()` and `LiteralPrefix::find`; DFA entry points share the exact prefix reader and report a zero-word minimum for these programs. +- Advance the relocatable program format to 16 and validate the prefix descriptor against the complete instruction sequence; older program formats are rejected. diff --git a/CLAUDE.md b/CLAUDE.md index b004c71..c4b33d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ One compiler, one program format, one semantics with two exact executors: the ev **Pattern text → program.** `compiler::compile` parses into a caller-owned arena of `Node`/`Range` scratch (indices only, no source pointers survive) and emits a validated immutable `u32` program. `compiler::prepare` splits the same pipeline so the host can drop its pattern borrow before allocating final output: `Prepared::required_words()` gives the exact size, `emit` consumes the plan. Emission walks the topologically ordered arena iteratively — no native-stack recursion on long concatenations, no per-count expansion of `a{n}`. Submodules under `src/compiler/` add optimizer analyses (`admission`, `candidate`, `classes`, `repetition`) and grammar corners (`escapes` for Annex B, `names`). -**Program format** (`src/program.rs`). One relocatable buffer: header, three words per instruction, class-table pairs, eight-word repeat records, a filter section for properties of strings, optional packed name section. Everything is relative offsets/indices — never a host pointer. `Program::from_words` revalidates opcodes, operand ranges, property IDs, branch/table bounds and name/capture lists under a work budget, so a relocated or re-read buffer is always checked. `HEADER` and `VERSION` in `src/program.rs` are authoritative (currently 11 words / version 15); older versions are rejected outright, and a few docs still quote an earlier number. Word 9 (leading-literal run) is *re-derived* from the instructions during validation rather than trusted — hint words that can't be re-derived (words 7/8) are only allowed to be conservative, never to supply capture semantics. +**Program format** (`src/program.rs`). One relocatable buffer: header, three words per instruction, class-table pairs, eight-word repeat records, a filter section for properties of strings, optional packed name section. Everything is relative offsets/indices — never a host pointer. `Program::from_words` revalidates opcodes, operand ranges, property IDs, branch/table bounds and name/capture lists under a work budget, so a relocated or re-read buffer is always checked. `HEADER` and `VERSION` in `src/program.rs` are authoritative (currently 11 words / version 16); older versions are rejected outright, and a few docs still quote an earlier number. Word 9 (leading-literal run) is *re-derived* from the instructions during validation rather than trusted — hint words that can't be re-derived (words 7/8) are only allowed to be conservative, never to supply capture semantics. **Subject access** (`src/input.rs`). `Input` borrows original storage — `&str`, generalized WTF-8 bytes (lone and separately-encoded surrogates accepted), or host `&[u16]`. **Never build a UTF-16 conversion buffer or copy the subject**; that is the core project constraint. All public positions count UTF-16 units, including positions between surrogate halves. ASCII byte storage gets a distinct internal representation with O(1) seeks; non-ASCII byte cursors pack a half-pair flag into the high bit of the byte offset. diff --git a/Cargo.lock b/Cargo.lock index 95f1557..be4d212 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,4 +4,4 @@ version = 4 [[package]] name = "perex" -version = "0.1.13" +version = "0.1.14" diff --git a/Cargo.toml b/Cargo.toml index f4b77df..84d29cd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "perex" -version = "0.1.13" +version = "0.1.14" edition = "2024" rust-version = "1.88" description = "An ECMAScript regex engine designed for explicit host memory ownership" @@ -21,6 +21,7 @@ include = [ "src/**/*.rs", "Cargo.toml", "README.md", + "CHANGELOG.md", "LICENSE", "docs/**/*.md", "third_party/unicode/17.0.0/LICENSE.txt", diff --git a/README.md b/README.md index b74c821..db34850 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,8 @@ the current offset when it shortens the seek. Adjacent reads need no prefix rescan or subject index. Seeking still consumes the caller's cumulative work budget in bounded steps; retargeting cannot revive a failed reader. +Release notes are in [the changelog](CHANGELOG.md). + ## Development ```sh diff --git a/docs/engine.md b/docs/engine.md index cae9fdd..eed390c 100644 --- a/docs/engine.md +++ b/docs/engine.md @@ -20,7 +20,7 @@ Unicode-mode [character properties](properties.md), including general categories `compile` receives a borrowed pattern, caller-owned `Node`/`Range` scratch, final `u32` storage and a work budget. Nodes contain indices only. The emitter walks the topologically ordered arena iteratively: a long concatenation does not recurse on the native stack, and a counted repetition is not expanded into one program copy per count. Group parsing has an explicit nesting limit of 128; sizes/counters use checked arithmetic and u32 limits. These remain implementation limits, not claims that deeper/larger valid expressions are invalid JavaScript. -The output is one immutable relocatable buffer: an eleven-word header, three words per instruction, class-table pairs, eight-word repetition records, a filter section for the [properties of strings](sets.md), and an optional [name section](names.md). A class-table pair is a literal interval or shared Unicode property reference/complement. Capture indices are u32, with tests above 254 groups. The native-u32 format needs four-byte alignment and is versioned but unstable; version 15 adds the validated [automaton eligibility](dfa.md) bit to word 7, retaining version 14's first-character branch instruction and stored start-anchored claim described in [branches](branch.md), and version 13's properties of strings — an instruction naming a shared set of sequences and the filter an operator left on it — and version 10's final-character descriptor for strict end-anchored patterns, version 9's [sorted literal classes](classes.md), version 8's candidate-start descriptor and version 7's consuming-atom repeat records and lexical flag instructions. Older versions are rejected; this is not cross-endian serialization. `Program::from_words` validates all opcodes, operand ranges, property IDs/modes and branch/table bounds, packed identifier names and capture-index lists under a work budget. It also validates the exact five-instruction shape and capture exclusion for optimized atom repeats. It does not prove arbitrary hand-authored bytecode has compiler-generated control structure or that a candidate/admission hint is a semantic consequence of its instructions; execution can reject invalid control/capture states and is always work-limited. +The output is one immutable relocatable buffer: an eleven-word header, three words per instruction, class-table pairs, eight-word repetition records, a filter section for the [properties of strings](sets.md), and an optional [name section](names.md). A class-table pair is a literal interval or shared Unicode property reference/complement. Capture indices are u32, with tests above 254 groups. The native-u32 format needs four-byte alignment and is versioned but unstable; version 16 adds the exact [literal-prefix executor](literal-prefix.md) descriptor; version 15 added the validated [automaton eligibility](dfa.md) bit to word 7, retaining version 14's first-character branch instruction and stored start-anchored claim described in [branches](branch.md), and version 13's properties of strings — an instruction naming a shared set of sequences and the filter an operator left on it — and version 10's final-character descriptor for strict end-anchored patterns, version 9's [sorted literal classes](classes.md), version 8's candidate-start descriptor and version 7's consuming-atom repeat records and lexical flag instructions. Older versions are rejected; this is not cross-endian serialization. `Program::from_words` validates all opcodes, operand ranges, property IDs/modes and branch/table bounds, packed identifier names and capture-index lists under a work budget. It also validates the exact five-instruction shape and capture exclusion for optimized atom repeats. It does not prove arbitrary hand-authored bytecode has compiler-generated control structure or that a candidate/admission hint is a semantic consequence of its instructions; execution can reject invalid control/capture states and is always work-limited. [Required-text admission](admission.md) can reject a long search when a compiler-proved literal/class condition is absent from the original subject. It uses existing instructions and header bits, without increasing program size. Presence still executes the same VM for complete answers. diff --git a/docs/leading.md b/docs/leading.md index 66bff38..84f09ec 100644 --- a/docs/leading.md +++ b/docs/leading.md @@ -23,7 +23,7 @@ because word 7 already carries that claim. ASCII bytes cannot occur inside a multibyte UTF-8/WTF-8 encoding, so a run of ASCII characters is also a claim about original bytes. -Program format 15 has eleven header words. Word 9's low byte holds the run's +Program format 16 has eleven header words. Word 9's low byte holds the run's character count, or zero when disabled; its top bit is the separate forward-admission claim described in [admission](admission.md). The word costs four used program bytes, and the two claims share it rather than adding another. diff --git a/docs/literal-prefix.md b/docs/literal-prefix.md new file mode 100644 index 0000000..45a348b --- /dev/null +++ b/docs/literal-prefix.md @@ -0,0 +1,26 @@ +# Start-anchored literal execution + +Program format 16 adds word 7 bit 26. The compiler derives it only for the +complete instruction sequence `SAVE 0; START; CHAR+; SAVE 1; MATCH`, with +one capture (group zero) and at most 32 characters. Validation re-derives +it and rejects either a forged or a missing claim. The instructions remain +the program data; there is no second program, native code or retained state. + +`Program::literal_prefix()` obtains an exact executor. Its `find` compares +those literal characters with the original input and returns group zero's +span. ASCII byte storage is compared directly without constructing cursor +state; other storage uses the lossless cursor. Multiline assertions, case folding, captures and other suffixes +use the ordinary evaluator or automaton. Unicode starts between a leading +surrogate pair's halves normalize exactly as the evaluator does. Work is +charged for the check and each compared character; the 32-character bound +keeps synchronous work bounded. No scratch or cache storage is needed. +The DFA entry points dispatch these programs to the same executor before +reading cache storage, and `minimum_words` returns zero for them. + +## Verification + +`cargo test --test literal_prefix` compares complete evaluator answers for +ASCII, non-ASCII, astral and lone-surrogate inputs in original UTF-16 and +WTF-8 storage, all starts, global/sticky/Unicode flags, relocation, work +exhaustion and forged descriptors. `cargo test --test dfa` checks both +bounds and Boolean entry points against the evaluator. diff --git a/src/compiler/prepared.rs b/src/compiler/prepared.rs index d145a45..93a644a 100644 --- a/src/compiler/prepared.rs +++ b/src/compiler/prepared.rs @@ -281,6 +281,9 @@ impl Prepared<'_> { } else { 0 }; + if crate::literal::eligible(Program { words: output }) { + output[7] |= crate::program::LITERAL_PREFIX; + } output[8] = parser.end_candidate_descriptor(root); // Derived from the emitted instructions with the same function the // validator re-runs, so the stored word cannot disagree with the program. diff --git a/src/dfa.rs b/src/dfa.rs index 9f765d3..7dd7bcf 100644 --- a/src/dfa.rs +++ b/src/dfa.rs @@ -178,6 +178,9 @@ pub fn eligible(program: Program<'_>) -> bool { /// The least cache storage, in words, that `program` can be searched in, or /// `None` when it is not eligible. More storage holds more states. pub fn minimum_words(program: Program<'_>) -> Option { + if program.literal_prefix().is_some() { + return Some(0); + } if !eligible(program) { return None; } @@ -287,6 +290,12 @@ fn search( budget: &mut Budget, bounds: bool, ) -> Result { + if let Some(prefix) = program.literal_prefix() { + return Ok(match prefix.find(input, start_utf16, budget)? { + Some(span) => Outcome::Matched(bounds.then_some(span)), + None => Outcome::NoMatch, + }); + } // A validated program holds its whole header; reading it as one array // checks that once. let Some(words) = program.words.first_chunk::
() else { diff --git a/src/lib.rs b/src/lib.rs index cfda95f..7653d8f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ pub mod binding; pub mod input; +pub mod literal; pub mod span; mod casefold; diff --git a/src/literal.rs b/src/literal.rs new file mode 100644 index 0000000..4e04f92 --- /dev/null +++ b/src/literal.rs @@ -0,0 +1,109 @@ +//! Exact execution of bounded start-anchored literal programs. This is a +//! reading of the same immutable instructions as the evaluator, with no +//! automaton, scratch, allocation, or retained subject storage. +use crate::{Budget, executor::ExecError, input::Input, program::*, span::Span}; + +/// The largest run answered without yielding. Longer programs use the ordinary +/// resumable evaluator or automaton, preserving the host's polling openings. +const MAX_CHARS: usize = 32; + +pub(crate) fn eligible(p: Program<'_>) -> bool { + let count = p.instructions(); + p.capture_count() == 1 + && (5..=MAX_CHARS + 4).contains(&count) + && p.instruction(0) == [SAVE, 0, 0] + && p.instruction(1)[0] == START + && p.instruction(count - 2) == [SAVE, 1, 0] + && p.instruction(count - 1)[0] == MATCH + && (2..count - 2).all(|pc| p.instruction(pc)[0] == CHAR) +} + +/// An exact, capture-free `^` followed by literal characters. Obtained only +/// from a validated program whose descriptor was derived from its instructions. +#[derive(Clone, Copy)] +pub struct LiteralPrefix<'a> { + program: Program<'a>, +} + +impl<'a> Program<'a> { + /// A bounded literal-prefix executor, or `None` for every other program. + /// In particular, multiline `^`, folded characters, captures and trailing + /// assertions remain the ordinary evaluator's or automaton's responsibility. + #[inline] + pub fn literal_prefix(self) -> Option> { + (self.words()[7] & LITERAL_PREFIX != 0).then_some(LiteralPrefix { program: self }) + } +} + +impl LiteralPrefix<'_> { + /// The complete group-zero answer the evaluator would produce at or after + /// `start_utf16`. Reads the original subject; no view or address is retained. + /// Work exhaustion is an error, never a no-match answer. + #[inline] + pub fn find( + self, + input: Input<'_>, + start_utf16: usize, + budget: &mut Budget, + ) -> Result, ExecError> { + budget.charge(1).map_err(|_| ExecError::WorkLimit)?; + // A counted ASCII subject needs no cursor state or decoding. Compare + // the original bytes directly with the immutable CHAR operands; a + // non-ASCII operand simply cannot match this storage. + if let Some(bytes) = input.ascii_bytes() { + if start_utf16 != 0 { + return Ok(None); + } + let count = self.program.instructions() - 4; + for index in 0..count { + budget.charge(1).map_err(|_| ExecError::WorkLimit)?; + if bytes.get(index).copied().map(u32::from) + != Some(self.program.instruction(index + 2)[1]) + { + return Ok(None); + } + } + return Ok(Span::new(0, count)); + } + self.find_cursor(input, start_utf16, budget) + } + + // Keep the lossless cursor machinery outside the ASCII caller's frame. + #[inline(never)] + fn find_cursor( + self, + input: Input<'_>, + start_utf16: usize, + budget: &mut Budget, + ) -> Result, ExecError> { + let unicode = self.program.words()[2] & U != 0; + // Unicode RegExpBuiltinExec can normalize a start between a leading + // surrogate pair's halves back to zero. No later start can reach ^. + if start_utf16 != 0 { + if start_utf16 != 1 || !unicode { + return Ok(None); + } + let Some(mut start) = input.cursor_at(1) else { + return Ok(None); + }; + start.normalize_unicode_start(); + if start.position() != 0 { + return Ok(None); + } + } + let mut cursor = input.cursor(); + for pc in 2..self.program.instructions() - 2 { + budget.charge(1).map_err(|_| ExecError::WorkLimit)?; + let expected = self.program.instruction(pc)[1]; + let actual = if unicode { + cursor.next_point() + } else { + cursor.next_unit().map(u32::from) + }; + if actual != Some(expected) { + return Ok(None); + } + } + Ok(Span::new(0, cursor.position())) + } +} diff --git a/src/program.rs b/src/program.rs index fb221a5..eac429a 100644 --- a/src/program.rs +++ b/src/program.rs @@ -3,7 +3,7 @@ use crate::{Budget, properties}; pub(crate) const HEADER: usize = 11; pub(crate) const MAGIC: u32 = 0x50525831; -pub(crate) const VERSION: u32 = 15; +pub(crate) const VERSION: u32 = 16; pub(crate) const U: u32 = 1; pub(crate) const M: u32 = 2; pub(crate) const S: u32 = 4; @@ -104,6 +104,9 @@ pub(crate) const ANCHORED: u32 = 1 << 24; /// and every repeat's counter fits its state key. Derived by [`derive_dfa`] /// and re-derived on validation; see `docs/dfa.md`. pub(crate) const DFA: u32 = 1 << 25; +/// Word 7 flag: the entire program is a bounded, capture-free start-anchored +/// literal run. Re-derived from the instructions during validation. +pub(crate) const LITERAL_PREFIX: u32 = 1 << 26; /// Largest repeat bound the automaton counts to: a maximum, or the minimum of /// an unbounded repeat. pub(crate) const DFA_COUNT_MAX: u32 = 1023; @@ -632,7 +635,7 @@ impl<'a> Program<'a> { // semantic guarantee belongs to the compiler. for (index, &candidate) in words[7..9].iter().enumerate() { let descriptor = candidate & DESCRIPTOR; - if index == 0 && candidate & !(DESCRIPTOR | ANCHORED | DFA) != 0 { + if index == 0 && candidate & !(DESCRIPTOR | ANCHORED | DFA | LITERAL_PREFIX) != 0 { return Err(bad); } let lo = (descriptor >> 8) & 255; @@ -867,7 +870,9 @@ impl<'a> Program<'a> { budget .charge(p.instructions() / 8 + 1) .map_err(|_| ProgramError::WorkLimit)?; - if (words[7] & DFA != 0) != derive_dfa(p) { + if (words[7] & LITERAL_PREFIX != 0) != crate::literal::eligible(p) + || (words[7] & DFA != 0) != derive_dfa(p) + { return Err(bad); } Ok(p) diff --git a/tests/atom_repeat.rs b/tests/atom_repeat.rs index 78951bc..8951eaf 100644 --- a/tests/atom_repeat.rs +++ b/tests/atom_repeat.rs @@ -213,7 +213,7 @@ fn every_insufficient_work_allowance_preserves_output() { #[test] fn optimized_record_validation_rejects_shape_and_version_corruption() { let w = words("^([a-z]+)$", ""); - assert_eq!(w[1], 15); + assert_eq!(w[1], 16); let table = 11 + w[4] as usize * 3 + w[5] as usize * 2; assert_eq!(w[table + 7], 1); let entry = w[table + 3] as usize - 2; diff --git a/tests/classes.rs b/tests/classes.rs index 28189f1..bad91a5 100644 --- a/tests/classes.rs +++ b/tests/classes.rs @@ -57,7 +57,7 @@ fn members() -> String { fn sorted_classes_preserve_original_membership_and_reject_invalid_order() { let source = format!("[{}]", members()); let words = program(&source, "u"); - assert_eq!(words[1], 15); + assert_eq!(words[1], 16); assert_eq!(words[11 + 3], 27); // Sorted class following capture-zero start. for unit in 0..0x400u16 { let expected = (0..128).any(|i| unit == 0x100 + i * 3); diff --git a/tests/literal_prefix.rs b/tests/literal_prefix.rs new file mode 100644 index 0000000..2b249f0 --- /dev/null +++ b/tests/literal_prefix.rs @@ -0,0 +1,154 @@ +//! Complete answers against the evaluator, original storage and relocation. +use perex::{ + Budget, + compiler::{Node, Range, compile}, + executor::{ExecError, Frame, Scratch, Undo, find}, + input::Input, + program::Program, + span::Span, +}; + +fn compile_words(pattern: &str, flags: &str) -> Vec { + let mut nodes = vec![Node::default(); 1024]; + let mut ranges = vec![Range::default(); 1024]; + let mut words = vec![0; 8192]; + compile( + Input::utf8(pattern), + flags, + &mut nodes, + &mut ranges, + &mut words, + &mut Budget::new(1_000_000), + ) + .unwrap() + .words() + .to_vec() +} +fn view(words: &[u32]) -> Program<'_> { + Program::from_words(words, &mut Budget::new(1_000_000)).unwrap() +} +fn evaluator(program: Program<'_>, input: Input<'_>, start: usize) -> Option { + let mut registers = vec![0; program.register_count()]; + let mut frames = [Frame::default(); 64]; + let mut undo = [Undo::default(); 128]; + let mut captures = vec![None; program.capture_count()]; + let matched = find( + program, + input, + start, + Scratch { + registers: &mut registers, + frames: &mut frames, + undo: &mut undo, + }, + &mut captures, + &mut Budget::new(1_000_000), + ) + .unwrap(); + assert_eq!(captures.len(), 1, "the prefix answer is every capture"); + matched.then(|| captures[0].unwrap()) +} +fn agree(program: Program<'_>, input: Input<'_>) { + for start in 0..=input.len_utf16() + 1 { + let expected = evaluator(program, input, start); + let prefix = program + .literal_prefix() + .expect("fixture must use the exact prefix executor"); + assert_eq!( + prefix.find(input, start, &mut Budget::new(1000)).unwrap(), + expected, + "start {start}" + ); + let mut cache = []; + assert_eq!(perex::dfa::minimum_words(program), Some(0)); + assert_eq!( + perex::dfa::find(program, input, start, &mut cache, &mut Budget::new(1000)).unwrap(), + expected.map_or(perex::dfa::Found::NoMatch, perex::dfa::Found::Match) + ); + } +} +#[test] +fn complete_answers_all_flags_and_original_storage() { + for pattern in [ + "^--", + "^-", + "^no-", + r"^\u002d\x2d", + "^é", + "^😀", + r"^\ud800", + "^abcdefghijklmnopqrstuvwxyz012345", + ] { + for flags in ["", "g", "y", "gy", "u", "gu", "yu", "v", "dgs"] { + let words = compile_words(pattern, flags); + let program = view(&words); + for subject in [ + "", + "-", + "--", + "--option", + "no-option", + "x--", + "\n--", + "éclair", + "😀tail", + "a😀", + ] { + agree(program, Input::utf8(subject)); + let units: Vec<_> = subject.encode_utf16().collect(); + agree(program, Input::utf16(&units)); + } + agree(program, Input::wtf8(&[0xed, 0xa0, 0x80, b'x']).unwrap()); + // Separately encoded surrogate halves are valid generalized WTF-8. + agree( + program, + Input::wtf8(&[0xed, 0xa0, 0xbd, 0xed, 0xb8, 0x80, b'x']).unwrap(), + ); + } + } +} +#[test] +fn other_programs_decline_and_descriptor_cannot_be_forged() { + for (pattern, flags) in [ + ("^--", "m"), + ("^--", "i"), + ("^(--)", ""), + ("^--$", ""), + ("--", ""), + ("^--|x", ""), + ("^--?", ""), + ("^", ""), + ("^abcdefghijklmnopqrstuvwxyz0123456", ""), + ] { + let mut words = compile_words(pattern, flags); + assert!( + view(&words).literal_prefix().is_none(), + "/{pattern}/{flags}" + ); + words[7] |= 1 << 26; + assert!(Program::from_words(&words, &mut Budget::new(1_000_000)).is_err()); + } + let mut words = compile_words("^--", ""); + words[7] &= !(1 << 26); + assert!(Program::from_words(&words, &mut Budget::new(1_000_000)).is_err()); +} +#[test] +fn relocation_and_work_exhaustion() { + let mut words = compile_words("^--", ""); + let moved = words.clone(); + words.fill(0xdead_beef); + drop(words); + let prefix = view(&moved).literal_prefix().unwrap(); + assert_eq!( + prefix.find(Input::utf8("--abc"), 0, &mut Budget::new(3)), + Ok(Span::new(0, 2)) + ); + assert_eq!( + prefix.find(Input::utf8("--abc"), 0, &mut Budget::new(2)), + Err(ExecError::WorkLimit) + ); + assert_eq!( + prefix.find(Input::utf8("xx"), 0, &mut Budget::new(0)), + Err(ExecError::WorkLimit) + ); +} diff --git a/tests/modifiers.rs b/tests/modifiers.rs index 8f0b75e..e6be029 100644 --- a/tests/modifiers.rs +++ b/tests/modifiers.rs @@ -146,7 +146,7 @@ fn modifier_errors_and_instruction_corruption_are_explicit() { (r"(?a)|(?b)\k", "i"), ] { let words = program(source, flags).unwrap(); - assert_eq!(words[1], 15); + assert_eq!(words[1], 16); for pc in 0..words[4] as usize { let at = 11 + pc * 3; if (19..=26).contains(&words[at]) {