diff --git a/CLAUDE.md b/CLAUDE.md index c5abaf2..f732396 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,7 +57,7 @@ One compiler, one program format, one evaluator. There is no fallback engine and **Pattern text → program.** `compiler::compile` parses into a caller-owned arena of `Node`/`Range` scratch (indices only, no source pointers survive) and emits a validated immutable `u32` program. `compiler::prepare` splits the same pipeline so the host can drop its pattern borrow before allocating final output: `Prepared::required_words()` gives the exact size, `emit` consumes the plan. Emission walks the topologically ordered arena iteratively — no native-stack recursion on long concatenations, no per-count expansion of `a{n}`. Submodules under `src/compiler/` add optimizer analyses (`admission`, `candidate`, `classes`, `repetition`) and grammar corners (`escapes` for Annex B, `names`). -**Program format** (`src/program.rs`). One relocatable buffer: header, three words per instruction, class-table pairs, eight-word repeat records, a filter section for properties of strings, optional packed name section. Everything is relative offsets/indices — never a host pointer. `Program::from_words` revalidates opcodes, operand ranges, property IDs, branch/table bounds and name/capture lists under a work budget, so a relocated or re-read buffer is always checked. `HEADER` and `VERSION` in `src/program.rs` are authoritative (currently 11 words / version 13); older versions are rejected outright, and a few docs still quote an earlier number. Word 9 (leading-literal run) is *re-derived* from the instructions during validation rather than trusted — hint words that can't be re-derived (words 7/8) are only allowed to be conservative, never to supply capture semantics. +**Program format** (`src/program.rs`). One relocatable buffer: header, three words per instruction, class-table pairs, eight-word repeat records, a filter section for properties of strings, optional packed name section. Everything is relative offsets/indices — never a host pointer. `Program::from_words` revalidates opcodes, operand ranges, property IDs, branch/table bounds and name/capture lists under a work budget, so a relocated or re-read buffer is always checked. `HEADER` and `VERSION` in `src/program.rs` are authoritative (currently 11 words / version 14); older versions are rejected outright, and a few docs still quote an earlier number. Word 9 (leading-literal run) is *re-derived* from the instructions during validation rather than trusted — hint words that can't be re-derived (words 7/8) are only allowed to be conservative, never to supply capture semantics. **Subject access** (`src/input.rs`). `Input` borrows original storage — `&str`, generalized WTF-8 bytes (lone and separately-encoded surrogates accepted), or host `&[u16]`. **Never build a UTF-16 conversion buffer or copy the subject**; that is the core project constraint. All public positions count UTF-16 units, including positions between surrogate halves. ASCII byte storage gets a distinct internal representation with O(1) seeks; non-ASCII byte cursors pack a half-pair flag into the high bit of the byte offset. diff --git a/Cargo.lock b/Cargo.lock index 556bc72..16cf11c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,4 +4,4 @@ version = 4 [[package]] name = "perex" -version = "0.1.11" +version = "0.1.12" diff --git a/Cargo.toml b/Cargo.toml index 2913c97..485b664 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "perex" -version = "0.1.11" +version = "0.1.12" edition = "2024" rust-version = "1.88" description = "An ECMAScript regex engine designed for explicit host memory ownership" diff --git a/README.md b/README.md index 62dce9a..b19b624 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,8 @@ Case equivalence uses generated Unicode 17.0.0 data under the [Unicode License V [Test262 pattern conformance](docs/conformance.md) compares every pattern harvested from the suite's regular-expression tests against Node, for syntax acceptance and complete match answers. There is no remaining gap: 48,718 cases compared, nothing unsupported and no difference, on the harvest recorded there. +[Branches](docs/branch.md) lay alternation out flat and skip an alternative, and its backtracking frame, when the next character cannot begin it, by a first-character set re-derived from the instructions during validation. + [Leading literal starts](docs/leading.md) reject an impossible start with a byte comparison instead of an initialized trial. The claim is re-derived from the instructions during program validation, so it cannot disagree with them. [Required-text admission](docs/admission.md) also bounds where a match can begin when the condition is one the match itself consumes, so text present only before every possible start stops the search instead of retrying it. diff --git a/bench/Cargo.lock b/bench/Cargo.lock index ef6b0f1..5c77dc8 100644 --- a/bench/Cargo.lock +++ b/bench/Cargo.lock @@ -80,7 +80,7 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "perex" -version = "0.1.11" +version = "0.1.12" [[package]] name = "perex-bench" diff --git a/docs/branch.md b/docs/branch.md new file mode 100644 index 0000000..083b45f --- /dev/null +++ b/docs/branch.md @@ -0,0 +1,132 @@ +# Branches + +An alternation that fails on its first alternative used to pay for every +other one in turn: a frame popped, a rollback, a dispatcher round trip, and the +alternative's first instruction executed only to fail. Claude Code's emoji +pattern, an alternation of eleven top-level alternatives and about two hundred +nested ones, cost 6,664 instructions per search against the one-character +subject `"1"`; V8's whole `test` call costs about 240. Format 14 changes three +things, each a property of the program alone, and keeps the one evaluator. + +## Flat layout + +`a|b|c` parses as `(a|b)|c`. Emitted as parsed, the first alternative was +entered under one live frame per alternative, and a success left through one +jump per nesting level. Ordered choice is associative, so the emitter now lays +a left-nested chain out flat: + +``` +SPLIT → a, R1 a JUMP end +R1: SPLIT → b, R2 b JUMP end +R2: c +end: +``` + +Each alternative but the last sits behind its own branch, whose frame resumes +at the next alternative; each jumps straight to the end. The size is the same +— two instructions per alternation either way — and the order in which +alternatives are tried, and so every answer, is unchanged. The left spine's +inner alternation nodes become no instruction. + +## First-character sets + +A new instruction, `BRANCH` (opcode 31), is a `SPLIT` whose first arm begins +at the next instruction and whose operand `a` is the set of characters that arm +can consume first. When the character the arm would read — in the direction the +code runs, so backwards inside a lookbehind — is absent or outside the set, the +arm cannot match: trying it would fail and resume at `b` with nothing changed. +The evaluator goes to `b` directly, without a frame. When `b` is itself a +`BRANCH`, it is decided against the same character in the same step, so a +chain of alternatives that cannot begin here costs one read and one comparison +each. Each branch passed over is charged one work unit, exactly as executing it +would be, and the chain stops at the quantum the next instruction would have +been given, so totals do not depend on how a search is paused. + +The set is a 32-bit mask of character buckets. ASCII characters fall in 24 +buckets by their low five bits, so both cases of a letter share one; every +other character falls in one of 8 buckets by its high bits, which separates +surrogate halves, the symbol blocks and the CJK blocks. It is derived from the +emitted instructions by `derive_first_mask`: from the arm's first instruction, +capture `SAVE`s and position assertions are passed over (dropping a condition +only widens the set), jumps are followed, and both arms of a nested branch are +taken; each path ends at its first consuming instruction, which contributes +its characters. A path that can succeed without consuming, or that reaches +anything else — an assertion body, a repeat, a backreference, a property, a +complement, a folded class, a folded non-ASCII character — makes the set +unknown, and the branch stays a `SPLIT`. A folded ASCII letter under `u` also +admits every non-ASCII bucket, for U+212A and U+017F. The walk inspects at most +32 instructions and eight pending paths. + +`Program::from_words` derives every `BRANCH` mask again and rejects a program +whose operand differs, or whose mask admits everything. So, like the leading +run, a branch's set cannot claim less than its arm can consume. + +## Start-anchored claim and start registers + +The claim that every match begins at the subject's start was derived at the +start of every search; on the nested emoji alternation that walk was 313 of the +654 instructions a miss on a letter cost. It is now stored as word 7's bit 24 +and re-derived on validation; see [candidate starts](candidate.md#start-anchored-starts). + +A trial cleared every register before it began: two per capture and two per +repeat. The emoji pattern has 194 repeats, one per `?`, so every start cleared +390 registers and charged 390 work units for it. A repeat's registers are +written by its `REPEAT_INIT`, the only way into the repeat, before anything in +it reads them, and a frame or undo entry that restores an older value restores +it to a point before that write. A trial now clears only the capture registers. + +## Measurement + +Instructions per search (`instructions:u`, difference between 200,000 searches +and none) of `Search::run` over a counted binding at a quantum of 4096, as a +host calls it per `test`, against 0.1.11 in the same harness. "cc mix" cycles +the subjects `t o k e n 1 7` and a space, the graphemes string-width tests in +Claude Code's streamed output. + +| Case | 0.1.11 | 0.1.12 | Change | +|---|---:|---:|---:| +| `/^\p{DI}$/u`, cc mix | 432 | 391 | −9.5% | +| `/^\p{DI}$/u` on 中 | 1,505 | 1,459 | −3.1% | +| emoji `/g`, cc mix | 2,212 | 946 | −57.2% | +| emoji on `t` | 728 | 406 | −44.2% | +| emoji on `1` | 6,664 | 2,569 | −61.4% | +| emoji on 中 | 6,741 | 1,637 | −75.7% | +| emoji on 😀, a match | 11,014 | 5,638 | −48.8% | +| `/ab\|cd\|ef\|gh/` on `xxgh` | 3,183 | 2,124 | −33.3% | +| `/(foo\|bar\|baz)=(\d+)/` on `a=1&baz=42` | 4,241 | 3,819 | −10.0% | +| `/a/` on `t` | 446 | 406 | −9.0% | +| `/a/` on `a`, a match | 1,206 | 1,162 | −3.6% | +| `/qq_[0-9]+/` on `record_12345` | 498 | 458 | −8.0% | +| `/\w+/g` on `foo bar` | 2,231 | 2,189 | −1.9% | +| `/^[a-z]+_[0-9]+$/`, a match | 3,041 | 2,993 | −1.6% | +| `/\s+$/` on a trailing run | 4,143 | 4,095 | −1.2% | + +No case measured slower. What remains is mostly the fixed cost of a search +that enters the evaluator: about 1,100 instructions to match `/a/` against +`"a"`, spread over a dozen phase round trips (admission, the start scan, the +seek, clearing, the trial, validation) of 30 to 100 instructions each. + +## Verification + +`tests/branch.rs` compares 36 patterns against the same patterns with every +alternative, at every depth, written between empty lookaheads — whose arms +begin with an assertion, so no branch carries a set and every arm is tried — +from every start including past the end, over ASCII, folded, non-ASCII, +astral and keycap subjects: alternations of literals, classes, captures and +named groups, lookbehind and negative lookbehind alternatives, folding with +U+212A and U+017F under `i` and `iu`, empty and end-asserting alternatives, +backreferences, repeated alternations and sticky searches. It checks the flat +layout and source order, that repeat registers holding arbitrary values do not +change any answer, and that validation rejects every single-bit change to a +mask, a mask admitting everything, either flip of the anchored bit and a +reserved bit above it. Six injected faults are each caught by the suite: +non-ASCII characters bucketed as ASCII, `REPEAT_INIT` leaving its position +register unwritten, a folded non-ASCII character claiming only its own bucket, +an arm that can match empty contributing nothing, a chain resuming one +instruction past the next alternative, and a trial clearing no capture +register. The engine differential (plain, one- and seventeen-unit resumption +with relocation and growth, from positions, decided in one call), the Test262 +pattern corpus, and the input, candidate, classes, names, legacy, admission, +lookbehind, run-skip, repetition, modifiers, bound, end-candidate, sets, +sequences, Unicode-sets, casefold, properties and atom-filter harnesses, and +the native tier's check, report no differences. diff --git a/docs/candidate.md b/docs/candidate.md index e027ef6..6d0bdc9 100644 --- a/docs/candidate.md +++ b/docs/candidate.md @@ -35,6 +35,8 @@ Program format 10 has nine header words, retaining the first-character descripto no ASCII start is possible, or `2 | (lo << 8) | (hi << 16)` for an inclusive interval with `0 <= lo <= hi <= 127`. Reserved bits, malformed bounds and old format versions are rejected. The additional word costs four used program bytes. +Since format 14 the descriptor occupies word 7's low 24 bits; bit 24 holds the +validated start-anchored claim and the bits above it are reserved. Bindings check the full eleven-word header when reacquiring a view. AOT and runtime programs must use this same format and its version checks. @@ -144,12 +146,10 @@ the 256 KiB literal lookbehind, which measured between 0 and 2.9 percent slower across five runs of 21 rounds each: its remainder is too long to scan, and it pays only for being told so. -The claim that a program is anchored at the subject's start is derived at the +The claim that a program is anchored at the subject's start was derived at the start of every search, and the same measurement priced that derivation at -1.3 ns of the empty search. It now decides the common case — a consuming -instruction straight after the entry `SAVE`s — before building its branch stack. -Without that, the misses were about 1 ns slower and the literal hits up to -1 ns slower; `[^0-9]+!` was 0.8 percent slower than before rather than 1.1. +1.3 ns of the empty search. Format 14 stores it instead, as a validated word 7 +flag; see [start-anchored starts](#start-anchored-starts). ### Checks @@ -273,14 +273,20 @@ where one attempt charges a handful. Perry's `test` and `exec` calls pay a search per call, so a failing anchored pattern paid it on every call (Perry issue #10166). -The claim is derived from the instructions rather than stored, like the leading -run, so no program word can assert it falsely. From instruction zero, capture +The claim is derived from the instructions by the compiler and stored as word +7's bit 24, above the descriptor; `Program::from_words` derives it again and +rejects a program whose bit disagrees, as it does for the leading run, so no +program word can assert it falsely. Format 13 derived it at the start of every +search instead, which for an alternation of ten nested branches walked the +branch stack to its limit each time: 313 of the 654 instructions a miss of +Claude Code's emoji pattern cost. A search now reads the bit. From instruction zero, capture bookkeeping is skipped and a `SPLIT` is followed into both branches; every path must reach `START`. Anything first that could consume or match elsewhere — a character, a class, a repeat, an assertion, a jump, or `^` under `m` — disables it, so `^a|b`, `(?:^)?a`, `(?:^a)+`, `(?<=^)a` and `(?m)^a` are searched as -before. The walk inspects at most 128 instructions and eight pending branches, -once per search, on entering admission. +before. A `BRANCH` is followed like a `SPLIT`. The walk inspects at most 128 +instructions and eight pending branches, once per compilation and once per +validation. A start-anchored search then behaves as a sticky one does for choosing starts: the candidate scan examines only its first position, and no later start is diff --git a/docs/engine.md b/docs/engine.md index 8ce4304..362b30e 100644 --- a/docs/engine.md +++ b/docs/engine.md @@ -20,7 +20,7 @@ Unicode-mode [character properties](properties.md), including general categories `compile` receives a borrowed pattern, caller-owned `Node`/`Range` scratch, final `u32` storage and a work budget. Nodes contain indices only. The emitter walks the topologically ordered arena iteratively: a long concatenation does not recurse on the native stack, and a counted repetition is not expanded into one program copy per count. Group parsing has an explicit nesting limit of 128; sizes/counters use checked arithmetic and u32 limits. These remain implementation limits, not claims that deeper/larger valid expressions are invalid JavaScript. -The output is one immutable relocatable buffer: an eleven-word header, three words per instruction, class-table pairs, eight-word repetition records, a filter section for the [properties of strings](sets.md), and an optional [name section](names.md). A class-table pair is a literal interval or shared Unicode property reference/complement. Capture indices are u32, with tests above 254 groups. The native-u32 format needs four-byte alignment and is versioned but unstable; version 13 adds the properties of strings — an instruction naming a shared set of sequences and the filter an operator left on it — retaining version 10's final-character descriptor for strict end-anchored patterns, version 9's [sorted literal classes](classes.md), version 8's candidate-start descriptor and version 7's consuming-atom repeat records and lexical flag instructions. Older versions are rejected; this is not cross-endian serialization. `Program::from_words` validates all opcodes, operand ranges, property IDs/modes and branch/table bounds, packed identifier names and capture-index lists under a work budget. It also validates the exact five-instruction shape and capture exclusion for optimized atom repeats. It does not prove arbitrary hand-authored bytecode has compiler-generated control structure or that a candidate/admission hint is a semantic consequence of its instructions; execution can reject invalid control/capture states and is always work-limited. +The output is one immutable relocatable buffer: an eleven-word header, three words per instruction, class-table pairs, eight-word repetition records, a filter section for the [properties of strings](sets.md), and an optional [name section](names.md). A class-table pair is a literal interval or shared Unicode property reference/complement. Capture indices are u32, with tests above 254 groups. The native-u32 format needs four-byte alignment and is versioned but unstable; version 14 adds the first-character branch instruction and the stored start-anchored claim described in [branches](branch.md), retaining version 13's properties of strings — an instruction naming a shared set of sequences and the filter an operator left on it — and version 10's final-character descriptor for strict end-anchored patterns, version 9's [sorted literal classes](classes.md), version 8's candidate-start descriptor and version 7's consuming-atom repeat records and lexical flag instructions. Older versions are rejected; this is not cross-endian serialization. `Program::from_words` validates all opcodes, operand ranges, property IDs/modes and branch/table bounds, packed identifier names and capture-index lists under a work budget. It also validates the exact five-instruction shape and capture exclusion for optimized atom repeats. It does not prove arbitrary hand-authored bytecode has compiler-generated control structure or that a candidate/admission hint is a semantic consequence of its instructions; execution can reject invalid control/capture states and is always work-limited. [Required-text admission](admission.md) can reject a long search when a compiler-proved literal/class condition is absent from the original subject. It uses existing instructions and header bits, without increasing program size. Presence still executes the same VM for complete answers. diff --git a/docs/leading.md b/docs/leading.md index 97595a1..b7a2e0f 100644 --- a/docs/leading.md +++ b/docs/leading.md @@ -23,7 +23,7 @@ because word 7 already carries that claim. ASCII bytes cannot occur inside a multibyte UTF-8/WTF-8 encoding, so a run of ASCII characters is also a claim about original bytes. -Program format 13 has eleven header words. Word 9's low byte holds the run's +Program format 14 has eleven header words. Word 9's low byte holds the run's character count, or zero when disabled; its top bit is the separate forward-admission claim described in [admission](admission.md). The word costs four used program bytes, and the two claims share it rather than adding another. diff --git a/src/binding.rs b/src/binding.rs index 8754f0d..3c3a9e9 100644 --- a/src/binding.rs +++ b/src/binding.rs @@ -212,6 +212,21 @@ impl BoundSubject { } } +/// Whether `words` begins with `header`. Every view checks this, so it is +/// written as one pass of word differences rather than a call to a general +/// slice comparison: the header is fixed-size and nearly always equal. +#[inline(always)] +fn same_header(words: &[u32], header: &[u32; crate::program::HEADER]) -> bool { + words.len() >= header.len() + && words[..header.len()] + .iter() + .zip(header) + .fold(0, |difference, (word, expected)| { + difference | (word ^ expected) + }) + == 0 +} + /// What validating a program established, as plain data: its length and /// header. Obtained only from a [`BoundProgram`] that validated it, and small /// enough for a host to keep beside the program it describes, so later @@ -269,10 +284,7 @@ impl BoundProgram

{ witness: ProgramWitness, ) -> Result>> { let result = storage - .with_words(|words| { - words.len() == witness.words - && words.get(..witness.header.len()) == Some(&witness.header) - }) + .with_words(|words| words.len() == witness.words && same_header(words, &witness.header)) .map_err(BoundProgramError::Resource) .and_then(|same| { if same { @@ -315,8 +327,7 @@ impl BoundProgram

{ ) -> Result> { self.storage .with_words(|words| { - if words.len() != self.words || words.get(..self.header.len()) != Some(&self.header) - { + if words.len() != self.words || !same_header(words, &self.header) { return Err(BoundProgramError::ChangedLayout); } Ok(f(Program { words })) diff --git a/src/compiler/prepared.rs b/src/compiler/prepared.rs index 8db23c0..ed74e6f 100644 --- a/src/compiler/prepared.rs +++ b/src/compiler/prepared.rs @@ -118,14 +118,44 @@ impl Prepared<'_> { parser.nodes[b as usize].reverse = n.reverse; } ALT => { - let len = parser.nodes[n.a as usize].len; - let right = pc + len + 2; - parser.nodes[n.a as usize].start = pc + 1; - parser.nodes[n.a as usize].reverse = n.reverse; - parser.nodes[n.b as usize].start = right; + // `a|b|c` parses as `(a|b)|c`. Nested as parsed, the + // first alternative is entered under one live frame per + // alternative and a success leaves through one jump per + // level. Ordered choice is associative, so the chain is + // laid out flat instead: each alternative but the last + // behind its own `SPLIT`, whose frame resumes at the next + // alternative, and each jumping straight to the end. The + // left spine's inner alternations are laid out here, so + // their own nodes become no instruction. The size is the + // same: two instructions per alternation either way. + let end = pc + n.len; + let last = parser.nodes[n.b as usize]; + let mut next = end - last.len; + parser.nodes[n.b as usize].start = next; parser.nodes[n.b as usize].reverse = n.reverse; - write(output, pc, SPLIT, pc + 1, right); - write(output, right - 1, JUMP, pc + n.len, 0); + let mut left = n.a; + loop { + parser.step()?; + let node = parser.nodes[left as usize]; + let (alternative, inner) = if node.kind == ALT { + parser.nodes[left as usize].start = 0; + (node.b, Some(node.a)) + } else { + (left, None) + }; + let len = parser.nodes[alternative as usize].len; + let split = next - len - 2; + parser.nodes[alternative as usize].start = split + 1; + parser.nodes[alternative as usize].reverse = n.reverse; + write(output, split, SPLIT, split + 1, next); + write(output, next - 1, JUMP, end, 0); + next = split; + match inner { + Some(inner) => left = inner, + None => break, + } + } + debug_assert_eq!(next, pc); } GROUP => { child(n.a, pc + 1, n.reverse); @@ -216,12 +246,36 @@ impl Prepared<'_> { output[at + 7] = 1; } } + // A branch whose first arm can only begin with characters from a known + // set carries that set, so a search skips the arm, and its frame, + // where the next character is outside it. Derived from the final + // instructions with the function the validator re-runs; an earlier + // branch already rewritten is followed exactly as a `SPLIT` would be. + for pc in 0..count as usize { + let at = HEADER + pc * 3; + if output[at] == SPLIT && output[at + 1] as usize == pc + 1 { + let mask = crate::program::derive_first_mask(output, pc + 1, parser.budget) + .map_err(|e| match e { + ProgramError::WorkLimit => CompileError::WorkLimit, + ProgramError::Invalid => CompileError::InvalidProgram, + })?; + if mask != crate::program::ALL_FIRST { + output[at] = crate::program::BRANCH; + output[at + 1] = mask; + } + } + } if parser.repeats != 0 { let hint = parser.admission(Program { words: output }, root)?; output[2] |= hint; } parser.candidate()?; - output[7] = parser.candidate_descriptor(root); + output[7] = parser.candidate_descriptor(root) + | if crate::program::derive_start_anchored(output, count as usize) { + crate::program::ANCHORED + } else { + 0 + }; output[8] = parser.end_candidate_descriptor(root); // Derived from the emitted instructions with the same function the // validator re-runs, so the stored word cannot disagree with the program. diff --git a/src/executor.rs b/src/executor.rs index cc573a6..05ec488 100644 --- a/src/executor.rs +++ b/src/executor.rs @@ -1371,18 +1371,24 @@ impl Vm<'_, '_, '_, '_> { } /// Clear the registers a trial starts from, in bounded steps, and enter the /// trial once they are all clear. + /// + /// Only the capture registers start a trial cleared. A repeat's two + /// registers follow them, and the only way into a repeat is its + /// `REPEAT_INIT`, which writes both before anything in the repeat reads + /// them; a frame or undo entry that restores an earlier value restores it + /// to a point before that write, where nothing reads it either. So a + /// pattern with many repeats, such as one `?` per alternative, does not + /// pay for clearing them at every start. fn initialize(&mut self, index: usize, available: usize) -> Result<(), ExecError> { - if index == self.program.register_count() { + let count = self.program.capture_count() * 2; + if index == count { self.begin_trial(); return self.skip_verified(); } - let end = self - .program - .register_count() - .min(index + available.min(256)); + let end = count.min(index + available.min(256)); self.charge(end - index)?; self.scratch.registers[index..end].fill(UNSET); - if end == self.program.register_count() { + if end == count { self.begin_trial(); self.skip_verified() } else { @@ -1840,6 +1846,33 @@ impl Vm<'_, '_, '_, '_> { self.push(b as usize, 0)?; self.state.pc = a as usize; } + BRANCH => { + // The arm begins at the next instruction, where `pc` already + // is. One that cannot consume the next character would fail + // and resume at `b` with nothing changed; go there directly. + // When `b` is the next alternative's own branch, it is decided + // here against the same character, and charged as executing + // it would be, so a chain of alternatives costs one read. + let mut ahead = self.cursor; + let bucket = read(&mut ahead, self.program.unicode(), self.state.reverse) + .map_or(0, |c| 1 << first_bucket(c)); + // The chain stays within the quantum the next instruction + // would have been given; past it, the trial loop takes the + // next branch as it takes any instruction. + let (mut mask, mut right, mut chained) = (a, b as usize, 1); + while mask & bucket == 0 { + let [op, a, b] = self.program.instruction(right); + if op != BRANCH || chained >= available { + self.state.pc = right; + return Ok(Step::Next); + } + chained += 1; + self.charge(1)?; + self.state.pc = right + 1; + (mask, right) = (a, b as usize); + } + self.push(right, 0)?; + } JUMP => self.state.pc = a as usize, START | START_M => { let mut before = self.cursor; diff --git a/src/executor/admission.rs b/src/executor/admission.rs index 43b6abd..3c3060d 100644 --- a/src/executor/admission.rs +++ b/src/executor/admission.rs @@ -121,8 +121,7 @@ impl Vm<'_, '_, '_, '_> { // Admission still runs for these: when the condition is absent // it rejects in one scan a single attempt whose backtracking // could otherwise cost far more than the scan. - self.state.anchored = - derive_start_anchored(self.program.words, self.program.instructions()); + self.state.anchored = self.program.start_anchored(); self.state.one_start = self.program.words[2] & Y != 0 || self.state.anchored; if !self.end_candidate()? { self.state.phase = Phase::Finished(false); diff --git a/src/executor/candidate.rs b/src/executor/candidate.rs index 33cabd1..da874c0 100644 --- a/src/executor/candidate.rs +++ b/src/executor/candidate.rs @@ -66,7 +66,7 @@ impl Vm<'_, '_, '_, '_> { /// scan's chunk, so the step stays inside the bound a pause documents. #[inline(always)] pub(super) fn short_start(&mut self) -> Result { - let descriptor = self.program.words[7]; + let descriptor = self.program.first_descriptor(); let from = self.state.requested_start; let Some(rest) = self.input.ascii_bytes().and_then(|bytes| bytes.get(from..)) else { return Ok(ShortStart::Unknown); @@ -104,11 +104,12 @@ impl Vm<'_, '_, '_, '_> { } pub(super) fn start_candidate(&mut self) { - self.state.phase = if self.program.words[7] != 0 && self.input.original_bytes().is_some() { - Phase::Candidate - } else { - Phase::Initialize(0) - }; + self.state.phase = + if self.program.first_descriptor() != 0 && self.input.original_bytes().is_some() { + Phase::Candidate + } else { + Phase::Initialize(0) + }; } /// What the scan should look for, for a program that carries a leading /// claim. @@ -134,15 +135,14 @@ impl Vm<'_, '_, '_, '_> { let mut depth = 0; let mut pc = self.program.leading_pc(); loop { - let [op, a, b] = self.program.instruction(pc); - if op == SPLIT { + if let Some((first, second)) = branch_arms(pc, self.program.instruction(pc)) { if depth == pending.len() { prefix.members = 0; return prefix; } - pending[depth] = b as usize; + pending[depth] = second; depth += 1; - pc = a as usize; + pc = first; continue; } if prefix.members == prefix.first.len() { @@ -205,15 +205,14 @@ impl Vm<'_, '_, '_, '_> { let mut pc = self.program.leading_pc(); loop { self.charge(1)?; - let [op, a, b] = self.program.instruction(pc); - if op == SPLIT { + if let Some((first, second)) = branch_arms(pc, self.program.instruction(pc)) { // The derivation proved the branch count fits. if depth == pending.len() { return Ok(true); } - pending[depth] = b as usize; + pending[depth] = second; depth += 1; - pc = a as usize; + pc = first; continue; } let mut matched = true; @@ -280,7 +279,7 @@ impl Vm<'_, '_, '_, '_> { return self.mixed_candidate_step(available); }; let start = self.cursor.position(); - let descriptor = self.program.words[7]; + let descriptor = self.program.first_descriptor(); if descriptor == 1 || start == bytes.len() { self.charge(1)?; self.state.phase = Phase::Finished(false); @@ -402,7 +401,7 @@ impl Vm<'_, '_, '_, '_> { self.state.phase = Phase::Finished(false); return Ok(()); } - let descriptor = self.program.words[7]; + let descriptor = self.program.first_descriptor(); let (lo, hi) = if descriptor == 1 { // No ASCII member is possible, but every non-ASCII lead byte must // still go through the ordinary matcher. diff --git a/src/program.rs b/src/program.rs index 57d67c9..a9b2895 100644 --- a/src/program.rs +++ b/src/program.rs @@ -3,7 +3,7 @@ use crate::{Budget, properties}; pub(crate) const HEADER: usize = 11; pub(crate) const MAGIC: u32 = 0x50525831; -pub(crate) const VERSION: u32 = 13; +pub(crate) const VERSION: u32 = 14; pub(crate) const U: u32 = 1; pub(crate) const M: u32 = 2; pub(crate) const S: u32 = 4; @@ -86,6 +86,163 @@ pub(crate) const CLASS_SORTED_I: u32 = 28; /// an operator around it left. pub(crate) const SEQUENCE: u32 = 29; pub(crate) const SEQUENCE_I: u32 = 30; +/// A `SPLIT` whose first arm begins at the next instruction and can only +/// succeed by consuming a first character in the set operand `a` describes, +/// one bit per [`first_bucket`]. When the character the arm would read first +/// is absent or falls in no set bucket, the arm cannot match, and execution +/// continues at `b` without a backtracking frame: what trying the arm, failing +/// and backtracking would have reached. Operand `a` is derived from the +/// instructions by [`derive_first_mask`] and checked against it on validation, +/// so no program can claim a narrower set than its arm has. +pub(crate) const BRANCH: u32 = 31; +/// Word 7 flag, above its first-character descriptor: every successful match +/// asserts `^` without `m` before it consumes anything. Derived by +/// [`derive_start_anchored`] and re-derived on validation, so a search reads +/// it instead of walking the entry each time. +pub(crate) const ANCHORED: u32 = 1 << 24; +/// Word 7 bits holding the first-character descriptor. +pub(crate) const DESCRIPTOR: u32 = 0xff_ffff; +/// A first-character mask admitting every character: the set is unknown. +pub(crate) const ALL_FIRST: u32 = u32::MAX; +/// Instructions [`derive_first_mask`] inspects from one arm before it gives +/// up and reports [`ALL_FIRST`]. +const FIRST_SCAN: usize = 32; +/// Paths [`derive_first_mask`] follows at once through nested branches. +const FIRST_PATHS: usize = 8; + +/// The bucket of a character in a first-character mask. ASCII characters fall +/// in 24 buckets by their low five bits, so the two cases of a letter share +/// one; every other character falls in one of 8 buckets by its high bits, so +/// surrogate halves, the CJK blocks and the symbol blocks are told apart. +#[inline(always)] +pub(crate) fn first_bucket(c: u32) -> u32 { + if c < 128 { + (c & 31) % 24 + } else { + 24 + (c >> 13).min(7) + } +} + +/// The buckets a character range covers. +fn range_mask(lo: u32, hi: u32) -> u32 { + let mut mask = 0; + if lo < 128 { + let top = hi.min(127); + if top - lo >= 31 { + mask |= (1 << 24) - 1; + } else { + for c in lo..=top { + mask |= 1 << first_bucket(c); + } + } + } + if hi >= 128 { + for bucket in first_bucket(lo.max(128))..=first_bucket(hi) { + mask |= 1 << bucket; + } + } + mask +} + +/// The arms of a branching instruction at `pc`: where it continues first and +/// where its backtracking frame resumes. `None` for any other instruction. +#[inline(always)] +pub(crate) fn branch_arms(pc: usize, [op, a, b]: [u32; 3]) -> Option<(usize, usize)> { + match op { + SPLIT => Some((a as usize, b as usize)), + BRANCH => Some((pc + 1, b as usize)), + _ => None, + } +} + +/// The first-character mask of the code starting at `pc`: one bit per bucket +/// that can hold the first character any successful path from there consumes, +/// read in the direction that code runs. [`ALL_FIRST`] when some path can +/// succeed without consuming, or reaches an instruction this does not follow +/// — an assertion body, a repeat, a backreference, a fold that can reach ASCII +/// from outside it, a property, a complement — or when the walk is too long. +/// +/// Position assertions consume nothing and are passed over: dropping a +/// condition can only widen the set. A path ends at its first consuming +/// instruction, whose own characters are the path's contribution. +pub(crate) fn derive_first_mask( + words: &[u32], + pc: usize, + budget: &mut Budget, +) -> Result { + let instructions = words[4] as usize; + let unicode = words[2] & U != 0; + let mut pending = [0usize; FIRST_PATHS]; + let mut depth = 0; + let mut pc = pc; + let mut mask = 0u32; + let mut inspected = 0; + loop { + inspected += 1; + budget.charge(1).map_err(|_| ProgramError::WorkLimit)?; + if inspected > FIRST_SCAN || pc >= instructions { + return Ok(ALL_FIRST); + } + let at = HEADER + pc * 3; + let instruction: [u32; 3] = words[at..at + 3].try_into().unwrap(); + let [op, a, b] = instruction; + if let Some((first, second)) = branch_arms(pc, instruction) { + if depth == pending.len() { + return Ok(ALL_FIRST); + } + pending[depth] = second; + depth += 1; + pc = first; + continue; + } + match op { + SAVE | START | START_M | END | END_M | WORD | WORD_I => { + pc += 1; + continue; + } + JUMP => { + pc = a as usize; + continue; + } + CHAR => mask |= 1 << first_bucket(a), + // Both cases of an ASCII letter share a bucket. Under `u`, `k` and + // `s` also fold with U+212A and U+017F, so a folded ASCII letter + // admits non-ASCII characters as well; a folded non-ASCII + // character can fold into ASCII, so it admits everything. + CHAR_I if a < 128 => { + mask |= 1 << first_bucket(a); + if (a as u8).is_ascii_alphabetic() && unicode { + mask |= !((1 << 24) - 1); + } + } + CLASS | CLASS_SORTED if b & NEGATED == 0 => { + let start = a as usize; + let end = start + b as usize; + if end > words[5] as usize { + return Err(ProgramError::Invalid); + } + for index in start..end { + budget.charge(1).map_err(|_| ProgramError::WorkLimit)?; + let base = HEADER + instructions * 3 + index * 2; + let (lo, hi) = (words[base], words[base + 1]); + if lo & PROPERTY != 0 || lo > hi { + return Ok(ALL_FIRST); + } + mask |= range_mask(lo, hi); + } + } + _ => return Ok(ALL_FIRST), + } + if mask == ALL_FIRST { + return Ok(ALL_FIRST); + } + if depth == 0 { + return Ok(mask); + } + depth -= 1; + pc = pending[depth]; + } +} /// Operand `a`'s group when the instruction covers every group of its set. pub(crate) const EVERY_GROUP: u32 = 255 << 8; /// The set and the group of equal-length members an instruction addresses. @@ -208,7 +365,7 @@ pub(crate) fn derive_leading(words: &[u32], instructions: usize) -> u32 { /// undecidable here and disables the claim. fn derive_alternation(words: &[u32], instructions: usize) -> bool { let entry = leading_pc(words, instructions); - if entry >= instructions || words[HEADER + entry * 3] != SPLIT { + if entry >= instructions || !matches!(words[HEADER + entry * 3], SPLIT | BRANCH) { return false; } let mut pending = [0usize; LEADING_BRANCHES]; @@ -222,13 +379,13 @@ fn derive_alternation(words: &[u32], instructions: usize) -> bool { return false; } let at = HEADER + pc * 3; - if words[at] == SPLIT { + if let Some((first, second)) = branch_arms(pc, words[at..at + 3].try_into().unwrap()) { if depth == pending.len() { return false; } - pending[depth] = words[at + 2] as usize; + pending[depth] = second; depth += 1; - pc = words[at + 1] as usize; + pc = first; continue; } let mut run = 0; @@ -263,17 +420,17 @@ fn derive_alternation(words: &[u32], instructions: usize) -> bool { /// such a program tries only its first start, as a sticky one does: every later /// start fails at the same `^`. /// -/// Derived from the instructions each search, like [`derive_leading`], rather -/// than stored, so no program word can claim it falsely. +/// Derived from the instructions when the program is emitted and again when it +/// is validated, which rejects a word 2 [`ANCHORED`] flag that disagrees, so no +/// program word can claim it falsely and a search only reads the flag. pub(crate) fn derive_start_anchored(words: &[u32], instructions: usize) -> bool { // Almost every program reaches a consuming instruction straight after its // entry `SAVE`s, which decides the claim without the branch stack below. - // This runs at the start of every search, so the common case stays short. let entry = leading_pc(words, instructions); if entry < instructions && entry < LEADING_SCAN { match words[HEADER + entry * 3] { START => return true, - SPLIT => {} + SPLIT | BRANCH => {} _ => return false, } } @@ -287,16 +444,18 @@ pub(crate) fn derive_start_anchored(words: &[u32], instructions: usize) -> bool return false; } let at = HEADER + pc * 3; - match words[at] { - SAVE => pc += 1, - SPLIT => { - if depth == pending.len() { - return false; - } - pending[depth] = words[at + 2] as usize; - depth += 1; - pc = words[at + 1] as usize; + let instruction: [u32; 3] = words[at..at + 3].try_into().unwrap(); + if let Some((first, second)) = branch_arms(pc, instruction) { + if depth == pending.len() { + return false; } + pending[depth] = second; + depth += 1; + pc = first; + continue; + } + match instruction[0] { + SAVE => pc += 1, START => { if depth == 0 { return true; @@ -380,17 +539,14 @@ impl<'a> Program<'a> { { return Err(bad); } - // Word 8's upper byte carries the end-anchored length bound; word 7 has - // no such field. Both share the character-range encoding below it. Like - // the range descriptors, the bound's representation is checked here and - // its semantic guarantee belongs to the compiler. + // Word 8's upper byte carries the end-anchored length bound; word 7's + // carries only the anchored flag, checked against the instructions + // below. Both share the character-range encoding below it. Like the + // range descriptors, the bound's representation is checked here and its + // semantic guarantee belongs to the compiler. for (index, &candidate) in words[7..9].iter().enumerate() { - let descriptor = if index == 1 { - candidate & 0xff_ffff - } else { - candidate - }; - if index == 0 && candidate != descriptor { + let descriptor = candidate & DESCRIPTOR; + if index == 0 && candidate & !(DESCRIPTOR | ANCHORED) != 0 { return Err(bad); } let lo = (descriptor >> 8) & 255; @@ -430,6 +586,7 @@ impl<'a> Program<'a> { // so every field is checked on its own. if words[9] & RUN_SKIP != derive_run_skip(words, p) || words[9] & LEADING_MASK != derive_leading(words, p.instructions()) + || (words[7] & ANCHORED != 0) != derive_start_anchored(words, p.instructions()) || (words[9] & ADMISSION_FORWARD != 0 && words[2] & ADMISSION == 0) { return Err(bad); @@ -528,6 +685,12 @@ impl<'a> Program<'a> { } SAVE => a < words[3] * 2 && b == 0, SPLIT => a < words[4] && b < words[4], + BRANCH => { + b < words[4] + && pc + 1 < p.instructions() + && a != ALL_FIRST + && derive_first_mask(words, pc + 1, budget)? == a + } JUMP => a < words[4] && b == 0, WORD | WORD_I => a <= 1 && b == 0, BACKREF | BACKREF_I => a > 0 && a < words[3] && b == 0, @@ -663,6 +826,15 @@ impl<'a> Program<'a> { } } /// Instruction holding the first character of [`Program::leading`]. + /// Word 7's first-character descriptor; see `docs/candidate.md`. + pub(crate) fn first_descriptor(self) -> u32 { + self.words[7] & DESCRIPTOR + } + /// Whether every match begins at the subject's start. See + /// [`derive_start_anchored`]. + pub(crate) fn start_anchored(self) -> bool { + self.words[7] & ANCHORED != 0 + } pub(crate) fn leading_pc(self) -> usize { leading_pc(self.words, self.instructions()) } diff --git a/tests/atom_repeat.rs b/tests/atom_repeat.rs index dec988e..fcfa149 100644 --- a/tests/atom_repeat.rs +++ b/tests/atom_repeat.rs @@ -213,7 +213,7 @@ fn every_insufficient_work_allowance_preserves_output() { #[test] fn optimized_record_validation_rejects_shape_and_version_corruption() { let w = words("^([a-z]+)$", ""); - assert_eq!(w[1], 13); + assert_eq!(w[1], 14); let table = 11 + w[4] as usize * 3 + w[5] as usize * 2; assert_eq!(w[table + 7], 1); let entry = w[table + 3] as usize - 2; diff --git a/tests/branch.rs b/tests/branch.rs new file mode 100644 index 0000000..15ad550 --- /dev/null +++ b/tests/branch.rs @@ -0,0 +1,358 @@ +//! Alternation is laid out flat, and a branch whose first arm can only begin +//! with characters from a known set skips that arm, and its frame, when the +//! next character is outside it. Programs are anchored by a stored, validated +//! claim. A trial clears only its capture registers. See `docs/branch.md`. +use perex::{ + Budget, + compiler::{Node, Range, compile}, + executor::{ExecError, Frame, Scratch, Undo, find}, + input::Input, + program::{Program, ProgramError}, + span::Span, +}; + +fn words(source: &str, flags: &str) -> Vec { + let mut nodes = vec![Node::default(); source.len() * 3 + 64]; + let mut ranges = vec![Range::default(); source.len() * 12 + 64]; + let mut storage = vec![0; source.len() * 48 + 256]; + compile( + Input::utf8(source), + flags, + &mut nodes, + &mut ranges, + &mut storage, + &mut Budget::new(2_000_000), + ) + .unwrap_or_else(|e| panic!("/{source}/{flags}: {e:?}")) + .words() + .to_vec() +} + +type Answer = Result>>, ExecError>; + +/// A search's complete answer from `start`, with registers that start out +/// holding `garbage` rather than anything the engine wrote. +fn run(words: &[u32], subject: &str, start: usize, garbage: usize) -> Answer { + let p = Program::from_words(words, &mut Budget::new(1_000_000)).unwrap(); + let mut registers = vec![garbage; p.register_count()]; + let mut frames = vec![Frame::default(); 4096]; + let mut undo = vec![Undo::default(); 32768]; + let mut captures = vec![None; p.capture_count()]; + let mut budget = Budget::new(10_000_000); + find( + p, + Input::utf8(subject), + start, + Scratch { + registers: &mut registers, + frames: &mut frames, + undo: &mut undo, + }, + &mut captures, + &mut budget, + ) + .map(|found| found.then(|| captures.clone())) +} + +fn branches(words: &[u32]) -> usize { + (0..words[4] as usize) + .filter(|pc| words[11 + pc * 3] == 31) + .count() +} + +/// Each alternative, at every depth, written between empty lookaheads, so its +/// arm begins with an assertion, so no branch carries a set: the same +/// answers, decided by trying every arm. +fn unfiltered(source: &str) -> String { + const EMPTY: &str = "(?=)"; + let chars: Vec = source.chars().collect(); + let mut out = String::from("(?:"); + out.push_str(EMPTY); + let mut i = 0; + let mut class = false; + while i < chars.len() { + let c = chars[i]; + // A lookbehind runs its alternatives backwards, so each one gets the + // assertion at its end as well as at its start. + if !class && matches!(c, '|' | ')') { + out.push_str(EMPTY); + } + out.push(c); + i += 1; + match c { + '\\' => { + if i < chars.len() { + out.push(chars[i]); + i += 1; + } + } + '[' if !class => class = true, + ']' if class => class = false, + '|' if !class => out.push_str(EMPTY), + '(' if !class => { + if chars.get(i) == Some(&'?') { + // Copy the group's prefix: `?:`, `?=`, `?!`, `?<=`, `?`. + out.push('?'); + i += 1; + if chars[i] == '<' && !matches!(chars[i + 1], '=' | '!') { + while chars[i] != '>' { + out.push(chars[i]); + i += 1; + } + out.push('>'); + i += 1; + } else if chars[i] == '<' { + out.push('<'); + out.push(chars[i + 1]); + i += 2; + } else { + out.push(chars[i]); + i += 1; + } + } + out.push_str(EMPTY); + } + _ => {} + } + } + out.push_str(EMPTY); + out.push(')'); + out +} + +const PATTERNS: &[(&str, &str)] = &[ + ("ab|cd|ef|gh", ""), + ("a|b|c|d|e|f|g|h|i|j|k", ""), + ("(a)|(b)|(c)", ""), + ("(a)|b(c)|(d)e", "d"), + ("x|xy|xyz", ""), + ("xyz|xy|x", ""), + ("[0-9]+|[a-z]+|_", ""), + ( + "[#*0-9]\\uFE0F?\\u20E3|[\\xA9\\xAE]\\uFE0F?|\\uD83C[\\uDDE6-\\uDDFF]|\\u26D3", + "g", + ), + ( + "[#*0-9]\\uFE0F?\\u20E3|[\\xA9\\xAE]\\uFE0F?|\\u{1F1E6}|\\u26D3", + "u", + ), + ("K|s|é|\\u212A", "i"), + ("K|s|é|\\u212A", "iu"), + ("k|\\u017F|x", "iu"), + ("\\u212A|q", "iu"), + ("(?<=a|bc)d|e", ""), + ("(?<=x|[0-9]y)z", ""), + ("(?a)|(?b)", ""), + ("(a)\\1|b", ""), + ("[^a]|a", ""), + ("\\p{Lu}|[a-c]", "u"), + ("\\d|\\s|\\w", ""), + ("é|中|😀|a", "u"), + ("é|中|😀|a", ""), + ("(?:x|y|z){2,3}w|v", ""), + ("(?:(a)|(b))+", ""), + ("token|tok|t", "y"), +]; + +const SUBJECTS: &[&str] = &[ + "", + "a", + "b", + "e", + "z", + "1", + "#", + "#\u{20E3}", + "1\u{FE0F}\u{20E3}", + "\u{A9}", + "\u{1F1E6}", + "\u{26D3}", + "k", + "K", + "\u{212A}", + "s", + "\u{17F}", + "S", + "é", + "É", + "中", + "😀", + "abcd", + "bcd", + "xyz", + "xy", + "x", + "cd ef gh", + "__9", + "foo", + "bar", + "baz", + "foo\nbar", + "ad", + "bcd", + "ac", + "abbcd", + "aa", + "ba", + "xyzw", + "xyxw", + "v", + "token", + "tokenx", + "ab", + " ", + "-", + "1a_", +]; + +#[test] +fn branches_answer_as_trying_every_arm() { + let mut filtered = 0; + for &(source, flags) in PATTERNS { + let w = words(source, flags); + let plain = words(&unfiltered(source), flags); + assert_eq!(branches(&plain), 0, "/{}/", unfiltered(source)); + filtered += branches(&w); + for subject in SUBJECTS { + let units = subject.encode_utf16().count(); + for start in 0..=units + 1 { + assert_eq!( + run(&w, subject, start, usize::MAX), + run(&plain, subject, start, usize::MAX), + "/{source}/{flags} on {subject:?} from {start}" + ); + } + } + } + assert!(filtered > 30, "{filtered} filtered branches"); +} + +#[test] +fn alternation_is_flat_and_ordered() { + // One frame live per attempted alternative and one jump to the end, in + // source order. + let w = words("ab|cd|ef|gh", ""); + let at = |pc: usize| [w[11 + pc * 3], w[12 + pc * 3], w[13 + pc * 3]]; + assert_eq!(at(1)[0], 31); + assert_eq!(at(2), [1, 'a' as u32, 0]); + assert_eq!(at(4), [6, 15, 0]); + assert_eq!(at(12), [6, 15, 0]); + assert_eq!(at(1)[2], 5); + assert_eq!(at(5)[0], 31); + assert_eq!(at(6), [1, 'c' as u32, 0]); + assert_eq!(at(13), [1, 'g' as u32, 0]); + // Leftmost alternative wins even when a later one is longer. + let w = words("x|xy|xyz", ""); + assert_eq!( + run(&w, "xyz", 0, usize::MAX), + Ok(Some(vec![Span::new(0, 1)])) + ); +} + +#[test] +fn repeat_registers_need_no_clearing() { + // Repeat counters start as whatever the host's scratch held; every repeat + // writes its own before reading them, so answers do not depend on it. + for (source, flags) in [ + ("(?:ab){2,3}c", ""), + ("a{1,3}?b", ""), + ("(?:(a)|b){2,}?(c)", ""), + ("(?:x(?:y{0,2}z)+)*w", ""), + ("(a*)+b", ""), + ("\\uFE0F?\\u20E3|[0-9]\\uFE0F?", ""), + ("((a)|b)+", "d"), + ("(?:a?){3,5}", ""), + ] { + let w = words(source, flags); + for subject in [ + "", + "abababc", + "aab", + "aaab", + "bac", + "acbc", + "xyyzxzw", + "xzw", + "w", + "aaaab", + "b", + "1\u{FE0F}", + "\u{20E3}", + "abba", + "aaaaaa", + ] { + let expected = run(&w, subject, 0, usize::MAX); + for garbage in [0, 1, 2, 3, 7, 1000, usize::MAX - 1] { + assert_eq!( + run(&w, subject, 0, garbage), + expected, + "/{source}/ on {subject:?} with {garbage}" + ); + } + } + } +} + +#[test] +fn validation_rederives_masks_and_the_anchored_claim() { + let w = words("ab|cd|[0-9]x", ""); + assert!(Program::from_words(&w, &mut Budget::new(100_000)).is_ok()); + let first = (0..w[4] as usize).find(|pc| w[11 + pc * 3] == 31).unwrap(); + for bit in 0..32 { + let mut bad = w.clone(); + bad[12 + first * 3] ^= 1 << bit; + assert_eq!( + Program::from_words(&bad, &mut Budget::new(100_000)).unwrap_err(), + ProgramError::Invalid, + "bit {bit}" + ); + } + // A set every character is in is not a claim; such a branch is a `SPLIT`. + let mut bad = w.clone(); + bad[12 + first * 3] = u32::MAX; + assert!(Program::from_words(&bad, &mut Budget::new(100_000)).is_err()); + + let anchored = words("^a|^b", ""); + let free = words("^a|b", ""); + assert_ne!(anchored[7] & (1 << 24), 0); + assert_eq!(free[7] & (1 << 24), 0); + for (program, flip) in [(anchored, true), (free, false)] { + let mut bad = program.clone(); + bad[7] ^= 1 << 24; + assert_eq!( + Program::from_words(&bad, &mut Budget::new(100_000)).unwrap_err(), + ProgramError::Invalid, + "anchored {flip}" + ); + let mut reserved = program.clone(); + reserved[7] |= 1 << 25; + assert!(Program::from_words(&reserved, &mut Budget::new(100_000)).is_err()); + } +} + +#[test] +fn anchored_claim_still_limits_starts() { + let w = words("^(?:ab|cd)", ""); + assert_eq!(run(&w, "xab", 0, usize::MAX), Ok(None)); + assert_eq!( + run(&w, "cdab", 0, usize::MAX), + Ok(Some(vec![Span::new(0, 2)])) + ); + let w = words("^(?:ab|cd)", "m"); + assert_eq!( + run(&w, "x\ncd", 0, usize::MAX), + Ok(Some(vec![Span::new(2, 4)])) + ); +} diff --git a/tests/candidate.rs b/tests/candidate.rs index 6f4eb42..6bed81c 100644 --- a/tests/candidate.rs +++ b/tests/candidate.rs @@ -154,7 +154,7 @@ fn descriptors_are_validated_and_old_formats_rejected() { let mut bounded = original.clone(); bounded[8] = original[8] | (7 << 24); assert!(Program::from_words(&bounded, &mut Budget::new(1000)).is_ok()); - for version in [7, 8, 9] { + for version in [7, 8, 9, 13] { let mut bad = original.clone(); bad[1] = version; assert_eq!( diff --git a/tests/classes.rs b/tests/classes.rs index ef0e33b..c4e54b7 100644 --- a/tests/classes.rs +++ b/tests/classes.rs @@ -57,7 +57,7 @@ fn members() -> String { fn sorted_classes_preserve_original_membership_and_reject_invalid_order() { let source = format!("[{}]", members()); let words = program(&source, "u"); - assert_eq!(words[1], 13); + assert_eq!(words[1], 14); assert_eq!(words[11 + 3], 27); // Sorted class following capture-zero start. for unit in 0..0x400u16 { let expected = (0..128).any(|i| unit == 0x100 + i * 3); diff --git a/tests/modifiers.rs b/tests/modifiers.rs index 5f329cb..682cbb0 100644 --- a/tests/modifiers.rs +++ b/tests/modifiers.rs @@ -146,7 +146,7 @@ fn modifier_errors_and_instruction_corruption_are_explicit() { (r"(?a)|(?b)\k", "i"), ] { let words = program(source, flags).unwrap(); - assert_eq!(words[1], 13); + assert_eq!(words[1], 14); for pc in 0..words[4] as usize { let at = 11 + pc * 3; if (19..=26).contains(&words[at]) {