From 63ac910fc2646186e9f4863af4d8fdc34e908bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 28 Oct 2026 23:38:07 +0000 Subject: [PATCH 1/2] runtime: literal-aware concat, inline operand checks, pointer push gate (string-build -21%) A: ToPrimitive and Symbol checks on + operands run behind one inline test of the live bits (plain number, heap string, inline string skip), and a literal or provably non-pointer operand carries none. B: js_string_concat_parts takes literal operands as rodata address + lengths; two-digit integer formatter written straight into the result; js_string_concat_lit_value_box for a literal prefix + number, consulting the concat memo exactly where the header-flag test admits it today. C: the pointer-push gate admits a heap string: one header test, refcount read, demote only for an unshared string; write barrier unchanged. --- .../src/codegen/declared_string_add_tests.rs | 165 +++++- crates/perry-codegen/src/codegen/mod.rs | 1 + crates/perry-codegen/src/expr/array_push.rs | 68 ++- .../src/expr/array_push_string_gate_tests.rs | 199 +++++++ .../perry-codegen/src/lower_string_concat.rs | 290 +++++++++- .../src/runtime_decls/strings.rs | 3 + .../src/stmt/indexed_owner_loop_tests.rs | 2 - crates/perry-codegen/src/strings.rs | 13 + .../perry-codegen/src/wasm32/runtime_abi.tsv | 2 + .../tests/temp_root_operand_temporaries.rs | 23 + crates/perry-runtime/src/string/concat.rs | 80 ++- .../perry-runtime/src/string/concat_parts.rs | 509 ++++++++++++++++++ .../src/string/concat_parts_tests.rs | 311 +++++++++++ crates/perry-runtime/src/string/mod.rs | 7 +- scripts/gc_root_dominance_check.py | 5 + test-files/test_gap_jst5_concat_parts.ts | 143 +++++ test-files/test_gap_jst5_push_gate.ts | 179 ++++++ 17 files changed, 1938 insertions(+), 62 deletions(-) create mode 100644 crates/perry-codegen/src/expr/array_push_string_gate_tests.rs create mode 100644 crates/perry-runtime/src/string/concat_parts.rs create mode 100644 crates/perry-runtime/src/string/concat_parts_tests.rs create mode 100644 test-files/test_gap_jst5_concat_parts.ts create mode 100644 test-files/test_gap_jst5_push_gate.ts diff --git a/crates/perry-codegen/src/codegen/declared_string_add_tests.rs b/crates/perry-codegen/src/codegen/declared_string_add_tests.rs index 45fbf3ff1cd..798617a333c 100644 --- a/crates/perry-codegen/src/codegen/declared_string_add_tests.rs +++ b/crates/perry-codegen/src/codegen/declared_string_add_tests.rs @@ -198,7 +198,8 @@ fn a_string_literal_operand_keeps_the_fused_concat() { add(Expr::String("item_".to_string()), Expr::LocalGet(1)), ); assert!( - ir.contains("call double @js_string_concat_value_box("), + ir.contains("call double @js_string_concat_value_box(") + || ir.contains("call double @js_string_concat_lit_value_box("), "a proven string must keep the fused concat (the `_box` twin, which \ returns SSO for short results) — the guard is for claims, not for \ proofs:\n{ir}" @@ -290,6 +291,13 @@ fn a_string_method_name_on_an_unproven_receiver_is_guarded() { // -------------------------------------------------------------- chain fold +/// The N-way fold's runtime entry: the generic chain, or the literal-operand +/// entry that carries literals as constant data (JST slice 3). +fn has_nway_fold(ir: &str) -> bool { + ir.contains("call i64 @js_string_concat_chain(") + || ir.contains("call i64 @js_string_concat_parts(") +} + #[test] fn a_chain_whose_head_pair_is_all_declared_does_not_fold() { // `s + t + "x"`. `js_string_concat_chain` formats EVERY part as a string, @@ -304,7 +312,8 @@ fn a_chain_whose_head_pair_is_all_declared_does_not_fold() { ), ); assert!( - !ir.contains("call i64 @js_string_concat_chain("), + !ir.contains("call i64 @js_string_concat_chain(") + && !ir.contains("call i64 @js_string_concat_parts("), "the head pair carries no proof, so the N-way fold is unsound \ here:\n{ir}" ); @@ -323,7 +332,7 @@ fn a_chain_led_by_a_literal_still_folds() { ), ); assert!( - ir.contains("call i64 @js_string_concat_chain("), + has_nway_fold(&ir), "a proven string in the head pair keeps the N-way fold:\n{ir}" ); } @@ -339,7 +348,7 @@ fn a_chain_whose_second_part_is_proven_still_folds() { ), ); assert!( - ir.contains("call i64 @js_string_concat_chain("), + has_nway_fold(&ir), "`s + \",\" + t` concatenates at every node whatever `s` holds:\n{ir}" ); } @@ -375,7 +384,8 @@ fn a_self_append_chain_fuses_the_accumulator_and_suffix() { "the growing prefix must reach the amortized append path:\n{ir}" ); assert_eq!( - ir.matches("call i64 @js_string_concat_chain(").count(), + ir.matches("call i64 @js_string_concat_chain(").count() + + ir.matches("call i64 @js_string_concat_parts(").count(), 0, "the suffix must not be allocated before it is appended:\n{ir}" ); @@ -759,3 +769,148 @@ fn annotation_only_numeric_assignment_keeps_string_alias_guard() { "{ir}" ); } + +// -------------------------------------- JST slice 3: operand checks on live bits + +/// Text of every block whose label starts with `prefix`. +fn blocks_named(ir: &str, prefix: &str) -> Vec { + let mut blocks = Vec::new(); + let mut current: Option> = None; + for line in ir.lines() { + if let Some(label) = line.strip_suffix(':') { + if !label.starts_with(char::is_whitespace) { + if let Some(done) = current.take() { + blocks.push(done.join("\n")); + } + if label.starts_with(prefix) { + current = Some(Vec::new()); + } + continue; + } + } + if let Some(body) = current.as_mut() { + body.push(line); + } + } + if let Some(done) = current.take() { + blocks.push(done.join("\n")); + } + blocks +} + +fn literal_chain(middle: Expr, params: Vec) -> String { + ir( + params, + add( + add(Expr::String("item-".to_string()), middle), + Expr::String(";".to_string()), + ), + ) +} + +#[test] +fn literal_parts_get_no_symbol_check_and_the_value_part_is_tested_inline() { + // `"item-" + n + ";"` with a declared-only `n`: ToPrimitive and the Symbol + // check each have exactly one site, for the one value part, and each sits + // in a block reached only through the inline test of the live bits. + let ir = literal_chain(Expr::LocalGet(1), vec![param(1, "n", Type::Number)]); + assert!(ir.contains("call i64 @js_string_concat_parts("), "{ir}"); + assert_eq!( + ir.matches("call void @js_add_throw_if_symbol(").count(), + 1, + "only the value part can be a Symbol:\n{ir}" + ); + assert_eq!( + ir.matches("call double @js_to_primitive_default_for_add(") + .count(), + 1, + "{ir}" + ); + let prim = blocks_named(&ir, "cat.prim.call"); + let sym = blocks_named(&ir, "cat.sym.call"); + assert_eq!(prim.len(), 1, "{ir}"); + assert_eq!(sym.len(), 1, "{ir}"); + assert!( + prim[0].contains("@js_to_primitive_default_for_add("), + "{}", + prim[0] + ); + assert!(sym[0].contains("@js_add_throw_if_symbol("), "{}", sym[0]); + // The test is on the value's own bits: plain number, STRING_TAG, SSO tag. + assert!( + ir.contains("9223372036854775807"), + "plain-number mask missing:\n{ir}" + ); + assert!( + ir.contains(", 32767") && ir.contains(", 32761"), + "string/SSO tag tests missing:\n{ir}" + ); + // Literals travel as the address of their image-resident text + lengths. + assert_eq!( + ir.matches( + "ptrtoint (ptr getelementptr (i8, ptr getelementptr (i8, ptr @perry_image_literal_" + ) + .count(), + 2, + "both literals are passed as constant data:\n{ir}" + ); + // The literals are still lowered in source position (pool interning order + // is unchanged), but nothing consumes their handle loads. + for handle in ir.lines().filter(|l| l.contains(".handle")) { + let reg = handle.trim().split(" = ").next().unwrap(); + assert_eq!( + ir.matches(reg).count(), + 1, + "a literal's handle load must be unused ({reg}):\n{ir}" + ); + } +} + +#[test] +fn an_operand_that_cannot_be_a_symbol_carries_no_check_at_all() { + // `"a" + n + 5 + "z"`: the integer is non-pointer by construction. + let ir = ir( + vec![param(1, "n", Type::Number)], + add( + add( + add(Expr::String("a".to_string()), Expr::LocalGet(1)), + Expr::Integer(5), + ), + Expr::String("z".to_string()), + ), + ); + assert!(ir.contains("call i64 @js_string_concat_parts("), "{ir}"); + assert_eq!( + ir.matches("call void @js_add_throw_if_symbol(").count(), + 1, + "only `n` needs the check:\n{ir}" + ); +} + +#[test] +fn the_symbol_and_conversion_calls_are_never_on_the_straight_line() { + // Sabotage guard: an unconditional call (the inline test dropped) puts the + // call outside every `cat.*.call` block. + let ir = literal_chain(Expr::LocalGet(1), vec![param(1, "n", Type::Any)]); + let mut in_call_block = false; + for line in ir.lines() { + if let Some(label) = line.strip_suffix(':') { + if !label.starts_with(char::is_whitespace) { + in_call_block = + label.starts_with("cat.prim.call") || label.starts_with("cat.sym.call"); + continue; + } + } + if line.contains("@js_add_throw_if_symbol(") + || line.contains("@js_to_primitive_default_for_add(") + { + if line.trim_start().starts_with("declare") { + continue; + } + assert!( + in_call_block, + "call outside its inline-test arm: {line}\n{ir}" + ); + } + } +} diff --git a/crates/perry-codegen/src/codegen/mod.rs b/crates/perry-codegen/src/codegen/mod.rs index ac2cc2bb4e2..a8ed4b887f7 100644 --- a/crates/perry-codegen/src/codegen/mod.rs +++ b/crates/perry-codegen/src/codegen/mod.rs @@ -755,6 +755,7 @@ fn lower_module( .unwrap_or(&module_prefix); let mut strings = StringPool::with_prefix(module_prefix.clone()); strings.set_thread_literal_callback_prefix(thread_literal_callback_prefix.clone()); + strings.set_image_literals(opts.output_type != "dylib" && opts.output_type != "staticlib"); strings.tdz_binding_names = tdz_names::collect(hir); // #5247: install per-module source-location context for the dynamic // call-dispatch throw path, but only under `--debug-symbols` (which sets diff --git a/crates/perry-codegen/src/expr/array_push.rs b/crates/perry-codegen/src/expr/array_push.rs index 1820cbb15a1..5063dd683d6 100644 --- a/crates/perry-codegen/src/expr/array_push.rs +++ b/crates/perry-codegen/src/expr/array_push.rs @@ -204,14 +204,72 @@ fn emit_dynamic_pointer_push_store( let done_idx = ctx.new_block("apush.pointer_layout.done"); let bookkeeping_label = ctx.block_label(bookkeeping_idx); let done_label = ctx.block_label(done_idx); - { + // A heap string is pointer-bearing exactly like an object pointer, so it + // is admitted by the same header proof; what it still owes is the shared- + // string demote, and only when the string is not already shared. + // `layout_pointer_bearing_bits` accepts a `STRING_TAG` payload iff it is + // non-zero; the `>= 0x1000` floor is what makes the refcount read below + // safe, and a string that fails it takes the full sequence. + let admit_strings = string_addref_needed; + // Created only for the string arm: an unterminated block is invalid IR. + let (fast_idx, string_addref_idx, string_demote_idx) = if admit_strings { + ( + ctx.new_block("apush.pointer_layout.fast"), + ctx.new_block("apush.pointer_layout.str_addref"), + ctx.new_block("apush.pointer_layout.str_demote"), + ) + } else { + (done_idx, done_idx, done_idx) + }; + let fast_label = ctx.block_label(fast_idx); + let string_addref_label = ctx.block_label(string_addref_idx); + let string_demote_label = ctx.block_label(string_demote_idx); + let (is_heap_string, payload) = { let blk = ctx.block(); let top16 = blk.lshr(I64, &value_bits, "48"); let is_object_pointer = blk.icmp_eq(I64, &top16, crate::nanbox::POINTER_TAG_TOP16_I64); + let is_heap_string = blk.icmp_eq(I64, &top16, crate::nanbox::STRING_TAG_TOP16_I64); + let payload = blk.and(I64, &value_bits, crate::nanbox::POINTER_MASK_I64); + let admitted_value = if admit_strings { + let payload_valid = blk.icmp_uge(I64, &payload, "4096"); + let string_ok = blk.and(I1, &is_heap_string, &payload_valid); + blk.or(I1, &is_object_pointer, &string_ok) + } else { + is_object_pointer + }; let proof_bits = blk.and(I16, object_flags, ARRAY_PUSH_POINTER_LAYOUT_MASK_I16); let pointer_layout = blk.icmp_eq(I16, &proof_bits, ARRAY_PUSH_POINTER_LAYOUT_EXPECT_I16); - let fast = blk.and(I1, &is_object_pointer, &pointer_layout); - blk.cond_br(&fast, &done_label, &bookkeeping_label); + let fast = blk.and(I1, &admitted_value, &pointer_layout); + if admit_strings { + blk.cond_br(&fast, &fast_label, &bookkeeping_label); + } else { + blk.cond_br(&fast, &done_label, &bookkeeping_label); + } + (is_heap_string, payload) + }; + + if admit_strings { + ctx.current_block = fast_idx; + ctx.block() + .cond_br(&is_heap_string, &string_addref_label, &done_label); + ctx.current_block = string_addref_idx; + { + let blk = ctx.block(); + // `StringHeader.refcount` (offset 12): 0 is already shared. A fresh + // concat result is born shared, so the helper runs only for a + // uniquely-owned string that must be demoted. + let refcount_addr = blk.add(I64, &payload, "12"); + let refcount_ptr = blk.inttoptr(I64, &refcount_addr); + let refcount = blk.load(I32, &refcount_ptr); + let unique = blk.icmp_ne(I32, &refcount, "0"); + blk.cond_br(&unique, &string_demote_label, &done_label); + } + ctx.current_block = string_demote_idx; + { + let blk = ctx.block(); + blk.call_void("js_string_addref_if_heap_string", &[(DOUBLE, value_double)]); + blk.br(&done_label); + } } ctx.current_block = bookkeeping_idx; @@ -1873,3 +1931,7 @@ mod parent_gate_tests { ); } } + +#[cfg(test)] +#[path = "array_push_string_gate_tests.rs"] +mod string_gate_tests; diff --git a/crates/perry-codegen/src/expr/array_push_string_gate_tests.rs b/crates/perry-codegen/src/expr/array_push_string_gate_tests.rs new file mode 100644 index 00000000000..172d5278d5e --- /dev/null +++ b/crates/perry-codegen/src/expr/array_push_string_gate_tests.rs @@ -0,0 +1,199 @@ +//! JST slice 3 (C): the pointer-push gate admits a heap string. +//! +//! An append of a heap string onto an all-pointers array at its append +//! position makes the layout note, the numeric note and (for an already shared +//! string) the demote provable no-ops. The test is one header test beside the +//! value-tag test; anything that fails it takes today's full sequence. + +use perry_hir::types::Type; +use perry_hir::{Expr, Function, Module as HirModule, Stmt}; + +fn push_ir(value: Expr) -> String { + let mut hir = HirModule::new("apush_string_gate_test"); + hir.functions.push(Function { + id: 0, + name: "pushes".to_string(), + type_params: Vec::new(), + params: Vec::new(), + return_type: Type::Any, + body: vec![ + Stmt::Let { + id: 0, + name: "a".to_string(), + ty: Type::Any, + mutable: true, + init: Some(Expr::Array(Vec::new(), None)), + }, + Stmt::Expr(Expr::ArrayPush { + array_id: 0, + value: Box::new(value), + field_writeback: None, + }), + Stmt::Return(Some(Expr::LocalGet(0))), + ], + is_async: false, + is_generator: false, + is_strict: true, + is_exported: false, + captures: Vec::new(), + decorators: Vec::new(), + was_plain_async: false, + was_unrolled: false, + }); + let opts = crate::CompileOptions { + emit_ir_only: true, + ..Default::default() + }; + let bytes = crate::compile_module(&hir, opts).expect("compile"); + String::from_utf8(bytes).expect("ir is text") +} + +/// Body of every block whose label starts with `prefix` (labels carry a +/// uniquing suffix, so walk labels rather than substrings). +fn blocks_named(ir: &str, prefix: &str) -> Vec { + let mut blocks = Vec::new(); + let mut current: Option> = None; + for line in ir.lines() { + if let Some(label) = line.strip_suffix(':') { + if !label.starts_with(char::is_whitespace) { + if let Some(done) = current.take() { + blocks.push(done.join("\n")); + } + if label.starts_with(prefix) { + current = Some(Vec::new()); + } + continue; + } + } + if let Some(body) = current.as_mut() { + body.push(line); + } + } + if let Some(done) = current.take() { + blocks.push(done.join("\n")); + } + blocks +} + +fn block(ir: &str, prefix: &str) -> String { + let mut found = blocks_named(ir, prefix); + assert_eq!(found.len(), 1, "expected one `{prefix}*` block:\n{ir}"); + found.remove(0) +} + +const NOTE_CALLS: [&str; 3] = [ + "@js_string_addref_if_heap_string(", + "@js_gc_note_slot_layout(", + "@js_array_note_numeric_write(", +]; + +#[test] +fn a_string_push_has_one_header_test_and_no_note_calls_on_the_admitted_arm() { + let ir = push_ir(Expr::String("x".to_string())); + // The header proof (mask 0xF880 == 63616, expected 0xA000 == 40960) is + // tested once, beside the value-tag test. + assert_eq!( + ir.matches(", 63616").count(), + 1, + "exactly one header-layout mask test:\n{ir}" + ); + assert_eq!(ir.matches(", 40960").count(), 1, "{ir}"); + // The admitted arm: no notes and no demote call before the refcount test. + let fast = block(&ir, "apush.pointer_layout.fast"); + for call in NOTE_CALLS { + assert!( + !fast.contains(call), + "admitted arm must not call {call}:\n{fast}" + ); + } + let tag_arm = block(&ir, "apush.pointer_layout.str_addref"); + for call in NOTE_CALLS { + assert!( + !tag_arm.contains(call), + "refcount arm must not call {call}:\n{tag_arm}" + ); + } + assert!( + tag_arm.contains(", 12") && tag_arm.contains("load i32"), + "the string arm reads StringHeader.refcount (offset 12):\n{tag_arm}" + ); + // The demote runs only for a not-yet-shared string. + let demote = block(&ir, "apush.pointer_layout.str_demote"); + assert!( + demote.contains("@js_string_addref_if_heap_string("), + "{demote}" + ); + assert!(!demote.contains("@js_gc_note_slot_layout("), "{demote}"); + // The slow arm is today's exact sequence. + let slow = block(&ir, "apush.pointer_layout.bookkeeping"); + for call in NOTE_CALLS { + assert!(slow.contains(call), "slow arm must keep {call}:\n{slow}"); + } + // The barrier is untouched: still gated on the parent's header after the + // bookkeeping blocks. + assert!( + ir.contains("apush.barrier."), + "write barrier arm missing:\n{ir}" + ); +} + +#[test] +fn a_value_that_cannot_be_a_string_owes_no_string_arm() { + let ir = push_ir(Expr::Object(vec![("v".to_string(), Expr::Number(1.0))])); + assert!( + blocks_named(&ir, "apush.pointer_layout.str_addref").is_empty() + && blocks_named(&ir, "apush.pointer_layout.str_demote").is_empty() + && blocks_named(&ir, "apush.pointer_layout.fast").is_empty(), + "an object literal never owes the string demote, so no string arm:\n{ir}" + ); + for slow in blocks_named(&ir, "apush.pointer_layout.bookkeeping") { + assert!( + !slow.contains("@js_string_addref_if_heap_string("), + "{slow}" + ); + } +} + +fn def_of<'a>(ir: &'a str, reg: &str) -> &'a str { + ir.lines() + .find(|line| line.trim_start().starts_with(&format!("{reg} = "))) + .unwrap_or_else(|| panic!("no definition of {reg} in:\n{ir}")) +} + +/// Sabotage guard for the gate itself: the branch into the admitted arm must +/// be an `and` whose operand is the header-proof compare (`icmp eq i16 ..., +/// 40960`). Dropping the header test (admitting on the value's tag alone) +/// leaves a branch on the tag test only and turns this red. +#[test] +fn the_admission_branch_is_conditioned_on_the_header_proof() { + let ir = push_ir(Expr::String("x".to_string())); + let branch = ir + .lines() + .find(|line| line.contains("br i1") && line.contains("label %apush.pointer_layout.fast")) + .unwrap_or_else(|| panic!("no branch into the admitted arm:\n{ir}")); + let condition = branch + .split_whitespace() + .nth(2) + .unwrap() + .trim_end_matches(',') + .to_string(); + let def = def_of(&ir, &condition); + assert!( + def.contains("and i1"), + "admission must conjoin its tests: {def}" + ); + let operands: Vec = def + .split("and i1") + .nth(1) + .unwrap() + .split(',') + .map(|s| s.trim().to_string()) + .collect(); + assert!( + operands.iter().any(|reg| { + let d = def_of(&ir, reg); + d.contains("icmp eq i16") && d.trim_end().ends_with(", 40960") + }), + "the header proof must be one operand of the admission test: {def}\n{ir}" + ); +} diff --git a/crates/perry-codegen/src/lower_string_concat.rs b/crates/perry-codegen/src/lower_string_concat.rs index 4acd2ed006a..4db2eeb5c96 100644 --- a/crates/perry-codegen/src/lower_string_concat.rs +++ b/crates/perry-codegen/src/lower_string_concat.rs @@ -621,6 +621,25 @@ fn coerce_concat_body( { return Ok(value); } + // A non-empty ASCII source literal as the prefix: its bytes, length + // and UTF-16 length go to the runtime as constants, so the call skips + // the prefix validation, ASCII scan and header reads (JST slice 3). + // Same answers and the same memo consultation as the fused call below. + if let Expr::String(prefix) = left { + if !prefix.is_empty() && prefix.is_ascii() && prefix.len() < (1 << 31) { + if let Some(literal) = chain_literal_part(ctx, left) { + return Ok(ctx.block().call( + DOUBLE, + "js_string_concat_lit_value_box", + &[ + (I64, &literal_address_operand(&literal)), + (I64, &literal_value_meta_operand(&literal)), + (DOUBLE, r_box), + ], + )); + } + } + } // Issue #214: SSO-safe unbox; repsel Phase 3a: inline `bitcast+and` // for proven-heap operands (string literals — the `"user_" + i` // shape) and tag-dispatch for canonical-Str locals. @@ -876,6 +895,152 @@ fn chain_part_needs_default_primitive(ctx: &FnCtx<'_>, part: &Expr) -> bool { && !crate::expr::expr_produces_non_pointer_bits_by_construction(ctx, part) } +/// A string literal operand of a concat chain, passed to the runtime as +/// `(rodata address, byte length, UTF-16 length)` instead of as a heap handle. +/// The bytes are the literal's own `.str.N.bytes` constant, which the module +/// already emits for the string pool; nothing is recorded per site. +#[derive(Clone)] +pub(crate) struct LiteralPart { + /// The literal's text as a constant `i64` address operand. + address: String, + byte_len: usize, + utf16_len: usize, + /// The literal's header would pass the concat memo's admission test: it is + /// created at run time, not placed in the program image. + memo_eligible: bool, +} + +/// One operand of `js_string_concat_parts`. +pub(crate) enum ConcatPartArg { + Value(String), + Literal(LiteralPart), +} + +/// The literal form of `part`, when it is a source string literal. A source +/// `Expr::String` is valid UTF-8 with no lone surrogate (`Expr::WtfString` is +/// the other kind and stays a value operand). +pub(crate) fn chain_literal_part(ctx: &mut FnCtx<'_>, part: &Expr) -> Option { + let Expr::String(text) = part else { + return None; + }; + let utf16_len = text.encode_utf16().count(); + if text.len() >= (1 << 31) || utf16_len >= (1 << 31) { + return None; + } + let idx = ctx.strings.intern(text); + let entry = ctx.strings.entry(idx); + // An executable's literals are image-resident string objects: the text is + // the 20 bytes after the record's `StringHeader`, already in the binary. + // Pointing at it adds no data. A library output (`dylib`/`staticlib`) + // creates its literals at run time and keeps the pool's `.rodata` bytes. + let address = if ctx.strings.image_literals() { + format!( + "ptrtoint (ptr getelementptr (i8, ptr {}, i64 20) to i64)", + crate::codegen::string_pool_image_pointer(entry) + ) + } else { + format!("ptrtoint (ptr @{} to i64)", entry.bytes_global) + }; + Some(LiteralPart { + address, + byte_len: entry.byte_len, + utf16_len, + memo_eligible: !ctx.strings.image_literals(), + }) +} + +/// Whether `part` is a source string literal `chain_literal_part` accepts. +pub(crate) fn is_chain_literal(part: &Expr) -> bool { + matches!(part, Expr::String(text) + if text.len() < (1 << 31) && text.encode_utf16().count() < (1 << 31)) +} + +/// The literal's address as a constant operand. +pub(crate) fn literal_address_operand(literal: &LiteralPart) -> String { + literal.address.clone() +} + +/// `byte_len | utf16_len << 32`, as the signed immediate the IR spells. +pub(crate) fn literal_meta_operand(literal: &LiteralPart) -> String { + let meta = (literal.byte_len as u64) | ((literal.utf16_len as u64) << 32); + (meta as i64).to_string() +} + +/// The `js_string_concat_lit_value_box` form: `literal_meta_operand` plus bit +/// 63 when the memo may be consulted (`CONCAT_LIT_MEMO_ELIGIBLE`). +pub(crate) fn literal_value_meta_operand(literal: &LiteralPart) -> String { + let meta = (literal.byte_len as u64) + | ((literal.utf16_len as u64) << 32) + | if literal.memo_eligible { 1 << 63 } else { 0 }; + (meta as i64).to_string() +} + +/// True when `bits` may need ToPrimitive / may be a Symbol: everything except +/// a plain number, a heap string and an inline string. Those three are +/// primitive and never a Symbol. INT32-tagged values stay on the call because +/// a class reference is an INT32 class id (#9087), and so do pointers, BigInts, +/// undefined/null/booleans and handles. +fn emit_value_needs_conversion(ctx: &mut FnCtx<'_>, bits: &str) -> String { + use crate::nanbox::{SHORT_STRING_TAG_TOP16_I64, STRING_TAG_TOP16_I64}; + let blk = ctx.block(); + let plain = crate::codegen::emit_plain_number_bits_test(blk, bits); + let top16 = blk.lshr(I64, bits, "48"); + let is_string = blk.icmp_eq(I64, &top16, STRING_TAG_TOP16_I64); + let is_inline = blk.icmp_eq(I64, &top16, SHORT_STRING_TAG_TOP16_I64); + let skip = blk.or(crate::types::I1, &plain, &is_string); + let skip = blk.or(crate::types::I1, &skip, &is_inline); + blk.xor(crate::types::I1, &skip, "true") +} + +/// `js_to_primitive_default_for_add(raw)` behind one inline test of the live +/// bits: a plain number, heap string or inline string is its own primitive. +fn emit_to_primitive_for_add_tested(ctx: &mut FnCtx<'_>, raw: &str) -> String { + let bits = ctx.block().bitcast_double_to_i64(raw); + let needs = emit_value_needs_conversion(ctx, &bits); + let call_idx = ctx.new_block("cat.prim.call"); + let merge_idx = ctx.new_block("cat.prim.merge"); + let call_label = ctx.block_label(call_idx); + let merge_label = ctx.block_label(merge_idx); + let pred = ctx.block().label.clone(); + ctx.block().cond_br(&needs, &call_label, &merge_label); + + ctx.current_block = call_idx; + let converted = ctx + .block() + .call(DOUBLE, "js_to_primitive_default_for_add", &[(DOUBLE, raw)]); + let call_pred = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + ctx.block() + .phi(DOUBLE, &[(raw, &pred), (&converted, &call_pred)]) +} + +/// `js_add_throw_if_symbol(value)` behind the same inline test. +fn emit_symbol_check_tested(ctx: &mut FnCtx<'_>, value: &str) { + let bits = ctx.block().bitcast_double_to_i64(value); + let needs = emit_value_needs_conversion(ctx, &bits); + let call_idx = ctx.new_block("cat.sym.call"); + let done_idx = ctx.new_block("cat.sym.done"); + let call_label = ctx.block_label(call_idx); + let done_label = ctx.block_label(done_idx); + ctx.block().cond_br(&needs, &call_label, &done_label); + ctx.current_block = call_idx; + ctx.block() + .call_void("js_add_throw_if_symbol", &[(DOUBLE, value)]); + ctx.block().br(&done_label); + ctx.current_block = done_idx; +} + +/// A part that cannot be a Symbol: a literal, a proven runtime string, or a +/// value whose bits are non-pointer by construction. +fn chain_part_cannot_be_symbol(ctx: &FnCtx<'_>, part: &Expr, literal: bool) -> bool { + literal + || matches!(part, Expr::String(_) | Expr::WtfString(_)) + || crate::type_analysis::string_value_is_runtime_guaranteed(ctx, part) + || crate::expr::expr_produces_non_pointer_bits_by_construction(ctx, part) +} + pub(crate) fn lower_string_concat_chain(ctx: &mut FnCtx<'_>, parts: &[&Expr]) -> Result { debug_assert!(parts.len() >= 2); debug_assert!(parts.len() <= CONCAT_CHAIN_MAX_PARTS); @@ -888,52 +1053,82 @@ pub(crate) fn lower_string_concat_chain(ctx: &mut FnCtx<'_>, parts: &[&Expr]) -> .map(|part| chain_part_without_redundant_coerce(ctx, part)) .collect(); let parts = parts.as_slice(); + // A literal has no value to convert, root or check: it goes to the runtime + // as constant data. It is still LOWERED in its source position, and the + // result simply unused, so the string pool interns literals in exactly the + // order it always did (the image layout, and every address derived from + // it, stays the same); the dead handle load is removed by the optimizer. + let is_literal: Vec = parts.iter().map(|part| is_chain_literal(part)).collect(); if needs_primitive.iter().any(|needed| *needed) { + let may_be_symbol: Vec = parts + .iter() + .zip(&is_literal) + .map(|(part, literal)| !chain_part_cannot_be_symbol(ctx, part, *literal)) + .collect(); return with_rooted_group(ctx, parts.len() + 2, |ctx, group| { // For the first Add, both operands are evaluated before either is // coerced. Later parts are evaluated and coerced one at a time, // before the next Add's right operand is evaluated. let mut first = Vec::with_capacity(2); - for part in parts.iter().take(2) { + for (part, literal) in parts.iter().zip(&is_literal).take(2) { let raw = lower_expr(ctx, part)?; - first.push(group.adopt_emitted(ctx, Repr::Boxed, &raw, true)); + if *literal { + first.push(None); + continue; + } + first.push(Some(group.adopt_emitted(ctx, Repr::Boxed, &raw, true))); } let mut converted = Vec::with_capacity(parts.len()); for i in 0..parts.len() { - let raw = if i < 2 { - group.reread_emitted(ctx, first[i]) + if !is_literal[i] { + let raw = if i < 2 { + group.reread_emitted(ctx, first[i].expect("value part has a root")) + } else { + lower_expr(ctx, parts[i])? + }; + let value = if needs_primitive[i] { + emit_to_primitive_for_add_tested(ctx, &raw) + } else { + raw + }; + converted.push(Some(group.adopt_emitted(ctx, Repr::Boxed, &value, true))); } else { - lower_expr(ctx, parts[i])? - }; - let value = if needs_primitive[i] { - ctx.block() - .call(DOUBLE, "js_to_primitive_default_for_add", &[(DOUBLE, &raw)]) - } else { - raw - }; - converted.push(group.adopt_emitted(ctx, Repr::Boxed, &value, true)); + if i >= 2 { + lower_expr(ctx, parts[i])?; + } + converted.push(None); + } // A Symbol throws only after both operands of this Add have // been converted. Do that check after the head pair, then // after each later right operand and before the next one. + // An operand that cannot be a Symbol carries no check at all; + // any other is checked behind an inline test of its bits. if i == 1 { - for part in converted.iter().take(2) { - let value = group.reread_emitted(ctx, *part); - ctx.block() - .call_void("js_add_throw_if_symbol", &[(DOUBLE, &value)]); + for index in 0..2 { + if let (true, Some(root)) = (may_be_symbol[index], converted[index]) { + let value = group.reread_emitted(ctx, root); + emit_symbol_check_tested(ctx, &value); + } } } else if i > 1 { - let value = group.reread_emitted(ctx, converted[i]); - ctx.block() - .call_void("js_add_throw_if_symbol", &[(DOUBLE, &value)]); + if let (true, Some(root)) = (may_be_symbol[i], converted[i]) { + let value = group.reread_emitted(ctx, root); + emit_symbol_check_tested(ctx, &value); + } } } - let lowered: Vec = converted - .into_iter() - .map(|part| group.reread_emitted(ctx, part)) - .collect(); - Ok(emit_string_concat_chain(ctx, &lowered)) + let mut lowered = Vec::with_capacity(parts.len()); + for (index, root) in converted.into_iter().enumerate() { + lowered.push(match root { + Some(root) => ConcatPartArg::Value(group.reread_emitted(ctx, root)), + None => ConcatPartArg::Literal( + chain_literal_part(ctx, parts[index]).expect("literal part"), + ), + }); + } + Ok(emit_string_concat_parts(ctx, &lowered)) }); } @@ -944,10 +1139,53 @@ pub(crate) fn lower_string_concat_chain(ctx: &mut FnCtx<'_>, parts: &[&Expr]) -> // one allocating interpolation was enough to sweep the parts already // lowered. Parts that nothing allocating follows emit no rooting calls. with_operands_rooted(ctx, parts, |ctx, lowered| { - Ok(emit_string_concat_chain(ctx, lowered)) + let args: Vec = parts + .iter() + .zip(lowered) + .map(|(part, value)| match chain_literal_part(ctx, part) { + Some(literal) => ConcatPartArg::Literal(literal), + None => ConcatPartArg::Value(value.clone()), + }) + .collect(); + Ok(emit_string_concat_parts(ctx, &args)) }) } +/// Emit the parts buffer for `js_string_concat_parts`: two words per operand, +/// `{bits, -1}` for a value and `{rodata address, byte_len | utf16_len << 32}` +/// for a literal. The buffer is hoisted to the entry block like the chain's +/// (issue #167). +pub(crate) fn emit_string_concat_parts(ctx: &mut FnCtx<'_>, parts: &[ConcatPartArg]) -> String { + debug_assert!(parts.len() >= 2); + debug_assert!(parts.len() <= CONCAT_CHAIN_MAX_PARTS); + let n = parts.len(); + let buf_reg = ctx.func.alloca_entry_array(I64, 2 * CONCAT_CHAIN_MAX_PARTS); + let blk = ctx.block(); + for (i, part) in parts.iter().enumerate() { + let word = blk.gep(I64, &buf_reg, &[(I64, &(2 * i).to_string())]); + let meta = blk.gep(I64, &buf_reg, &[(I64, &(2 * i + 1).to_string())]); + match part { + ConcatPartArg::Value(value) => { + let bits = blk.bitcast_double_to_i64(value); + blk.store(I64, &bits, &word); + blk.store(I64, "-1", &meta); + } + ConcatPartArg::Literal(literal) => { + blk.store(I64, &literal_address_operand(literal), &word); + blk.store(I64, &literal_meta_operand(literal), &meta); + } + } + } + let base_i64 = blk.next_reg(); + blk.emit_raw(format!("{} = ptrtoint ptr {} to i64", base_i64, buf_reg)); + let result_handle = blk.call( + I64, + "js_string_concat_parts", + &[(I64, &base_i64), (I32, &n.to_string())], + ); + nanbox_string_inline(blk, &result_handle) +} + /// Emit the shared stack-buffer + runtime-call core for values that have /// already been lowered, coerced where required, and re-read from their roots. /// diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index e12eaa41893..b5a5b2e3734 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -61,6 +61,9 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { // second arg is the count. Returns a raw string handle. // (`crates/perry-runtime/src/string.rs::js_string_concat_chain`) module.declare_function("js_string_concat_chain", I64, &[I64, I32]); + // Literal operands as constant data (`string/concat_parts.rs`). + module.declare_function("js_string_concat_parts", I64, &[I64, I32]); + module.declare_function("js_string_concat_lit_value_box", DOUBLE, &[I64, I64, DOUBLE]); module.declare_function("js_to_primitive_default_for_add", DOUBLE, &[DOUBLE]); module.declare_function("js_add_throw_if_symbol", VOID, &[DOUBLE]); // Self-append variant of the N-way chain. The first part is the binding's diff --git a/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs b/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs index baedd2961c9..fd86d8cabae 100644 --- a/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs +++ b/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs @@ -376,7 +376,6 @@ fn string_code(index: Expr) -> Expr { args: vec![index], type_args: vec![], byte_offset: 0, - byte_end: 0, } } @@ -461,7 +460,6 @@ fn string_region_rejects_rebinding_and_calls_before_any_body_effect() { args: vec![], type_args: vec![], byte_offset: 0, - byte_end: 0, }), ], vec![Stmt::If { diff --git a/crates/perry-codegen/src/strings.rs b/crates/perry-codegen/src/strings.rs index 32eafc3979f..cc3d004a4f8 100644 --- a/crates/perry-codegen/src/strings.rs +++ b/crates/perry-codegen/src/strings.rs @@ -46,6 +46,9 @@ pub struct StringPool { /// the `js_set_call_location` is emitted with the *outer* call's offset /// immediately before the throwing dispatch. `0` = none. pending_call_offset: std::cell::Cell, + /// Whether this module's literals are image-resident string objects + /// (`output_type` not `dylib`/`staticlib`); see `string_pool.rs`. + image_literals: bool, } pub struct StringEntry { @@ -93,9 +96,19 @@ impl StringPool { debug_location_ctx: None, debug_source_line_offset: 0, pending_call_offset: std::cell::Cell::new(0), + image_literals: false, } } + pub fn set_image_literals(&mut self, image_literals: bool) { + self.image_literals = image_literals; + } + + /// True when literals are image-resident (static header flags). + pub fn image_literals(&self) -> bool { + self.image_literals + } + pub fn module_prefix(&self) -> &str { &self.module_prefix } diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index 798c2bcbbf5..a7e8a228796 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -4007,6 +4007,8 @@ js_string_compare_value i32s f64,f64 js_string_concat ptr ptr,ptr js_string_concat_box f64 f64,f64 js_string_concat_chain ptr ptr,i32s +js_string_concat_lit_value_box f64 ptr,i64,f64 +js_string_concat_parts ptr ptr,i32s js_string_concat_site_value f64 ptr,ptr,f64 js_string_concat_value ptr ptr,f64 js_string_concat_value_box f64 ptr,f64 diff --git a/crates/perry-codegen/tests/temp_root_operand_temporaries.rs b/crates/perry-codegen/tests/temp_root_operand_temporaries.rs index 178d3e2a470..b0a2b130307 100644 --- a/crates/perry-codegen/tests/temp_root_operand_temporaries.rs +++ b/crates/perry-codegen/tests/temp_root_operand_temporaries.rs @@ -801,6 +801,29 @@ fn string_literal_concat_operand_is_re_derived_below_the_allocating_sibling() { // are unambiguous rather than "whichever matched first". let f = init_ir(&ir); let handle = "load double, ptr @concat_reload_ts_.str."; + // JST slice 3: an ASCII source-literal prefix reaches the runtime as its + // rodata address and lengths (`js_string_concat_lit_value_box`), a + // link-time constant. There is no register holding the literal to go + // stale across the sibling's collection point, so the invariant below + // holds by construction; assert that form directly. + if f.contains("call double @js_string_concat_lit_value_box(") { + assert_eq!( + f.matches("call double @js_string_concat_lit_value_box(") + .count(), + 1, + "exactly one fused string+value concat in @main:\n{f}" + ); + assert_eq!(object_allocation_count(f), 1, "{f}"); + assert!( + f.contains( + "@js_string_concat_lit_value_box(i64 ptrtoint (ptr getelementptr (i8, ptr getelementptr (i8, ptr @perry_image_literal_" + ), + "the literal must travel as a constant image address, never a \ + register that a collection could leave stale:\n{f}" + ); + assert_no_temp_rooting(f, "a constant operand needs no slot"); + return; + } assert_eq!( f.matches("call double @js_string_concat_value_box(") .count(), diff --git a/crates/perry-runtime/src/string/concat.rs b/crates/perry-runtime/src/string/concat.rs index 52282958c04..3a2262ce450 100644 --- a/crates/perry-runtime/src/string/concat.rs +++ b/crates/perry-runtime/src/string/concat.rs @@ -326,7 +326,7 @@ pub extern "C" fn js_string_concat_box(l_value: f64, r_value: f64) -> f64 { /// regression traded for wall time, which is the wrong trade /// (`memory-parity-directive`). const CONCAT_MEMO_SIZE: usize = 512; -const CONCAT_MEMO_MAX_BYTES: u32 = 12; +pub(super) const CONCAT_MEMO_MAX_BYTES: u32 = 12; // Candidates per governor window. const MEMO_WINDOW: u32 = 4096; @@ -378,7 +378,7 @@ per_test_global! { /// Count this candidate and report whether the memo path is worth walking. /// Closes a window every `MEMO_WINDOW` candidates. #[inline] -fn concat_memo_should_probe() -> bool { +pub(super) fn concat_memo_should_probe() -> bool { use std::sync::atomic::Ordering::Relaxed; let v = MEMO_GOV.load(Relaxed); let count = (v & GOV_COUNT_MASK) + 1; @@ -409,7 +409,7 @@ fn concat_memo_should_probe() -> bool { } #[inline] -fn concat_memo_note_hit() { +pub(super) fn concat_memo_note_hit() { MEMO_GOV.fetch_add(1u64 << GOV_HIT_SHIFT, std::sync::atomic::Ordering::Relaxed); } @@ -472,7 +472,7 @@ crate::perry_thread_local! { /// Otherwise records it and declines, so the first sighting costs nothing but /// a byte. #[inline] -fn concat_memo_admit(slot: usize, tag: u8) -> bool { +pub(super) fn concat_memo_admit(slot: usize, tag: u8) -> bool { CONCAT_MEMO_TAGS.with(|t| unsafe { let slot_tag = &mut (*t.get())[slot]; if *slot_tag == tag { @@ -561,17 +561,17 @@ fn concat_memo_lookup_parts(slot: usize, a: &[u8], b: &[u8]) -> *mut StringHeade /// primitives with an empty second operand — one hash/lookup definition, /// not two. #[inline] -fn concat_memo_slot_and_tag(bytes: &[u8]) -> (usize, u8) { +pub(super) fn concat_memo_slot_and_tag(bytes: &[u8]) -> (usize, u8) { concat_memo_slot_and_tag_parts(bytes, &[]) } #[inline] -fn concat_memo_lookup(slot: usize, bytes: &[u8]) -> *mut StringHeader { +pub(super) fn concat_memo_lookup(slot: usize, bytes: &[u8]) -> *mut StringHeader { concat_memo_lookup_parts(slot, bytes, &[]) } #[inline] -fn concat_memo_insert(slot: usize, ptr: *mut StringHeader) { +pub(super) fn concat_memo_insert(slot: usize, ptr: *mut StringHeader) { CONCAT_MEMO.with(|c| unsafe { // GC_STORE_AUDIT(ROOT): CONCAT_MEMO is scanned by scan_concat_memo_roots_mut. crate::gc::runtime_store_root_raw_mut_ptr_slot(&raw mut (*c.get())[slot], ptr); @@ -1892,27 +1892,57 @@ pub(crate) fn format_ryu_js_into(value: f64, buf: &mut [u8; 32]) -> usize { len } -/// Fast integer-to-ASCII formatting into a provided buffer. -/// Returns the number of bytes written. Digits are written to the END -/// of the buffer and then shifted to the front. -#[inline] -pub(crate) fn fast_itoa_u32(mut n: u32, buf: &mut [u8; 32]) -> usize { - if n == 0 { - buf[0] = b'0'; - return 1; - } - // Size first, then write digits in place back-to-front. The old - // write-at-the-end-then-`copy_within` shape paid a libc `memmove` PLT - // call per conversion (runtime-length overlapping copy — it showed up - // as ~15% of the `"id-" + i` loop in `sample`, inlined into both - // concat entry points). - let len = n.ilog10() as usize + 1; +/// Two decimal digits per entry, `"00".."99"`. A constant of the formatter, +/// not a cache. +const DIGIT_PAIRS: &[u8; 200] = b"0001020304050607080910111213141516171819\ +2021222324252627282930313233343536373839\ +4041424344454647484950515253545556575859\ +6061626364656667686970717273747576777879\ +8081828384858687888990919293949596979899"; + +/// Decimal digit count of `n` (`0` has one). +#[inline(always)] +pub(crate) fn decimal_len(n: u32) -> usize { + (n | 1).ilog10() as usize + 1 +} + +/// Write the `len` decimal digits of `n` at `out..out+len`, two per step, +/// back to front. `len` must be `decimal_len(n)`. +/// +/// # Safety +/// `out` is valid for `len` bytes. +#[inline(always)] +pub(crate) unsafe fn write_decimal(out: *mut u8, mut n: u32, len: usize) { let mut pos = len; - while n > 0 { + while n >= 100 { + let pair = (n % 100) as usize * 2; + n /= 100; + pos -= 2; + out.add(pos).copy_from_nonoverlapping(DIGIT_PAIRS.as_ptr().add(pair), 2); + } + if n >= 10 { + pos -= 2; + let pair = n as usize * 2; + out.add(pos).copy_from_nonoverlapping(DIGIT_PAIRS.as_ptr().add(pair), 2); + } else { pos -= 1; - buf[pos] = b'0' + (n % 10) as u8; - n /= 10; + *out.add(pos) = b'0' + n as u8; } + debug_assert_eq!(pos, 0); +} + +/// Fast integer-to-ASCII formatting into a provided buffer. +/// Returns the number of bytes written. +/// +/// The length is known first (one `ilog10`), then digits are written in place +/// back-to-front two at a time: one divide by 100 per pair halves the serial +/// multiply-high chain a digit-at-a-time loop pays (6-7 digit ids are the +/// common shape). +#[inline] +pub(crate) fn fast_itoa_u32(n: u32, buf: &mut [u8; 32]) -> usize { + let len = decimal_len(n); + // SAFETY: `len <= 10 <= buf.len()`. + unsafe { write_decimal(buf.as_mut_ptr(), n, len) }; len } diff --git a/crates/perry-runtime/src/string/concat_parts.rs b/crates/perry-runtime/src/string/concat_parts.rs new file mode 100644 index 00000000000..7e53faf5890 --- /dev/null +++ b/crates/perry-runtime/src/string/concat_parts.rs @@ -0,0 +1,509 @@ +//! Literal-aware concatenation (JST slice 3, design B). +//! +//! `"item-" + (i + r) + ";"` used to hand `js_string_concat_chain` a heap +//! `StringHeader` for each literal and let it re-derive everything about them +//! (tag dispatch, header loads, a failed all-heap attempt, a root per literal). +//! Codegen already knows the literal bytes, their length and their UTF-16 +//! length, and the bytes are constants in the program's rodata. This module +//! takes them as operands of that form: `(ptr, byte_len | utf16_len << 32)` +//! next to ordinary value operands, so the work per call is classification of +//! the value operands, one allocation and the copies. +//! +//! Nothing here is a cache or a table. `js_string_concat_lit_value_box` is the +//! two-operand `"literal" + number` form and consults the process concat memo +//! exactly where `concat_formatted_number` does. + +use super::concat::{ + canonicalize_surrogate_pairs, concat_memo_admit, concat_memo_insert, concat_memo_lookup, + concat_memo_note_hit, concat_memo_should_probe, concat_memo_slot_and_tag, copy_bytes_small, + decimal_len, fast_itoa_i64, fast_itoa_u32, format_number_into, js_string_concat, write_decimal, + CONCAT_MEMO_MAX_BYTES, +}; +use super::*; + +/// One operand slot as codegen writes it. A value operand is +/// `{ NaN-boxed bits, CONCAT_PART_VALUE }`. A literal operand is +/// `{ address of its rodata bytes, byte_len | utf16_len << 32 }`; codegen +/// admits only source literals, which are valid UTF-8 with no lone surrogate. +#[repr(C)] +#[derive(Clone, Copy)] +pub struct ConcatPartSlot { + pub word: u64, + pub meta: u64, +} + +/// `meta` of a value operand. A literal's `meta` carries a UTF-16 length below +/// 2^31 in its high word, so it can never equal this. +pub const CONCAT_PART_VALUE: u64 = u64::MAX; + +const MAX_PARTS: usize = 32; + +#[inline(always)] +fn lit_blen(meta: u64) -> u32 { + meta as u32 +} + +#[inline(always)] +fn lit_units(meta: u64) -> u32 { + ((meta >> 32) & 0x7FFF_FFFF) as u32 +} + +/// `js_string_concat_lit_value_box` only: the literal's own header would pass +/// `concat_formatted_number`'s memo admission. Today a literal that lives in +/// the program image carries static header flags (`perry_image_literal_*`), +/// which fail that test, so the memo is consulted for a literal prefix only in +/// a build whose literals are created at run time. Codegen knows which build it +/// is emitting and sets this bit accordingly; the runtime does not guess. +pub const CONCAT_LIT_MEMO_ELIGIBLE: u64 = 1 << 63; + +/// N-way concatenation whose literal operands are constant data. +/// +/// Returns a fresh shared (`refcount == 0`) heap string, like +/// `js_string_concat_chain`. Value operands that are heap strings, inline +/// strings or numbers stay on the fast path; any other kind (a class id, +/// `undefined`, a BigInt, ...) takes the general arm, which converts it the way +/// the chain's general arm does. +#[no_mangle] +pub extern "C" fn js_string_concat_parts( + parts: *const ConcatPartSlot, + n: i32, +) -> *mut StringHeader { + let n = (n.max(0) as usize).min(MAX_PARTS); + if n == 0 || parts.is_null() { + return crate::string::js_string_from_bytes(b"".as_ptr(), 0); + } + if n <= 4 { + concat_parts_sized::<4>(parts, n) + } else if n <= 8 { + concat_parts_sized::<8>(parts, n) + } else { + concat_parts_sized::(parts, n) + } +} + +#[inline] +fn concat_parts_sized(parts: *const ConcatPartSlot, n: usize) -> *mut StringHeader { + if let Some(result) = concat_parts_simple::(parts, n) { + return result; + } + concat_parts_general::(parts, n) +} + +/// A value operand that is an exact `u32` is formatted straight into the +/// result: its digit count sizes the allocation and its digits are written at +/// the copy position, so it needs neither a scratch buffer nor a second copy. +#[inline(always)] +fn exact_u32(value: f64) -> Option { + if (0.0..=u32::MAX as f64).contains(&value) { + // SAFETY: the range test above admits only finite values in + // `0..=u32::MAX`, which an `i64` holds exactly. + let n = unsafe { value.to_int_unchecked::() }; + if n as f64 == value { + return Some(n as u32); + } + } + None +} + +/// Literals, heap strings, inline strings and numbers: no user code, and no +/// allocation before the single result allocation. +#[inline] +fn concat_parts_simple( + parts: *const ConcatPartSlot, + n: usize, +) -> Option<*mut StringHeader> { + let mut scratch: [core::mem::MaybeUninit<[u8; 32]>; M] = [core::mem::MaybeUninit::uninit(); M]; + // `piece_ptrs[i]` is null for a part written as digits of `digits[i]`. + let mut piece_ptrs: [*const u8; M] = [ptr::null(); M]; + let mut piece_lens: [u32; M] = [0; M]; + let mut digits: [u32; M] = [0; M]; + let mut flags = STRING_FLAG_WTF8_VALIDATED; + // 32 parts of at most `u32::MAX` bytes cannot overflow a u64; the sum is + // checked once, below. + let mut total_blen: u64 = 0; + let mut total_u16: u64 = 0; + for i in 0..n { + let slot = unsafe { *parts.add(i) }; + if slot.meta != CONCAT_PART_VALUE { + let blen = lit_blen(slot.meta); + piece_ptrs[i] = slot.word as *const u8; + piece_lens[i] = blen; + total_blen += blen as u64; + total_u16 += lit_units(slot.meta) as u64; + continue; + } + let bits = slot.word; + if bits >> 48 == 0x7FFF { + let header = (bits & 0x0000_FFFF_FFFF_FFFF) as *const StringHeader; + if !is_valid_string_ptr(header) { + return None; + } + let blen = unsafe { (*header).byte_len }; + // An empty operand contributes neither bytes nor provenance. + if blen == 0 { + continue; + } + piece_ptrs[i] = string_data(header); + piece_lens[i] = blen; + flags = combine_string_flags(flags, unsafe { (*header).flags }); + total_blen += blen as u64; + total_u16 += unsafe { (*header).utf16_len } as u64; + continue; + } + let value = crate::value::JSValue::from_bits(bits); + if value.is_number() { + let number = f64::from_bits(bits); + let len = if let Some(n) = exact_u32(number) { + digits[i] = n; + decimal_len(n) + } else { + let buf = scratch[i].write([0u8; 32]); + let len = format_number_into(number, buf); + piece_ptrs[i] = buf.as_ptr(); + len + }; + piece_lens[i] = len as u32; + total_blen += len as u64; + total_u16 += len as u64; // ASCII + continue; + } + if value.is_short_string() { + let buf = scratch[i].write([0u8; 32]); + let mut bytes = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + let len = value.short_string_to_buf(&mut bytes); + if len == 0 { + continue; + } + buf[..len].copy_from_slice(&bytes[..len]); + let (units, part_flags) = raw_string_metadata(buf.as_ptr(), len as u32); + flags = combine_string_flags(flags, part_flags); + piece_ptrs[i] = buf.as_ptr(); + piece_lens[i] = len as u32; + total_blen += len as u64; + total_u16 += units as u64; + continue; + } + return None; + } + if total_blen > u32::MAX as u64 || total_u16 > u32::MAX as u64 { + return None; + } + let (total_blen, total_u16) = (total_blen as u32, total_u16 as u32); + + // `Some` proves nothing moved, so the borrowed heap payloads are valid. + let (result, mut cursor) = match string_storage_alloc_no_collect(total_blen) { + Some(allocated) => allocated, + None => { + return Some(concat_parts_rooted_copy::( + parts, + n, + &mut piece_ptrs, + &piece_lens, + &digits, + total_blen, + total_u16, + flags, + )) + } + }; + unsafe { + init_string_header(result, total_u16, total_blen, total_blen, 0, flags); + for i in 0..n { + let len = piece_lens[i] as usize; + // An empty operand has a null pointer too: it writes nothing. + if len == 0 { + continue; + } + if piece_ptrs[i].is_null() { + write_decimal(cursor, digits[i], len); + } else { + // A literal or a number is a handful of bytes: keep them out + // of libc's memcpy. Long heap payloads still take it. + copy_bytes_small(piece_ptrs[i], cursor, len); + } + cursor = cursor.add(len); + } + } + // Only a heap or inline operand can carry a lone surrogate, and only a + // join of two such operands can mint a pair. + if flags & STRING_FLAG_HAS_LONE_SURROGATES != 0 { + Some(canonicalize_surrogate_pairs(result)) + } else { + Some(result) + } +} + +/// The block boundary refused the no-collect allocation: root every heap +/// operand, take the collecting allocation, and re-read each from its handle. +/// Literals, digits and the stack conversions do not move. +#[cold] +#[inline(never)] +#[allow(clippy::too_many_arguments)] +fn concat_parts_rooted_copy( + parts: *const ConcatPartSlot, + n: usize, + piece_ptrs: &mut [*const u8; M], + piece_lens: &[u32; M], + digits: &[u32; M], + total_blen: u32, + total_u16: u32, + flags: u32, +) -> *mut StringHeader { + let scope = crate::gc::RuntimeHandleScope::new(); + let mut handles = [None; M]; + for i in 0..n { + let slot = unsafe { *parts.add(i) }; + if slot.meta == CONCAT_PART_VALUE && slot.word >> 48 == 0x7FFF && piece_lens[i] != 0 { + handles[i] = Some( + scope.root_string_ptr((slot.word & 0x0000_FFFF_FFFF_FFFF) as *const StringHeader), + ); + } + } + let (result, mut cursor) = string_storage_alloc(total_blen); + unsafe { + init_string_header(result, total_u16, total_blen, total_blen, 0, flags); + for i in 0..n { + let len = piece_lens[i] as usize; + if len == 0 { + continue; + } + if let Some(handle) = handles[i] { + piece_ptrs[i] = string_data(handle.get_raw_const_ptr::()); + } + if piece_ptrs[i].is_null() { + write_decimal(cursor, digits[i], len); + } else { + ptr::copy_nonoverlapping(piece_ptrs[i], cursor, len); + } + cursor = cursor.add(len); + } + } + canonicalize_surrogate_pairs(result) +} + +/// Every other value kind. The converted text of a value operand is a rooted +/// heap string; a literal or a number is borrowed from rodata or the stack. +#[cold] +#[inline(never)] +fn concat_parts_general( + parts: *const ConcatPartSlot, + n: usize, +) -> *mut StringHeader { + let mut num_bufs: [core::mem::MaybeUninit<[u8; 32]>; M] = [core::mem::MaybeUninit::uninit(); M]; + let scope = crate::gc::RuntimeHandleScope::new(); + let mut handles = [None; M]; + let mut piece_ptrs: [*const u8; M] = [ptr::null(); M]; + let mut piece_lens: [u32; M] = [0; M]; + let mut flags = STRING_FLAG_WTF8_VALIDATED; + let mut total_blen: u32 = 0; + let mut total_u16: u32 = 0; + + for i in 0..n { + let slot = unsafe { *parts.add(i) }; + if slot.meta != CONCAT_PART_VALUE { + let blen = lit_blen(slot.meta); + piece_ptrs[i] = slot.word as *const u8; + piece_lens[i] = blen; + total_blen = total_blen.saturating_add(blen); + total_u16 = total_u16.saturating_add(lit_units(slot.meta)); + continue; + } + let value = f64::from_bits(slot.word); + let bits = slot.word; + let tag = bits >> 48; + + if tag == 0x7FFF { + let header = (bits & 0x0000_FFFF_FFFF_FFFF) as *const StringHeader; + if is_valid_string_ptr(header) { + let blen = unsafe { (*header).byte_len }; + if blen > 0 { + handles[i] = Some(scope.root_string_ptr(header)); + piece_lens[i] = blen; + flags = combine_string_flags(flags, unsafe { (*header).flags }); + total_blen = total_blen.saturating_add(blen); + total_u16 = total_u16.saturating_add(unsafe { (*header).utf16_len }); + } + continue; + } + } + if crate::value::JSValue::from_bits(bits).is_number() { + let len = format_number_into(value, num_bufs[i].write([0u8; 32])); + piece_ptrs[i] = num_bufs[i].as_ptr() as *const u8; + piece_lens[i] = len as u32; + total_blen = total_blen.saturating_add(len as u32); + total_u16 = total_u16.saturating_add(len as u32); + continue; + } + // A registered class id stringifies as its function source, so only an + // ordinary INT32 takes the digit arm. + if tag == 0x7FFE && !crate::object::is_class_id_registered((bits & 0xFFFF_FFFF) as u32) { + let v = (bits & 0xFFFF_FFFF) as u32 as i32; + let len = { + let buf = num_bufs[i].write([0u8; 32]); + if v >= 0 { + fast_itoa_u32(v as u32, buf) + } else { + fast_itoa_i64(v as i64, buf) + } + }; + piece_ptrs[i] = num_bufs[i].as_ptr() as *const u8; + piece_lens[i] = len as u32; + total_blen = total_blen.saturating_add(len as u32); + total_u16 = total_u16.saturating_add(len as u32); + continue; + } + // Inline strings, class ids, undefined, null, booleans, BigInts, + // objects: the shared ToString. + let converted = crate::value::js_jsvalue_to_string(value); + if is_valid_string_ptr(converted) { + let blen = unsafe { (*converted).byte_len }; + if blen > 0 { + handles[i] = Some(scope.root_string_ptr(converted)); + piece_lens[i] = blen; + flags = combine_string_flags(flags, unsafe { (*converted).flags }); + total_blen = total_blen.saturating_add(blen); + total_u16 = total_u16.saturating_add(unsafe { (*converted).utf16_len }); + } + } + } + + let (result, mut cursor) = string_storage_alloc(total_blen); + unsafe { + init_string_header(result, total_u16, total_blen, total_blen, 0, flags); + for i in 0..n { + let len = piece_lens[i] as usize; + if len == 0 { + continue; + } + if let Some(handle) = handles[i] { + let piece = handle.get_raw_const_ptr::(); + if is_valid_string_ptr(piece) { + ptr::copy_nonoverlapping(string_data(piece), cursor, len); + cursor = cursor.add(len); + } + } else if !piece_ptrs[i].is_null() { + ptr::copy_nonoverlapping(piece_ptrs[i], cursor, len); + cursor = cursor.add(len); + } + } + } + canonicalize_surrogate_pairs(result) +} + +/// NaN-box-returning `"literal" + value` for a non-empty ASCII source literal, +/// the literal-operand twin of `js_string_concat_value_box`: the same answers +/// (SSO immediate for an exact-u32 result of at most five ASCII bytes, heap +/// `STRING_TAG` otherwise), the same memo consultation, minus the per-call +/// validation of a prefix codegen already knows. The literal is ASCII, so +/// `meta`'s UTF-16 length equals its byte length; bit 63 is +/// [`CONCAT_LIT_MEMO_ELIGIBLE`]. +#[no_mangle] +pub extern "C" fn js_string_concat_lit_value_box(lit: *const u8, meta: u64, value: f64) -> f64 { + let prefix_blen = lit_blen(meta) as usize; + let bits = value.to_bits(); + if crate::value::JSValue::from_bits(bits).is_number() { + let mut num_buf = [0u8; 32]; + let exact_u32 = (0.0..=u32::MAX as f64).contains(&value) && (value as u32) as f64 == value; + let num_len = if exact_u32 { + fast_itoa_u32(value as u32, &mut num_buf) + } else { + format_number_into(value, &mut num_buf) + }; + // Same admission as `js_string_concat_value_box`: only an exact u32 + // can become an inline string. + if exact_u32 + && prefix_blen < crate::value::SHORT_STRING_MAX_LEN + && prefix_blen + num_len <= crate::value::SHORT_STRING_MAX_LEN + { + let mut sso = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + unsafe { + copy_bytes_small(lit, sso.as_mut_ptr(), prefix_blen); + copy_bytes_small(num_buf.as_ptr(), sso.as_mut_ptr().add(prefix_blen), num_len); + } + return f64::from_bits( + crate::value::JSValue::short_string_unchecked(&sso[..prefix_blen + num_len]).bits(), + ); + } + let ptr = concat_literal_formatted_number(lit, meta, &num_buf, num_len); + return f64::from_bits(crate::value::js_nanbox_string(ptr as i64).to_bits()); + } + // Not a number: ToString the value first (it may run user code and + // collect), root that text, then build the prefix string and join. + let value_str = crate::value::js_jsvalue_to_string(value); + let scope = crate::gc::RuntimeHandleScope::new(); + let value_handle = scope.root_string_ptr(value_str); + let prefix = crate::string::js_string_from_bytes_known_utf16( + lit, + prefix_blen as u32, + lit_units(meta), + STRING_FLAG_WTF8_VALIDATED, + ); + let joined = js_string_concat(prefix, value_handle.get_raw_const_ptr::()); + f64::from_bits(crate::value::js_nanbox_string(joined as i64).to_bits()) +} + +/// `concat_formatted_number` for a literal prefix: the identical memo +/// protocol, with the prefix facts (ASCII, no surrogates, length) taken from +/// codegen instead of validated per call. +fn concat_literal_formatted_number( + lit: *const u8, + meta: u64, + num_buf: &[u8; 32], + num_len: usize, +) -> *mut StringHeader { + let prefix_blen = lit_blen(meta) as usize; + let total_blen = prefix_blen + num_len; + let memoizable = meta & CONCAT_LIT_MEMO_ELIGIBLE != 0 + && total_blen <= CONCAT_MEMO_MAX_BYTES as usize + && concat_memo_should_probe(); + let mut memo_buf = [0u8; CONCAT_MEMO_MAX_BYTES as usize]; + let mut memo_slot = 0usize; + let mut memo_admitted = false; + if memoizable { + unsafe { + copy_bytes_small(lit, memo_buf.as_mut_ptr(), prefix_blen); + copy_bytes_small( + num_buf.as_ptr(), + memo_buf.as_mut_ptr().add(prefix_blen), + num_len, + ); + } + let bytes = &memo_buf[..total_blen]; + let (slot, tag) = concat_memo_slot_and_tag(bytes); + memo_slot = slot; + let hit = concat_memo_lookup(memo_slot, bytes); + if !hit.is_null() { + concat_memo_note_hit(); + return hit; + } + memo_admitted = concat_memo_admit(memo_slot, tag); + } + // Prefer the open nursery block; a refusal needs no root because the + // prefix is rodata and the digits are on the stack. + let (result, data) = match string_storage_alloc_no_collect(total_blen as u32) { + Some(allocated) => allocated, + None => string_storage_alloc(total_blen as u32), + }; + unsafe { + // `concat_formatted_number` copies the prefix header's flags; a source + // literal's are the validated proof alone. + init_string_header( + result, + total_blen as u32, + total_blen as u32, + total_blen as u32, + 0, + STRING_FLAG_WTF8_VALIDATED, + ); + copy_bytes_small(lit, data, prefix_blen); + copy_bytes_small(num_buf.as_ptr(), data.add(prefix_blen), num_len); + } + if memoizable && memo_admitted { + concat_memo_insert(memo_slot, result); + } + result +} + +#[cfg(test)] +#[path = "concat_parts_tests.rs"] +mod tests; diff --git a/crates/perry-runtime/src/string/concat_parts_tests.rs b/crates/perry-runtime/src/string/concat_parts_tests.rs new file mode 100644 index 00000000000..4e9c68a7094 --- /dev/null +++ b/crates/perry-runtime/src/string/concat_parts_tests.rs @@ -0,0 +1,311 @@ +use super::*; +use crate::value::{JSValue, STRING_TAG}; + +fn inline(text: &[u8]) -> f64 { + f64::from_bits(JSValue::short_string_unchecked(text).bits()) +} + +fn heap(text: &str) -> f64 { + f64::from_bits(STRING_TAG | js_string_from_str(text) as u64) +} + +fn bytes(s: *const StringHeader) -> Vec { + unsafe { std::slice::from_raw_parts(string_data(s), (*s).byte_len as usize).to_vec() } +} + +fn value(v: f64) -> ConcatPartSlot { + ConcatPartSlot { + word: v.to_bits(), + meta: CONCAT_PART_VALUE, + } +} + +/// A literal operand over `text`, which must outlive the call. +fn literal(text: &'static str) -> ConcatPartSlot { + ConcatPartSlot { + word: text.as_ptr() as u64, + meta: text.len() as u64 | ((text.encode_utf16().count() as u64) << 32), + } +} + +fn parts_call(parts: &[ConcatPartSlot]) -> *mut StringHeader { + js_string_concat_parts(parts.as_ptr(), parts.len() as i32) +} + +fn chain_call(values: &[f64]) -> *mut StringHeader { + super::super::concat::js_string_concat_chain(values.as_ptr(), values.len() as i32) +} + +fn same_string(a: *const StringHeader, b: *const StringHeader) { + unsafe { + assert_eq!(bytes(a), bytes(b)); + assert_eq!((*a).utf16_len, (*b).utf16_len); + assert_eq!( + (*a).flags & STRING_FLAG_HAS_LONE_SURROGATES, + (*b).flags & STRING_FLAG_HAS_LONE_SURROGATES + ); + assert_eq!((*a).refcount, 0); + assert_eq!((*a).capacity, (*a).byte_len); + } +} + +#[test] +fn parts_agree_with_the_chain_on_every_value_kind() { + let numbers = [ + 0.0, + -0.0, + 1.0, + -1.0, + 9.0, + 10.0, + 99.0, + 100.0, + 999_999_999.0, + 1_000_000_000.0, + 2_147_483_647.0, + -2_147_483_648.0, + 4_294_967_295.0, + 4_294_967_296.0, + -4_294_967_295.0, + 1e15, + 1e21, + 1.5, + -0.25, + 1e-7, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + f64::MAX, + ]; + for number in numbers { + let literal_chain = chain_call(&[heap("item-"), number, heap(";")]); + let literal_parts = parts_call(&[literal("item-"), value(number), literal(";")]); + same_string(literal_parts, literal_chain); + } + // Heap, inline and empty operands around a number. + let operands: [f64; 6] = [ + heap("heap-string"), + inline(b"ab"), + inline(b""), + heap(""), + heap("é€😀"), + inline("é".as_bytes()), + ]; + for operand in operands { + let a = parts_call(&[literal("x"), value(operand), value(42.0), literal("y")]); + let b = chain_call(&[heap("x"), operand, 42.0, heap("y")]); + same_string(a, b); + } +} + +#[test] +fn other_value_kinds_take_the_general_arm_with_chain_results() { + let others = [ + f64::from_bits(0x7FFC_0000_0000_0001), + f64::from_bits(0x7FFC_0000_0000_0002), + f64::from_bits(0x7FFC_0000_0000_0004), + f64::from_bits(0x7FFC_0000_0000_0003), + f64::from_bits(JSValue::int32(7).bits()), + f64::from_bits(JSValue::int32(-7).bits()), + ]; + for other in others { + let a = parts_call(&[literal("<"), value(other), literal(">")]); + let b = chain_call(&[heap("<"), other, heap(">")]); + same_string(a, b); + } +} + +#[test] +fn literal_and_value_boundaries_around_surrogates() { + // A lone high surrogate in a heap operand followed by a lone low one joins + // into the astral scalar, exactly as the chain's canonicalization does. + let high = js_string_from_wtf8_bytes([0xED, 0xA0, 0xBD].as_ptr(), 3); + let low = js_string_from_wtf8_bytes([0xED, 0xB8, 0x80].as_ptr(), 3); + let high_box = f64::from_bits(STRING_TAG | high as u64); + let low_box = f64::from_bits(STRING_TAG | low as u64); + let joined = parts_call(&[value(high_box), value(low_box)]); + assert_eq!(bytes(joined), "😀".as_bytes()); + assert_eq!(unsafe { (*joined).utf16_len }, 2); + assert_eq!( + unsafe { (*joined).flags } & STRING_FLAG_HAS_LONE_SURROGATES, + 0 + ); + // A lone surrogate next to literals stays lone, with its flag. + let lone = parts_call(&[literal("a"), value(high_box), literal("b")]); + assert_eq!(bytes(lone), vec![b'a', 0xED, 0xA0, 0xBD, b'b']); + assert_ne!( + unsafe { (*lone).flags } & STRING_FLAG_HAS_LONE_SURROGATES, + 0 + ); + let reference = chain_call(&[heap("a"), high_box, heap("b")]); + same_string(lone, reference); + // Multi-byte literals carry their own UTF-16 length. + let wide = parts_call(&[literal("é€😀"), value(1.0), literal("😀")]); + assert_eq!(bytes(wide), "é€😀1😀".as_bytes()); + assert_eq!(unsafe { (*wide).utf16_len }, 1 + 1 + 2 + 1 + 2); +} + +#[test] +fn empty_literals_and_32_parts() { + let empty = parts_call(&[literal(""), literal("")]); + assert_eq!(bytes(empty), b""); + let mut slots = vec![literal("ab"); 32]; + slots[5] = value(12345.0); + slots[31] = value(heap("end")); + let mut expected = String::new(); + for i in 0..32 { + expected.push_str(match i { + 5 => "12345", + 31 => "end", + _ => "ab", + }); + } + assert_eq!(bytes(parts_call(&slots)), expected.as_bytes()); + // Counts that select each scratch size. + for count in [2, 4, 5, 8, 9, 31] { + let slots = vec![literal("z"); count]; + assert_eq!(bytes(parts_call(&slots)), vec![b'z'; count]); + } +} + +#[test] +fn very_long_results_cross_the_copy_thresholds() { + for len in [15usize, 16, 17, 31, 32, 33, 4096, 100_000] { + let text = "q".repeat(len); + let long = heap(&text); + let a = parts_call(&[literal("item-"), value(long), literal(";"), value(7.0)]); + assert_eq!(unsafe { (*a).byte_len } as usize, len + 5 + 1 + 1); + let out = bytes(a); + assert!(out.starts_with(b"item-q") && out.ends_with(b"q;7")); + } +} + +#[test] +fn a_refused_no_collect_allocation_takes_the_rooted_copy() { + // Heap operands must survive a collecting allocation: drive the cold arm + // directly with a payload pointer taken before the collection. + let scope = crate::gc::RuntimeHandleScope::new(); + let kept = scope.root_string_ptr(js_string_from_str("kept-across-collection")); + let boxed = || f64::from_bits(STRING_TAG | kept.get_raw_const_ptr::() as u64); + let slots = [literal("<"), value(boxed()), value(5.0), literal(">")]; + let mut piece_ptrs: [*const u8; 4] = [std::ptr::null(); 4]; + let mut piece_lens: [u32; 4] = [0; 4]; + let text = "kept-across-collection"; + piece_ptrs[0] = b"<".as_ptr(); + piece_lens[0] = 1; + piece_ptrs[1] = string_data(kept.get_raw_const_ptr::()); + piece_lens[1] = text.len() as u32; + let five = b"5"; + piece_ptrs[2] = five.as_ptr(); + piece_lens[2] = 1; + piece_ptrs[3] = b">".as_ptr(); + piece_lens[3] = 1; + let total = 1 + text.len() as u32 + 1 + 1; + let result = concat_parts_rooted_copy::<4>( + slots.as_ptr(), + 4, + &mut piece_ptrs, + &piece_lens, + &[0u32; 4], + total, + total, + STRING_FLAG_WTF8_VALIDATED, + ); + assert_eq!(bytes(result), format!("<{text}5>").as_bytes()); +} + +#[test] +fn literal_value_box_matches_the_fused_value_box() { + use super::super::concat::js_string_concat_value_box; + let prefixes: [(&'static str, f64); 4] = [ + ("k", heap("k")), + ("item-", heap("item-")), + ("user_id_", heap("user_id_")), + ("abcd", heap("abcd")), + ]; + let values = [ + 0.0, + 7.0, + 42.0, + 99_999.0, + 100_000.0, + 750_000.0, + 4_294_967_295.0, + 4_294_967_296.0, + -1.0, + -0.0, + 1.5, + f64::NAN, + f64::INFINITY, + 1e21, + ]; + for (text, prefix_box) in prefixes { + let prefix = (prefix_box.to_bits() & 0x0000_FFFF_FFFF_FFFF) as *const StringHeader; + for (v, eligible) in values + .iter() + .flat_map(|v| [(*v, 0), (*v, CONCAT_LIT_MEMO_ELIGIBLE)]) + { + let meta = text.len() as u64 | ((text.len() as u64) << 32) | eligible; + let new = js_string_concat_lit_value_box(text.as_ptr(), meta, v); + let old = js_string_concat_value_box(prefix, v); + assert_eq!( + JSValue::from_bits(new.to_bits()).is_short_string(), + JSValue::from_bits(old.to_bits()).is_short_string(), + "{text}+{v}: inline/heap representation" + ); + let text_of = |boxed: f64| -> Vec { + let bits = boxed.to_bits(); + if JSValue::from_bits(bits).is_short_string() { + let mut buf = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + let len = JSValue::from_bits(bits).short_string_to_buf(&mut buf); + buf[..len].to_vec() + } else { + bytes((bits & 0x0000_FFFF_FFFF_FFFF) as *const StringHeader) + } + }; + assert_eq!(text_of(new), text_of(old), "{text}+{v}"); + } + } + // A non-number takes the ToString arm. + let meta = 1u64 | (1u64 << 32); + let joined = js_string_concat_lit_value_box(b"k".as_ptr(), meta, heap("tail")); + let ptr = (joined.to_bits() & 0x0000_FFFF_FFFF_FFFF) as *const StringHeader; + assert_eq!(bytes(ptr), b"ktail"); +} + +#[test] +fn digit_pair_formatter_is_exact() { + let mut buf = [0u8; 32]; + for n in 0..=10_000_000u32 { + let len = fast_itoa_u32(n, &mut buf); + let text = n.to_string(); + assert_eq!(&buf[..len], text.as_bytes(), "{n}"); + } + // Digit-count boundaries and extremes, plus a deterministic spread. + let mut samples = vec![ + u32::MAX, + u32::MAX - 1, + 99, + 100, + 101, + 999, + 1000, + 9_999_999, + 10_000_000, + ]; + let mut x = 0x9E37_79B9u32; + for _ in 0..200_000 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + samples.push(x); + } + for p in 1..10u32 { + samples.push(10u32.pow(p) - 1); + samples.push(10u32.pow(p)); + } + for n in samples { + let len = fast_itoa_u32(n, &mut buf); + assert_eq!(&buf[..len], n.to_string().as_bytes(), "{n}"); + } +} diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index f4da22a0e7a..a3487493d33 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -113,6 +113,7 @@ mod compare_value_eq; #[cfg(test)] pub(crate) use compare::test_key_byte_reads; pub(crate) mod concat; +pub(crate) mod concat_parts; mod concat_boundary; pub(crate) mod concat_site; mod format; @@ -203,6 +204,7 @@ pub use concat::{ js_string_concat_chain, js_string_concat_value, js_value_add_string, js_value_concat_string, scan_concat_memo_roots, scan_concat_memo_roots_mut, }; +pub use concat_parts::{js_string_concat_lit_value_box, js_string_concat_parts}; pub use concat_site::{js_string_concat_site_value, CONCAT_SITE_SLOTS}; pub(crate) use format::ascii_char_string; pub(crate) use format::fix_exponent_format; @@ -558,13 +560,16 @@ pub struct StringHeader { /// cannot share a constant. This assertion is the link: reordering, resizing, /// or padding this struct fails the runtime BUILD here, at the definition, /// rather than silently miscompiling every `.length` and `charCodeAt` in -/// every user program. The literals are duplicated in +/// every user program. (12) is read by the inline pointer-push +/// arm in . The literals are duplicated in /// `lower_string_method.rs`'s `STRING_HEADER_*` constants, which name this /// item. const STRING_HEADER_ABI_MATCHES_CODEGEN: () = { assert!(std::mem::size_of::() == 20); assert!(std::mem::offset_of!(StringHeader, utf16_len) == 0); assert!(std::mem::offset_of!(StringHeader, byte_len) == 4); + assert!(std::mem::offset_of!(StringHeader, refcount) == 12); + assert!(std::mem::offset_of!(StringHeader, refcount) == 12); assert!(std::mem::offset_of!(StringHeader, flags) == 16); }; const _: () = STRING_HEADER_ABI_MATCHES_CODEGEN; diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index 48360a248e2..b8b22d2698f 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -1912,6 +1912,11 @@ def is_collecting(callee): # `js_string_concat_value_box` on its miss path, so it can allocate and # re-enter a moving minor exactly like its callee (the #7616 shape). "js_string_concat_site_value", + # JST slice 3: the literal-operand concat entries. `js_string_concat_parts` + # takes the collecting allocation (and the general arm's ToString) like + # `js_string_concat_chain`; `js_string_concat_lit_value_box` is the + # literal-prefix twin of `js_string_concat_value_box`. + "js_string_concat_parts", "js_string_concat_lit_value_box", # `js_private_brand_add` is the referent-with-no-name that NEITHER audit can # ask for: `--audit-poll-reach` only walks symbols `ALLOC_RE` matches, and # `js_private_brand_add` matches no alloc/new/create convention, so the one diff --git a/test-files/test_gap_jst5_concat_parts.ts b/test-files/test_gap_jst5_concat_parts.ts new file mode 100644 index 00000000000..d4a57730c24 --- /dev/null +++ b/test-files/test_gap_jst5_concat_parts.ts @@ -0,0 +1,143 @@ +// JST slice 3: literal-aware concat (`js_string_concat_parts`), the inline +// ToPrimitive/Symbol tests on the operands of a `+` chain, and the literal +// prefix + number form. Every line is compared byte-for-byte with Node. + +function attempt(f: () => string): string { + try { + return f(); + } catch (e) { + return "throw:" + (e instanceof TypeError ? "TypeError" : String(e)); + } +} + +function cps(s: string): string { + const out: string[] = []; + for (let i = 0; i < s.length; i++) out.push(s.charCodeAt(i).toString(16)); + return out.join(","); +} + +// ---- numbers: -0, NaN, infinities, large and negative integers, fractions ---- +const numbers: number[] = [ + 0, -0, 1, -1, 9, 10, 99, 100, 12345, 999999999, 1000000000, 2147483647, + -2147483648, 4294967295, 4294967296, -4294967295, 1e15, 1e21, 1.5, -0.25, + 1e-7, NaN, Infinity, -Infinity, 123456789012, 0.1 + 0.2, +]; +for (const n of numbers) { + console.log("item-" + n + ";", "k" + n, "<" + n + ">" + "!", `${n}:` + n); +} +let acc = ""; +for (let i = 0; i < 50; i++) acc += "r" + (i * 7919 % 1000003) + "-" + (i - 25) + ";"; +console.log(acc.length, acc.slice(0, 60), acc.slice(-30)); + +// ---- Symbol operands throw TypeError, after earlier operands convert ---- +const sym = Symbol("s"); +const log: string[] = []; +const o = { + valueOf() { + log.push("o.valueOf"); + return 1; + }, + toString() { + log.push("o.toString"); + return "O"; + }, +}; +console.log(attempt(() => "a" + (sym as any) + "b")); +console.log(attempt(() => (sym as any) + "a" + "b")); +console.log(attempt(() => "a" + "b" + (sym as any))); +console.log(attempt(() => "a" + (o as any) + (sym as any)), log.join("|")); +log.length = 0; +console.log(attempt(() => "a" + (sym as any) + (o as any)), log.join("|")); +log.length = 0; +// A lying number annotation that holds a Symbol or an object. +function lie(x: number): string { + return "v=" + x + ";"; +} +console.log(attempt(() => lie(sym as any))); +console.log(attempt(() => lie(o as any)), log.join("|")); +log.length = 0; + +// ---- ToPrimitive: valueOf/toString/Symbol.toPrimitive, order across 3+ parts ---- +const p = { + [Symbol.toPrimitive](hint: string) { + log.push("p:" + hint); + return "P"; + }, +}; +const q = { + valueOf() { + log.push("q.valueOf"); + return {}; + }, + toString() { + log.push("q.toString"); + return "Q"; + }, +}; +const r = { + valueOf() { + log.push("r.valueOf"); + return 2.5; + }, +}; +console.log("x" + (o as any) + "y" + (p as any) + "z" + (q as any) + "w" + (r as any) + "!"); +console.log(log.join("|")); +log.length = 0; +function f(tag: string, v: any): any { + log.push("eval:" + tag); + return v; +} +console.log("a" + f("1", o) + f("2", p) + "b" + f("3", r) + f("4", 7)); +console.log(log.join("|")); +log.length = 0; +console.log(attempt(() => "a" + f("1", o) + f("2", sym) + f("3", p)), log.join("|")); +log.length = 0; +// booleans, null, undefined, bigint, arrays and plain objects as operands +console.log("t=" + (true as any) + (null as any) + (undefined as any) + (10n as any) + ([1, [2, 3]] as any) + ({} as any)); + +// ---- surrogates: pairs minted across operand boundaries, lone halves kept ---- +const hi = String.fromCharCode(0xd83d); +const lo = String.fromCharCode(0xde00); +const pair = "<" + hi + lo + ">"; +console.log(pair.length, cps(pair), [...pair].length, pair.codePointAt(1)!.toString(16)); +const loneHi = "a" + hi + "b"; +const loneLo = "a" + lo + "b"; +console.log(loneHi.length, cps(loneHi), loneLo.length, cps(loneLo)); +const two = "" + hi + lo + hi + lo; +console.log(two.length, cps(two), [...two].length); +const mixed = "é€" + hi + "😀" + lo + 5 + "ü"; +console.log(mixed.length, cps(mixed)); +console.log(("x" + hi + lo + "y") === "x😀y", ("x" + hi + "y" + lo) === "x😀y"); + +// ---- boundaries: SSO sizes, empty operands, many parts ---- +for (let i = 0; i < 12; i++) { + const n = i * 11; + console.log("k" + n, "ab" + n, "abc" + n, "abcd" + n, "abcde" + n, "" + n + ""); +} +const e1 = ""; +const e2 = ""; +console.log(("a" + e1 + e2 + "b").length, ("" + e1 + e2).length); +// Empty operands (heap and inline) next to numbers and literals. +function blank(): string { + return process.argv.length > 999 ? "z" : ""; +} +const emptyA = blank(); +const emptyB = blank() + blank(); +console.log("x" + emptyA + 42 + "y", "x" + 42 + emptyB + "y", emptyA + 7 + emptyB + ";", 5 + emptyA + "!" + emptyB); +const s32 = "0" + 1 + "2" + 3 + "4" + 5 + "6" + 7 + "8" + 9 + "a" + 10 + "b" + 11 + "c" + 12 + "d" + 13 + "e" + 14 + "f" + 15 + "g" + 16 + "h" + 17 + "i" + 18 + "j" + 19 + "k" + 20; +console.log(s32, s32.length); + +// ---- very long results ---- +for (const len of [15, 16, 17, 31, 32, 33, 1000, 100000, 3000000]) { + const body = "x".repeat(len); + const s = "item-" + body + ";" + len; + console.log(len, s.length, s.charCodeAt(5), s.charCodeAt(s.length - 1), s.slice(-8)); +} + +// ---- results used as keys and compared by content ---- +const keys = new Map(); +for (let i = 0; i < 2000; i++) keys.set("k" + (i % 750), i); +console.log(keys.size, keys.get("k0"), keys.get("k749"), keys.has("k750")); +const seen = new Set(); +for (let i = 0; i < 3000; i++) seen.add("id-" + (i % 97) + "-" + (i % 13)); +console.log(seen.size); diff --git a/test-files/test_gap_jst5_push_gate.ts b/test-files/test_gap_jst5_push_gate.ts new file mode 100644 index 00000000000..bf0ffdda1de --- /dev/null +++ b/test-files/test_gap_jst5_push_gate.ts @@ -0,0 +1,179 @@ +// JST slice 3: the pointer-push gate. A string pushed onto an all-pointers +// array at its append position skips the layout/numeric notes and the shared- +// string demote only when it is already shared. Everything else keeps the full +// sequence. Compared byte-for-byte with Node. + +function attempt(f: () => any): string { + try { + return String(f()); + } catch (e) { + return "throw:" + (e instanceof TypeError ? "TypeError" : String(e)); + } +} + +// Fresh concat results (shared) appended in a loop. +const built: string[] = []; +for (let i = 0; i < 2000; i++) built.push("item-" + i + ";"); +console.log(built.length, built[0], built[1999], built.join("").length); + +// A uniquely-owned accumulator pushed and then appended to: the push must +// demote it, or the in-place append rewrites the stored element. +const grown: string[] = []; +let acc = ""; +for (let i = 0; i < 300; i++) { + acc += "q"; + grown.push(acc); +} +let ok = true; +for (let i = 0; i < grown.length; i++) if (grown[i].length !== i + 1) ok = false; +console.log(ok, grown[0], grown[299].length); +let s2 = "ab"; +s2 += "cd"; +const holder: string[] = []; +holder.push("seed" + 1 + "x"); +holder.push(s2); +s2 += "ef"; +holder.push(s2); +console.log(holder.join("|")); + +// Mixed element kinds on an array that started all-pointers. +const mixed: any[] = []; +mixed.push("a" + 1 + "b"); +mixed.push("c" + 2 + "d"); +mixed.push(5); +mixed.push("k" + 7); +mixed.push(undefined); +mixed.push({ x: 1 }); +mixed.push("e" + 3 + "f"); +mixed.push(null); +mixed.push(-0); +mixed.push(""); +console.log(mixed.length, JSON.stringify(mixed), Object.is(mixed[8], -0)); + +// A number[] demoted by a string push. +const nums: any[] = [1, 2, 3]; +nums.push("s" + 4 + "t"); +nums.push(5); +console.log(nums.join(","), nums.length, typeof nums[3], typeof nums[4]); + +// Holey arrays, non-append positions, shrunk and grown lengths. +const holey: string[] = []; +holey.push("a" + 0 + "!"); +holey[4] = "far"; +holey.push("b" + 1 + "!"); +console.log(holey.length, holey.join("|"), 2 in holey, 5 in holey); +const shrunk: string[] = []; +for (let i = 0; i < 10; i++) shrunk.push("v" + i + ";"); +shrunk.length = 4; +shrunk.push("w" + 9 + ";"); +console.log(shrunk.join(""), shrunk.length); +const grownLen: string[] = []; +grownLen.push("a" + 1 + "b"); +grownLen.length = 3; +grownLen.push("c" + 2 + "d"); +console.log(grownLen.length, grownLen.join("|"), 1 in grownLen); +const popped: string[] = []; +for (let i = 0; i < 6; i++) popped.push("p" + i + ";"); +popped.pop(); +popped.shift(); +popped.push("n" + 1 + ";"); +popped.unshift("u" + 2 + ";"); +console.log(popped.join("")); +const sorted: string[] = []; +for (let i = 0; i < 8; i++) sorted.push("z" + ((i * 5) % 8) + ";"); +sorted.sort(); +sorted.push("last" + 1 + ";"); +sorted.splice(2, 1); +sorted.push("after" + 2 + ";"); +console.log(sorted.join("")); + +// Frozen, sealed and non-extensible arrays. +const frozen: string[] = ["a" + 1 + "b"]; +Object.freeze(frozen); +console.log(attempt(() => frozen.push("c" + 2 + "d")), frozen.length); +const sealed: string[] = ["a" + 1 + "b"]; +Object.seal(sealed); +console.log(attempt(() => sealed.push("c" + 2 + "d")), sealed.length); +const closed: string[] = ["a" + 1 + "b"]; +Object.preventExtensions(closed); +console.log(attempt(() => closed.push("c" + 2 + "d")), closed.length); + +// Own properties and non-writable length. +const withProp: string[] = []; +(withProp as any).tag = "t"; +for (let i = 0; i < 5; i++) withProp.push("x" + i + ";"); +console.log(withProp.length, withProp.join(""), (withProp as any).tag); +const fixedLen: string[] = ["a" + 1 + "b"]; +Object.defineProperty(fixedLen, "length", { writable: false }); +console.log(attempt(() => fixedLen.push("c" + 2 + "d")), fixedLen.length); + +// Inherited setters on the prototype intercept the append. +const writes: string[] = []; +const proto: string[] = []; +for (let i = 0; i < 3; i++) proto.push("m" + i + ";"); +Object.defineProperty(Array.prototype, "3", { + configurable: true, + set(v: any) { + writes.push("setter:" + String(v)); + }, + get() { + return "got"; + }, +}); +console.log(attempt(() => proto.push("n" + 3 + ";")), proto.length, writes.join("|"), Object.keys(proto).join(",")); +delete (Array.prototype as any)[3]; +const afterDelete: string[] = []; +for (let i = 0; i < 5; i++) afterDelete.push("d" + i + ";"); +console.log(afterDelete.join(""), writes.length); + +// A subclass and a proxy receiver. +class Bag extends Array {} +const bag = new Bag(); +for (let i = 0; i < 4; i++) bag.push("g" + i + ";"); +console.log(bag.length, bag.join(""), bag instanceof Bag); +const proxied = new Proxy([] as string[], {}); +for (let i = 0; i < 3; i++) proxied.push("y" + i + ";"); +console.log(proxied.length, proxied.join("")); + +// Aliased growth: a pushed array element stays intact when the source grows. +let live = "head"; +const refs: string[] = []; +for (let i = 0; i < 40; i++) { + live += "+" + i; + refs.push(live); +} +console.log(refs[0], refs[1], refs[39].length, refs[20].slice(-6)); + +// GC pressure: a pointer pushed through the gate must stay traced. Garbage is +// allocated between pushes so that collections run while the strings are held +// only by the arrays (a demoted or never-declared layout that the gate skipped +// would let the collector free or move them unseen). +function churn(n: number): number { + let total = 0; + for (let i = 0; i < n; i++) { + const s = "garbage-" + i + "-" + ((i * 3) % 1000); + total += s.length; + } + return total; +} +const kept: string[] = []; +for (let i = 0; i < 40; i++) { + kept.push("keep-" + i + ";"); + churn(30000); +} +console.log(kept.length, kept.join("")); +const demoted: any[] = [1, 2, 3]; +demoted.push("s" + 4 + "t"); +churn(300000); +console.log(demoted.join(","), demoted.length, typeof demoted[3]); +const grownKept: string[] = []; +let accKept = ""; +for (let i = 0; i < 40; i++) { + accKept += "z"; + grownKept.push(accKept); + churn(20000); + accKept += "y"; +} +let intact = true; +for (let i = 0; i < grownKept.length; i++) if (grownKept[i].length !== 2 * i + 1) intact = false; +console.log(intact, grownKept[0], grownKept[39].length); From 0ce785c6dfe1b1b81af9375259eb46059a838185 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Thu, 29 Oct 2026 03:10:54 +0000 Subject: [PATCH 2/2] changelog: fragment for #13450 --- changelog.d/13450-jst5-string-concat.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/13450-jst5-string-concat.md diff --git a/changelog.d/13450-jst5-string-concat.md b/changelog.d/13450-jst5-string-concat.md new file mode 100644 index 00000000000..fd436491f0e --- /dev/null +++ b/changelog.d/13450-jst5-string-concat.md @@ -0,0 +1 @@ +String concatenation is cheaper: literal-aware concat folds image literals without re-reading headers, operand checks are inlined at the call site, and array push of freshly built strings uses a pointer-only gate. Allocation and GC schedules are unchanged.