diff --git a/changelog.d/13459-jst6-string-slice4.md b/changelog.d/13459-jst6-string-slice4.md new file mode 100644 index 00000000000..b3c471381ab --- /dev/null +++ b/changelog.d/13459-jst6-string-slice4.md @@ -0,0 +1 @@ +String building is cheaper again: split births each piece without a per-piece root and reload when the allocation provably cannot collect, join walks its pieces with a heap-string-first dispatch and inline small copies, string-array length loops are admitted as one call-free region, and concatenations of two to five operands call count-specialized entries. Allocation order, GC schedule and memory use are unchanged. diff --git a/crates/perry-codegen/src/codegen/declared_string_add_tests.rs b/crates/perry-codegen/src/codegen/declared_string_add_tests.rs index 798617a333c..bbde8f999a4 100644 --- a/crates/perry-codegen/src/codegen/declared_string_add_tests.rs +++ b/crates/perry-codegen/src/codegen/declared_string_add_tests.rs @@ -295,7 +295,7 @@ fn a_string_method_name_on_an_unproven_receiver_is_guarded() { /// entry that carries literals as constant data (JST slice 3). fn has_nway_fold(ir: &str) -> bool { ir.contains("call i64 @js_string_concat_chain(") - || ir.contains("call i64 @js_string_concat_parts(") + || ir.contains("call i64 @js_string_concat_parts") } #[test] @@ -313,7 +313,7 @@ fn a_chain_whose_head_pair_is_all_declared_does_not_fold() { ); assert!( !ir.contains("call i64 @js_string_concat_chain(") - && !ir.contains("call i64 @js_string_concat_parts("), + && !ir.contains("call i64 @js_string_concat_parts"), "the head pair carries no proof, so the N-way fold is unsound \ here:\n{ir}" ); @@ -385,7 +385,7 @@ fn a_self_append_chain_fuses_the_accumulator_and_suffix() { ); assert_eq!( ir.matches("call i64 @js_string_concat_chain(").count() - + ir.matches("call i64 @js_string_concat_parts(").count(), + + ir.matches("call i64 @js_string_concat_parts").count(), 0, "the suffix must not be allocated before it is appended:\n{ir}" ); @@ -814,7 +814,7 @@ fn literal_parts_get_no_symbol_check_and_the_value_part_is_tested_inline() { // check each have exactly one site, for the one value part, and each sits // in a block reached only through the inline test of the live bits. let ir = literal_chain(Expr::LocalGet(1), vec![param(1, "n", Type::Number)]); - assert!(ir.contains("call i64 @js_string_concat_parts("), "{ir}"); + assert!(ir.contains("call i64 @js_string_concat_parts"), "{ir}"); assert_eq!( ir.matches("call void @js_add_throw_if_symbol(").count(), 1, @@ -879,7 +879,7 @@ fn an_operand_that_cannot_be_a_symbol_carries_no_check_at_all() { Expr::String("z".to_string()), ), ); - assert!(ir.contains("call i64 @js_string_concat_parts("), "{ir}"); + assert!(ir.contains("call i64 @js_string_concat_parts"), "{ir}"); assert_eq!( ir.matches("call void @js_add_throw_if_symbol(").count(), 1, diff --git a/crates/perry-codegen/src/expr/array_proto_guard.rs b/crates/perry-codegen/src/expr/array_proto_guard.rs index f9a06ab3d0c..5d9f51df7ea 100644 --- a/crates/perry-codegen/src/expr/array_proto_guard.rs +++ b/crates/perry-codegen/src/expr/array_proto_guard.rs @@ -97,3 +97,20 @@ pub(crate) fn emit_array_element_descriptors(blk: &mut LlBlock, arr_addr: &str) ); blk.icmp_ne(crate::types::I32, &kind, "0") } + +/// Emit the i1 "an ordinary, unforwarded `GC_TYPE_ARRAY` with no element +/// descriptors" test for the live heap handle `handle` (an i64 address the +/// caller has already placed in the heap-pointer band): the receiver guard of +/// every inline element read. A call-free region reads the dense slots of an +/// array admitted by it. +pub(crate) fn emit_ordinary_array_admission(blk: &mut LlBlock, handle: &str) -> String { + let word = super::index_get::guarded_array::emit_array_guard_word(blk, handle); + super::index_get::guarded_array::emit_array_guard_word_ok(blk, &word, handle) +} + +/// `bits - POINTER_TAG - 2 MiB`, compared unsigned against this span, holds +/// exactly for a heap-pointer-tagged value above the runtime-id band: the +/// precondition of reading the header behind it. +pub(crate) const HEAP_POINTER_BAND_BASE_I64: &str = super::receiver_range::RECEIVER_BIAS_LITERAL; +/// The span of the band that [`HEAP_POINTER_BAND_BASE_I64`] biases into. +pub(crate) const HEAP_POINTER_BAND_SPAN_I64: &str = super::receiver_range::RECEIVER_SPAN_LITERAL; diff --git a/crates/perry-codegen/src/expr/index_get/guarded_array.rs b/crates/perry-codegen/src/expr/index_get/guarded_array.rs index 46ad712d62b..577f4a2efbd 100644 --- a/crates/perry-codegen/src/expr/index_get/guarded_array.rs +++ b/crates/perry-codegen/src/expr/index_get/guarded_array.rs @@ -221,7 +221,11 @@ pub(in crate::expr) fn emit_array_guard_word( /// `(word & MASK) == EXPECT` and a null named layout: an ordinary array, not /// forwarded, no element descriptors. -fn emit_array_guard_word_ok(blk: &mut crate::block::LlBlock, word: &str, handle: &str) -> String { +pub(in crate::expr) fn emit_array_guard_word_ok( + blk: &mut crate::block::LlBlock, + word: &str, + handle: &str, +) -> String { let masked = blk.and(I32, word, ARRAY_READ_GUARD_MASK_I32); let word_ok = blk.icmp_eq(I32, &masked, ARRAY_READ_GUARD_EXPECT_I32); emit_and_no_named_layout(blk, &word_ok, handle) diff --git a/crates/perry-codegen/src/lower_string_concat.rs b/crates/perry-codegen/src/lower_string_concat.rs index 4db2eeb5c96..f5a6b2b128f 100644 --- a/crates/perry-codegen/src/lower_string_concat.rs +++ b/crates/perry-codegen/src/lower_string_concat.rs @@ -1178,11 +1178,21 @@ pub(crate) fn emit_string_concat_parts(ctx: &mut FnCtx<'_>, parts: &[ConcatPartA } let base_i64 = blk.next_reg(); blk.emit_raw(format!("{} = ptrtoint ptr {} to i64", base_i64, buf_reg)); - let result_handle = blk.call( - I64, - "js_string_concat_parts", - &[(I64, &base_i64), (I32, &n.to_string())], - ); + // A count the runtime has a specialized entry for takes it (the count is + // then a constant of the callee); any other count takes the generic entry. + let result_handle = if (2..=5).contains(&n) { + blk.call( + I64, + &format!("js_string_concat_parts_{n}"), + &[(I64, &base_i64)], + ) + } else { + blk.call( + I64, + "js_string_concat_parts", + &[(I64, &base_i64), (I32, &n.to_string())], + ) + }; nanbox_string_inline(blk, &result_handle) } diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index b5a5b2e3734..4801244690d 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -63,6 +63,10 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { module.declare_function("js_string_concat_chain", I64, &[I64, I32]); // Literal operands as constant data (`string/concat_parts.rs`). module.declare_function("js_string_concat_parts", I64, &[I64, I32]); + // The same operation for a part count the compiler knows (2..=5). + for n in 2..=5 { + module.declare_function(&format!("js_string_concat_parts_{n}"), I64, &[I64]); + } module.declare_function("js_string_concat_lit_value_box", DOUBLE, &[I64, I64, DOUBLE]); module.declare_function("js_to_primitive_default_for_add", DOUBLE, &[DOUBLE]); module.declare_function("js_add_throw_if_symbol", VOID, &[DOUBLE]); diff --git a/crates/perry-codegen/src/stmt/indexed_owner_loop.rs b/crates/perry-codegen/src/stmt/indexed_owner_loop.rs index 21a4331ee65..41b7e61aece 100644 --- a/crates/perry-codegen/src/stmt/indexed_owner_loop.rs +++ b/crates/perry-codegen/src/stmt/indexed_owner_loop.rs @@ -15,12 +15,16 @@ use std::collections::{BTreeMap, BTreeSet}; #[path = "indexed_owner_string.rs"] mod string; const STRING_KIND: u8 = 12; +#[path = "indexed_owner_sarray.rs"] +mod sarray; +const SARRAY_KIND: u8 = 13; #[derive(Default)] struct Plan { counter: u32, arrays: BTreeMap, strings: BTreeSet, + sarrays: BTreeSet, aliases: BTreeMap, defs: BTreeMap, inputs: BTreeSet, @@ -63,7 +67,7 @@ impl Plan { .and_then(crate::expr::typed_element::type_kind) }) .or_else(|| crate::expr::ta_element_read::receiver_kind(ctx, &Expr::LocalGet(root)))?; - if matches!(kind, 9 | 10) || self.strings.contains(&root) { + if matches!(kind, 9 | 10) || self.strings.contains(&root) || self.sarrays.contains(&root) { return None; } self.arrays.insert(root, kind); @@ -85,6 +89,7 @@ impl Plan { if self.body_bindings.contains(&id) || !plain_binding(ctx, id) || self.arrays.contains_key(&id) + || self.sarrays.contains(&id) { return None; } @@ -92,6 +97,25 @@ impl Plan { Some((id, index)) } + /// `array[i].length` of a declared `string[]` binding. + fn sarray_read<'a>(&mut self, ctx: &FnCtx<'_>, e: &'a Expr) -> Option<(u32, &'a Expr)> { + let (id, index) = sarray::read(e)?; + if !self.sarray_binding(ctx, id) { + return None; + } + self.sarrays.insert(id); + Some((id, index)) + } + + fn sarray_binding(&self, ctx: &FnCtx<'_>, id: u32) -> bool { + !self.body_bindings.contains(&id) + && !self.aliases.contains_key(&id) + && plain_binding(ctx, id) + && !self.arrays.contains_key(&id) + && !self.strings.contains(&id) + && sarray::declared(ctx, id) + } + fn condition(&mut self, ctx: &FnCtx<'_>, e: &Expr) -> bool { match e { Expr::Bool(_) => true, @@ -110,6 +134,13 @@ impl Plan { self.indices.push((root, index.clone())); return true; } + if let Some((root, index)) = self.sarray_read(ctx, e) { + if !self.numeric(ctx, index) { + return false; + } + self.indices.push((root, index.clone())); + return true; + } if let Some((root, index)) = self.read(ctx, e) { if !self.numeric(ctx, index) { return false; @@ -891,6 +922,11 @@ fn numeric( values: &mut BTreeMap, storage: &BTreeMap, ) -> Number { + if let Some((id, index)) = sarray::read(e) { + let s = &storage[&id]; + let index = index_value(ctx, p, index, values, storage); + return Number::U(sarray::element_length(ctx, s, &index)); + } if let Some((id, index)) = string::read(e) { let s = &storage[&id]; let index = index_value(ctx, p, index, values, storage); @@ -1403,7 +1439,10 @@ pub(super) fn lower( object, property, .. } if property == "length" => { if let Expr::LocalGet(id) = object.as_ref() { - if p.strings.contains(id) { + if p.strings.contains(id) || p.sarrays.contains(id) { + Some(*id) + } else if p.sarray_binding(ctx, *id) { + p.sarrays.insert(*id); Some(*id) } else { p.receiver(ctx, object) @@ -1443,11 +1482,12 @@ pub(super) fn lower( // captured cells. The raw binding and any i32 shadow are flushed on exit. if p.strings .iter() + .chain(p.sarrays.iter()) .any(|id| p.inputs.contains(id) || p.writes.contains(id)) { return Ok(false); } - if !p.strings.is_empty() { + if !p.strings.is_empty() || !p.sarrays.is_empty() { // The original condition/body can drain work deferred by a preceding // allocation even though this admitted region cannot allocate. Keep // that entry safepoint before loading any owner or raw string bytes; @@ -1486,6 +1526,9 @@ pub(super) fn lower( for id in &p.strings { storage.insert(*id, string::admit(ctx, *id, &slow_l)); } + for id in &p.sarrays { + storage.insert(*id, sarray::admit(ctx, *id, &slow_l)); + } // Read-only regions require no alias separation. For writes, allocation // identities can prove separation before falling back to live spans. if !p.stores.is_empty() { @@ -1493,6 +1536,7 @@ pub(super) fn lower( } if let Some(id) = length_receiver.filter(|id| { storage[id].kind != STRING_KIND + && storage[id].kind != SARRAY_KIND && !(ctx.wpfacts.indexed_metadata_pristine && owner_proof(ctx, *id, storage[id].kind).is_some()) }) { @@ -1703,7 +1747,10 @@ pub(super) fn lower( false, false, vec![ - format!("owners={:?};ascii_strings={:?}", p.arrays, p.strings), + format!( + "owners={:?};ascii_strings={:?};string_arrays={:?}", + p.arrays, p.strings, p.sarrays + ), format!("index_windows={}", indices.len()), format!( "dynamic_entry_guards={dynamic_entry_guards};nested_loops={}", diff --git a/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs b/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs index fd86d8cabae..fa697d315d7 100644 --- a/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs +++ b/crates/perry-codegen/src/stmt/indexed_owner_loop_tests.rs @@ -31,6 +31,14 @@ fn ir(kind: &str, body: Vec) -> String { ir_with_facts(kind, body, Default::default()) } fn ir_with_facts(kind: &str, body: Vec, facts: crate::codegen::ElisionFacts) -> String { + ir_with_param_type(Type::Named(kind.into()), body, facts) +} +fn ir_with_param_type( + receiver: Type, + body: Vec, + facts: crate::codegen::ElisionFacts, +) -> String { + let kind = receiver; let mut m = Module::new("indexed_owner_contract"); let param = |id, name: &str, ty| Param { id, @@ -46,9 +54,9 @@ fn ir_with_facts(kind: &str, body: Vec, facts: crate::codegen::ElisionFact name: "operate".into(), type_params: vec![], params: vec![ - param(1, "a", Type::Named(kind.into())), + param(1, "a", kind.clone()), param(2, "scale", Type::Any), - param(7, "b", Type::Named(kind.into())), + param(7, "b", kind.clone()), ], return_type: Type::Any, body: vec![ @@ -475,3 +483,82 @@ fn string_region_rejects_rebinding_and_calls_before_any_body_effect() { assert!(!ir("string", body).contains("indexed.string.storage")); } } + +fn string_array_length(index: Expr) -> Expr { + Expr::PropertyGet { + object: Box::new(index_get_of(1, index)), + property: "length".into(), + byte_offset: 0, + byte_end: 0, + } +} +fn index_get_of(a: u32, i: Expr) -> Expr { + index(a, i) +} +fn string_array() -> Type { + Type::Array(Box::new(Type::String)) +} + +// JST slice 4 (F): `lenSum += pieces[i].length` over a declared `string[]`. +#[test] +fn string_array_length_region_scans_the_slots_once_and_stays_call_free() { + let body = vec![Stmt::Expr(Expr::LocalSet( + 2, + Box::new(bin(BinaryOp::Add, get(2), string_array_length(get(3)))), + ))]; + let ir = ir_with_param_type(string_array(), body, Default::default()); + assert!(ir.contains("indexed.sarray.scan"), "{ir}"); + assert!(ir.contains("indexed.owner.slow"), "{ir}"); + let admission = ir.find("indexed.sarray.scan").unwrap(); + let first_poll = ir + .find("call void @js_gc_loop_safepoint()") + .expect("pending GC work must be serviced before the array is read"); + assert!( + first_poll < admission, + "the entry poll must precede the slot scan: {ir}" + ); + let fast = ir + .split("\nindexed.owner.fast.") + .nth(1) + .unwrap() + .split("\nindexed.owner.exit.") + .next() + .unwrap(); + assert!( + !fast.contains("call "), + "the string-array region stays call-free: {fast}" + ); + // The per-element read is one slot load and one header load. + assert!(fast.contains("lshr i64"), "slot tag test: {fast}"); +} + +#[test] +fn string_array_length_region_rejects_stores_and_calls_before_any_body_effect() { + for body in [ + // The array is rebound in the body. + vec![ + Stmt::Expr(Expr::LocalSet(1, Box::new(Expr::Integer(4)))), + Stmt::Expr(string_array_length(get(3))), + ], + // A call can run user code between two element reads. + vec![ + Stmt::Expr(string_array_length(get(3))), + Stmt::Expr(Expr::Call { + byte_end: 0, + callee: Box::new(get(2)), + args: vec![], + type_args: vec![], + byte_offset: 0, + }), + ], + // An element store. + vec![Stmt::Expr(Expr::IndexSet { + object: Box::new(get(1)), + index: Box::new(get(3)), + value: Box::new(Expr::String("x".into())), + })], + ] { + let ir = ir_with_param_type(string_array(), body, Default::default()); + assert!(!ir.contains("indexed.sarray.scan"), "{ir}"); + } +} diff --git a/crates/perry-codegen/src/stmt/indexed_owner_sarray.rs b/crates/perry-codegen/src/stmt/indexed_owner_sarray.rs new file mode 100644 index 00000000000..d616d9dde3c --- /dev/null +++ b/crates/perry-codegen/src/stmt/indexed_owner_sarray.rs @@ -0,0 +1,158 @@ +//! Region-local string-array admission (JST slice 4, design F): the read +//! `array[i].length` of a declared `string[]`. +//! +//! Admission proves the receiver is an ordinary unforwarded array without +//! element descriptors, then walks its `[0, length)` slots ONCE in the +//! preheader and misses to the original loop unless every slot holds a heap +//! string or an inline ASCII string. Inside the region (no call, no +//! allocation, no poll) nothing can store into the array or move a string, so +//! the proof holds for every iteration and the per-element read is a tag test +//! and one header load. A hole, `undefined`, an object or a non-ASCII inline +//! string anywhere in the array declines the whole region before any body +//! effect. +use super::*; + +pub(super) fn read(e: &Expr) -> Option<(u32, &Expr)> { + let Expr::PropertyGet { + object, property, .. + } = e + else { + return None; + }; + if property != "length" { + return None; + } + let Expr::IndexGet { object, index, .. } = object.as_ref() else { + return None; + }; + let Expr::LocalGet(id) = object.as_ref() else { + return None; + }; + Some((*id, index.as_ref())) +} + +/// `true` for a declared `string[]` binding the region may read. +pub(super) fn declared(ctx: &FnCtx<'_>, id: u32) -> bool { + super::super::string_length_loop::is_declared_string_array(ctx, id) +} + +pub(super) fn admit(ctx: &mut FnCtx<'_>, id: u32, slow: &str) -> Storage { + let boxed = boxed_binding(ctx, id); + let bits = ctx.block().bitcast_double_to_i64(&boxed); + // A heap-pointer-tagged value above the runtime-id band: the header + // behind it is readable. + let offset = ctx.block().sub( + I64, + &bits, + crate::expr::array_proto_guard::HEAP_POINTER_BAND_BASE_I64, + ); + let heap = ctx.block().icmp_ult( + I64, + &offset, + crate::expr::array_proto_guard::HEAP_POINTER_BAND_SPAN_I64, + ); + branch_guard(ctx, &heap, slow); + let handle = ctx.block().and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + let ordinary = crate::expr::array_proto_guard::emit_ordinary_array_admission( + ctx.block(), + &handle, + ); + branch_guard(ctx, &ordinary, slow); + let ptr = ctx.block().inttoptr(I64, &handle); + let len = ctx.block().load(I32, &ptr); + let cap_ptr = ctx.block().gep_inbounds(I8, &ptr, &[(I64, "4")]); + let cap = ctx.block().load(I32, &cap_ptr); + let within = ctx.block().icmp_ule(I32, &len, &cap); + branch_guard(ctx, &within, slow); + let data = ctx.block().array_elements_addr_with_capacity(&handle, &cap); + scan(ctx, &data, &len, slow); + Storage { + raw: handle, + data, + len, + kind: SARRAY_KIND, + metadata: String::new(), + } +} + +/// Miss unless every slot below `len` holds a heap string or an inline ASCII +/// string. One OR-accumulated verdict, one branch after the walk. +fn scan(ctx: &mut FnCtx<'_>, data: &str, len: &str, slow: &str) { + let index_slot = ctx.func.alloca_entry(I32); + let bad_slot = ctx.func.alloca_entry(I32); + ctx.block().store(I32, "0", &index_slot); + ctx.block().store(I32, "0", &bad_slot); + let head = ctx.new_block("indexed.sarray.scan"); + let body = ctx.new_block("indexed.sarray.slot"); + let done = ctx.new_block("indexed.sarray.scanned"); + let (head_l, body_l, done_l) = ( + ctx.block_label(head), + ctx.block_label(body), + ctx.block_label(done), + ); + ctx.block().br(&head_l); + + ctx.current_block = head; + let index = ctx.block().load(I32, &index_slot); + let more = ctx.block().icmp_ult(I32, &index, len); + ctx.block().cond_br(&more, &body_l, &done_l); + + ctx.current_block = body; + let wide = ctx.block().zext(I32, &index, I64); + let off = ctx.block().shl(I64, &wide, "3"); + let base = ctx.block().inttoptr(I64, data); + let slot = ctx.block().gep(I8, &base, &[(I64, &off)]); + let bits = ctx.block().load(I64, &slot); + let tag = ctx.block().lshr(I64, &bits, "48"); + let is_heap = ctx + .block() + .icmp_eq(I64, &tag, crate::nanbox::STRING_TAG_TOP16_I64); + let is_short = ctx + .block() + .icmp_eq(I64, &tag, crate::nanbox::SHORT_STRING_TAG_TOP16_I64); + let high = ctx.block().and(I64, &bits, SHORT_NON_ASCII_MASK); + let ascii = ctx.block().icmp_eq(I64, &high, "0"); + let short_ok = ctx.block().and(I1, &is_short, &ascii); + let ok = ctx.block().or(I1, &is_heap, &short_ok); + let not_ok = ctx.block().xor(I1, &ok, "true"); + let not_ok = ctx.block().zext(I1, ¬_ok, I32); + let bad = ctx.block().load(I32, &bad_slot); + let next_bad = ctx.block().or(I32, &bad, ¬_ok); + ctx.block().store(I32, &next_bad, &bad_slot); + let next = ctx.block().add(I32, &index, "1"); + ctx.block().store(I32, &next, &index_slot); + ctx.block().br(&head_l); + + ctx.current_block = done; + let bad = ctx.block().load(I32, &bad_slot); + let clean = ctx.block().icmp_eq(I32, &bad, "0"); + branch_guard(ctx, &clean, slow); +} + +/// The inline-string bytes of a slot that are not ASCII payload: bit 7 of each +/// of the five payload bytes (`string::admit` tests the same mask). +const SHORT_NON_ASCII_MASK: &str = "551911719040"; + +/// The UTF-16 length of the element at `index`, as a u32. The slot was proven +/// by [`scan`] to be a heap string or an inline ASCII string. +pub(super) fn element_length(ctx: &mut FnCtx<'_>, s: &Storage, index: &str) -> String { + let off = ctx.block().shl(I64, index, "3"); + let base = ctx.block().inttoptr(I64, &s.data); + let slot = ctx.block().gep(I8, &base, &[(I64, &off)]); + let bits = ctx.block().load(I64, &slot); + let tag = ctx.block().lshr(I64, &bits, "48"); + let is_heap = ctx + .block() + .icmp_eq(I64, &tag, crate::nanbox::STRING_TAG_TOP16_I64); + // Read through the string's address when it is a heap string; the array's + // own header (always readable) stands in for an inline string so the + // load needs no branch. + let raw = ctx.block().and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + let from = ctx.block().select(I1, &is_heap, I64, &raw, &s.raw); + let from_ptr = ctx.block().inttoptr(I64, &from); + let heap_len = ctx.block().load(I32, &from_ptr); + let short = ctx.block().lshr(I64, &bits, "40"); + let short = ctx.block().and(I64, &short, "255"); + let short_len = ctx.block().trunc(I64, &short, I32); + ctx.block().select(I1, &is_heap, I32, &heap_len, &short_len) +} diff --git a/crates/perry-codegen/src/stmt/string_length_loop.rs b/crates/perry-codegen/src/stmt/string_length_loop.rs index 71777805770..62f8bf2610f 100644 --- a/crates/perry-codegen/src/stmt/string_length_loop.rs +++ b/crates/perry-codegen/src/stmt/string_length_loop.rs @@ -19,7 +19,7 @@ struct Matched { max_idx_exclusive: i64, } -fn is_declared_string_array(ctx: &FnCtx<'_>, id: u32) -> bool { +pub(super) fn is_declared_string_array(ctx: &FnCtx<'_>, id: u32) -> bool { use perry_hir::types::Type; let is_string = |ty: &Type| matches!(ty, Type::String | Type::StringLiteral(_)); match crate::type_analysis::static_type_of(ctx, &Expr::LocalGet(id)) { diff --git a/crates/perry-codegen/src/wasm32/runtime_abi.tsv b/crates/perry-codegen/src/wasm32/runtime_abi.tsv index a7e8a228796..6b3d035aad3 100644 --- a/crates/perry-codegen/src/wasm32/runtime_abi.tsv +++ b/crates/perry-codegen/src/wasm32/runtime_abi.tsv @@ -4009,6 +4009,10 @@ js_string_concat_box f64 f64,f64 js_string_concat_chain ptr ptr,i32s js_string_concat_lit_value_box f64 ptr,i64,f64 js_string_concat_parts ptr ptr,i32s +js_string_concat_parts_2 ptr ptr +js_string_concat_parts_3 ptr ptr +js_string_concat_parts_4 ptr ptr +js_string_concat_parts_5 ptr ptr js_string_concat_site_value f64 ptr,ptr,f64 js_string_concat_value ptr ptr,f64 js_string_concat_value_box f64 ptr,f64 diff --git a/crates/perry-runtime/src/arena/allocators.rs b/crates/perry-runtime/src/arena/allocators.rs index c6316551471..137650d5423 100644 --- a/crates/perry-runtime/src/arena/allocators.rs +++ b/crates/perry-runtime/src/arena/allocators.rs @@ -386,6 +386,57 @@ pub(crate) fn arena_alloc_gc_old_born_tenured(size: usize, align: usize, obj_typ arena_alloc_gc_old(size, align, obj_type) } +/// [`arena_alloc_gc_old_born_tenured`] minus its collection point (JST slice 4, +/// design D): serve the request from a swept same-size hole or from the old +/// block that is already open, or return null. +/// +/// The collection points of the collecting form are exactly two: the +/// `gc_check_trigger` call and the fresh-block reservation, both inside +/// `arena_cell_alloc` after `try_alloc_current` has refused. Neither is +/// reachable from this body: the hole probe (`old_free_take_exact`) only +/// unlinks a listed hole, `try_alloc_current` is a bump inside the open block +/// ("Never collects"), and the header stamping and deferred page registration +/// touch only the new object's own header and page lists. So `non-null` PROVES +/// that nothing on the heap moved or was freed, and every raw pointer the +/// caller read before the call (its source string, its result array) is still +/// valid. A null is a refusal, not an event: the caller roots what it holds +/// and re-issues through [`arena_alloc_gc_old_born_tenured`], which then takes +/// the same collection decision the unconditional call would have taken. +/// +/// Written out rather than sharing a body with `arena_alloc_gc_old_registered` +/// for the same reason `arena_alloc_gc_no_collect` is: that body is inlined +/// into every old-generation birth. +#[inline(always)] +pub(crate) fn arena_alloc_gc_old_no_collect(size: usize, align: usize, obj_type: u8) -> *mut u8 { + use crate::gc::{GcHeader, GC_FLAG_ARENA, GC_FLAG_TENURED, GC_HEADER_SIZE}; + + let pad = align.max(8); + let total = (GC_HEADER_SIZE + size + pad - 1) & !(pad - 1); + let user = if let Some(user_ptr) = crate::gc::old_free_take_exact(total, None) { + OLD_ARENA.with(|a| unsafe { (&mut *a.get()).note_allocation(total, total >= 16 * 1024) }); + (user_ptr - GC_HEADER_SIZE) as *mut u8 + } else { + let Some(raw) = OLD_ARENA + .with(|a| unsafe { crate::arena::arena_cell_try_alloc_current(a.get(), total, align) }) + else { + return std::ptr::null_mut(); + }; + raw + }; + super::alloc_sample::note(total, obj_type); + unsafe { + let header = user as *mut GcHeader; + (*header).obj_type = obj_type; + (*header).gc_flags = GC_FLAG_ARENA | GC_FLAG_TENURED | crate::gc::gc_birth_extra_flags(); + crate::gc::gc_note_black_birth(header); + (*header)._reserved = 0; + (*header).size = crate::gc::gc_header_size_word(total); + } + record_arena_object_start(user as usize, obj_type); + defer_old_object_page_registration(user as usize, total); + unsafe { user.add(GC_HEADER_SIZE) } +} + /// #7624: registration stays EAGER here, unlike `arena_alloc_gc_old`. This is /// old-page defrag's relocation allocator (`gc/oldgen.rs`'s /// `evacuate_selected_old_pages_collecting`), which runs from INSIDE diff --git a/crates/perry-runtime/src/arena/mod.rs b/crates/perry-runtime/src/arena/mod.rs index f7b4d55ca74..5ab6cbe4690 100644 --- a/crates/perry-runtime/src/arena/mod.rs +++ b/crates/perry-runtime/src/arena/mod.rs @@ -120,6 +120,7 @@ pub use allocators::{ }; pub(crate) use allocators::{ arena_alloc_gc_collecting_holder, arena_alloc_gc_no_collect, arena_alloc_gc_old_born_tenured, + arena_alloc_gc_old_no_collect, arena_alloc_gc_old_excluding_pages, arena_alloc_gc_promoted, arena_alloc_gc_survivor, }; diff --git a/crates/perry-runtime/src/arena/tests.rs b/crates/perry-runtime/src/arena/tests.rs index 0c1ec075942..6625f58fcf0 100644 --- a/crates/perry-runtime/src/arena/tests.rs +++ b/crates/perry-runtime/src/arena/tests.rs @@ -1920,3 +1920,70 @@ fn no_collect_alloc_refuses_an_oversized_request() { assert_eq!(gc_trigger_arena_calls(), 0); }); } + +// --------------------------------------------------------------------------- +// JST slice 4 (design D): `arena_alloc_gc_old_no_collect`, the old-generation +// twin. `split` drops the per-piece root/reload when this returns non-null, so +// the guarantee is the whole point: a non-null return means no collection +// point was reached, and a full open block is REFUSED, not served through +// `gc_check_trigger`. +// --------------------------------------------------------------------------- + +#[test] +fn old_no_collect_alloc_refuses_a_full_block_instead_of_collecting() { + run_with_fresh_arenas(|| { + // A fresh old arena has no open block, so the first request is refused + // by construction; open one through the collecting entry first. + assert!(!arena_alloc_gc_old_born_tenured(64, 8, GC_TYPE_STRING).is_null()); + reset_gc_trigger_arena_probe(); + let chunk = LARGE_OBJECT_THRESHOLD_BYTES / 4; + let bound = 8 * BLOCK_SIZE / chunk; + let mut served = 0usize; + let mut refused = false; + for _ in 0..bound { + if arena_alloc_gc_old_no_collect(chunk, 8, GC_TYPE_STRING).is_null() { + refused = true; + break; + } + served += 1; + } + assert!( + refused, + "the old no-collect entry must REFUSE once the open old block is full; \ + it served {served} chunks without declining" + ); + assert!(served > 0, "test premise: it must serve from an open block"); + assert_eq!( + gc_trigger_arena_calls(), + 0, + "the old no-collect entry reached the allocation-point GC trigger" + ); + // The refusal leaves the collecting entry's decision intact. + assert!(!arena_alloc_gc_old_born_tenured(chunk, 8, GC_TYPE_STRING).is_null()); + }); +} + +#[test] +fn old_no_collect_alloc_births_exactly_what_the_collecting_entry_births() { + run_with_fresh_arenas(|| { + use crate::gc::{GcHeader, GC_FLAG_ARENA, GC_FLAG_TENURED, GC_HEADER_SIZE}; + let a = arena_alloc_gc_old_born_tenured(40, 8, GC_TYPE_STRING); + let b = arena_alloc_gc_old_no_collect(40, 8, GC_TYPE_STRING); + assert!(!b.is_null()); + unsafe { + let (ha, hb) = ( + &*(a.sub(GC_HEADER_SIZE) as *const GcHeader), + &*(b.sub(GC_HEADER_SIZE) as *const GcHeader), + ); + assert_eq!(ha.obj_type, hb.obj_type); + assert_eq!(ha.size, hb.size); + assert_eq!(ha.gc_flags, hb.gc_flags); + assert_ne!(hb.gc_flags & GC_FLAG_ARENA, 0); + assert_ne!(hb.gc_flags & GC_FLAG_TENURED, 0); + } + assert!(matches!( + classify_heap_generation(b as usize), + HeapGeneration::Old + )); + }); +} diff --git a/crates/perry-runtime/src/array/join.rs b/crates/perry-runtime/src/array/join.rs index 7fb1add7748..325ef355e63 100644 --- a/crates/perry-runtime/src/array/join.rs +++ b/crates/perry-runtime/src/array/join.rs @@ -122,8 +122,28 @@ unsafe fn well_formed_join_size( }; let elements = unsafe { array_elements_ptr(arr) }; + // The element sums cannot overflow a u64: at most 2^32 elements of at most + // 2^32 bytes each, plus the separators above (which saturate once). + let mut byte_total = size.byte_len; + let mut utf16_total = size.utf16_len; + let mut proof = size.encoding_proof; for index in 0..length as usize { let bits = unsafe { (*elements.add(index)).to_bits() }; + // A heap string is the common element: one tag test, one header read. + if bits >> 48 == STRING_TAG >> 48 { + let string = (bits & POINTER_MASK) as *const StringHeader; + let (flags, blen, ulen) = + unsafe { ((*string).flags, (*string).byte_len, (*string).utf16_len) }; + // Copying proven pieces preserves canonical encoding because this + // path excludes surrogate halves. Unknown pieces cannot grant proof. + if flags & STRING_FLAG_HAS_LONE_SURROGATES != 0 { + return None; + } + proof &= flags; + byte_total += blen as u64; + utf16_total += ulen as u64; + continue; + } if bits == TAG_HOLE { continue; } @@ -141,9 +161,7 @@ unsafe fn well_formed_join_size( } (len as u64, len as u64) } else if let Some(string) = unsafe { direct_string(bits) } { - // Copying proven pieces preserves canonical encoding because this - // path excludes surrogate halves. Unknown pieces cannot grant proof. - size.encoding_proof &= unsafe { (*string).flags }; + proof &= unsafe { (*string).flags }; if unsafe { (*string).flags } & STRING_FLAG_HAS_LONE_SURROGATES != 0 { return None; } @@ -156,9 +174,12 @@ unsafe fn well_formed_join_size( } else { return None; }; - size.byte_len = size.byte_len.saturating_add(byte_len); - size.utf16_len = size.utf16_len.saturating_add(utf16_len); + byte_total += byte_len; + utf16_total += utf16_len; } + size.byte_len = byte_total; + size.utf16_len = utf16_total; + size.encoding_proof = proof; Some(size) } @@ -210,6 +231,14 @@ fn try_join_well_formed( } let bits = (*elements.add(index)).to_bits(); + if bits >> 48 == STRING_TAG >> 48 { + let string = (bits & POINTER_MASK) as *const StringHeader; + let len = (*string).byte_len as usize; + // A piece is usually a handful of bytes: keep it out of libc. + crate::string::concat::copy_bytes_small(crate::string::string_data(string), cursor, len); + cursor = cursor.add(len); + continue; + } if bits == TAG_HOLE { continue; } diff --git a/crates/perry-runtime/src/string/concat_parts.rs b/crates/perry-runtime/src/string/concat_parts.rs index 7e53faf5890..46b0ff48736 100644 --- a/crates/perry-runtime/src/string/concat_parts.rs +++ b/crates/perry-runtime/src/string/concat_parts.rs @@ -72,18 +72,50 @@ pub extern "C" fn js_string_concat_parts( if n == 0 || parts.is_null() { return crate::string::js_string_from_bytes(b"".as_ptr(), 0); } - if n <= 4 { - concat_parts_sized::<4>(parts, n) - } else if n <= 8 { - concat_parts_sized::<8>(parts, n) - } else { - concat_parts_sized::(parts, n) + match n { + 2 => js_string_concat_parts_2(parts), + 3 => js_string_concat_parts_3(parts), + 4 => js_string_concat_parts_4(parts), + 5 => js_string_concat_parts_5(parts), + _ if n <= 8 => concat_parts_sized::<8>(parts, n), + _ => concat_parts_sized::(parts, n), + } +} + +/// Count-specialized form of [`js_string_concat_parts`] for a part count the +/// compiler knows (JST slice 4). `N` is a constant here, so the classification +/// loop, the scratch arrays and the copy loop are fully unrolled and sized +/// exactly, and the entry skips the count dispatch. The body is the one the +/// generic entry runs for that count; nothing is recorded per program. +#[inline(always)] +fn concat_parts_fixed(parts: *const ConcatPartSlot) -> *mut StringHeader { + if let Some(result) = concat_parts_simple::(parts, N) { + return result; } + concat_parts_general::(parts, N) } -#[inline] +macro_rules! concat_parts_fixed_entry { + ($($name:ident = $n:literal),* $(,)?) => {$( + /// `js_string_concat_parts` for exactly this many operands. + #[no_mangle] + #[inline(never)] + pub extern "C" fn $name(parts: *const ConcatPartSlot) -> *mut StringHeader { + concat_parts_fixed::<$n>(parts) + } + )*}; +} + +concat_parts_fixed_entry! { + js_string_concat_parts_2 = 2, + js_string_concat_parts_3 = 3, + js_string_concat_parts_4 = 4, + js_string_concat_parts_5 = 5, +} + +#[inline(never)] fn concat_parts_sized(parts: *const ConcatPartSlot, n: usize) -> *mut StringHeader { - if let Some(result) = concat_parts_simple::(parts, n) { + if let Some(result) = concat_parts_simple::(parts, n) { return result; } concat_parts_general::(parts, n) @@ -105,32 +137,86 @@ fn exact_u32(value: f64) -> Option { None } -/// Literals, heap strings, inline strings and numbers: no user code, and no -/// allocation before the single result allocation. -#[inline] -fn concat_parts_simple( - parts: *const ConcatPartSlot, - n: usize, -) -> Option<*mut StringHeader> { - let mut scratch: [core::mem::MaybeUninit<[u8; 32]>; M] = [core::mem::MaybeUninit::uninit(); M]; - // `piece_ptrs[i]` is null for a part written as digits of `digits[i]`. - let mut piece_ptrs: [*const u8; M] = [ptr::null(); M]; - let mut piece_lens: [u32; M] = [0; M]; - let mut digits: [u32; M] = [0; M]; - let mut flags = STRING_FLAG_WTF8_VALIDATED; +/// Evaluate `$call` (with `?` standing for the index, and its `Option` result +/// propagated) for each `i in 0..$n`. With `$unroll` the calls are written out. +macro_rules! drive_parts { + ($unroll:expr, $n:expr, $obj:ident . $method:ident ( $parts:ident , ? )) => {{ + if $unroll { + debug_assert!($n <= MAX_UNROLLED_PARTS); + if $n > 0 { $obj.$method($parts, 0)?; } + if $n > 1 { $obj.$method($parts, 1)?; } + if $n > 2 { $obj.$method($parts, 2)?; } + if $n > 3 { $obj.$method($parts, 3)?; } + if $n > 4 { $obj.$method($parts, 4)?; } + } else { + for i in 0..$n { + $obj.$method($parts, i)?; + } + } + }}; +} + +/// Fold `$cursor = $obj.$method(i, $cursor)` over `i in 0..$n`, unrolled like +/// [`drive_parts`]. +macro_rules! emit_parts { + ($unroll:expr, $n:expr, $obj:ident . $method:ident ( ? , $cursor:ident ) , $out:ident) => {{ + if $unroll { + if $n > 0 { $out = $obj.$method(0, $cursor); } + if $n > 1 { $out = $obj.$method(1, $out); } + if $n > 2 { $out = $obj.$method(2, $out); } + if $n > 3 { $out = $obj.$method(3, $out); } + if $n > 4 { $out = $obj.$method(4, $out); } + } else { + for i in 0..$n { + $out = $obj.$method(i, $out); + } + } + }}; +} + +/// The largest part count the count-specialized entries unroll. +const MAX_UNROLLED_PARTS: usize = 5; + +/// The classification state of [`concat_parts_simple`]: what each operand +/// contributes to the result, found before the single allocation. +struct SimpleParts { + scratch: [core::mem::MaybeUninit<[u8; 32]>; M], + /// `piece_ptrs[i]` is null for a part written as digits of `digits[i]`. + piece_ptrs: [*const u8; M], + piece_lens: [u32; M], + digits: [u32; M], + flags: u32, // 32 parts of at most `u32::MAX` bytes cannot overflow a u64; the sum is - // checked once, below. - let mut total_blen: u64 = 0; - let mut total_u16: u64 = 0; - for i in 0..n { + // checked once, by the caller. + total_blen: u64, + total_u16: u64, +} + +impl SimpleParts { + #[inline(always)] + fn new() -> Self { + Self { + scratch: [core::mem::MaybeUninit::uninit(); M], + piece_ptrs: [ptr::null(); M], + piece_lens: [0; M], + digits: [0; M], + flags: STRING_FLAG_WTF8_VALIDATED, + total_blen: 0, + total_u16: 0, + } + } + + /// Classify operand `i`. `None`: not a simple operand. + #[inline(always)] + fn classify(&mut self, parts: *const ConcatPartSlot, i: usize) -> Option<()> { let slot = unsafe { *parts.add(i) }; if slot.meta != CONCAT_PART_VALUE { let blen = lit_blen(slot.meta); - piece_ptrs[i] = slot.word as *const u8; - piece_lens[i] = blen; - total_blen += blen as u64; - total_u16 += lit_units(slot.meta) as u64; - continue; + self.piece_ptrs[i] = slot.word as *const u8; + self.piece_lens[i] = blen; + self.total_blen += blen as u64; + self.total_u16 += lit_units(slot.meta) as u64; + return Some(()); } let bits = slot.word; if bits >> 48 == 0x7FFF { @@ -141,54 +227,92 @@ fn concat_parts_simple( let blen = unsafe { (*header).byte_len }; // An empty operand contributes neither bytes nor provenance. if blen == 0 { - continue; + return Some(()); } - piece_ptrs[i] = string_data(header); - piece_lens[i] = blen; - flags = combine_string_flags(flags, unsafe { (*header).flags }); - total_blen += blen as u64; - total_u16 += unsafe { (*header).utf16_len } as u64; - continue; + self.piece_ptrs[i] = string_data(header); + self.piece_lens[i] = blen; + self.flags = combine_string_flags(self.flags, unsafe { (*header).flags }); + self.total_blen += blen as u64; + self.total_u16 += unsafe { (*header).utf16_len } as u64; + return Some(()); } let value = crate::value::JSValue::from_bits(bits); if value.is_number() { let number = f64::from_bits(bits); let len = if let Some(n) = exact_u32(number) { - digits[i] = n; + self.digits[i] = n; decimal_len(n) } else { - let buf = scratch[i].write([0u8; 32]); + let buf = self.scratch[i].write([0u8; 32]); let len = format_number_into(number, buf); - piece_ptrs[i] = buf.as_ptr(); + self.piece_ptrs[i] = buf.as_ptr(); len }; - piece_lens[i] = len as u32; - total_blen += len as u64; - total_u16 += len as u64; // ASCII - continue; + self.piece_lens[i] = len as u32; + self.total_blen += len as u64; + self.total_u16 += len as u64; // ASCII + return Some(()); } if value.is_short_string() { - let buf = scratch[i].write([0u8; 32]); + let buf = self.scratch[i].write([0u8; 32]); let mut bytes = [0u8; crate::value::SHORT_STRING_MAX_LEN]; let len = value.short_string_to_buf(&mut bytes); if len == 0 { - continue; + return Some(()); } buf[..len].copy_from_slice(&bytes[..len]); let (units, part_flags) = raw_string_metadata(buf.as_ptr(), len as u32); - flags = combine_string_flags(flags, part_flags); - piece_ptrs[i] = buf.as_ptr(); - piece_lens[i] = len as u32; - total_blen += len as u64; - total_u16 += units as u64; - continue; + self.flags = combine_string_flags(self.flags, part_flags); + self.piece_ptrs[i] = buf.as_ptr(); + self.piece_lens[i] = len as u32; + self.total_blen += len as u64; + self.total_u16 += units as u64; + return Some(()); } - return None; + None } - if total_blen > u32::MAX as u64 || total_u16 > u32::MAX as u64 { + + /// Write operand `i` at `cursor` and return the cursor after it. + /// + /// # Safety + /// `cursor` has room for the operand's `piece_lens[i]` bytes, and every + /// non-null `piece_ptrs[i]` is still valid. + #[inline(always)] + unsafe fn emit(&self, i: usize, cursor: *mut u8) -> *mut u8 { + let len = self.piece_lens[i] as usize; + // An empty operand has a null pointer too: it writes nothing. + if len == 0 { + return cursor; + } + if self.piece_ptrs[i].is_null() { + write_decimal(cursor, self.digits[i], len); + } else { + // A literal or a number is a handful of bytes: keep them out + // of libc's memcpy. Long heap payloads still take it. + copy_bytes_small(self.piece_ptrs[i], cursor, len); + } + cursor.add(len) + } +} + +/// Literals, heap strings, inline strings and numbers: no user code, and no +/// allocation before the single result allocation. +/// +/// `UNROLL` (the count is a compile-time constant of the caller, at most +/// [`MAX_UNROLLED_PARTS`]) writes the per-operand steps out, so every index is +/// a constant and the per-operand arrays live in registers. +#[inline(always)] +fn concat_parts_simple( + parts: *const ConcatPartSlot, + n: usize, +) -> Option<*mut StringHeader> { + let mut st = SimpleParts::::new(); + drive_parts!(UNROLL, n, st.classify(parts, ?)); + if st.total_blen > u32::MAX as u64 || st.total_u16 > u32::MAX as u64 { return None; } - let (total_blen, total_u16) = (total_blen as u32, total_u16 as u32); + let (total_blen, total_u16) = (st.total_blen as u32, st.total_u16 as u32); + let flags = st.flags; // `Some` proves nothing moved, so the borrowed heap payloads are valid. let (result, mut cursor) = match string_storage_alloc_no_collect(total_blen) { @@ -197,9 +321,9 @@ fn concat_parts_simple( return Some(concat_parts_rooted_copy::( parts, n, - &mut piece_ptrs, - &piece_lens, - &digits, + &mut st.piece_ptrs, + &st.piece_lens, + &st.digits, total_blen, total_u16, flags, @@ -208,21 +332,7 @@ fn concat_parts_simple( }; unsafe { init_string_header(result, total_u16, total_blen, total_blen, 0, flags); - for i in 0..n { - let len = piece_lens[i] as usize; - // An empty operand has a null pointer too: it writes nothing. - if len == 0 { - continue; - } - if piece_ptrs[i].is_null() { - write_decimal(cursor, digits[i], len); - } else { - // A literal or a number is a handful of bytes: keep them out - // of libc's memcpy. Long heap payloads still take it. - copy_bytes_small(piece_ptrs[i], cursor, len); - } - cursor = cursor.add(len); - } + emit_parts!(UNROLL, n, st.emit(?, cursor), cursor); } // Only a heap or inline operand can carry a lone surrogate, and only a // join of two such operands can mint a pair. diff --git a/crates/perry-runtime/src/string/mod.rs b/crates/perry-runtime/src/string/mod.rs index a3487493d33..9882a7f0d61 100644 --- a/crates/perry-runtime/src/string/mod.rs +++ b/crates/perry-runtime/src/string/mod.rs @@ -204,7 +204,10 @@ pub use concat::{ js_string_concat_chain, js_string_concat_value, js_value_add_string, js_value_concat_string, scan_concat_memo_roots, scan_concat_memo_roots_mut, }; -pub use concat_parts::{js_string_concat_lit_value_box, js_string_concat_parts}; +pub use concat_parts::{ + js_string_concat_lit_value_box, js_string_concat_parts, js_string_concat_parts_2, + js_string_concat_parts_3, js_string_concat_parts_4, js_string_concat_parts_5, +}; pub use concat_site::{js_string_concat_site_value, CONCAT_SITE_SLOTS}; pub(crate) use format::ascii_char_string; pub(crate) use format::fix_exponent_format; @@ -880,6 +883,38 @@ pub(crate) fn string_storage_alloc_no_collect( Some((ptr, data)) } +/// [`string_storage_alloc_for`] with **no collection point** (JST slice 4, +/// design D). `Some` proves nothing on the heap moved, so a raw string or array +/// pointer the caller read before the call is still valid. `None` is a refusal, +/// not an event: root the operands and call [`string_storage_alloc_for`]. +/// +/// Both arms are the no-collect forms of the allocators the collecting call +/// reaches, so a `Some` is byte-identical to what it would have returned. +#[inline(always)] +pub(crate) fn string_storage_alloc_for_no_collect( + birth: crate::gc::ProducerChildBirth, + capacity: u32, +) -> Option<(*mut StringHeader, *mut u8)> { + match birth { + crate::gc::ProducerChildBirth::Young => string_storage_alloc_no_collect(capacity), + crate::gc::ProducerChildBirth::Old => { + let payload_size = std::mem::size_of::() + capacity as usize; + let raw = crate::arena::arena_alloc_gc_old_no_collect( + payload_size, + 8, + crate::gc::GC_TYPE_STRING, + ); + if raw.is_null() { + return None; + } + let ptr = raw as *mut StringHeader; + let data = unsafe { raw.add(std::mem::size_of::()) }; + zero_alignment_padding_tail(raw, payload_size); + Some((ptr, data)) + } + } +} + #[inline] pub(crate) fn string_storage_alloc_longlived(capacity: u32) -> (*mut StringHeader, *mut u8) { let payload_size = std::mem::size_of::() + capacity as usize; diff --git a/crates/perry-runtime/src/string/split.rs b/crates/perry-runtime/src/string/split.rs index d64e6192c63..5dfbe8384b6 100644 --- a/crates/perry-runtime/src/string/split.rs +++ b/crates/perry-runtime/src/string/split.rs @@ -624,7 +624,7 @@ fn split_by_delimiter(s: *const StringHeader, delimiter: &str, limit: i32) -> *m let scope = crate::gc::RuntimeHandleScope::new(); let s_handle = scope.root_string_ptr(s); - let (arr, _) = s_handle.across_const::(|| { + let (arr, mut s_cur) = s_handle.across_const::(|| { crate::array::js_array_alloc_pointer_elements(n as u32) }); // Decided once, before the first part allocation and before the result @@ -632,17 +632,33 @@ fn split_by_delimiter(s: *const StringHeader, delimiter: &str, limit: i32) -> *m // the remembered set nothing (`crate::gc::ProducerChildBirth`). let birth = crate::gc::ProducerChildBirth::for_fresh_container(arr as usize); let arr_handle = scope.root_raw_mut_ptr(arr); + // JST slice 4 (D): `s_cur` and `arr_cur` are the live addresses of the + // source and the result. A piece taken from the block that is already open + // (`string_storage_alloc_for_no_collect` returns `Some`) cannot have + // collected, so both are still valid and neither is reloaded from its + // handle. A refusal takes the rooted collecting allocation exactly as every + // piece did before, and refreshes both from the handles. + let mut arr_cur = arr; unsafe { for (i, &(offset, byte_len)) in part_ranges.iter().enumerate() { // Allocate the destination FIRST (it may move the source), then // re-read the source address before touching its bytes. - let (((sh, data_ptr), s_now), arr_now) = - arr_handle.across_mut::(|| { - s_handle.across_const::(|| { - string_storage_alloc_for(birth, byte_len) - }) - }); + let ((sh, data_ptr), s_now, arr_now) = + match crate::string::string_storage_alloc_for_no_collect(birth, byte_len) { + Some(allocated) => (allocated, s_cur, arr_cur), + None => { + let (((sh, data_ptr), s_now), arr_now) = + arr_handle.across_mut::(|| { + s_handle.across_const::(|| { + string_storage_alloc_for(birth, byte_len) + }) + }); + s_cur = s_now; + arr_cur = arr_now; + ((sh, data_ptr), s_now, arr_now) + } + }; let part_ptr = string_data(s_now).add(offset as usize); // Derive metadata from THIS PART's own bytes. The only shortcut // taken is the all-ASCII one, which was verified by scanning the @@ -668,7 +684,7 @@ fn split_by_delimiter(s: *const StringHeader, delimiter: &str, limit: i32) -> *m } } - arr_handle.get_raw_mut_ptr::() + arr_cur } /// `String.prototype.split` for the shape nearly every call has: a heap string diff --git a/scripts/gc_root_dominance_check.py b/scripts/gc_root_dominance_check.py index b8b22d2698f..b5d6330a2b4 100755 --- a/scripts/gc_root_dominance_check.py +++ b/scripts/gc_root_dominance_check.py @@ -1917,6 +1917,9 @@ def is_collecting(callee): # `js_string_concat_chain`; `js_string_concat_lit_value_box` is the # literal-prefix twin of `js_string_concat_value_box`. "js_string_concat_parts", "js_string_concat_lit_value_box", + # JST slice 4: the count-specialized forms of `js_string_concat_parts`. + "js_string_concat_parts_2", "js_string_concat_parts_3", + "js_string_concat_parts_4", "js_string_concat_parts_5", # `js_private_brand_add` is the referent-with-no-name that NEITHER audit can # ask for: `--audit-poll-reach` only walks symbols `ALLOC_RE` matches, and # `js_private_brand_add` matches no alloc/new/create convention, so the one diff --git a/test-files/test_gap_jst6_split_join.ts b/test-files/test_gap_jst6_split_join.ts new file mode 100644 index 00000000000..c758c758e63 --- /dev/null +++ b/test-files/test_gap_jst6_split_join.ts @@ -0,0 +1,150 @@ +// JST slice 4: split pieces built without a per-piece root/reload, join's +// size+copy walk, the string-array `.length` loop region and the +// count-specialized concat entries. Every line is compared with Node. + +function attempt(f: () => string): string { + try { + return f(); + } catch (e) { + return "throw:" + (e instanceof TypeError ? "TypeError" : String(e)); + } +} + +function cps(s: string): string { + const out: string[] = []; + for (let i = 0; i < s.length; i++) out.push(s.charCodeAt(i).toString(16)); + return out.join(","); +} + +// ---- split: empty strings, separators, unicode, surrogates ---- +const sources: string[] = [ + "", "a", ";", ";;", "a;", ";a", "a;b", "a;;b", ";a;b;", "item-1;item-2;item-3;", + "héllo;wörld;日本語;", "x\u{1F600}y;\u{1F600};z", "lone\uD800;high\uD83D;\uDE00low;", + "abcdefghijklmnopqrstuvwxyz;0123456789;ABCDEFGHIJKLMNOPQRSTUVWXYZ", +]; +const seps: string[] = [";", ";;", "item", "é", "\u{1F600}", "\uD800", "\uDE00", "ab", "z;0"]; +for (const s of sources) { + for (const sep of seps) { + const parts = s.split(sep); + console.log(JSON.stringify(s), JSON.stringify(sep), parts.length, parts.map((p) => p.length).join("/"), parts.map(cps).join("|")); + console.log(cps(parts.join("+")), cps(parts.join("")), cps(parts.join())); + } + console.log(s.split(";", 0).length, s.split(";", 1).length, cps(s.split(";", 2).join("|")), s.split(";", 100).length); +} + +// ---- very long: enough pieces that the old/young birth and the block +// boundary refusals (collecting fallback) are crossed repeatedly ---- +let big = ""; +for (let i = 0; i < 20000; i++) big += "p" + i + ";"; +const bigParts = big.split(";"); +let lenSum = 0; +for (let i = 0; i < bigParts.length; i++) lenSum += bigParts[i].length; +console.log(bigParts.length, lenSum, bigParts[0], bigParts[19999], JSON.stringify(bigParts[20000])); +console.log(bigParts.join(";") === big, bigParts.join("").length); +let wide = ""; +for (let i = 0; i < 3000; i++) wide += "é" + i + "\u{1F600}" + (i % 7 === 0 ? "\uD800" : "") + "|"; +const wideParts = wide.split("|"); +let wideSum = 0; +for (let i = 0; i < wideParts.length; i++) wideSum += wideParts[i].length; +console.log(wideParts.length, wideSum, wideParts.join("|") === wide, cps(wideParts[7]), cps(wideParts[8])); +const longPiece = "q".repeat(100000) + ";" + "r".repeat(70000); +const longParts = longPiece.split(";"); +console.log(longParts.length, longParts[0].length, longParts[1].length, longParts.join("--").length); + +// ---- join: holey arrays, non-string elements, toString, null/undefined ---- +const holey: string[] = new Array(6); +holey[1] = "b"; +holey[4] = "e"; +console.log(JSON.stringify(holey.join("-")), JSON.stringify(holey.join()), JSON.stringify(holey.join(""))); +const mixed: any[] = ["a", 1, -0, 1.5, true, false, null, undefined, "", "é", { toString() { return "obj"; } }, [1, 2], 10n, "\u{1F600}"]; +console.log(JSON.stringify(mixed.join("|")), JSON.stringify(mixed.join()), JSON.stringify(mixed.join(""))); +console.log(attempt(() => ["a", Symbol("s")].join("-") as string)); +const surr: string[] = ["\uD83D", "\uDE00", "x", "\uD83D", "\uDE00"]; +console.log(cps(surr.join("")), cps(surr.join("\uDE00")), surr.join("").length); +const shorts: string[] = ["a", "bc", "def", "ghij", "klmno", "pqrstu", "", "", "é", "日"]; +console.log(JSON.stringify(shorts.join("")), JSON.stringify(shorts.join(", ")), JSON.stringify(shorts.join("\u{1F600}"))); +const lens: string[] = []; +for (let n = 0; n < 40; n++) lens.push("x".repeat(n)); +console.log(lens.join("").length, lens.join(",").length, lens.join("ab").slice(-30)); +console.log(JSON.stringify([].join("-")), JSON.stringify([""].join("-")), JSON.stringify(["", ""].join("-"))); + +// ---- string-array `.length` loops: heap, inline, empty, mixed, non-ASCII ---- +function totalLength(a: string[]): number { + let total = 0; + for (let i = 0; i < a.length; i++) total += a[i].length; + return total; +} +function totalLengthLocal(n: number): number { + const arr: string[] = []; + for (let i = 0; i < n; i++) arr.push("s" + i); + let total = 0; + for (let i = 0; i < arr.length; i++) total += arr[i].length; + return total; +} +console.log(totalLength([]), totalLength([""]), totalLength(["a", "bb", "ccc"])); +console.log(totalLength(["item-123456", "item-1234567", "x".repeat(300)])); +console.log(totalLength(["é", "日本", "\u{1F600}", "a\u{1F600}b", "\uD800", "\uDE00x"])); +console.log(totalLength("a;bb;ccc;;dddd;eeeee;ffffff;".split(";"))); +console.log(totalLengthLocal(0), totalLengthLocal(1), totalLengthLocal(10), totalLengthLocal(100000)); +const withHole: string[] = new Array(4); +withHole[0] = "ab"; +withHole[1] = "cd"; +withHole[3] = "ef"; +console.log(attempt(() => String(totalLength(withHole)))); +const withUndefined: any[] = ["ab", undefined, "cd"]; +console.log(attempt(() => String(totalLength(withUndefined)))); +const withNull: any[] = ["ab", null]; +console.log(attempt(() => String(totalLength(withNull)))); +const withObjects: any[] = ["ab", new String("xyz"), { length: 7 }, [1, 2, 3, 4]]; +console.log(attempt(() => String(totalLength(withObjects)))); +const withNumber: any[] = ["ab", 12345]; +console.log(attempt(() => String(totalLength(withNumber)))); +const withLateBad: any[] = []; +for (let i = 0; i < 1000; i++) withLateBad.push("v" + i); +withLateBad.push(undefined); +console.log(attempt(() => String(totalLength(withLateBad)))); +const proxied: string[] = new Proxy(["a", "bb"], {}) as any; +console.log(attempt(() => String(totalLength(proxied)))); +class Sub extends Array {} +const sub = new Sub(); +sub.push("a", "bcd"); +console.log(totalLength(sub)); +const accessor: string[] = ["a", "b", "c"]; +Object.defineProperty(accessor, 1, { get() { return "zzzz"; } }); +console.log(totalLength(accessor)); +const stray: string[] = ["a", "bb"]; +(stray as any).foo = 1; +console.log(totalLength(stray)); +const shifted: string[] = ["x", "yy", "zzz", "wwww"]; +shifted.shift(); +console.log(totalLength(shifted), shifted.length); +let pieceSum = 0; +const pieces: string[] = "k1;k22;k333;".split(";"); +for (let i = 0; i < pieces.length; i++) pieceSum += pieces[i].length; +console.log(pieceSum, pieces.length); + +// ---- concat with 2..6 operands (the count-specialized entries) ---- +const objs: any = { toString() { return "O"; }, valueOf() { return 7; } }; +const vals: any[] = [0, -0, 1, 99, 100, 12345, 4294967295, 4294967296, 1.5, NaN, -Infinity, "", "é", "\u{1F600}", "\uD800", "\uDE00", "abcdefg", true, null, undefined, objs, 10n]; +for (const v of vals) { + const a = attempt(() => "<" + v + ">"); + const b = attempt(() => "[" + v + "|" + v + "]"); + const c = attempt(() => "a" + v + "b" + v + "c"); + const d = attempt(() => "" + v + "-" + v + "-" + v + "-" + v); + const e = attempt(() => "k" + v + ":" + v + ":" + v + ":" + v + ":" + v); + const f = attempt(() => v + "x" + v); + const g = attempt(() => "1" + "2" + v + "3" + "4" + v + "5" + "6" + v); + console.log(String(typeof v), cps(a), cps(b), cps(c), cps(d), cps(e), cps(f), cps(g)); +} +for (let i = 0; i < 40; i++) { + const s = "item-" + i + ";" + (i * 31) + "," + (i % 3 === 0 ? "t" : "") + "|" + (i - 20); + console.log(s, s.length, "[" + s + "]" + i, i + "#" + s); +} +console.log(attempt(() => "a" + Symbol("q") + "b")); +console.log(attempt(() => ("a" + Symbol("q")) as string)); +console.log(attempt(() => "a" + "b" + Symbol("q"))); +const order: string[] = []; +const p1 = { valueOf() { order.push("p1"); return 1; } }; +const p2 = { valueOf() { order.push("p2"); return 2; } }; +const p3 = { toString() { order.push("p3"); return "3"; }, valueOf() { order.push("p3v"); return 3; } }; +console.log("x" + (p1 as any) + "y" + (p2 as any) + "z" + (p3 as any), order.join(","));