diff --git a/shape.c b/shape.c index 9052bd658b1ded..688635971fdc73 100644 --- a/shape.c +++ b/shape.c @@ -729,29 +729,6 @@ rb_shape_object_id(shape_id_t original_shape_id) return SHAPE_ID(shape, original_shape_id) | SHAPE_ID_FL_HAS_OBJECT_ID; } -/* - * This function is used for assertions where we don't want to increment - * max_iv_count - */ -static inline rb_shape_t * -shape_get_next_iv_shape(rb_shape_t *shape, ID id) -{ - RUBY_ASSERT(!is_instance_id(id) || RTEST(rb_sym2str(ID2SYM(id)))); - bool dont_care; - return get_next_shape_internal(shape, id, SHAPE_IVAR, &dont_care, true); -} - -shape_id_t -rb_shape_get_next_iv_shape(shape_id_t shape_id, ID id) -{ - rb_shape_t *shape = RSHAPE(shape_id); - rb_shape_t *next_shape = shape_get_next_iv_shape(shape, id); - if (!next_shape) { - return INVALID_SHAPE_ID; - } - return SHAPE_OFFSET(next_shape); -} - static bool shape_get_iv_index(rb_shape_t *shape, ID id, attr_index_t *value) { @@ -1099,9 +1076,11 @@ shape_rebuild(rb_shape_t *initial_shape, rb_shape_t *dest_shape) } switch ((enum shape_type)dest_shape->type) { - case SHAPE_IVAR: - midway_shape = shape_get_next_iv_shape(midway_shape, dest_shape->edge_name); + case SHAPE_IVAR: { + bool dont_care; + midway_shape = get_next_shape_internal(midway_shape, dest_shape->edge_name, SHAPE_IVAR, &dont_care, true); break; + } case SHAPE_OBJ_ID: case SHAPE_ROOT: break; diff --git a/shape.h b/shape.h index a319449988e8fc..7df66f045f8fce 100644 --- a/shape.h +++ b/shape.h @@ -215,7 +215,6 @@ RSHAPE(shape_id_t shape_id) int32_t rb_shape_id_offset(void); RUBY_FUNC_EXPORTED shape_id_t rb_obj_shape_id(VALUE obj); -shape_id_t rb_shape_get_next_iv_shape(shape_id_t shape_id, ID id); bool rb_shape_get_iv_index(shape_id_t shape_id, ID id, attr_index_t *value); bool rb_shape_get_iv_index_with_hint(shape_id_t shape_id, ID id, attr_index_t *value, shape_id_t *shape_id_hint); bool rb_shape_find_ivar(shape_id_t shape_id, ID id, shape_id_t *ivar_shape); diff --git a/zjit/src/codegen.rs b/zjit/src/codegen.rs index 94adf122b43e14..69f71d1f13ed21 100644 --- a/zjit/src/codegen.rs +++ b/zjit/src/codegen.rs @@ -21,7 +21,7 @@ use crate::stats::{counter_ptr, with_time_stat, trace_compile_phase, Counter, Co use crate::{asm::CodeBlock, cruby::*, options::debug, virtualmem::CodePtr}; use crate::backend::lir::{self, Assembler, C_ARG_OPNDS, C_RET_OPND, CFP, EC, NATIVE_BASE_PTR, Opnd, SP, SideExit, SideExitRecompile, Target, asm_ccall, asm_comment}; use crate::hir::{iseq_to_hir, BlockId, Invariant, RangeType, SideExitReason::{self, *}, SpecialBackrefSymbol, SpecialObjectType}; -use crate::hir::{BlockHandler, Const, FieldName, FrameState, Function, Insn, InsnId, Recompile, SendFallbackReason}; +use crate::hir::{BlockHandler, CCallVariadicData, CCallWithFrameData, Const, FieldName, FrameState, Function, Insn, InsnId, Recompile, SendFallbackReason}; use crate::hir_type::{types, Type}; use crate::options::{get_option, InlineDepth, PerfMap}; use crate::cast::IntoUsize; @@ -661,7 +661,7 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::InvokeBlock { cd, state, reason, .. } => gen_invokeblock(jit, asm, cd, &function.frame_state(state), reason), Insn::InvokeBlockIfunc { cd, block_handler, args, state, .. } => gen_invokeblock_ifunc(jit, asm, *cd, opnd!(block_handler), opnds!(args), &function.frame_state(*state)), Insn::InvokeProc { recv, args, state, kw_splat } => gen_invokeproc(jit, asm, opnd!(recv), opnds!(args), *kw_splat, &function.frame_state(*state)), - Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), bf, *leaf, opnds!(args)), + Insn::InvokeBuiltin { bf, leaf, args, state, .. } => gen_invokebuiltin(jit, asm, &function.frame_state(*state), unsafe { &**bf }, *leaf, opnds!(args)), &Insn::EntryPoint { jit_entry_idx } => no_output!(gen_entry_point(jit, asm, jit_entry_idx)), Insn::Return { val } => no_output!(gen_return(asm, opnd!(val))), Insn::FixnumAdd { left, right, state } => gen_fixnum_add(jit, asm, opnd!(left), opnd!(right), &function.frame_state(*state)), @@ -721,9 +721,12 @@ fn gen_insn(cb: &mut CodeBlock, jit: &mut JITState, asm: &mut Assembler, functio &Insn::GuardGreaterEq { left, right, state, .. } => gen_guard_greater_eq(jit, asm, opnd!(left), opnd!(right), &function.frame_state(state)), Insn::PatchPoint { invariant, state } => no_output!(gen_patch_point(jit, asm, invariant, &function.frame_state(*state))), Insn::CCall { cfunc, recv, args, name, owner: _, return_type: _, elidable: _ } => gen_ccall(asm, *cfunc, *name, opnd!(recv), opnds!(args)), - Insn::CCallWithFrame { cfunc, recv, name, args, cme, state, block, .. } => - gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)), - Insn::CCallVariadic { cfunc, recv, name, args, cme, state, block, return_type: _, elidable: _ } => { + Insn::CCallWithFrame(insn) => { + let CCallWithFrameData { cfunc, recv, name, args, cme, state, block, .. } = &**insn; + gen_ccall_with_frame(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) + } + Insn::CCallVariadic(insn) => { + let CCallVariadicData { cfunc, recv, name, args, cme, state, block, .. } = &**insn; gen_ccall_variadic(jit, asm, *cfunc, *name, opnd!(recv), opnds!(args), *cme, *block, &function.frame_state(*state)) } Insn::GetIvar { self_val, id, ic, state } => gen_getivar(asm, opnd!(self_val), *id, *ic, &function.frame_state(*state)), diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index d8bd1d95574a98..7777faef0fbfc1 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -667,7 +667,7 @@ impl VALUE { i.try_into().unwrap() } - pub fn as_usize(self) -> usize { + pub const fn as_usize(self) -> usize { let VALUE(us) = self; us } diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 8a1f7b87980148..69f79c5e853496 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -841,6 +841,35 @@ impl From for FieldName { } } +/// Payload of [`Insn::CCallWithFrame`]. Boxed in the enum to keep `Insn` small. +#[derive(Debug, Clone)] +pub struct CCallWithFrameData { + pub cd: *const rb_call_data, // cd for falling back to Send + pub cfunc: *const u8, + pub recv: InsnId, + pub args: Vec, + pub cme: *const rb_callable_method_entry_t, + pub name: ID, + pub state: InsnId, + pub return_type: Type, + pub elidable: bool, + pub block: Option, +} + +/// Payload of [`Insn::CCallVariadic`]. Boxed in the enum to keep `Insn` small. +#[derive(Debug, Clone)] +pub struct CCallVariadicData { + pub cfunc: *const u8, + pub recv: InsnId, + pub args: Vec, + pub cme: *const rb_callable_method_entry_t, + pub name: ID, + pub state: InsnId, + pub return_type: Type, + pub elidable: bool, + pub block: Option, +} + /// An instruction in the SSA IR. The output of an instruction is referred to by the index of /// the instruction ([`InsnId`]). SSA form enables this, and [`UnionFind`] ([`Function::find`]) /// helps with editing. @@ -1001,7 +1030,7 @@ pub enum Insn { /// Own a FrameState so that instructions can look up their dominating FrameState when /// generating deopt side-exits and frame reconstruction metadata. Does not directly generate /// any code. - Snapshot { state: FrameState }, + Snapshot { state: Box }, /// Unconditional jump Jump(BranchEdge), @@ -1014,32 +1043,11 @@ pub enum Insn { CCall { cfunc: *const u8, recv: InsnId, args: Vec, name: ID, owner: VALUE, return_type: Type, elidable: bool }, /// Call a C function that pushes a frame - CCallWithFrame { - cd: *const rb_call_data, // cd for falling back to Send - cfunc: *const u8, - recv: InsnId, - args: Vec, - cme: *const rb_callable_method_entry_t, - name: ID, - state: InsnId, - return_type: Type, - elidable: bool, - block: Option, - }, + CCallWithFrame(Box), /// Call a variadic C function with signature: func(int argc, VALUE *argv, VALUE recv) /// This handles frame setup, argv creation, and frame teardown all in one - CCallVariadic { - cfunc: *const u8, - recv: InsnId, - args: Vec, - cme: *const rb_callable_method_entry_t, - name: ID, - state: InsnId, - return_type: Type, - elidable: bool, - block: Option, - }, + CCallVariadic(Box), /// Un-optimized fallback implementation (dynamic dispatch) for send-ish instructions /// Ignoring keyword arguments etc for now @@ -1128,7 +1136,7 @@ pub enum Insn { // Invoke a builtin function InvokeBuiltin { - bf: rb_builtin_function, + bf: *const rb_builtin_function, recv: InsnId, args: Vec, state: InsnId, @@ -1241,25 +1249,25 @@ macro_rules! for_each_operand_impl { | Insn::IncrCounterPtr { .. } => {} Insn::IsBlockGiven { lep } => { - $visit_one!(lep); + $visit_one!(*lep); } Insn::IsBlockParamModified { flags } => { - $visit_one!(flags); + $visit_one!(*flags); } Insn::CheckMatch { target, pattern, state, .. } => { - $visit_one!(target); - $visit_one!(pattern); - $visit_one!(state); + $visit_one!(*target); + $visit_one!(*pattern); + $visit_one!(*state); } Insn::PatchPoint { state, .. } | Insn::CheckInterrupts { state } | Insn::PutSpecialObject { state, .. } | Insn::GetBlockParam { state, .. } | Insn::GetConstantPath { state, .. } => { - $visit_one!(state); + $visit_one!(*state); } Insn::FixnumBitCheck { val, .. } => { - $visit_one!(val); + $visit_one!(*val); } Insn::ArrayMax { elements, state, .. } | Insn::ArrayMin { elements, state, .. } @@ -1267,57 +1275,57 @@ macro_rules! for_each_operand_impl { | Insn::NewHash { elements, state, .. } | Insn::NewArray { elements, state, .. } => { $visit_many!(elements); - $visit_one!(state); + $visit_one!(*state); } Insn::ArrayInclude { elements, target, state, .. } => { $visit_many!(elements); - $visit_one!(target); - $visit_one!(state); + $visit_one!(*target); + $visit_one!(*state); } Insn::ArrayPackBuffer { elements, fmt, buffer, state, .. } => { $visit_many!(elements); - $visit_one!(fmt); + $visit_one!(*fmt); if let Some(buffer) = buffer { - $visit_one!(buffer); + $visit_one!(*buffer); } - $visit_one!(state); + $visit_one!(*state); } Insn::DupArrayInclude { target, state, .. } => { - $visit_one!(target); - $visit_one!(state); + $visit_one!(*target); + $visit_one!(*state); } Insn::NewRange { low, high, state, .. } | Insn::NewRangeFixnum { low, high, state, .. } => { - $visit_one!(low); - $visit_one!(high); - $visit_one!(state); + $visit_one!(*low); + $visit_one!(*high); + $visit_one!(*state); } Insn::StringConcat { strings, state, .. } => { $visit_many!(strings); - $visit_one!(state); + $visit_one!(*state); } Insn::StringGetbyte { string, index } => { - $visit_one!(string); - $visit_one!(index); + $visit_one!(*string); + $visit_one!(*index); } Insn::StringSetbyteFixnum { string, index, value } => { - $visit_one!(string); - $visit_one!(index); - $visit_one!(value); + $visit_one!(*string); + $visit_one!(*index); + $visit_one!(*value); } Insn::StringAppend { recv, other, state } | Insn::StringAppendCodepoint { recv, other, state } => { - $visit_one!(recv); - $visit_one!(other); - $visit_one!(state); + $visit_one!(*recv); + $visit_one!(*other); + $visit_one!(*state); } Insn::StringEqual { left, right } => { - $visit_one!(left); - $visit_one!(right); + $visit_one!(*left); + $visit_one!(*right); } Insn::ToRegexp { values, state, .. } => { $visit_many!(values); - $visit_one!(state); + $visit_one!(*state); } Insn::RefineType { val, .. } | Insn::HasType { val, .. } @@ -1325,7 +1333,7 @@ macro_rules! for_each_operand_impl { | Insn::Test { val } | Insn::SetLocal { val, .. } | Insn::BoxBool { val } => { - $visit_one!(val); + $visit_one!(*val); } Insn::SetGlobal { val, state, .. } | Insn::Defined { v: val, state, .. } @@ -1340,14 +1348,14 @@ macro_rules! for_each_operand_impl { | Insn::IsMethodCfunc { val, state, .. } | Insn::ToNewArray { val, state } | Insn::BoxFixnum { val, state } => { - $visit_one!(val); - $visit_one!(state); + $visit_one!(*val); + $visit_one!(*state); } Insn::GuardGreaterEq { left, right, state, .. } | Insn::GuardLess { left, right, state, .. } => { - $visit_one!(left); - $visit_one!(right); - $visit_one!(state); + $visit_one!(*left); + $visit_one!(*right); + $visit_one!(*state); } Insn::Snapshot { state } => { $visit_many!(state.stack); @@ -1360,21 +1368,21 @@ macro_rules! for_each_operand_impl { | Insn::FixnumMod { left, right, state } | Insn::ArrayExtend { left, right, state } | Insn::FixnumLShift { left, right, state } => { - $visit_one!(left); - $visit_one!(right); - $visit_one!(state); + $visit_one!(*left); + $visit_one!(*right); + $visit_one!(*state); } Insn::FloatAdd { recv, other, state } | Insn::FloatSub { recv, other, state } | Insn::FloatMul { recv, other, state } | Insn::FloatDiv { recv, other, state } => { - $visit_one!(recv); - $visit_one!(other); - $visit_one!(state); + $visit_one!(*recv); + $visit_one!(*other); + $visit_one!(*state); } Insn::FloatToInt { recv, state } => { - $visit_one!(recv); - $visit_one!(state); + $visit_one!(*recv); + $visit_one!(*state); } Insn::FixnumLt { left, right } | Insn::FixnumLe { left, right } @@ -1390,139 +1398,150 @@ macro_rules! for_each_operand_impl { | Insn::FixnumRShift { left, right } | Insn::IsBitEqual { left, right } | Insn::IsBitNotEqual { left, right } => { - $visit_one!(left); - $visit_one!(right); + $visit_one!(*left); + $visit_one!(*right); } Insn::Jump(BranchEdge { args, .. }) => { $visit_many!(args); } Insn::CondBranch { val, if_true: BranchEdge { args: true_args, .. }, if_false: BranchEdge { args: false_args, .. } } => { - $visit_one!(val); + $visit_one!(*val); $visit_many!(true_args); $visit_many!(false_args); } Insn::ArrayDup { val, state } | Insn::Throw { val, state, .. } | Insn::HashDup { val, state } => { - $visit_one!(val); - $visit_one!(state); + $visit_one!(*val); + $visit_one!(*state); } Insn::ArrayAref { array, index } => { - $visit_one!(array); - $visit_one!(index); + $visit_one!(*array); + $visit_one!(*index); } Insn::ArrayAset { array, index, val } => { - $visit_one!(array); - $visit_one!(index); - $visit_one!(val); + $visit_one!(*array); + $visit_one!(*index); + $visit_one!(*val); } Insn::ArrayPop { array, state } => { - $visit_one!(array); - $visit_one!(state); + $visit_one!(*array); + $visit_one!(*state); } Insn::ArrayLength { array } => { - $visit_one!(array); + $visit_one!(*array); } Insn::AdjustBounds { index, length } => { - $visit_one!(index); - $visit_one!(length); + $visit_one!(*index); + $visit_one!(*length); } Insn::HashAref { hash, key, state } => { - $visit_one!(hash); - $visit_one!(key); - $visit_one!(state); + $visit_one!(*hash); + $visit_one!(*key); + $visit_one!(*state); } Insn::HashAset { hash, key, val, state } => { - $visit_one!(hash); - $visit_one!(key); - $visit_one!(val); - $visit_one!(state); + $visit_one!(*hash); + $visit_one!(*key); + $visit_one!(*val); + $visit_one!(*state); } Insn::Send { recv, args, state, .. } | Insn::SendForward { recv, args, state, .. } - | Insn::CCallVariadic { recv, args, state, .. } - | Insn::CCallWithFrame { recv, args, state, .. } | Insn::SendDirect { recv, args, state, .. } | Insn::PushInlineFrame { recv, args, state, .. } | Insn::InvokeBuiltin { recv, args, state, .. } | Insn::InvokeSuper { recv, args, state, .. } | Insn::InvokeSuperForward { recv, args, state, .. } | Insn::InvokeProc { recv, args, state, .. } => { - $visit_one!(recv); + $visit_one!(*recv); $visit_many!(args); - $visit_one!(state); + $visit_one!(*state); + } + // CCallWithFrame/CCallVariadic carry their operands behind a Box, which + // stable Rust can't destructure in a pattern. visit_one takes a place, so + // a box field works the same as the deref'd bindings used by other arms. + Insn::CCallWithFrame(insn) => { + $visit_one!(insn.recv); + $visit_many!(insn.args); + $visit_one!(insn.state); + } + Insn::CCallVariadic(insn) => { + $visit_one!(insn.recv); + $visit_many!(insn.args); + $visit_one!(insn.state); } Insn::InvokeBlock { args, state, .. } => { $visit_many!(args); - $visit_one!(state); + $visit_one!(*state); } Insn::InvokeBlockIfunc { block_handler, args, state, .. } => { - $visit_one!(block_handler); + $visit_one!(*block_handler); $visit_many!(args); - $visit_one!(state); + $visit_one!(*state); } Insn::CCall { recv, args, .. } => { - $visit_one!(recv); + $visit_one!(*recv); $visit_many!(args); } Insn::GetIvar { self_val, state, .. } | Insn::DefinedIvar { self_val, state, .. } => { - $visit_one!(self_val); - $visit_one!(state); + $visit_one!(*self_val); + $visit_one!(*state); } Insn::GetConstant { klass, allow_nil, state, .. } => { - $visit_one!(klass); - $visit_one!(allow_nil); - $visit_one!(state); + $visit_one!(*klass); + $visit_one!(*allow_nil); + $visit_one!(*state); } Insn::SetIvar { self_val, val, state, .. } => { - $visit_one!(self_val); - $visit_one!(val); - $visit_one!(state); + $visit_one!(*self_val); + $visit_one!(*val); + $visit_one!(*state); } Insn::GetClassVar { state, .. } | Insn::PopInlineFrame { state, .. } => { - $visit_one!(state); + $visit_one!(*state); } Insn::SetClassVar { val, state, .. } => { - $visit_one!(val); - $visit_one!(state); + $visit_one!(*val); + $visit_one!(*state); } Insn::ArrayPush { array, val, state } => { - $visit_one!(array); - $visit_one!(val); - $visit_one!(state); + $visit_one!(*array); + $visit_one!(*val); + $visit_one!(*state); } Insn::AnyToString { val, str, state, .. } => { - $visit_one!(val); - $visit_one!(str); - $visit_one!(state); + $visit_one!(*val); + $visit_one!(*str); + $visit_one!(*state); } Insn::LoadField { recv, .. } => { - $visit_one!(recv); + $visit_one!(*recv); } Insn::StoreField { recv, val, .. } | Insn::WriteBarrier { recv, val } => { - $visit_one!(recv); - $visit_one!(val); + $visit_one!(*recv); + $visit_one!(*val); } Insn::GetGlobal { state, .. } | Insn::GetSpecialSymbol { state, .. } | Insn::GetSpecialNumber { state, .. } | Insn::ObjectAllocClass { state, .. } | Insn::SideExit { state, .. } => { - $visit_one!(state); + $visit_one!(*state); } Insn::UnboxFixnum { val } => { - $visit_one!(val); + $visit_one!(*val); } Insn::FixnumAref { recv, index } => { - $visit_one!(recv); - $visit_one!(index); + $visit_one!(*recv); + $visit_one!(*index); } Insn::IsA { val, class } => { - $visit_one!(val); - $visit_one!(class); + $visit_one!(*val); + $visit_one!(*class); } } }; @@ -1565,21 +1584,21 @@ impl Insn { /// Call `f` on each operand (InsnId) of this instruction. pub fn for_each_operand(&self, mut f: impl FnMut(InsnId)) { - macro_rules! visit_one { ($id:expr) => { f(*$id) }; } + macro_rules! visit_one { ($p:expr) => { f($p) }; } macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id) } }; } for_each_operand_impl!(self, visit_one, visit_many); } /// Call `f` on a mutable reference to each operand (InsnId) of this instruction. pub fn for_each_operand_mut(&mut self, mut f: impl FnMut(&mut InsnId)) { - macro_rules! visit_one { ($id:expr) => { f($id) }; } + macro_rules! visit_one { ($p:expr) => { f(&mut $p) }; } macro_rules! visit_many { ($s:expr) => { for id in ($s).iter_mut() { f(id) } }; } for_each_operand_impl!(self, visit_one, visit_many); } /// Call `f` on each operand, short-circuiting on the first error. pub fn try_for_each_operand(&self, mut f: impl FnMut(InsnId) -> Result<(), E>) -> Result<(), E> { - macro_rules! visit_one { ($id:expr) => { f(*$id)? }; } + macro_rules! visit_one { ($p:expr) => { f($p)? }; } macro_rules! visit_many { ($s:expr) => { for id in ($s).iter() { f(*id)? } }; } for_each_operand_impl!(self, visit_one, visit_many); Ok(()) @@ -1693,15 +1712,15 @@ impl Insn { effects::Any } }, - Insn::CCallWithFrame { elidable, .. } => { - if *elidable { + Insn::CCallWithFrame(insn) => { + if insn.elidable { Effect::write(abstract_heaps::Allocator) } else { effects::Any } }, - Insn::CCallVariadic { .. } => effects::Any, + Insn::CCallVariadic(_) => effects::Any, Insn::Send { .. } => effects::Any, Insn::SendForward { .. } => effects::Any, Insn::InvokeSuper { .. } => effects::Any, @@ -2092,7 +2111,7 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { Ok(()) } Insn::InvokeBuiltin { bf, args, leaf, .. } => { - let bf_name = unsafe { CStr::from_ptr(bf.name) }.to_str().unwrap(); + let bf_name = unsafe { CStr::from_ptr((**bf).name) }.to_str().unwrap(); write!(f, "InvokeBuiltin{} {}", if *leaf { " leaf" } else { "" }, // e.g. Code that use `Primitive.cexpr!`. From BUILTIN_INLINE_PREFIX. @@ -2173,7 +2192,8 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { write_separated!(f, ", ", ", ", args); Ok(()) }, - Insn::CCallWithFrame { cfunc, recv, args, name, cme, block, .. } => { + Insn::CCallWithFrame(insn) => { + let CCallWithFrameData { cfunc, recv, args, name, cme, block, .. } = &**insn; write!(f, "CCallWithFrame {recv}, :{}@{:p}", qualified_method_name(unsafe { (**cme).owner }, *name), self.ptr_map.map_ptr(cfunc))?; write_separated!(f, ", ", ", ", args); match block { @@ -2185,7 +2205,8 @@ impl<'a> std::fmt::Display for InsnPrinter<'a> { } Ok(()) }, - Insn::CCallVariadic { cfunc, recv, args, name, cme, .. } => { + Insn::CCallVariadic(insn) => { + let CCallVariadicData { cfunc, recv, args, name, cme, .. } = &**insn; write!(f, "CCallVariadic {recv}, :{}@{:p}", qualified_method_name(unsafe { (**cme).owner }, *name), self.ptr_map.map_ptr(cfunc))?; write_separated!(f, ", ", ", ", args); Ok(()) @@ -2606,7 +2627,7 @@ enum IseqReturn { LocalVariable(u32), Receiver, // Builtin descriptor and return type (if known) - InvokeLeafBuiltin(rb_builtin_function, Option), + InvokeLeafBuiltin(*const rb_builtin_function, Option), } unsafe extern "C" { @@ -2668,15 +2689,15 @@ fn iseq_get_return_value(iseq: IseqPtr, captured_opnd: Option, ci_flags: YARVINSN_putself if captured_opnd.is_none() => Some(IseqReturn::Receiver), YARVINSN_opt_invokebuiltin_delegate_leave => { let pc = unsafe { rb_iseq_pc_at_idx(iseq, 0) }; - let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; - let argc = bf.argc as usize; + let bf: *const rb_builtin_function = get_arg(pc, 0).as_ptr(); + let argc = unsafe { (*bf).argc } as usize; if argc != 0 { return None; } let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; let leaf = builtin_attrs & BUILTIN_ATTR_LEAF != 0; if !leaf { return None; } // Check if this builtin is annotated let return_type = ZJITState::get_method_annotations() - .get_builtin_properties(&bf) + .get_builtin_properties(bf) .map(|props| props.return_type); Some(IseqReturn::InvokeLeafBuiltin(bf, return_type)) } @@ -2762,7 +2783,7 @@ impl Function { /// Return a FrameState at the given instruction index. pub fn frame_state(&self, insn_id: InsnId) -> FrameState { match self.find(insn_id) { - Insn::Snapshot { state } => state, + Insn::Snapshot { state } => *state, insn => panic!("Unexpected non-Snapshot {insn} when looking up FrameState"), } } @@ -3026,9 +3047,9 @@ impl Function { Insn::NewRangeFixnum { .. } => types::RangeExact, Insn::ObjectAlloc { .. } => types::HeapBasicObject, Insn::ObjectAllocClass { class, .. } => Type::from_class(*class), - &Insn::CCallWithFrame { return_type, .. } => return_type, + Insn::CCallWithFrame(insn) => insn.return_type, Insn::CCall { return_type, .. } => *return_type, - &Insn::CCallVariadic { return_type, .. } => return_type, + Insn::CCallVariadic(insn) => insn.return_type, Insn::CheckMatch { .. } => types::BasicObject, Insn::GuardType { val, guard_type, .. } => self.type_of(*val).intersection(*guard_type), Insn::RefineType { val, new_type, .. } => self.type_of(*val).intersection(*new_type), @@ -3274,7 +3295,7 @@ impl Function { // If args were reordered or synthesized, create a new snapshot with the updated stack let send_state = if processed_args != args { let new_state = self.frame_state(state).with_replaced_args(&processed_args, caller_argc); - self.push_insn(block, Insn::Snapshot { state: new_state }) + self.push_insn(block, Insn::Snapshot { state: Box::new(new_state) }) } else { state }; @@ -4280,7 +4301,7 @@ impl Function { if get_option!(stats) { self.count_not_inlined_cfunc(block, super_cme); } - self.push_insn(block, Insn::CCallWithFrame { + self.push_insn(block, Insn::CCallWithFrame(Box::new(CCallWithFrameData { cd, cfunc: cfunc_ptr, recv, @@ -4291,7 +4312,7 @@ impl Function { return_type, elidable, block: None, - }) + }))) }; self.make_equal_to(insn_id, ccall); } @@ -4330,7 +4351,7 @@ impl Function { if get_option!(stats) { self.count_not_inlined_cfunc(block, super_cme); } - self.push_insn(block, Insn::CCallVariadic { + self.push_insn(block, Insn::CCallVariadic(Box::new(CCallVariadicData { cfunc: cfunc_ptr, recv, args: args.clone(), @@ -4340,7 +4361,7 @@ impl Function { return_type, elidable, block: None, - }) + }))) }; self.make_equal_to(insn_id, ccall); } @@ -4958,7 +4979,7 @@ impl Function { let mut reload_state = state.clone(); reload_state.insn_idx = insn_idx as usize; reload_state.pc = unsafe { rb_iseq_pc_at_idx(iseq, insn_idx) }; - let reload_exit_id = self.push_insn(block, Insn::Snapshot { state: reload_state.without_locals() }); + let reload_exit_id = self.push_insn(block, Insn::Snapshot { state: Box::new(reload_state.without_locals()) }); self.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: reload_exit_id }); } @@ -5152,7 +5173,7 @@ impl Function { if get_option!(stats) { fun.count_not_inlined_cfunc(block, cme); } - let ccall = fun.push_insn(block, Insn::CCallWithFrame { + let ccall = fun.push_insn(block, Insn::CCallWithFrame(Box::new(CCallWithFrameData { cd, cfunc: cfunc_ptr, recv, @@ -5163,7 +5184,7 @@ impl Function { return_type, elidable, block: blockiseq.map(BlockHandler::BlockIseq), - }); + }))); fun.make_equal_to(send_insn_id, ccall); Ok(()) } @@ -5219,7 +5240,7 @@ impl Function { fun.count_not_inlined_cfunc(block, cme); } - let ccall = fun.push_insn(block, Insn::CCallVariadic { + let ccall = fun.push_insn(block, Insn::CCallVariadic(Box::new(CCallVariadicData { cfunc: cfunc_ptr, recv, args, @@ -5229,7 +5250,7 @@ impl Function { return_type, elidable, block: blockiseq.map(BlockHandler::BlockIseq), - }); + }))); fun.make_equal_to(send_insn_id, ccall); Ok(()) @@ -5256,7 +5277,7 @@ impl Function { } } Insn::InvokeBuiltin { bf, recv, args, state, .. } => { - let props = ZJITState::get_method_annotations().get_builtin_properties(&bf).unwrap_or_default(); + let props = ZJITState::get_method_annotations().get_builtin_properties(bf).unwrap_or_default(); // Try inlining the cfunc into HIR let tmp_block = self.new_block(u32::MAX); if let Some(replacement) = (props.inline)(self, tmp_block, recv, &args, state) { @@ -6451,8 +6472,6 @@ impl Function { | Insn::SendForward { recv, ref args, .. } | Insn::InvokeSuper { recv, ref args, .. } | Insn::InvokeSuperForward { recv, ref args, .. } - | Insn::CCallWithFrame { recv, ref args, .. } - | Insn::CCallVariadic { recv, ref args, .. } | Insn::InvokeBuiltin { recv, ref args, .. } | Insn::InvokeProc { recv, ref args, .. } | Insn::ArrayInclude { target: recv, elements: ref args, .. } => { @@ -6462,6 +6481,20 @@ impl Function { } Ok(()) } + Insn::CCallWithFrame(insn) => { + self.assert_subtype(insn_id, insn.recv, types::BasicObject)?; + for &arg in &insn.args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } + Insn::CCallVariadic(insn) => { + self.assert_subtype(insn_id, insn.recv, types::BasicObject)?; + for &arg in &insn.args { + self.assert_subtype(insn_id, arg, types::BasicObject)?; + } + Ok(()) + } Insn::ArrayPackBuffer { ref elements, fmt, buffer, .. } => { self.assert_subtype(insn_id, fmt, types::BasicObject)?; if let Some(buffer) = buffer { @@ -7369,7 +7402,7 @@ fn add_iseq_to_hir( // Start the block off with a Snapshot so that if we need to insert a new Guard later on // and we don't have a Snapshot handy, we can just iterate backward (at the earliest, to // the beginning of the block). - fun.push_insn(block, Insn::Snapshot { state: state.clone() }); + fun.push_insn(block, Insn::Snapshot { state: Box::new(state.clone()) }); while insn_idx < iseq_size { state.insn_idx = insn_idx as usize; // Get the current pc and opcode @@ -7389,7 +7422,7 @@ fn add_iseq_to_hir( unsafe extern "C" { fn rb_iseq_event_flags(iseq: IseqPtr, pos: usize) -> rb_event_flag_t; } - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.clone() }); + let exit_id = fun.push_insn(block, Insn::Snapshot { state: Box::new(exit_state.clone()) }); // If TracePoint has been enabled after we have collected profiles, we'll see // trace_getinstancevariable in the ISEQ. We have to treat it like getinstancevariable @@ -7807,7 +7840,7 @@ fn add_iseq_to_hir( let ep = fun.push_insn(block, Insn::GetEP { level: 0 }); fun.get_local_from_ep(block, iseq, ep, ep_offset, 0, types::BasicObject) } else { - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); + let exit_id = fun.push_insn(block, Insn::Snapshot { state: Box::new(exit_state.without_locals()) }); fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); local_inval = false; state.getlocal(ep_offset) @@ -8017,7 +8050,7 @@ fn add_iseq_to_hir( assert!(local_inval); // if check above // There has been some non-leaf call since JIT entry or the last patch point, // so add a patch point to make sure locals have not been escaped. - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + let exit_id = fun.push_insn(block, Insn::Snapshot { state: Box::new(exit_state.without_locals()) }); // skip spilling locals fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); local_inval = false; @@ -8035,7 +8068,7 @@ fn add_iseq_to_hir( } else if local_inval { // If there has been any non-leaf call since JIT entry or the last patch point, // add a patch point to make sure locals have not been escaped. - let exit_id = fun.push_insn(block, Insn::Snapshot { state: exit_state.without_locals() }); // skip spilling locals + let exit_id = fun.push_insn(block, Insn::Snapshot { state: Box::new(exit_state.without_locals()) }); // skip spilling locals fun.push_insn(block, Insn::PatchPoint { invariant: Invariant::NoEPEscape(iseq), state: exit_id }); local_inval = false; } @@ -8606,7 +8639,7 @@ fn add_iseq_to_hir( // reusing exit_id so type specialization resolves the receiver from // its refined, exact type instead of the polymorphic profile that is // keyed at exit_id. - let snapshot = fun.push_insn(iftrue_block, Insn::Snapshot { state: exit_state.clone() }); + let snapshot = fun.push_insn(iftrue_block, Insn::Snapshot { state: Box::new(exit_state.clone()) }); let refined_recv = fun.push_insn(iftrue_block, Insn::RefineType { val: recv, new_type: expected }); let send = fun.push_insn(iftrue_block, Insn::Send { recv: refined_recv, cd, block: None, args: args.clone(), state: snapshot, reason: Uncategorized(opcode) }); fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); @@ -8967,7 +9000,7 @@ fn add_iseq_to_hir( if let Some(profiled_type) = fun.monomorphic_summary(&profiles, self_param, exit_id) { let result = fun.try_emit_optimized_getivar(block, self_param, id, profiled_type, exit_id).unwrap_or_else(|counter| { fun.count(block, counter); - fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic: std::ptr::null(), state: exit_id }) + fun.push_insn(block, Insn::GetIvar { self_val: self_param, id, ic, state: exit_id }) }); state.stack_push(result); } else { @@ -9034,16 +9067,16 @@ fn add_iseq_to_hir( state.stack_push(insn_id); } YARVINSN_invokebuiltin => { - let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let bf: *const rb_builtin_function = get_arg(pc, 0).as_ptr(); // TODO: Support passing arguments on the stack in C calls // +2 for ec, self - if (bf.argc + 2) as usize > C_ARG_OPNDS.len() { + if (unsafe { (*bf).argc } + 2) as usize > C_ARG_OPNDS.len() { fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyArgsForLir, recompile: None }); break; // End the block } let mut args = vec![]; - for _ in 0..bf.argc { + for _ in 0..unsafe { (*bf).argc } { args.push(state.stack_pop()?); } args.push(self_param); @@ -9051,7 +9084,7 @@ fn add_iseq_to_hir( // Check if this builtin is annotated let return_type = ZJITState::get_method_annotations() - .get_builtin_properties(&bf) + .get_builtin_properties(bf) .map(|props| props.return_type); let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; @@ -9069,16 +9102,16 @@ fn add_iseq_to_hir( } YARVINSN_opt_invokebuiltin_delegate | YARVINSN_opt_invokebuiltin_delegate_leave => { - let bf: rb_builtin_function = unsafe { *get_arg(pc, 0).as_ptr() }; + let bf: *const rb_builtin_function = get_arg(pc, 0).as_ptr(); // TODO: Support passing arguments on the stack in C calls // +2 for ec, self - if (bf.argc + 2) as usize > C_ARG_OPNDS.len() { + let argc = unsafe { (*bf).argc } as usize; + if argc + 2 > C_ARG_OPNDS.len() { fun.push_insn(block, Insn::SideExit { state: exit_id, reason: SideExitReason::TooManyArgsForLir, recompile: None }); break; // End the block } let index = get_arg(pc, 1).as_usize(); - let argc = bf.argc as usize; let mut args = vec![self_param]; for &local in state.locals().skip(index).take(argc) { @@ -9087,7 +9120,7 @@ fn add_iseq_to_hir( // Check if this builtin is annotated let return_type = ZJITState::get_method_annotations() - .get_builtin_properties(&bf) + .get_builtin_properties(bf) .map(|props| props.return_type); let builtin_attrs = unsafe { rb_jit_iseq_builtin_attrs(iseq) }; diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index 9eb83b3ea9b6c7..426352b75c9f82 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -2520,7 +2520,7 @@ mod hir_opt_tests { let mut function = Function::new(std::ptr::null()); let entry = function.entry_block; - let state = function.push_insn(entry, Insn::Snapshot { state: FrameState::new(std::ptr::null()) }); + let state = function.push_insn(entry, Insn::Snapshot { state: Box::new(FrameState::new(std::ptr::null())) }); // A nil constant is a NilClass, which is disjoint from Fixnum, so the guard below can // never pass and the optimizer infers its result as Empty. let nil = function.push_insn(entry, Insn::Const { val: Const::Value(Qnil) }); diff --git a/zjit/src/hir/tests.rs b/zjit/src/hir/tests.rs index 0d44650486b4d6..8289a43d7f924e 100644 --- a/zjit/src/hir/tests.rs +++ b/zjit/src/hir/tests.rs @@ -1,6 +1,21 @@ #[cfg(test)] use super::*; +#[cfg(test)] +mod size_tests { + use super::*; + + #[test] + fn test_size_of_insn() { + assert_eq!(std::mem::size_of::(), 88); + } + + #[test] + fn test_size_of_type() { + assert_eq!(std::mem::size_of::(), 16); + } +} + #[cfg(test)] mod snapshot_tests { use super::*; diff --git a/zjit/src/hir_type/mod.rs b/zjit/src/hir_type/mod.rs index ef67de616e0c34..d005562d9ef4ab 100644 --- a/zjit/src/hir_type/mod.rs +++ b/zjit/src/hir_type/mod.rs @@ -43,6 +43,33 @@ pub enum Specialization { Empty, } +/// The integer type backing `Type::bits`. The masks and shift below derive their +/// width from this, so changing it updates everything. +type Bits = u64; + +/// Number of high bits of `Type::bits` stolen for the specialization tag. +const NUM_SPEC_BITS: u32 = 3; +const SPEC_SHIFT: u32 = Bits::BITS - NUM_SPEC_BITS; +/// Mask that isolates the spec tag (the top `NUM_SPEC_BITS` bits). +const SPEC_MASK: Bits = (((1 as Bits) << NUM_SPEC_BITS) - 1) << SPEC_SHIFT; +/// Mask that isolates the type-lattice bits (everything below the spec tag). +const BITS_MASK: Bits = !SPEC_MASK; + +/// Tag stored in the top `NUM_SPEC_BITS` of `Type::bits`, identifying which +/// [`Specialization`] variant the `spec_val` payload belongs to. `repr(u64)` pins +/// the discriminants to `Bits`-wide values for the `as Bits` casts in +/// `Type::new`/`Type::spec`. +#[repr(u64)] +enum SpecTag { + Any = 0, + Type, + TypeExact, + Object, + Int, + Double, + Empty, +} + // NOTE: Type very intentionally does not support Eq or PartialEq; we almost never want to check // bit equality of types in the compiler but instead check subtyping, intersection, union, etc. #[derive(Copy, Clone, Debug)] @@ -53,6 +80,10 @@ pub enum Specialization { /// * union/meet type A and type B /// /// Most questions can be rewritten in terms of these operations. +/// +/// Packed layout (16 bytes total): +/// - `bits`: low `NumTypeBits` bits are the type lattice bitset, bits 61-63 encode the [`Specialization`] tag +/// - `spec_val`: raw 8-byte payload for the specialization (VALUE bits, u64, or f64 bits) pub struct Type { /// A bitset representing type information about the object. Specific bits are assigned for /// leaf types (for example, static symbols) and union-ing bitsets together represents @@ -62,18 +93,24 @@ pub struct Type { /// Capable of also representing cvalue types (bool, i32, etc). /// /// This field should not be directly read or written except by internal `Type` APIs. - bits: u64, - /// Specialization of the type. See [`Specialization`]. + /// + /// The top `NUM_SPEC_BITS` are stolen for encoding the specialization tag. + bits: Bits, + /// Payload for `Specialization`. /// /// This field should not be directly read or written except by internal `Type` APIs. - spec: Specialization + spec_val: u64, } include!("hir_type.inc.rs"); +// Compile-time check that type lattice bits don't overlap with the spec tag. +const _: () = assert!(bits::NumTypeBits <= SPEC_SHIFT as u64, + "type lattice bits overlap with spec tag bits"); + fn write_spec(f: &mut std::fmt::Formatter, printer: &TypePrinter) -> std::fmt::Result { let ty = printer.inner; - match ty.spec { + match ty.spec() { Specialization::Any | Specialization::Empty => { Ok(()) }, Specialization::Object(val) if val == unsafe { rb_mRubyVMFrozenCore } => write!(f, "[VMFrozenCore]"), Specialization::Object(val) if val == unsafe { rb_block_param_proxy } => write!(f, "[BlockParamProxy]"), @@ -131,13 +168,13 @@ impl<'a> std::fmt::Display for TypePrinter<'a> { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { let ty = self.inner; for (name, pattern) in bits::AllBitPatterns { - if ty.bits == pattern { + if ty.type_bits() == pattern { write!(f, "{name}")?; return write_spec(f, self); } } assert!(bits::AllBitPatterns.is_sorted_by(|(_, left), (_, right)| left > right)); - let mut bits = ty.bits; + let mut bits = ty.type_bits(); let mut sep = ""; for (name, pattern) in bits::AllBitPatterns { if bits == 0 { break; } @@ -179,13 +216,49 @@ fn is_range_exact(val: VALUE) -> bool { impl Type { /// Create a `Type` from the given integer. - pub const fn fixnum(val: i64) -> Type { - Type { - bits: bits::Fixnum, - spec: Specialization::Object(VALUE::fixnum_from_usize(val as usize)), + /// Construct a Type from raw lattice bits and a Specialization. + const fn new(bits: u64, spec: Specialization) -> Type { + let (tag, val) = match spec { + Specialization::Any => (SpecTag::Any, 0u64), + Specialization::Type(v) => (SpecTag::Type, v.as_usize() as u64), + Specialization::TypeExact(v) => (SpecTag::TypeExact, v.as_usize() as u64), + Specialization::Object(v) => (SpecTag::Object, v.as_usize() as u64), + Specialization::Int(v) => (SpecTag::Int, v), + Specialization::Double(v) => (SpecTag::Double, v.to_bits()), + Specialization::Empty => (SpecTag::Empty, 0u64), + }; + debug_assert!(bits & SPEC_MASK == 0, "type lattice bits overlap with spec tag"); + Type { bits: ((tag as Bits) << SPEC_SHIFT) | bits, spec_val: val } + } + + /// Return the type-lattice bits (low 61 bits, excluding the spec tag). + const fn type_bits(&self) -> u64 { + self.bits & BITS_MASK + } + + const fn spec_bits(&self) -> u64 { + self.bits >> SPEC_SHIFT + } + + /// Reconstruct the Specialization enum from the packed representation. + pub fn spec(&self) -> Specialization { + match self.spec_bits() { + t if t == SpecTag::Any as Bits => Specialization::Any, + t if t == SpecTag::Type as Bits => Specialization::Type(VALUE(self.spec_val as usize)), + t if t == SpecTag::TypeExact as Bits => Specialization::TypeExact(VALUE(self.spec_val as usize)), + t if t == SpecTag::Object as Bits => Specialization::Object(VALUE(self.spec_val as usize)), + t if t == SpecTag::Int as Bits => Specialization::Int(self.spec_val), + t if t == SpecTag::Double as Bits => Specialization::Double(f64::from_bits(self.spec_val)), + t if t == SpecTag::Empty as Bits => Specialization::Empty, + _ => unreachable!(), } } + /// Create a `Type` from the given integer. + pub const fn fixnum(val: i64) -> Type { + Type::new(bits::Fixnum, Specialization::Object(VALUE::fixnum_from_usize(val as usize))) + } + fn bits_from_exact_class(class: VALUE) -> Option { types::ExactBitsAndClass .iter() @@ -218,8 +291,7 @@ impl Type { unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", get_class_name(val.class_of())) }; - let spec = Specialization::Object(val); - Type { bits, spec } + Type::new(bits, Specialization::Object(val)) } /// Create a `Type` from a Ruby `VALUE`. The type is not guaranteed to have object @@ -228,13 +300,13 @@ impl Type { pub fn from_value(val: VALUE) -> Type { // Immediates if val.fixnum_p() { - Type { bits: bits::Fixnum, spec: Specialization::Object(val) } + Type::new(bits::Fixnum, Specialization::Object(val)) } else if val.flonum_p() { - Type { bits: bits::Flonum, spec: Specialization::Object(val) } + Type::new(bits::Flonum, Specialization::Object(val)) } else if val.static_sym_p() { - Type { bits: bits::StaticSymbol, spec: Specialization::Object(val) } + Type::new(bits::StaticSymbol, Specialization::Object(val)) } // Singleton objects; don't specialize else if val == Qnil { types::NilClass } @@ -243,7 +315,7 @@ impl Type { else if val.cme_p() { // NB: Checking for CME has to happen before looking at class_of because that's not // valid on imemo. - Type { bits: bits::CallableMethodEntry, spec: Specialization::Object(val) } + Type::new(bits::CallableMethodEntry, Specialization::Object(val)) } else { Self::from_heap_object(val) @@ -285,7 +357,7 @@ impl Type { return Type::from_bits(bits); } if let Some(bits) = Self::bits_from_subclass(class) { - return Type { bits, spec: Specialization::TypeExact(class) } + return Type::new(bits, Specialization::TypeExact(class)) } unreachable!("Class {} is not a subclass of BasicObject! Don't know what to do.", get_class_name(class)) @@ -296,44 +368,41 @@ impl Type { .iter() .find(|&(_, class_object)| class.is_subclass_of(unsafe { **class_object }) == ClassRelationship::Subclass) .unwrap_or_else(|| panic!("Class {} is not a subclass of BasicObject! Don't know what to do.", get_class_name(class))).0; - Type { bits, spec: Specialization::Type(class) } + Type::new(bits, Specialization::Type(class)) } /// Private. Only for creating type globals. const fn from_bits(bits: u64) -> Type { - Type { - bits, - spec: if bits == bits::Empty { - Specialization::Empty - } else { - Specialization::Any - }, - } + Type::new(bits, if bits == bits::Empty { + Specialization::Empty + } else { + Specialization::Any + }) } /// Create a `Type` from a cvalue integer. Use the `ty` given to specify what size the /// `specialization` represents. For example, `Type::from_cint(types::CBool, 1)` or /// `Type::from_cint(types::CUInt16, 12)`. pub fn from_cint(ty: Type, val: i64) -> Type { - assert_eq!(ty.spec, Specialization::Any); + assert_eq!(ty.spec(), Specialization::Any); assert!((ty.is_subtype(types::CUnsigned) || ty.is_subtype(types::CSigned)) && - ty.bits != types::CUnsigned.bits && ty.bits != types::CSigned.bits, + ty.type_bits() != types::CUnsigned.type_bits() && ty.type_bits() != types::CSigned.type_bits(), "ty must be a specific int size"); - Type { bits: ty.bits, spec: Specialization::Int(val as u64) } + Type::new(ty.type_bits(), Specialization::Int(val as u64)) } pub fn from_cptr(val: *const u8) -> Type { - Type { bits: bits::CPtr, spec: Specialization::Int(val as u64) } + Type::new(bits::CPtr, Specialization::Int(val as u64)) } /// Create a `Type` (a `CDouble` with double specialization) from a f64. pub fn from_double(val: f64) -> Type { - Type { bits: bits::CDouble, spec: Specialization::Double(val) } + Type::new(bits::CDouble, Specialization::Double(val)) } /// Create a `Type` from a cvalue boolean. pub fn from_cbool(val: bool) -> Type { - Type { bits: bits::CBool, spec: Specialization::Int(val as u64) } + Type::new(bits::CBool, Specialization::Int(val as u64)) } /// Return true if the value with this type is definitely truthy. @@ -347,7 +416,7 @@ impl Type { } pub fn has_value(&self, val: Const) -> bool { - match (self.spec, val) { + match (self.spec(), val) { (Specialization::Object(v1), Const::Value(v2)) => v1 == v2, (Specialization::Int(v1), Const::CBool(v2)) if self.is_subtype(types::CBool) => v1 == (v2 as u64), (Specialization::Int(v1), Const::CInt8(v2)) if self.is_subtype(types::CInt8) => v1 == (v2 as u64), @@ -367,35 +436,14 @@ impl Type { /// Return the object specialization, if any. pub fn ruby_object(&self) -> Option { - match self.spec { - Specialization::Object(val) => Some(val), - _ => None, - } - } - - /// Return a Ruby object that needs to be marked on GC. - /// This covers Type and TypeExact unlike ruby_object(). - pub fn gc_object(&self) -> Option { - match self.spec { - Specialization::Type(val) | - Specialization::TypeExact(val) | - Specialization::Object(val) => Some(val), - _ => None, - } - } - - /// Mutable version of gc_object(). - pub fn gc_object_mut(&mut self) -> Option<&mut VALUE> { - match &mut self.spec { - Specialization::Type(val) | - Specialization::TypeExact(val) | + match self.spec() { Specialization::Object(val) => Some(val), _ => None, } } pub fn unspecialized(&self) -> Self { - Type { spec: Specialization::Any, ..*self } + Type::new(self.type_bits(), Specialization::Any) } pub fn fixnum_value(&self) -> Option { @@ -407,15 +455,15 @@ impl Type { } pub fn cint64_value(&self) -> Option { - match (self.is_subtype(types::CInt64), &self.spec) { - (true, Specialization::Int(val)) => Some(*val as i64), + match (self.is_subtype(types::CInt64), self.spec()) { + (true, Specialization::Int(val)) => Some(val as i64), _ => None, } } fn int_spec_signed(&self) -> Option { assert!(self.is_subtype(types::CSigned), "int_spec_signed() only makes sense for signed integer types"); - match self.spec { + match self.spec() { Specialization::Int(val) => Some(val as i64), _ => None, } @@ -428,7 +476,7 @@ impl Type { /// Return true if the Type has object specialization and false otherwise. pub fn ruby_object_known(&self) -> bool { - matches!(self.spec, Specialization::Object(_)) + matches!(self.spec(), Specialization::Object(_)) } /// Find a `T_*` type that is exactly as wide as `self`. @@ -459,7 +507,7 @@ impl Type { // Easy cases first if self.is_subtype(other) { return other; } if other.is_subtype(*self) { return *self; } - let bits = self.bits | other.bits; + let bits = self.type_bits() | other.type_bits(); let result = Type::from_bits(bits); // If one type isn't type specialized, we can't return a specialized Type if !self.type_known() || !other.type_known() { return result; } @@ -478,19 +526,19 @@ impl Type { if let Some(self_class) = self.exact_ruby_class() { if let Some(other_class) = other.exact_ruby_class() { if self_class == other_class { - return Type { bits, spec: Specialization::TypeExact(self_class) }; + return Type::new(bits, Specialization::TypeExact(self_class)); } } } - Type { bits, spec: Specialization::Type(super_class) } + Type::new(bits, Specialization::Type(super_class)) } /// Intersect both types, preserving specialization if possible. pub fn intersection(&self, other: Type) -> Type { - let bits = self.bits & other.bits; + let bits = self.type_bits() & other.type_bits(); if bits == bits::Empty { return types::Empty; } - if self.spec_is_subtype_of(other) { return Type { bits, spec: self.spec }; } - if other.spec_is_subtype_of(*self) { return Type { bits, spec: other.spec }; } + if self.spec_is_subtype_of(other) { return Type::new(bits, self.spec()); } + if other.spec_is_subtype_of(*self) { return Type::new(bits, other.spec()); } types::Empty } @@ -501,27 +549,27 @@ impl Type { /// Check if the type field of `self` is a subtype of the type field of `other` and also check /// if the specialization of `self` is a subtype of the specialization of `other`. pub fn is_subtype(&self, other: Type) -> bool { - (self.bits & other.bits) == self.bits && self.spec_is_subtype_of(other) + (self.type_bits() & other.type_bits()) == self.type_bits() && self.spec_is_subtype_of(other) } /// Return the type specialization, if any. Type specialization asks if we know the Ruby type /// (including potentially its subclasses) corresponding to a `Type`, including knowing exactly /// what object is is. pub fn type_known(&self) -> bool { - matches!(self.spec, Specialization::TypeExact(_) | Specialization::Type(_) | Specialization::Object(_)) + matches!(self.spec(), Specialization::TypeExact(_) | Specialization::Type(_) | Specialization::Object(_)) } /// Return the exact type specialization, if any. Type specialization asks if we know the /// *exact* Ruby type corresponding to a `Type`, including knowing exactly what object is is. pub fn exact_class_known(&self) -> bool { - matches!(self.spec, Specialization::TypeExact(_) | Specialization::Object(_)) + matches!(self.spec(), Specialization::TypeExact(_) | Specialization::Object(_)) } /// Return the exact type specialization, if any. Type specialization asks if we know the exact /// Ruby type corresponding to a `Type` (no subclasses), including knowing exactly what object /// it is. pub fn exact_ruby_class(&self) -> Option { - match self.spec { + match self.spec() { // If we're looking at a precise object, we can pull out its class. Specialization::Object(val) => Some(val.class_of()), Specialization::TypeExact(val) => Some(val), @@ -532,7 +580,7 @@ impl Type { /// Return the type specialization, if any. Type specialization asks if we know the inexact /// Ruby type corresponding to a `Type`, including knowing exactly what object is is. pub fn inexact_ruby_class(&self) -> Option { - match self.spec { + match self.spec() { // If we're looking at a precise object, we can pull out its class. Specialization::Object(val) => Some(val.class_of()), Specialization::TypeExact(val) | Specialization::Type(val) => Some(val), @@ -555,13 +603,13 @@ impl Type { /// Check bit equality of two `Type`s. Do not use! You are probably looking for [`Type::is_subtype`]. pub fn bit_equal(&self, other: Type) -> bool { - self.bits == other.bits && self.spec == other.spec + self.bits == other.bits && self.spec_val == other.spec_val } /// Check *only* if `self`'s specialization is a subtype of `other`'s specialization. Private. /// You probably want [`Type::is_subtype`] instead. fn spec_is_subtype_of(&self, other: Type) -> bool { - match (self.spec, other.spec) { + match (self.spec(), other.spec()) { // Empty is a subtype of everything; Any is a supertype of everything (Specialization::Empty, _) | (_, Specialization::Any) => true, // Other is not Any from the previous case, so Any is definitely not a subtype @@ -569,7 +617,7 @@ impl Type { // Int and double specialization requires exact equality (Specialization::Int(_), _) | (_, Specialization::Int(_)) | (Specialization::Double(_), _) | (_, Specialization::Double(_)) => - self.bits == other.bits && self.spec == other.spec, + self.bits == other.bits && self.spec_val == other.spec_val, // Check other's specialization type in decreasing order of specificity (_, Specialization::Object(_)) => self.ruby_object_known() && self.ruby_object() == other.ruby_object(), @@ -625,8 +673,8 @@ mod tests { #[track_caller] fn assert_bit_equal(left: Type, right: Type) { - assert_eq!(left.bits, right.bits, "{left} bits are not equal to {right} bits"); - assert_eq!(left.spec, right.spec, "{left} spec is not equal to {right} spec"); + assert_eq!(left.type_bits(), right.type_bits(), "{left} bits are not equal to {right} bits"); + assert_eq!(left.spec(), right.spec(), "{left} spec is not equal to {right} spec"); } #[track_caller] @@ -667,26 +715,26 @@ mod tests { fn from_const() { let cint32 = Type::from_const(Const::CInt32(12)); assert_subtype(cint32, types::CInt32); - assert_eq!(cint32.spec, Specialization::Int(12)); + assert_eq!(cint32.spec(), Specialization::Int(12)); assert_eq!(format!("{}", cint32), "CInt32[12]"); let cint32 = Type::from_const(Const::CInt32(-12)); assert_subtype(cint32, types::CInt32); - assert_eq!(cint32.spec, Specialization::Int((-12i64) as u64)); + assert_eq!(cint32.spec(), Specialization::Int((-12i64) as u64)); assert_eq!(format!("{}", cint32), "CInt32[-12]"); let cuint32 = Type::from_const(Const::CInt32(12)); assert_subtype(cuint32, types::CInt32); - assert_eq!(cuint32.spec, Specialization::Int(12)); + assert_eq!(cuint32.spec(), Specialization::Int(12)); let cuint32 = Type::from_const(Const::CUInt32(0xffffffff)); assert_subtype(cuint32, types::CUInt32); - assert_eq!(cuint32.spec, Specialization::Int(0xffffffff)); + assert_eq!(cuint32.spec(), Specialization::Int(0xffffffff)); assert_eq!(format!("{}", cuint32), "CUInt32[4294967295]"); let cuint32 = Type::from_const(Const::CUInt32(0xc00087)); assert_subtype(cuint32, types::CUInt32); - assert_eq!(cuint32.spec, Specialization::Int(0xc00087)); + assert_eq!(cuint32.spec(), Specialization::Int(0xc00087)); assert_eq!(format!("{}", cuint32), "CUInt32[12583047]"); } @@ -845,7 +893,7 @@ mod tests { assert_bit_equal(Type::from_class(unsafe { rb_cTrueClass }), types::TrueClass); assert_bit_equal(Type::from_class(unsafe { rb_cFalseClass }), types::FalseClass); let c_class = define_class("C", unsafe { rb_cObject }); - assert_bit_equal(Type::from_class(c_class), Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class) }); + assert_bit_equal(Type::from_class(c_class), Type::new(bits::ObjectSubclass, Specialization::TypeExact(c_class))); }); } @@ -916,7 +964,7 @@ mod tests { #[test] fn union_bits_unions_bits() { - assert_bit_equal(types::Fixnum.union(types::StaticSymbol), Type { bits: bits::Fixnum | bits::StaticSymbol, spec: Specialization::Any }); + assert_bit_equal(types::Fixnum.union(types::StaticSymbol), Type::new(bits::Fixnum | bits::StaticSymbol, Specialization::Any)); } #[test] @@ -934,8 +982,8 @@ mod tests { crate::cruby::with_rubyvm(|| { let specialized = Type::from_value(unsafe { rb_ary_new_capa(0) }); let unspecialized = types::StringExact; - assert_bit_equal(specialized.union(unspecialized), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); - assert_bit_equal(unspecialized.union(specialized), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + assert_bit_equal(specialized.union(unspecialized), Type::new(bits::ArrayExact | bits::StringExact, Specialization::Any)); + assert_bit_equal(unspecialized.union(specialized), Type::new(bits::ArrayExact | bits::StringExact, Specialization::Any)); }); } @@ -1012,7 +1060,7 @@ mod tests { crate::cruby::with_rubyvm(|| { let string = Type::from_value(rust_str_to_ruby("hello")); let array = Type::from_value(unsafe { rb_ary_new_capa(0) }); - assert_bit_equal(string.union(array), Type { bits: bits::ArrayExact | bits::StringExact, spec: Specialization::Any }); + assert_bit_equal(string.union(array), Type::new(bits::ArrayExact | bits::StringExact, Specialization::Any)); }); } @@ -1021,11 +1069,11 @@ mod tests { crate::cruby::with_rubyvm(|| { let c_class = define_class("C", unsafe { rb_cObject }); let d_class = define_class("D", c_class); - let c_instance = Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class) }; - let d_instance = Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(d_class) }; - assert_bit_equal(c_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::TypeExact(c_class)}); - assert_bit_equal(c_instance.union(d_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); - assert_bit_equal(d_instance.union(c_instance), Type { bits: bits::ObjectSubclass, spec: Specialization::Type(c_class)}); + let c_instance = Type::new(bits::ObjectSubclass, Specialization::TypeExact(c_class)); + let d_instance = Type::new(bits::ObjectSubclass, Specialization::TypeExact(d_class)); + assert_bit_equal(c_instance.union(c_instance), Type::new(bits::ObjectSubclass, Specialization::TypeExact(c_class))); + assert_bit_equal(c_instance.union(d_instance), Type::new(bits::ObjectSubclass, Specialization::Type(c_class))); + assert_bit_equal(d_instance.union(c_instance), Type::new(bits::ObjectSubclass, Specialization::Type(c_class))); }); }