class RubyVM::RJIT::InsnCompiler
Constants
- CallingInfo
struct rb_calling_info. Storing flags instead of ci.
Public Instance Methods
# File ruby_vm/rjit/insn_compiler.rb, line 5834 def assert(cond) assert_equal(cond, true) end
# File ruby_vm/rjit/insn_compiler.rb, line 5838 def assert_equal(left, right) if left != right raise "'#{left.inspect}' was not '#{right.inspect}'" end end
# File ruby_vm/rjit/insn_compiler.rb, line 5961 def build_calling(ci:, block_handler:) CallingInfo.new( argc: C.vm_ci_argc(ci), flags: C.vm_ci_flag(ci), kwarg: C.vm_ci_kwarg(ci), ci_addr: ci.to_i, send_shift: 0, block_handler:, ) end
Return a builtin function if a given iseq consists of only that builtin function
# File ruby_vm/rjit/insn_compiler.rb, line 5951 def builtin_function(iseq) opt_invokebuiltin_delegate_leave = INSNS.values.find { |i| i.name == :opt_invokebuiltin_delegate_leave } leave = INSNS.values.find { |i| i.name == :leave } if iseq.body.iseq_size == opt_invokebuiltin_delegate_leave.len + leave.len && C.rb_vm_insn_decode(iseq.body.iseq_encoded[0]) == opt_invokebuiltin_delegate_leave.bin && C.rb_vm_insn_decode(iseq.body.iseq_encoded[opt_invokebuiltin_delegate_leave.len]) == leave.bin C.rb_builtin_function.new(iseq.body.iseq_encoded[1]) end end
# File ruby_vm/rjit/insn_compiler.rb, line 5946 def c_method_tracing_currently_enabled? C.rb_rjit_global_events & (C::RUBY_EVENT_C_CALL | C::RUBY_EVENT_C_RETURN) != 0 end
# File ruby_vm/rjit/insn_compiler.rb, line 5895 def compile_jit_direct_jump(branch_stub, comment:) # Proc escapes arguments in memory proc do |branch_asm| branch_asm.comment(comment) branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.jmp(branch_stub.target0.address) in Next0 # Just write the block without a jump end end end end
# File ruby_vm/rjit/insn_compiler.rb, line 5658 def compile_jit_return(branch_stub, cfp_offset:) # Proc escapes arguments in memory proc do |branch_asm| branch_asm.comment('set jit_return to callee CFP') branch_asm.stub(branch_stub) do case branch_stub.shape in Default branch_asm.mov(:rax, branch_stub.target0.address) branch_asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:jit_return)], :rax) end end end end
# File ruby_vm/rjit/insn_compiler.rb, line 5923 def counted_exit(side_exit, name) asm = Assembler.new asm.incr_counter(name) asm.jmp(side_exit) @ocb.write(asm) end
# File ruby_vm/rjit/insn_compiler.rb, line 5930 def def_iseq_ptr(cme_def) C.rb_iseq_check(cme_def.body.iseq.iseqptr) end
@param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5872 def defer_compilation(jit, ctx, asm) # Make a stub to compile the current insn if ctx.chain_depth != 0 raise "double defer!" end ctx.chain_depth += 1 jit_direct_jump(jit.iseq, jit.pc, ctx, asm, comment: 'defer_compilation') end
# File ruby_vm/rjit/insn_compiler.rb, line 5860 def dynamic_symbol?(obj) return false if C::SPECIAL_CONST_P(obj) C.RB_TYPE_P(obj, C::RUBY_T_SYMBOL) end
# File ruby_vm/rjit/insn_compiler.rb, line 5844 def fixnum?(obj) (C.to_value(obj) & C::RUBY_FIXNUM_FLAG) == C::RUBY_FIXNUM_FLAG end
# File ruby_vm/rjit/insn_compiler.rb, line 5848 def flonum?(obj) (C.to_value(obj) & C::RUBY_FLONUM_MASK) == C::RUBY_FLONUM_FLAG end
# File ruby_vm/rjit/insn_compiler.rb, line 5939 def full_cfunc_return @full_cfunc_return ||= Assembler.new.then do |asm| @exit_compiler.compile_full_cfunc_return(asm) @ocb.write(asm) end end
vm_caller_setup_arg_block: Handle VM_CALL_ARGS_BLOCKARG cases. @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4188 def guard_block_arg(jit, ctx, asm, calling) if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 block_arg_type = ctx.get_opnd_type(StackOpnd[0]) case block_arg_type in Type::Nil calling.block_handler = C::VM_BLOCK_HANDLER_NONE in Type::BlockParamProxy calling.block_handler = C.rb_block_param_proxy else asm.incr_counter(:send_block_arg) return CantCompile end end end
vm_call_opt_send (lazy part) @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5477 def handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) # We don't support `send(:send, ...)` for now. assert_equal(1, send_shift) asm.comment('shift stack') (0...argc).reverse_each do |i| opnd = ctx.stack_opnd(i) opnd2 = ctx.stack_opnd(i + 1) asm.mov(:rax, opnd) asm.mov(opnd2, :rax) end ctx.shift_stack(argc) end
Generate RARRAY_LEN. For array_opnd, use Opnd::Reg to reduce memory access, and use Opnd::Mem to save registers.
# File ruby_vm/rjit/insn_compiler.rb, line 5804 def jit_array_len(asm, array_reg, len_reg) asm.comment('get array length for embedded or heap') # Pull out the embed flag to check if it's an embedded array. asm.mov(len_reg, [array_reg, C.RBasic.offsetof(:flags)]) # Get the length of the array asm.and(len_reg, C::RARRAY_EMBED_LEN_MASK) asm.sar(len_reg, C::RARRAY_EMBED_LEN_SHIFT) # Conditionally move the length of the heap array asm.test([array_reg, C.RBasic.offsetof(:flags)], C::RARRAY_EMBED_FLAG) # Select the array length value asm.cmovz(len_reg, [array_reg, C.RArray.offsetof(:as, :heap, :len)]) end
Generate RARRAY_CONST_PTR (part of RARRAY_AREF)
# File ruby_vm/rjit/insn_compiler.rb, line 5822 def jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg asm.comment('get array pointer for embedded or heap') flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) # Load the address of the embedded array # (struct RArray *)(obj)->as.ary asm.mov(ary_opnd, [array_reg, C.RArray.offsetof(:as, :heap, :ptr)]) asm.lea(array_reg, [array_reg, C.RArray.offsetof(:as, :ary)]) # clobbers array_reg asm.cmovnz(ary_opnd, array_reg) end
vm_call_alias @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5314 def jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) cme = C.rb_aliased_callable_method_entry(cme) jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) end
vm_call_attrset @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5184 def jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) argc = calling.argc flags = calling.flags send_shift = calling.send_shift if flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_attrset_splat) return CantCompile end if flags & C::VM_CALL_KWARG != 0 asm.incr_counter(:send_attrset_kwarg) return CantCompile elsif argc != 1 || !C.RB_TYPE_P(comptime_recv, C::RUBY_T_OBJECT) asm.incr_counter(:send_attrset_method) return CantCompile elsif c_method_tracing_currently_enabled? # Can't generate code for firing c_call and c_return events # See :attr-tracing: asm.incr_counter(:send_c_tracingg) return CantCompile elsif flags & C::VM_CALL_ARGS_BLOCKARG != 0 asm.incr_counter(:send_block_arg) return CantCompile end ivar_name = cme.def.body.attr.id # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end # Save the PC and SP because the callee may allocate # Note that this modifies REG_SP, which is why we do it first jit_prepare_routine_call(jit, ctx, asm) # Get the operands from the stack val_opnd = ctx.stack_pop(1) recv_opnd = ctx.stack_pop(1) # Call rb_vm_set_ivar_id with the receiver, the ivar name, and the value asm.mov(C_ARGS[0], recv_opnd) asm.mov(C_ARGS[1], ivar_name) asm.mov(C_ARGS[2], val_opnd) asm.call(C.rb_vm_set_ivar_id) out_opnd = ctx.stack_push(Type::Unknown) asm.mov(out_opnd, C_RET) KeepCompiling end
vm_call_bmethod @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5275 def jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) proc_addr = cme.def.body.bmethod.proc proc_t = C.rb_yjit_get_proc_ptr(proc_addr) proc_block = proc_t.block if proc_block.type != C.block_type_iseq asm.incr_counter(:send_bmethod_not_iseq) return CantCompile end capture = proc_block.as.captured iseq = capture.code.iseq # TODO: implement this # Optimize for single ractor mode and avoid runtime check for # "defined with an un-shareable Proc in a different Ractor" # if !assume_single_ractor_mode(jit, ocb) # return CantCompile; # end # Passing a block to a block needs logic different from passing # a block to a method and sometimes requires allocation. Bail for now. if calling.block_handler != C::VM_BLOCK_HANDLER_NONE asm.incr_counter(:send_bmethod_blockarg) return CantCompile end jit_call_iseq( jit, ctx, asm, cme, calling, iseq, frame_type: C::VM_FRAME_MAGIC_BLOCK | C::VM_FRAME_FLAG_BMETHOD | C::VM_FRAME_FLAG_LAMBDA, prev_ep: capture.ep, ) end
vm_call_cfunc @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4956 def jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class: nil) argc = calling.argc flags = calling.flags cfunc = cme.def.body.cfunc cfunc_argc = cfunc.argc # If the function expects a Ruby array of arguments if cfunc_argc < 0 && cfunc_argc != -1 asm.incr_counter(:send_cfunc_ruby_array_varg) return CantCompile end # We aren't handling a vararg cfuncs with splat currently. if flags & C::VM_CALL_ARGS_SPLAT != 0 && cfunc_argc == -1 asm.incr_counter(:send_args_splat_cfunc_var_args) return CantCompile end if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0 # zsuper methods are super calls without any arguments. # They are also marked as splat, but don't actually have an array # they pull arguments from, instead we need to change to call # a different method with the current stack. asm.incr_counter(:send_args_splat_cfunc_zuper) return CantCompile; end # In order to handle backwards compatibility between ruby 3 and 2 # ruby2_keywords was introduced. It is called only on methods # with splat and changes they way they handle them. # We are just going to not compile these. # https://docs.ruby-lang.org/en/3.2/Module.html#method-i-ruby2_keywords if jit.iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_args_splat_cfunc_ruby2_keywords) return CantCompile; end kw_arg = calling.kwarg kw_arg_num = if kw_arg.nil? 0 else kw_arg.keyword_len end if kw_arg_num != 0 && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_cfunc_splat_with_kw) return CantCompile end if c_method_tracing_currently_enabled? # Don't JIT if tracing c_call or c_return asm.incr_counter(:send_cfunc_tracing) return CantCompile end # Delegate to codegen for C methods if we have it. if kw_arg.nil? && flags & C::VM_CALL_OPT_SEND == 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 && (cfunc_argc == -1 || argc == cfunc_argc) known_cfunc_codegen = lookup_cfunc_codegen(cme.def) if known_cfunc_codegen&.call(jit, ctx, asm, argc, known_recv_class) # cfunc codegen generated code. Terminate the block so # there isn't multiple calls in the same block. jump_to_next_insn(jit, ctx, asm) return EndBlock end end # Check for interrupts jit_check_ints(jit, ctx, asm) # Stack overflow check # #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) # REG_CFP <= REG_SP + 4 * SIZEOF_VALUE + sizeof(rb_control_frame_t) asm.comment('stack overflow check') asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * 4 + 2 * C.rb_control_frame_t.size)) asm.cmp(CFP, :rax) asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow)) # Number of args which will be passed through to the callee # This is adjusted by the kwargs being combined into a hash. passed_argc = if kw_arg.nil? argc else argc - kw_arg_num + 1 end # If the argument count doesn't match if cfunc_argc >= 0 && cfunc_argc != passed_argc && flags & C::VM_CALL_ARGS_SPLAT == 0 asm.incr_counter(:send_cfunc_argc_mismatch) return CantCompile end # Don't JIT functions that need C stack arguments for now if cfunc_argc >= 0 && passed_argc + 1 > C_ARGS.size asm.incr_counter(:send_cfunc_toomany_args) return CantCompile end block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0 # Guard block_arg_type if guard_block_arg(jit, ctx, asm, calling) == CantCompile return CantCompile end if block_arg ctx.stack_pop(1) end # push_splat_args does stack manipulation so we can no longer side exit if flags & C::VM_CALL_ARGS_SPLAT != 0 assert_equal(true, cfunc_argc >= 0) required_args = cfunc_argc - (argc - 1) # + 1 because we pass self if required_args + 1 >= C_ARGS.size asm.incr_counter(:send_cfunc_toomany_args) return CantCompile end # We are going to assume that the splat fills # all the remaining arguments. So the number of args # should just equal the number of args the cfunc takes. # In the generated code we test if this is true # and if not side exit. argc = cfunc_argc passed_argc = argc push_splat_args(required_args, jit, ctx, asm) end # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift: calling.send_shift) end # Points to the receiver operand on the stack # Store incremented PC into current control frame in case callee raises. jit_save_pc(jit, asm) # Increment the stack pointer by 3 (in the callee) # sp += 3 frame_type = C::VM_FRAME_MAGIC_CFUNC | C::VM_FRAME_FLAG_CFRAME | C::VM_ENV_FLAG_LOCAL if kw_arg frame_type |= C::VM_FRAME_FLAG_CFRAME_KW end jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, calling.block_handler) if kw_arg # Build a hash from all kwargs passed asm.comment('build_kwhash') imemo_ci = calling.ci_addr # we assume all callinfos with kwargs are on the GC heap assert_equal(true, C.imemo_type_p(imemo_ci, C.imemo_callinfo)) asm.mov(C_ARGS[0], imemo_ci) asm.lea(C_ARGS[1], ctx.sp_opnd(0)) asm.call(C.rjit_build_kwhash) # Replace the stack location at the start of kwargs with the new hash stack_opnd = ctx.stack_opnd(argc - passed_argc) asm.mov(stack_opnd, C_RET) end # Copy SP because REG_SP will get overwritten sp = :rax asm.lea(sp, ctx.sp_opnd(0)) # Pop the C function arguments from the stack (in the caller) ctx.stack_pop(argc + 1) # Write interpreter SP into CFP. # Needed in case the callee yields to the block. jit_save_sp(ctx, asm) # Non-variadic method case cfunc_argc in (0..) # Non-variadic method # Copy the arguments from the stack to the C argument registers # self is the 0th argument and is at index argc from the stack top (0..passed_argc).each do |i| asm.mov(C_ARGS[i], [sp, -(argc + 1 - i) * C.VALUE.size]) end in -1 # Variadic method: rb_f_puts(int argc, VALUE *argv, VALUE recv) # The method gets a pointer to the first argument # rb_f_puts(int argc, VALUE *argv, VALUE recv) asm.mov(C_ARGS[0], passed_argc) asm.lea(C_ARGS[1], [sp, -argc * C.VALUE.size]) # argv asm.mov(C_ARGS[2], [sp, -(argc + 1) * C.VALUE.size]) # recv end # Call the C function # VALUE ret = (cfunc->func)(recv, argv[0], argv[1]); # cfunc comes from compile-time cme->def, which we assume to be stable. # Invalidation logic is in yjit_method_lookup_change() asm.comment('call C function') asm.mov(:rax, cfunc.func) asm.call(:rax) # TODO: use rel32 if close enough # Record code position for TracePoint patching. See full_cfunc_return(). Invariants.record_global_inval_patch(asm, full_cfunc_return) # Push the return value on the Ruby stack stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) # Pop the stack frame (ec->cfp++) # Instead of recalculating, we can reuse the previous CFP, which is stored in a callee-saved # register asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], CFP) # cfunc calls may corrupt types ctx.clear_local_types # Note: the return block of jit_call_iseq has ctx->sp_offset == 1 # which allows for sharing the same successor. # Jump (fall through) to the call continuation block # We do this to end the current block after the call assert_equal(1, ctx.sp_offset) jump_to_next_insn(jit, ctx, asm) EndBlock end
vm_call_general @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4247 def jit_call_general(jit, ctx, asm, mid, calling, cme, known_recv_class) jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) end
vm_call_iseq_setup @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4342 def jit_call_iseq(jit, ctx, asm, cme, calling, iseq, frame_type: nil, prev_ep: nil) argc = calling.argc flags = calling.flags send_shift = calling.send_shift # When you have keyword arguments, there is an extra object that gets # placed on the stack the represents a bitmap of the keywords that were not # specified at the call site. We need to keep track of the fact that this # value is present on the stack in order to properly set up the callee's # stack pointer. doing_kw_call = iseq.body.param.flags.has_kw supplying_kws = flags & C::VM_CALL_KWARG != 0 if flags & C::VM_CALL_TAILCALL != 0 # We can't handle tailcalls asm.incr_counter(:send_tailcall) return CantCompile end # No support for callees with these parameters yet as they require allocation # or complex handling. if iseq.body.param.flags.has_post asm.incr_counter(:send_iseq_has_opt) return CantCompile end if iseq.body.param.flags.has_kwrest asm.incr_counter(:send_iseq_has_kwrest) return CantCompile end # In order to handle backwards compatibility between ruby 3 and 2 # ruby2_keywords was introduced. It is called only on methods # with splat and changes they way they handle them. # We are just going to not compile these. # https://www.rubydoc.info/stdlib/core/Proc:ruby2_keywords if iseq.body.param.flags.ruby2_keywords && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_iseq_ruby2_keywords) return CantCompile end iseq_has_rest = iseq.body.param.flags.has_rest if iseq_has_rest && calling.block_handler == :captured asm.incr_counter(:send_iseq_has_rest_and_captured) return CantCompile end if iseq_has_rest && iseq.body.param.flags.has_kw && supplying_kws asm.incr_counter(:send_iseq_has_rest_and_kw_supplied) return CantCompile end # If we have keyword arguments being passed to a callee that only takes # positionals, then we need to allocate a hash. For now we're going to # call that too complex and bail. if supplying_kws && !iseq.body.param.flags.has_kw asm.incr_counter(:send_iseq_has_no_kw) return CantCompile end # If we have a method accepting no kwargs (**nil), exit if we have passed # it any kwargs. if supplying_kws && iseq.body.param.flags.accepts_no_kwarg asm.incr_counter(:send_iseq_accepts_no_kwarg) return CantCompile end # For computing number of locals to set up for the callee num_params = iseq.body.param.size # Block parameter handling. This mirrors setup_parameters_complex(). if iseq.body.param.flags.has_block if iseq.body.local_iseq.to_i == iseq.to_i num_params -= 1 else # In this case (param.flags.has_block && local_iseq != iseq), # the block argument is setup as a local variable and requires # materialization (allocation). Bail. asm.incr_counter(:send_iseq_materialized_block) return CantCompile end end if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_ZSUPER != 0 # zsuper methods are super calls without any arguments. # They are also marked as splat, but don't actually have an array # they pull arguments from, instead we need to change to call # a different method with the current stack. asm.incr_counter(:send_iseq_zsuper) return CantCompile end start_pc_offset = 0 required_num = iseq.body.param.lead_num # This struct represents the metadata about the caller-specified # keyword arguments. kw_arg = calling.kwarg kw_arg_num = if kw_arg.nil? 0 else kw_arg.keyword_len end # Arity handling and optional parameter setup opts_filled = argc - required_num - kw_arg_num opt_num = iseq.body.param.opt_num opts_missing = opt_num - opts_filled if doing_kw_call && flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_iseq_splat_with_kw) return CantCompile end if iseq_has_rest && opt_num != 0 asm.incr_counter(:send_iseq_has_rest_and_optional) return CantCompile end if opts_filled < 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 # Too few arguments and no splat to make up for it asm.incr_counter(:send_iseq_arity_error) return CantCompile end if opts_filled > opt_num && !iseq_has_rest # Too many arguments and no place to put them (i.e. rest arg) asm.incr_counter(:send_iseq_arity_error) return CantCompile end block_arg = flags & C::VM_CALL_ARGS_BLOCKARG != 0 # Guard block_arg_type if guard_block_arg(jit, ctx, asm, calling) == CantCompile return CantCompile end # If we have unfilled optional arguments and keyword arguments then we # would need to adjust the arguments location to account for that. # For now we aren't handling this case. if doing_kw_call && opts_missing > 0 asm.incr_counter(:send_iseq_missing_optional_kw) return CantCompile end # We will handle splat case later if opt_num > 0 && flags & C::VM_CALL_ARGS_SPLAT == 0 num_params -= opts_missing start_pc_offset = iseq.body.param.opt_table[opts_filled] end if doing_kw_call # Here we're calling a method with keyword arguments and specifying # keyword arguments at this call site. # This struct represents the metadata about the callee-specified # keyword parameters. keyword = iseq.body.param.keyword keyword_num = keyword.num keyword_required_num = keyword.required_num required_kwargs_filled = 0 if keyword_num > 30 # We have so many keywords that (1 << num) encoded as a FIXNUM # (which shifts it left one more) no longer fits inside a 32-bit # immediate. asm.incr_counter(:send_iseq_too_many_kwargs) return CantCompile end # Check that the kwargs being passed are valid if supplying_kws # This is the list of keyword arguments that the callee specified # in its initial declaration. # SAFETY: see compile.c for sizing of this slice. callee_kwargs = keyword_num.times.map { |i| keyword.table[i] } # Here we're going to build up a list of the IDs that correspond to # the caller-specified keyword arguments. If they're not in the # same order as the order specified in the callee declaration, then # we're going to need to generate some code to swap values around # on the stack. caller_kwargs = [] kw_arg.keyword_len.times do |kwarg_idx| sym = C.to_ruby(kw_arg[:keywords][kwarg_idx]) caller_kwargs << C.rb_sym2id(sym) end # First, we're going to be sure that the names of every # caller-specified keyword argument correspond to a name in the # list of callee-specified keyword parameters. caller_kwargs.each do |caller_kwarg| search_result = callee_kwargs.map.with_index.find { |kwarg, _| kwarg == caller_kwarg } case search_result in nil # If the keyword was never found, then we know we have a # mismatch in the names of the keyword arguments, so we need to # bail. asm.incr_counter(:send_iseq_kwargs_mismatch) return CantCompile in _, callee_idx if callee_idx < keyword_required_num # Keep a count to ensure all required kwargs are specified required_kwargs_filled += 1 else end end end assert_equal(true, required_kwargs_filled <= keyword_required_num) if required_kwargs_filled != keyword_required_num asm.incr_counter(:send_iseq_kwargs_mismatch) return CantCompile end end # Check if we need the arg0 splat handling of vm_callee_setup_block_arg arg_setup_block = (calling.block_handler == :captured) # arg_setup_type: arg_setup_block (invokeblock) block_arg0_splat = arg_setup_block && argc == 1 && (iseq.body.param.flags.has_lead || opt_num > 1) && !iseq.body.param.flags.ambiguous_param0 if block_arg0_splat # If block_arg0_splat, we still need side exits after splat, but # doing push_splat_args here disallows it. So bail out. if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest asm.incr_counter(:invokeblock_iseq_arg0_args_splat) return CantCompile end # The block_arg0_splat implementation is for the rb_simple_iseq_p case, # but doing_kw_call means it's not a simple ISEQ. if doing_kw_call asm.incr_counter(:invokeblock_iseq_arg0_has_kw) return CantCompile end # The block_arg0_splat implementation cannot deal with optional parameters. # This is a setup_parameters_complex() situation and interacts with the # starting position of the callee. if opt_num > 1 asm.incr_counter(:invokeblock_iseq_arg0_optional) return CantCompile end end if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest array = jit.peek_at_stack(block_arg ? 1 : 0) splat_array_length = if array.nil? 0 else array.length end if opt_num == 0 && required_num != splat_array_length + argc - 1 asm.incr_counter(:send_iseq_splat_arity_error) return CantCompile end end # We will not have CantCompile from here. if block_arg ctx.stack_pop(1) end if calling.block_handler == C::VM_BLOCK_HANDLER_NONE && iseq.body.builtin_attrs & C::BUILTIN_ATTR_LEAF != 0 if jit_leaf_builtin_func(jit, ctx, asm, flags, iseq) return KeepCompiling end end # Number of locals that are not parameters num_locals = iseq.body.local_table_size - num_params # Stack overflow check # Note that vm_push_frame checks it against a decremented cfp, hence the multiply by 2. # #define CHECK_VM_STACK_OVERFLOW0(cfp, sp, margin) asm.comment('stack overflow check') locals_offs = C.VALUE.size * (num_locals + iseq.body.stack_max) + 2 * C.rb_control_frame_t.size asm.lea(:rax, ctx.sp_opnd(locals_offs)) asm.cmp(CFP, :rax) asm.jbe(counted_exit(side_exit(jit, ctx), :send_stackoverflow)) # push_splat_args does stack manipulation so we can no longer side exit if splat_array_length remaining_opt = (opt_num + required_num) - (splat_array_length + (argc - 1)) if opt_num > 0 # We are going to jump to the correct offset based on how many optional # params are remaining. offset = opt_num - remaining_opt start_pc_offset = iseq.body.param.opt_table[offset] end # We are going to assume that the splat fills # all the remaining arguments. In the generated code # we test if this is true and if not side exit. argc = argc - 1 + splat_array_length + remaining_opt push_splat_args(splat_array_length, jit, ctx, asm) remaining_opt.times do # We need to push nil for the optional arguments stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, Qnil) end end # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end if iseq_has_rest # We are going to allocate so setting pc and sp. jit_save_pc(jit, asm) # clobbers rax jit_save_sp(ctx, asm) if flags & C::VM_CALL_ARGS_SPLAT != 0 non_rest_arg_count = argc - 1 # We start by dupping the array because someone else might have # a reference to it. array = ctx.stack_pop(1) asm.mov(C_ARGS[0], array) asm.call(C.rb_ary_dup) array = C_RET if non_rest_arg_count > required_num # If we have more arguments than required, we need to prepend # the items from the stack onto the array. diff = (non_rest_arg_count - required_num) # diff is >0 so no need to worry about null pointer asm.comment('load pointer to array elements') offset_magnitude = C.VALUE.size * diff values_opnd = ctx.sp_opnd(-offset_magnitude) values_ptr = :rcx asm.lea(values_ptr, values_opnd) asm.comment('prepend stack values to rest array') asm.mov(C_ARGS[0], diff) asm.mov(C_ARGS[1], values_ptr) asm.mov(C_ARGS[2], array) asm.call(C.rb_ary_unshift_m) ctx.stack_pop(diff) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) # We now should have the required arguments # and an array of all the rest arguments argc = required_num + 1 elsif non_rest_arg_count < required_num # If we have fewer arguments than required, we need to take some # from the array and move them to the stack. diff = (required_num - non_rest_arg_count) # This moves the arguments onto the stack. But it doesn't modify the array. move_rest_args_to_stack(array, diff, jit, ctx, asm) # We will now slice the array to give us a new array of the correct size asm.mov(C_ARGS[0], array) asm.mov(C_ARGS[1], diff) asm.call(C.rjit_rb_ary_subseq_length) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) # We now should have the required arguments # and an array of all the rest arguments argc = required_num + 1 else # The arguments are equal so we can just push to the stack assert_equal(non_rest_arg_count, required_num) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, array) end else assert_equal(true, argc >= required_num) n = (argc - required_num) argc = required_num + 1 # If n is 0, then elts is never going to be read, so we can just pass null if n == 0 values_ptr = 0 else asm.comment('load pointer to array elements') offset_magnitude = C.VALUE.size * n values_opnd = ctx.sp_opnd(-offset_magnitude) values_ptr = :rcx asm.lea(values_ptr, values_opnd) end asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], n) asm.mov(C_ARGS[2], values_ptr) asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) end end if doing_kw_call # Here we're calling a method with keyword arguments and specifying # keyword arguments at this call site. # Number of positional arguments the callee expects before the first # keyword argument args_before_kw = required_num + opt_num # This struct represents the metadata about the caller-specified # keyword arguments. ci_kwarg = calling.kwarg caller_keyword_len = if ci_kwarg.nil? 0 else ci_kwarg.keyword_len end # This struct represents the metadata about the callee-specified # keyword parameters. keyword = iseq.body.param.keyword asm.comment('keyword args') # This is the list of keyword arguments that the callee specified # in its initial declaration. callee_kwargs = keyword.table total_kwargs = keyword.num # Here we're going to build up a list of the IDs that correspond to # the caller-specified keyword arguments. If they're not in the # same order as the order specified in the callee declaration, then # we're going to need to generate some code to swap values around # on the stack. caller_kwargs = [] caller_keyword_len.times do |kwarg_idx| sym = C.to_ruby(ci_kwarg[:keywords][kwarg_idx]) caller_kwargs << C.rb_sym2id(sym) end kwarg_idx = caller_keyword_len unspecified_bits = 0 keyword_required_num = keyword.required_num (keyword_required_num...total_kwargs).each do |callee_idx| already_passed = false callee_kwarg = callee_kwargs[callee_idx] caller_keyword_len.times do |caller_idx| if caller_kwargs[caller_idx] == callee_kwarg already_passed = true break end end unless already_passed # Reserve space on the stack for each default value we'll be # filling in (which is done in the next loop). Also increments # argc so that the callee's SP is recorded correctly. argc += 1 default_arg = ctx.stack_push(Type::Unknown) # callee_idx - keyword->required_num is used in a couple of places below. req_num = keyword.required_num extra_args = callee_idx - req_num # VALUE default_value = keyword->default_values[callee_idx - keyword->required_num]; default_value = keyword.default_values[extra_args] if default_value == Qundef # Qundef means that this value is not constant and must be # recalculated at runtime, so we record it in unspecified_bits # (Qnil is then used as a placeholder instead of Qundef). unspecified_bits |= 0x01 << extra_args default_value = Qnil end asm.mov(:rax, default_value) asm.mov(default_arg, :rax) caller_kwargs[kwarg_idx] = callee_kwarg kwarg_idx += 1 end end assert_equal(kwarg_idx, total_kwargs) # Next, we're going to loop through every keyword that was # specified by the caller and make sure that it's in the correct # place. If it's not we're going to swap it around with another one. total_kwargs.times do |kwarg_idx| callee_kwarg = callee_kwargs[kwarg_idx] # If the argument is already in the right order, then we don't # need to generate any code since the expected value is already # in the right place on the stack. if callee_kwarg == caller_kwargs[kwarg_idx] next end # In this case the argument is not in the right place, so we # need to find its position where it _should_ be and swap with # that location. ((kwarg_idx + 1)...total_kwargs).each do |swap_idx| if callee_kwarg == caller_kwargs[swap_idx] # First we're going to generate the code that is going # to perform the actual swapping at runtime. offset0 = argc - 1 - swap_idx - args_before_kw offset1 = argc - 1 - kwarg_idx - args_before_kw stack_swap(jit, ctx, asm, offset0, offset1) # Next we're going to do some bookkeeping on our end so # that we know the order that the arguments are # actually in now. caller_kwargs[kwarg_idx], caller_kwargs[swap_idx] = caller_kwargs[swap_idx], caller_kwargs[kwarg_idx] break end end end # Keyword arguments cause a special extra local variable to be # pushed onto the stack that represents the parameters that weren't # explicitly given a value and have a non-constant default. asm.mov(ctx.stack_opnd(-1), C.to_value(unspecified_bits)) end # Same as vm_callee_setup_block_arg_arg0_check and vm_callee_setup_block_arg_arg0_splat # on vm_callee_setup_block_arg for arg_setup_block. This is done after CALLER_SETUP_ARG # and CALLER_REMOVE_EMPTY_KW_SPLAT, so this implementation is put here. This may need # side exits, so you still need to allow side exits here if block_arg0_splat is true. # Note that you can't have side exits after this arg0 splat. if block_arg0_splat asm.incr_counter(:send_iseq_block_arg0_splat) return CantCompile end # Create a context for the callee callee_ctx = Context.new # Set the argument types in the callee's context argc.times do |arg_idx| stack_offs = argc - arg_idx - 1 arg_type = ctx.get_opnd_type(StackOpnd[stack_offs]) callee_ctx.set_local_type(arg_idx, arg_type) end recv_type = if calling.block_handler == :captured Type::Unknown # we don't track the type information of captured->self for now else ctx.get_opnd_type(StackOpnd[argc]) end callee_ctx.upgrade_opnd_type(SelfOpnd, recv_type) # Setup the new frame frame_type ||= C::VM_FRAME_MAGIC_METHOD | C::VM_ENV_FLAG_LOCAL jit_push_frame( jit, ctx, asm, cme, flags, argc, frame_type, calling.block_handler, iseq: iseq, local_size: num_locals, stack_max: iseq.body.stack_max, prev_ep:, doing_kw_call:, ) # Directly jump to the entry point of the callee pc = (iseq.body.iseq_encoded + start_pc_offset).to_i jit_direct_jump(iseq, pc, callee_ctx, asm) EndBlock end
vm_call_ivar (+ part of vm_call_method_each_type) @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5240 def jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) argc = calling.argc flags = calling.flags if flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_ivar_splat) return CantCompile end if argc != 0 asm.incr_counter(:send_arity) return CantCompile end # We don't support handle_opt_send_shift_stack for this yet. if flags & C::VM_CALL_OPT_SEND != 0 asm.incr_counter(:send_ivar_opt_send) return CantCompile end ivar_id = cme.def.body.attr.id # Not handling block_handler if flags & C::VM_CALL_ARGS_BLOCKARG != 0 asm.incr_counter(:send_block_arg) return CantCompile end jit_getivar(jit, ctx, asm, comptime_recv, ivar_id, recv_opnd, StackOpnd[0]) end
vm_call_method @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler] @param send_shift [Integer] The number of shifts needed for VM_CALL_OPT_SEND
# File ruby_vm/rjit/insn_compiler.rb, line 4256 def jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) # The main check of vm_call_method before vm_call_method_each_type case C::METHOD_ENTRY_VISI(cme) in C::METHOD_VISI_PUBLIC # You can always call public methods in C::METHOD_VISI_PRIVATE # Allow only callsites without a receiver if calling.flags & C::VM_CALL_FCALL == 0 asm.incr_counter(:send_private) return CantCompile end in C::METHOD_VISI_PROTECTED # If the method call is an FCALL, it is always valid if calling.flags & C::VM_CALL_FCALL == 0 # otherwise we need an ancestry check to ensure the receiver is valid to be called as protected jit_protected_callee_ancestry_guard(asm, cme, side_exit(jit, ctx)) end end # Get a compile-time receiver recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet recv_idx += calling.send_shift comptime_recv = jit.peek_at_stack(recv_idx) recv_opnd = ctx.stack_opnd(recv_idx) jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) end
vm_call_method_each_type @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4303 def jit_call_method_each_type(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) case cme.def.type in C::VM_METHOD_TYPE_ISEQ iseq = def_iseq_ptr(cme.def) jit_call_iseq(jit, ctx, asm, cme, calling, iseq) in C::VM_METHOD_TYPE_NOTIMPLEMENTED asm.incr_counter(:send_notimplemented) return CantCompile in C::VM_METHOD_TYPE_CFUNC jit_call_cfunc(jit, ctx, asm, cme, calling, known_recv_class:) in C::VM_METHOD_TYPE_ATTRSET jit_call_attrset(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) in C::VM_METHOD_TYPE_IVAR jit_call_ivar(jit, ctx, asm, cme, calling, comptime_recv, recv_opnd) in C::VM_METHOD_TYPE_MISSING asm.incr_counter(:send_missing) return CantCompile in C::VM_METHOD_TYPE_BMETHOD jit_call_bmethod(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) in C::VM_METHOD_TYPE_ALIAS jit_call_alias(jit, ctx, asm, calling, cme, comptime_recv, recv_opnd, known_recv_class) in C::VM_METHOD_TYPE_OPTIMIZED jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class) in C::VM_METHOD_TYPE_UNDEF asm.incr_counter(:send_undef) return CantCompile in C::VM_METHOD_TYPE_ZSUPER asm.incr_counter(:send_zsuper) return CantCompile in C::VM_METHOD_TYPE_REFINED asm.incr_counter(:send_refined) return CantCompile end end
vm_call_opt_call @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5376 def jit_call_opt_call(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:) if block_handler != C::VM_BLOCK_HANDLER_NONE asm.incr_counter(:send_optimized_call_block) return CantCompile end if flags & C::VM_CALL_KWARG != 0 asm.incr_counter(:send_optimized_call_kwarg) return CantCompile end if flags & C::VM_CALL_ARGS_SPLAT != 0 asm.incr_counter(:send_optimized_call_splat) return CantCompile end # TODO: implement this # Optimize for single ractor mode and avoid runtime check for # "defined with an un-shareable Proc in a different Ractor" # if !assume_single_ractor_mode(jit, ocb) # return CantCompile # end # If this is a .send call we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end # About to reset the SP, need to load this here recv_idx = argc # blockarg is not supported. send_shift is already handled. asm.mov(:rcx, ctx.stack_opnd(recv_idx)) # recv # Save the PC and SP because the callee can make Ruby calls jit_prepare_routine_call(jit, ctx, asm) # NOTE: clobbers rax asm.lea(:rax, ctx.sp_opnd(0)) # sp kw_splat = flags & C::VM_CALL_KW_SPLAT asm.mov(C_ARGS[0], :rcx) asm.mov(C_ARGS[1], EC) asm.mov(C_ARGS[2], argc) asm.lea(C_ARGS[3], [:rax, -argc * C.VALUE.size]) # stack_argument_pointer. NOTE: C_ARGS[3] is rcx asm.mov(C_ARGS[4], kw_splat) asm.mov(C_ARGS[5], C::VM_BLOCK_HANDLER_NONE) asm.call(C.rjit_optimized_call) ctx.stack_pop(argc + 1) stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) return KeepCompiling end
vm_call_opt_send @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5350 def jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class) if jit_caller_setup_arg(jit, ctx, asm, calling.flags) == CantCompile return CantCompile end if calling.argc == 0 asm.incr_counter(:send_optimized_send_no_args) return CantCompile end calling.argc -= 1 # We aren't handling `send(:send, ...)` yet. This might work, but not tested yet. if calling.send_shift > 0 asm.incr_counter(:send_optimized_send_send) return CantCompile end # Lazily handle stack shift in handle_opt_send_shift_stack calling.send_shift += 1 jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, C::VM_CALL_FCALL) end
vm_call_opt_struct_aref @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5434 def jit_call_opt_struct_aref(jit, ctx, asm, cme, flags, argc, block_handler, known_recv_class, send_shift:) if argc != 0 asm.incr_counter(:send_optimized_struct_aref_error) return CantCompile end off = cme.def.body.optimized.index recv_idx = argc # blockarg is not supported recv_idx += send_shift comptime_recv = jit.peek_at_stack(recv_idx) # This is a .send call and we need to adjust the stack if flags & C::VM_CALL_OPT_SEND != 0 handle_opt_send_shift_stack(asm, argc, ctx, send_shift:) end # All structs from the same Struct class should have the same # length. So if our comptime_recv is embedded all runtime # structs of the same class should be as well, and the same is # true of the converse. embedded = C::FL_TEST_RAW(comptime_recv, C::RSTRUCT_EMBED_LEN_MASK) asm.comment('struct aref') asm.mov(:rax, ctx.stack_pop(1)) # recv if embedded asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :ary) + (C.VALUE.size * off)]) else asm.mov(:rax, [:rax, C.RStruct.offsetof(:as, :heap, :ptr)]) asm.mov(:rax, [:rax, C.VALUE.size * off]) end ret = ctx.stack_push(Type::Unknown) asm.mov(ret, :rax) jump_to_next_insn(jit, ctx, asm) EndBlock end
vm_call_optimized @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5323 def jit_call_optimized(jit, ctx, asm, cme, calling, known_recv_class) if calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 # Not working yet asm.incr_counter(:send_block_arg) return CantCompile end case cme.def.body.optimized.type in C::OPTIMIZED_METHOD_TYPE_SEND jit_call_opt_send(jit, ctx, asm, cme, calling, known_recv_class) in C::OPTIMIZED_METHOD_TYPE_CALL jit_call_opt_call(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift) in C::OPTIMIZED_METHOD_TYPE_BLOCK_CALL asm.incr_counter(:send_optimized_block_call) return CantCompile in C::OPTIMIZED_METHOD_TYPE_STRUCT_AREF jit_call_opt_struct_aref(jit, ctx, asm, cme, calling.flags, calling.argc, calling.block_handler, known_recv_class, send_shift: calling.send_shift) in C::OPTIMIZED_METHOD_TYPE_STRUCT_ASET asm.incr_counter(:send_optimized_struct_aset) return CantCompile end end
vm_call_symbol @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5496 def jit_call_symbol(jit, ctx, asm, cme, calling, known_recv_class, flags) flags |= C::VM_CALL_OPT_SEND | (calling.kw_splat ? C::VM_CALL_KW_SPLAT : 0) comptime_symbol = jit.peek_at_stack(calling.argc) if comptime_symbol.class != String && !static_symbol?(comptime_symbol) asm.incr_counter(:send_optimized_send_not_sym_or_str) return CantCompile end mid = C.get_symbol_id(comptime_symbol) if mid == 0 asm.incr_counter(:send_optimized_send_null_mid) return CantCompile end asm.comment("Guard #{comptime_symbol.inspect} is on stack") class_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_class_changed) jit_guard_known_klass( jit, ctx, asm, C.rb_class_of(comptime_symbol), ctx.stack_opnd(calling.argc), StackOpnd[calling.argc], comptime_symbol, class_changed_exit, ) asm.mov(C_ARGS[0], ctx.stack_opnd(calling.argc)) asm.call(C.rb_get_symbol_id) asm.cmp(C_RET, mid) id_changed_exit = counted_exit(side_exit(jit, ctx), :send_optimized_send_mid_id_changed) jit_chain_guard(:jne, jit, ctx, asm, id_changed_exit) # rb_callable_method_entry_with_refinements calling.flags = flags cme, _ = jit_search_method(jit, ctx, asm, mid, calling) if cme == CantCompile return CantCompile end if flags & C::VM_CALL_FCALL != 0 return jit_call_method(jit, ctx, asm, mid, calling, cme, known_recv_class) end raise NotImplementedError # unreachable for now end
CALLER_SETUP_ARG: Return CantCompile if not supported @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5675 def jit_caller_setup_arg(jit, ctx, asm, flags) if flags & C::VM_CALL_ARGS_SPLAT != 0 && flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_args_splat_kw_splat) return CantCompile elsif flags & C::VM_CALL_ARGS_SPLAT != 0 # splat is not supported in this path asm.incr_counter(:send_args_splat) return CantCompile elsif flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_args_kw_splat) return CantCompile elsif flags & C::VM_CALL_KWARG != 0 asm.incr_counter(:send_kwarg) return CantCompile end end
# File ruby_vm/rjit/insn_compiler.rb, line 5881 def jit_direct_jump(iseq, pc, ctx, asm, comment: 'jit_direct_jump') branch_stub = BranchStub.new( iseq:, shape: Default, target0: BranchTarget.new(ctx:, pc:), ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.compile = compile_jit_direct_jump(branch_stub, comment:) branch_stub.compile.call(asm) end
vm_get_ep @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4063 def jit_get_ep(asm, level, reg:) asm.mov(reg, [CFP, C.rb_control_frame_t.offsetof(:ep)]) level.times do # GET_PREV_EP: ep[VM_ENV_DATA_INDEX_SPECVAL] & ~0x03 asm.mov(reg, [reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) asm.and(reg, ~0x03) end end
GET_LEP @param jit [RubyVM::RJIT::JITState] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4056 def jit_get_lep(jit, asm, reg:) level = get_lvar_level(jit.iseq) jit_get_ep(asm, level, reg:) end
vm_getivar @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4076 def jit_getivar(jit, ctx, asm, comptime_obj, ivar_id, obj_opnd, obj_yarv_opnd) side_exit = side_exit(jit, ctx) starting_ctx = ctx.dup # copy for jit_chain_guard # Guard not special const if C::SPECIAL_CONST_P(comptime_obj) asm.incr_counter(:getivar_special_const) return CantCompile end case C::BUILTIN_TYPE(comptime_obj) when C::T_OBJECT # This is the only supported case for now (ROBJECT_IVPTR) else # General case. Call rb_ivar_get(). # VALUE rb_ivar_get(VALUE obj, ID id) asm.comment('call rb_ivar_get()') asm.mov(C_ARGS[0], obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)]) asm.mov(C_ARGS[1], ivar_id) # The function could raise exceptions. jit_prepare_routine_call(jit, ctx, asm) # clobbers obj_opnd and :rax asm.call(C.rb_ivar_get) if obj_opnd # attr_reader ctx.stack_pop end # Push the ivar on the stack out_opnd = ctx.stack_push(Type::Unknown) asm.mov(out_opnd, C_RET) # Jump to next instruction. This allows guard chains to share the same successor. jump_to_next_insn(jit, ctx, asm) return EndBlock end asm.mov(:rax, obj_opnd ? obj_opnd : [CFP, C.rb_control_frame_t.offsetof(:self)]) guard_object_is_heap(jit, ctx, asm, :rax, obj_yarv_opnd, :getivar_not_heap) shape_id = C.rb_shape_get_shape_id(comptime_obj) if shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID asm.incr_counter(:getivar_too_complex) return CantCompile end asm.comment('guard shape') asm.cmp(DwordPtr[:rax, C.rb_shape_id_offset], shape_id) jit_chain_guard(:jne, jit, starting_ctx, asm, counted_exit(side_exit, :getivar_megamorphic)) if obj_opnd ctx.stack_pop # pop receiver for attr_reader end index = C.rb_shape_get_iv_index(shape_id, ivar_id) # If there is no IVAR index, then the ivar was undefined # when we entered the compiler. That means we can just return # nil for this shape + iv name if index.nil? stack_opnd = ctx.stack_push(Type::Nil) val_opnd = Qnil else asm.comment('ROBJECT_IVPTR') if C::FL_TEST_RAW(comptime_obj, C::ROBJECT_EMBED) # Access embedded array asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :ary) + (index * C.VALUE.size)]) else # Pull out an ivar table on heap asm.mov(:rax, [:rax, C.RObject.offsetof(:as, :heap, :ivptr)]) # Read the table asm.mov(:rax, [:rax, index * C.VALUE.size]) end stack_opnd = ctx.stack_push(Type::Unknown) val_opnd = :rax end asm.mov(stack_opnd, val_opnd) # Let guard chains share the same successor jump_to_next_insn(jit, ctx, asm) EndBlock end
# File ruby_vm/rjit/insn_compiler.rb, line 4909 def jit_leaf_builtin_func(jit, ctx, asm, flags, iseq) builtin_func = builtin_function(iseq) if builtin_func.nil? return false end # this is a .send call not currently supported for builtins if flags & C::VM_CALL_OPT_SEND != 0 return false end builtin_argc = builtin_func.argc if builtin_argc + 1 >= C_ARGS.size return false end asm.comment('inlined leaf builtin') # Skip this if it doesn't trigger GC if iseq.body.builtin_attrs & C::BUILTIN_ATTR_NO_GC == 0 # The callee may allocate, e.g. Integer#abs on a Bignum. # Save SP for GC, save PC for allocation tracing, and prepare # for global invalidation after GC's VM lock contention. jit_prepare_routine_call(jit, ctx, asm) end # Call the builtin func (ec, recv, arg1, arg2, ...) asm.mov(C_ARGS[0], EC) # Copy self and arguments (0..builtin_argc).each do |i| stack_opnd = ctx.stack_opnd(builtin_argc - i) asm.mov(C_ARGS[i + 1], stack_opnd) end ctx.stack_pop(builtin_argc + 1) asm.call(builtin_func.func_ptr) # Push the return value stack_ret = ctx.stack_push(Type::Unknown) asm.mov(stack_ret, C_RET) return true end
Generate ancestry guard for protected callee. Calls to protected callees only go through when self.is_a?(klass_that_defines_the_callee).
# File ruby_vm/rjit/insn_compiler.rb, line 4286 def jit_protected_callee_ancestry_guard(asm, cme, side_exit) # See vm_call_method(). def_class = cme.defined_class # Note: PC isn't written to current control frame as rb_is_kind_of() shouldn't raise. # VALUE rb_obj_is_kind_of(VALUE obj, VALUE klass); asm.mov(C_ARGS[0], [CFP, C.rb_control_frame_t.offsetof(:self)]) asm.mov(C_ARGS[1], to_value(def_class)) asm.call(C.rb_obj_is_kind_of) asm.test(C_RET, C_RET) asm.jz(counted_exit(side_exit, :send_protected_check_failed)) end
vm_push_frame
Frame structure: | args | locals | cme/cref | block_handler/prev EP | frame type (EP here) | stack bottom (SP here)
@param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5545 def jit_push_frame(jit, ctx, asm, cme, flags, argc, frame_type, block_handler, iseq: nil, local_size: 0, stack_max: 0, prev_ep: nil, doing_kw_call: nil) # Save caller SP and PC before pushing a callee frame for backtrace and side exits asm.comment('save SP to caller CFP') recv_idx = argc # blockarg is already popped recv_idx += (block_handler == :captured) ? 0 : 1 # receiver is not on stack when captured->self is used if iseq # Skip setting this to SP register. This cfp->sp will be copied to SP on leave insn. asm.lea(:rax, ctx.sp_opnd(C.VALUE.size * -recv_idx)) # Pop receiver and arguments to prepare for side exits asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], :rax) else asm.lea(SP, ctx.sp_opnd(C.VALUE.size * -recv_idx)) asm.mov([CFP, C.rb_control_frame_t.offsetof(:sp)], SP) ctx.sp_offset = recv_idx end jit_save_pc(jit, asm, comment: 'save PC to caller CFP') sp_offset = ctx.sp_offset + 3 + local_size + (doing_kw_call ? 1 : 0) # callee_sp local_size.times do |i| asm.comment('set local variables') if i == 0 local_index = sp_offset + i - local_size - 3 asm.mov([SP, C.VALUE.size * local_index], Qnil) end asm.comment('set up EP with managing data') ep_offset = sp_offset - 1 # ep[-2]: cref_or_me asm.mov(:rax, cme.to_i) asm.mov([SP, C.VALUE.size * (ep_offset - 2)], :rax) # ep[-1]: block handler or prev env ptr (specval) if prev_ep asm.mov(:rax, prev_ep.to_i | 1) # tagged prev ep asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) elsif block_handler == :captured # Set captured->ep, saving captured in :rcx for captured->self ep_reg = :rcx jit_get_lep(jit, asm, reg: ep_reg) asm.mov(:rcx, [ep_reg, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # block_handler asm.and(:rcx, ~0x3) # captured asm.mov(:rax, [:rcx, C.VALUE.size]) # captured->ep asm.or(:rax, 0x1) # GC_GUARDED_PTR asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) elsif block_handler == C::VM_BLOCK_HANDLER_NONE asm.mov([SP, C.VALUE.size * (ep_offset - 1)], C::VM_BLOCK_HANDLER_NONE) elsif block_handler == C.rb_block_param_proxy # vm_caller_setup_arg_block: block_code == rb_block_param_proxy jit_get_lep(jit, asm, reg: :rax) # VM_CF_BLOCK_HANDLER: VM_CF_LEP asm.mov(:rax, [:rax, C.VALUE.size * C::VM_ENV_DATA_INDEX_SPECVAL]) # VM_CF_BLOCK_HANDLER: VM_ENV_BLOCK_HANDLER asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax) # reg_cfp->block_code = handler asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) # return handler; else # assume blockiseq asm.mov(:rax, block_handler) asm.mov([CFP, C.rb_control_frame_t.offsetof(:block_code)], :rax) asm.lea(:rax, [CFP, C.rb_control_frame_t.offsetof(:self)]) # VM_CFP_TO_CAPTURED_BLOCK asm.or(:rax, 1) # VM_BH_FROM_ISEQ_BLOCK asm.mov([SP, C.VALUE.size * (ep_offset - 1)], :rax) end # ep[-0]: ENV_FLAGS asm.mov([SP, C.VALUE.size * (ep_offset - 0)], frame_type) asm.comment('set up new frame') cfp_offset = -C.rb_control_frame_t.size # callee CFP # For ISEQ, JIT code will set it as needed. However, C func needs 0 there for svar frame detection. if iseq.nil? asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:pc)], 0) end asm.mov(:rax, iseq.to_i) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:iseq)], :rax) if block_handler == :captured asm.mov(:rax, [:rcx]) # captured->self else self_index = ctx.sp_offset - (1 + argc) # blockarg has been popped asm.mov(:rax, [SP, C.VALUE.size * self_index]) end asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:self)], :rax) asm.lea(:rax, [SP, C.VALUE.size * ep_offset]) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:ep)], :rax) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:block_code)], 0) # Update SP register only for ISEQ calls. SP-relative operations should be done above this. sp_reg = iseq ? SP : :rax asm.lea(sp_reg, [SP, C.VALUE.size * sp_offset]) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:sp)], sp_reg) # cfp->jit_return is used only for ISEQs if iseq # The callee might change locals through Kernel#binding and other means. ctx.clear_local_types # Stub cfp->jit_return return_ctx = ctx.dup return_ctx.stack_pop(argc + ((block_handler == :captured) ? 0 : 1)) # Pop args and receiver. blockarg has been popped return_ctx.stack_push(Type::Unknown) # push callee's return value return_ctx.sp_offset = 1 # SP is in the position after popping a receiver and arguments return_ctx.chain_depth = 0 branch_stub = BranchStub.new( iseq: jit.iseq, shape: Default, target0: BranchTarget.new(ctx: return_ctx, pc: jit.pc + jit.insn.len * C.VALUE.size), ) branch_stub.target0.address = Assembler.new.then do |ocb_asm| @exit_compiler.compile_branch_stub(return_ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end branch_stub.compile = compile_jit_return(branch_stub, cfp_offset:) branch_stub.compile.call(asm) end asm.comment('switch to callee CFP') # Update CFP register only for ISEQ calls cfp_reg = iseq ? CFP : :rax asm.lea(cfp_reg, [CFP, cfp_offset]) asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], cfp_reg) end
vm_search_method @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 4207 def jit_search_method(jit, ctx, asm, mid, calling) assert_equal(true, jit.at_current_insn?) # Generate a side exit side_exit = side_exit(jit, ctx) # kw_splat is not supported yet if calling.flags & C::VM_CALL_KW_SPLAT != 0 asm.incr_counter(:send_kw_splat) return CantCompile end # Get a compile-time receiver and its class recv_idx = calling.argc + (calling.flags & C::VM_CALL_ARGS_BLOCKARG != 0 ? 1 : 0) # blockarg is not popped yet recv_idx += calling.send_shift comptime_recv = jit.peek_at_stack(recv_idx) comptime_recv_klass = C.rb_class_of(comptime_recv) # Guard the receiver class (part of vm_search_method_fastpath) recv_opnd = ctx.stack_opnd(recv_idx) megamorphic_exit = counted_exit(side_exit, :send_klass_megamorphic) jit_guard_known_klass(jit, ctx, asm, comptime_recv_klass, recv_opnd, StackOpnd[recv_idx], comptime_recv, megamorphic_exit) # Do method lookup (vm_cc_cme(cc) != NULL) cme = C.rb_callable_method_entry(comptime_recv_klass, mid) if cme.nil? asm.incr_counter(:send_missing_cme) return CantCompile # We don't support vm_call_method_name end # Invalidate on redefinition (part of vm_search_method_fastpath) Invariants.assume_method_lookup_stable(jit, cme) return cme, comptime_recv_klass end
# File ruby_vm/rjit/insn_compiler.rb, line 4159 def jit_write_iv(asm, comptime_receiver, recv_reg, temp_reg, ivar_index, set_value, needs_extension) # Compile time self is embedded and the ivar index lands within the object embed_test_result = C::FL_TEST_RAW(comptime_receiver, C::ROBJECT_EMBED) && !needs_extension if embed_test_result # Find the IV offset offs = C.RObject.offsetof(:as, :ary) + ivar_index * C.VALUE.size # Write the IV asm.comment('write IV') asm.mov(temp_reg, set_value) asm.mov([recv_reg, offs], temp_reg) else # Compile time value is *not* embedded. # Get a pointer to the extended table asm.mov(recv_reg, [recv_reg, C.RObject.offsetof(:as, :heap, :ivptr)]) # Write the ivar in to the extended table asm.comment("write IV"); asm.mov(temp_reg, set_value) asm.mov([recv_reg, C.VALUE.size * ivar_index], temp_reg) end end
Pushes arguments from an array to the stack. Differs from push splat because the array can have items left over. @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5697 def move_rest_args_to_stack(array, num_args, jit, ctx, asm) side_exit = side_exit(jit, ctx) asm.comment('move_rest_args_to_stack') # array is :rax array_len_opnd = :rcx jit_array_len(asm, array, array_len_opnd) asm.comment('Side exit if length is less than required') asm.cmp(array_len_opnd, num_args) asm.jl(counted_exit(side_exit, :send_iseq_has_rest_and_splat_not_equal)) asm.comment('Push arguments from array') # Load the address of the embedded array # (struct RArray *)(obj)->as.ary array_reg = array # Conditionally load the address of the heap array # (struct RArray *)(obj)->as.heap.ptr flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) heap_ptr_opnd = [array_reg, C.RArray.offsetof(:as, :heap, :ptr)] # Load the address of the embedded array # (struct RArray *)(obj)->as.ary ary_opnd = :rdx # NOTE: array :rax is used after move_rest_args_to_stack too asm.lea(:rcx, [array_reg, C.RArray.offsetof(:as, :ary)]) asm.mov(ary_opnd, heap_ptr_opnd) asm.cmovnz(ary_opnd, :rcx) num_args.times do |i| top = ctx.stack_push(Type::Unknown) asm.mov(:rcx, [ary_opnd, i * C.VALUE.size]) asm.mov(top, :rcx) end end
vm_caller_setup_arg_splat (+ CALLER_SETUP_ARG): Pushes arguments from an array to the stack that are passed with a splat (i.e. *args). It optimistically compiles to a static size that is the exact number of arguments needed for the function. @param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context] @param asm [RubyVM::RJIT::Assembler]
# File ruby_vm/rjit/insn_compiler.rb, line 5741 def push_splat_args(required_args, jit, ctx, asm) side_exit = side_exit(jit, ctx) asm.comment('push_splat_args') array_opnd = ctx.stack_opnd(0) array_stack_opnd = StackOpnd[0] array_reg = :rax asm.mov(array_reg, array_opnd) guard_object_is_array(jit, ctx, asm, array_reg, :rcx, array_stack_opnd, :send_args_splat_not_array) array_len_opnd = :rcx jit_array_len(asm, array_reg, array_len_opnd) asm.comment('Side exit if length is not equal to remaining args') asm.cmp(array_len_opnd, required_args) asm.jne(counted_exit(side_exit, :send_args_splat_length_not_equal)) asm.comment('Check last argument is not ruby2keyword hash') ary_opnd = :rcx jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg last_array_value = :rax asm.mov(last_array_value, [ary_opnd, (required_args - 1) * C.VALUE.size]) ruby2_exit = counted_exit(side_exit, :send_args_splat_ruby2_hash); guard_object_is_not_ruby2_keyword_hash(asm, last_array_value, :rcx, ruby2_exit) # clobbers :rax asm.comment('Push arguments from array') array_opnd = ctx.stack_pop(1) if required_args > 0 # Load the address of the embedded array # (struct RArray *)(obj)->as.ary array_reg = :rax asm.mov(array_reg, array_opnd) # Conditionally load the address of the heap array # (struct RArray *)(obj)->as.heap.ptr flags_opnd = [array_reg, C.RBasic.offsetof(:flags)] asm.test(flags_opnd, C::RARRAY_EMBED_FLAG) heap_ptr_opnd = [array_reg, C.RArray.offsetof(:as, :heap, :ptr)] # Load the address of the embedded array # (struct RArray *)(obj)->as.ary asm.lea(:rcx, [array_reg, C.RArray.offsetof(:as, :ary)]) asm.mov(:rax, heap_ptr_opnd) asm.cmovnz(:rax, :rcx) ary_opnd = :rax (0...required_args).each do |i| top = ctx.stack_push(Type::Unknown) asm.mov(:rcx, [ary_opnd, i * C.VALUE.size]) asm.mov(top, :rcx) end asm.comment('end push_each') end end
# File ruby_vm/rjit/insn_compiler.rb, line 5865 def shape_too_complex?(obj) C.rb_shape_get_shape_id(obj) == C::OBJ_TOO_COMPLEX_SHAPE_ID end
@param jit [RubyVM::RJIT::JITState] @param ctx [RubyVM::RJIT::Context]
# File ruby_vm/rjit/insn_compiler.rb, line 5911 def side_exit(jit, ctx) # We use the latest ctx.sp_offset to generate a side exit to tolerate sp_offset changes by jit_save_sp. # However, we want to simulate an old stack_size when we take a side exit. We do that by adjusting the # sp_offset because gen_outlined_exit uses ctx.sp_offset to move SP. ctx = ctx.with_stack_size(jit.stack_size_for_pc) jit.side_exit_for_pc[ctx.sp_offset] ||= Assembler.new.then do |asm| @exit_compiler.compile_side_exit(jit.pc, ctx, asm) @ocb.write(asm) end end
# File ruby_vm/rjit/insn_compiler.rb, line 5856 def static_symbol?(obj) (C.to_value(obj) & 0xff) == C::RUBY_SYMBOL_FLAG end
# File ruby_vm/rjit/insn_compiler.rb, line 5852 def symbol?(obj) static_symbol?(obj) || dynamic_symbol?(obj) end
# File ruby_vm/rjit/insn_compiler.rb, line 5934 def to_value(obj) GC_REFS << obj C.to_value(obj) end