10#include "internal/sanitizers.h"
11#include "internal/string.h"
12#include "internal/hash.h"
13#include "internal/variable.h"
14#include "internal/compile.h"
15#include "internal/class.h"
16#include "internal/fixnum.h"
19#include "vm_callinfo.h"
22#include "insns_info.inc"
25#include "vm_insnhelper.h"
27#include "probes_helper.h"
30#include "internal/cont.h"
42STATIC_ASSERT(64b_size_t, SIZE_MAX == UINT64_MAX);
45STATIC_ASSERT(size_t_no_padding_bits,
sizeof(
size_t) ==
sizeof(uint64_t));
49STATIC_ASSERT(pointer_tagging_scheme,
USE_FLONUM);
67rb_yjit_mark_writable(
void *mem_block, uint32_t mem_size)
69 return mprotect(mem_block, mem_size, PROT_READ | PROT_WRITE) == 0;
73rb_yjit_mark_executable(
void *mem_block, uint32_t mem_size)
80 if (mprotect(mem_block, mem_size, PROT_READ | PROT_EXEC)) {
81 rb_bug(
"Couldn't make JIT page (%p, %lu bytes) executable, errno: %s\n",
82 mem_block, (
unsigned long)mem_size, strerror(errno));
88rb_yjit_mark_unused(
void *mem_block, uint32_t mem_size)
93 madvise(mem_block, mem_size, MADV_DONTNEED);
97 return mprotect(mem_block, mem_size, PROT_NONE) == 0;
102rb_yjit_icache_invalidate(
void *start,
void *end)
108 __builtin___clear_cache(start, end);
109#elif defined(__aarch64__)
110#error No instruction cache clear available with this compiler on Aarch64!
114# define PTR2NUM(x) (rb_int2inum((intptr_t)(void *)(x)))
121 VALUE frame_id = PTR2NUM(frame);
123 if (
RTEST(rb_hash_aref(hash, frame_id))) {
127 VALUE frame_info = rb_hash_new();
151 rb_hash_aset(hash, frame_id, frame_info);
161rb_yjit_exit_locations_dict(
VALUE *yjit_raw_samples,
int *yjit_line_samples,
int samples_len)
163 VALUE result = rb_hash_new();
164 VALUE raw_samples = rb_ary_new_capa(samples_len);
165 VALUE line_samples = rb_ary_new_capa(samples_len);
166 VALUE frames = rb_hash_new();
171 while (idx < samples_len) {
172 int num = (int)yjit_raw_samples[idx];
173 int line_num = (int)yjit_line_samples[idx];
176 rb_ary_push(raw_samples,
SIZET2NUM(num));
177 rb_ary_push(line_samples,
INT2NUM(line_num));
182 for (
int o = 0; o < num; o++) {
183 rb_yjit_add_frame(frames, yjit_raw_samples[idx]);
184 rb_ary_push(raw_samples,
SIZET2NUM(yjit_raw_samples[idx]));
185 rb_ary_push(line_samples,
INT2NUM(yjit_line_samples[idx]));
189 rb_ary_push(raw_samples,
SIZET2NUM(yjit_raw_samples[idx]));
190 rb_ary_push(line_samples,
INT2NUM(yjit_line_samples[idx]));
193 rb_ary_push(raw_samples,
SIZET2NUM(yjit_raw_samples[idx]));
194 rb_ary_push(line_samples,
INT2NUM(yjit_line_samples[idx]));
208rb_yjit_get_page_size(
void)
210#if defined(_SC_PAGESIZE)
211 long page_size = sysconf(_SC_PAGESIZE);
212 if (page_size <= 0)
rb_bug(
"yjit: failed to get page size");
217 if (page_size > 0x40000000l)
rb_bug(
"yjit page size too large");
219 return (uint32_t)page_size;
221#error "YJIT supports POSIX only for now"
225#if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
228align_ptr(uint8_t *ptr, uint32_t multiple)
231 uint32_t rem = ((uint32_t)(uintptr_t)ptr) % multiple;
238 uint32_t pad = multiple - rem;
247rb_yjit_reserve_addr_space(uint32_t mem_size)
253 #if defined(MAP_FIXED_NOREPLACE) && defined(_SC_PAGESIZE)
254 uint32_t
const page_size = (uint32_t)sysconf(_SC_PAGESIZE);
255 uint8_t *
const cfunc_sample_addr = (
void *)&rb_yjit_reserve_addr_space;
256 uint8_t *
const probe_region_end = cfunc_sample_addr + INT32_MAX;
258 uint8_t *req_addr = align_ptr(cfunc_sample_addr, page_size);
267 MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE,
273 if (mem_block != MAP_FAILED) {
278 req_addr += 4 * 1024 * 1024;
279 }
while (req_addr < probe_region_end);
285 (
void *)rb_yjit_reserve_addr_space,
288 MAP_PRIVATE | MAP_ANONYMOUS,
295 if (mem_block == MAP_FAILED) {
301 MAP_PRIVATE | MAP_ANONYMOUS,
308 if (mem_block == MAP_FAILED) {
309 perror(
"ruby: yjit: mmap:");
310 if(errno == ENOMEM) {
329 if (rb_multi_ractor_p()) {
330 tracing_events = ruby_vm_event_enabled_global_flags;
336 tracing_events = rb_ec_ractor_hooks(ec)->events;
360 EXEC_EVENT_HOOK(ec,
RUBY_EVENT_C_RETURN, cfp->self, me->def->original_id, me->called_id, me->owner, return_value);
364 RUBY_DTRACE_CMETHOD_RETURN_HOOK(ec, me->owner, me->def->original_id);
368 ec->cfp->sp[0] = return_value;
373rb_iseq_encoded_size(
const rb_iseq_t *iseq)
375 return iseq->body->iseq_size;
380rb_iseq_get_yjit_payload(
const rb_iseq_t *iseq)
384 return iseq->body->yjit_payload;
393rb_iseq_set_yjit_payload(
const rb_iseq_t *iseq,
void *payload)
398 iseq->body->yjit_payload = payload;
402rb_iseq_reset_jit_func(
const rb_iseq_t *iseq)
405 iseq->body->jit_func = NULL;
408 iseq->body->total_calls = 0;
413rb_iseq_pc_at_idx(
const rb_iseq_t *iseq, uint32_t insn_idx)
417 VALUE *encoded = iseq->body->iseq_encoded;
418 VALUE *pc = &encoded[insn_idx];
429 const VALUE at_pc = *pc;
430 return rb_vm_insn_addr2opcode((
const void *)at_pc);
435rb_str_bytesize(
VALUE str)
441rb_RSTRING_LEN(
VALUE str)
447rb_RSTRING_PTR(
VALUE str)
453rb_yjit_get_proc_ptr(
VALUE procv)
456 GetProcPtr(procv, proc);
463typedef struct rb_iseq_param_keyword rb_seq_param_keyword_struct;
466rb_insn_name(
VALUE insn)
468 return insn_name(insn);
473rb_insn_len(
VALUE insn)
475 return insn_len(insn);
481 return vm_ci_argc(ci);
487 return vm_ci_mid(ci);
493 return vm_ci_flag(ci);
499 return vm_ci_kwarg(ci);
505 return cikw->keyword_len;
511 return cikw->keywords[idx];
514rb_method_visibility_t
517 return METHOD_ENTRY_VISI(me);
523 if (UNDEFINED_METHOD_ENTRY_P(cme)) {
524 return VM_METHOD_TYPE_UNDEF;
527 return cme->def->type;
534 return cme->def->body.attr.id;
537ID rb_get_symbol_id(
VALUE namep);
539enum method_optimized_type
542 return cme->def->body.optimized.type;
548 return cme->def->body.optimized.index;
554 return UNALIGNED_MEMBER_PTR(cme->def, body.cfunc);
560 return def->method_serial;
566 return def->original_id;
578 return (
void*)mct->func;
584 return def_iseq_ptr(def);
591 return def->body.bmethod.proc;
595rb_get_iseq_body_local_iseq(
const rb_iseq_t *iseq)
597 return iseq->body->local_iseq;
601rb_get_iseq_body_parent_iseq(
const rb_iseq_t *iseq)
603 return iseq->body->parent_iseq;
607rb_get_iseq_body_local_table_size(
const rb_iseq_t *iseq)
609 return iseq->body->local_table_size;
613rb_get_iseq_body_iseq_encoded(
const rb_iseq_t *iseq)
615 return iseq->body->iseq_encoded;
619rb_get_iseq_body_builtin_inline_p(
const rb_iseq_t *iseq)
621 return iseq->body->builtin_inline_p;
625rb_get_iseq_body_stack_max(
const rb_iseq_t *iseq)
627 return iseq->body->stack_max;
631rb_get_iseq_flags_has_lead(
const rb_iseq_t *iseq)
633 return iseq->body->
param.flags.has_lead;
637rb_get_iseq_flags_has_opt(
const rb_iseq_t *iseq)
639 return iseq->body->
param.flags.has_opt;
643rb_get_iseq_flags_has_kw(
const rb_iseq_t *iseq)
645 return iseq->body->
param.flags.has_kw;
649rb_get_iseq_flags_has_post(
const rb_iseq_t *iseq)
651 return iseq->body->
param.flags.has_post;
655rb_get_iseq_flags_has_kwrest(
const rb_iseq_t *iseq)
657 return iseq->body->
param.flags.has_kwrest;
661rb_get_iseq_flags_has_rest(
const rb_iseq_t *iseq)
663 return iseq->body->
param.flags.has_rest;
667rb_get_iseq_flags_ruby2_keywords(
const rb_iseq_t *iseq)
669 return iseq->body->
param.flags.ruby2_keywords;
673rb_get_iseq_flags_has_block(
const rb_iseq_t *iseq)
675 return iseq->body->
param.flags.has_block;
679rb_get_iseq_flags_ambiguous_param0(
const rb_iseq_t *iseq)
681 return iseq->body->
param.flags.ambiguous_param0;
685rb_get_iseq_flags_accepts_no_kwarg(
const rb_iseq_t *iseq)
687 return iseq->body->
param.flags.accepts_no_kwarg;
690const rb_seq_param_keyword_struct *
691rb_get_iseq_body_param_keyword(
const rb_iseq_t *iseq)
693 return iseq->body->
param.keyword;
697rb_get_iseq_body_param_size(
const rb_iseq_t *iseq)
699 return iseq->body->
param.size;
703rb_get_iseq_body_param_lead_num(
const rb_iseq_t *iseq)
705 return iseq->body->
param.lead_num;
709rb_get_iseq_body_param_opt_num(
const rb_iseq_t *iseq)
711 return iseq->body->
param.opt_num;
715rb_get_iseq_body_param_opt_table(
const rb_iseq_t *iseq)
717 return iseq->body->
param.opt_table;
724 GetProcPtr(recv, proc);
725 return rb_vm_invoke_proc(ec, proc, argc, argv, kw_splat, block_handler);
731rb_leaf_invokebuiltin_iseq_p(
const rb_iseq_t *iseq)
733 unsigned int invokebuiltin_len = insn_len(BIN(opt_invokebuiltin_delegate_leave));
734 unsigned int leave_len = insn_len(BIN(leave));
736 return (iseq->body->iseq_size == (invokebuiltin_len + leave_len) &&
737 rb_vm_insn_addr2opcode((
void *)iseq->body->iseq_encoded[0]) == BIN(opt_invokebuiltin_delegate_leave) &&
738 rb_vm_insn_addr2opcode((
void *)iseq->body->iseq_encoded[invokebuiltin_len]) == BIN(leave) &&
739 iseq->body->builtin_inline_p
745rb_leaf_builtin_function(
const rb_iseq_t *iseq)
747 if (!rb_leaf_invokebuiltin_iseq_p(iseq))
753rb_yjit_str_simple_append(
VALUE str1,
VALUE str2)
767 return (
VALUE*)cfp->pc;
804 return (
VALUE*)cfp->ep;
812 for (i = 0; i < lv; i++) {
813 ep = VM_ENV_PREV_EP(ep);
819rb_yarv_class_of(
VALUE obj)
826rb_yarv_str_eql_internal(
VALUE str1,
VALUE str2)
829 return rb_str_eql_internal(str1, str2);
834rb_yarv_ary_entry_internal(
VALUE ary,
long offset)
836 return rb_ary_entry_internal(ary, offset);
842 return rb_fix_mod_fix(recv, obj);
847rb_yjit_dump_iseq_loc(
const rb_iseq_t *iseq, uint32_t insn_idx)
851 VALUE path = rb_iseq_path(iseq);
853 fprintf(stderr,
"%s %.*s:%u\n", __func__, (
int)len, ptr, rb_iseq_line_no(iseq, insn_idx));
878rb_RSTRUCT_LEN(
VALUE st)
880 return RSTRUCT_LEN(st);
889 RSTRUCT_SET(st, k, v);
899rb_BASIC_OP_UNREDEFINED_P(
enum ruby_basic_operators bop, uint32_t klass)
901 return BASIC_OP_UNREDEFINED_P(bop, klass);
905rb_RCLASS_ORIGIN(
VALUE c)
907 return RCLASS_ORIGIN(c);
912rb_ENCODING_GET(
VALUE obj)
918rb_yjit_multi_ractor_p(
void)
920 return rb_multi_ractor_p();
925rb_assert_iseq_handle(
VALUE handle)
932rb_IMEMO_TYPE_P(
VALUE imemo,
enum imemo_type imemo_type)
934 return IMEMO_TYPE_P(imemo, imemo_type);
938rb_assert_cme_handle(
VALUE handle)
946 rb_iseq_callback callback;
952for_each_iseq_i(
void *vstart,
void *vend,
size_t stride,
void *data)
956 for (; v != (
VALUE)vend; v += stride) {
957 void *ptr = asan_poisoned_object_p(v);
958 asan_unpoison_object(v,
false);
960 if (rb_obj_is_iseq(v)) {
962 callback_data->callback(iseq, callback_data->data);
965 asan_poison_object_if(ptr, v);
973rb_yjit_for_each_iseq(rb_iseq_callback callback,
void *data)
976 rb_objspace_each_objects(for_each_iseq_i, (
void *)&callback_data);
982rb_yjit_obj_written(
VALUE old,
VALUE young,
const char *file,
int line)
984 rb_obj_written(old,
Qundef, young, file, line);
992rb_yjit_vm_lock_then_barrier(
unsigned int *recursive_lock_level,
const char *file,
int line)
994 rb_vm_lock_enter(recursive_lock_level, file, line);
1001rb_yjit_vm_unlock(
unsigned int *recursive_lock_level,
const char *file,
int line)
1003 rb_vm_lock_leave(recursive_lock_level, file, line);
1012 bool success =
true;
1018 uint8_t *code_ptr = rb_yjit_iseq_gen_entry_point(iseq, ec);
1021 iseq->body->jit_func = (yjit_func_t)code_ptr;
1024 iseq->body->jit_func = 0;
1038yjit_root_free(
void *ptr)
1044yjit_root_memsize(
const void *ptr)
1052yjit_root_update_references(
void *ptr)
1057void rb_yjit_root_mark(
void *ptr);
1063 {rb_yjit_root_mark, yjit_root_free, yjit_root_memsize, yjit_root_update_references},
1064 0, 0, RUBY_TYPED_FREE_IMMEDIATELY
1069rb_yjit_invalidate_all_method_lookup_assumptions(
void)
1080 return ULONG2NUM((
unsigned long)GET_VM()->next_shape_id);
1095#include "yjit.rbinc"
1102 void rb_yjit_init_rust(
void);
1103 rb_yjit_init_rust();
1108 rb_gc_register_mark_object(yjit_root);
#define RUBY_ASSERT(expr)
Asserts that the given expression is truthy if and only if RUBY_DEBUG is truthy.
#define RUBY_ASSERT_ALWAYS(expr)
A variant of RUBY_ASSERT that does not interface with RUBY_DEBUG.
VALUE rb_profile_frame_full_label(VALUE frame)
Identical to rb_profile_frame_label(), except it returns a qualified result.
VALUE rb_profile_frame_absolute_path(VALUE frame)
Identical to rb_profile_frame_path(), except it tries to expand the returning path.
VALUE rb_profile_frame_path(VALUE frame)
Queries the path of the passed backtrace.
VALUE rb_profile_frame_first_lineno(VALUE frame)
Queries the first line of the method of the passed frame pointer.
#define RUBY_EVENT_C_CALL
A method, written in C, is called.
#define RUBY_EVENT_C_RETURN
Return from a method, written in C.
uint32_t rb_event_flag_t
Represents event(s).
static VALUE RB_FL_TEST(VALUE obj, VALUE flags)
Tests if the given flag(s) are set or not.
#define Qundef
Old name of RUBY_Qundef.
#define INT2FIX
Old name of RB_INT2FIX.
#define ID2SYM
Old name of RB_ID2SYM.
#define ULONG2NUM
Old name of RB_ULONG2NUM.
#define SIZET2NUM
Old name of RB_SIZE2NUM.
#define FL_TEST_RAW
Old name of RB_FL_TEST_RAW.
#define LONG2NUM
Old name of RB_LONG2NUM.
#define INT2NUM
Old name of RB_INT2NUM.
#define NIL_P
Old name of RB_NIL_P.
void rb_bug(const char *fmt,...)
Interpreter panic switch.
static VALUE rb_class_of(VALUE obj)
Object to class mapping function.
static int RB_ENCODING_GET(VALUE obj)
Just another name of rb_enc_get_index.
VALUE rb_str_cat(VALUE dst, const char *src, long srclen)
Destructively appends the passed contents to the string.
ID rb_intern(const char *name)
Finds or creates a symbol of the given name.
#define RSTRING_GETMEM(str, ptrvar, lenvar)
Convenient macro to obtain the contents and length at once.
static long RSTRING_LEN(VALUE str)
Queries the length of the string.
static char * RSTRING_PTR(VALUE str)
Queries the contents pointer of the string.
#define TypedData_Make_Struct(klass, type, data_type, sval)
Identical to TypedData_Wrap_Struct, except it allocates a new data region internally instead of takin...
#define RTEST
This is an old name of RB_TEST.
This is the struct that holds necessary info for a struct.
struct rb_iseq_constant_body::@131 param
parameter information
uintptr_t ID
Type that represents a Ruby identifier such as a variable name.
uintptr_t VALUE
Type that represents a Ruby object.
static bool RB_TYPE_P(VALUE obj, enum ruby_value_type t)
Queries if the given object is of given type.
ruby_value_type
C-level type of an object.