Skip to main content

cranelift_jit/
compiled_blob.rs

1use std::ptr;
2
3use cranelift_codegen::binemit::{Addend, Reloc};
4use cranelift_module::{ModuleError, ModuleReloc, ModuleRelocTarget, ModuleResult};
5
6use crate::JITMemoryProvider;
7use crate::memory::JITMemoryKind;
8
9/// Size reserved per veneer. This is exactly the size of the largest veneer:
10/// 16 bytes on AArch64 (`ldr` + `br` + 8-byte target address). The x86_64
11/// veneer is 14 bytes (`jmp qword ptr [rip]` + 8-byte target address).
12///
13/// Veneers are placed directly after the compiled code, followed by the GOT
14/// entries: `[code][veneers][GOT entries]`.
15const VENEER_SIZE: usize = 16;
16
17/// Size of a GOT entry: the absolute 4/8-byte address of a symbol.
18const GOT_ENTRY_SIZE: usize = size_of::<usize>();
19
20/// Reads a 32bit instruction at `iptr`, and writes it again after
21/// being altered by `modifier`
22unsafe fn modify_inst32(iptr: *mut u32, modifier: impl FnOnce(u32) -> u32) {
23    let inst = iptr.read_unaligned();
24    let new_inst = modifier(inst);
25    iptr.write_unaligned(new_inst);
26}
27
28#[derive(Clone)]
29pub(crate) struct CompiledBlob {
30    ptr: *mut u8,
31    size: usize,
32    relocs: Vec<ModuleReloc>,
33    veneer_count: usize,
34    got_count: usize,
35    #[cfg(feature = "wasmtime-unwinder")]
36    wasmtime_exception_data: Option<Vec<u8>>,
37}
38
39unsafe impl Send for CompiledBlob {}
40
41impl CompiledBlob {
42    pub(crate) fn new(
43        memory: &mut dyn JITMemoryProvider,
44        data: &[u8],
45        align: u64,
46        relocs: Vec<ModuleReloc>,
47        #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,
48        kind: JITMemoryKind,
49    ) -> ModuleResult<Self> {
50        // Reserve a worst-case veneer slot for every branch relocation and a
51        // GOT entry for every GOT-relative load in case its target is out of
52        // range.
53        let mut veneer_count = 0;
54        let mut got_count = 0;
55        for reloc in &relocs {
56            match reloc.kind {
57                Reloc::Arm64Call | Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {
58                    veneer_count += 1
59                }
60                Reloc::X86GOTPCRel4 => got_count += 1,
61                _ => {}
62            }
63        }
64
65        let mut alloc_size = data.len() + veneer_count * VENEER_SIZE;
66        if got_count > 0 {
67            alloc_size = alloc_size.next_multiple_of(GOT_ENTRY_SIZE) + got_count * GOT_ENTRY_SIZE;
68        }
69
70        let ptr = memory
71            .allocate(alloc_size, align, kind)
72            .map_err(|e| ModuleError::Allocation { err: e })?;
73
74        unsafe {
75            ptr::copy_nonoverlapping(data.as_ptr(), ptr, data.len());
76        }
77
78        Ok(CompiledBlob {
79            ptr,
80            size: data.len(),
81            relocs,
82            veneer_count,
83            got_count,
84            #[cfg(feature = "wasmtime-unwinder")]
85            wasmtime_exception_data,
86        })
87    }
88
89    pub(crate) fn new_zeroed(
90        memory: &mut dyn JITMemoryProvider,
91        size: usize,
92        align: u64,
93        relocs: Vec<ModuleReloc>,
94        #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,
95        kind: JITMemoryKind,
96    ) -> ModuleResult<Self> {
97        let ptr = memory
98            .allocate(size, align, kind)
99            .map_err(|e| ModuleError::Allocation { err: e })?;
100
101        unsafe { ptr::write_bytes(ptr, 0, size) };
102
103        Ok(CompiledBlob {
104            ptr,
105            size,
106            relocs,
107            veneer_count: 0,
108            got_count: 0,
109            #[cfg(feature = "wasmtime-unwinder")]
110            wasmtime_exception_data,
111        })
112    }
113
114    pub(crate) fn ptr(&self) -> *const u8 {
115        self.ptr
116    }
117
118    pub(crate) fn size(&self) -> usize {
119        self.size
120    }
121
122    #[cfg(feature = "wasmtime-unwinder")]
123    pub(crate) fn wasmtime_exception_data(&self) -> Option<&[u8]> {
124        self.wasmtime_exception_data.as_deref()
125    }
126
127    /// Pointer to a GOT entry within the allocation.
128    unsafe fn got_entry(&self, idx: usize) -> *mut u8 {
129        // Offset of the GOT entries within the allocation: after the code and
130        // the veneers, aligned to the entry size.
131        let got_offset =
132            (self.size + self.veneer_count * VENEER_SIZE).next_multiple_of(GOT_ENTRY_SIZE);
133
134        unsafe { self.ptr.byte_add(got_offset + idx * GOT_ENTRY_SIZE) }
135    }
136
137    pub(crate) fn perform_relocations(
138        &self,
139        get_address: impl Fn(&ModuleRelocTarget) -> *const u8,
140    ) {
141        use std::ptr::write_unaligned;
142
143        let mut next_veneer_idx = 0;
144        let mut next_got_idx = 0;
145        let relocation_target_addr = |name: &ModuleRelocTarget, addend: Addend| {
146            let addend = isize::try_from(addend).unwrap();
147            get_address(name)
148                .expose_provenance()
149                .checked_add_signed(addend)
150                .unwrap()
151        };
152        let pcrel_offset = |what: usize, at: *mut u8| {
153            i32::try_from(what.cast_signed() - at.expose_provenance().cast_signed())
154        };
155
156        for (
157            i,
158            &ModuleReloc {
159                kind,
160                offset,
161                ref name,
162                addend,
163            },
164        ) in self.relocs.iter().enumerate()
165        {
166            // The writes below rely on the relocation lying within the definition.
167            let end = (offset as usize).checked_add(kind.patched_len() as usize);
168            assert!(
169                end.is_some_and(|end| end <= self.size),
170                "{kind:?} relocation at offset {offset} is out of bounds of a {}-byte definition",
171                self.size,
172            );
173            let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) };
174            match kind {
175                Reloc::Abs4 => {
176                    let what = relocation_target_addr(name, addend);
177                    unsafe { write_unaligned(at.cast::<u32>(), u32::try_from(what).unwrap()) };
178                }
179                Reloc::Abs8 => {
180                    let what = relocation_target_addr(name, addend);
181                    unsafe { write_unaligned(at.cast::<u64>(), u64::try_from(what).unwrap()) };
182                }
183
184                Reloc::X86PCRel4 => {
185                    let what = relocation_target_addr(name, addend);
186                    let pcrel = pcrel_offset(what, at).unwrap();
187                    unsafe { write_unaligned(at.cast::<i32>(), pcrel) };
188                }
189                Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {
190                    // These relocations are applied to the 32-bit displacement of a `call`
191                    // or `jmp` instruction, which is relative to the end of the
192                    // instruction, i.e. 4 bytes past `at`. The addend (always -4 as
193                    // emitted by the x64 backend) accounts for this, so the instruction
194                    // transfers control to `what + 4`. The PLT form additionally permits
195                    // resolution through a stub, which the veneer below provides.
196                    let what = relocation_target_addr(name, addend);
197                    if let Ok(pcrel) = pcrel_offset(what, at) {
198                        unsafe { write_unaligned(at.cast::<i32>(), pcrel) };
199                    } else {
200                        // The target is out of range for the 32-bit displacement, so
201                        // redirect the call through a veneer at the end of the function,
202                        // just like for `Arm64Call` below: `jmp qword ptr [rip]` followed
203                        // by the absolute target address.
204                        let veneer_idx = next_veneer_idx;
205                        next_veneer_idx += 1;
206                        assert!(veneer_idx < self.veneer_count);
207                        let veneer =
208                            unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };
209
210                        let target = u64::try_from(what.checked_add(4).unwrap()).unwrap();
211                        unsafe {
212                            ptr::copy_nonoverlapping([0xff, 0x25, 0, 0, 0, 0].as_ptr(), veneer, 6);
213                            write_unaligned(veneer.byte_add(6).cast::<u64>(), target);
214                        }
215
216                        // Point the original instruction at the veneer instead; the
217                        // displacement is again relative to the end of the 4-byte field.
218                        let pcrel = pcrel_offset(veneer.expose_provenance() - 4, at).unwrap();
219                        unsafe { write_unaligned(at.cast::<i32>(), pcrel) };
220                    }
221                }
222                Reloc::X86GOTPCRel4 => {
223                    // This relocation is applied to the 32-bit displacement of a
224                    // `mov reg, qword ptr [rip + disp]` instruction that loads the
225                    // address of a symbol from its GOT entry. The addend (always -4 as
226                    // emitted by the x64 backend) accounts for the displacement being
227                    // relative to the end of the instruction, and any symbol offset is
228                    // added by a separately emitted instruction, so the GOT entry holds
229                    // the address of the symbol itself without the addend applied.
230                    let symbol_addr = get_address(name).expose_provenance();
231
232                    // If the symbol is within displacement range, relax the load to a
233                    // `lea` computing its address directly, the same way linkers relax
234                    // `R_X86_64_REX_GOTPCRELX`. The instruction is expected to be a REX
235                    // prefix, the `mov` opcode 0x8b, and a ModRM byte with mod=0b00 and
236                    // r/m=0b101 (RIP-relative); only rewrite it if it matches.
237                    let insn = (offset >= 3)
238                        .then(|| unsafe { at.cast_const().sub(3).cast::<[u8; 3]>().read() });
239                    let is_rip_relative_mov = insn.is_some_and(
240                        |insn| matches!(insn, [0x48..=0x4f, 0x8b, modrm] if modrm & 0xc7 == 0x05),
241                    );
242                    let direct_pcrel = pcrel_offset(symbol_addr.wrapping_sub(4), at);
243                    match (is_rip_relative_mov, direct_pcrel) {
244                        (true, Ok(pcrel)) if addend == -4 => unsafe {
245                            at.sub(2).write(0x8d); // mov -> lea
246                            write_unaligned(at.cast::<i32>(), pcrel);
247                        },
248                        _ => {
249                            // Resolve through a GOT entry at the end of the allocation,
250                            // which is always in range of the load.
251                            let got_idx = next_got_idx;
252                            next_got_idx += 1;
253                            assert!(got_idx < self.got_count);
254                            let entry = unsafe { self.got_entry(got_idx) };
255                            unsafe { write_unaligned(entry.cast::<usize>(), symbol_addr) };
256
257                            let pcrel = i32::try_from(
258                                entry.expose_provenance().cast_signed()
259                                    - at.expose_provenance().cast_signed()
260                                    + (addend as isize),
261                            )
262                            .unwrap();
263                            unsafe { write_unaligned(at.cast::<i32>(), pcrel) };
264                        }
265                    }
266                }
267
268                Reloc::S390xGOTRel32Dbl => {
269                    panic!("GOT relocation shouldn't be generated when !is_pic");
270                }
271                Reloc::S390xPCRel32Dbl | Reloc::S390xPLTRel32Dbl => {
272                    let what = relocation_target_addr(name, addend);
273                    let pcrel = i32::try_from(
274                        (what.cast_signed() - at.expose_provenance().cast_signed()) >> 1,
275                    )
276                    .unwrap();
277                    unsafe { write_unaligned(at.cast::<i32>(), pcrel) };
278                }
279
280                Reloc::Arm64Call => {
281                    let what = relocation_target_addr(name, addend);
282                    // The instruction is 32 bits long.
283                    let iptr = at.cast::<u32>();
284
285                    // The offset encoded in the `bl` instruction is the
286                    // number of bytes divided by 4.
287                    let diff = (what.cast_signed() - at.expose_provenance().cast_signed()) >> 2;
288                    // Sign propagating right shift disposes of the
289                    // included bits, so the result is expected to be
290                    // either all sign bits or 0 when in-range, depending
291                    // on if the original value was negative or positive.
292                    if (diff >> 25 == -1) || (diff >> 25 == 0) {
293                        // The lower 26 bits of the `bl` instruction form the
294                        // immediate offset argument.
295                        let chop = 32 - 26;
296                        let imm26 = (diff as u32) << chop >> chop;
297                        unsafe { modify_inst32(iptr, |inst| inst | imm26) };
298                    } else {
299                        // If the target is out of range for a direct call, insert a veneer at the
300                        // end of the function.
301                        let veneer_idx = next_veneer_idx;
302                        next_veneer_idx += 1;
303                        assert!(veneer_idx < self.veneer_count);
304                        let veneer =
305                            unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };
306
307                        // Write the veneer
308                        // x16 is reserved as scratch register to be used by veneers and PLT entries
309                        unsafe {
310                            write_unaligned(
311                                veneer.cast::<u32>(),
312                                0x58000050, // ldr x16, 0x8
313                            );
314                            write_unaligned(
315                                veneer.byte_add(4).cast::<u32>(),
316                                0xd61f0200, // br x16
317                            );
318                            write_unaligned(veneer.byte_add(8).cast::<u64>(), what as u64);
319                        };
320
321                        // Set the veneer as target of the call
322                        let diff = (veneer.expose_provenance().cast_signed()
323                            - at.expose_provenance().cast_signed())
324                            >> 2;
325                        assert!((diff >> 25 == -1) || (diff >> 25 == 0));
326                        let chop = 32 - 26;
327                        let imm26 = (diff as u32) << chop >> chop;
328                        unsafe { modify_inst32(iptr, |inst| inst | imm26) };
329                    }
330                }
331                Reloc::Aarch64AdrGotPage21 => {
332                    panic!("GOT relocation shouldn't be generated when !is_pic");
333                }
334                Reloc::Aarch64Ld64GotLo12Nc => {
335                    panic!("GOT relocation shouldn't be generated when !is_pic");
336                }
337                Reloc::Aarch64AdrPrelPgHi21 => {
338                    let what = relocation_target_addr(name, addend);
339                    let get_page = |x| x & (!0xfff);
340                    // NOTE: This should technically be i33 given that this relocation type allows
341                    // a range from -4GB to +4GB, not -2GB to +2GB. But this doesn't really matter
342                    // as the target is unlikely to be more than 2GB from the adrp instruction. We
343                    // need to be careful to not cast to an unsigned int until after doing >> 12 to
344                    // compute the upper 21bits of the pcrel address however as otherwise the top
345                    // bit of the 33bit pcrel address would be forced 0 through zero extension
346                    // instead of being sign extended as it should be.
347                    let pcrel = i32::try_from(
348                        get_page(what.cast_signed())
349                            - get_page(at.expose_provenance().cast_signed()),
350                    )
351                    .unwrap();
352                    let iptr = at.cast::<u32>();
353                    let hi21 = (pcrel >> 12).cast_unsigned();
354                    let lo = (hi21 & 0x3) << 29;
355                    let hi = (hi21 & 0x1ffffc) << 3;
356                    unsafe { modify_inst32(iptr, |inst| inst | lo | hi) };
357                }
358                Reloc::Aarch64AddAbsLo12Nc => {
359                    let what = relocation_target_addr(name, addend);
360                    let iptr = at.cast::<u32>();
361                    let imm12 = (what as u32 & 0xfff) << 10;
362                    unsafe { modify_inst32(iptr, |inst| inst | imm12) };
363                }
364
365                Reloc::RiscvCallPlt => {
366                    // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
367                    // It is the equivalent of two relocations:
368                    // 1. R_RISCV_PCREL_HI20 on the `auipc`
369                    // 2. R_RISCV_PCREL_LO12_I on the `jalr`
370
371                    let what = relocation_target_addr(name, addend);
372                    let pcrel = pcrel_offset(what, at).unwrap().cast_unsigned();
373
374                    // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
375                    // for a better explanation of the following code.
376                    //
377                    // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
378                    //
379                    // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
380                    // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
381                    // get a valid address.
382                    //
383                    // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
384                    //
385                    // `hi20` should also be shifted right to be the "true" value. But we also need it
386                    // left shifted for the `lo12` calculation and it also matches the instruction encoding.
387                    let hi20 = pcrel.wrapping_add(0x800) & 0xFFFFF000;
388                    let lo12 = pcrel.wrapping_sub(hi20) & 0xFFF;
389
390                    unsafe {
391                        // Do a R_RISCV_PCREL_HI20 on the `auipc`
392                        let auipc_addr = at.cast::<u32>();
393                        modify_inst32(auipc_addr, |auipc| (auipc & 0xFFF) | hi20);
394
395                        // Do a R_RISCV_PCREL_LO12_I on the `jalr`
396                        let jalr_addr = at.offset(4).cast::<u32>();
397                        modify_inst32(jalr_addr, |jalr| (jalr & 0xFFFFF) | (lo12 << 20));
398                    }
399                }
400                Reloc::RiscvPCRelHi20 => {
401                    // See <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
402                    // for why `0x800` is added here.
403                    let what = relocation_target_addr(name, addend);
404                    let pcrel = i32::try_from(
405                        what.cast_signed() - at.expose_provenance().cast_signed() + 0x800,
406                    )
407                    .unwrap()
408                    .cast_unsigned();
409                    let at = at.cast::<u32>();
410                    unsafe {
411                        modify_inst32(at, |i| i | (pcrel & 0xfffff000));
412                    }
413                }
414                Reloc::RiscvPCRelLo12I => {
415                    // The target of this relocation is the `auipc` preceding this
416                    // instruction which should be `RiscvPCRelHi20`, and the actual
417                    // target that we're relocating against is the target of that
418                    // relocation. Assume for now that the previous relocation is
419                    // the target of this relocation, and then use that.
420                    let prev_reloc = &self.relocs[i - 1];
421                    assert_eq!(prev_reloc.kind, Reloc::RiscvPCRelHi20);
422                    let lo_target = get_address(name);
423                    let hi_address =
424                        unsafe { self.ptr.offset(isize::try_from(prev_reloc.offset).unwrap()) };
425                    assert_eq!(lo_target, hi_address);
426                    let hi_target = get_address(&prev_reloc.name);
427                    let pcrel = i32::try_from(
428                        hi_target.expose_provenance().cast_signed()
429                            - hi_address.expose_provenance().cast_signed(),
430                    )
431                    .unwrap()
432                    .cast_unsigned();
433                    let at = at.cast::<u32>();
434                    unsafe {
435                        modify_inst32(at, |i| i | ((pcrel & 0xfff) << 20));
436                    }
437                }
438
439                Reloc::PulleyPcRel => {
440                    let what = relocation_target_addr(name, addend);
441                    let pcrel = pcrel_offset(what, at).unwrap();
442                    let at = at.cast::<i32>();
443                    unsafe {
444                        at.write_unaligned(at.read_unaligned().wrapping_add(pcrel));
445                    }
446                }
447
448                other => unimplemented!("unimplemented reloc {other:?}"),
449            }
450        }
451    }
452}