cranelift_jit/compiled_blob.rs
1use std::ptr;
2
3use cranelift_codegen::binemit::{Addend, Reloc};
4use cranelift_module::{ModuleError, ModuleReloc, ModuleRelocTarget, ModuleResult};
5
6use crate::JITMemoryProvider;
7use crate::memory::JITMemoryKind;
8
9/// Size reserved per veneer. This is exactly the size of the largest veneer:
10/// 16 bytes on AArch64 (`ldr` + `br` + 8-byte target address). The x86_64
11/// veneer is 14 bytes (`jmp qword ptr [rip]` + 8-byte target address).
12///
13/// Veneers are placed directly after the compiled code, followed by the GOT
14/// entries: `[code][veneers][GOT entries]`.
15const VENEER_SIZE: usize = 16;
16
17/// Size of a GOT entry: the absolute 8-byte address of a symbol.
18const GOT_ENTRY_SIZE: usize = 8;
19
20/// Reads a 32bit instruction at `iptr`, and writes it again after
21/// being altered by `modifier`
22unsafe fn modify_inst32(iptr: *mut u32, modifier: impl FnOnce(u32) -> u32) {
23 let inst = iptr.read_unaligned();
24 let new_inst = modifier(inst);
25 iptr.write_unaligned(new_inst);
26}
27
28#[derive(Clone)]
29pub(crate) struct CompiledBlob {
30 ptr: *mut u8,
31 size: usize,
32 relocs: Vec<ModuleReloc>,
33 veneer_count: usize,
34 got_count: usize,
35 #[cfg(feature = "wasmtime-unwinder")]
36 wasmtime_exception_data: Option<Vec<u8>>,
37}
38
39unsafe impl Send for CompiledBlob {}
40
41impl CompiledBlob {
42 pub(crate) fn new(
43 memory: &mut dyn JITMemoryProvider,
44 data: &[u8],
45 align: u64,
46 relocs: Vec<ModuleReloc>,
47 #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,
48 kind: JITMemoryKind,
49 ) -> ModuleResult<Self> {
50 // Reserve a worst-case veneer slot for every branch relocation and a
51 // GOT entry for every GOT-relative load in case its target is out of
52 // range.
53 let mut veneer_count = 0;
54 let mut got_count = 0;
55 for reloc in &relocs {
56 match reloc.kind {
57 Reloc::Arm64Call | Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {
58 veneer_count += 1
59 }
60 Reloc::X86GOTPCRel4 => got_count += 1,
61 _ => {}
62 }
63 }
64
65 let mut alloc_size = data.len() + veneer_count * VENEER_SIZE;
66 if got_count > 0 {
67 alloc_size = alloc_size.next_multiple_of(GOT_ENTRY_SIZE) + got_count * GOT_ENTRY_SIZE;
68 }
69
70 let ptr = memory
71 .allocate(alloc_size, align, kind)
72 .map_err(|e| ModuleError::Allocation { err: e })?;
73
74 unsafe {
75 ptr::copy_nonoverlapping(data.as_ptr(), ptr, data.len());
76 }
77
78 Ok(CompiledBlob {
79 ptr,
80 size: data.len(),
81 relocs,
82 veneer_count,
83 got_count,
84 #[cfg(feature = "wasmtime-unwinder")]
85 wasmtime_exception_data,
86 })
87 }
88
89 pub(crate) fn new_zeroed(
90 memory: &mut dyn JITMemoryProvider,
91 size: usize,
92 align: u64,
93 relocs: Vec<ModuleReloc>,
94 #[cfg(feature = "wasmtime-unwinder")] wasmtime_exception_data: Option<Vec<u8>>,
95 kind: JITMemoryKind,
96 ) -> ModuleResult<Self> {
97 let ptr = memory
98 .allocate(size, align, kind)
99 .map_err(|e| ModuleError::Allocation { err: e })?;
100
101 unsafe { ptr::write_bytes(ptr, 0, size) };
102
103 Ok(CompiledBlob {
104 ptr,
105 size,
106 relocs,
107 veneer_count: 0,
108 got_count: 0,
109 #[cfg(feature = "wasmtime-unwinder")]
110 wasmtime_exception_data,
111 })
112 }
113
114 pub(crate) fn ptr(&self) -> *const u8 {
115 self.ptr
116 }
117
118 pub(crate) fn size(&self) -> usize {
119 self.size
120 }
121
122 #[cfg(feature = "wasmtime-unwinder")]
123 pub(crate) fn wasmtime_exception_data(&self) -> Option<&[u8]> {
124 self.wasmtime_exception_data.as_deref()
125 }
126
127 /// Offset of the GOT entries within the allocation: after the code and
128 /// the veneers, aligned to the entry size.
129 fn got_offset(&self) -> usize {
130 (self.size + self.veneer_count * VENEER_SIZE).next_multiple_of(GOT_ENTRY_SIZE)
131 }
132
133 pub(crate) fn perform_relocations(
134 &self,
135 get_address: impl Fn(&ModuleRelocTarget) -> *const u8,
136 ) {
137 use std::ptr::write_unaligned;
138
139 let mut next_veneer_idx = 0;
140 let mut next_got_idx = 0;
141 let relocation_target_addr = |name: &ModuleRelocTarget, addend: Addend| {
142 let addend = isize::try_from(addend).unwrap();
143 get_address(name)
144 .expose_provenance()
145 .checked_add_signed(addend)
146 .unwrap()
147 };
148
149 for (
150 i,
151 &ModuleReloc {
152 kind,
153 offset,
154 ref name,
155 addend,
156 },
157 ) in self.relocs.iter().enumerate()
158 {
159 debug_assert!((offset as usize) < self.size);
160 let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) };
161 match kind {
162 Reloc::Abs4 => {
163 let what = relocation_target_addr(name, addend);
164 unsafe { write_unaligned(at as *mut u32, u32::try_from(what).unwrap()) };
165 }
166 Reloc::Abs8 => {
167 let what = relocation_target_addr(name, addend);
168 unsafe { write_unaligned(at as *mut u64, u64::try_from(what).unwrap()) };
169 }
170 Reloc::X86PCRel4 => {
171 let what = relocation_target_addr(name, addend);
172 let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
173 unsafe { write_unaligned(at as *mut i32, pcrel) };
174 }
175 Reloc::X86CallPCRel4 | Reloc::X86CallPLTRel4 => {
176 // These relocations are applied to the 32-bit displacement of a `call`
177 // or `jmp` instruction, which is relative to the end of the
178 // instruction, i.e. 4 bytes past `at`. The addend (always -4 as
179 // emitted by the x64 backend) accounts for this, so the instruction
180 // transfers control to `what + 4`. The PLT form additionally permits
181 // resolution through a stub, which the veneer below provides.
182 let what = relocation_target_addr(name, addend);
183 if let Ok(pcrel) = i32::try_from((what as isize) - (at as isize)) {
184 unsafe { write_unaligned(at as *mut i32, pcrel) };
185 } else {
186 // The target is out of range for the 32-bit displacement, so
187 // redirect the call through a veneer at the end of the function,
188 // just like for `Arm64Call` below: `jmp qword ptr [rip]` followed
189 // by the absolute target address.
190 let veneer_idx = next_veneer_idx;
191 next_veneer_idx += 1;
192 assert!(veneer_idx < self.veneer_count);
193 let veneer =
194 unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };
195
196 let target = u64::try_from(what.checked_add(4).unwrap()).unwrap();
197 unsafe {
198 ptr::copy_nonoverlapping([0xff, 0x25, 0, 0, 0, 0].as_ptr(), veneer, 6);
199 write_unaligned(veneer.byte_add(6).cast::<u64>(), target);
200 }
201
202 // Point the original instruction at the veneer instead; the
203 // displacement is again relative to the end of the 4-byte field.
204 let pcrel = i32::try_from((veneer as isize) - (at as isize) - 4).unwrap();
205 unsafe { write_unaligned(at as *mut i32, pcrel) };
206 }
207 }
208 Reloc::X86GOTPCRel4 => {
209 // This relocation is applied to the 32-bit displacement of a
210 // `mov reg, qword ptr [rip + disp]` instruction that loads the
211 // address of a symbol from its GOT entry. The addend (always -4 as
212 // emitted by the x64 backend) accounts for the displacement being
213 // relative to the end of the instruction, and any symbol offset is
214 // added by a separately emitted instruction, so the GOT entry holds
215 // the address of the symbol itself: `what + 4`.
216 let what = relocation_target_addr(name, addend);
217 let symbol_addr = u64::try_from(what.checked_add(4).unwrap()).unwrap();
218
219 // If the symbol is within displacement range, relax the load to a
220 // `lea` computing its address directly, the same way linkers relax
221 // `R_X86_64_REX_GOTPCRELX`. The instruction is expected to be a REX
222 // prefix, the `mov` opcode 0x8b, and a ModRM byte with mod=0b00 and
223 // r/m=0b101 (RIP-relative); only rewrite it if it matches.
224 let insn = (offset >= 3)
225 .then(|| unsafe { at.cast_const().sub(3).cast::<[u8; 3]>().read() });
226 let is_rip_relative_mov = insn.is_some_and(
227 |insn| matches!(insn, [0x48..=0x4f, 0x8b, modrm] if modrm & 0xc7 == 0x05),
228 );
229 let direct_pcrel = i32::try_from((what as isize) - (at as isize));
230 match (is_rip_relative_mov, direct_pcrel) {
231 (true, Ok(pcrel)) => unsafe {
232 at.sub(2).write(0x8d); // mov -> lea
233 write_unaligned(at as *mut i32, pcrel);
234 },
235 _ => {
236 // Resolve through a GOT entry at the end of the allocation,
237 // which is always in range of the load.
238 let got_idx = next_got_idx;
239 next_got_idx += 1;
240 assert!(got_idx < self.got_count);
241 let entry = unsafe {
242 self.ptr
243 .byte_add(self.got_offset() + got_idx * GOT_ENTRY_SIZE)
244 };
245 unsafe { write_unaligned(entry.cast::<u64>(), symbol_addr) };
246
247 let pcrel =
248 i32::try_from((entry as isize) - (at as isize) - 4).unwrap();
249 unsafe { write_unaligned(at as *mut i32, pcrel) };
250 }
251 }
252 }
253 Reloc::S390xPCRel32Dbl | Reloc::S390xPLTRel32Dbl => {
254 let what = relocation_target_addr(name, addend);
255 let pcrel = i32::try_from(((what as isize) - (at as isize)) >> 1).unwrap();
256 unsafe { write_unaligned(at as *mut i32, pcrel) };
257 }
258 Reloc::Arm64Call => {
259 let what = relocation_target_addr(name, addend);
260 // The instruction is 32 bits long.
261 let iptr = at as *mut u32;
262
263 // The offset encoded in the `bl` instruction is the
264 // number of bytes divided by 4.
265 let diff = ((what as isize) - (at as isize)) >> 2;
266 // Sign propagating right shift disposes of the
267 // included bits, so the result is expected to be
268 // either all sign bits or 0 when in-range, depending
269 // on if the original value was negative or positive.
270 if (diff >> 25 == -1) || (diff >> 25 == 0) {
271 // The lower 26 bits of the `bl` instruction form the
272 // immediate offset argument.
273 let chop = 32 - 26;
274 let imm26 = (diff as u32) << chop >> chop;
275 unsafe { modify_inst32(iptr, |inst| inst | imm26) };
276 } else {
277 // If the target is out of range for a direct call, insert a veneer at the
278 // end of the function.
279 let veneer_idx = next_veneer_idx;
280 next_veneer_idx += 1;
281 assert!(veneer_idx < self.veneer_count);
282 let veneer =
283 unsafe { self.ptr.byte_add(self.size + veneer_idx * VENEER_SIZE) };
284
285 // Write the veneer
286 // x16 is reserved as scratch register to be used by veneers and PLT entries
287 unsafe {
288 write_unaligned(
289 veneer.cast::<u32>(),
290 0x58000050, // ldr x16, 0x8
291 );
292 write_unaligned(
293 veneer.byte_add(4).cast::<u32>(),
294 0xd61f0200, // br x16
295 );
296 write_unaligned(veneer.byte_add(8).cast::<u64>(), what as u64);
297 };
298
299 // Set the veneer as target of the call
300 let diff = ((veneer as isize) - (at as isize)) >> 2;
301 assert!((diff >> 25 == -1) || (diff >> 25 == 0));
302 let chop = 32 - 26;
303 let imm26 = (diff as u32) << chop >> chop;
304 unsafe { modify_inst32(iptr, |inst| inst | imm26) };
305 }
306 }
307 Reloc::Aarch64AdrGotPage21 => {
308 panic!("GOT relocation shouldn't be generated when !is_pic");
309 }
310 Reloc::Aarch64Ld64GotLo12Nc => {
311 panic!("GOT relocation shouldn't be generated when !is_pic");
312 }
313 Reloc::Aarch64AdrPrelPgHi21 => {
314 let what = relocation_target_addr(name, addend);
315 let get_page = |x| x & (!0xfff);
316 // NOTE: This should technically be i33 given that this relocation type allows
317 // a range from -4GB to +4GB, not -2GB to +2GB. But this doesn't really matter
318 // as the target is unlikely to be more than 2GB from the adrp instruction. We
319 // need to be careful to not cast to an unsigned int until after doing >> 12 to
320 // compute the upper 21bits of the pcrel address however as otherwise the top
321 // bit of the 33bit pcrel address would be forced 0 through zero extension
322 // instead of being sign extended as it should be.
323 let pcrel =
324 i32::try_from(get_page(what as isize) - get_page(at as isize)).unwrap();
325 let iptr = at as *mut u32;
326 let hi21 = (pcrel >> 12).cast_unsigned();
327 let lo = (hi21 & 0x3) << 29;
328 let hi = (hi21 & 0x1ffffc) << 3;
329 unsafe { modify_inst32(iptr, |inst| inst | lo | hi) };
330 }
331 Reloc::Aarch64AddAbsLo12Nc => {
332 let what = relocation_target_addr(name, addend);
333 let iptr = at as *mut u32;
334 let imm12 = (what as u32 & 0xfff) << 10;
335 unsafe { modify_inst32(iptr, |inst| inst | imm12) };
336 }
337 Reloc::RiscvCallPlt => {
338 // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
339 // It is the equivalent of two relocations:
340 // 1. R_RISCV_PCREL_HI20 on the `auipc`
341 // 2. R_RISCV_PCREL_LO12_I on the `jalr`
342
343 let what = relocation_target_addr(name, addend);
344 let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap() as u32;
345
346 // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
347 // for a better explanation of the following code.
348 //
349 // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
350 //
351 // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
352 // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
353 // get a valid address.
354 //
355 // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
356 //
357 // `hi20` should also be shifted right to be the "true" value. But we also need it
358 // left shifted for the `lo12` calculation and it also matches the instruction encoding.
359 let hi20 = pcrel.wrapping_add(0x800) & 0xFFFFF000;
360 let lo12 = pcrel.wrapping_sub(hi20) & 0xFFF;
361
362 unsafe {
363 // Do a R_RISCV_PCREL_HI20 on the `auipc`
364 let auipc_addr = at as *mut u32;
365 modify_inst32(auipc_addr, |auipc| (auipc & 0xFFF) | hi20);
366
367 // Do a R_RISCV_PCREL_LO12_I on the `jalr`
368 let jalr_addr = at.offset(4) as *mut u32;
369 modify_inst32(jalr_addr, |jalr| (jalr & 0xFFFFF) | (lo12 << 20));
370 }
371 }
372 Reloc::PulleyPcRel => {
373 let what = relocation_target_addr(name, addend);
374 let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
375 let at = at as *mut i32;
376 unsafe {
377 at.write_unaligned(at.read_unaligned().wrapping_add(pcrel));
378 }
379 }
380
381 // See <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
382 // for why `0x800` is added here.
383 Reloc::RiscvPCRelHi20 => {
384 let what = relocation_target_addr(name, addend);
385 let pcrel = i32::try_from((what as isize) - (at as isize) + 0x800)
386 .unwrap()
387 .cast_unsigned();
388 let at = at as *mut u32;
389 unsafe {
390 modify_inst32(at, |i| i | (pcrel & 0xfffff000));
391 }
392 }
393
394 // The target of this relocation is the `auipc` preceding this
395 // instruction which should be `RiscvPCRelHi20`, and the actual
396 // target that we're relocating against is the target of that
397 // relocation. Assume for now that the previous relocation is
398 // the target of this relocation, and then use that.
399 Reloc::RiscvPCRelLo12I => {
400 let prev_reloc = &self.relocs[i - 1];
401 assert_eq!(prev_reloc.kind, Reloc::RiscvPCRelHi20);
402 let lo_target = get_address(name);
403 let hi_address =
404 unsafe { self.ptr.offset(isize::try_from(prev_reloc.offset).unwrap()) };
405 assert_eq!(lo_target, hi_address);
406 let hi_target = get_address(&prev_reloc.name);
407 let pcrel = i32::try_from((hi_target as isize) - (hi_address as isize))
408 .unwrap()
409 .cast_unsigned();
410 let at = at as *mut u32;
411 unsafe {
412 modify_inst32(at, |i| i | ((pcrel & 0xfff) << 20));
413 }
414 }
415
416 other => unimplemented!("unimplemented reloc {other:?}"),
417 }
418 }
419 }
420}