cranelift_jit/compiled_blob.rs
1use cranelift_codegen::binemit::Reloc;
2use cranelift_module::ModuleReloc;
3use cranelift_module::ModuleRelocTarget;
4
5/// Reads a 32bit instruction at `iptr`, and writes it again after
6/// being altered by `modifier`
7unsafe fn modify_inst32(iptr: *mut u32, modifier: impl FnOnce(u32) -> u32) {
8 let inst = iptr.read_unaligned();
9 let new_inst = modifier(inst);
10 iptr.write_unaligned(new_inst);
11}
12
13#[derive(Clone)]
14pub(crate) struct CompiledBlob {
15 pub(crate) ptr: *mut u8,
16 pub(crate) size: usize,
17 pub(crate) relocs: Vec<ModuleReloc>,
18 #[cfg(feature = "wasmtime-unwinder")]
19 pub(crate) exception_data: Option<Vec<u8>>,
20}
21
22unsafe impl Send for CompiledBlob {}
23
24impl CompiledBlob {
25 pub(crate) fn perform_relocations(
26 &self,
27 get_address: impl Fn(&ModuleRelocTarget) -> *const u8,
28 ) {
29 use std::ptr::write_unaligned;
30
31 for (
32 i,
33 &ModuleReloc {
34 kind,
35 offset,
36 ref name,
37 addend,
38 },
39 ) in self.relocs.iter().enumerate()
40 {
41 debug_assert!((offset as usize) < self.size);
42 let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) };
43 match kind {
44 Reloc::Abs4 => {
45 let base = get_address(name);
46 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
47 unsafe {
48 write_unaligned(at as *mut u32, u32::try_from(what as usize).unwrap())
49 };
50 }
51 Reloc::Abs8 => {
52 let base = get_address(name);
53 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
54 unsafe {
55 write_unaligned(at as *mut u64, u64::try_from(what as usize).unwrap())
56 };
57 }
58 Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
59 let base = get_address(name);
60 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
61 let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
62 unsafe { write_unaligned(at as *mut i32, pcrel) };
63 }
64 Reloc::X86GOTPCRel4 => {
65 panic!("GOT relocation shouldn't be generated when !is_pic");
66 }
67 Reloc::X86CallPLTRel4 => {
68 panic!("PLT relocation shouldn't be generated when !is_pic");
69 }
70 Reloc::S390xPCRel32Dbl | Reloc::S390xPLTRel32Dbl => {
71 let base = get_address(name);
72 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
73 let pcrel = i32::try_from(((what as isize) - (at as isize)) >> 1).unwrap();
74 unsafe { write_unaligned(at as *mut i32, pcrel) };
75 }
76 Reloc::Arm64Call => {
77 let base = get_address(name);
78 // The instruction is 32 bits long.
79 let iptr = at as *mut u32;
80 // The offset encoded in the `bl` instruction is the
81 // number of bytes divided by 4.
82 let diff = ((base as isize) - (at as isize)) >> 2;
83 // Sign propagating right shift disposes of the
84 // included bits, so the result is expected to be
85 // either all sign bits or 0, depending on if the original
86 // value was negative or positive.
87 assert!((diff >> 26 == -1) || (diff >> 26 == 0));
88 // The lower 26 bits of the `bl` instruction form the
89 // immediate offset argument.
90 let chop = 32 - 26;
91 let imm26 = (diff as u32) << chop >> chop;
92 unsafe { modify_inst32(iptr, |inst| inst | imm26) };
93 }
94 Reloc::Aarch64AdrGotPage21 => {
95 panic!("GOT relocation shouldn't be generated when !is_pic");
96 }
97 Reloc::Aarch64Ld64GotLo12Nc => {
98 panic!("GOT relocation shouldn't be generated when !is_pic");
99 }
100 Reloc::Aarch64AdrPrelPgHi21 => {
101 let base = get_address(name);
102 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
103 let get_page = |x| x & (!0xfff);
104 // NOTE: This should technically be i33 given that this relocation type allows
105 // a range from -4GB to +4GB, not -2GB to +2GB. But this doesn't really matter
106 // as the target is unlikely to be more than 2GB from the adrp instruction. We
107 // need to be careful to not cast to an unsigned int until after doing >> 12 to
108 // compute the upper 21bits of the pcrel address however as otherwise the top
109 // bit of the 33bit pcrel address would be forced 0 through zero extension
110 // instead of being sign extended as it should be.
111 let pcrel =
112 i32::try_from(get_page(what as isize) - get_page(at as isize)).unwrap();
113 let iptr = at as *mut u32;
114 let hi21 = (pcrel >> 12).cast_unsigned();
115 let lo = (hi21 & 0x3) << 29;
116 let hi = (hi21 & 0x1ffffc) << 3;
117 unsafe { modify_inst32(iptr, |inst| inst | lo | hi) };
118 }
119 Reloc::Aarch64AddAbsLo12Nc => {
120 let base = get_address(name);
121 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
122 let iptr = at as *mut u32;
123 let imm12 = (what.addr() as u32 & 0xfff) << 10;
124 unsafe { modify_inst32(iptr, |inst| inst | imm12) };
125 }
126 Reloc::RiscvCallPlt => {
127 // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
128 // It is the equivalent of two relocations:
129 // 1. R_RISCV_PCREL_HI20 on the `auipc`
130 // 2. R_RISCV_PCREL_LO12_I on the `jalr`
131
132 let base = get_address(name);
133 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
134 let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap() as u32;
135
136 // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
137 // for a better explanation of the following code.
138 //
139 // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
140 //
141 // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
142 // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
143 // get a valid address.
144 //
145 // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
146 //
147 // `hi20` should also be shifted right to be the "true" value. But we also need it
148 // left shifted for the `lo12` calculation and it also matches the instruction encoding.
149 let hi20 = pcrel.wrapping_add(0x800) & 0xFFFFF000;
150 let lo12 = pcrel.wrapping_sub(hi20) & 0xFFF;
151
152 unsafe {
153 // Do a R_RISCV_PCREL_HI20 on the `auipc`
154 let auipc_addr = at as *mut u32;
155 modify_inst32(auipc_addr, |auipc| (auipc & 0xFFF) | hi20);
156
157 // Do a R_RISCV_PCREL_LO12_I on the `jalr`
158 let jalr_addr = at.offset(4) as *mut u32;
159 modify_inst32(jalr_addr, |jalr| (jalr & 0xFFFFF) | (lo12 << 20));
160 }
161 }
162 Reloc::PulleyPcRel => {
163 let base = get_address(name);
164 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
165 let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
166 let at = at as *mut i32;
167 unsafe {
168 at.write_unaligned(at.read_unaligned().wrapping_add(pcrel));
169 }
170 }
171
172 // See <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
173 // for why `0x800` is added here.
174 Reloc::RiscvPCRelHi20 => {
175 let base = get_address(name);
176 let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
177 let pcrel = i32::try_from((what as isize) - (at as isize) + 0x800)
178 .unwrap()
179 .cast_unsigned();
180 let at = at as *mut u32;
181 unsafe {
182 modify_inst32(at, |i| i | (pcrel & 0xfffff000));
183 }
184 }
185
186 // The target of this relocation is the `auipc` preceding this
187 // instruction which should be `RiscvPCRelHi20`, and the actual
188 // target that we're relocating against is the target of that
189 // relocation. Assume for now that the previous relocation is
190 // the target of this relocation, and then use that.
191 Reloc::RiscvPCRelLo12I => {
192 let prev_reloc = &self.relocs[i - 1];
193 assert_eq!(prev_reloc.kind, Reloc::RiscvPCRelHi20);
194 let lo_target = get_address(name);
195 let hi_address =
196 unsafe { self.ptr.offset(isize::try_from(prev_reloc.offset).unwrap()) };
197 assert_eq!(lo_target, hi_address);
198 let hi_target = get_address(&prev_reloc.name);
199 let pcrel = i32::try_from((hi_target as isize) - (hi_address as isize))
200 .unwrap()
201 .cast_unsigned();
202 let at = at as *mut u32;
203 unsafe {
204 modify_inst32(at, |i| i | ((pcrel & 0xfff) << 20));
205 }
206 }
207
208 other => unimplemented!("unimplemented reloc {other:?}"),
209 }
210 }
211 }
212}