cranelift_jit/
compiled_blob.rs

1use cranelift_codegen::binemit::Reloc;
2use cranelift_module::ModuleReloc;
3use cranelift_module::ModuleRelocTarget;
4
5/// Reads a 32bit instruction at `iptr`, and writes it again after
6/// being altered by `modifier`
7unsafe fn modify_inst32(iptr: *mut u32, modifier: impl FnOnce(u32) -> u32) {
8    let inst = iptr.read_unaligned();
9    let new_inst = modifier(inst);
10    iptr.write_unaligned(new_inst);
11}
12
13#[derive(Clone)]
14pub(crate) struct CompiledBlob {
15    pub(crate) ptr: *mut u8,
16    pub(crate) size: usize,
17    pub(crate) relocs: Vec<ModuleReloc>,
18    #[cfg(feature = "wasmtime-unwinder")]
19    pub(crate) exception_data: Option<Vec<u8>>,
20}
21
22unsafe impl Send for CompiledBlob {}
23
24impl CompiledBlob {
25    pub(crate) fn perform_relocations(
26        &self,
27        get_address: impl Fn(&ModuleRelocTarget) -> *const u8,
28    ) {
29        use std::ptr::write_unaligned;
30
31        for (
32            i,
33            &ModuleReloc {
34                kind,
35                offset,
36                ref name,
37                addend,
38            },
39        ) in self.relocs.iter().enumerate()
40        {
41            debug_assert!((offset as usize) < self.size);
42            let at = unsafe { self.ptr.offset(isize::try_from(offset).unwrap()) };
43            match kind {
44                Reloc::Abs4 => {
45                    let base = get_address(name);
46                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
47                    unsafe {
48                        write_unaligned(at as *mut u32, u32::try_from(what as usize).unwrap())
49                    };
50                }
51                Reloc::Abs8 => {
52                    let base = get_address(name);
53                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
54                    unsafe {
55                        write_unaligned(at as *mut u64, u64::try_from(what as usize).unwrap())
56                    };
57                }
58                Reloc::X86PCRel4 | Reloc::X86CallPCRel4 => {
59                    let base = get_address(name);
60                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
61                    let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
62                    unsafe { write_unaligned(at as *mut i32, pcrel) };
63                }
64                Reloc::X86GOTPCRel4 => {
65                    panic!("GOT relocation shouldn't be generated when !is_pic");
66                }
67                Reloc::X86CallPLTRel4 => {
68                    panic!("PLT relocation shouldn't be generated when !is_pic");
69                }
70                Reloc::S390xPCRel32Dbl | Reloc::S390xPLTRel32Dbl => {
71                    let base = get_address(name);
72                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
73                    let pcrel = i32::try_from(((what as isize) - (at as isize)) >> 1).unwrap();
74                    unsafe { write_unaligned(at as *mut i32, pcrel) };
75                }
76                Reloc::Arm64Call => {
77                    let base = get_address(name);
78                    // The instruction is 32 bits long.
79                    let iptr = at as *mut u32;
80                    // The offset encoded in the `bl` instruction is the
81                    // number of bytes divided by 4.
82                    let diff = ((base as isize) - (at as isize)) >> 2;
83                    // Sign propagating right shift disposes of the
84                    // included bits, so the result is expected to be
85                    // either all sign bits or 0, depending on if the original
86                    // value was negative or positive.
87                    assert!((diff >> 26 == -1) || (diff >> 26 == 0));
88                    // The lower 26 bits of the `bl` instruction form the
89                    // immediate offset argument.
90                    let chop = 32 - 26;
91                    let imm26 = (diff as u32) << chop >> chop;
92                    unsafe { modify_inst32(iptr, |inst| inst | imm26) };
93                }
94                Reloc::Aarch64AdrGotPage21 => {
95                    panic!("GOT relocation shouldn't be generated when !is_pic");
96                }
97                Reloc::Aarch64Ld64GotLo12Nc => {
98                    panic!("GOT relocation shouldn't be generated when !is_pic");
99                }
100                Reloc::Aarch64AdrPrelPgHi21 => {
101                    let base = get_address(name);
102                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
103                    let get_page = |x| x & (!0xfff);
104                    let pcrel = i32::try_from(get_page(what as isize) - get_page(at as isize))
105                        .unwrap()
106                        .cast_unsigned();
107                    let iptr = at as *mut u32;
108                    let hi21 = pcrel >> 12;
109                    let lo = (hi21 & 0x3) << 29;
110                    let hi = (hi21 & 0x1ffffc) << 3;
111                    unsafe { modify_inst32(iptr, |inst| inst | lo | hi) };
112                }
113                Reloc::Aarch64AddAbsLo12Nc => {
114                    let base = get_address(name);
115                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
116                    let iptr = at as *mut u32;
117                    let imm12 = (what.addr() as u32 & 0xfff) << 10;
118                    unsafe { modify_inst32(iptr, |inst| inst | imm12) };
119                }
120                Reloc::RiscvCallPlt => {
121                    // A R_RISCV_CALL_PLT relocation expects auipc+jalr instruction pair.
122                    // It is the equivalent of two relocations:
123                    // 1. R_RISCV_PCREL_HI20 on the `auipc`
124                    // 2. R_RISCV_PCREL_LO12_I on the `jalr`
125
126                    let base = get_address(name);
127                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
128                    let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap() as u32;
129
130                    // See https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses
131                    // for a better explanation of the following code.
132                    //
133                    // Unlike the regular symbol relocations, here both "sub-relocations" point to the same address.
134                    //
135                    // `pcrel` is a signed value (+/- 2GiB range), when splitting it into two parts, we need to
136                    // ensure that `hi20` is close enough to `pcrel` to be able to add `lo12` to it and still
137                    // get a valid address.
138                    //
139                    // `lo12` is also a signed offset (+/- 2KiB range) relative to the `hi20` value.
140                    //
141                    // `hi20` should also be shifted right to be the "true" value. But we also need it
142                    // left shifted for the `lo12` calculation and it also matches the instruction encoding.
143                    let hi20 = pcrel.wrapping_add(0x800) & 0xFFFFF000;
144                    let lo12 = pcrel.wrapping_sub(hi20) & 0xFFF;
145
146                    unsafe {
147                        // Do a R_RISCV_PCREL_HI20 on the `auipc`
148                        let auipc_addr = at as *mut u32;
149                        modify_inst32(auipc_addr, |auipc| (auipc & 0xFFF) | hi20);
150
151                        // Do a R_RISCV_PCREL_LO12_I on the `jalr`
152                        let jalr_addr = at.offset(4) as *mut u32;
153                        modify_inst32(jalr_addr, |jalr| (jalr & 0xFFFFF) | (lo12 << 20));
154                    }
155                }
156                Reloc::PulleyPcRel => {
157                    let base = get_address(name);
158                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
159                    let pcrel = i32::try_from((what as isize) - (at as isize)).unwrap();
160                    let at = at as *mut i32;
161                    unsafe {
162                        at.write_unaligned(at.read_unaligned().wrapping_add(pcrel));
163                    }
164                }
165
166                // See <https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-elf.adoc#pc-relative-symbol-addresses>
167                // for why `0x800` is added here.
168                Reloc::RiscvPCRelHi20 => {
169                    let base = get_address(name);
170                    let what = unsafe { base.offset(isize::try_from(addend).unwrap()) };
171                    let pcrel = i32::try_from((what as isize) - (at as isize) + 0x800)
172                        .unwrap()
173                        .cast_unsigned();
174                    let at = at as *mut u32;
175                    unsafe {
176                        modify_inst32(at, |i| i | (pcrel & 0xfffff000));
177                    }
178                }
179
180                // The target of this relocation is the `auipc` preceding this
181                // instruction which should be `RiscvPCRelHi20`, and the actual
182                // target that we're relocating against is the target of that
183                // relocation. Assume for now that the previous relocation is
184                // the target of this relocation, and then use that.
185                Reloc::RiscvPCRelLo12I => {
186                    let prev_reloc = &self.relocs[i - 1];
187                    assert_eq!(prev_reloc.kind, Reloc::RiscvPCRelHi20);
188                    let lo_target = get_address(name);
189                    let hi_address =
190                        unsafe { self.ptr.offset(isize::try_from(prev_reloc.offset).unwrap()) };
191                    assert_eq!(lo_target, hi_address);
192                    let hi_target = get_address(&prev_reloc.name);
193                    let pcrel = i32::try_from((hi_target as isize) - (hi_address as isize))
194                        .unwrap()
195                        .cast_unsigned();
196                    let at = at as *mut u32;
197                    unsafe {
198                        modify_inst32(at, |i| i | ((pcrel & 0xfff) << 20));
199                    }
200                }
201
202                other => unimplemented!("unimplemented reloc {other:?}"),
203            }
204        }
205    }
206}
cranelift_jit/compiled_blob.rs

cranelift_jit/
compiled_blob.rs
