wasmtime_cranelift/
lib.rs

1//! Support for compiling with Cranelift.
2//!
3//! This crate provides an implementation of the `wasmtime_environ::Compiler`
4//! and `wasmtime_environ::CompilerBuilder` traits.
5
6// See documentation in crates/wasmtime/src/runtime.rs for why this is
7// selectively enabled here.
8#![warn(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
9
10use cranelift_codegen::{
11    binemit,
12    cursor::FuncCursor,
13    ir::{self, AbiParam, ArgumentPurpose, ExternalName, InstBuilder, Signature, TrapCode},
14    isa::{CallConv, TargetIsa},
15    settings, FinalizedMachReloc, FinalizedRelocTarget, MachTrap,
16};
17use cranelift_entity::PrimaryMap;
18
19use target_lexicon::Architecture;
20use wasmtime_environ::{
21    BuiltinFunctionIndex, FlagValue, FuncIndex, RelocationTarget, Trap, TrapInformation, Tunables,
22    WasmFuncType, WasmHeapTopType, WasmHeapType, WasmValType,
23};
24
25pub use builder::builder;
26
27pub mod isa_builder;
28mod obj;
29pub use obj::*;
30mod compiled_function;
31pub use compiled_function::*;
32
33mod builder;
34mod compiler;
35mod debug;
36mod func_environ;
37mod gc;
38mod translate;
39
40use self::compiler::Compiler;
41
42const TRAP_INTERNAL_ASSERT: TrapCode = TrapCode::unwrap_user(1);
43const TRAP_OFFSET: u8 = 2;
44pub const TRAP_ALWAYS: TrapCode =
45    TrapCode::unwrap_user(Trap::AlwaysTrapAdapter as u8 + TRAP_OFFSET);
46pub const TRAP_CANNOT_ENTER: TrapCode =
47    TrapCode::unwrap_user(Trap::CannotEnterComponent as u8 + TRAP_OFFSET);
48pub const TRAP_INDIRECT_CALL_TO_NULL: TrapCode =
49    TrapCode::unwrap_user(Trap::IndirectCallToNull as u8 + TRAP_OFFSET);
50pub const TRAP_BAD_SIGNATURE: TrapCode =
51    TrapCode::unwrap_user(Trap::BadSignature as u8 + TRAP_OFFSET);
52pub const TRAP_NULL_REFERENCE: TrapCode =
53    TrapCode::unwrap_user(Trap::NullReference as u8 + TRAP_OFFSET);
54pub const TRAP_ALLOCATION_TOO_LARGE: TrapCode =
55    TrapCode::unwrap_user(Trap::AllocationTooLarge as u8 + TRAP_OFFSET);
56pub const TRAP_ARRAY_OUT_OF_BOUNDS: TrapCode =
57    TrapCode::unwrap_user(Trap::ArrayOutOfBounds as u8 + TRAP_OFFSET);
58pub const TRAP_UNREACHABLE: TrapCode =
59    TrapCode::unwrap_user(Trap::UnreachableCodeReached as u8 + TRAP_OFFSET);
60pub const TRAP_HEAP_MISALIGNED: TrapCode =
61    TrapCode::unwrap_user(Trap::HeapMisaligned as u8 + TRAP_OFFSET);
62pub const TRAP_TABLE_OUT_OF_BOUNDS: TrapCode =
63    TrapCode::unwrap_user(Trap::TableOutOfBounds as u8 + TRAP_OFFSET);
64pub const TRAP_CAST_FAILURE: TrapCode =
65    TrapCode::unwrap_user(Trap::CastFailure as u8 + TRAP_OFFSET);
66
67/// Creates a new cranelift `Signature` with no wasm params/results for the
68/// given calling convention.
69///
70/// This will add the default vmctx/etc parameters to the signature returned.
71fn blank_sig(isa: &dyn TargetIsa, call_conv: CallConv) -> ir::Signature {
72    let pointer_type = isa.pointer_type();
73    let mut sig = ir::Signature::new(call_conv);
74    // Add the caller/callee `vmctx` parameters.
75    sig.params.push(ir::AbiParam::special(
76        pointer_type,
77        ir::ArgumentPurpose::VMContext,
78    ));
79    sig.params.push(ir::AbiParam::new(pointer_type));
80    return sig;
81}
82
83/// Emit code for the following unbarriered memory write of the given type:
84///
85/// ```ignore
86/// *(base + offset) = value
87/// ```
88///
89/// This is intended to be used with things like `ValRaw` and the array calling
90/// convention.
91fn unbarriered_store_type_at_offset(
92    pos: &mut FuncCursor,
93    flags: ir::MemFlags,
94    base: ir::Value,
95    offset: i32,
96    value: ir::Value,
97) {
98    pos.ins().store(flags, value, base, offset);
99}
100
101/// Emit code to do the following unbarriered memory read of the given type and
102/// with the given flags:
103///
104/// ```ignore
105/// result = *(base + offset)
106/// ```
107///
108/// This is intended to be used with things like `ValRaw` and the array calling
109/// convention.
110fn unbarriered_load_type_at_offset(
111    isa: &dyn TargetIsa,
112    pos: &mut FuncCursor,
113    ty: WasmValType,
114    flags: ir::MemFlags,
115    base: ir::Value,
116    offset: i32,
117) -> ir::Value {
118    let ir_ty = value_type(isa, ty);
119    pos.ins().load(ir_ty, flags, base, offset)
120}
121
122/// Returns the corresponding cranelift type for the provided wasm type.
123fn value_type(isa: &dyn TargetIsa, ty: WasmValType) -> ir::types::Type {
124    match ty {
125        WasmValType::I32 => ir::types::I32,
126        WasmValType::I64 => ir::types::I64,
127        WasmValType::F32 => ir::types::F32,
128        WasmValType::F64 => ir::types::F64,
129        WasmValType::V128 => ir::types::I8X16,
130        WasmValType::Ref(rt) => reference_type(rt.heap_type, isa.pointer_type()),
131    }
132}
133
134/// Get the Cranelift signature for all array-call functions, that is:
135///
136/// ```ignore
137/// unsafe extern "C" fn(
138///     callee_vmctx: *mut VMOpaqueContext,
139///     caller_vmctx: *mut VMOpaqueContext,
140///     values_ptr: *mut ValRaw,
141///     values_len: usize,
142/// )
143/// ```
144///
145/// This signature uses the target's default calling convention.
146///
147/// Note that regardless of the Wasm function type, the array-call calling
148/// convention always uses that same signature.
149fn array_call_signature(isa: &dyn TargetIsa) -> ir::Signature {
150    let mut sig = blank_sig(isa, CallConv::triple_default(isa.triple()));
151    // The array-call signature has an added parameter for the `values_vec`
152    // input/output buffer in addition to the size of the buffer, in units
153    // of `ValRaw`.
154    sig.params.push(ir::AbiParam::new(isa.pointer_type()));
155    sig.params.push(ir::AbiParam::new(isa.pointer_type()));
156    // boolean return value of whether this function trapped
157    sig.returns.push(ir::AbiParam::new(ir::types::I8));
158    sig
159}
160
161/// Get the internal Wasm calling convention for the target/tunables combo
162fn wasm_call_conv(isa: &dyn TargetIsa, tunables: &Tunables) -> CallConv {
163    // The default calling convention is `CallConv::Tail` to enable the use of
164    // tail calls in modules when needed. Note that this is used even if the
165    // tail call proposal is disabled in wasm. This is not interacted with on
166    // the host so it's purely an internal detail of wasm itself.
167    //
168    // The Winch calling convention is used instead when generating trampolines
169    // which call Winch-generated functions. The winch calling convention is
170    // only implemented for x64 and aarch64, so assert that here and panic on
171    // other architectures.
172    if tunables.winch_callable {
173        assert!(
174            matches!(
175                isa.triple().architecture,
176                Architecture::X86_64 | Architecture::Aarch64(_)
177            ),
178            "The Winch calling convention is only implemented for x86_64 and aarch64"
179        );
180        CallConv::Winch
181    } else {
182        CallConv::Tail
183    }
184}
185
186/// Get the internal Wasm calling convention signature for the given type.
187fn wasm_call_signature(
188    isa: &dyn TargetIsa,
189    wasm_func_ty: &WasmFuncType,
190    tunables: &Tunables,
191) -> ir::Signature {
192    let call_conv = wasm_call_conv(isa, tunables);
193    let mut sig = blank_sig(isa, call_conv);
194    let cvt = |ty: &WasmValType| ir::AbiParam::new(value_type(isa, *ty));
195    sig.params.extend(wasm_func_ty.params().iter().map(&cvt));
196    sig.returns.extend(wasm_func_ty.returns().iter().map(&cvt));
197    sig
198}
199
200/// Returns the reference type to use for the provided wasm type.
201fn reference_type(wasm_ht: WasmHeapType, pointer_type: ir::Type) -> ir::Type {
202    match wasm_ht.top() {
203        WasmHeapTopType::Func => pointer_type,
204        WasmHeapTopType::Any | WasmHeapTopType::Extern => ir::types::I32,
205        WasmHeapTopType::Cont => todo!(), // FIXME: #10248 stack switching support.
206    }
207}
208
209// List of namespaces which are processed in `mach_reloc_to_reloc` below.
210
211/// Namespace corresponding to wasm functions, the index is the index of the
212/// defined function that's being referenced.
213pub const NS_WASM_FUNC: u32 = 0;
214
215/// Namespace for builtin function trampolines. The index is the index of the
216/// builtin that's being referenced. These trampolines invoke the real host
217/// function through an indirect function call loaded by the `VMContext`.
218pub const NS_WASMTIME_BUILTIN: u32 = 1;
219
220/// Namespace used to when a call from Pulley to the host is being made. This is
221/// used with a `colocated: false` name to trigger codegen for a special opcode
222/// for pulley-to-host communication. The index of the functions used in this
223/// namespace correspond to the function signature of `for_each_host_signature!`
224/// in the pulley_interpreter crate.
225pub const NS_PULLEY_HOSTCALL: u32 = 2;
226
227/// A record of a relocation to perform.
228#[derive(Debug, Clone, PartialEq, Eq)]
229pub struct Relocation {
230    /// The relocation code.
231    pub reloc: binemit::Reloc,
232    /// Relocation target.
233    pub reloc_target: RelocationTarget,
234    /// The offset where to apply the relocation.
235    pub offset: binemit::CodeOffset,
236    /// The addend to add to the relocation value.
237    pub addend: binemit::Addend,
238}
239
240/// Converts cranelift_codegen settings to the wasmtime_environ equivalent.
241pub fn clif_flags_to_wasmtime(
242    flags: impl IntoIterator<Item = settings::Value>,
243) -> Vec<(&'static str, FlagValue<'static>)> {
244    flags
245        .into_iter()
246        .map(|val| (val.name, to_flag_value(&val)))
247        .collect()
248}
249
250fn to_flag_value(v: &settings::Value) -> FlagValue<'static> {
251    match v.kind() {
252        settings::SettingKind::Enum => FlagValue::Enum(v.as_enum().unwrap()),
253        settings::SettingKind::Num => FlagValue::Num(v.as_num().unwrap()),
254        settings::SettingKind::Bool => FlagValue::Bool(v.as_bool().unwrap()),
255        settings::SettingKind::Preset => unreachable!(),
256    }
257}
258
259/// Converts machine traps to trap information.
260pub fn mach_trap_to_trap(trap: &MachTrap) -> Option<TrapInformation> {
261    let &MachTrap { offset, code } = trap;
262    Some(TrapInformation {
263        code_offset: offset,
264        trap_code: clif_trap_to_env_trap(code)?,
265    })
266}
267
268fn clif_trap_to_env_trap(trap: ir::TrapCode) -> Option<Trap> {
269    Some(match trap {
270        ir::TrapCode::STACK_OVERFLOW => Trap::StackOverflow,
271        ir::TrapCode::HEAP_OUT_OF_BOUNDS => Trap::MemoryOutOfBounds,
272        ir::TrapCode::INTEGER_OVERFLOW => Trap::IntegerOverflow,
273        ir::TrapCode::INTEGER_DIVISION_BY_ZERO => Trap::IntegerDivisionByZero,
274        ir::TrapCode::BAD_CONVERSION_TO_INTEGER => Trap::BadConversionToInteger,
275
276        // These do not get converted to wasmtime traps, since they
277        // shouldn't ever be hit in theory. Instead of catching and handling
278        // these, we let the signal crash the process.
279        TRAP_INTERNAL_ASSERT => return None,
280
281        other => Trap::from_u8(other.as_raw().get() - TRAP_OFFSET).unwrap(),
282    })
283}
284
285/// Converts machine relocations to relocation information
286/// to perform.
287fn mach_reloc_to_reloc(
288    reloc: &FinalizedMachReloc,
289    name_map: &PrimaryMap<ir::UserExternalNameRef, ir::UserExternalName>,
290) -> Relocation {
291    let &FinalizedMachReloc {
292        offset,
293        kind,
294        ref target,
295        addend,
296    } = reloc;
297    let reloc_target = match *target {
298        FinalizedRelocTarget::ExternalName(ExternalName::User(user_func_ref)) => {
299            let name = &name_map[user_func_ref];
300            match name.namespace {
301                NS_WASM_FUNC => RelocationTarget::Wasm(FuncIndex::from_u32(name.index)),
302                NS_WASMTIME_BUILTIN => {
303                    RelocationTarget::Builtin(BuiltinFunctionIndex::from_u32(name.index))
304                }
305                NS_PULLEY_HOSTCALL => RelocationTarget::PulleyHostcall(name.index),
306                _ => panic!("unknown namespace {}", name.namespace),
307            }
308        }
309        FinalizedRelocTarget::ExternalName(ExternalName::LibCall(libcall)) => {
310            let libcall = libcall_cranelift_to_wasmtime(libcall);
311            RelocationTarget::HostLibcall(libcall)
312        }
313        _ => panic!("unrecognized external name"),
314    };
315    Relocation {
316        reloc: kind,
317        reloc_target,
318        offset,
319        addend,
320    }
321}
322
323fn libcall_cranelift_to_wasmtime(call: ir::LibCall) -> wasmtime_environ::obj::LibCall {
324    use wasmtime_environ::obj::LibCall as LC;
325    match call {
326        ir::LibCall::FloorF32 => LC::FloorF32,
327        ir::LibCall::FloorF64 => LC::FloorF64,
328        ir::LibCall::NearestF32 => LC::NearestF32,
329        ir::LibCall::NearestF64 => LC::NearestF64,
330        ir::LibCall::CeilF32 => LC::CeilF32,
331        ir::LibCall::CeilF64 => LC::CeilF64,
332        ir::LibCall::TruncF32 => LC::TruncF32,
333        ir::LibCall::TruncF64 => LC::TruncF64,
334        ir::LibCall::FmaF32 => LC::FmaF32,
335        ir::LibCall::FmaF64 => LC::FmaF64,
336        ir::LibCall::X86Pshufb => LC::X86Pshufb,
337        _ => panic!("cranelift emitted a libcall wasmtime does not support: {call:?}"),
338    }
339}
340
341/// Helper structure for creating a `Signature` for all builtins.
342struct BuiltinFunctionSignatures {
343    pointer_type: ir::Type,
344
345    host_call_conv: CallConv,
346    wasm_call_conv: CallConv,
347    argument_extension: ir::ArgumentExtension,
348}
349
350impl BuiltinFunctionSignatures {
351    fn new(compiler: &Compiler) -> Self {
352        Self {
353            pointer_type: compiler.isa().pointer_type(),
354            host_call_conv: CallConv::triple_default(compiler.isa().triple()),
355            wasm_call_conv: wasm_call_conv(compiler.isa(), compiler.tunables()),
356            argument_extension: compiler.isa().default_argument_extension(),
357        }
358    }
359
360    fn vmctx(&self) -> AbiParam {
361        AbiParam::special(self.pointer_type, ArgumentPurpose::VMContext)
362    }
363
364    fn pointer(&self) -> AbiParam {
365        AbiParam::new(self.pointer_type)
366    }
367
368    fn u32(&self) -> AbiParam {
369        AbiParam::new(ir::types::I32)
370    }
371
372    fn u64(&self) -> AbiParam {
373        AbiParam::new(ir::types::I64)
374    }
375
376    fn u8(&self) -> AbiParam {
377        AbiParam::new(ir::types::I8)
378    }
379
380    fn bool(&self) -> AbiParam {
381        AbiParam::new(ir::types::I8)
382    }
383
384    fn wasm_signature(&self, builtin: BuiltinFunctionIndex) -> Signature {
385        let mut _cur = 0;
386        macro_rules! iter {
387            (
388                $(
389                    $( #[$attr:meta] )*
390                    $name:ident( $( $pname:ident: $param:ident ),* ) $( -> $result:ident )?;
391                )*
392            ) => {
393                $(
394                    $( #[$attr] )*
395                    if _cur == builtin.index() {
396                        return Signature {
397                            params: vec![ $( self.$param() ),* ],
398                            returns: vec![ $( self.$result() )? ],
399                            call_conv: self.wasm_call_conv,
400                        };
401                    }
402                    _cur += 1;
403                )*
404            };
405        }
406
407        wasmtime_environ::foreach_builtin_function!(iter);
408
409        unreachable!();
410    }
411
412    fn host_signature(&self, builtin: BuiltinFunctionIndex) -> Signature {
413        let mut sig = self.wasm_signature(builtin);
414        sig.call_conv = self.host_call_conv;
415
416        // Once we're declaring the signature of a host function we must
417        // respect the default ABI of the platform which is where argument
418        // extension of params/results may come into play.
419        for arg in sig.params.iter_mut().chain(sig.returns.iter_mut()) {
420            if arg.value_type.is_int() {
421                arg.extension = self.argument_extension;
422            }
423        }
424
425        sig
426    }
427}
428
429/// If this bit is set on a GC reference, then the GC reference is actually an
430/// unboxed `i31`.
431///
432/// Must be kept in sync with
433/// `crate::runtime::vm::gc::VMGcRef::I31_REF_DISCRIMINANT`.
434const I31_REF_DISCRIMINANT: u32 = 1;
wasmtime_cranelift/lib.rs

wasmtime_cranelift/
lib.rs
