cranelift_filetests/
function_runner.rs

1//! Provides functionality for compiling and running CLIF IR for `run` tests.
2use anyhow::{Context as _, Result, anyhow};
3use core::mem;
4use cranelift::prelude::Imm64;
5use cranelift_codegen::cursor::{Cursor, FuncCursor};
6use cranelift_codegen::data_value::DataValue;
7use cranelift_codegen::ir::{
8    ExternalName, Function, InstBuilder, InstructionData, LibCall, Opcode, Signature,
9    UserExternalName, UserFuncName,
10};
11use cranelift_codegen::isa::{OwnedTargetIsa, TargetIsa};
12use cranelift_codegen::{CodegenError, Context, ir, settings};
13use cranelift_control::ControlPlane;
14use cranelift_frontend::{FunctionBuilder, FunctionBuilderContext};
15use cranelift_jit::{JITBuilder, JITModule};
16use cranelift_module::{FuncId, Linkage, Module, ModuleError};
17use cranelift_native::builder_with_options;
18use cranelift_reader::TestFile;
19use pulley_interpreter::interp as pulley;
20use std::cell::Cell;
21use std::cmp::max;
22use std::collections::hash_map::Entry;
23use std::collections::{HashMap, HashSet};
24use std::ptr::NonNull;
25use target_lexicon::Architecture;
26use thiserror::Error;
27
28const TESTFILE_NAMESPACE: u32 = 0;
29
30/// Holds information about a previously defined function.
31#[derive(Debug)]
32struct DefinedFunction {
33    /// This is the name that the function is internally known as.
34    ///
35    /// The JIT module does not support linking / calling [TestcaseName]'s, so
36    /// we rename every function into a [UserExternalName].
37    ///
38    /// By doing this we also have to rename functions that previously were using a
39    /// [UserFuncName], since they may now be in conflict after the renaming that
40    /// occurred.
41    new_name: UserExternalName,
42
43    /// The function signature
44    signature: ir::Signature,
45
46    /// JIT [FuncId]
47    func_id: FuncId,
48}
49
50/// Compile a test case.
51///
52/// Several Cranelift functions need the ability to run Cranelift IR (e.g. `test_run`); this
53/// [TestFileCompiler] provides a way for compiling Cranelift [Function]s to
54/// `CompiledFunction`s and subsequently calling them through the use of a `Trampoline`. As its
55/// name indicates, this compiler is limited: any functionality that requires knowledge of things
56/// outside the [Function] will likely not work (e.g. global values, calls). For an example of this
57/// "outside-of-function" functionality, see `cranelift_jit::backend::JITBackend`.
58///
59/// ```
60/// # let ctrl_plane = &mut Default::default();
61/// use cranelift_filetests::TestFileCompiler;
62/// use cranelift_reader::parse_functions;
63/// use cranelift_codegen::data_value::DataValue;
64///
65/// let code = "test run \n function %add(i32, i32) -> i32 {  block0(v0:i32, v1:i32):  v2 = iadd v0, v1  return v2 }".into();
66/// let func = parse_functions(code).unwrap().into_iter().nth(0).unwrap();
67/// let Ok(mut compiler) = TestFileCompiler::with_default_host_isa() else {
68///     return;
69/// };
70/// compiler.declare_function(&func).unwrap();
71/// compiler.define_function(func.clone(), ctrl_plane).unwrap();
72/// compiler.create_trampoline_for_function(&func, ctrl_plane).unwrap();
73/// let compiled = compiler.compile().unwrap();
74/// let trampoline = compiled.get_trampoline(&func).unwrap();
75///
76/// let returned = trampoline.call(&compiled, &vec![DataValue::I32(2), DataValue::I32(40)]);
77/// assert_eq!(vec![DataValue::I32(42)], returned);
78/// ```
79pub struct TestFileCompiler {
80    module: JITModule,
81    ctx: Context,
82
83    /// Holds info about the functions that have already been defined.
84    /// Use look them up by their original [UserFuncName] since that's how the caller
85    /// passes them to us.
86    defined_functions: HashMap<UserFuncName, DefinedFunction>,
87
88    /// We deduplicate trampolines by the signature of the function that they target.
89    /// This map holds as a key the [Signature] of the target function, and as a value
90    /// the [UserFuncName] of the trampoline for that [Signature].
91    ///
92    /// The trampoline is defined in `defined_functions` as any other regular function.
93    trampolines: HashMap<Signature, UserFuncName>,
94}
95
96impl TestFileCompiler {
97    /// Build a [TestFileCompiler] from a [TargetIsa]. For functions to be runnable on the
98    /// host machine, this [TargetIsa] must match the host machine's ISA (see
99    /// [TestFileCompiler::with_host_isa]).
100    pub fn new(isa: OwnedTargetIsa) -> Self {
101        let mut builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names());
102        let _ = &mut builder; // require mutability on all architectures
103        #[cfg(target_arch = "x86_64")]
104        {
105            builder.symbol_lookup_fn(Box::new(|name| {
106                if name == "__cranelift_x86_pshufb" {
107                    Some(__cranelift_x86_pshufb as *const u8)
108                } else {
109                    None
110                }
111            }));
112        }
113
114        // On Unix platforms force `libm` to get linked into this executable
115        // because tests that use libcalls rely on this library being present.
116        // Without this it's been seen that when cross-compiled to riscv64 the
117        // final binary doesn't link in `libm`.
118        #[cfg(unix)]
119        {
120            unsafe extern "C" {
121                safe fn cosf(f: f32) -> f32;
122            }
123            let f = std::hint::black_box(1.2_f32);
124            assert_eq!(f.cos(), cosf(f));
125        }
126
127        let module = JITModule::new(builder);
128        let ctx = module.make_context();
129
130        Self {
131            module,
132            ctx,
133            defined_functions: HashMap::new(),
134            trampolines: HashMap::new(),
135        }
136    }
137
138    /// Build a [TestFileCompiler] using the host machine's ISA and the passed flags.
139    pub fn with_host_isa(flags: settings::Flags) -> Result<Self> {
140        let builder = builder_with_options(true)
141            .map_err(anyhow::Error::msg)
142            .context("Unable to build a TargetIsa for the current host")?;
143        let isa = builder.finish(flags)?;
144        Ok(Self::new(isa))
145    }
146
147    /// Build a [TestFileCompiler] using the host machine's ISA and the default flags for this
148    /// ISA.
149    pub fn with_default_host_isa() -> Result<Self> {
150        let flags = settings::Flags::new(settings::builder());
151        Self::with_host_isa(flags)
152    }
153
154    /// Declares and compiles all functions in `functions`. Additionally creates a trampoline for
155    /// each one of them.
156    pub fn add_functions(
157        &mut self,
158        functions: &[Function],
159        ctrl_planes: Vec<ControlPlane>,
160    ) -> Result<()> {
161        // Declare all functions in the file, so that they may refer to each other.
162        for func in functions {
163            self.declare_function(func)?;
164        }
165
166        let ctrl_planes = ctrl_planes
167            .into_iter()
168            .chain(std::iter::repeat(ControlPlane::default()));
169
170        // Define all functions and trampolines
171        for (func, ref mut ctrl_plane) in functions.iter().zip(ctrl_planes) {
172            self.define_function(func.clone(), ctrl_plane)?;
173            self.create_trampoline_for_function(func, ctrl_plane)?;
174        }
175
176        Ok(())
177    }
178
179    /// Registers all functions in a [TestFile]. Additionally creates a trampoline for each one
180    /// of them.
181    pub fn add_testfile(&mut self, testfile: &TestFile) -> Result<()> {
182        let functions = testfile
183            .functions
184            .iter()
185            .map(|(f, _)| f)
186            .cloned()
187            .collect::<Vec<_>>();
188
189        self.add_functions(&functions[..], Vec::new())?;
190        Ok(())
191    }
192
193    /// Declares a function an registers it as a linkable and callable target internally
194    pub fn declare_function(&mut self, func: &Function) -> Result<()> {
195        let next_id = self.defined_functions.len() as u32;
196        match self.defined_functions.entry(func.name.clone()) {
197            Entry::Occupied(_) => {
198                anyhow::bail!("Duplicate function with name {} found!", &func.name)
199            }
200            Entry::Vacant(v) => {
201                let name = func.name.to_string();
202                let func_id =
203                    self.module
204                        .declare_function(&name, Linkage::Local, &func.signature)?;
205
206                v.insert(DefinedFunction {
207                    new_name: UserExternalName::new(TESTFILE_NAMESPACE, next_id),
208                    signature: func.signature.clone(),
209                    func_id,
210                });
211            }
212        };
213
214        Ok(())
215    }
216
217    /// Renames the function to its new [UserExternalName], as well as any other function that
218    /// it may reference.
219    ///
220    /// We have to do this since the JIT cannot link Testcase functions.
221    fn apply_func_rename(
222        &self,
223        mut func: Function,
224        defined_func: &DefinedFunction,
225    ) -> Result<Function> {
226        // First, rename the function
227        let func_original_name = func.name;
228        func.name = UserFuncName::User(defined_func.new_name.clone());
229
230        // Rename any functions that it references
231        // Do this in stages to appease the borrow checker
232        let mut redefines = Vec::with_capacity(func.dfg.ext_funcs.len());
233        for (ext_ref, ext_func) in &func.dfg.ext_funcs {
234            let old_name = match &ext_func.name {
235                ExternalName::TestCase(tc) => UserFuncName::Testcase(tc.clone()),
236                ExternalName::User(username) => {
237                    UserFuncName::User(func.params.user_named_funcs()[*username].clone())
238                }
239                // The other cases don't need renaming, so lets just continue...
240                _ => continue,
241            };
242
243            let target_df = self.defined_functions.get(&old_name).ok_or(anyhow!(
244                "Undeclared function {} is referenced by {}!",
245                &old_name,
246                &func_original_name
247            ))?;
248
249            redefines.push((ext_ref, target_df.new_name.clone()));
250        }
251
252        // Now register the redefines
253        for (ext_ref, new_name) in redefines.into_iter() {
254            // Register the new name in the func, so that we can get a reference to it.
255            let new_name_ref = func.params.ensure_user_func_name(new_name);
256
257            // Finally rename the ExtFunc
258            func.dfg.ext_funcs[ext_ref].name = ExternalName::User(new_name_ref);
259        }
260
261        Ok(func)
262    }
263
264    /// Defines the body of a function
265    pub fn define_function(
266        &mut self,
267        mut func: Function,
268        ctrl_plane: &mut ControlPlane,
269    ) -> Result<()> {
270        Self::replace_hostcall_references(&mut func);
271
272        let defined_func = self
273            .defined_functions
274            .get(&func.name)
275            .ok_or(anyhow!("Undeclared function {} found!", &func.name))?;
276
277        self.ctx.func = self.apply_func_rename(func, defined_func)?;
278        self.module.define_function_with_control_plane(
279            defined_func.func_id,
280            &mut self.ctx,
281            ctrl_plane,
282        )?;
283        self.module.clear_context(&mut self.ctx);
284        Ok(())
285    }
286
287    fn replace_hostcall_references(func: &mut Function) {
288        // For every `func_addr` referring to a hostcall that we
289        // define, replace with an `iconst` with the actual
290        // address. Then modify the external func references to
291        // harmless libcall references (that will be unused so
292        // ignored).
293        let mut funcrefs_to_remove = HashSet::new();
294        let mut cursor = FuncCursor::new(func);
295        while let Some(_block) = cursor.next_block() {
296            while let Some(inst) = cursor.next_inst() {
297                match &cursor.func.dfg.insts[inst] {
298                    InstructionData::FuncAddr {
299                        opcode: Opcode::FuncAddr,
300                        func_ref,
301                    } => {
302                        let ext_func = &cursor.func.dfg.ext_funcs[*func_ref];
303                        let hostcall_addr = match &ext_func.name {
304                            ExternalName::TestCase(tc) if tc.raw() == b"__cranelift_throw" => {
305                                Some(__cranelift_throw as usize)
306                            }
307                            _ => None,
308                        };
309
310                        if let Some(addr) = hostcall_addr {
311                            funcrefs_to_remove.insert(*func_ref);
312                            cursor.func.dfg.insts[inst] = InstructionData::UnaryImm {
313                                opcode: Opcode::Iconst,
314                                imm: Imm64::new(addr as i64),
315                            };
316                        }
317                    }
318                    _ => {}
319                }
320            }
321        }
322
323        for to_remove in funcrefs_to_remove {
324            func.dfg.ext_funcs[to_remove].name = ExternalName::LibCall(LibCall::Probestack);
325        }
326    }
327
328    /// Creates and registers a trampoline for a function if none exists.
329    pub fn create_trampoline_for_function(
330        &mut self,
331        func: &Function,
332        ctrl_plane: &mut ControlPlane,
333    ) -> Result<()> {
334        if !self.defined_functions.contains_key(&func.name) {
335            anyhow::bail!("Undeclared function {} found!", &func.name);
336        }
337
338        // Check if a trampoline for this function signature already exists
339        if self.trampolines.contains_key(&func.signature) {
340            return Ok(());
341        }
342
343        // Create a trampoline and register it
344        let name = UserFuncName::user(TESTFILE_NAMESPACE, self.defined_functions.len() as u32);
345        let trampoline = make_trampoline(name.clone(), &func.signature, self.module.isa());
346
347        self.declare_function(&trampoline)?;
348        self.define_function(trampoline, ctrl_plane)?;
349
350        self.trampolines.insert(func.signature.clone(), name);
351
352        Ok(())
353    }
354
355    /// Finalize this TestFile and link all functions.
356    pub fn compile(mut self) -> Result<CompiledTestFile, CompilationError> {
357        // Finalize the functions which we just defined, which resolves any
358        // outstanding relocations (patching in addresses, now that they're
359        // available).
360        self.module.finalize_definitions()?;
361
362        Ok(CompiledTestFile {
363            module: Some(self.module),
364            defined_functions: self.defined_functions,
365            trampolines: self.trampolines,
366        })
367    }
368}
369
370/// A finalized Test File
371pub struct CompiledTestFile {
372    /// We need to store [JITModule] since it contains the underlying memory for the functions.
373    /// Store it in an [Option] so that we can later drop it.
374    module: Option<JITModule>,
375
376    /// Holds info about the functions that have been registered in `module`.
377    /// See [TestFileCompiler] for more info.
378    defined_functions: HashMap<UserFuncName, DefinedFunction>,
379
380    /// Trampolines available in this [JITModule].
381    /// See [TestFileCompiler] for more info.
382    trampolines: HashMap<Signature, UserFuncName>,
383}
384
385impl CompiledTestFile {
386    /// Return a trampoline for calling.
387    ///
388    /// Returns None if [TestFileCompiler::create_trampoline_for_function] wasn't called for this function.
389    pub fn get_trampoline(&self, func: &Function) -> Option<Trampoline<'_>> {
390        let defined_func = self.defined_functions.get(&func.name)?;
391        let trampoline_id = self
392            .trampolines
393            .get(&func.signature)
394            .and_then(|name| self.defined_functions.get(name))
395            .map(|df| df.func_id)?;
396        Some(Trampoline {
397            module: self.module.as_ref()?,
398            func_id: defined_func.func_id,
399            func_signature: &defined_func.signature,
400            trampoline_id,
401        })
402    }
403}
404
405impl Drop for CompiledTestFile {
406    fn drop(&mut self) {
407        // Freeing the module's memory erases the compiled functions.
408        // This should be safe since their pointers never leave this struct.
409        unsafe { self.module.take().unwrap().free_memory() }
410    }
411}
412
413std::thread_local! {
414    /// TLS slot used to store a CompiledTestFile reference so that it
415    /// can be recovered when a hostcall (such as the exception-throw
416    /// handler) is invoked.
417    pub static COMPILED_TEST_FILE: Cell<*const CompiledTestFile> = Cell::new(std::ptr::null());
418}
419
420/// A callable trampoline
421pub struct Trampoline<'a> {
422    module: &'a JITModule,
423    func_id: FuncId,
424    func_signature: &'a Signature,
425    trampoline_id: FuncId,
426}
427
428impl<'a> Trampoline<'a> {
429    /// Call the target function of this trampoline, passing in [DataValue]s using a compiled trampoline.
430    pub fn call(&self, compiled: &CompiledTestFile, arguments: &[DataValue]) -> Vec<DataValue> {
431        let mut values = UnboxedValues::make_arguments(arguments, &self.func_signature);
432        let arguments_address = values.as_mut_ptr();
433
434        let function_ptr = self.module.get_finalized_function(self.func_id);
435        let trampoline_ptr = self.module.get_finalized_function(self.trampoline_id);
436
437        COMPILED_TEST_FILE.set(compiled as *const _);
438        unsafe {
439            self.call_raw(trampoline_ptr, function_ptr, arguments_address);
440        }
441        COMPILED_TEST_FILE.set(std::ptr::null());
442
443        values.collect_returns(&self.func_signature)
444    }
445
446    unsafe fn call_raw(
447        &self,
448        trampoline_ptr: *const u8,
449        function_ptr: *const u8,
450        arguments_address: *mut u128,
451    ) {
452        match self.module.isa().triple().architecture {
453            // For the pulley target this is pulley bytecode, not machine code,
454            // so run the interpreter.
455            Architecture::Pulley32
456            | Architecture::Pulley64
457            | Architecture::Pulley32be
458            | Architecture::Pulley64be => {
459                let mut state = pulley::Vm::new();
460                unsafe {
461                    state.call(
462                        NonNull::new(trampoline_ptr.cast_mut()).unwrap(),
463                        &[
464                            pulley::XRegVal::new_ptr(function_ptr.cast_mut()).into(),
465                            pulley::XRegVal::new_ptr(arguments_address).into(),
466                        ],
467                        [],
468                    );
469                }
470            }
471
472            // Other targets natively execute this machine code.
473            _ => {
474                let callable_trampoline: fn(*const u8, *mut u128) -> () =
475                    unsafe { mem::transmute(trampoline_ptr) };
476                callable_trampoline(function_ptr, arguments_address);
477            }
478        }
479    }
480}
481
482/// Compilation Error when compiling a function.
483#[derive(Error, Debug)]
484pub enum CompilationError {
485    /// Cranelift codegen error.
486    #[error("Cranelift codegen error")]
487    CodegenError(#[from] CodegenError),
488    /// Module Error
489    #[error("Module error")]
490    ModuleError(#[from] ModuleError),
491    /// Memory mapping error.
492    #[error("Memory mapping error")]
493    IoError(#[from] std::io::Error),
494}
495
496/// A container for laying out the [ValueData]s in memory in a way that the [Trampoline] can
497/// understand.
498struct UnboxedValues(Vec<u128>);
499
500impl UnboxedValues {
501    /// The size in bytes of each slot location in the allocated [DataValue]s. Though [DataValue]s
502    /// could be smaller than 16 bytes (e.g. `I16`), this simplifies the creation of the [DataValue]
503    /// array and could be used to align the slots to the largest used [DataValue] (i.e. 128-bit
504    /// vectors).
505    const SLOT_SIZE: usize = 16;
506
507    /// Build the arguments vector for passing the [DataValue]s into the [Trampoline]. The size of
508    /// `u128` used here must match [Trampoline::SLOT_SIZE].
509    pub fn make_arguments(arguments: &[DataValue], signature: &ir::Signature) -> Self {
510        assert_eq!(arguments.len(), signature.params.len());
511        let mut values_vec = vec![0; max(signature.params.len(), signature.returns.len())];
512
513        // Store the argument values into `values_vec`.
514        for ((arg, slot), param) in arguments.iter().zip(&mut values_vec).zip(&signature.params) {
515            assert!(
516                arg.ty() == param.value_type || arg.is_vector(),
517                "argument type mismatch: {} != {}",
518                arg.ty(),
519                param.value_type
520            );
521            unsafe {
522                arg.write_value_to(slot);
523            }
524        }
525
526        Self(values_vec)
527    }
528
529    /// Return a pointer to the underlying memory for passing to the trampoline.
530    pub fn as_mut_ptr(&mut self) -> *mut u128 {
531        self.0.as_mut_ptr()
532    }
533
534    /// Collect the returned [DataValue]s into a [Vec]. The size of `u128` used here must match
535    /// [Trampoline::SLOT_SIZE].
536    pub fn collect_returns(&self, signature: &ir::Signature) -> Vec<DataValue> {
537        assert!(self.0.len() >= signature.returns.len());
538        let mut returns = Vec::with_capacity(signature.returns.len());
539
540        // Extract the returned values from this vector.
541        for (slot, param) in self.0.iter().zip(&signature.returns) {
542            let value = unsafe { DataValue::read_value_from(slot, param.value_type) };
543            returns.push(value);
544        }
545
546        returns
547    }
548}
549
550/// Build the Cranelift IR for moving the memory-allocated [DataValue]s to their correct location
551/// (e.g. register, stack) prior to calling a [CompiledFunction]. The [Function] returned by
552/// [make_trampoline] is compiled to a [Trampoline]. Note that this uses the [TargetIsa]'s default
553/// calling convention so we must also check that the [CompiledFunction] has the same calling
554/// convention (see [TestFileCompiler::compile]).
555fn make_trampoline(name: UserFuncName, signature: &ir::Signature, isa: &dyn TargetIsa) -> Function {
556    // Create the trampoline signature: (callee_address: pointer, values_vec: pointer) -> ()
557    let pointer_type = isa.pointer_type();
558    let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv);
559    wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); // Add the `callee_address` parameter.
560    wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); // Add the `values_vec` parameter.
561
562    let mut func = ir::Function::with_name_signature(name, wrapper_sig);
563
564    // The trampoline has a single block filled with loads, one call to callee_address, and some loads.
565    let mut builder_context = FunctionBuilderContext::new();
566    let mut builder = FunctionBuilder::new(&mut func, &mut builder_context);
567    let block0 = builder.create_block();
568    builder.append_block_params_for_function_params(block0);
569    builder.switch_to_block(block0);
570    builder.seal_block(block0);
571
572    // Extract the incoming SSA values.
573    let (callee_value, values_vec_ptr_val) = {
574        let params = builder.func.dfg.block_params(block0);
575        (params[0], params[1])
576    };
577
578    // Load the argument values out of `values_vec`.
579    let callee_args = signature
580        .params
581        .iter()
582        .enumerate()
583        .map(|(i, param)| {
584            // We always store vector types in little-endian byte order as DataValue.
585            let mut flags = ir::MemFlags::trusted();
586            if param.value_type.is_vector() {
587                flags.set_endianness(ir::Endianness::Little);
588            }
589
590            // Load the value.
591            builder.ins().load(
592                param.value_type,
593                flags,
594                values_vec_ptr_val,
595                (i * UnboxedValues::SLOT_SIZE) as i32,
596            )
597        })
598        .collect::<Vec<_>>();
599
600    // Call the passed function.
601    let new_sig = builder.import_signature(signature.clone());
602    let call = builder
603        .ins()
604        .call_indirect(new_sig, callee_value, &callee_args);
605
606    // Store the return values into `values_vec`.
607    let results = builder.func.dfg.inst_results(call).to_vec();
608    for ((i, value), param) in results.iter().enumerate().zip(&signature.returns) {
609        // We always store vector types in little-endian byte order as DataValue.
610        let mut flags = ir::MemFlags::trusted();
611        if param.value_type.is_vector() {
612            flags.set_endianness(ir::Endianness::Little);
613        }
614        // Store the value.
615        builder.ins().store(
616            flags,
617            *value,
618            values_vec_ptr_val,
619            (i * UnboxedValues::SLOT_SIZE) as i32,
620        );
621    }
622
623    builder.ins().return_(&[]);
624    builder.finalize();
625
626    func
627}
628
629/// Hostcall invoked directly from a compiled function body to test
630/// exception throws.
631///
632/// This function does not return normally: it either uses the
633/// unwinder to jump directly to a Cranelift frame further up the
634/// stack, if a handler is found; or it panics, if not.
635#[cfg(any(
636    target_arch = "x86_64",
637    target_arch = "aarch64",
638    target_arch = "s390x",
639    target_arch = "riscv64"
640))]
641extern "C-unwind" fn __cranelift_throw(
642    entry_fp: usize,
643    exit_fp: usize,
644    exit_pc: usize,
645    tag: u32,
646    payload1: usize,
647    payload2: usize,
648) -> ! {
649    let compiled_test_file = unsafe { &*COMPILED_TEST_FILE.get() };
650    let unwind_host = wasmtime_unwinder::UnwindHost;
651    let frame_handler = |frame: &wasmtime_unwinder::Frame| -> Option<(usize, usize)> {
652        let (base, table) = compiled_test_file
653            .module
654            .as_ref()
655            .unwrap()
656            .lookup_wasmtime_exception_data(frame.pc())?;
657        let relative_pc = u32::try_from(
658            frame
659                .pc()
660                .checked_sub(base)
661                .expect("module lookup did not return a module base below the PC"),
662        )
663        .expect("module larger than 4GiB");
664
665        table
666            .lookup_pc_tag(relative_pc, tag)
667            .map(|(frame_offset, handler)| {
668                let handler_sp = frame
669                    .fp()
670                    .wrapping_sub(usize::try_from(frame_offset).unwrap());
671                let handler_pc = base
672                    .checked_add(usize::try_from(handler).unwrap())
673                    .expect("Handler address computation overflowed");
674                (handler_pc, handler_sp)
675            })
676    };
677    unsafe {
678        match wasmtime_unwinder::Handler::find(
679            &unwind_host,
680            frame_handler,
681            exit_pc,
682            exit_fp,
683            entry_fp,
684        ) {
685            Some(handler) => handler.resume_tailcc(payload1, payload2),
686            None => {
687                panic!("Expected a handler to exit for throw of tag {tag} at pc {exit_pc:x}");
688            }
689        }
690    }
691}
692
693#[cfg(not(any(
694    target_arch = "x86_64",
695    target_arch = "aarch64",
696    target_arch = "s390x",
697    target_arch = "riscv64"
698)))]
699extern "C-unwind" fn __cranelift_throw(
700    _entry_fp: usize,
701    _exit_fp: usize,
702    _exit_pc: usize,
703    _tag: u32,
704    _payload1: usize,
705    _payload2: usize,
706) -> ! {
707    panic!("Throw not implemented on platforms without native backends.");
708}
709
710#[cfg(target_arch = "x86_64")]
711use std::arch::x86_64::__m128i;
712#[cfg(target_arch = "x86_64")]
713#[expect(
714    improper_ctypes_definitions,
715    reason = "manually verified to work for now"
716)]
717extern "C" fn __cranelift_x86_pshufb(a: __m128i, b: __m128i) -> __m128i {
718    union U {
719        reg: __m128i,
720        mem: [u8; 16],
721    }
722
723    unsafe {
724        let a = U { reg: a }.mem;
725        let b = U { reg: b }.mem;
726
727        let select = |arr: &[u8; 16], byte: u8| {
728            if byte & 0x80 != 0 {
729                0x00
730            } else {
731                arr[(byte & 0xf) as usize]
732            }
733        };
734
735        U {
736            mem: [
737                select(&a, b[0]),
738                select(&a, b[1]),
739                select(&a, b[2]),
740                select(&a, b[3]),
741                select(&a, b[4]),
742                select(&a, b[5]),
743                select(&a, b[6]),
744                select(&a, b[7]),
745                select(&a, b[8]),
746                select(&a, b[9]),
747                select(&a, b[10]),
748                select(&a, b[11]),
749                select(&a, b[12]),
750                select(&a, b[13]),
751                select(&a, b[14]),
752                select(&a, b[15]),
753            ],
754        }
755        .reg
756    }
757}
758
759#[cfg(test)]
760mod test {
761    use super::*;
762    use cranelift_reader::{ParseOptions, parse_functions, parse_test};
763
764    fn parse(code: &str) -> Function {
765        parse_functions(code).unwrap().into_iter().nth(0).unwrap()
766    }
767
768    #[test]
769    fn nop() {
770        // Skip this test when cranelift doesn't support the native platform.
771        if cranelift_native::builder().is_err() {
772            return;
773        }
774        let code = String::from(
775            "
776            test run
777            function %test() -> i8 {
778            block0:
779                nop
780                v1 = iconst.i8 -1
781                return v1
782            }",
783        );
784        let ctrl_plane = &mut ControlPlane::default();
785
786        // extract function
787        let test_file = parse_test(code.as_str(), ParseOptions::default()).unwrap();
788        assert_eq!(1, test_file.functions.len());
789        let function = test_file.functions[0].0.clone();
790
791        // execute function
792        let mut compiler = TestFileCompiler::with_default_host_isa().unwrap();
793        compiler.declare_function(&function).unwrap();
794        compiler
795            .define_function(function.clone(), ctrl_plane)
796            .unwrap();
797        compiler
798            .create_trampoline_for_function(&function, ctrl_plane)
799            .unwrap();
800        let compiled = compiler.compile().unwrap();
801        let trampoline = compiled.get_trampoline(&function).unwrap();
802        let returned = trampoline.call(&compiled, &[]);
803        assert_eq!(returned, vec![DataValue::I8(-1)])
804    }
805
806    #[test]
807    fn trampolines() {
808        // Skip this test when cranelift doesn't support the native platform.
809        if cranelift_native::builder().is_err() {
810            return;
811        }
812        let function = parse(
813            "
814            function %test(f32, i8, i64x2, i8) -> f32x4, i64 {
815            block0(v0: f32, v1: i8, v2: i64x2, v3: i8):
816                v4 = vconst.f32x4 [0x0.1 0x0.2 0x0.3 0x0.4]
817                v5 = iconst.i64 -1
818                return v4, v5
819            }",
820        );
821
822        let compiler = TestFileCompiler::with_default_host_isa().unwrap();
823        let trampoline = make_trampoline(
824            UserFuncName::user(0, 0),
825            &function.signature,
826            compiler.module.isa(),
827        );
828        println!("{trampoline}");
829        assert!(format!("{trampoline}").ends_with(
830            "sig0 = (f32, i8, i64x2, i8) -> f32x4, i64 fast
831
832block0(v0: i64, v1: i64):
833    v2 = load.f32 notrap aligned v1
834    v3 = load.i8 notrap aligned v1+16
835    v4 = load.i64x2 notrap aligned little v1+32
836    v5 = load.i8 notrap aligned v1+48
837    v6, v7 = call_indirect sig0, v0(v2, v3, v4, v5)
838    store notrap aligned little v6, v1
839    store notrap aligned v7, v1+16
840    return
841}
842"
843        ));
844    }
845}
cranelift_filetests/function_runner.rs

cranelift_filetests/
function_runner.rs
