Skip to main content

wasmtime_environ/
fact.rs

1//! Wasmtime's Fused Adapter Compiler of Trampolines (FACT)
2//!
3//! This module contains a compiler which emits trampolines to implement fused
4//! adapters for the component model. A fused adapter is when a core wasm
5//! function is lifted from one component instance and then lowered into another
6//! component instance. This communication between components is well-defined by
7//! the spec and ends up creating what's called a "fused adapter".
8//!
9//! Adapters are currently implemented with WebAssembly modules. This submodule
10//! will generate a core wasm binary which contains the adapters specified
11//! during compilation. The actual wasm is then later processed by standard
12//! paths in Wasmtime to create native machine code and runtime representations
13//! of modules.
14//!
15//! Note that identification of precisely what goes into an adapter module is
16//! not handled in this file, instead that's all done in `translate/adapt.rs`.
17//! Otherwise this module is only responsible for taking a set of adapters and
18//! their imports and then generating a core wasm module to implement all of
19//! that.
20
21use crate::component::dfg::{AdapterId, ComponentDfg, CoreDef};
22use crate::component::{
23    AdapterOptions as AdapterOptionsDfg, CanonicalAbiInfo, ComponentTypesBuilder, FlatType,
24    InterfaceType, RuntimeComponentInstanceIndex, StringEncoding, Transcode, TypeFuncIndex,
25    TypeResourceTableIndex, UnsafeIntrinsic,
26};
27use crate::fact::transcode::Transcoder;
28use crate::prelude::*;
29use crate::{
30    EntityRef, FuncIndex, GlobalIndex, IndexType, Memory, MemoryIndex, ModuleInternedTypeIndex,
31    PrimaryMap, Trap, Tunables, WasmValType,
32};
33use std::collections::HashMap;
34use wasm_encoder::*;
35use wasmparser::WasmFeatures;
36
37mod core_types;
38mod signature;
39mod trampoline;
40mod transcode;
41
42/// Fixed parameter types for the `prepare_call` built-in function.
43///
44/// Note that `prepare_call` also takes a variable number of parameters in
45/// addition to these, determined by the signature of the function for which
46/// we're generating an adapter.
47pub static PREPARE_CALL_FIXED_PARAMS: &[ValType] = &[
48    ValType::FUNCREF, // start
49    ValType::FUNCREF, // return
50    ValType::I32,     // caller_instance
51    ValType::I32,     // callee_instance
52    ValType::I32,     // task_return_type
53    ValType::I32,     // callee_async
54    ValType::I32,     // string_encoding
55    ValType::I32,     // result_count_or_max_if_async
56];
57
58/// Representation of an adapter module.
59pub struct Module<'a> {
60    /// Compilation configuration
61    tunables: &'a Tunables,
62    /// Type information from the creator of this `Module`
63    types: &'a ComponentTypesBuilder,
64
65    /// The component being adapted.
66    component: &'a ComponentDfg,
67
68    /// The Wasm features enabled for validation of this module.
69    features: WasmFeatures,
70
71    /// Core wasm type section that's incrementally built
72    core_types: core_types::CoreTypes,
73
74    /// Core wasm import section which is built as adapters are inserted. Note
75    /// that imports here are intern'd to avoid duplicate imports of the same
76    /// item.
77    core_imports: ImportSection,
78    /// Final list of imports that this module ended up using, in the same order
79    /// as the imports in the import section.
80    imports: Vec<Import>,
81    /// Intern'd imports and what index they were assigned. Note that this map
82    /// covers all the index spaces for imports, not just one.
83    imported: HashMap<CoreDef, usize>,
84    /// Intern'd transcoders and what index they were assigned.
85    imported_transcoders: HashMap<Transcoder, FuncIndex>,
86
87    /// Cached versions of imported trampolines for working with resources.
88    imported_resource_transfer_own: Option<FuncIndex>,
89    imported_resource_transfer_borrow: Option<FuncIndex>,
90    imported_resource_transfer_borrow_to_owner: HashMap<TypeResourceTableIndex, FuncIndex>,
91
92    // Cached versions of imported trampolines for working with the async ABI.
93    imported_start_calls: HashMap<(Option<FuncIndex>, Option<FuncIndex>, Vec<ValType>), FuncIndex>,
94
95    // Cached versions of imported trampolines for working with `stream`s,
96    // `future`s, and `error-context`s.
97    imported_future_transfer: Option<FuncIndex>,
98    imported_stream_transfer: Option<FuncIndex>,
99    imported_error_context_transfer: Option<FuncIndex>,
100
101    imported_enter_sync_call: Option<FuncIndex>,
102    imported_exit_sync_call: Option<FuncIndex>,
103    imported_hostcall_fuel: Option<FuncIndex>,
104
105    /// Whether any adapter uses the defined global holding the hostcall fuel
106    /// that's remaining for strings copied by the host.
107    hostcall_fuel_global: bool,
108
109    /// Cached versions of unsafe intrinsics and where they were imported.
110    imported_unsafe_intrinsics: HashMap<UnsafeIntrinsic, FuncIndex>,
111
112    /// Cached versions of the imported `trap` intrinsic, one per trap code.
113    imported_traps: HashMap<Trap, FuncIndex>,
114
115    // Current status of index spaces from the imports generated so far.
116    imported_funcs: PrimaryMap<FuncIndex, Option<CoreDef>>,
117    imported_memories: PrimaryMap<MemoryIndex, CoreDef>,
118    imported_globals: PrimaryMap<GlobalIndex, CoreDef>,
119
120    funcs: PrimaryMap<FunctionId, Function>,
121    helper_funcs: HashMap<Helper, FunctionId>,
122    helper_worklist: Vec<(FunctionId, Helper)>,
123
124    exports: Vec<(u32, String)>,
125}
126
127struct AdapterData {
128    /// Export name of this adapter
129    name: String,
130    /// Options specified during the `canon lift` operation
131    lift: AdapterOptions,
132    /// Options specified during the `canon lower` operation
133    lower: AdapterOptions,
134    /// The core wasm function that this adapter will be calling (the original
135    /// function that was `canon lift`'d)
136    callee: FuncIndex,
137    /// Whether nothing this adapter can reach is able to observe or mutate the
138    /// thread state that `enter-sync-call`/`exit-sync-call` maintain, meaning
139    /// that pair can be omitted entirely.
140    ///
141    /// See `crates/environ/src/component/thread_transparency.rs` for details.
142    thread_transparent: bool,
143}
144
145/// Configuration options which apply at the "global adapter" level.
146///
147/// These options are typically unique per-adapter and generally aren't needed
148/// when translating recursive types within an adapter.
149struct AdapterOptions {
150    /// The Wasmtime-assigned component instance index where the options were
151    /// originally specified.
152    instance: RuntimeComponentInstanceIndex,
153    /// The ascribed type of this adapter.
154    ty: TypeFuncIndex,
155    /// The global that represents the instance flags for where this adapter
156    /// came from.
157    flags: GlobalIndex,
158    /// The configured post-return function, if any.
159    post_return: Option<FuncIndex>,
160    /// Other, more general, options configured.
161    options: Options,
162}
163
164#[derive(PartialEq, Eq, Hash, Copy, Clone)]
165/// Linear memory.
166struct LinearMemoryOptions {
167    /// An optionally-specified memory where values may travel through for
168    /// types like lists.
169    memory: Option<(MemoryIndex, Memory)>,
170    /// An optionally-specified function to be used to allocate space for
171    /// types such as strings as they go into a module.
172    realloc: Option<FuncIndex>,
173}
174
175impl LinearMemoryOptions {
176    fn ptr(&self) -> ValType {
177        if self.memory64() {
178            ValType::I64
179        } else {
180            ValType::I32
181        }
182    }
183
184    fn ptr_size(&self) -> u8 {
185        if self.memory64() { 8 } else { 4 }
186    }
187
188    fn memory64(&self) -> bool {
189        self.memory
190            .as_ref()
191            .map(|(_, ty)| ty.idx_type == IndexType::I64)
192            .unwrap_or(false)
193    }
194
195    fn sizealign(&self, abi: &CanonicalAbiInfo) -> (u32, u32) {
196        if self.memory64() {
197            (abi.size64, abi.align64)
198        } else {
199            (abi.size32, abi.align32)
200        }
201    }
202}
203
204/// The data model for objects passed through an adapter.
205#[derive(PartialEq, Eq, Hash, Copy, Clone)]
206enum DataModel {
207    Gc {},
208    LinearMemory(LinearMemoryOptions),
209}
210
211impl DataModel {
212    #[track_caller]
213    fn unwrap_memory(&self) -> &LinearMemoryOptions {
214        match self {
215            DataModel::Gc {} => panic!("`unwrap_memory` on GC"),
216            DataModel::LinearMemory(opts) => opts,
217        }
218    }
219}
220
221/// This type is split out of `AdapterOptions` and is specifically used to
222/// deduplicate translation functions within a module. Consequently this has
223/// as few fields as possible to minimize the number of functions generated
224/// within an adapter module.
225#[derive(PartialEq, Eq, Hash, Copy, Clone)]
226struct Options {
227    /// The encoding that strings use from this adapter.
228    string_encoding: StringEncoding,
229    callback: Option<FuncIndex>,
230    async_: bool,
231    core_type: ModuleInternedTypeIndex,
232    data_model: DataModel,
233}
234
235/// Representation of a "helper function" which may be generated as part of
236/// generating an adapter trampoline.
237///
238/// Helper functions are created when inlining the translation for a type in its
239/// entirety would make a function excessively large. This is currently done via
240/// a simple fuel/cost heuristic based on the type being translated but may get
241/// fancier over time.
242#[derive(Copy, Clone, PartialEq, Eq, Hash)]
243struct Helper {
244    /// Metadata about the source type of what's being translated.
245    src: HelperType,
246    /// Metadata about the destination type which is being translated to.
247    dst: HelperType,
248}
249
250/// Information about a source or destination type in a `Helper` which is
251/// generated.
252#[derive(Copy, Clone, PartialEq, Eq, Hash)]
253struct HelperType {
254    /// The concrete type being translated.
255    ty: InterfaceType,
256    /// The configuration options (memory, etc) for the adapter.
257    opts: Options,
258    /// Where the type is located (either the stack or in memory)
259    loc: HelperLocation,
260}
261
262/// Where a `HelperType` is located, dictating the signature of the helper
263/// function.
264#[derive(Copy, Clone, PartialEq, Eq, Hash)]
265enum HelperLocation {
266    /// Located on the stack in wasm locals.
267    Stack,
268    /// Located in linear memory as configured by `opts`.
269    Memory,
270    /// Located in a GC struct field.
271    #[expect(dead_code, reason = "CM+GC is still WIP")]
272    StructField,
273    /// Located in a GC array element.
274    #[expect(dead_code, reason = "CM+GC is still WIP")]
275    ArrayElement,
276}
277
278impl<'a> Module<'a> {
279    /// Creates an empty module.
280    pub fn new(
281        types: &'a ComponentTypesBuilder,
282        component: &'a ComponentDfg,
283        tunables: &'a Tunables,
284        features: WasmFeatures,
285    ) -> Module<'a> {
286        Module {
287            tunables,
288            types,
289            component,
290            features,
291            core_types: Default::default(),
292            core_imports: Default::default(),
293            imported: Default::default(),
294            imports: Default::default(),
295            imported_transcoders: Default::default(),
296            imported_funcs: PrimaryMap::new(),
297            imported_memories: PrimaryMap::new(),
298            imported_globals: PrimaryMap::new(),
299            funcs: PrimaryMap::new(),
300            helper_funcs: HashMap::new(),
301            helper_worklist: Vec::new(),
302            imported_resource_transfer_own: None,
303            imported_resource_transfer_borrow: None,
304            imported_resource_transfer_borrow_to_owner: HashMap::new(),
305            imported_start_calls: HashMap::new(),
306            imported_future_transfer: None,
307            imported_stream_transfer: None,
308            imported_error_context_transfer: None,
309            imported_enter_sync_call: None,
310            imported_exit_sync_call: None,
311            imported_hostcall_fuel: None,
312            hostcall_fuel_global: false,
313            imported_unsafe_intrinsics: HashMap::new(),
314            imported_traps: HashMap::new(),
315            exports: Vec::new(),
316        }
317    }
318
319    /// Registers a new adapter within this adapter module.
320    ///
321    /// The `name` provided is the export name of the adapter from the final
322    /// module, and `adapter` indexes into this module's component for all the
323    /// metadata necessary for compilation.
324    pub fn adapt(&mut self, name: &str, adapter: AdapterId) {
325        let thread_transparent = self.component.transparent_adapters.contains(adapter);
326        let adapter = &self.component.adapters[adapter];
327
328        // Import any items required by the various canonical options
329        // (memories, reallocs, etc)
330        let mut lift = self.import_options(adapter.lift_ty, &adapter.lift_options);
331        let lower = self.import_options(adapter.lower_ty, &adapter.lower_options);
332
333        // Lowering options are not allowed to specify post-return as per the
334        // current canonical abi specification.
335        assert!(adapter.lower_options.post_return.is_none());
336
337        // Import the core wasm function which was lifted using its appropriate
338        // signature since the exported function this adapter generates will
339        // call the lifted function.
340        let signature = self.types.signature(&lift);
341        let ty = self
342            .core_types
343            .function(&signature.params, &signature.results);
344        let callee = self.import_func("callee", name, ty, adapter.func.clone());
345
346        // Handle post-return specifically here where we have `core_ty` and the
347        // results of `core_ty` are the parameters to the post-return function.
348        lift.post_return = adapter.lift_options.post_return.as_ref().map(|func| {
349            let ty = self.core_types.function(&signature.results, &[]);
350            self.import_func("post_return", name, ty, func.clone())
351        });
352
353        // This will internally create the adapter as specified and append
354        // anything necessary to `self.funcs`.
355        trampoline::compile(
356            self,
357            &AdapterData {
358                name: name.to_string(),
359                lift,
360                lower,
361                callee,
362                thread_transparent,
363            },
364        );
365
366        while let Some((result, helper)) = self.helper_worklist.pop() {
367            trampoline::compile_helper(self, result, helper);
368        }
369    }
370
371    fn import_options(&mut self, ty: TypeFuncIndex, options: &AdapterOptionsDfg) -> AdapterOptions {
372        let AdapterOptionsDfg {
373            instance,
374            string_encoding,
375            post_return: _, // handled above
376            callback,
377            async_,
378            core_type,
379            data_model,
380        } = options;
381
382        let flags = self.import_global(
383            "flags",
384            &format!("instance{}", instance.as_u32()),
385            GlobalType {
386                val_type: ValType::I32,
387                mutable: true,
388                shared: false,
389            },
390            CoreDef::InstanceFlags(*instance),
391        );
392
393        let data_model = match data_model {
394            crate::component::DataModel::Gc {} => DataModel::Gc {},
395            crate::component::DataModel::LinearMemory { memory, realloc } => {
396                let memory = memory.as_ref().map(|(memory, ty)| {
397                    (
398                        self.import_memory(
399                            "memory",
400                            &format!("m{}", self.imported_memories.len()),
401                            MemoryType {
402                                minimum: 0,
403                                maximum: None,
404                                shared: ty.shared,
405                                memory64: ty.idx_type == IndexType::I64,
406                                page_size_log2: if ty.page_size_log2 == 16 {
407                                    None
408                                } else {
409                                    Some(ty.page_size_log2.into())
410                                },
411                            },
412                            memory.clone().into(),
413                        ),
414                        *ty,
415                    )
416                });
417                let realloc = realloc.as_ref().map(|func| {
418                    let ptr = match memory.as_ref().unwrap().1.idx_type {
419                        IndexType::I32 => ValType::I32,
420                        IndexType::I64 => ValType::I64,
421                    };
422                    let ty = self.core_types.function(&[ptr, ptr, ptr, ptr], &[ptr]);
423                    self.import_func(
424                        "realloc",
425                        &format!("f{}", self.imported_funcs.len()),
426                        ty,
427                        func.clone(),
428                    )
429                });
430                DataModel::LinearMemory(LinearMemoryOptions { memory, realloc })
431            }
432        };
433
434        let callback = callback.as_ref().map(|func| {
435            let ty = self
436                .core_types
437                .function(&[ValType::I32, ValType::I32, ValType::I32], &[ValType::I32]);
438            self.import_func(
439                "callback",
440                &format!("f{}", self.imported_funcs.len()),
441                ty,
442                func.clone(),
443            )
444        });
445
446        AdapterOptions {
447            instance: *instance,
448            ty,
449            flags,
450            post_return: None,
451            options: Options {
452                string_encoding: *string_encoding,
453                callback,
454                async_: *async_,
455                core_type: *core_type,
456                data_model,
457            },
458        }
459    }
460
461    fn import_func(&mut self, module: &str, name: &str, ty: u32, def: CoreDef) -> FuncIndex {
462        self.import(module, name, EntityType::Function(ty), def, |m| {
463            &mut m.imported_funcs
464        })
465    }
466
467    fn import_global(
468        &mut self,
469        module: &str,
470        name: &str,
471        ty: GlobalType,
472        def: CoreDef,
473    ) -> GlobalIndex {
474        self.import(module, name, EntityType::Global(ty), def, |m| {
475            &mut m.imported_globals
476        })
477    }
478
479    fn import_memory(
480        &mut self,
481        module: &str,
482        name: &str,
483        ty: MemoryType,
484        def: CoreDef,
485    ) -> MemoryIndex {
486        self.import(module, name, EntityType::Memory(ty), def, |m| {
487            &mut m.imported_memories
488        })
489    }
490
491    fn import<K: EntityRef, V: From<CoreDef>>(
492        &mut self,
493        module: &str,
494        name: &str,
495        ty: EntityType,
496        def: CoreDef,
497        map: impl FnOnce(&mut Self) -> &mut PrimaryMap<K, V>,
498    ) -> K {
499        if let Some(prev) = self.imported.get(&def) {
500            return K::new(*prev);
501        }
502        let idx = map(self).push(def.clone().into());
503        self.core_imports.import(module, name, ty);
504        self.imported.insert(def.clone(), idx.index());
505        self.imports.push(Import::CoreDef(def));
506        idx
507    }
508
509    fn import_transcoder(&mut self, transcoder: transcode::Transcoder) -> FuncIndex {
510        *self
511            .imported_transcoders
512            .entry(transcoder)
513            .or_insert_with(|| {
514                // Add the import to the core wasm import section...
515                let name = transcoder.name();
516                let ty = transcoder.ty(&mut self.core_types);
517                self.core_imports.import("transcode", &name, ty);
518
519                // ... and also record the metadata for what this import
520                // corresponds to.
521                let from = self.imported_memories[transcoder.from_memory].clone();
522                let to = self.imported_memories[transcoder.to_memory].clone();
523                self.imports.push(Import::Transcode {
524                    op: transcoder.op,
525                    from,
526                    from64: transcoder.from_memory64,
527                    to,
528                    to64: transcoder.to_memory64,
529                });
530
531                self.imported_funcs.push(None)
532            })
533    }
534
535    fn import_simple(
536        &mut self,
537        module: &str,
538        name: &str,
539        params: &[ValType],
540        results: &[ValType],
541        import: Import,
542        get: impl Fn(&mut Self) -> &mut Option<FuncIndex>,
543    ) -> FuncIndex {
544        self.import_simple_get_and_set(
545            module,
546            name,
547            params,
548            results,
549            import,
550            |me| *get(me),
551            |me, v| *get(me) = Some(v),
552        )
553    }
554
555    fn import_simple_get_and_set(
556        &mut self,
557        module: &str,
558        name: &str,
559        params: &[ValType],
560        results: &[ValType],
561        import: Import,
562        get: impl Fn(&mut Self) -> Option<FuncIndex>,
563        set: impl Fn(&mut Self, FuncIndex),
564    ) -> FuncIndex {
565        if let Some(idx) = get(self) {
566            return idx;
567        }
568        let ty = self.core_types.function(params, results);
569        let ty = EntityType::Function(ty);
570        self.core_imports.import(module, name, ty);
571
572        self.imports.push(import);
573        let idx = self.imported_funcs.push(None);
574        set(self, idx);
575        idx
576    }
577
578    /// Import a host built-in function to set up a subtask for a sync-lowered
579    /// import call to an async-lifted export.
580    ///
581    /// Given that the callee may exert backpressure before the host can copy
582    /// the parameters, the adapter must use this function to set up the subtask
583    /// and stash the parameters as part of that subtask until any backpressure
584    /// has cleared.
585    fn import_prepare_call(
586        &mut self,
587        suffix: &str,
588        params: &[ValType],
589        memory: Option<MemoryIndex>,
590    ) -> FuncIndex {
591        let ty = self.core_types.function(
592            &PREPARE_CALL_FIXED_PARAMS
593                .iter()
594                .copied()
595                .chain(params.iter().copied())
596                .collect::<Vec<_>>(),
597            &[],
598        );
599        self.core_imports.import(
600            "sync",
601            &format!("[prepare-call]{suffix}"),
602            EntityType::Function(ty),
603        );
604        let import = Import::PrepareCall {
605            memory: memory.map(|v| self.imported_memories[v].clone()),
606        };
607        self.imports.push(import);
608        self.imported_funcs.push(None)
609    }
610
611    /// Import a host built-in function to start a subtask previously set up
612    /// with `prepare-call`.
613    ///
614    /// For async-lowered imports `results` will be `[i32]`, and for
615    /// sync-lowered imports it'll be the function's normal return type.
616    ///
617    /// Note that this could potentially be combined with the `prepare-call`
618    /// built-in into a single built-in function that does both jobs. However,
619    /// we've kept them separate to allow a future optimization where the caller
620    /// calls the callee directly rather than using `start-call` to have the
621    /// host do it.
622    fn import_start_call(
623        &mut self,
624        suffix: &str,
625        callback: Option<FuncIndex>,
626        post_return: Option<FuncIndex>,
627        results: &[ValType],
628    ) -> FuncIndex {
629        let key = (callback, post_return, results.to_vec());
630        self.import_simple_get_and_set(
631            "async",
632            &format!("[start-call]{suffix}"),
633            &[ValType::FUNCREF, ValType::I32, ValType::I32, ValType::I32],
634            results,
635            Import::StartCall {
636                callback: callback
637                    .map(|callback| self.imported_funcs.get(callback).unwrap().clone().unwrap()),
638                post_return: post_return.map(|post_return| {
639                    self.imported_funcs
640                        .get(post_return)
641                        .unwrap()
642                        .clone()
643                        .unwrap()
644                }),
645            },
646            |me| me.imported_start_calls.get(&key).copied(),
647            |me, v| assert!(me.imported_start_calls.insert(key.clone(), v).is_none()),
648        )
649    }
650
651    fn import_future_transfer(&mut self) -> FuncIndex {
652        self.import_simple(
653            "future",
654            "transfer",
655            &[ValType::I32; 3],
656            &[ValType::I32],
657            Import::FutureTransfer,
658            |me| &mut me.imported_future_transfer,
659        )
660    }
661
662    fn import_stream_transfer(&mut self) -> FuncIndex {
663        self.import_simple(
664            "stream",
665            "transfer",
666            &[ValType::I32; 3],
667            &[ValType::I32],
668            Import::StreamTransfer,
669            |me| &mut me.imported_stream_transfer,
670        )
671    }
672
673    fn import_error_context_transfer(&mut self) -> FuncIndex {
674        self.import_simple(
675            "error-context",
676            "transfer",
677            &[ValType::I32; 3],
678            &[ValType::I32],
679            Import::ErrorContextTransfer,
680            |me| &mut me.imported_error_context_transfer,
681        )
682    }
683
684    fn import_resource_transfer_own(&mut self) -> FuncIndex {
685        self.import_simple(
686            "resource",
687            "transfer-own",
688            &[ValType::I32, ValType::I32, ValType::I32],
689            &[ValType::I32],
690            Import::ResourceTransferOwn,
691            |me| &mut me.imported_resource_transfer_own,
692        )
693    }
694
695    fn import_resource_transfer_borrow(&mut self) -> FuncIndex {
696        self.import_simple(
697            "resource",
698            "transfer-borrow",
699            &[ValType::I32, ValType::I32, ValType::I32],
700            &[ValType::I32],
701            Import::ResourceTransferBorrow,
702            |me| &mut me.imported_resource_transfer_borrow,
703        )
704    }
705
706    /// Returns whether a borrow lowered into the resource table `ty` has its
707    /// resource owned by `ty`'s own instance.
708    fn resource_owned_by_own_instance(&self, ty: TypeResourceTableIndex) -> bool {
709        self.types
710            .resource_owned_by_own_instance(ty, |idx| self.component.defined_resource_instance(idx))
711    }
712
713    fn import_resource_transfer_borrow_to_owner(
714        &mut self,
715        ty: TypeResourceTableIndex,
716    ) -> FuncIndex {
717        self.import_simple_get_and_set(
718            "resource",
719            &format!("transfer-borrow-to-owner{}", ty.as_u32()),
720            &[ValType::I32],
721            &[ValType::I32],
722            Import::ResourceTransferBorrowToOwner { ty },
723            |me| {
724                me.imported_resource_transfer_borrow_to_owner
725                    .get(&ty)
726                    .copied()
727            },
728            |me, v| {
729                assert!(
730                    me.imported_resource_transfer_borrow_to_owner
731                        .insert(ty, v)
732                        .is_none()
733                )
734            },
735        )
736    }
737
738    fn import_enter_sync_call(&mut self) -> FuncIndex {
739        self.import_simple(
740            "async",
741            "enter-sync-call",
742            &[ValType::I32; 2],
743            &[],
744            Import::EnterSyncCall,
745            |me| &mut me.imported_enter_sync_call,
746        )
747    }
748
749    fn import_exit_sync_call(&mut self) -> FuncIndex {
750        self.import_simple(
751            "async",
752            "exit-sync-call",
753            &[],
754            &[],
755            Import::ExitSyncCall,
756            |me| &mut me.imported_exit_sync_call,
757        )
758    }
759
760    fn import_hostcall_fuel(&mut self) -> FuncIndex {
761        self.import_simple(
762            "runtime",
763            "hostcall-fuel",
764            &[],
765            &[ValType::I64],
766            Import::HostcallFuel,
767            |me| &mut me.imported_hostcall_fuel,
768        )
769    }
770
771    /// Imports the `context.get` intrinsic for the `slot`th context slot.
772    fn import_context_get(&mut self, slot: usize) -> FuncIndex {
773        let intrinsic = match slot {
774            0 => UnsafeIntrinsic::ContextGetI32_0,
775            1 => UnsafeIntrinsic::ContextGetI32_1,
776            _ => unreachable!(),
777        };
778        self.import_unsafe_intrinsic(intrinsic, &format!("get{slot}"))
779    }
780
781    /// Imports the `context.set` intrinsic for the `slot`th context slot.
782    fn import_context_set(&mut self, slot: usize) -> FuncIndex {
783        let intrinsic = match slot {
784            0 => UnsafeIntrinsic::ContextSetI32_0,
785            1 => UnsafeIntrinsic::ContextSetI32_1,
786            _ => unreachable!(),
787        };
788        self.import_unsafe_intrinsic(intrinsic, &format!("set{slot}"))
789    }
790
791    fn import_unsafe_intrinsic(&mut self, intrinsic: UnsafeIntrinsic, name: &str) -> FuncIndex {
792        let map = |ty: &WasmValType| match ty {
793            crate::WasmValType::I32 => ValType::I32,
794            crate::WasmValType::I64 => ValType::I64,
795            crate::WasmValType::F32 => ValType::F32,
796            crate::WasmValType::F64 => ValType::F64,
797            crate::WasmValType::V128 => ValType::V128,
798            crate::WasmValType::Ref(_) => unreachable!(),
799        };
800        let params = intrinsic.core_params().iter().map(map).collect::<Vec<_>>();
801        let results = intrinsic.core_results().iter().map(map).collect::<Vec<_>>();
802
803        self.import_simple_get_and_set(
804            "context",
805            name,
806            &params,
807            &results,
808            Import::UnsafeIntrinsic(intrinsic),
809            |me| me.imported_unsafe_intrinsics.get(&intrinsic).copied(),
810            |me, idx| {
811                me.imported_unsafe_intrinsics.insert(intrinsic, idx);
812            },
813        )
814    }
815
816    fn import_trap(&mut self, trap: Trap) -> FuncIndex {
817        let name = format!("trap{}", trap as u8);
818        self.import_simple_get_and_set(
819            "runtime",
820            &name,
821            &[],
822            &[],
823            Import::Trap(trap),
824            |me| me.imported_traps.get(&trap).copied(),
825            |me, idx| {
826                me.imported_traps.insert(trap, idx);
827            },
828        )
829    }
830
831    fn translate_helper(&mut self, helper: Helper) -> FunctionId {
832        *self.helper_funcs.entry(helper).or_insert_with(|| {
833            // Generate a fresh `Function` with a unique id for what we're about to
834            // generate.
835            let ty = helper.core_type(self.types, &mut self.core_types);
836            let id = self.funcs.push(Function::new(None, ty));
837            self.helper_worklist.push((id, helper));
838            id
839        })
840    }
841
842    /// Encodes this module into a WebAssembly binary.
843    pub fn encode(&mut self) -> Vec<u8> {
844        // Build the function/export sections of the wasm module in a first pass
845        // which will assign a final `FuncIndex` to all functions defined in
846        // `self.funcs`.
847        let mut funcs = FunctionSection::new();
848        let mut exports = ExportSection::new();
849        let mut id_to_index = PrimaryMap::<FunctionId, FuncIndex>::new();
850        for (id, func) in self.funcs.iter() {
851            assert!(func.filled_in);
852            let idx = FuncIndex::from_u32(self.imported_funcs.next_key().as_u32() + id.as_u32());
853            let id2 = id_to_index.push(idx);
854            assert_eq!(id2, id);
855
856            funcs.function(func.ty);
857
858            if let Some(name) = &func.export {
859                exports.export(name, ExportKind::Func, idx.as_u32());
860            }
861        }
862        for (idx, name) in &self.exports {
863            exports.export(name, ExportKind::Func, *idx);
864        }
865
866        // With all functions numbered the fragments of the body of each
867        // function can be assigned into one final adapter function.
868        let mut code = CodeSection::new();
869        for (_, func) in self.funcs.iter() {
870            let mut body = Vec::new();
871
872            // Encode all locals used for this function
873            func.locals.len().encode(&mut body);
874            for (count, ty) in func.locals.iter() {
875                count.encode(&mut body);
876                ty.encode(&mut body);
877            }
878
879            // Then encode each "chunk" of a body which may have optional traps
880            // specified within it. Traps get offset by the current length of
881            // the body and otherwise our `Call` instructions are "relocated"
882            // here to the final function index.
883            for chunk in func.body.iter() {
884                match chunk {
885                    Body::Raw(code) => {
886                        body.extend_from_slice(code);
887                    }
888                    Body::Call(id) => {
889                        Instruction::Call(id_to_index[*id].as_u32()).encode(&mut body);
890                    }
891                    Body::RefFunc(id) => {
892                        Instruction::RefFunc(id_to_index[*id].as_u32()).encode(&mut body);
893                    }
894                    Body::HostcallFuelGlobalGet => {
895                        let idx = u32::try_from(self.imported_globals.len()).unwrap();
896                        Instruction::GlobalGet(idx).encode(&mut body);
897                    }
898                    Body::HostcallFuelGlobalSet => {
899                        let idx = u32::try_from(self.imported_globals.len()).unwrap();
900                        Instruction::GlobalSet(idx).encode(&mut body);
901                    }
902                }
903            }
904            code.raw(&body);
905        }
906
907        let mut result = wasm_encoder::Module::new();
908        result.section(&self.core_types.section);
909        result.section(&self.core_imports);
910        result.section(&funcs);
911        if self.hostcall_fuel_global {
912            let mut globals = GlobalSection::new();
913            globals.global(
914                GlobalType {
915                    val_type: ValType::I64,
916                    mutable: true,
917                    shared: false,
918                },
919                &ConstExpr::i64_const(0),
920            );
921            result.section(&globals);
922        }
923        result.section(&exports);
924        result.section(&code);
925        result.finish()
926    }
927
928    /// Returns the imports that were used, in order, to create this adapter
929    /// module.
930    pub fn imports(&self) -> &[Import] {
931        &self.imports
932    }
933}
934
935/// Possible imports into an adapter module.
936#[derive(Clone)]
937pub enum Import {
938    /// A definition required in the configuration of an `Adapter`.
939    CoreDef(CoreDef),
940    /// A transcoding function from the host to convert between string encodings.
941    Transcode {
942        /// The transcoding operation this performs.
943        op: Transcode,
944        /// The memory being read
945        from: CoreDef,
946        /// Whether or not `from` is a 64-bit memory
947        from64: bool,
948        /// The memory being written
949        to: CoreDef,
950        /// Whether or not `to` is a 64-bit memory
951        to64: bool,
952    },
953    /// Transfers an owned resource from one table to another.
954    ResourceTransferOwn,
955    /// Transfers a borrowed resource from one table to another.
956    ResourceTransferBorrow,
957    /// Lifts a borrowed resource for a destination which owns the resource and
958    /// therefore receives its representation rather than a handle.
959    ResourceTransferBorrowToOwner {
960        /// The resource table the borrow is being transferred out of.
961        ty: TypeResourceTableIndex,
962    },
963    /// An intrinsic used by FACT-generated modules to begin a call involving
964    /// an async-lowered import and/or an async-lifted export.
965    PrepareCall {
966        /// The memory used to verify that the memory specified for the
967        /// `task.return` that is called at runtime (if any) matches the one
968        /// specified in the lifted export.
969        memory: Option<CoreDef>,
970    },
971    /// An intrinsic used by FACT-generated modules to start a call previously
972    /// set up with `PrepareCall`.
973    StartCall {
974        /// The callee's callback function, if any.
975        callback: Option<CoreDef>,
976
977        /// The callee's post-return function, if any.
978        post_return: Option<CoreDef>,
979    },
980    /// An intrinisic used by FACT-generated modules to (partially or entirely) transfer
981    /// ownership of a `future`.
982    FutureTransfer,
983    /// An intrinisic used by FACT-generated modules to (partially or entirely) transfer
984    /// ownership of a `stream`.
985    StreamTransfer,
986    /// An intrinisic used by FACT-generated modules to (partially or entirely) transfer
987    /// ownership of an `error-context`.
988    ErrorContextTransfer,
989    /// An intrinsic for trapping the instance with a specific trap code.
990    Trap(Trap),
991    /// An intrinsic returning the store's hostcall fuel.
992    HostcallFuel,
993    /// An intrinsic used by FACT-generated modules to check whether an instance
994    /// may be entered for a sync-to-sync call and push a task onto the stack if
995    /// so.
996    EnterSyncCall,
997    /// An intrinsic used by FACT-generated modules to pop the task previously
998    /// pushed by `EnterSyncCall`.
999    ExitSyncCall,
1000    /// An unsafe intrinsic, such as reading/writing `context.{get,set}` slots.
1001    UnsafeIntrinsic(UnsafeIntrinsic),
1002}
1003
1004impl Options {
1005    fn flat_types<'a>(
1006        &self,
1007        ty: &InterfaceType,
1008        types: &'a ComponentTypesBuilder,
1009    ) -> Option<&'a [FlatType]> {
1010        let flat = types.flat_types(ty)?;
1011        match self.data_model {
1012            // These types lower to a scalar core value type under the GC data
1013            // model, which is the same flattening that the linear memory data
1014            // model uses, and which does not depend on the width of memory.
1015            // Therefore the `memory32` list is reused here.
1016            //
1017            // See https://github.com/WebAssembly/component-model/issues/525.
1018            DataModel::Gc {} => match ty {
1019                InterfaceType::Bool
1020                | InterfaceType::U8
1021                | InterfaceType::S8
1022                | InterfaceType::U16
1023                | InterfaceType::S16
1024                | InterfaceType::U32
1025                | InterfaceType::S32
1026                | InterfaceType::U64
1027                | InterfaceType::S64
1028                | InterfaceType::Float32
1029                | InterfaceType::Float64
1030                | InterfaceType::Char
1031                | InterfaceType::Enum(_)
1032                | InterfaceType::Flags(_) => Some(flat.memory32),
1033
1034                // These lower to a GC `struct`/`array` reference or to
1035                // `externref`, neither of which FACT can translate yet.
1036                InterfaceType::String
1037                | InterfaceType::Record(_)
1038                | InterfaceType::Variant(_)
1039                | InterfaceType::List(_)
1040                | InterfaceType::Map(_)
1041                | InterfaceType::Tuple(_)
1042                | InterfaceType::Option(_)
1043                | InterfaceType::Result(_)
1044                | InterfaceType::Own(_)
1045                | InterfaceType::Borrow(_)
1046                | InterfaceType::Future(_)
1047                | InterfaceType::Stream(_)
1048                | InterfaceType::ErrorContext(_)
1049                | InterfaceType::FixedLengthList(_) => todo!("CM+GC"),
1050            },
1051            DataModel::LinearMemory(mem_opts) => Some(if mem_opts.memory64() {
1052                flat.memory64
1053            } else {
1054                flat.memory32
1055            }),
1056        }
1057    }
1058}
1059
1060/// Temporary index which is not the same as `FuncIndex`.
1061///
1062/// This represents the nth generated function in the adapter module where the
1063/// final index of the function is not known at the time of generation since
1064/// more imports may be discovered (specifically string transcoders).
1065#[derive(Debug, Copy, Clone, PartialEq, Eq)]
1066struct FunctionId(u32);
1067cranelift_entity::entity_impl!(FunctionId);
1068
1069/// A generated function to be added to an adapter module.
1070///
1071/// At least one function is created per-adapter and depending on the type
1072/// hierarchy multiple functions may be generated per-adapter.
1073struct Function {
1074    /// Whether or not the `body` has been finished.
1075    ///
1076    /// Functions are added to a `Module` before they're defined so this is used
1077    /// to assert that the function was in fact actually filled in by the
1078    /// time we reach `Module::encode`.
1079    filled_in: bool,
1080
1081    /// The type signature that this function has, as an index into the core
1082    /// wasm type index space of the generated adapter module.
1083    ty: u32,
1084
1085    /// The locals that are used by this function, organized by the number of
1086    /// types of each local.
1087    locals: Vec<(u32, ValType)>,
1088
1089    /// If specified, the export name of this function.
1090    export: Option<String>,
1091
1092    /// The contents of the function.
1093    ///
1094    /// See `Body` for more information, and the `Vec` here represents the
1095    /// concatenation of all the `Body` fragments.
1096    body: Vec<Body>,
1097}
1098
1099/// Representation of a fragment of the body of a core wasm function generated
1100/// for adapters.
1101///
1102/// The purpose of this representation is to handle relocations sort of. For
1103/// example `Body::Call` doesn't actually know the function index being called
1104/// until all adapters are generated because generating an adapter may generate
1105/// imports which would shift the index space.
1106///
1107/// One other possible representation in the future would be to encode indexes
1108/// with a 5-byte leb to fill in later, but for now this felt easier to
1109/// represent. A 5-byte leb may be more efficient at compile-time if necessary,
1110/// however.
1111enum Body {
1112    /// Raw wasm opcodes which aren't affected by imports.
1113    Raw(Vec<u8>),
1114    /// A wasm `call` instruction to the specified function.
1115    Call(FunctionId),
1116    /// A wasm `ref.func` instruction to the specified function.
1117    RefFunc(FunctionId),
1118    /// A `global.get` of the hostcall fuel global.
1119    HostcallFuelGlobalGet,
1120    /// A `global.set` of the hostcall fuel global.
1121    HostcallFuelGlobalSet,
1122}
1123
1124impl Function {
1125    fn new(export: Option<String>, ty: u32) -> Function {
1126        Function {
1127            filled_in: false,
1128            ty,
1129            locals: Vec::new(),
1130            export,
1131            body: Vec::new(),
1132        }
1133    }
1134}
1135
1136impl Helper {
1137    fn core_type(
1138        &self,
1139        types: &ComponentTypesBuilder,
1140        core_types: &mut core_types::CoreTypes,
1141    ) -> u32 {
1142        let mut params = Vec::new();
1143        let mut results = Vec::new();
1144        // The source type being translated is always pushed onto the
1145        // parameters first, either a pointer for memory or its flat
1146        // representation.
1147        self.src.push_flat(&mut params, types);
1148
1149        // The destination type goes into the parameter list if it's from
1150        // memory or otherwise is the result of the function itself for a
1151        // stack-based representation.
1152        match self.dst.loc {
1153            HelperLocation::Stack => self.dst.push_flat(&mut results, types),
1154            HelperLocation::Memory => params.push(self.dst.opts.data_model.unwrap_memory().ptr()),
1155            HelperLocation::StructField | HelperLocation::ArrayElement => todo!("CM+GC"),
1156        }
1157
1158        core_types.function(&params, &results)
1159    }
1160}
1161
1162impl HelperType {
1163    fn push_flat(&self, dst: &mut Vec<ValType>, types: &ComponentTypesBuilder) {
1164        match self.loc {
1165            HelperLocation::Stack => {
1166                for ty in self.opts.flat_types(&self.ty, types).unwrap() {
1167                    dst.push((*ty).into());
1168                }
1169            }
1170            HelperLocation::Memory => {
1171                dst.push(self.opts.data_model.unwrap_memory().ptr());
1172            }
1173            HelperLocation::StructField | HelperLocation::ArrayElement => todo!("CM+GC"),
1174        }
1175    }
1176}