wasmtime_environ/
fact.rs

1//! Wasmtime's Fused Adapter Compiler of Trampolines (FACT)
2//!
3//! This module contains a compiler which emits trampolines to implement fused
4//! adapters for the component model. A fused adapter is when a core wasm
5//! function is lifted from one component instance and then lowered into another
6//! component instance. This communication between components is well-defined by
7//! the spec and ends up creating what's called a "fused adapter".
8//!
9//! Adapters are currently implemented with WebAssembly modules. This submodule
10//! will generate a core wasm binary which contains the adapters specified
11//! during compilation. The actual wasm is then later processed by standard
12//! paths in Wasmtime to create native machine code and runtime representations
13//! of modules.
14//!
15//! Note that identification of precisely what goes into an adapter module is
16//! not handled in this file, instead that's all done in `translate/adapt.rs`.
17//! Otherwise this module is only responsible for taking a set of adapters and
18//! their imports and then generating a core wasm module to implement all of
19//! that.
20
21use crate::component::dfg::CoreDef;
22use crate::component::{
23    Adapter, AdapterOptions as AdapterOptionsDfg, ComponentTypesBuilder, FlatType, InterfaceType,
24    RuntimeComponentInstanceIndex, StringEncoding, Transcode, TypeFuncIndex,
25};
26use crate::fact::transcode::Transcoder;
27use crate::prelude::*;
28use crate::{EntityRef, FuncIndex, GlobalIndex, MemoryIndex, PrimaryMap};
29use std::borrow::Cow;
30use std::collections::HashMap;
31use wasm_encoder::*;
32
33mod core_types;
34mod signature;
35mod trampoline;
36mod transcode;
37mod traps;
38
39/// Bit flag for indicating async-lifted exports
40///
41/// This flag may be passed to the `async-start` built-in function (which is
42/// called from both async->async and async->sync adapters) to indicate that the
43/// callee is an async-lifted export.
44pub const START_FLAG_ASYNC_CALLEE: i32 = 1 << 0;
45
46/// Fixed parameter types for the `prepare_call` built-in function.
47///
48/// Note that `prepare_call` also takes a variable number of parameters in
49/// addition to these, determined by the signature of the function for which
50/// we're generating an adapter.
51pub static PREPARE_CALL_FIXED_PARAMS: &[ValType] = &[
52    ValType::FUNCREF, // start
53    ValType::FUNCREF, // return
54    ValType::I32,     // caller_instance
55    ValType::I32,     // callee_instance
56    ValType::I32,     // task_return_type
57    ValType::I32,     // string_encoding
58    ValType::I32,     // result_count_or_max_if_async
59];
60
61/// Representation of an adapter module.
62pub struct Module<'a> {
63    /// Whether or not debug code is inserted into the adapters themselves.
64    debug: bool,
65    /// Type information from the creator of this `Module`
66    types: &'a ComponentTypesBuilder,
67
68    /// Core wasm type section that's incrementally built
69    core_types: core_types::CoreTypes,
70
71    /// Core wasm import section which is built as adapters are inserted. Note
72    /// that imports here are intern'd to avoid duplicate imports of the same
73    /// item.
74    core_imports: ImportSection,
75    /// Final list of imports that this module ended up using, in the same order
76    /// as the imports in the import section.
77    imports: Vec<Import>,
78    /// Intern'd imports and what index they were assigned. Note that this map
79    /// covers all the index spaces for imports, not just one.
80    imported: HashMap<CoreDef, usize>,
81    /// Intern'd transcoders and what index they were assigned.
82    imported_transcoders: HashMap<Transcoder, FuncIndex>,
83
84    /// Cached versions of imported trampolines for working with resources.
85    imported_resource_transfer_own: Option<FuncIndex>,
86    imported_resource_transfer_borrow: Option<FuncIndex>,
87    imported_resource_enter_call: Option<FuncIndex>,
88    imported_resource_exit_call: Option<FuncIndex>,
89
90    // Cached versions of imported trampolines for working with the async ABI.
91    imported_async_start_calls: HashMap<(Option<FuncIndex>, Option<FuncIndex>), FuncIndex>,
92
93    // Cached versions of imported trampolines for working with `stream`s,
94    // `future`s, and `error-context`s.
95    imported_future_transfer: Option<FuncIndex>,
96    imported_stream_transfer: Option<FuncIndex>,
97    imported_error_context_transfer: Option<FuncIndex>,
98
99    // Current status of index spaces from the imports generated so far.
100    imported_funcs: PrimaryMap<FuncIndex, Option<CoreDef>>,
101    imported_memories: PrimaryMap<MemoryIndex, CoreDef>,
102    imported_globals: PrimaryMap<GlobalIndex, CoreDef>,
103
104    funcs: PrimaryMap<FunctionId, Function>,
105    helper_funcs: HashMap<Helper, FunctionId>,
106    helper_worklist: Vec<(FunctionId, Helper)>,
107
108    exports: Vec<(u32, String)>,
109}
110
111struct AdapterData {
112    /// Export name of this adapter
113    name: String,
114    /// Options specified during the `canon lift` operation
115    lift: AdapterOptions,
116    /// Options specified during the `canon lower` operation
117    lower: AdapterOptions,
118    /// The core wasm function that this adapter will be calling (the original
119    /// function that was `canon lift`'d)
120    callee: FuncIndex,
121    /// FIXME(#4185) should be plumbed and handled as part of the new reentrance
122    /// rules not yet implemented here.
123    called_as_export: bool,
124}
125
126/// Configuration options which apply at the "global adapter" level.
127///
128/// These options are typically unique per-adapter and generally aren't needed
129/// when translating recursive types within an adapter.
130struct AdapterOptions {
131    instance: RuntimeComponentInstanceIndex,
132    /// The ascribed type of this adapter.
133    ty: TypeFuncIndex,
134    /// The global that represents the instance flags for where this adapter
135    /// came from.
136    flags: GlobalIndex,
137    /// The configured post-return function, if any.
138    post_return: Option<FuncIndex>,
139    /// Other, more general, options configured.
140    options: Options,
141}
142
143/// This type is split out of `AdapterOptions` and is specifically used to
144/// deduplicate translation functions within a module. Consequently this has
145/// as few fields as possible to minimize the number of functions generated
146/// within an adapter module.
147#[derive(PartialEq, Eq, Hash, Copy, Clone)]
148struct Options {
149    /// The encoding that strings use from this adapter.
150    string_encoding: StringEncoding,
151    /// Whether or not the `memory` field, if present, is a 64-bit memory.
152    memory64: bool,
153    /// An optionally-specified memory where values may travel through for
154    /// types like lists.
155    memory: Option<MemoryIndex>,
156    /// An optionally-specified function to be used to allocate space for
157    /// types such as strings as they go into a module.
158    realloc: Option<FuncIndex>,
159    callback: Option<FuncIndex>,
160    async_: bool,
161}
162
163enum Context {
164    Lift,
165    Lower,
166}
167
168/// Representation of a "helper function" which may be generated as part of
169/// generating an adapter trampoline.
170///
171/// Helper functions are created when inlining the translation for a type in its
172/// entirety would make a function excessively large. This is currently done via
173/// a simple fuel/cost heuristic based on the type being translated but may get
174/// fancier over time.
175#[derive(Copy, Clone, PartialEq, Eq, Hash)]
176struct Helper {
177    /// Metadata about the source type of what's being translated.
178    src: HelperType,
179    /// Metadata about the destination type which is being translated to.
180    dst: HelperType,
181}
182
183/// Information about a source or destination type in a `Helper` which is
184/// generated.
185#[derive(Copy, Clone, PartialEq, Eq, Hash)]
186struct HelperType {
187    /// The concrete type being translated.
188    ty: InterfaceType,
189    /// The configuration options (memory, etc) for the adapter.
190    opts: Options,
191    /// Where the type is located (either the stack or in memory)
192    loc: HelperLocation,
193}
194
195/// Where a `HelperType` is located, dictating the signature of the helper
196/// function.
197#[derive(Copy, Clone, PartialEq, Eq, Hash)]
198enum HelperLocation {
199    /// Located on the stack in wasm locals.
200    Stack,
201    /// Located in linear memory as configured by `opts`.
202    Memory,
203}
204
205impl<'a> Module<'a> {
206    /// Creates an empty module.
207    pub fn new(types: &'a ComponentTypesBuilder, debug: bool) -> Module<'a> {
208        Module {
209            debug,
210            types,
211            core_types: Default::default(),
212            core_imports: Default::default(),
213            imported: Default::default(),
214            imports: Default::default(),
215            imported_transcoders: Default::default(),
216            imported_funcs: PrimaryMap::new(),
217            imported_memories: PrimaryMap::new(),
218            imported_globals: PrimaryMap::new(),
219            funcs: PrimaryMap::new(),
220            helper_funcs: HashMap::new(),
221            helper_worklist: Vec::new(),
222            imported_resource_transfer_own: None,
223            imported_resource_transfer_borrow: None,
224            imported_resource_enter_call: None,
225            imported_resource_exit_call: None,
226            imported_async_start_calls: HashMap::new(),
227            imported_future_transfer: None,
228            imported_stream_transfer: None,
229            imported_error_context_transfer: None,
230            exports: Vec::new(),
231        }
232    }
233
234    /// Registers a new adapter within this adapter module.
235    ///
236    /// The `name` provided is the export name of the adapter from the final
237    /// module, and `adapter` contains all metadata necessary for compilation.
238    pub fn adapt(&mut self, name: &str, adapter: &Adapter) {
239        // Import any items required by the various canonical options
240        // (memories, reallocs, etc)
241        let mut lift = self.import_options(adapter.lift_ty, &adapter.lift_options);
242        let lower = self.import_options(adapter.lower_ty, &adapter.lower_options);
243
244        // Lowering options are not allowed to specify post-return as per the
245        // current canonical abi specification.
246        assert!(adapter.lower_options.post_return.is_none());
247
248        // Import the core wasm function which was lifted using its appropriate
249        // signature since the exported function this adapter generates will
250        // call the lifted function.
251        let signature = self.types.signature(&lift, Context::Lift);
252        let ty = self
253            .core_types
254            .function(&signature.params, &signature.results);
255        let callee = self.import_func("callee", name, ty, adapter.func.clone());
256
257        // Handle post-return specifically here where we have `core_ty` and the
258        // results of `core_ty` are the parameters to the post-return function.
259        lift.post_return = adapter.lift_options.post_return.as_ref().map(|func| {
260            let ty = self.core_types.function(&signature.results, &[]);
261            self.import_func("post_return", name, ty, func.clone())
262        });
263
264        // This will internally create the adapter as specified and append
265        // anything necessary to `self.funcs`.
266        trampoline::compile(
267            self,
268            &AdapterData {
269                name: name.to_string(),
270                lift,
271                lower,
272                callee,
273                // FIXME(#4185) should be plumbed and handled as part of the new
274                // reentrance rules not yet implemented here.
275                called_as_export: true,
276            },
277        );
278
279        while let Some((result, helper)) = self.helper_worklist.pop() {
280            trampoline::compile_helper(self, result, helper);
281        }
282    }
283
284    fn import_options(&mut self, ty: TypeFuncIndex, options: &AdapterOptionsDfg) -> AdapterOptions {
285        let AdapterOptionsDfg {
286            instance,
287            string_encoding,
288            memory,
289            memory64,
290            realloc,
291            post_return: _, // handled above
292            callback,
293            async_,
294        } = options;
295
296        let flags = self.import_global(
297            "flags",
298            &format!("instance{}", instance.as_u32()),
299            GlobalType {
300                val_type: ValType::I32,
301                mutable: true,
302                shared: false,
303            },
304            CoreDef::InstanceFlags(*instance),
305        );
306        let memory = memory.as_ref().map(|memory| {
307            self.import_memory(
308                "memory",
309                &format!("m{}", self.imported_memories.len()),
310                MemoryType {
311                    minimum: 0,
312                    maximum: None,
313                    shared: false,
314                    memory64: *memory64,
315                    page_size_log2: None,
316                },
317                memory.clone().into(),
318            )
319        });
320        let realloc = realloc.as_ref().map(|func| {
321            let ptr = if *memory64 {
322                ValType::I64
323            } else {
324                ValType::I32
325            };
326            let ty = self.core_types.function(&[ptr, ptr, ptr, ptr], &[ptr]);
327            self.import_func(
328                "realloc",
329                &format!("f{}", self.imported_funcs.len()),
330                ty,
331                func.clone(),
332            )
333        });
334        let callback = callback.as_ref().map(|func| {
335            let ty = self
336                .core_types
337                .function(&[ValType::I32, ValType::I32, ValType::I32], &[ValType::I32]);
338            self.import_func(
339                "callback",
340                &format!("f{}", self.imported_funcs.len()),
341                ty,
342                func.clone(),
343            )
344        });
345
346        AdapterOptions {
347            instance: *instance,
348            ty,
349            flags,
350            post_return: None,
351            options: Options {
352                string_encoding: *string_encoding,
353                memory64: *memory64,
354                memory,
355                realloc,
356                callback,
357                async_: *async_,
358            },
359        }
360    }
361
362    fn import_func(&mut self, module: &str, name: &str, ty: u32, def: CoreDef) -> FuncIndex {
363        self.import(module, name, EntityType::Function(ty), def, |m| {
364            &mut m.imported_funcs
365        })
366    }
367
368    fn import_global(
369        &mut self,
370        module: &str,
371        name: &str,
372        ty: GlobalType,
373        def: CoreDef,
374    ) -> GlobalIndex {
375        self.import(module, name, EntityType::Global(ty), def, |m| {
376            &mut m.imported_globals
377        })
378    }
379
380    fn import_memory(
381        &mut self,
382        module: &str,
383        name: &str,
384        ty: MemoryType,
385        def: CoreDef,
386    ) -> MemoryIndex {
387        self.import(module, name, EntityType::Memory(ty), def, |m| {
388            &mut m.imported_memories
389        })
390    }
391
392    fn import<K: EntityRef, V: From<CoreDef>>(
393        &mut self,
394        module: &str,
395        name: &str,
396        ty: EntityType,
397        def: CoreDef,
398        map: impl FnOnce(&mut Self) -> &mut PrimaryMap<K, V>,
399    ) -> K {
400        if let Some(prev) = self.imported.get(&def) {
401            return K::new(*prev);
402        }
403        let idx = map(self).push(def.clone().into());
404        self.core_imports.import(module, name, ty);
405        self.imported.insert(def.clone(), idx.index());
406        self.imports.push(Import::CoreDef(def));
407        idx
408    }
409
410    fn import_transcoder(&mut self, transcoder: transcode::Transcoder) -> FuncIndex {
411        *self
412            .imported_transcoders
413            .entry(transcoder)
414            .or_insert_with(|| {
415                // Add the import to the core wasm import section...
416                let name = transcoder.name();
417                let ty = transcoder.ty(&mut self.core_types);
418                self.core_imports.import("transcode", &name, ty);
419
420                // ... and also record the metadata for what this import
421                // corresponds to.
422                let from = self.imported_memories[transcoder.from_memory].clone();
423                let to = self.imported_memories[transcoder.to_memory].clone();
424                self.imports.push(Import::Transcode {
425                    op: transcoder.op,
426                    from,
427                    from64: transcoder.from_memory64,
428                    to,
429                    to64: transcoder.to_memory64,
430                });
431
432                self.imported_funcs.push(None)
433            })
434    }
435
436    fn import_simple(
437        &mut self,
438        module: &str,
439        name: &str,
440        params: &[ValType],
441        results: &[ValType],
442        import: Import,
443        get: impl Fn(&mut Self) -> &mut Option<FuncIndex>,
444    ) -> FuncIndex {
445        self.import_simple_get_and_set(
446            module,
447            name,
448            params,
449            results,
450            import,
451            |me| *get(me),
452            |me, v| *get(me) = Some(v),
453        )
454    }
455
456    fn import_simple_get_and_set(
457        &mut self,
458        module: &str,
459        name: &str,
460        params: &[ValType],
461        results: &[ValType],
462        import: Import,
463        get: impl Fn(&mut Self) -> Option<FuncIndex>,
464        set: impl Fn(&mut Self, FuncIndex),
465    ) -> FuncIndex {
466        if let Some(idx) = get(self) {
467            return idx;
468        }
469        let ty = self.core_types.function(params, results);
470        let ty = EntityType::Function(ty);
471        self.core_imports.import(module, name, ty);
472
473        self.imports.push(import);
474        let idx = self.imported_funcs.push(None);
475        set(self, idx);
476        idx
477    }
478
479    /// Import a host built-in function to set up a subtask for a sync-lowered
480    /// import call to an async-lifted export.
481    ///
482    /// Given that the callee may exert backpressure before the host can copy
483    /// the parameters, the adapter must use this function to set up the subtask
484    /// and stash the parameters as part of that subtask until any backpressure
485    /// has cleared.
486    fn import_prepare_call(
487        &mut self,
488        suffix: &str,
489        params: &[ValType],
490        memory: Option<MemoryIndex>,
491    ) -> FuncIndex {
492        let ty = self.core_types.function(
493            &PREPARE_CALL_FIXED_PARAMS
494                .iter()
495                .copied()
496                .chain(params.iter().copied())
497                .collect::<Vec<_>>(),
498            &[],
499        );
500        self.core_imports.import(
501            "sync",
502            &format!("[prepare-call]{suffix}"),
503            EntityType::Function(ty),
504        );
505        let import = Import::PrepareCall {
506            memory: memory.map(|v| self.imported_memories[v].clone()),
507        };
508        self.imports.push(import);
509        self.imported_funcs.push(None)
510    }
511
512    /// Import a host built-in function to start a subtask for a sync-lowered
513    /// import call to an async-lifted export.
514    ///
515    /// This call with block until the subtask has produced result(s) via the
516    /// `task.return` intrinsic.
517    ///
518    /// Note that this could potentially be combined with the `sync-prepare`
519    /// built-in into a single built-in function that does both jobs.  However,
520    /// we've kept them separate to allow a future optimization where the caller
521    /// calls the callee directly rather than using `sync-start` to have the host
522    /// do it.
523    fn import_sync_start_call(
524        &mut self,
525        suffix: &str,
526        callback: Option<FuncIndex>,
527        results: &[ValType],
528    ) -> FuncIndex {
529        let ty = self
530            .core_types
531            .function(&[ValType::FUNCREF, ValType::I32], results);
532        self.core_imports.import(
533            "sync",
534            &format!("[start-call]{suffix}"),
535            EntityType::Function(ty),
536        );
537        let import = Import::SyncStartCall {
538            callback: callback
539                .map(|callback| self.imported_funcs.get(callback).unwrap().clone().unwrap()),
540        };
541        self.imports.push(import);
542        self.imported_funcs.push(None)
543    }
544
545    /// Import a host built-in function to start a subtask for an async-lowered
546    /// import call to an async- or sync-lifted export.
547    ///
548    /// Note that this could potentially be combined with the `async-prepare`
549    /// built-in into a single built-in function that does both jobs.  However,
550    /// we've kept them separate to allow a future optimization where the caller
551    /// calls the callee directly rather than using `async-start` to have the
552    /// host do it.
553    fn import_async_start_call(
554        &mut self,
555        suffix: &str,
556        callback: Option<FuncIndex>,
557        post_return: Option<FuncIndex>,
558    ) -> FuncIndex {
559        self.import_simple_get_and_set(
560            "async",
561            &format!("[start-call]{suffix}"),
562            &[ValType::FUNCREF, ValType::I32, ValType::I32, ValType::I32],
563            &[ValType::I32],
564            Import::AsyncStartCall {
565                callback: callback
566                    .map(|callback| self.imported_funcs.get(callback).unwrap().clone().unwrap()),
567                post_return: post_return.map(|post_return| {
568                    self.imported_funcs
569                        .get(post_return)
570                        .unwrap()
571                        .clone()
572                        .unwrap()
573                }),
574            },
575            |me| {
576                me.imported_async_start_calls
577                    .get(&(callback, post_return))
578                    .copied()
579            },
580            |me, v| {
581                assert!(
582                    me.imported_async_start_calls
583                        .insert((callback, post_return), v)
584                        .is_none()
585                )
586            },
587        )
588    }
589
590    fn import_future_transfer(&mut self) -> FuncIndex {
591        self.import_simple(
592            "future",
593            "transfer",
594            &[ValType::I32; 3],
595            &[ValType::I32],
596            Import::FutureTransfer,
597            |me| &mut me.imported_future_transfer,
598        )
599    }
600
601    fn import_stream_transfer(&mut self) -> FuncIndex {
602        self.import_simple(
603            "stream",
604            "transfer",
605            &[ValType::I32; 3],
606            &[ValType::I32],
607            Import::StreamTransfer,
608            |me| &mut me.imported_stream_transfer,
609        )
610    }
611
612    fn import_error_context_transfer(&mut self) -> FuncIndex {
613        self.import_simple(
614            "error-context",
615            "transfer",
616            &[ValType::I32; 3],
617            &[ValType::I32],
618            Import::ErrorContextTransfer,
619            |me| &mut me.imported_error_context_transfer,
620        )
621    }
622
623    fn import_resource_transfer_own(&mut self) -> FuncIndex {
624        self.import_simple(
625            "resource",
626            "transfer-own",
627            &[ValType::I32, ValType::I32, ValType::I32],
628            &[ValType::I32],
629            Import::ResourceTransferOwn,
630            |me| &mut me.imported_resource_transfer_own,
631        )
632    }
633
634    fn import_resource_transfer_borrow(&mut self) -> FuncIndex {
635        self.import_simple(
636            "resource",
637            "transfer-borrow",
638            &[ValType::I32, ValType::I32, ValType::I32],
639            &[ValType::I32],
640            Import::ResourceTransferBorrow,
641            |me| &mut me.imported_resource_transfer_borrow,
642        )
643    }
644
645    fn import_resource_enter_call(&mut self) -> FuncIndex {
646        self.import_simple(
647            "resource",
648            "enter-call",
649            &[],
650            &[],
651            Import::ResourceEnterCall,
652            |me| &mut me.imported_resource_enter_call,
653        )
654    }
655
656    fn import_resource_exit_call(&mut self) -> FuncIndex {
657        self.import_simple(
658            "resource",
659            "exit-call",
660            &[],
661            &[],
662            Import::ResourceExitCall,
663            |me| &mut me.imported_resource_exit_call,
664        )
665    }
666
667    fn translate_helper(&mut self, helper: Helper) -> FunctionId {
668        *self.helper_funcs.entry(helper).or_insert_with(|| {
669            // Generate a fresh `Function` with a unique id for what we're about to
670            // generate.
671            let ty = helper.core_type(self.types, &mut self.core_types);
672            let id = self.funcs.push(Function::new(None, ty));
673            self.helper_worklist.push((id, helper));
674            id
675        })
676    }
677
678    /// Encodes this module into a WebAssembly binary.
679    pub fn encode(&mut self) -> Vec<u8> {
680        // Build the function/export sections of the wasm module in a first pass
681        // which will assign a final `FuncIndex` to all functions defined in
682        // `self.funcs`.
683        let mut funcs = FunctionSection::new();
684        let mut exports = ExportSection::new();
685        let mut id_to_index = PrimaryMap::<FunctionId, FuncIndex>::new();
686        for (id, func) in self.funcs.iter() {
687            assert!(func.filled_in);
688            let idx = FuncIndex::from_u32(self.imported_funcs.next_key().as_u32() + id.as_u32());
689            let id2 = id_to_index.push(idx);
690            assert_eq!(id2, id);
691
692            funcs.function(func.ty);
693
694            if let Some(name) = &func.export {
695                exports.export(name, ExportKind::Func, idx.as_u32());
696            }
697        }
698        for (idx, name) in &self.exports {
699            exports.export(name, ExportKind::Func, *idx);
700        }
701
702        // With all functions numbered the fragments of the body of each
703        // function can be assigned into one final adapter function.
704        let mut code = CodeSection::new();
705        let mut traps = traps::TrapSection::default();
706        for (id, func) in self.funcs.iter() {
707            let mut func_traps = Vec::new();
708            let mut body = Vec::new();
709
710            // Encode all locals used for this function
711            func.locals.len().encode(&mut body);
712            for (count, ty) in func.locals.iter() {
713                count.encode(&mut body);
714                ty.encode(&mut body);
715            }
716
717            // Then encode each "chunk" of a body which may have optional traps
718            // specified within it. Traps get offset by the current length of
719            // the body and otherwise our `Call` instructions are "relocated"
720            // here to the final function index.
721            for chunk in func.body.iter() {
722                match chunk {
723                    Body::Raw(code, traps) => {
724                        let start = body.len();
725                        body.extend_from_slice(code);
726                        for (offset, trap) in traps {
727                            func_traps.push((start + offset, *trap));
728                        }
729                    }
730                    Body::Call(id) => {
731                        Instruction::Call(id_to_index[*id].as_u32()).encode(&mut body);
732                    }
733                    Body::RefFunc(id) => {
734                        Instruction::RefFunc(id_to_index[*id].as_u32()).encode(&mut body);
735                    }
736                }
737            }
738            code.raw(&body);
739            traps.append(id_to_index[id].as_u32(), func_traps);
740        }
741
742        let traps = traps.finish();
743
744        let mut result = wasm_encoder::Module::new();
745        result.section(&self.core_types.section);
746        result.section(&self.core_imports);
747        result.section(&funcs);
748        result.section(&exports);
749        result.section(&code);
750        if self.debug {
751            result.section(&CustomSection {
752                name: "wasmtime-trampoline-traps".into(),
753                data: Cow::Borrowed(&traps),
754            });
755        }
756        result.finish()
757    }
758
759    /// Returns the imports that were used, in order, to create this adapter
760    /// module.
761    pub fn imports(&self) -> &[Import] {
762        &self.imports
763    }
764}
765
766/// Possible imports into an adapter module.
767#[derive(Clone)]
768pub enum Import {
769    /// A definition required in the configuration of an `Adapter`.
770    CoreDef(CoreDef),
771    /// A transcoding function from the host to convert between string encodings.
772    Transcode {
773        /// The transcoding operation this performs.
774        op: Transcode,
775        /// The memory being read
776        from: CoreDef,
777        /// Whether or not `from` is a 64-bit memory
778        from64: bool,
779        /// The memory being written
780        to: CoreDef,
781        /// Whether or not `to` is a 64-bit memory
782        to64: bool,
783    },
784    /// Transfers an owned resource from one table to another.
785    ResourceTransferOwn,
786    /// Transfers a borrowed resource from one table to another.
787    ResourceTransferBorrow,
788    /// Sets up entry metadata for a borrow resources when a call starts.
789    ResourceEnterCall,
790    /// Tears down a previous entry and handles checking borrow-related
791    /// metadata.
792    ResourceExitCall,
793    /// An intrinsic used by FACT-generated modules to begin a call involving
794    /// an async-lowered import and/or an async-lifted export.
795    PrepareCall {
796        /// The memory used to verify that the memory specified for the
797        /// `task.return` that is called at runtime (if any) matches the one
798        /// specified in the lifted export.
799        memory: Option<CoreDef>,
800    },
801    /// An intrinsic used by FACT-generated modules to complete a call involving
802    /// a sync-lowered import and async-lifted export.
803    SyncStartCall {
804        /// The callee's callback function, if any.
805        callback: Option<CoreDef>,
806    },
807    /// An intrinsic used by FACT-generated modules to complete a call involving
808    /// an async-lowered import function.
809    AsyncStartCall {
810        /// The callee's callback function, if any.
811        callback: Option<CoreDef>,
812
813        /// The callee's post-return function, if any.
814        post_return: Option<CoreDef>,
815    },
816    /// An intrinisic used by FACT-generated modules to (partially or entirely) transfer
817    /// ownership of a `future`.
818    FutureTransfer,
819    /// An intrinisic used by FACT-generated modules to (partially or entirely) transfer
820    /// ownership of a `stream`.
821    StreamTransfer,
822    /// An intrinisic used by FACT-generated modules to (partially or entirely) transfer
823    /// ownership of an `error-context`.
824    ErrorContextTransfer,
825}
826
827impl Options {
828    fn ptr(&self) -> ValType {
829        if self.memory64 {
830            ValType::I64
831        } else {
832            ValType::I32
833        }
834    }
835
836    fn ptr_size(&self) -> u8 {
837        if self.memory64 { 8 } else { 4 }
838    }
839
840    fn flat_types<'a>(
841        &self,
842        ty: &InterfaceType,
843        types: &'a ComponentTypesBuilder,
844    ) -> Option<&'a [FlatType]> {
845        let flat = types.flat_types(ty)?;
846        Some(if self.memory64 {
847            flat.memory64
848        } else {
849            flat.memory32
850        })
851    }
852}
853
854/// Temporary index which is not the same as `FuncIndex`.
855///
856/// This represents the nth generated function in the adapter module where the
857/// final index of the function is not known at the time of generation since
858/// more imports may be discovered (specifically string transcoders).
859#[derive(Debug, Copy, Clone, PartialEq, Eq)]
860struct FunctionId(u32);
861cranelift_entity::entity_impl!(FunctionId);
862
863/// A generated function to be added to an adapter module.
864///
865/// At least one function is created per-adapter and depending on the type
866/// hierarchy multiple functions may be generated per-adapter.
867struct Function {
868    /// Whether or not the `body` has been finished.
869    ///
870    /// Functions are added to a `Module` before they're defined so this is used
871    /// to assert that the function was in fact actually filled in by the
872    /// time we reach `Module::encode`.
873    filled_in: bool,
874
875    /// The type signature that this function has, as an index into the core
876    /// wasm type index space of the generated adapter module.
877    ty: u32,
878
879    /// The locals that are used by this function, organized by the number of
880    /// types of each local.
881    locals: Vec<(u32, ValType)>,
882
883    /// If specified, the export name of this function.
884    export: Option<String>,
885
886    /// The contents of the function.
887    ///
888    /// See `Body` for more information, and the `Vec` here represents the
889    /// concatenation of all the `Body` fragments.
890    body: Vec<Body>,
891}
892
893/// Representation of a fragment of the body of a core wasm function generated
894/// for adapters.
895///
896/// This variant comes in one of two flavors:
897///
898/// 1. First a `Raw` variant is used to contain general instructions for the
899///    wasm function. This is populated by `Compiler::instruction` primarily.
900///    This also comes with a list of traps. and the byte offset within the
901///    first vector of where the trap information applies to.
902///
903/// 2. A `Call` instruction variant for a `FunctionId` where the final
904///    `FuncIndex` isn't known until emission time.
905///
906/// The purpose of this representation is the `Body::Call` variant. This can't
907/// be encoded as an instruction when it's generated due to not knowing the
908/// final index of the function being called. During `Module::encode`, however,
909/// all indices are known and `Body::Call` is turned into a final
910/// `Instruction::Call`.
911///
912/// One other possible representation in the future would be to encode a `Call`
913/// instruction with a 5-byte leb to fill in later, but for now this felt
914/// easier to represent. A 5-byte leb may be more efficient at compile-time if
915/// necessary, however.
916enum Body {
917    Raw(Vec<u8>, Vec<(usize, traps::Trap)>),
918    Call(FunctionId),
919    RefFunc(FunctionId),
920}
921
922impl Function {
923    fn new(export: Option<String>, ty: u32) -> Function {
924        Function {
925            filled_in: false,
926            ty,
927            locals: Vec::new(),
928            export,
929            body: Vec::new(),
930        }
931    }
932}
933
934impl Helper {
935    fn core_type(
936        &self,
937        types: &ComponentTypesBuilder,
938        core_types: &mut core_types::CoreTypes,
939    ) -> u32 {
940        let mut params = Vec::new();
941        let mut results = Vec::new();
942        // The source type being translated is always pushed onto the
943        // parameters first, either a pointer for memory or its flat
944        // representation.
945        self.src.push_flat(&mut params, types);
946
947        // The destination type goes into the parameter list if it's from
948        // memory or otherwise is the result of the function itself for a
949        // stack-based representation.
950        match self.dst.loc {
951            HelperLocation::Stack => self.dst.push_flat(&mut results, types),
952            HelperLocation::Memory => params.push(self.dst.opts.ptr()),
953        }
954
955        core_types.function(&params, &results)
956    }
957}
958
959impl HelperType {
960    fn push_flat(&self, dst: &mut Vec<ValType>, types: &ComponentTypesBuilder) {
961        match self.loc {
962            HelperLocation::Stack => {
963                for ty in self.opts.flat_types(&self.ty, types).unwrap() {
964                    dst.push((*ty).into());
965                }
966            }
967            HelperLocation::Memory => {
968                dst.push(self.opts.ptr());
969            }
970        }
971    }
972}