wasmtime/runtime/component/func/
typed.rs

1use crate::component::Instance;
2use crate::component::func::{Func, LiftContext, LowerContext, Options};
3use crate::component::matching::InstanceType;
4use crate::component::storage::{storage_as_slice, storage_as_slice_mut};
5use crate::prelude::*;
6use crate::{AsContextMut, StoreContext, StoreContextMut, ValRaw};
7use alloc::borrow::Cow;
8use core::fmt;
9use core::iter;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::str;
13use wasmtime_environ::component::{
14    CanonicalAbiInfo, InterfaceType, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, StringEncoding, VariantInfo,
15};
16
17#[cfg(feature = "component-model-async")]
18use crate::component::concurrent::{self, AsAccessor, PreparedCall};
19#[cfg(feature = "component-model-async")]
20use crate::component::func::TaskExit;
21
22/// A statically-typed version of [`Func`] which takes `Params` as input and
23/// returns `Return`.
24///
25/// This is an efficient way to invoke a WebAssembly component where if the
26/// inputs and output are statically known this can eschew the vast majority of
27/// machinery and checks when calling WebAssembly. This is the most optimized
28/// way to call a WebAssembly component.
29///
30/// Note that like [`Func`] this is a pointer within a [`Store`](crate::Store)
31/// and usage will panic if used with the wrong store.
32///
33/// This type is primarily created with the [`Func::typed`] API.
34///
35/// See [`ComponentType`] for more information about supported types.
36pub struct TypedFunc<Params, Return> {
37    func: Func,
38
39    // The definition of this field is somewhat subtle and may be surprising.
40    // Naively one might expect something like
41    //
42    //      _marker: marker::PhantomData<fn(Params) -> Return>,
43    //
44    // Since this is a function pointer after all. The problem with this
45    // definition though is that it imposes the wrong variance on `Params` from
46    // what we want. Abstractly a `fn(Params)` is able to store `Params` within
47    // it meaning you can only give it `Params` that live longer than the
48    // function pointer.
49    //
50    // With a component model function, however, we're always copying data from
51    // the host into the guest, so we are never storing pointers to `Params`
52    // into the guest outside the duration of a `call`, meaning we can actually
53    // accept values in `TypedFunc::call` which live for a shorter duration
54    // than the `Params` argument on the struct.
55    //
56    // This all means that we don't use a phantom function pointer, but instead
57    // feign phantom storage here to get the variance desired.
58    _marker: marker::PhantomData<(Params, Return)>,
59}
60
61impl<Params, Return> Copy for TypedFunc<Params, Return> {}
62
63impl<Params, Return> Clone for TypedFunc<Params, Return> {
64    fn clone(&self) -> TypedFunc<Params, Return> {
65        *self
66    }
67}
68
69impl<Params, Return> TypedFunc<Params, Return>
70where
71    Params: ComponentNamedList + Lower,
72    Return: ComponentNamedList + Lift,
73{
74    /// Creates a new [`TypedFunc`] from the provided component [`Func`],
75    /// unsafely asserting that the underlying function takes `Params` as
76    /// input and returns `Return`.
77    ///
78    /// # Unsafety
79    ///
80    /// This is an unsafe function because it does not verify that the [`Func`]
81    /// provided actually implements this signature. It's up to the caller to
82    /// have performed some other sort of check to ensure that the signature is
83    /// correct.
84    pub unsafe fn new_unchecked(func: Func) -> TypedFunc<Params, Return> {
85        TypedFunc {
86            _marker: marker::PhantomData,
87            func,
88        }
89    }
90
91    /// Returns the underlying un-typed [`Func`] that this [`TypedFunc`]
92    /// references.
93    pub fn func(&self) -> &Func {
94        &self.func
95    }
96
97    /// Calls the underlying WebAssembly component function using the provided
98    /// `params` as input.
99    ///
100    /// This method is used to enter into a component. Execution happens within
101    /// the `store` provided. The `params` are copied into WebAssembly memory
102    /// as appropriate and a core wasm function is invoked.
103    ///
104    /// # Post-return
105    ///
106    /// In the component model each function can have a "post return" specified
107    /// which allows cleaning up the arguments returned to the host. For example
108    /// if WebAssembly returns a string to the host then it might be a uniquely
109    /// allocated string which, after the host finishes processing it, needs to
110    /// be deallocated in the wasm instance's own linear memory to prevent
111    /// memory leaks in wasm itself. The `post-return` canonical abi option is
112    /// used to configured this.
113    ///
114    /// To accommodate this feature of the component model after invoking a
115    /// function via [`TypedFunc::call`] you must next invoke
116    /// [`TypedFunc::post_return`]. Note that the return value of the function
117    /// should be processed between these two function calls. The return value
118    /// continues to be usable from an embedder's perspective after
119    /// `post_return` is called, but after `post_return` is invoked it may no
120    /// longer retain the same value that the wasm module originally returned.
121    ///
122    /// Also note that [`TypedFunc::post_return`] must be invoked irrespective
123    /// of whether the canonical ABI option `post-return` was configured or not.
124    /// This means that embedders must unconditionally call
125    /// [`TypedFunc::post_return`] when a function returns. If this function
126    /// call returns an error, however, then [`TypedFunc::post_return`] is not
127    /// required.
128    ///
129    /// # Errors
130    ///
131    /// This function can return an error for a number of reasons:
132    ///
133    /// * If the wasm itself traps during execution.
134    /// * If the wasm traps while copying arguments into memory.
135    /// * If the wasm provides bad allocation pointers when copying arguments
136    ///   into memory.
137    /// * If the wasm returns a value which violates the canonical ABI.
138    /// * If this function's instances cannot be entered, for example if the
139    ///   instance is currently calling a host function.
140    /// * If a previous function call occurred and the corresponding
141    ///   `post_return` hasn't been invoked yet.
142    ///
143    /// In general there are many ways that things could go wrong when copying
144    /// types in and out of a wasm module with the canonical ABI, and certain
145    /// error conditions are specific to certain types. For example a
146    /// WebAssembly module can't return an invalid `char`. When allocating space
147    /// for this host to copy a string into the returned pointer must be
148    /// in-bounds in memory.
149    ///
150    /// If an error happens then the error should contain detailed enough
151    /// information to understand which part of the canonical ABI went wrong
152    /// and what to inspect.
153    ///
154    /// # Panics
155    ///
156    /// Panics if this is called on a function in an asynchronous store. This
157    /// only works with functions defined within a synchronous store. Also
158    /// panics if `store` does not own this function.
159    pub fn call(&self, store: impl AsContextMut, params: Params) -> Result<Return> {
160        assert!(
161            !store.as_context().async_support(),
162            "must use `call_async` when async support is enabled on the config"
163        );
164        self.call_impl(store, params)
165    }
166
167    /// Exactly like [`Self::call`], except for use on asynchronous stores.
168    ///
169    /// # Panics
170    ///
171    /// Panics if this is called on a function in a synchronous store. This
172    /// only works with functions defined within an asynchronous store. Also
173    /// panics if `store` does not own this function.
174    #[cfg(feature = "async")]
175    pub async fn call_async(
176        &self,
177        mut store: impl AsContextMut<Data: Send>,
178        params: Params,
179    ) -> Result<Return>
180    where
181        Return: 'static,
182    {
183        let mut store = store.as_context_mut();
184        assert!(
185            store.0.async_support(),
186            "cannot use `call_async` when async support is not enabled on the config"
187        );
188        #[cfg(feature = "component-model-async")]
189        {
190            use crate::component::concurrent::TaskId;
191            use crate::runtime::vm::SendSyncPtr;
192            use core::ptr::NonNull;
193
194            let ptr = SendSyncPtr::from(NonNull::from(&params).cast::<u8>());
195            let prepared =
196                self.prepare_call(store.as_context_mut(), false, false, move |cx, ty, dst| {
197                    // SAFETY: The goal here is to get `Params`, a non-`'static`
198                    // value, to live long enough to the lowering of the
199                    // parameters. We're guaranteed that `Params` lives in the
200                    // future of the outer function (we're in an `async fn`) so it'll
201                    // stay alive as long as the future itself. That is distinct,
202                    // for example, from the signature of `call_concurrent` below.
203                    //
204                    // Here a pointer to `Params` is smuggled to this location
205                    // through a `SendSyncPtr<u8>` to thwart the `'static` check
206                    // of rustc and the signature of `prepare_call`.
207                    //
208                    // Note the use of `RemoveOnDrop` in the code that follows
209                    // this closure, which ensures that the task will be removed
210                    // from the concurrent state to which it belongs when the
211                    // containing `Future` is dropped, thereby ensuring that
212                    // this closure will never be called if it hasn't already,
213                    // meaning it will never see a dangling pointer.
214                    let params = unsafe { ptr.cast::<Params>().as_ref() };
215                    Self::lower_args(cx, ty, dst, params)
216                })?;
217
218            struct RemoveOnDrop<'a, T: 'static> {
219                store: StoreContextMut<'a, T>,
220                task: TaskId,
221            }
222
223            impl<'a, T> Drop for RemoveOnDrop<'a, T> {
224                fn drop(&mut self) {
225                    self.task.remove(self.store.as_context_mut()).unwrap();
226                }
227            }
228
229            let mut wrapper = RemoveOnDrop {
230                store,
231                task: prepared.task_id(),
232            };
233
234            let result = concurrent::queue_call(wrapper.store.as_context_mut(), prepared)?;
235            wrapper
236                .store
237                .as_context_mut()
238                .run_concurrent_trap_on_idle(async |_| Ok(result.await?.0))
239                .await?
240        }
241        #[cfg(not(feature = "component-model-async"))]
242        {
243            store
244                .on_fiber(|store| self.call_impl(store, params))
245                .await?
246        }
247    }
248
249    /// Start a concurrent call to this function.
250    ///
251    /// Unlike [`Self::call`] and [`Self::call_async`] (both of which require
252    /// exclusive access to the store until the completion of the call), calls
253    /// made using this method may run concurrently with other calls to the same
254    /// instance.  In addition, the runtime will call the `post-return` function
255    /// (if any) automatically when the guest task completes -- no need to
256    /// explicitly call `Func::post_return` afterward.
257    ///
258    /// Besides the task's return value, this returns a [`TaskExit`]
259    /// representing the completion of the guest task and any transitive
260    /// subtasks it might create.
261    ///
262    /// # Panics
263    ///
264    /// Panics if the store that the [`Accessor`] is derived from does not own
265    /// this function.
266    #[cfg(feature = "component-model-async")]
267    pub async fn call_concurrent(
268        self,
269        accessor: impl AsAccessor<Data: Send>,
270        params: Params,
271    ) -> Result<(Return, TaskExit)>
272    where
273        Params: 'static,
274        Return: 'static,
275    {
276        let result = accessor.as_accessor().with(|mut store| {
277            let mut store = store.as_context_mut();
278            assert!(
279                store.0.async_support(),
280                "cannot use `call_concurrent` when async support is not enabled on the config"
281            );
282
283            let prepared =
284                self.prepare_call(store.as_context_mut(), true, true, move |cx, ty, dst| {
285                    Self::lower_args(cx, ty, dst, &params)
286                })?;
287            concurrent::queue_call(store, prepared)
288        });
289        let (result, rx) = result?.await?;
290        Ok((result, TaskExit(rx)))
291    }
292
293    fn lower_args<T>(
294        cx: &mut LowerContext<T>,
295        ty: InterfaceType,
296        dst: &mut [MaybeUninit<ValRaw>],
297        params: &Params,
298    ) -> Result<()> {
299        use crate::component::storage::slice_to_storage_mut;
300
301        if Params::flatten_count() <= MAX_FLAT_PARAMS {
302            // SAFETY: the safety of `slice_to_storage_mut` relies on
303            // `Params::Lower` being represented by a sequence of
304            // `ValRaw`, and that's a guarantee upheld by the `Lower`
305            // trait itself.
306            let dst: &mut MaybeUninit<Params::Lower> = unsafe { slice_to_storage_mut(dst) };
307            Self::lower_stack_args(cx, &params, ty, dst)
308        } else {
309            Self::lower_heap_args(cx, &params, ty, &mut dst[0])
310        }
311    }
312
313    /// Calls `concurrent::prepare_call` with monomorphized functions for
314    /// lowering the parameters and lifting the result according to the number
315    /// of core Wasm parameters and results in the signature of the function to
316    /// be called.
317    #[cfg(feature = "component-model-async")]
318    fn prepare_call<T>(
319        self,
320        store: StoreContextMut<'_, T>,
321        remove_task_automatically: bool,
322        call_post_return_automatically: bool,
323        lower: impl FnOnce(
324            &mut LowerContext<T>,
325            InterfaceType,
326            &mut [MaybeUninit<ValRaw>],
327        ) -> Result<()>
328        + Send
329        + Sync
330        + 'static,
331    ) -> Result<PreparedCall<Return>>
332    where
333        Return: 'static,
334    {
335        use crate::component::storage::slice_to_storage;
336
337        let param_count = if Params::flatten_count() <= MAX_FLAT_PARAMS {
338            Params::flatten_count()
339        } else {
340            1
341        };
342        let max_results = if self.func.abi_async(store.0) {
343            MAX_FLAT_PARAMS
344        } else {
345            MAX_FLAT_RESULTS
346        };
347        concurrent::prepare_call(
348            store,
349            self.func,
350            param_count,
351            remove_task_automatically,
352            call_post_return_automatically,
353            move |func, store, params_out| {
354                func.with_lower_context(store, call_post_return_automatically, |cx, ty| {
355                    lower(cx, ty, params_out)
356                })
357            },
358            move |func, store, results| {
359                let result = if Return::flatten_count() <= max_results {
360                    func.with_lift_context(store, |cx, ty| {
361                        // SAFETY: Per the safety requiments documented for the
362                        // `ComponentType` trait, `Return::Lower` must be
363                        // compatible at the binary level with a `[ValRaw; N]`,
364                        // where `N` is `mem::size_of::<Return::Lower>() /
365                        // mem::size_of::<ValRaw>()`.  And since this function
366                        // is only used when `Return::flatten_count() <=
367                        // MAX_FLAT_RESULTS` and `MAX_FLAT_RESULTS == 1`, `N`
368                        // can only either be 0 or 1.
369                        //
370                        // See `ComponentInstance::exit_call` for where we use
371                        // the result count passed from
372                        // `wasmtime_environ::fact::trampoline`-generated code
373                        // to ensure the slice has the correct length, and also
374                        // `concurrent::start_call` for where we conservatively
375                        // use a slice length of 1 unconditionally.  Also note
376                        // that, as of this writing `slice_to_storage`
377                        // double-checks the slice length is sufficient.
378                        let results: &Return::Lower = unsafe { slice_to_storage(results) };
379                        Self::lift_stack_result(cx, ty, results)
380                    })?
381                } else {
382                    func.with_lift_context(store, |cx, ty| {
383                        Self::lift_heap_result(cx, ty, &results[0])
384                    })?
385                };
386                Ok(Box::new(result))
387            },
388        )
389    }
390
391    fn call_impl(&self, mut store: impl AsContextMut, params: Params) -> Result<Return> {
392        let store = store.as_context_mut();
393
394        if self.func.abi_async(store.0) {
395            bail!("must enable the `component-model-async` feature to call async-lifted exports")
396        }
397
398        // Note that this is in theory simpler than it might read at this time.
399        // Here we're doing a runtime dispatch on the `flatten_count` for the
400        // params/results to see whether they're inbounds. This creates 4 cases
401        // to handle. In reality this is a highly optimizable branch where LLVM
402        // will easily figure out that only one branch here is taken.
403        //
404        // Otherwise this current construction is done to ensure that the stack
405        // space reserved for the params/results is always of the appropriate
406        // size (as the params/results needed differ depending on the "flatten"
407        // count)
408        //
409        // SAFETY: the safety of these invocations of `call_raw` depends on the
410        // correctness of the ascription of the `LowerParams` and `LowerReturn`
411        // types on the `call_raw` function. That's upheld here through the
412        // safety requirements of `Lift` and `Lower` on `Params` and `Return` in
413        // combination with checking the various possible branches here and
414        // dispatching to appropriately typed functions.
415        unsafe {
416            // This type is used as `LowerParams` for `call_raw` which is either
417            // `Params::Lower` or `ValRaw` representing it's either on the stack
418            // or it's on the heap. This allocates 1 extra `ValRaw` on the stack
419            // if `Params` is empty and `Return` is also empty, but that's a
420            // reasonable enough price to pay for now given the current code
421            // organization.
422            #[derive(Copy, Clone)]
423            union Union<T: Copy, U: Copy> {
424                _a: T,
425                _b: U,
426            }
427
428            if Return::flatten_count() <= MAX_FLAT_RESULTS {
429                self.func.call_raw(
430                    store,
431                    |cx, ty, dst: &mut MaybeUninit<Union<Params::Lower, ValRaw>>| {
432                        let dst = storage_as_slice_mut(dst);
433                        Self::lower_args(cx, ty, dst, &params)
434                    },
435                    Self::lift_stack_result,
436                )
437            } else {
438                self.func.call_raw(
439                    store,
440                    |cx, ty, dst: &mut MaybeUninit<Union<Params::Lower, ValRaw>>| {
441                        let dst = storage_as_slice_mut(dst);
442                        Self::lower_args(cx, ty, dst, &params)
443                    },
444                    Self::lift_heap_result,
445                )
446            }
447        }
448    }
449
450    /// Lower parameters directly onto the stack specified by the `dst`
451    /// location.
452    ///
453    /// This is only valid to call when the "flatten count" is small enough, or
454    /// when the canonical ABI says arguments go through the stack rather than
455    /// the heap.
456    fn lower_stack_args<T>(
457        cx: &mut LowerContext<'_, T>,
458        params: &Params,
459        ty: InterfaceType,
460        dst: &mut MaybeUninit<Params::Lower>,
461    ) -> Result<()> {
462        assert!(Params::flatten_count() <= MAX_FLAT_PARAMS);
463        params.linear_lower_to_flat(cx, ty, dst)?;
464        Ok(())
465    }
466
467    /// Lower parameters onto a heap-allocated location.
468    ///
469    /// This is used when the stack space to be used for the arguments is above
470    /// the `MAX_FLAT_PARAMS` threshold. Here the wasm's `realloc` function is
471    /// invoked to allocate space and then parameters are stored at that heap
472    /// pointer location.
473    fn lower_heap_args<T>(
474        cx: &mut LowerContext<'_, T>,
475        params: &Params,
476        ty: InterfaceType,
477        dst: &mut MaybeUninit<ValRaw>,
478    ) -> Result<()> {
479        // Memory must exist via validation if the arguments are stored on the
480        // heap, so we can create a `MemoryMut` at this point. Afterwards
481        // `realloc` is used to allocate space for all the arguments and then
482        // they're all stored in linear memory.
483        //
484        // Note that `realloc` will bake in a check that the returned pointer is
485        // in-bounds.
486        let ptr = cx.realloc(0, 0, Params::ALIGN32, Params::SIZE32)?;
487        params.linear_lower_to_memory(cx, ty, ptr)?;
488
489        // Note that the pointer here is stored as a 64-bit integer. This allows
490        // this to work with either 32 or 64-bit memories. For a 32-bit memory
491        // it'll just ignore the upper 32 zero bits, and for 64-bit memories
492        // this'll have the full 64-bits. Note that for 32-bit memories the call
493        // to `realloc` above guarantees that the `ptr` is in-bounds meaning
494        // that we will know that the zero-extended upper bits of `ptr` are
495        // guaranteed to be zero.
496        //
497        // This comment about 64-bit integers is also referred to below with
498        // "WRITEPTR64".
499        dst.write(ValRaw::i64(ptr as i64));
500
501        Ok(())
502    }
503
504    /// Lift the result of a function directly from the stack result.
505    ///
506    /// This is only used when the result fits in the maximum number of stack
507    /// slots.
508    fn lift_stack_result(
509        cx: &mut LiftContext<'_>,
510        ty: InterfaceType,
511        dst: &Return::Lower,
512    ) -> Result<Return> {
513        Return::linear_lift_from_flat(cx, ty, dst)
514    }
515
516    /// Lift the result of a function where the result is stored indirectly on
517    /// the heap.
518    fn lift_heap_result(
519        cx: &mut LiftContext<'_>,
520        ty: InterfaceType,
521        dst: &ValRaw,
522    ) -> Result<Return> {
523        assert!(Return::flatten_count() > MAX_FLAT_RESULTS);
524        // FIXME(#4311): needs to read an i64 for memory64
525        let ptr = usize::try_from(dst.get_u32())?;
526        if ptr % usize::try_from(Return::ALIGN32)? != 0 {
527            bail!("return pointer not aligned");
528        }
529
530        let bytes = cx
531            .memory()
532            .get(ptr..)
533            .and_then(|b| b.get(..Return::SIZE32))
534            .ok_or_else(|| anyhow::anyhow!("pointer out of bounds of memory"))?;
535        Return::linear_lift_from_memory(cx, ty, bytes)
536    }
537
538    /// See [`Func::post_return`]
539    pub fn post_return(&self, store: impl AsContextMut) -> Result<()> {
540        self.func.post_return(store)
541    }
542
543    /// See [`Func::post_return_async`]
544    #[cfg(feature = "async")]
545    pub async fn post_return_async<T: Send>(
546        &self,
547        store: impl AsContextMut<Data = T>,
548    ) -> Result<()> {
549        self.func.post_return_async(store).await
550    }
551}
552
553/// A trait representing a static list of named types that can be passed to or
554/// returned from a [`TypedFunc`].
555///
556/// This trait is implemented for a number of tuple types and is not expected
557/// to be implemented externally. The contents of this trait are hidden as it's
558/// intended to be an implementation detail of Wasmtime. The contents of this
559/// trait are not covered by Wasmtime's stability guarantees.
560///
561/// For more information about this trait see [`Func::typed`] and
562/// [`TypedFunc`].
563//
564// Note that this is an `unsafe` trait, and the unsafety means that
565// implementations of this trait must be correct or otherwise [`TypedFunc`]
566// would not be memory safe. The main reason this is `unsafe` is the
567// `typecheck` function which must operate correctly relative to the `AsTuple`
568// interpretation of the implementor.
569pub unsafe trait ComponentNamedList: ComponentType {}
570
571/// A trait representing types which can be passed to and read from components
572/// with the canonical ABI.
573///
574/// This trait is implemented for Rust types which can be communicated to
575/// components. The [`Func::typed`] and [`TypedFunc`] Rust items are the main
576/// consumers of this trait.
577///
578/// Supported Rust types include:
579///
580/// | Component Model Type              | Rust Type                            |
581/// |-----------------------------------|--------------------------------------|
582/// | `{s,u}{8,16,32,64}`               | `{i,u}{8,16,32,64}`                  |
583/// | `f{32,64}`                        | `f{32,64}`                           |
584/// | `bool`                            | `bool`                               |
585/// | `char`                            | `char`                               |
586/// | `tuple<A, B>`                     | `(A, B)`                             |
587/// | `option<T>`                       | `Option<T>`                          |
588/// | `result`                          | `Result<(), ()>`                     |
589/// | `result<T>`                       | `Result<T, ()>`                      |
590/// | `result<_, E>`                    | `Result<(), E>`                      |
591/// | `result<T, E>`                    | `Result<T, E>`                       |
592/// | `string`                          | `String`, `&str`, or [`WasmStr`]     |
593/// | `list<T>`                         | `Vec<T>`, `&[T]`, or [`WasmList`]    |
594/// | `own<T>`, `borrow<T>`             | [`Resource<T>`] or [`ResourceAny`]   |
595/// | `record`                          | [`#[derive(ComponentType)]`][d-cm]   |
596/// | `variant`                         | [`#[derive(ComponentType)]`][d-cm]   |
597/// | `enum`                            | [`#[derive(ComponentType)]`][d-cm]   |
598/// | `flags`                           | [`flags!`][f-m]                      |
599///
600/// [`Resource<T>`]: crate::component::Resource
601/// [`ResourceAny`]: crate::component::ResourceAny
602/// [d-cm]: macro@crate::component::ComponentType
603/// [f-m]: crate::component::flags
604///
605/// Rust standard library pointers such as `&T`, `Box<T>`, and `Arc<T>`
606/// additionally represent whatever type `T` represents in the component model.
607/// Note that types such as `record`, `variant`, `enum`, and `flags` are
608/// generated by the embedder at compile time. These macros derive
609/// implementation of this trait for custom types to map to custom types in the
610/// component model. Note that for `record`, `variant`, `enum`, and `flags`
611/// those types are often generated by the
612/// [`bindgen!`](crate::component::bindgen) macro from WIT definitions.
613///
614/// Types that implement [`ComponentType`] are used for `Params` and `Return`
615/// in [`TypedFunc`] and [`Func::typed`].
616///
617/// The contents of this trait are hidden as it's intended to be an
618/// implementation detail of Wasmtime. The contents of this trait are not
619/// covered by Wasmtime's stability guarantees.
620///
621/// # Safety
622///
623/// Note that this is an `unsafe` trait as `TypedFunc`'s safety heavily relies on
624/// the correctness of the implementations of this trait. Some ways in which this
625/// trait must be correct to be safe are:
626///
627/// * The `Lower` associated type must be a `ValRaw` sequence. It doesn't have to
628///   literally be `[ValRaw; N]` but when laid out in memory it must be adjacent
629///   `ValRaw` values and have a multiple of the size of `ValRaw` and the same
630///   alignment.
631///
632/// * The `lower` function must initialize the bits within `Lower` that are going
633///   to be read by the trampoline that's used to enter core wasm. A trampoline
634///   is passed `*mut Lower` and will read the canonical abi arguments in
635///   sequence, so all of the bits must be correctly initialized.
636///
637/// * The `size` and `align` functions must be correct for this value stored in
638///   the canonical ABI. The `Cursor<T>` iteration of these bytes rely on this
639///   for correctness as they otherwise eschew bounds-checking.
640///
641/// There are likely some other correctness issues which aren't documented as
642/// well, this isn't currently an exhaustive list. It suffices to say, though,
643/// that correctness bugs in this trait implementation are highly likely to
644/// lead to security bugs, which again leads to the `unsafe` in the trait.
645///
646/// Note that this trait specifically is not sealed because `bindgen!`-generated
647/// types must be able to implement this trait using a `#[derive]` macro. For
648/// users it's recommended to not implement this trait manually given the
649/// non-exhaustive list of safety requirements that must be upheld. This trait
650/// is implemented at your own risk if you do so.
651///
652/// # Send and Sync
653///
654/// While on the topic of safety it's worth discussing the `Send` and `Sync`
655/// bounds here as well. These bounds might naively seem like they shouldn't be
656/// required for all component types as they're host-level types not guest-level
657/// types persisted anywhere. Various subtleties lead to these bounds, however:
658///
659/// * Fibers require that all stack-local variables are `Send` and `Sync` for
660///   fibers themselves to be send/sync. Unfortunately we have no help from the
661///   compiler on this one so it's up to Wasmtime's discipline to maintain this.
662///   One instance of this is that return values are placed on the stack as
663///   they're lowered into guest memory. This lowering operation can involve
664///   malloc and context switches, so return values must be Send/Sync.
665///
666/// * In the implementation of component model async it's not uncommon for types
667///   to be "buffered" in the store temporarily. For example parameters might
668///   reside in a store temporarily while wasm has backpressure turned on.
669///
670/// Overall it's generally easiest to require `Send` and `Sync` for all
671/// component types. There additionally aren't known use case for non-`Send` or
672/// non-`Sync` types at this time.
673pub unsafe trait ComponentType: Send + Sync {
674    /// Representation of the "lowered" form of this component value.
675    ///
676    /// Lowerings lower into core wasm values which are represented by `ValRaw`.
677    /// This `Lower` type must be a list of `ValRaw` as either a literal array
678    /// or a struct where every field is a `ValRaw`. This must be `Copy` (as
679    /// `ValRaw` is `Copy`) and support all byte patterns. This being correct is
680    /// one reason why the trait is unsafe.
681    #[doc(hidden)]
682    type Lower: Copy;
683
684    /// The information about this type's canonical ABI (size/align/etc).
685    #[doc(hidden)]
686    const ABI: CanonicalAbiInfo;
687
688    #[doc(hidden)]
689    const SIZE32: usize = Self::ABI.size32 as usize;
690    #[doc(hidden)]
691    const ALIGN32: u32 = Self::ABI.align32;
692
693    #[doc(hidden)]
694    const IS_RUST_UNIT_TYPE: bool = false;
695
696    /// Whether this type might require a call to the guest's realloc function
697    /// to allocate linear memory when lowering (e.g. a non-empty `string`).
698    ///
699    /// If this is `false`, Wasmtime may optimize lowering by using
700    /// `LowerContext::new_without_realloc` and lowering values outside of any
701    /// fiber.  That will panic if the lowering process ends up needing realloc
702    /// after all, so `true` is a conservative default.
703    #[doc(hidden)]
704    const MAY_REQUIRE_REALLOC: bool = true;
705
706    /// Returns the number of core wasm abi values will be used to represent
707    /// this type in its lowered form.
708    ///
709    /// This divides the size of `Self::Lower` by the size of `ValRaw`.
710    #[doc(hidden)]
711    fn flatten_count() -> usize {
712        assert!(mem::size_of::<Self::Lower>() % mem::size_of::<ValRaw>() == 0);
713        assert!(mem::align_of::<Self::Lower>() == mem::align_of::<ValRaw>());
714        mem::size_of::<Self::Lower>() / mem::size_of::<ValRaw>()
715    }
716
717    /// Performs a type-check to see whether this component value type matches
718    /// the interface type `ty` provided.
719    #[doc(hidden)]
720    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()>;
721}
722
723#[doc(hidden)]
724pub unsafe trait ComponentVariant: ComponentType {
725    const CASES: &'static [Option<CanonicalAbiInfo>];
726    const INFO: VariantInfo = VariantInfo::new_static(Self::CASES);
727    const PAYLOAD_OFFSET32: usize = Self::INFO.payload_offset32 as usize;
728}
729
730/// Host types which can be passed to WebAssembly components.
731///
732/// This trait is implemented for all types that can be passed to components
733/// either as parameters of component exports or returns of component imports.
734/// This trait represents the ability to convert from the native host
735/// representation to the canonical ABI.
736///
737/// Built-in types to Rust such as `Option<T>` implement this trait as
738/// appropriate. For a mapping of component model to Rust types see
739/// [`ComponentType`].
740///
741/// For user-defined types, for example `record` types mapped to Rust `struct`s,
742/// this crate additionally has
743/// [`#[derive(Lower)]`](macro@crate::component::Lower).
744///
745/// Note that like [`ComponentType`] the definition of this trait is intended to
746/// be an internal implementation detail of Wasmtime at this time. It's
747/// recommended to use the `#[derive(Lower)]` implementation instead.
748pub unsafe trait Lower: ComponentType {
749    /// Performs the "lower" function in the linear memory version of the
750    /// canonical ABI.
751    ///
752    /// This method will lower the current value into a component. The `lower`
753    /// function performs a "flat" lowering into the `dst` specified which is
754    /// allowed to be uninitialized entering this method but is guaranteed to be
755    /// fully initialized if the method returns `Ok(())`.
756    ///
757    /// The `cx` context provided is the context within which this lowering is
758    /// happening. This contains information such as canonical options specified
759    /// (e.g. string encodings, memories, etc), the store itself, along with
760    /// type information.
761    ///
762    /// The `ty` parameter is the destination type that is being lowered into.
763    /// For example this is the component's "view" of the type that is being
764    /// lowered. This is guaranteed to have passed a `typecheck` earlier.
765    ///
766    /// This will only be called if `typecheck` passes for `Op::Lower`.
767    #[doc(hidden)]
768    fn linear_lower_to_flat<T>(
769        &self,
770        cx: &mut LowerContext<'_, T>,
771        ty: InterfaceType,
772        dst: &mut MaybeUninit<Self::Lower>,
773    ) -> Result<()>;
774
775    /// Performs the "store" operation in the linear memory version of the
776    /// canonical ABI.
777    ///
778    /// This function will store `self` into the linear memory described by
779    /// `cx` at the `offset` provided.
780    ///
781    /// It is expected that `offset` is a valid offset in memory for
782    /// `Self::SIZE32` bytes. At this time that's not an unsafe contract as it's
783    /// always re-checked on all stores, but this is something that will need to
784    /// be improved in the future to remove extra bounds checks. For now this
785    /// function will panic if there's a bug and `offset` isn't valid within
786    /// memory.
787    ///
788    /// The `ty` type information passed here is the same as the type
789    /// information passed to `lower` above, and is the component's own view of
790    /// what the resulting type should be.
791    ///
792    /// This will only be called if `typecheck` passes for `Op::Lower`.
793    #[doc(hidden)]
794    fn linear_lower_to_memory<T>(
795        &self,
796        cx: &mut LowerContext<'_, T>,
797        ty: InterfaceType,
798        offset: usize,
799    ) -> Result<()>;
800
801    /// Provided method to lower a list of `Self` into memory.
802    ///
803    /// Requires that `offset` has already been checked for alignment and
804    /// validity in terms of being in-bounds, otherwise this may panic.
805    ///
806    /// This is primarily here to get overridden for implementations of integers
807    /// which can avoid some extra fluff and use a pattern that's more easily
808    /// optimizable by LLVM.
809    #[doc(hidden)]
810    fn linear_store_list_to_memory<T>(
811        cx: &mut LowerContext<'_, T>,
812        ty: InterfaceType,
813        mut offset: usize,
814        items: &[Self],
815    ) -> Result<()>
816    where
817        Self: Sized,
818    {
819        for item in items {
820            item.linear_lower_to_memory(cx, ty, offset)?;
821            offset += Self::SIZE32;
822        }
823        Ok(())
824    }
825}
826
827/// Host types which can be created from the canonical ABI.
828///
829/// This is the mirror of the [`Lower`] trait where it represents the capability
830/// of acquiring items from WebAssembly and passing them to the host.
831///
832/// Built-in types to Rust such as `Option<T>` implement this trait as
833/// appropriate. For a mapping of component model to Rust types see
834/// [`ComponentType`].
835///
836/// For user-defined types, for example `record` types mapped to Rust `struct`s,
837/// this crate additionally has
838/// [`#[derive(Lift)]`](macro@crate::component::Lift).
839///
840/// Note that like [`ComponentType`] the definition of this trait is intended to
841/// be an internal implementation detail of Wasmtime at this time. It's
842/// recommended to use the `#[derive(Lift)]` implementation instead.
843pub unsafe trait Lift: Sized + ComponentType {
844    /// Performs the "lift" operation in the linear memory version of the
845    /// canonical ABI.
846    ///
847    /// This function performs a "flat" lift operation from the `src` specified
848    /// which is a sequence of core wasm values. The lifting operation will
849    /// validate core wasm values and produce a `Self` on success.
850    ///
851    /// The `cx` provided contains contextual information such as the store
852    /// that's being loaded from, canonical options, and type information.
853    ///
854    /// The `ty` parameter is the origin component's specification for what the
855    /// type that is being lifted is. For example this is the record type or the
856    /// resource type that is being lifted.
857    ///
858    /// Note that this has a default implementation but if `typecheck` passes
859    /// for `Op::Lift` this needs to be overridden.
860    #[doc(hidden)]
861    fn linear_lift_from_flat(
862        cx: &mut LiftContext<'_>,
863        ty: InterfaceType,
864        src: &Self::Lower,
865    ) -> Result<Self>;
866
867    /// Performs the "load" operation in the linear memory version of the
868    /// canonical ABI.
869    ///
870    /// This will read the `bytes` provided, which are a sub-slice into the
871    /// linear memory described by `cx`. The `bytes` array provided is
872    /// guaranteed to be `Self::SIZE32` bytes large. All of memory is then also
873    /// available through `cx` for bounds-checks and such as necessary for
874    /// strings/lists.
875    ///
876    /// The `ty` argument is the type that's being loaded, as described by the
877    /// original component.
878    ///
879    /// Note that this has a default implementation but if `typecheck` passes
880    /// for `Op::Lift` this needs to be overridden.
881    #[doc(hidden)]
882    fn linear_lift_from_memory(
883        cx: &mut LiftContext<'_>,
884        ty: InterfaceType,
885        bytes: &[u8],
886    ) -> Result<Self>;
887
888    /// Converts `list` into a `Vec<T>`, used in `Lift for Vec<T>`.
889    #[doc(hidden)]
890    fn linear_lift_list_from_memory(
891        cx: &mut LiftContext<'_>,
892        list: &WasmList<Self>,
893    ) -> Result<Vec<Self>>
894    where
895        Self: Sized,
896    {
897        let mut dst = Vec::with_capacity(list.len);
898        Self::linear_lift_into_from_memory(cx, list, &mut dst)?;
899        Ok(dst)
900    }
901
902    /// Load no more than `max_count` items from `list` into `dst`.
903    ///
904    /// This is primarily here to get overridden for implementations of integers
905    /// which can avoid some extra fluff and use a pattern that's more easily
906    /// optimizable by LLVM.
907    #[doc(hidden)]
908    fn linear_lift_into_from_memory(
909        cx: &mut LiftContext<'_>,
910        list: &WasmList<Self>,
911        dst: &mut impl Extend<Self>,
912    ) -> Result<()>
913    where
914        Self: Sized,
915    {
916        for i in 0..list.len {
917            dst.extend(Some(list.get_from_store(cx, i).unwrap()?));
918        }
919        Ok(())
920    }
921}
922
923// Macro to help generate "forwarding implementations" of `ComponentType` to
924// another type, used for wrappers in Rust like `&T`, `Box<T>`, etc. Note that
925// these wrappers only implement lowering because lifting native Rust types
926// cannot be done.
927macro_rules! forward_type_impls {
928    ($(($($generics:tt)*) $a:ty => $b:ty,)*) => ($(
929        unsafe impl <$($generics)*> ComponentType for $a {
930            type Lower = <$b as ComponentType>::Lower;
931
932            const ABI: CanonicalAbiInfo = <$b as ComponentType>::ABI;
933
934            #[inline]
935            fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
936                <$b as ComponentType>::typecheck(ty, types)
937            }
938        }
939    )*)
940}
941
942forward_type_impls! {
943    (T: ComponentType + ?Sized) &'_ T => T,
944    (T: ComponentType + ?Sized) Box<T> => T,
945    (T: ComponentType + ?Sized) alloc::sync::Arc<T> => T,
946    () String => str,
947    (T: ComponentType) Vec<T> => [T],
948}
949
950macro_rules! forward_lowers {
951    ($(($($generics:tt)*) $a:ty => $b:ty,)*) => ($(
952        unsafe impl <$($generics)*> Lower for $a {
953            fn linear_lower_to_flat<U>(
954                &self,
955                cx: &mut LowerContext<'_, U>,
956                ty: InterfaceType,
957                dst: &mut MaybeUninit<Self::Lower>,
958            ) -> Result<()> {
959                <$b as Lower>::linear_lower_to_flat(self, cx, ty, dst)
960            }
961
962            fn linear_lower_to_memory<U>(
963                &self,
964                cx: &mut LowerContext<'_, U>,
965                ty: InterfaceType,
966                offset: usize,
967            ) -> Result<()> {
968                <$b as Lower>::linear_lower_to_memory(self, cx, ty, offset)
969            }
970        }
971    )*)
972}
973
974forward_lowers! {
975    (T: Lower + ?Sized) &'_ T => T,
976    (T: Lower + ?Sized) Box<T> => T,
977    (T: Lower + ?Sized) alloc::sync::Arc<T> => T,
978    () String => str,
979    (T: Lower) Vec<T> => [T],
980}
981
982macro_rules! forward_string_lifts {
983    ($($a:ty,)*) => ($(
984        unsafe impl Lift for $a {
985            #[inline]
986            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
987                Ok(<WasmStr as Lift>::linear_lift_from_flat(cx, ty, src)?.to_str_from_memory(cx.memory())?.into())
988            }
989
990            #[inline]
991            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
992                Ok(<WasmStr as Lift>::linear_lift_from_memory(cx, ty, bytes)?.to_str_from_memory(cx.memory())?.into())
993            }
994        }
995    )*)
996}
997
998forward_string_lifts! {
999    Box<str>,
1000    alloc::sync::Arc<str>,
1001    String,
1002}
1003
1004macro_rules! forward_list_lifts {
1005    ($($a:ty,)*) => ($(
1006        unsafe impl <T: Lift> Lift for $a {
1007            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1008                let list = <WasmList::<T> as Lift>::linear_lift_from_flat(cx, ty, src)?;
1009                Ok(T::linear_lift_list_from_memory(cx, &list)?.into())
1010            }
1011
1012            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1013                let list = <WasmList::<T> as Lift>::linear_lift_from_memory(cx, ty, bytes)?;
1014                Ok(T::linear_lift_list_from_memory(cx, &list)?.into())
1015            }
1016        }
1017    )*)
1018}
1019
1020forward_list_lifts! {
1021    Box<[T]>,
1022    alloc::sync::Arc<[T]>,
1023    Vec<T>,
1024}
1025
1026// Macro to help generate `ComponentType` implementations for primitive types
1027// such as integers, char, bool, etc.
1028macro_rules! integers {
1029    ($($primitive:ident = $ty:ident in $field:ident/$get:ident with abi:$abi:ident,)*) => ($(
1030        unsafe impl ComponentType for $primitive {
1031            type Lower = ValRaw;
1032
1033            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::$abi;
1034
1035            const MAY_REQUIRE_REALLOC: bool = false;
1036
1037            fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1038                match ty {
1039                    InterfaceType::$ty => Ok(()),
1040                    other => bail!("expected `{}` found `{}`", desc(&InterfaceType::$ty), desc(other))
1041                }
1042            }
1043        }
1044
1045        unsafe impl Lower for $primitive {
1046            #[inline]
1047            #[allow(trivial_numeric_casts, reason = "macro-generated code")]
1048            fn linear_lower_to_flat<T>(
1049                &self,
1050                _cx: &mut LowerContext<'_, T>,
1051                ty: InterfaceType,
1052                dst: &mut MaybeUninit<Self::Lower>,
1053            ) -> Result<()> {
1054                debug_assert!(matches!(ty, InterfaceType::$ty));
1055                dst.write(ValRaw::$field(*self as $field));
1056                Ok(())
1057            }
1058
1059            #[inline]
1060            fn linear_lower_to_memory<T>(
1061                &self,
1062                cx: &mut LowerContext<'_, T>,
1063                ty: InterfaceType,
1064                offset: usize,
1065            ) -> Result<()> {
1066                debug_assert!(matches!(ty, InterfaceType::$ty));
1067                debug_assert!(offset % Self::SIZE32 == 0);
1068                *cx.get(offset) = self.to_le_bytes();
1069                Ok(())
1070            }
1071
1072            fn linear_store_list_to_memory<T>(
1073                cx: &mut LowerContext<'_, T>,
1074                ty: InterfaceType,
1075                offset: usize,
1076                items: &[Self],
1077            ) -> Result<()> {
1078                debug_assert!(matches!(ty, InterfaceType::$ty));
1079
1080                // Double-check that the CM alignment is at least the host's
1081                // alignment for this type which should be true for all
1082                // platforms.
1083                assert!((Self::ALIGN32 as usize) >= mem::align_of::<Self>());
1084
1085                // Slice `cx`'s memory to the window that we'll be modifying.
1086                // This should all have already been verified in terms of
1087                // alignment and sizing meaning that these assertions here are
1088                // not truly necessary but are instead double-checks.
1089                //
1090                // Note that we're casting a `[u8]` slice to `[Self]` with
1091                // `align_to_mut` which is not safe in general but is safe in
1092                // our specific case as all `u8` patterns are valid `Self`
1093                // patterns since `Self` is an integral type.
1094                let dst = &mut cx.as_slice_mut()[offset..][..items.len() * Self::SIZE32];
1095                let (before, middle, end) = unsafe { dst.align_to_mut::<Self>() };
1096                assert!(before.is_empty() && end.is_empty());
1097                assert_eq!(middle.len(), items.len());
1098
1099                // And with all that out of the way perform the copying loop.
1100                // This is not a `copy_from_slice` because endianness needs to
1101                // be handled here, but LLVM should pretty easily transform this
1102                // into a memcpy on little-endian platforms.
1103                for (dst, src) in middle.iter_mut().zip(items) {
1104                    *dst = src.to_le();
1105                }
1106                Ok(())
1107            }
1108        }
1109
1110        unsafe impl Lift for $primitive {
1111            #[inline]
1112            #[allow(
1113                trivial_numeric_casts,
1114                clippy::cast_possible_truncation,
1115                reason = "macro-generated code"
1116            )]
1117            fn linear_lift_from_flat(_cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1118                debug_assert!(matches!(ty, InterfaceType::$ty));
1119                Ok(src.$get() as $primitive)
1120            }
1121
1122            #[inline]
1123            fn linear_lift_from_memory(_cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1124                debug_assert!(matches!(ty, InterfaceType::$ty));
1125                debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1126                Ok($primitive::from_le_bytes(bytes.try_into().unwrap()))
1127            }
1128
1129            fn linear_lift_into_from_memory(
1130                cx: &mut LiftContext<'_>,
1131                list: &WasmList<Self>,
1132                dst: &mut impl Extend<Self>,
1133            ) -> Result<()>
1134            where
1135                Self: Sized,
1136            {
1137                dst.extend(list._as_le_slice(cx.memory())
1138                           .iter()
1139                           .map(|i| Self::from_le(*i)));
1140                Ok(())
1141            }
1142        }
1143    )*)
1144}
1145
1146integers! {
1147    i8 = S8 in i32/get_i32 with abi:SCALAR1,
1148    u8 = U8 in u32/get_u32 with abi:SCALAR1,
1149    i16 = S16 in i32/get_i32 with abi:SCALAR2,
1150    u16 = U16 in u32/get_u32 with abi:SCALAR2,
1151    i32 = S32 in i32/get_i32 with abi:SCALAR4,
1152    u32 = U32 in u32/get_u32 with abi:SCALAR4,
1153    i64 = S64 in i64/get_i64 with abi:SCALAR8,
1154    u64 = U64 in u64/get_u64 with abi:SCALAR8,
1155}
1156
1157macro_rules! floats {
1158    ($($float:ident/$get_float:ident = $ty:ident with abi:$abi:ident)*) => ($(const _: () = {
1159        unsafe impl ComponentType for $float {
1160            type Lower = ValRaw;
1161
1162            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::$abi;
1163
1164            fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1165                match ty {
1166                    InterfaceType::$ty => Ok(()),
1167                    other => bail!("expected `{}` found `{}`", desc(&InterfaceType::$ty), desc(other))
1168                }
1169            }
1170        }
1171
1172        unsafe impl Lower for $float {
1173            #[inline]
1174            fn linear_lower_to_flat<T>(
1175                &self,
1176                _cx: &mut LowerContext<'_, T>,
1177                ty: InterfaceType,
1178                dst: &mut MaybeUninit<Self::Lower>,
1179            ) -> Result<()> {
1180                debug_assert!(matches!(ty, InterfaceType::$ty));
1181                dst.write(ValRaw::$float(self.to_bits()));
1182                Ok(())
1183            }
1184
1185            #[inline]
1186            fn linear_lower_to_memory<T>(
1187                &self,
1188                cx: &mut LowerContext<'_, T>,
1189                ty: InterfaceType,
1190                offset: usize,
1191            ) -> Result<()> {
1192                debug_assert!(matches!(ty, InterfaceType::$ty));
1193                debug_assert!(offset % Self::SIZE32 == 0);
1194                let ptr = cx.get(offset);
1195                *ptr = self.to_bits().to_le_bytes();
1196                Ok(())
1197            }
1198
1199            fn linear_store_list_to_memory<T>(
1200                cx: &mut LowerContext<'_, T>,
1201                ty: InterfaceType,
1202                offset: usize,
1203                items: &[Self],
1204            ) -> Result<()> {
1205                debug_assert!(matches!(ty, InterfaceType::$ty));
1206
1207                // Double-check that the CM alignment is at least the host's
1208                // alignment for this type which should be true for all
1209                // platforms.
1210                assert!((Self::ALIGN32 as usize) >= mem::align_of::<Self>());
1211
1212                // Slice `cx`'s memory to the window that we'll be modifying.
1213                // This should all have already been verified in terms of
1214                // alignment and sizing meaning that these assertions here are
1215                // not truly necessary but are instead double-checks.
1216                let dst = &mut cx.as_slice_mut()[offset..][..items.len() * Self::SIZE32];
1217                assert!(dst.as_ptr().cast::<Self>().is_aligned());
1218
1219                // And with all that out of the way perform the copying loop.
1220                // This is not a `copy_from_slice` because endianness needs to
1221                // be handled here, but LLVM should pretty easily transform this
1222                // into a memcpy on little-endian platforms.
1223                // TODO use `as_chunks` when https://github.com/rust-lang/rust/issues/74985
1224                // is stabilized
1225                for (dst, src) in iter::zip(dst.chunks_exact_mut(Self::SIZE32), items) {
1226                    let dst: &mut [u8; Self::SIZE32] = dst.try_into().unwrap();
1227                    *dst = src.to_le_bytes();
1228                }
1229                Ok(())
1230            }
1231        }
1232
1233        unsafe impl Lift for $float {
1234            #[inline]
1235            fn linear_lift_from_flat(_cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1236                debug_assert!(matches!(ty, InterfaceType::$ty));
1237                Ok($float::from_bits(src.$get_float()))
1238            }
1239
1240            #[inline]
1241            fn linear_lift_from_memory(_cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1242                debug_assert!(matches!(ty, InterfaceType::$ty));
1243                debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1244                Ok($float::from_le_bytes(bytes.try_into().unwrap()))
1245            }
1246
1247            fn linear_lift_list_from_memory(cx: &mut LiftContext<'_>, list: &WasmList<Self>) -> Result<Vec<Self>> where Self: Sized {
1248                // See comments in `WasmList::get` for the panicking indexing
1249                let byte_size = list.len * mem::size_of::<Self>();
1250                let bytes = &cx.memory()[list.ptr..][..byte_size];
1251
1252                // The canonical ABI requires that everything is aligned to its
1253                // own size, so this should be an aligned array.
1254                assert!(bytes.as_ptr().cast::<Self>().is_aligned());
1255
1256                // Copy the resulting slice to a new Vec, handling endianness
1257                // in the process
1258                // TODO use `as_chunks` when https://github.com/rust-lang/rust/issues/74985
1259                // is stabilized
1260                Ok(
1261                    bytes
1262                        .chunks_exact(Self::SIZE32)
1263                        .map(|i| $float::from_le_bytes(i.try_into().unwrap()))
1264                        .collect()
1265                )
1266            }
1267        }
1268    };)*)
1269}
1270
1271floats! {
1272    f32/get_f32 = Float32 with abi:SCALAR4
1273    f64/get_f64 = Float64 with abi:SCALAR8
1274}
1275
1276unsafe impl ComponentType for bool {
1277    type Lower = ValRaw;
1278
1279    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR1;
1280
1281    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1282        match ty {
1283            InterfaceType::Bool => Ok(()),
1284            other => bail!("expected `bool` found `{}`", desc(other)),
1285        }
1286    }
1287}
1288
1289unsafe impl Lower for bool {
1290    fn linear_lower_to_flat<T>(
1291        &self,
1292        _cx: &mut LowerContext<'_, T>,
1293        ty: InterfaceType,
1294        dst: &mut MaybeUninit<Self::Lower>,
1295    ) -> Result<()> {
1296        debug_assert!(matches!(ty, InterfaceType::Bool));
1297        dst.write(ValRaw::i32(*self as i32));
1298        Ok(())
1299    }
1300
1301    fn linear_lower_to_memory<T>(
1302        &self,
1303        cx: &mut LowerContext<'_, T>,
1304        ty: InterfaceType,
1305        offset: usize,
1306    ) -> Result<()> {
1307        debug_assert!(matches!(ty, InterfaceType::Bool));
1308        debug_assert!(offset % Self::SIZE32 == 0);
1309        cx.get::<1>(offset)[0] = *self as u8;
1310        Ok(())
1311    }
1312}
1313
1314unsafe impl Lift for bool {
1315    #[inline]
1316    fn linear_lift_from_flat(
1317        _cx: &mut LiftContext<'_>,
1318        ty: InterfaceType,
1319        src: &Self::Lower,
1320    ) -> Result<Self> {
1321        debug_assert!(matches!(ty, InterfaceType::Bool));
1322        match src.get_i32() {
1323            0 => Ok(false),
1324            _ => Ok(true),
1325        }
1326    }
1327
1328    #[inline]
1329    fn linear_lift_from_memory(
1330        _cx: &mut LiftContext<'_>,
1331        ty: InterfaceType,
1332        bytes: &[u8],
1333    ) -> Result<Self> {
1334        debug_assert!(matches!(ty, InterfaceType::Bool));
1335        match bytes[0] {
1336            0 => Ok(false),
1337            _ => Ok(true),
1338        }
1339    }
1340}
1341
1342unsafe impl ComponentType for char {
1343    type Lower = ValRaw;
1344
1345    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
1346
1347    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1348        match ty {
1349            InterfaceType::Char => Ok(()),
1350            other => bail!("expected `char` found `{}`", desc(other)),
1351        }
1352    }
1353}
1354
1355unsafe impl Lower for char {
1356    #[inline]
1357    fn linear_lower_to_flat<T>(
1358        &self,
1359        _cx: &mut LowerContext<'_, T>,
1360        ty: InterfaceType,
1361        dst: &mut MaybeUninit<Self::Lower>,
1362    ) -> Result<()> {
1363        debug_assert!(matches!(ty, InterfaceType::Char));
1364        dst.write(ValRaw::u32(u32::from(*self)));
1365        Ok(())
1366    }
1367
1368    #[inline]
1369    fn linear_lower_to_memory<T>(
1370        &self,
1371        cx: &mut LowerContext<'_, T>,
1372        ty: InterfaceType,
1373        offset: usize,
1374    ) -> Result<()> {
1375        debug_assert!(matches!(ty, InterfaceType::Char));
1376        debug_assert!(offset % Self::SIZE32 == 0);
1377        *cx.get::<4>(offset) = u32::from(*self).to_le_bytes();
1378        Ok(())
1379    }
1380}
1381
1382unsafe impl Lift for char {
1383    #[inline]
1384    fn linear_lift_from_flat(
1385        _cx: &mut LiftContext<'_>,
1386        ty: InterfaceType,
1387        src: &Self::Lower,
1388    ) -> Result<Self> {
1389        debug_assert!(matches!(ty, InterfaceType::Char));
1390        Ok(char::try_from(src.get_u32())?)
1391    }
1392
1393    #[inline]
1394    fn linear_lift_from_memory(
1395        _cx: &mut LiftContext<'_>,
1396        ty: InterfaceType,
1397        bytes: &[u8],
1398    ) -> Result<Self> {
1399        debug_assert!(matches!(ty, InterfaceType::Char));
1400        debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1401        let bits = u32::from_le_bytes(bytes.try_into().unwrap());
1402        Ok(char::try_from(bits)?)
1403    }
1404}
1405
1406// FIXME(#4311): these probably need different constants for memory64
1407const UTF16_TAG: usize = 1 << 31;
1408const MAX_STRING_BYTE_LENGTH: usize = (1 << 31) - 1;
1409
1410// Note that this is similar to `ComponentType for WasmStr` except it can only
1411// be used for lowering, not lifting.
1412unsafe impl ComponentType for str {
1413    type Lower = [ValRaw; 2];
1414
1415    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1416
1417    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1418        match ty {
1419            InterfaceType::String => Ok(()),
1420            other => bail!("expected `string` found `{}`", desc(other)),
1421        }
1422    }
1423}
1424
1425unsafe impl Lower for str {
1426    fn linear_lower_to_flat<T>(
1427        &self,
1428        cx: &mut LowerContext<'_, T>,
1429        ty: InterfaceType,
1430        dst: &mut MaybeUninit<[ValRaw; 2]>,
1431    ) -> Result<()> {
1432        debug_assert!(matches!(ty, InterfaceType::String));
1433        let (ptr, len) = lower_string(cx, self)?;
1434        // See "WRITEPTR64" above for why this is always storing a 64-bit
1435        // integer.
1436        map_maybe_uninit!(dst[0]).write(ValRaw::i64(ptr as i64));
1437        map_maybe_uninit!(dst[1]).write(ValRaw::i64(len as i64));
1438        Ok(())
1439    }
1440
1441    fn linear_lower_to_memory<T>(
1442        &self,
1443        cx: &mut LowerContext<'_, T>,
1444        ty: InterfaceType,
1445        offset: usize,
1446    ) -> Result<()> {
1447        debug_assert!(matches!(ty, InterfaceType::String));
1448        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
1449        let (ptr, len) = lower_string(cx, self)?;
1450        // FIXME(#4311): needs memory64 handling
1451        *cx.get(offset + 0) = u32::try_from(ptr).unwrap().to_le_bytes();
1452        *cx.get(offset + 4) = u32::try_from(len).unwrap().to_le_bytes();
1453        Ok(())
1454    }
1455}
1456
1457fn lower_string<T>(cx: &mut LowerContext<'_, T>, string: &str) -> Result<(usize, usize)> {
1458    // Note that in general the wasm module can't assume anything about what the
1459    // host strings are encoded as. Additionally hosts are allowed to have
1460    // differently-encoded strings at runtime. Finally when copying a string
1461    // into wasm it's somewhat strict in the sense that the various patterns of
1462    // allocation and such are already dictated for us.
1463    //
1464    // In general what this means is that when copying a string from the host
1465    // into the destination we need to follow one of the cases of copying into
1466    // WebAssembly. It doesn't particularly matter which case as long as it ends
1467    // up in the right encoding. For example a destination encoding of
1468    // latin1+utf16 has a number of ways to get copied into and we do something
1469    // here that isn't the default "utf8 to latin1+utf16" since we have access
1470    // to simd-accelerated helpers in the `encoding_rs` crate. This is ok though
1471    // because we can fake that the host string was already stored in latin1
1472    // format and follow that copy pattern instead.
1473    match cx.options.string_encoding() {
1474        // This corresponds to `store_string_copy` in the canonical ABI where
1475        // the host's representation is utf-8 and the wasm module wants utf-8 so
1476        // a copy is all that's needed (and the `realloc` can be precise for the
1477        // initial memory allocation).
1478        StringEncoding::Utf8 => {
1479            if string.len() > MAX_STRING_BYTE_LENGTH {
1480                bail!(
1481                    "string length of {} too large to copy into wasm",
1482                    string.len()
1483                );
1484            }
1485            let ptr = cx.realloc(0, 0, 1, string.len())?;
1486            cx.as_slice_mut()[ptr..][..string.len()].copy_from_slice(string.as_bytes());
1487            Ok((ptr, string.len()))
1488        }
1489
1490        // This corresponds to `store_utf8_to_utf16` in the canonical ABI. Here
1491        // an over-large allocation is performed and then shrunk afterwards if
1492        // necessary.
1493        StringEncoding::Utf16 => {
1494            let size = string.len() * 2;
1495            if size > MAX_STRING_BYTE_LENGTH {
1496                bail!(
1497                    "string length of {} too large to copy into wasm",
1498                    string.len()
1499                );
1500            }
1501            let mut ptr = cx.realloc(0, 0, 2, size)?;
1502            let mut copied = 0;
1503            let bytes = &mut cx.as_slice_mut()[ptr..][..size];
1504            for (u, bytes) in string.encode_utf16().zip(bytes.chunks_mut(2)) {
1505                let u_bytes = u.to_le_bytes();
1506                bytes[0] = u_bytes[0];
1507                bytes[1] = u_bytes[1];
1508                copied += 1;
1509            }
1510            if (copied * 2) < size {
1511                ptr = cx.realloc(ptr, size, 2, copied * 2)?;
1512            }
1513            Ok((ptr, copied))
1514        }
1515
1516        StringEncoding::CompactUtf16 => {
1517            // This corresponds to `store_string_to_latin1_or_utf16`
1518            let bytes = string.as_bytes();
1519            let mut iter = string.char_indices();
1520            let mut ptr = cx.realloc(0, 0, 2, bytes.len())?;
1521            let mut dst = &mut cx.as_slice_mut()[ptr..][..bytes.len()];
1522            let mut result = 0;
1523            while let Some((i, ch)) = iter.next() {
1524                // Test if this `char` fits into the latin1 encoding.
1525                if let Ok(byte) = u8::try_from(u32::from(ch)) {
1526                    dst[result] = byte;
1527                    result += 1;
1528                    continue;
1529                }
1530
1531                // .. if utf16 is forced to be used then the allocation is
1532                // bumped up to the maximum size.
1533                let worst_case = bytes
1534                    .len()
1535                    .checked_mul(2)
1536                    .ok_or_else(|| anyhow!("byte length overflow"))?;
1537                if worst_case > MAX_STRING_BYTE_LENGTH {
1538                    bail!("byte length too large");
1539                }
1540                ptr = cx.realloc(ptr, bytes.len(), 2, worst_case)?;
1541                dst = &mut cx.as_slice_mut()[ptr..][..worst_case];
1542
1543                // Previously encoded latin1 bytes are inflated to their 16-bit
1544                // size for utf16
1545                for i in (0..result).rev() {
1546                    dst[2 * i] = dst[i];
1547                    dst[2 * i + 1] = 0;
1548                }
1549
1550                // and then the remainder of the string is encoded.
1551                for (u, bytes) in string[i..]
1552                    .encode_utf16()
1553                    .zip(dst[2 * result..].chunks_mut(2))
1554                {
1555                    let u_bytes = u.to_le_bytes();
1556                    bytes[0] = u_bytes[0];
1557                    bytes[1] = u_bytes[1];
1558                    result += 1;
1559                }
1560                if worst_case > 2 * result {
1561                    ptr = cx.realloc(ptr, worst_case, 2, 2 * result)?;
1562                }
1563                return Ok((ptr, result | UTF16_TAG));
1564            }
1565            if result < bytes.len() {
1566                ptr = cx.realloc(ptr, bytes.len(), 2, result)?;
1567            }
1568            Ok((ptr, result))
1569        }
1570    }
1571}
1572
1573/// Representation of a string located in linear memory in a WebAssembly
1574/// instance.
1575///
1576/// This type can be used in place of `String` and `str` for string-taking APIs
1577/// in some situations. The purpose of this type is to represent a range of
1578/// validated bytes within a component but does not actually copy the bytes. The
1579/// primary method, [`WasmStr::to_str`], attempts to return a reference to the
1580/// string directly located in the component's memory, avoiding a copy into the
1581/// host if possible.
1582///
1583/// The downside of this type, however, is that accessing a string requires a
1584/// [`Store`](crate::Store) pointer (via [`StoreContext`]). Bindings generated
1585/// by [`bindgen!`](crate::component::bindgen), for example, do not have access
1586/// to [`StoreContext`] and thus can't use this type.
1587///
1588/// This is intended for more advanced use cases such as defining functions
1589/// directly in a [`Linker`](crate::component::Linker). It's expected that in
1590/// the future [`bindgen!`](crate::component::bindgen) will also have a way to
1591/// use this type.
1592///
1593/// This type is used with [`TypedFunc`], for example, when WebAssembly returns
1594/// a string. This type cannot be used to give a string to WebAssembly, instead
1595/// `&str` should be used for that (since it's coming from the host).
1596///
1597/// Note that this type represents an in-bounds string in linear memory, but it
1598/// does not represent a valid string (e.g. valid utf-8). Validation happens
1599/// when [`WasmStr::to_str`] is called.
1600///
1601/// Also note that this type does not implement [`Lower`], it only implements
1602/// [`Lift`].
1603pub struct WasmStr {
1604    ptr: usize,
1605    len: usize,
1606    options: Options,
1607}
1608
1609impl WasmStr {
1610    pub(crate) fn new(ptr: usize, len: usize, cx: &mut LiftContext<'_>) -> Result<WasmStr> {
1611        let byte_len = match cx.options.string_encoding() {
1612            StringEncoding::Utf8 => Some(len),
1613            StringEncoding::Utf16 => len.checked_mul(2),
1614            StringEncoding::CompactUtf16 => {
1615                if len & UTF16_TAG == 0 {
1616                    Some(len)
1617                } else {
1618                    (len ^ UTF16_TAG).checked_mul(2)
1619                }
1620            }
1621        };
1622        match byte_len.and_then(|len| ptr.checked_add(len)) {
1623            Some(n) if n <= cx.memory().len() => {}
1624            _ => bail!("string pointer/length out of bounds of memory"),
1625        }
1626        Ok(WasmStr {
1627            ptr,
1628            len,
1629            options: *cx.options,
1630        })
1631    }
1632
1633    /// Returns the underlying string that this cursor points to.
1634    ///
1635    /// Note that this will internally decode the string from the wasm's
1636    /// encoding to utf-8 and additionally perform validation.
1637    ///
1638    /// The `store` provided must be the store where this string lives to
1639    /// access the correct memory.
1640    ///
1641    /// # Errors
1642    ///
1643    /// Returns an error if the string wasn't encoded correctly (e.g. invalid
1644    /// utf-8).
1645    ///
1646    /// # Panics
1647    ///
1648    /// Panics if this string is not owned by `store`.
1649    //
1650    // TODO: should add accessors for specifically utf-8 and utf-16 that perhaps
1651    // in an opt-in basis don't do validation. Additionally there should be some
1652    // method that returns `[u16]` after validating to avoid the utf16-to-utf8
1653    // transcode.
1654    pub fn to_str<'a, T: 'static>(
1655        &self,
1656        store: impl Into<StoreContext<'a, T>>,
1657    ) -> Result<Cow<'a, str>> {
1658        let store = store.into().0;
1659        let memory = self.options.memory(store);
1660        self.to_str_from_memory(memory)
1661    }
1662
1663    pub(crate) fn to_str_from_memory<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1664        match self.options.string_encoding() {
1665            StringEncoding::Utf8 => self.decode_utf8(memory),
1666            StringEncoding::Utf16 => self.decode_utf16(memory, self.len),
1667            StringEncoding::CompactUtf16 => {
1668                if self.len & UTF16_TAG == 0 {
1669                    self.decode_latin1(memory)
1670                } else {
1671                    self.decode_utf16(memory, self.len ^ UTF16_TAG)
1672                }
1673            }
1674        }
1675    }
1676
1677    fn decode_utf8<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1678        // Note that bounds-checking already happen in construction of `WasmStr`
1679        // so this is never expected to panic. This could theoretically be
1680        // unchecked indexing if we're feeling wild enough.
1681        Ok(str::from_utf8(&memory[self.ptr..][..self.len])?.into())
1682    }
1683
1684    fn decode_utf16<'a>(&self, memory: &'a [u8], len: usize) -> Result<Cow<'a, str>> {
1685        // See notes in `decode_utf8` for why this is panicking indexing.
1686        let memory = &memory[self.ptr..][..len * 2];
1687        Ok(core::char::decode_utf16(
1688            memory
1689                .chunks(2)
1690                .map(|chunk| u16::from_le_bytes(chunk.try_into().unwrap())),
1691        )
1692        .collect::<Result<String, _>>()?
1693        .into())
1694    }
1695
1696    fn decode_latin1<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1697        // See notes in `decode_utf8` for why this is panicking indexing.
1698        Ok(encoding_rs::mem::decode_latin1(
1699            &memory[self.ptr..][..self.len],
1700        ))
1701    }
1702}
1703
1704// Note that this is similar to `ComponentType for str` except it can only be
1705// used for lifting, not lowering.
1706unsafe impl ComponentType for WasmStr {
1707    type Lower = <str as ComponentType>::Lower;
1708
1709    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1710
1711    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1712        match ty {
1713            InterfaceType::String => Ok(()),
1714            other => bail!("expected `string` found `{}`", desc(other)),
1715        }
1716    }
1717}
1718
1719unsafe impl Lift for WasmStr {
1720    #[inline]
1721    fn linear_lift_from_flat(
1722        cx: &mut LiftContext<'_>,
1723        ty: InterfaceType,
1724        src: &Self::Lower,
1725    ) -> Result<Self> {
1726        debug_assert!(matches!(ty, InterfaceType::String));
1727        // FIXME(#4311): needs memory64 treatment
1728        let ptr = src[0].get_u32();
1729        let len = src[1].get_u32();
1730        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
1731        WasmStr::new(ptr, len, cx)
1732    }
1733
1734    #[inline]
1735    fn linear_lift_from_memory(
1736        cx: &mut LiftContext<'_>,
1737        ty: InterfaceType,
1738        bytes: &[u8],
1739    ) -> Result<Self> {
1740        debug_assert!(matches!(ty, InterfaceType::String));
1741        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
1742        // FIXME(#4311): needs memory64 treatment
1743        let ptr = u32::from_le_bytes(bytes[..4].try_into().unwrap());
1744        let len = u32::from_le_bytes(bytes[4..].try_into().unwrap());
1745        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
1746        WasmStr::new(ptr, len, cx)
1747    }
1748}
1749
1750unsafe impl<T> ComponentType for [T]
1751where
1752    T: ComponentType,
1753{
1754    type Lower = [ValRaw; 2];
1755
1756    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1757
1758    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
1759        match ty {
1760            InterfaceType::List(t) => T::typecheck(&types.types[*t].element, types),
1761            other => bail!("expected `list` found `{}`", desc(other)),
1762        }
1763    }
1764}
1765
1766unsafe impl<T> Lower for [T]
1767where
1768    T: Lower,
1769{
1770    fn linear_lower_to_flat<U>(
1771        &self,
1772        cx: &mut LowerContext<'_, U>,
1773        ty: InterfaceType,
1774        dst: &mut MaybeUninit<[ValRaw; 2]>,
1775    ) -> Result<()> {
1776        let elem = match ty {
1777            InterfaceType::List(i) => cx.types[i].element,
1778            _ => bad_type_info(),
1779        };
1780        let (ptr, len) = lower_list(cx, elem, self)?;
1781        // See "WRITEPTR64" above for why this is always storing a 64-bit
1782        // integer.
1783        map_maybe_uninit!(dst[0]).write(ValRaw::i64(ptr as i64));
1784        map_maybe_uninit!(dst[1]).write(ValRaw::i64(len as i64));
1785        Ok(())
1786    }
1787
1788    fn linear_lower_to_memory<U>(
1789        &self,
1790        cx: &mut LowerContext<'_, U>,
1791        ty: InterfaceType,
1792        offset: usize,
1793    ) -> Result<()> {
1794        let elem = match ty {
1795            InterfaceType::List(i) => cx.types[i].element,
1796            _ => bad_type_info(),
1797        };
1798        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
1799        let (ptr, len) = lower_list(cx, elem, self)?;
1800        *cx.get(offset + 0) = u32::try_from(ptr).unwrap().to_le_bytes();
1801        *cx.get(offset + 4) = u32::try_from(len).unwrap().to_le_bytes();
1802        Ok(())
1803    }
1804}
1805
1806// FIXME: this is not a memcpy for `T` where `T` is something like `u8`.
1807//
1808// Some attempts to fix this have proved not fruitful. In isolation an attempt
1809// was made where:
1810//
1811// * `MemoryMut` stored a `*mut [u8]` as its "last view" of memory to avoid
1812//   reloading the base pointer constantly. This view is reset on `realloc`.
1813// * The bounds-checks in `MemoryMut::get` were removed (replaced with unsafe
1814//   indexing)
1815//
1816// Even then though this didn't correctly vectorized for `Vec<u8>`. It's not
1817// entirely clear why but it appeared that it's related to reloading the base
1818// pointer to memory (I guess from `MemoryMut` itself?). Overall I'm not really
1819// clear on what's happening there, but this is surely going to be a performance
1820// bottleneck in the future.
1821fn lower_list<T, U>(
1822    cx: &mut LowerContext<'_, U>,
1823    ty: InterfaceType,
1824    list: &[T],
1825) -> Result<(usize, usize)>
1826where
1827    T: Lower,
1828{
1829    let elem_size = T::SIZE32;
1830    let size = list
1831        .len()
1832        .checked_mul(elem_size)
1833        .ok_or_else(|| anyhow!("size overflow copying a list"))?;
1834    let ptr = cx.realloc(0, 0, T::ALIGN32, size)?;
1835    T::linear_store_list_to_memory(cx, ty, ptr, list)?;
1836    Ok((ptr, list.len()))
1837}
1838
1839/// Representation of a list of values that are owned by a WebAssembly instance.
1840///
1841/// For some more commentary about the rationale for this type see the
1842/// documentation of [`WasmStr`]. In summary this type can avoid a copy when
1843/// passing data to the host in some situations but is additionally more
1844/// cumbersome to use by requiring a [`Store`](crate::Store) to be provided.
1845///
1846/// This type is used whenever a `(list T)` is returned from a [`TypedFunc`],
1847/// for example. This type represents a list of values that are stored in linear
1848/// memory which are waiting to be read.
1849///
1850/// Note that this type represents only a valid range of bytes for the list
1851/// itself, it does not represent validity of the elements themselves and that's
1852/// performed when they're iterated.
1853///
1854/// Note that this type does not implement the [`Lower`] trait, only [`Lift`].
1855pub struct WasmList<T> {
1856    ptr: usize,
1857    len: usize,
1858    options: Options,
1859    elem: InterfaceType,
1860    instance: Instance,
1861    _marker: marker::PhantomData<T>,
1862}
1863
1864impl<T: Lift> WasmList<T> {
1865    pub(crate) fn new(
1866        ptr: usize,
1867        len: usize,
1868        cx: &mut LiftContext<'_>,
1869        elem: InterfaceType,
1870    ) -> Result<WasmList<T>> {
1871        match len
1872            .checked_mul(T::SIZE32)
1873            .and_then(|len| ptr.checked_add(len))
1874        {
1875            Some(n) if n <= cx.memory().len() => {}
1876            _ => bail!("list pointer/length out of bounds of memory"),
1877        }
1878        if ptr % usize::try_from(T::ALIGN32)? != 0 {
1879            bail!("list pointer is not aligned")
1880        }
1881        Ok(WasmList {
1882            ptr,
1883            len,
1884            options: *cx.options,
1885            elem,
1886            instance: cx.instance_handle(),
1887            _marker: marker::PhantomData,
1888        })
1889    }
1890
1891    /// Returns the item length of this vector
1892    #[inline]
1893    pub fn len(&self) -> usize {
1894        self.len
1895    }
1896
1897    /// Gets the `n`th element of this list.
1898    ///
1899    /// Returns `None` if `index` is out of bounds. Returns `Some(Err(..))` if
1900    /// the value couldn't be decoded (it was invalid). Returns `Some(Ok(..))`
1901    /// if the value is valid.
1902    ///
1903    /// # Panics
1904    ///
1905    /// This function will panic if the string did not originally come from the
1906    /// `store` specified.
1907    //
1908    // TODO: given that interface values are intended to be consumed in one go
1909    // should we even expose a random access iteration API? In theory all
1910    // consumers should be validating through the iterator.
1911    pub fn get(&self, mut store: impl AsContextMut, index: usize) -> Option<Result<T>> {
1912        let store = store.as_context_mut().0;
1913        self.options.store_id().assert_belongs_to(store.id());
1914        let mut cx = LiftContext::new(store, &self.options, self.instance);
1915        self.get_from_store(&mut cx, index)
1916    }
1917
1918    fn get_from_store(&self, cx: &mut LiftContext<'_>, index: usize) -> Option<Result<T>> {
1919        if index >= self.len {
1920            return None;
1921        }
1922        // Note that this is using panicking indexing and this is expected to
1923        // never fail. The bounds-checking here happened during the construction
1924        // of the `WasmList` itself which means these should always be in-bounds
1925        // (and wasm memory can only grow). This could theoretically be
1926        // unchecked indexing if we're confident enough and it's actually a perf
1927        // issue one day.
1928        let bytes = &cx.memory()[self.ptr + index * T::SIZE32..][..T::SIZE32];
1929        Some(T::linear_lift_from_memory(cx, self.elem, bytes))
1930    }
1931
1932    /// Returns an iterator over the elements of this list.
1933    ///
1934    /// Each item of the list may fail to decode and is represented through the
1935    /// `Result` value of the iterator.
1936    pub fn iter<'a, U: 'static>(
1937        &'a self,
1938        store: impl Into<StoreContextMut<'a, U>>,
1939    ) -> impl ExactSizeIterator<Item = Result<T>> + 'a {
1940        let store = store.into().0;
1941        self.options.store_id().assert_belongs_to(store.id());
1942        let mut cx = LiftContext::new(store, &self.options, self.instance);
1943        (0..self.len).map(move |i| self.get_from_store(&mut cx, i).unwrap())
1944    }
1945}
1946
1947macro_rules! raw_wasm_list_accessors {
1948    ($($i:ident)*) => ($(
1949        impl WasmList<$i> {
1950            /// Get access to the raw underlying memory for this list.
1951            ///
1952            /// This method will return a direct slice into the original wasm
1953            /// module's linear memory where the data for this slice is stored.
1954            /// This allows the embedder to have efficient access to the
1955            /// underlying memory if needed and avoid copies and such if
1956            /// desired.
1957            ///
1958            /// Note that multi-byte integers are stored in little-endian format
1959            /// so portable processing of this slice must be aware of the host's
1960            /// byte-endianness. The `from_le` constructors in the Rust standard
1961            /// library should be suitable for converting from little-endian.
1962            ///
1963            /// # Panics
1964            ///
1965            /// Panics if the `store` provided is not the one from which this
1966            /// slice originated.
1967            pub fn as_le_slice<'a, T: 'static>(&self, store: impl Into<StoreContext<'a, T>>) -> &'a [$i] {
1968                let memory = self.options.memory(store.into().0);
1969                self._as_le_slice(memory)
1970            }
1971
1972            fn _as_le_slice<'a>(&self, all_of_memory: &'a [u8]) -> &'a [$i] {
1973                // See comments in `WasmList::get` for the panicking indexing
1974                let byte_size = self.len * mem::size_of::<$i>();
1975                let bytes = &all_of_memory[self.ptr..][..byte_size];
1976
1977                // The canonical ABI requires that everything is aligned to its
1978                // own size, so this should be an aligned array. Furthermore the
1979                // alignment of primitive integers for hosts should be smaller
1980                // than or equal to the size of the primitive itself, meaning
1981                // that a wasm canonical-abi-aligned list is also aligned for
1982                // the host. That should mean that the head/tail slices here are
1983                // empty.
1984                //
1985                // Also note that the `unsafe` here is needed since the type
1986                // we're aligning to isn't guaranteed to be valid, but in our
1987                // case it's just integers and bytes so this should be safe.
1988                unsafe {
1989                    let (head, body, tail) = bytes.align_to::<$i>();
1990                    assert!(head.is_empty() && tail.is_empty());
1991                    body
1992                }
1993            }
1994        }
1995    )*)
1996}
1997
1998raw_wasm_list_accessors! {
1999    i8 i16 i32 i64
2000    u8 u16 u32 u64
2001}
2002
2003// Note that this is similar to `ComponentType for str` except it can only be
2004// used for lifting, not lowering.
2005unsafe impl<T: ComponentType> ComponentType for WasmList<T> {
2006    type Lower = <[T] as ComponentType>::Lower;
2007
2008    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
2009
2010    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2011        <[T] as ComponentType>::typecheck(ty, types)
2012    }
2013}
2014
2015unsafe impl<T: Lift> Lift for WasmList<T> {
2016    fn linear_lift_from_flat(
2017        cx: &mut LiftContext<'_>,
2018        ty: InterfaceType,
2019        src: &Self::Lower,
2020    ) -> Result<Self> {
2021        let elem = match ty {
2022            InterfaceType::List(i) => cx.types[i].element,
2023            _ => bad_type_info(),
2024        };
2025        // FIXME(#4311): needs memory64 treatment
2026        let ptr = src[0].get_u32();
2027        let len = src[1].get_u32();
2028        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
2029        WasmList::new(ptr, len, cx, elem)
2030    }
2031
2032    fn linear_lift_from_memory(
2033        cx: &mut LiftContext<'_>,
2034        ty: InterfaceType,
2035        bytes: &[u8],
2036    ) -> Result<Self> {
2037        let elem = match ty {
2038            InterfaceType::List(i) => cx.types[i].element,
2039            _ => bad_type_info(),
2040        };
2041        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2042        // FIXME(#4311): needs memory64 treatment
2043        let ptr = u32::from_le_bytes(bytes[..4].try_into().unwrap());
2044        let len = u32::from_le_bytes(bytes[4..].try_into().unwrap());
2045        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
2046        WasmList::new(ptr, len, cx, elem)
2047    }
2048}
2049
2050/// Verify that the given wasm type is a tuple with the expected fields in the right order.
2051fn typecheck_tuple(
2052    ty: &InterfaceType,
2053    types: &InstanceType<'_>,
2054    expected: &[fn(&InterfaceType, &InstanceType<'_>) -> Result<()>],
2055) -> Result<()> {
2056    match ty {
2057        InterfaceType::Tuple(t) => {
2058            let tuple = &types.types[*t];
2059            if tuple.types.len() != expected.len() {
2060                bail!(
2061                    "expected {}-tuple, found {}-tuple",
2062                    expected.len(),
2063                    tuple.types.len()
2064                );
2065            }
2066            for (ty, check) in tuple.types.iter().zip(expected) {
2067                check(ty, types)?;
2068            }
2069            Ok(())
2070        }
2071        other => bail!("expected `tuple` found `{}`", desc(other)),
2072    }
2073}
2074
2075/// Verify that the given wasm type is a record with the expected fields in the right order and with the right
2076/// names.
2077pub fn typecheck_record(
2078    ty: &InterfaceType,
2079    types: &InstanceType<'_>,
2080    expected: &[(&str, fn(&InterfaceType, &InstanceType<'_>) -> Result<()>)],
2081) -> Result<()> {
2082    match ty {
2083        InterfaceType::Record(index) => {
2084            let fields = &types.types[*index].fields;
2085
2086            if fields.len() != expected.len() {
2087                bail!(
2088                    "expected record of {} fields, found {} fields",
2089                    expected.len(),
2090                    fields.len()
2091                );
2092            }
2093
2094            for (field, &(name, check)) in fields.iter().zip(expected) {
2095                check(&field.ty, types)
2096                    .with_context(|| format!("type mismatch for field {name}"))?;
2097
2098                if field.name != name {
2099                    bail!("expected record field named {}, found {}", name, field.name);
2100                }
2101            }
2102
2103            Ok(())
2104        }
2105        other => bail!("expected `record` found `{}`", desc(other)),
2106    }
2107}
2108
2109/// Verify that the given wasm type is a variant with the expected cases in the right order and with the right
2110/// names.
2111pub fn typecheck_variant(
2112    ty: &InterfaceType,
2113    types: &InstanceType<'_>,
2114    expected: &[(
2115        &str,
2116        Option<fn(&InterfaceType, &InstanceType<'_>) -> Result<()>>,
2117    )],
2118) -> Result<()> {
2119    match ty {
2120        InterfaceType::Variant(index) => {
2121            let cases = &types.types[*index].cases;
2122
2123            if cases.len() != expected.len() {
2124                bail!(
2125                    "expected variant of {} cases, found {} cases",
2126                    expected.len(),
2127                    cases.len()
2128                );
2129            }
2130
2131            for ((case_name, case_ty), &(name, check)) in cases.iter().zip(expected) {
2132                if *case_name != name {
2133                    bail!("expected variant case named {name}, found {case_name}");
2134                }
2135
2136                match (check, case_ty) {
2137                    (Some(check), Some(ty)) => check(ty, types)
2138                        .with_context(|| format!("type mismatch for case {name}"))?,
2139                    (None, None) => {}
2140                    (Some(_), None) => {
2141                        bail!("case `{name}` has no type but one was expected")
2142                    }
2143                    (None, Some(_)) => {
2144                        bail!("case `{name}` has a type but none was expected")
2145                    }
2146                }
2147            }
2148
2149            Ok(())
2150        }
2151        other => bail!("expected `variant` found `{}`", desc(other)),
2152    }
2153}
2154
2155/// Verify that the given wasm type is a enum with the expected cases in the right order and with the right
2156/// names.
2157pub fn typecheck_enum(
2158    ty: &InterfaceType,
2159    types: &InstanceType<'_>,
2160    expected: &[&str],
2161) -> Result<()> {
2162    match ty {
2163        InterfaceType::Enum(index) => {
2164            let names = &types.types[*index].names;
2165
2166            if names.len() != expected.len() {
2167                bail!(
2168                    "expected enum of {} names, found {} names",
2169                    expected.len(),
2170                    names.len()
2171                );
2172            }
2173
2174            for (name, expected) in names.iter().zip(expected) {
2175                if name != expected {
2176                    bail!("expected enum case named {}, found {}", expected, name);
2177                }
2178            }
2179
2180            Ok(())
2181        }
2182        other => bail!("expected `enum` found `{}`", desc(other)),
2183    }
2184}
2185
2186/// Verify that the given wasm type is a flags type with the expected flags in the right order and with the right
2187/// names.
2188pub fn typecheck_flags(
2189    ty: &InterfaceType,
2190    types: &InstanceType<'_>,
2191    expected: &[&str],
2192) -> Result<()> {
2193    match ty {
2194        InterfaceType::Flags(index) => {
2195            let names = &types.types[*index].names;
2196
2197            if names.len() != expected.len() {
2198                bail!(
2199                    "expected flags type with {} names, found {} names",
2200                    expected.len(),
2201                    names.len()
2202                );
2203            }
2204
2205            for (name, expected) in names.iter().zip(expected) {
2206                if name != expected {
2207                    bail!("expected flag named {}, found {}", expected, name);
2208                }
2209            }
2210
2211            Ok(())
2212        }
2213        other => bail!("expected `flags` found `{}`", desc(other)),
2214    }
2215}
2216
2217/// Format the specified bitflags using the specified names for debugging
2218pub fn format_flags(bits: &[u32], names: &[&str], f: &mut fmt::Formatter) -> fmt::Result {
2219    f.write_str("(")?;
2220    let mut wrote = false;
2221    for (index, name) in names.iter().enumerate() {
2222        if ((bits[index / 32] >> (index % 32)) & 1) != 0 {
2223            if wrote {
2224                f.write_str("|")?;
2225            } else {
2226                wrote = true;
2227            }
2228
2229            f.write_str(name)?;
2230        }
2231    }
2232    f.write_str(")")
2233}
2234
2235unsafe impl<T> ComponentType for Option<T>
2236where
2237    T: ComponentType,
2238{
2239    type Lower = TupleLower<<u32 as ComponentType>::Lower, T::Lower>;
2240
2241    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::variant_static(&[None, Some(T::ABI)]);
2242
2243    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2244        match ty {
2245            InterfaceType::Option(t) => T::typecheck(&types.types[*t].ty, types),
2246            other => bail!("expected `option` found `{}`", desc(other)),
2247        }
2248    }
2249}
2250
2251unsafe impl<T> ComponentVariant for Option<T>
2252where
2253    T: ComponentType,
2254{
2255    const CASES: &'static [Option<CanonicalAbiInfo>] = &[None, Some(T::ABI)];
2256}
2257
2258unsafe impl<T> Lower for Option<T>
2259where
2260    T: Lower,
2261{
2262    fn linear_lower_to_flat<U>(
2263        &self,
2264        cx: &mut LowerContext<'_, U>,
2265        ty: InterfaceType,
2266        dst: &mut MaybeUninit<Self::Lower>,
2267    ) -> Result<()> {
2268        let payload = match ty {
2269            InterfaceType::Option(ty) => cx.types[ty].ty,
2270            _ => bad_type_info(),
2271        };
2272        match self {
2273            None => {
2274                map_maybe_uninit!(dst.A1).write(ValRaw::i32(0));
2275                // Note that this is unsafe as we're writing an arbitrary
2276                // bit-pattern to an arbitrary type, but part of the unsafe
2277                // contract of the `ComponentType` trait is that we can assign
2278                // any bit-pattern. By writing all zeros here we're ensuring
2279                // that the core wasm arguments this translates to will all be
2280                // zeros (as the canonical ABI requires).
2281                unsafe {
2282                    map_maybe_uninit!(dst.A2).as_mut_ptr().write_bytes(0u8, 1);
2283                }
2284            }
2285            Some(val) => {
2286                map_maybe_uninit!(dst.A1).write(ValRaw::i32(1));
2287                val.linear_lower_to_flat(cx, payload, map_maybe_uninit!(dst.A2))?;
2288            }
2289        }
2290        Ok(())
2291    }
2292
2293    fn linear_lower_to_memory<U>(
2294        &self,
2295        cx: &mut LowerContext<'_, U>,
2296        ty: InterfaceType,
2297        offset: usize,
2298    ) -> Result<()> {
2299        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
2300        let payload = match ty {
2301            InterfaceType::Option(ty) => cx.types[ty].ty,
2302            _ => bad_type_info(),
2303        };
2304        match self {
2305            None => {
2306                cx.get::<1>(offset)[0] = 0;
2307            }
2308            Some(val) => {
2309                cx.get::<1>(offset)[0] = 1;
2310                val.linear_lower_to_memory(
2311                    cx,
2312                    payload,
2313                    offset + (Self::INFO.payload_offset32 as usize),
2314                )?;
2315            }
2316        }
2317        Ok(())
2318    }
2319}
2320
2321unsafe impl<T> Lift for Option<T>
2322where
2323    T: Lift,
2324{
2325    fn linear_lift_from_flat(
2326        cx: &mut LiftContext<'_>,
2327        ty: InterfaceType,
2328        src: &Self::Lower,
2329    ) -> Result<Self> {
2330        let payload = match ty {
2331            InterfaceType::Option(ty) => cx.types[ty].ty,
2332            _ => bad_type_info(),
2333        };
2334        Ok(match src.A1.get_i32() {
2335            0 => None,
2336            1 => Some(T::linear_lift_from_flat(cx, payload, &src.A2)?),
2337            _ => bail!("invalid option discriminant"),
2338        })
2339    }
2340
2341    fn linear_lift_from_memory(
2342        cx: &mut LiftContext<'_>,
2343        ty: InterfaceType,
2344        bytes: &[u8],
2345    ) -> Result<Self> {
2346        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2347        let payload_ty = match ty {
2348            InterfaceType::Option(ty) => cx.types[ty].ty,
2349            _ => bad_type_info(),
2350        };
2351        let discrim = bytes[0];
2352        let payload = &bytes[Self::INFO.payload_offset32 as usize..];
2353        match discrim {
2354            0 => Ok(None),
2355            1 => Ok(Some(T::linear_lift_from_memory(cx, payload_ty, payload)?)),
2356            _ => bail!("invalid option discriminant"),
2357        }
2358    }
2359}
2360
2361#[derive(Clone, Copy)]
2362#[repr(C)]
2363pub struct ResultLower<T: Copy, E: Copy> {
2364    tag: ValRaw,
2365    payload: ResultLowerPayload<T, E>,
2366}
2367
2368#[derive(Clone, Copy)]
2369#[repr(C)]
2370union ResultLowerPayload<T: Copy, E: Copy> {
2371    ok: T,
2372    err: E,
2373}
2374
2375unsafe impl<T, E> ComponentType for Result<T, E>
2376where
2377    T: ComponentType,
2378    E: ComponentType,
2379{
2380    type Lower = ResultLower<T::Lower, E::Lower>;
2381
2382    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::variant_static(&[Some(T::ABI), Some(E::ABI)]);
2383
2384    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2385        match ty {
2386            InterfaceType::Result(r) => {
2387                let result = &types.types[*r];
2388                match &result.ok {
2389                    Some(ty) => T::typecheck(ty, types)?,
2390                    None if T::IS_RUST_UNIT_TYPE => {}
2391                    None => bail!("expected no `ok` type"),
2392                }
2393                match &result.err {
2394                    Some(ty) => E::typecheck(ty, types)?,
2395                    None if E::IS_RUST_UNIT_TYPE => {}
2396                    None => bail!("expected no `err` type"),
2397                }
2398                Ok(())
2399            }
2400            other => bail!("expected `result` found `{}`", desc(other)),
2401        }
2402    }
2403}
2404
2405/// Lowers the payload of a variant into the storage for the entire payload,
2406/// handling writing zeros at the end of the representation if this payload is
2407/// smaller than the entire flat representation.
2408///
2409/// * `payload` - the flat storage space for the entire payload of the variant
2410/// * `typed_payload` - projection from the payload storage space to the
2411///   individual storage space for this variant.
2412/// * `lower` - lowering operation used to initialize the `typed_payload` return
2413///   value.
2414///
2415/// For more information on this se the comments in the `Lower for Result`
2416/// implementation below.
2417pub unsafe fn lower_payload<P, T>(
2418    payload: &mut MaybeUninit<P>,
2419    typed_payload: impl FnOnce(&mut MaybeUninit<P>) -> &mut MaybeUninit<T>,
2420    lower: impl FnOnce(&mut MaybeUninit<T>) -> Result<()>,
2421) -> Result<()> {
2422    let typed = typed_payload(payload);
2423    lower(typed)?;
2424
2425    let typed_len = unsafe { storage_as_slice(typed).len() };
2426    let payload = unsafe { storage_as_slice_mut(payload) };
2427    for slot in payload[typed_len..].iter_mut() {
2428        slot.write(ValRaw::u64(0));
2429    }
2430    Ok(())
2431}
2432
2433unsafe impl<T, E> ComponentVariant for Result<T, E>
2434where
2435    T: ComponentType,
2436    E: ComponentType,
2437{
2438    const CASES: &'static [Option<CanonicalAbiInfo>] = &[Some(T::ABI), Some(E::ABI)];
2439}
2440
2441unsafe impl<T, E> Lower for Result<T, E>
2442where
2443    T: Lower,
2444    E: Lower,
2445{
2446    fn linear_lower_to_flat<U>(
2447        &self,
2448        cx: &mut LowerContext<'_, U>,
2449        ty: InterfaceType,
2450        dst: &mut MaybeUninit<Self::Lower>,
2451    ) -> Result<()> {
2452        let (ok, err) = match ty {
2453            InterfaceType::Result(ty) => {
2454                let ty = &cx.types[ty];
2455                (ty.ok, ty.err)
2456            }
2457            _ => bad_type_info(),
2458        };
2459
2460        // This implementation of `Lower::lower`, if you're reading these from
2461        // the top of this file, is the first location that the "join" logic of
2462        // the component model's canonical ABI encountered. The rough problem is
2463        // that let's say we have a component model type of the form:
2464        //
2465        //      (result u64 (error (tuple f32 u16)))
2466        //
2467        // The flat representation of this is actually pretty tricky. Currently
2468        // it is:
2469        //
2470        //      i32 i64 i32
2471        //
2472        // The first `i32` is the discriminant for the `result`, and the payload
2473        // is represented by `i64 i32`. The "ok" variant will only use the `i64`
2474        // and the "err" variant will use both `i64` and `i32`.
2475        //
2476        // In the "ok" variant the first issue is encountered. The size of one
2477        // variant may not match the size of the other variants. All variants
2478        // start at the "front" but when lowering a type we need to be sure to
2479        // initialize the later variants (lest we leak random host memory into
2480        // the guest module). Due to how the `Lower` type is represented as a
2481        // `union` of all the variants what ends up happening here is that
2482        // internally within the `lower_payload` after the typed payload is
2483        // lowered the remaining bits of the payload that weren't initialized
2484        // are all set to zero. This will guarantee that we'll write to all the
2485        // slots for each variant.
2486        //
2487        // The "err" variant encounters the second issue, however, which is that
2488        // the flat representation for each type may differ between payloads. In
2489        // the "ok" arm an `i64` is written, but the `lower` implementation for
2490        // the "err" arm will write an `f32` and then an `i32`. For this
2491        // implementation of `lower` to be valid the `f32` needs to get inflated
2492        // to an `i64` with zero-padding in the upper bits. What may be
2493        // surprising, however, is that none of this is handled in this file.
2494        // This implementation looks like it's blindly deferring to `E::lower`
2495        // and hoping it does the right thing.
2496        //
2497        // In reality, however, the correctness of variant lowering relies on
2498        // two subtle details of the `ValRaw` implementation in Wasmtime:
2499        //
2500        // 1. First the `ValRaw` value always contains little-endian values.
2501        //    This means that if a `u32` is written, a `u64` is read, and then
2502        //    the `u64` has its upper bits truncated the original value will
2503        //    always be retained. This is primarily here for big-endian
2504        //    platforms where if it weren't little endian then the opposite
2505        //    would occur and the wrong value would be read.
2506        //
2507        // 2. Second, and perhaps even more subtly, the `ValRaw` constructors
2508        //    for 32-bit types actually always initialize 64-bits of the
2509        //    `ValRaw`. In the component model flat ABI only 32 and 64-bit types
2510        //    are used so 64-bits is big enough to contain everything. This
2511        //    means that when a `ValRaw` is written into the destination it will
2512        //    always, whether it's needed or not, be "ready" to get extended up
2513        //    to 64-bits.
2514        //
2515        // Put together these two subtle guarantees means that all `Lower`
2516        // implementations can be written "naturally" as one might naively
2517        // expect. Variants will, on each arm, zero out remaining fields and all
2518        // writes to the flat representation will automatically be 64-bit writes
2519        // meaning that if the value is read as a 64-bit value, which isn't
2520        // known at the time of the write, it'll still be correct.
2521        match self {
2522            Ok(e) => {
2523                map_maybe_uninit!(dst.tag).write(ValRaw::i32(0));
2524                unsafe {
2525                    lower_payload(
2526                        map_maybe_uninit!(dst.payload),
2527                        |payload| map_maybe_uninit!(payload.ok),
2528                        |dst| match ok {
2529                            Some(ok) => e.linear_lower_to_flat(cx, ok, dst),
2530                            None => Ok(()),
2531                        },
2532                    )
2533                }
2534            }
2535            Err(e) => {
2536                map_maybe_uninit!(dst.tag).write(ValRaw::i32(1));
2537                unsafe {
2538                    lower_payload(
2539                        map_maybe_uninit!(dst.payload),
2540                        |payload| map_maybe_uninit!(payload.err),
2541                        |dst| match err {
2542                            Some(err) => e.linear_lower_to_flat(cx, err, dst),
2543                            None => Ok(()),
2544                        },
2545                    )
2546                }
2547            }
2548        }
2549    }
2550
2551    fn linear_lower_to_memory<U>(
2552        &self,
2553        cx: &mut LowerContext<'_, U>,
2554        ty: InterfaceType,
2555        offset: usize,
2556    ) -> Result<()> {
2557        let (ok, err) = match ty {
2558            InterfaceType::Result(ty) => {
2559                let ty = &cx.types[ty];
2560                (ty.ok, ty.err)
2561            }
2562            _ => bad_type_info(),
2563        };
2564        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
2565        let payload_offset = Self::INFO.payload_offset32 as usize;
2566        match self {
2567            Ok(e) => {
2568                cx.get::<1>(offset)[0] = 0;
2569                if let Some(ok) = ok {
2570                    e.linear_lower_to_memory(cx, ok, offset + payload_offset)?;
2571                }
2572            }
2573            Err(e) => {
2574                cx.get::<1>(offset)[0] = 1;
2575                if let Some(err) = err {
2576                    e.linear_lower_to_memory(cx, err, offset + payload_offset)?;
2577                }
2578            }
2579        }
2580        Ok(())
2581    }
2582}
2583
2584unsafe impl<T, E> Lift for Result<T, E>
2585where
2586    T: Lift,
2587    E: Lift,
2588{
2589    #[inline]
2590    fn linear_lift_from_flat(
2591        cx: &mut LiftContext<'_>,
2592        ty: InterfaceType,
2593        src: &Self::Lower,
2594    ) -> Result<Self> {
2595        let (ok, err) = match ty {
2596            InterfaceType::Result(ty) => {
2597                let ty = &cx.types[ty];
2598                (ty.ok, ty.err)
2599            }
2600            _ => bad_type_info(),
2601        };
2602        // Note that this implementation specifically isn't trying to actually
2603        // reinterpret or alter the bits of `lower` depending on which variant
2604        // we're lifting. This ends up all working out because the value is
2605        // stored in little-endian format.
2606        //
2607        // When stored in little-endian format the `{T,E}::Lower`, when each
2608        // individual `ValRaw` is read, means that if an i64 value, extended
2609        // from an i32 value, was stored then when the i32 value is read it'll
2610        // automatically ignore the upper bits.
2611        //
2612        // This "trick" allows us to seamlessly pass through the `Self::Lower`
2613        // representation into the lifting/lowering without trying to handle
2614        // "join"ed types as per the canonical ABI. It just so happens that i64
2615        // bits will naturally be reinterpreted as f64. Additionally if the
2616        // joined type is i64 but only the lower bits are read that's ok and we
2617        // don't need to validate the upper bits.
2618        //
2619        // This is largely enabled by WebAssembly/component-model#35 where no
2620        // validation needs to be performed for ignored bits and bytes here.
2621        Ok(match src.tag.get_i32() {
2622            0 => Ok(unsafe { lift_option(cx, ok, &src.payload.ok)? }),
2623            1 => Err(unsafe { lift_option(cx, err, &src.payload.err)? }),
2624            _ => bail!("invalid expected discriminant"),
2625        })
2626    }
2627
2628    #[inline]
2629    fn linear_lift_from_memory(
2630        cx: &mut LiftContext<'_>,
2631        ty: InterfaceType,
2632        bytes: &[u8],
2633    ) -> Result<Self> {
2634        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2635        let discrim = bytes[0];
2636        let payload = &bytes[Self::INFO.payload_offset32 as usize..];
2637        let (ok, err) = match ty {
2638            InterfaceType::Result(ty) => {
2639                let ty = &cx.types[ty];
2640                (ty.ok, ty.err)
2641            }
2642            _ => bad_type_info(),
2643        };
2644        match discrim {
2645            0 => Ok(Ok(load_option(cx, ok, &payload[..T::SIZE32])?)),
2646            1 => Ok(Err(load_option(cx, err, &payload[..E::SIZE32])?)),
2647            _ => bail!("invalid expected discriminant"),
2648        }
2649    }
2650}
2651
2652fn lift_option<T>(cx: &mut LiftContext<'_>, ty: Option<InterfaceType>, src: &T::Lower) -> Result<T>
2653where
2654    T: Lift,
2655{
2656    match ty {
2657        Some(ty) => T::linear_lift_from_flat(cx, ty, src),
2658        None => Ok(empty_lift()),
2659    }
2660}
2661
2662fn load_option<T>(cx: &mut LiftContext<'_>, ty: Option<InterfaceType>, bytes: &[u8]) -> Result<T>
2663where
2664    T: Lift,
2665{
2666    match ty {
2667        Some(ty) => T::linear_lift_from_memory(cx, ty, bytes),
2668        None => Ok(empty_lift()),
2669    }
2670}
2671
2672fn empty_lift<T>() -> T
2673where
2674    T: Lift,
2675{
2676    assert!(T::IS_RUST_UNIT_TYPE);
2677    assert_eq!(mem::size_of::<T>(), 0);
2678    unsafe { MaybeUninit::uninit().assume_init() }
2679}
2680
2681/// Helper structure to define `Lower` for tuples below.
2682///
2683/// Uses default type parameters to have fields be zero-sized and not present
2684/// in memory for smaller tuple values.
2685#[expect(non_snake_case, reason = "more amenable to macro-generated code")]
2686#[doc(hidden)]
2687#[derive(Clone, Copy)]
2688#[repr(C)]
2689pub struct TupleLower<
2690    T1 = (),
2691    T2 = (),
2692    T3 = (),
2693    T4 = (),
2694    T5 = (),
2695    T6 = (),
2696    T7 = (),
2697    T8 = (),
2698    T9 = (),
2699    T10 = (),
2700    T11 = (),
2701    T12 = (),
2702    T13 = (),
2703    T14 = (),
2704    T15 = (),
2705    T16 = (),
2706    T17 = (),
2707> {
2708    // NB: these names match the names in `for_each_function_signature!`
2709    A1: T1,
2710    A2: T2,
2711    A3: T3,
2712    A4: T4,
2713    A5: T5,
2714    A6: T6,
2715    A7: T7,
2716    A8: T8,
2717    A9: T9,
2718    A10: T10,
2719    A11: T11,
2720    A12: T12,
2721    A13: T13,
2722    A14: T14,
2723    A15: T15,
2724    A16: T16,
2725    A17: T17,
2726    _align_tuple_lower0_correctly: [ValRaw; 0],
2727}
2728
2729macro_rules! impl_component_ty_for_tuples {
2730    ($n:tt $($t:ident)*) => {
2731        #[allow(non_snake_case, reason = "macro-generated code")]
2732        unsafe impl<$($t,)*> ComponentType for ($($t,)*)
2733            where $($t: ComponentType),*
2734        {
2735            type Lower = TupleLower<$($t::Lower),*>;
2736
2737            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::record_static(&[
2738                $($t::ABI),*
2739            ]);
2740
2741            const IS_RUST_UNIT_TYPE: bool = {
2742                let mut _is_unit = true;
2743                $(
2744                    let _anything_to_bind_the_macro_variable = $t::IS_RUST_UNIT_TYPE;
2745                    _is_unit = false;
2746                )*
2747                _is_unit
2748            };
2749
2750            fn typecheck(
2751                ty: &InterfaceType,
2752                types: &InstanceType<'_>,
2753            ) -> Result<()> {
2754                typecheck_tuple(ty, types, &[$($t::typecheck),*])
2755            }
2756        }
2757
2758        #[allow(non_snake_case, reason = "macro-generated code")]
2759        unsafe impl<$($t,)*> Lower for ($($t,)*)
2760            where $($t: Lower),*
2761        {
2762            fn linear_lower_to_flat<U>(
2763                &self,
2764                cx: &mut LowerContext<'_, U>,
2765                ty: InterfaceType,
2766                _dst: &mut MaybeUninit<Self::Lower>,
2767            ) -> Result<()> {
2768                let types = match ty {
2769                    InterfaceType::Tuple(t) => &cx.types[t].types,
2770                    _ => bad_type_info(),
2771                };
2772                let ($($t,)*) = self;
2773                let mut _types = types.iter();
2774                $(
2775                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2776                    $t.linear_lower_to_flat(cx, ty, map_maybe_uninit!(_dst.$t))?;
2777                )*
2778                Ok(())
2779            }
2780
2781            fn linear_lower_to_memory<U>(
2782                &self,
2783                cx: &mut LowerContext<'_, U>,
2784                ty: InterfaceType,
2785                mut _offset: usize,
2786            ) -> Result<()> {
2787                debug_assert!(_offset % (Self::ALIGN32 as usize) == 0);
2788                let types = match ty {
2789                    InterfaceType::Tuple(t) => &cx.types[t].types,
2790                    _ => bad_type_info(),
2791                };
2792                let ($($t,)*) = self;
2793                let mut _types = types.iter();
2794                $(
2795                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2796                    $t.linear_lower_to_memory(cx, ty, $t::ABI.next_field32_size(&mut _offset))?;
2797                )*
2798                Ok(())
2799            }
2800        }
2801
2802        #[allow(non_snake_case, reason = "macro-generated code")]
2803        unsafe impl<$($t,)*> Lift for ($($t,)*)
2804            where $($t: Lift),*
2805        {
2806            #[inline]
2807            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, _src: &Self::Lower) -> Result<Self> {
2808                let types = match ty {
2809                    InterfaceType::Tuple(t) => &cx.types[t].types,
2810                    _ => bad_type_info(),
2811                };
2812                let mut _types = types.iter();
2813                Ok(($(
2814                    $t::linear_lift_from_flat(
2815                        cx,
2816                        *_types.next().unwrap_or_else(bad_type_info),
2817                        &_src.$t,
2818                    )?,
2819                )*))
2820            }
2821
2822            #[inline]
2823            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
2824                debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2825                let types = match ty {
2826                    InterfaceType::Tuple(t) => &cx.types[t].types,
2827                    _ => bad_type_info(),
2828                };
2829                let mut _types = types.iter();
2830                let mut _offset = 0;
2831                $(
2832                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2833                    let $t = $t::linear_lift_from_memory(cx, ty, &bytes[$t::ABI.next_field32_size(&mut _offset)..][..$t::SIZE32])?;
2834                )*
2835                Ok(($($t,)*))
2836            }
2837        }
2838
2839        #[allow(non_snake_case, reason = "macro-generated code")]
2840        unsafe impl<$($t,)*> ComponentNamedList for ($($t,)*)
2841            where $($t: ComponentType),*
2842        {}
2843    };
2844}
2845
2846for_each_function_signature!(impl_component_ty_for_tuples);
2847
2848pub fn desc(ty: &InterfaceType) -> &'static str {
2849    match ty {
2850        InterfaceType::U8 => "u8",
2851        InterfaceType::S8 => "s8",
2852        InterfaceType::U16 => "u16",
2853        InterfaceType::S16 => "s16",
2854        InterfaceType::U32 => "u32",
2855        InterfaceType::S32 => "s32",
2856        InterfaceType::U64 => "u64",
2857        InterfaceType::S64 => "s64",
2858        InterfaceType::Float32 => "f32",
2859        InterfaceType::Float64 => "f64",
2860        InterfaceType::Bool => "bool",
2861        InterfaceType::Char => "char",
2862        InterfaceType::String => "string",
2863        InterfaceType::List(_) => "list",
2864        InterfaceType::Tuple(_) => "tuple",
2865        InterfaceType::Option(_) => "option",
2866        InterfaceType::Result(_) => "result",
2867
2868        InterfaceType::Record(_) => "record",
2869        InterfaceType::Variant(_) => "variant",
2870        InterfaceType::Flags(_) => "flags",
2871        InterfaceType::Enum(_) => "enum",
2872        InterfaceType::Own(_) => "owned resource",
2873        InterfaceType::Borrow(_) => "borrowed resource",
2874        InterfaceType::Future(_) => "future",
2875        InterfaceType::Stream(_) => "stream",
2876        InterfaceType::ErrorContext(_) => "error-context",
2877    }
2878}
2879
2880#[cold]
2881#[doc(hidden)]
2882pub fn bad_type_info<T>() -> T {
2883    // NB: should consider something like `unreachable_unchecked` here if this
2884    // becomes a performance bottleneck at some point, but that also comes with
2885    // a tradeoff of propagating a lot of unsafety, so it may not be worth it.
2886    panic!("bad type information detected");
2887}