wasmtime/runtime/component/func/
typed.rs

1use crate::component::Instance;
2use crate::component::func::{Func, LiftContext, LowerContext, Options};
3use crate::component::matching::InstanceType;
4use crate::component::storage::{storage_as_slice, storage_as_slice_mut};
5use crate::prelude::*;
6use crate::{AsContextMut, StoreContext, StoreContextMut, ValRaw};
7use alloc::borrow::Cow;
8use core::fmt;
9use core::iter;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::str;
13use wasmtime_environ::component::{
14    CanonicalAbiInfo, InterfaceType, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, StringEncoding, VariantInfo,
15};
16
17#[cfg(feature = "component-model-async")]
18use crate::component::concurrent::{self, AsAccessor, PreparedCall};
19#[cfg(feature = "component-model-async")]
20use crate::component::func::TaskExit;
21
22/// A statically-typed version of [`Func`] which takes `Params` as input and
23/// returns `Return`.
24///
25/// This is an efficient way to invoke a WebAssembly component where if the
26/// inputs and output are statically known this can eschew the vast majority of
27/// machinery and checks when calling WebAssembly. This is the most optimized
28/// way to call a WebAssembly component.
29///
30/// Note that like [`Func`] this is a pointer within a [`Store`](crate::Store)
31/// and usage will panic if used with the wrong store.
32///
33/// This type is primarily created with the [`Func::typed`] API.
34///
35/// See [`ComponentType`] for more information about supported types.
36pub struct TypedFunc<Params, Return> {
37    func: Func,
38
39    // The definition of this field is somewhat subtle and may be surprising.
40    // Naively one might expect something like
41    //
42    //      _marker: marker::PhantomData<fn(Params) -> Return>,
43    //
44    // Since this is a function pointer after all. The problem with this
45    // definition though is that it imposes the wrong variance on `Params` from
46    // what we want. Abstractly a `fn(Params)` is able to store `Params` within
47    // it meaning you can only give it `Params` that live longer than the
48    // function pointer.
49    //
50    // With a component model function, however, we're always copying data from
51    // the host into the guest, so we are never storing pointers to `Params`
52    // into the guest outside the duration of a `call`, meaning we can actually
53    // accept values in `TypedFunc::call` which live for a shorter duration
54    // than the `Params` argument on the struct.
55    //
56    // This all means that we don't use a phantom function pointer, but instead
57    // feign phantom storage here to get the variance desired.
58    _marker: marker::PhantomData<(Params, Return)>,
59}
60
61impl<Params, Return> Copy for TypedFunc<Params, Return> {}
62
63impl<Params, Return> Clone for TypedFunc<Params, Return> {
64    fn clone(&self) -> TypedFunc<Params, Return> {
65        *self
66    }
67}
68
69impl<Params, Return> TypedFunc<Params, Return>
70where
71    Params: ComponentNamedList + Lower,
72    Return: ComponentNamedList + Lift,
73{
74    /// Creates a new [`TypedFunc`] from the provided component [`Func`],
75    /// unsafely asserting that the underlying function takes `Params` as
76    /// input and returns `Return`.
77    ///
78    /// # Unsafety
79    ///
80    /// This is an unsafe function because it does not verify that the [`Func`]
81    /// provided actually implements this signature. It's up to the caller to
82    /// have performed some other sort of check to ensure that the signature is
83    /// correct.
84    pub unsafe fn new_unchecked(func: Func) -> TypedFunc<Params, Return> {
85        TypedFunc {
86            _marker: marker::PhantomData,
87            func,
88        }
89    }
90
91    /// Returns the underlying un-typed [`Func`] that this [`TypedFunc`]
92    /// references.
93    pub fn func(&self) -> &Func {
94        &self.func
95    }
96
97    /// Calls the underlying WebAssembly component function using the provided
98    /// `params` as input.
99    ///
100    /// This method is used to enter into a component. Execution happens within
101    /// the `store` provided. The `params` are copied into WebAssembly memory
102    /// as appropriate and a core wasm function is invoked.
103    ///
104    /// # Post-return
105    ///
106    /// In the component model each function can have a "post return" specified
107    /// which allows cleaning up the arguments returned to the host. For example
108    /// if WebAssembly returns a string to the host then it might be a uniquely
109    /// allocated string which, after the host finishes processing it, needs to
110    /// be deallocated in the wasm instance's own linear memory to prevent
111    /// memory leaks in wasm itself. The `post-return` canonical abi option is
112    /// used to configured this.
113    ///
114    /// To accommodate this feature of the component model after invoking a
115    /// function via [`TypedFunc::call`] you must next invoke
116    /// [`TypedFunc::post_return`]. Note that the return value of the function
117    /// should be processed between these two function calls. The return value
118    /// continues to be usable from an embedder's perspective after
119    /// `post_return` is called, but after `post_return` is invoked it may no
120    /// longer retain the same value that the wasm module originally returned.
121    ///
122    /// Also note that [`TypedFunc::post_return`] must be invoked irrespective
123    /// of whether the canonical ABI option `post-return` was configured or not.
124    /// This means that embedders must unconditionally call
125    /// [`TypedFunc::post_return`] when a function returns. If this function
126    /// call returns an error, however, then [`TypedFunc::post_return`] is not
127    /// required.
128    ///
129    /// # Errors
130    ///
131    /// This function can return an error for a number of reasons:
132    ///
133    /// * If the wasm itself traps during execution.
134    /// * If the wasm traps while copying arguments into memory.
135    /// * If the wasm provides bad allocation pointers when copying arguments
136    ///   into memory.
137    /// * If the wasm returns a value which violates the canonical ABI.
138    /// * If this function's instances cannot be entered, for example if the
139    ///   instance is currently calling a host function.
140    /// * If a previous function call occurred and the corresponding
141    ///   `post_return` hasn't been invoked yet.
142    ///
143    /// In general there are many ways that things could go wrong when copying
144    /// types in and out of a wasm module with the canonical ABI, and certain
145    /// error conditions are specific to certain types. For example a
146    /// WebAssembly module can't return an invalid `char`. When allocating space
147    /// for this host to copy a string into the returned pointer must be
148    /// in-bounds in memory.
149    ///
150    /// If an error happens then the error should contain detailed enough
151    /// information to understand which part of the canonical ABI went wrong
152    /// and what to inspect.
153    ///
154    /// # Panics
155    ///
156    /// Panics if this is called on a function in an asynchronous store. This
157    /// only works with functions defined within a synchronous store. Also
158    /// panics if `store` does not own this function.
159    pub fn call(&self, store: impl AsContextMut, params: Params) -> Result<Return> {
160        assert!(
161            !store.as_context().async_support(),
162            "must use `call_async` when async support is enabled on the config"
163        );
164        self.call_impl(store, params)
165    }
166
167    /// Exactly like [`Self::call`], except for use on asynchronous stores.
168    ///
169    /// # Panics
170    ///
171    /// Panics if this is called on a function in a synchronous store. This
172    /// only works with functions defined within an asynchronous store. Also
173    /// panics if `store` does not own this function.
174    #[cfg(feature = "async")]
175    pub async fn call_async(
176        &self,
177        mut store: impl AsContextMut<Data: Send>,
178        params: Params,
179    ) -> Result<Return>
180    where
181        Return: 'static,
182    {
183        let mut store = store.as_context_mut();
184        assert!(
185            store.0.async_support(),
186            "cannot use `call_async` when async support is not enabled on the config"
187        );
188        #[cfg(feature = "component-model-async")]
189        {
190            use crate::component::concurrent::TaskId;
191            use crate::runtime::vm::SendSyncPtr;
192            use core::ptr::NonNull;
193
194            let ptr = SendSyncPtr::from(NonNull::from(&params).cast::<u8>());
195            let prepared =
196                self.prepare_call(store.as_context_mut(), false, false, move |cx, ty, dst| {
197                    // SAFETY: The goal here is to get `Params`, a non-`'static`
198                    // value, to live long enough to the lowering of the
199                    // parameters. We're guaranteed that `Params` lives in the
200                    // future of the outer function (we're in an `async fn`) so it'll
201                    // stay alive as long as the future itself. That is distinct,
202                    // for example, from the signature of `call_concurrent` below.
203                    //
204                    // Here a pointer to `Params` is smuggled to this location
205                    // through a `SendSyncPtr<u8>` to thwart the `'static` check
206                    // of rustc and the signature of `prepare_call`.
207                    //
208                    // Note the use of `RemoveOnDrop` in the code that follows
209                    // this closure, which ensures that the task will be removed
210                    // from the concurrent state to which it belongs when the
211                    // containing `Future` is dropped, thereby ensuring that
212                    // this closure will never be called if it hasn't already,
213                    // meaning it will never see a dangling pointer.
214                    let params = unsafe { ptr.cast::<Params>().as_ref() };
215                    Self::lower_args(cx, ty, dst, params)
216                })?;
217
218            struct RemoveOnDrop<'a, T: 'static> {
219                store: StoreContextMut<'a, T>,
220                task: TaskId,
221            }
222
223            impl<'a, T> Drop for RemoveOnDrop<'a, T> {
224                fn drop(&mut self) {
225                    self.task.remove(self.store.as_context_mut()).unwrap();
226                }
227            }
228
229            let mut wrapper = RemoveOnDrop {
230                store,
231                task: prepared.task_id(),
232            };
233
234            let result = concurrent::queue_call(wrapper.store.as_context_mut(), prepared)?;
235            self.func
236                .instance
237                .run_concurrent(wrapper.store.as_context_mut(), async |_| {
238                    Ok(result.await?.0)
239                })
240                .await?
241        }
242        #[cfg(not(feature = "component-model-async"))]
243        {
244            store
245                .on_fiber(|store| self.call_impl(store, params))
246                .await?
247        }
248    }
249
250    /// Start a concurrent call to this function.
251    ///
252    /// Unlike [`Self::call`] and [`Self::call_async`] (both of which require
253    /// exclusive access to the store until the completion of the call), calls
254    /// made using this method may run concurrently with other calls to the same
255    /// instance.  In addition, the runtime will call the `post-return` function
256    /// (if any) automatically when the guest task completes -- no need to
257    /// explicitly call `Func::post_return` afterward.
258    ///
259    /// Besides the task's return value, this returns a [`TaskExit`]
260    /// representing the completion of the guest task and any transitive
261    /// subtasks it might create.
262    ///
263    /// # Panics
264    ///
265    /// Panics if the store that the [`Accessor`] is derived from does not own
266    /// this function.
267    #[cfg(feature = "component-model-async")]
268    pub async fn call_concurrent(
269        self,
270        accessor: impl AsAccessor<Data: Send>,
271        params: Params,
272    ) -> Result<(Return, TaskExit)>
273    where
274        Params: 'static,
275        Return: 'static,
276    {
277        let result = accessor.as_accessor().with(|mut store| {
278            let mut store = store.as_context_mut();
279            assert!(
280                store.0.async_support(),
281                "cannot use `call_concurrent` when async support is not enabled on the config"
282            );
283
284            let prepared =
285                self.prepare_call(store.as_context_mut(), true, true, move |cx, ty, dst| {
286                    Self::lower_args(cx, ty, dst, &params)
287                })?;
288            concurrent::queue_call(store, prepared)
289        });
290        let (result, rx) = result?.await?;
291        Ok((result, TaskExit(rx)))
292    }
293
294    fn lower_args<T>(
295        cx: &mut LowerContext<T>,
296        ty: InterfaceType,
297        dst: &mut [MaybeUninit<ValRaw>],
298        params: &Params,
299    ) -> Result<()> {
300        use crate::component::storage::slice_to_storage_mut;
301
302        if Params::flatten_count() <= MAX_FLAT_PARAMS {
303            // SAFETY: the safety of `slice_to_storage_mut` relies on
304            // `Params::Lower` being represented by a sequence of
305            // `ValRaw`, and that's a guarantee upheld by the `Lower`
306            // trait itself.
307            let dst: &mut MaybeUninit<Params::Lower> = unsafe { slice_to_storage_mut(dst) };
308            Self::lower_stack_args(cx, &params, ty, dst)
309        } else {
310            Self::lower_heap_args(cx, &params, ty, &mut dst[0])
311        }
312    }
313
314    /// Calls `concurrent::prepare_call` with monomorphized functions for
315    /// lowering the parameters and lifting the result according to the number
316    /// of core Wasm parameters and results in the signature of the function to
317    /// be called.
318    #[cfg(feature = "component-model-async")]
319    fn prepare_call<T>(
320        self,
321        store: StoreContextMut<'_, T>,
322        remove_task_automatically: bool,
323        call_post_return_automatically: bool,
324        lower: impl FnOnce(
325            &mut LowerContext<T>,
326            InterfaceType,
327            &mut [MaybeUninit<ValRaw>],
328        ) -> Result<()>
329        + Send
330        + Sync
331        + 'static,
332    ) -> Result<PreparedCall<Return>>
333    where
334        Return: 'static,
335    {
336        use crate::component::storage::slice_to_storage;
337
338        let param_count = if Params::flatten_count() <= MAX_FLAT_PARAMS {
339            Params::flatten_count()
340        } else {
341            1
342        };
343        let max_results = if self.func.abi_async(store.0) {
344            MAX_FLAT_PARAMS
345        } else {
346            MAX_FLAT_RESULTS
347        };
348        concurrent::prepare_call(
349            store,
350            self.func,
351            param_count,
352            remove_task_automatically,
353            call_post_return_automatically,
354            move |func, store, params_out| {
355                func.with_lower_context(store, call_post_return_automatically, |cx, ty| {
356                    lower(cx, ty, params_out)
357                })
358            },
359            move |func, store, results| {
360                let result = if Return::flatten_count() <= max_results {
361                    func.with_lift_context(store, |cx, ty| {
362                        // SAFETY: Per the safety requiments documented for the
363                        // `ComponentType` trait, `Return::Lower` must be
364                        // compatible at the binary level with a `[ValRaw; N]`,
365                        // where `N` is `mem::size_of::<Return::Lower>() /
366                        // mem::size_of::<ValRaw>()`.  And since this function
367                        // is only used when `Return::flatten_count() <=
368                        // MAX_FLAT_RESULTS` and `MAX_FLAT_RESULTS == 1`, `N`
369                        // can only either be 0 or 1.
370                        //
371                        // See `ComponentInstance::exit_call` for where we use
372                        // the result count passed from
373                        // `wasmtime_environ::fact::trampoline`-generated code
374                        // to ensure the slice has the correct length, and also
375                        // `concurrent::start_call` for where we conservatively
376                        // use a slice length of 1 unconditionally.  Also note
377                        // that, as of this writing `slice_to_storage`
378                        // double-checks the slice length is sufficient.
379                        let results: &Return::Lower = unsafe { slice_to_storage(results) };
380                        Self::lift_stack_result(cx, ty, results)
381                    })?
382                } else {
383                    func.with_lift_context(store, |cx, ty| {
384                        Self::lift_heap_result(cx, ty, &results[0])
385                    })?
386                };
387                Ok(Box::new(result))
388            },
389        )
390    }
391
392    fn call_impl(&self, mut store: impl AsContextMut, params: Params) -> Result<Return> {
393        let store = store.as_context_mut();
394
395        if self.func.abi_async(store.0) {
396            bail!("must enable the `component-model-async` feature to call async-lifted exports")
397        }
398
399        // Note that this is in theory simpler than it might read at this time.
400        // Here we're doing a runtime dispatch on the `flatten_count` for the
401        // params/results to see whether they're inbounds. This creates 4 cases
402        // to handle. In reality this is a highly optimizable branch where LLVM
403        // will easily figure out that only one branch here is taken.
404        //
405        // Otherwise this current construction is done to ensure that the stack
406        // space reserved for the params/results is always of the appropriate
407        // size (as the params/results needed differ depending on the "flatten"
408        // count)
409        //
410        // SAFETY: the safety of these invocations of `call_raw` depends on the
411        // correctness of the ascription of the `LowerParams` and `LowerReturn`
412        // types on the `call_raw` function. That's upheld here through the
413        // safety requirements of `Lift` and `Lower` on `Params` and `Return` in
414        // combination with checking the various possible branches here and
415        // dispatching to appropriately typed functions.
416        unsafe {
417            // This type is used as `LowerParams` for `call_raw` which is either
418            // `Params::Lower` or `ValRaw` representing it's either on the stack
419            // or it's on the heap. This allocates 1 extra `ValRaw` on the stack
420            // if `Params` is empty and `Return` is also empty, but that's a
421            // reasonable enough price to pay for now given the current code
422            // organization.
423            #[derive(Copy, Clone)]
424            union Union<T: Copy, U: Copy> {
425                _a: T,
426                _b: U,
427            }
428
429            if Return::flatten_count() <= MAX_FLAT_RESULTS {
430                self.func.call_raw(
431                    store,
432                    |cx, ty, dst: &mut MaybeUninit<Union<Params::Lower, ValRaw>>| {
433                        let dst = storage_as_slice_mut(dst);
434                        Self::lower_args(cx, ty, dst, &params)
435                    },
436                    Self::lift_stack_result,
437                )
438            } else {
439                self.func.call_raw(
440                    store,
441                    |cx, ty, dst: &mut MaybeUninit<Union<Params::Lower, ValRaw>>| {
442                        let dst = storage_as_slice_mut(dst);
443                        Self::lower_args(cx, ty, dst, &params)
444                    },
445                    Self::lift_heap_result,
446                )
447            }
448        }
449    }
450
451    /// Lower parameters directly onto the stack specified by the `dst`
452    /// location.
453    ///
454    /// This is only valid to call when the "flatten count" is small enough, or
455    /// when the canonical ABI says arguments go through the stack rather than
456    /// the heap.
457    fn lower_stack_args<T>(
458        cx: &mut LowerContext<'_, T>,
459        params: &Params,
460        ty: InterfaceType,
461        dst: &mut MaybeUninit<Params::Lower>,
462    ) -> Result<()> {
463        assert!(Params::flatten_count() <= MAX_FLAT_PARAMS);
464        params.linear_lower_to_flat(cx, ty, dst)?;
465        Ok(())
466    }
467
468    /// Lower parameters onto a heap-allocated location.
469    ///
470    /// This is used when the stack space to be used for the arguments is above
471    /// the `MAX_FLAT_PARAMS` threshold. Here the wasm's `realloc` function is
472    /// invoked to allocate space and then parameters are stored at that heap
473    /// pointer location.
474    fn lower_heap_args<T>(
475        cx: &mut LowerContext<'_, T>,
476        params: &Params,
477        ty: InterfaceType,
478        dst: &mut MaybeUninit<ValRaw>,
479    ) -> Result<()> {
480        // Memory must exist via validation if the arguments are stored on the
481        // heap, so we can create a `MemoryMut` at this point. Afterwards
482        // `realloc` is used to allocate space for all the arguments and then
483        // they're all stored in linear memory.
484        //
485        // Note that `realloc` will bake in a check that the returned pointer is
486        // in-bounds.
487        let ptr = cx.realloc(0, 0, Params::ALIGN32, Params::SIZE32)?;
488        params.linear_lower_to_memory(cx, ty, ptr)?;
489
490        // Note that the pointer here is stored as a 64-bit integer. This allows
491        // this to work with either 32 or 64-bit memories. For a 32-bit memory
492        // it'll just ignore the upper 32 zero bits, and for 64-bit memories
493        // this'll have the full 64-bits. Note that for 32-bit memories the call
494        // to `realloc` above guarantees that the `ptr` is in-bounds meaning
495        // that we will know that the zero-extended upper bits of `ptr` are
496        // guaranteed to be zero.
497        //
498        // This comment about 64-bit integers is also referred to below with
499        // "WRITEPTR64".
500        dst.write(ValRaw::i64(ptr as i64));
501
502        Ok(())
503    }
504
505    /// Lift the result of a function directly from the stack result.
506    ///
507    /// This is only used when the result fits in the maximum number of stack
508    /// slots.
509    fn lift_stack_result(
510        cx: &mut LiftContext<'_>,
511        ty: InterfaceType,
512        dst: &Return::Lower,
513    ) -> Result<Return> {
514        Return::linear_lift_from_flat(cx, ty, dst)
515    }
516
517    /// Lift the result of a function where the result is stored indirectly on
518    /// the heap.
519    fn lift_heap_result(
520        cx: &mut LiftContext<'_>,
521        ty: InterfaceType,
522        dst: &ValRaw,
523    ) -> Result<Return> {
524        assert!(Return::flatten_count() > MAX_FLAT_RESULTS);
525        // FIXME(#4311): needs to read an i64 for memory64
526        let ptr = usize::try_from(dst.get_u32())?;
527        if ptr % usize::try_from(Return::ALIGN32)? != 0 {
528            bail!("return pointer not aligned");
529        }
530
531        let bytes = cx
532            .memory()
533            .get(ptr..)
534            .and_then(|b| b.get(..Return::SIZE32))
535            .ok_or_else(|| anyhow::anyhow!("pointer out of bounds of memory"))?;
536        Return::linear_lift_from_memory(cx, ty, bytes)
537    }
538
539    /// See [`Func::post_return`]
540    pub fn post_return(&self, store: impl AsContextMut) -> Result<()> {
541        self.func.post_return(store)
542    }
543
544    /// See [`Func::post_return_async`]
545    #[cfg(feature = "async")]
546    pub async fn post_return_async<T: Send>(
547        &self,
548        store: impl AsContextMut<Data = T>,
549    ) -> Result<()> {
550        self.func.post_return_async(store).await
551    }
552}
553
554/// A trait representing a static list of named types that can be passed to or
555/// returned from a [`TypedFunc`].
556///
557/// This trait is implemented for a number of tuple types and is not expected
558/// to be implemented externally. The contents of this trait are hidden as it's
559/// intended to be an implementation detail of Wasmtime. The contents of this
560/// trait are not covered by Wasmtime's stability guarantees.
561///
562/// For more information about this trait see [`Func::typed`] and
563/// [`TypedFunc`].
564//
565// Note that this is an `unsafe` trait, and the unsafety means that
566// implementations of this trait must be correct or otherwise [`TypedFunc`]
567// would not be memory safe. The main reason this is `unsafe` is the
568// `typecheck` function which must operate correctly relative to the `AsTuple`
569// interpretation of the implementor.
570pub unsafe trait ComponentNamedList: ComponentType {}
571
572/// A trait representing types which can be passed to and read from components
573/// with the canonical ABI.
574///
575/// This trait is implemented for Rust types which can be communicated to
576/// components. The [`Func::typed`] and [`TypedFunc`] Rust items are the main
577/// consumers of this trait.
578///
579/// Supported Rust types include:
580///
581/// | Component Model Type              | Rust Type                            |
582/// |-----------------------------------|--------------------------------------|
583/// | `{s,u}{8,16,32,64}`               | `{i,u}{8,16,32,64}`                  |
584/// | `f{32,64}`                        | `f{32,64}`                           |
585/// | `bool`                            | `bool`                               |
586/// | `char`                            | `char`                               |
587/// | `tuple<A, B>`                     | `(A, B)`                             |
588/// | `option<T>`                       | `Option<T>`                          |
589/// | `result`                          | `Result<(), ()>`                     |
590/// | `result<T>`                       | `Result<T, ()>`                      |
591/// | `result<_, E>`                    | `Result<(), E>`                      |
592/// | `result<T, E>`                    | `Result<T, E>`                       |
593/// | `string`                          | `String`, `&str`, or [`WasmStr`]     |
594/// | `list<T>`                         | `Vec<T>`, `&[T]`, or [`WasmList`]    |
595/// | `own<T>`, `borrow<T>`             | [`Resource<T>`] or [`ResourceAny`]   |
596/// | `record`                          | [`#[derive(ComponentType)]`][d-cm]   |
597/// | `variant`                         | [`#[derive(ComponentType)]`][d-cm]   |
598/// | `enum`                            | [`#[derive(ComponentType)]`][d-cm]   |
599/// | `flags`                           | [`flags!`][f-m]                      |
600///
601/// [`Resource<T>`]: crate::component::Resource
602/// [`ResourceAny`]: crate::component::ResourceAny
603/// [d-cm]: macro@crate::component::ComponentType
604/// [f-m]: crate::component::flags
605///
606/// Rust standard library pointers such as `&T`, `Box<T>`, and `Arc<T>`
607/// additionally represent whatever type `T` represents in the component model.
608/// Note that types such as `record`, `variant`, `enum`, and `flags` are
609/// generated by the embedder at compile time. These macros derive
610/// implementation of this trait for custom types to map to custom types in the
611/// component model. Note that for `record`, `variant`, `enum`, and `flags`
612/// those types are often generated by the
613/// [`bindgen!`](crate::component::bindgen) macro from WIT definitions.
614///
615/// Types that implement [`ComponentType`] are used for `Params` and `Return`
616/// in [`TypedFunc`] and [`Func::typed`].
617///
618/// The contents of this trait are hidden as it's intended to be an
619/// implementation detail of Wasmtime. The contents of this trait are not
620/// covered by Wasmtime's stability guarantees.
621///
622/// # Safety
623///
624/// Note that this is an `unsafe` trait as `TypedFunc`'s safety heavily relies on
625/// the correctness of the implementations of this trait. Some ways in which this
626/// trait must be correct to be safe are:
627///
628/// * The `Lower` associated type must be a `ValRaw` sequence. It doesn't have to
629///   literally be `[ValRaw; N]` but when laid out in memory it must be adjacent
630///   `ValRaw` values and have a multiple of the size of `ValRaw` and the same
631///   alignment.
632///
633/// * The `lower` function must initialize the bits within `Lower` that are going
634///   to be read by the trampoline that's used to enter core wasm. A trampoline
635///   is passed `*mut Lower` and will read the canonical abi arguments in
636///   sequence, so all of the bits must be correctly initialized.
637///
638/// * The `size` and `align` functions must be correct for this value stored in
639///   the canonical ABI. The `Cursor<T>` iteration of these bytes rely on this
640///   for correctness as they otherwise eschew bounds-checking.
641///
642/// There are likely some other correctness issues which aren't documented as
643/// well, this isn't currently an exhaustive list. It suffices to say, though,
644/// that correctness bugs in this trait implementation are highly likely to
645/// lead to security bugs, which again leads to the `unsafe` in the trait.
646///
647/// Note that this trait specifically is not sealed because `bindgen!`-generated
648/// types must be able to implement this trait using a `#[derive]` macro. For
649/// users it's recommended to not implement this trait manually given the
650/// non-exhaustive list of safety requirements that must be upheld. This trait
651/// is implemented at your own risk if you do so.
652///
653/// # Send and Sync
654///
655/// While on the topic of safety it's worth discussing the `Send` and `Sync`
656/// bounds here as well. These bounds might naively seem like they shouldn't be
657/// required for all component types as they're host-level types not guest-level
658/// types persisted anywhere. Various subtleties lead to these bounds, however:
659///
660/// * Fibers require that all stack-local variables are `Send` and `Sync` for
661///   fibers themselves to be send/sync. Unfortunately we have no help from the
662///   compiler on this one so it's up to Wasmtime's discipline to maintain this.
663///   One instance of this is that return values are placed on the stack as
664///   they're lowered into guest memory. This lowering operation can involve
665///   malloc and context switches, so return values must be Send/Sync.
666///
667/// * In the implementation of component model async it's not uncommon for types
668///   to be "buffered" in the store temporarily. For example parameters might
669///   reside in a store temporarily while wasm has backpressure turned on.
670///
671/// Overall it's generally easiest to require `Send` and `Sync` for all
672/// component types. There additionally aren't known use case for non-`Send` or
673/// non-`Sync` types at this time.
674pub unsafe trait ComponentType: Send + Sync {
675    /// Representation of the "lowered" form of this component value.
676    ///
677    /// Lowerings lower into core wasm values which are represented by `ValRaw`.
678    /// This `Lower` type must be a list of `ValRaw` as either a literal array
679    /// or a struct where every field is a `ValRaw`. This must be `Copy` (as
680    /// `ValRaw` is `Copy`) and support all byte patterns. This being correct is
681    /// one reason why the trait is unsafe.
682    #[doc(hidden)]
683    type Lower: Copy;
684
685    /// The information about this type's canonical ABI (size/align/etc).
686    #[doc(hidden)]
687    const ABI: CanonicalAbiInfo;
688
689    #[doc(hidden)]
690    const SIZE32: usize = Self::ABI.size32 as usize;
691    #[doc(hidden)]
692    const ALIGN32: u32 = Self::ABI.align32;
693
694    #[doc(hidden)]
695    const IS_RUST_UNIT_TYPE: bool = false;
696
697    /// Whether this type might require a call to the guest's realloc function
698    /// to allocate linear memory when lowering (e.g. a non-empty `string`).
699    ///
700    /// If this is `false`, Wasmtime may optimize lowering by using
701    /// `LowerContext::new_without_realloc` and lowering values outside of any
702    /// fiber.  That will panic if the lowering process ends up needing realloc
703    /// after all, so `true` is a conservative default.
704    #[doc(hidden)]
705    const MAY_REQUIRE_REALLOC: bool = true;
706
707    /// Returns the number of core wasm abi values will be used to represent
708    /// this type in its lowered form.
709    ///
710    /// This divides the size of `Self::Lower` by the size of `ValRaw`.
711    #[doc(hidden)]
712    fn flatten_count() -> usize {
713        assert!(mem::size_of::<Self::Lower>() % mem::size_of::<ValRaw>() == 0);
714        assert!(mem::align_of::<Self::Lower>() == mem::align_of::<ValRaw>());
715        mem::size_of::<Self::Lower>() / mem::size_of::<ValRaw>()
716    }
717
718    /// Performs a type-check to see whether this component value type matches
719    /// the interface type `ty` provided.
720    #[doc(hidden)]
721    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()>;
722}
723
724#[doc(hidden)]
725pub unsafe trait ComponentVariant: ComponentType {
726    const CASES: &'static [Option<CanonicalAbiInfo>];
727    const INFO: VariantInfo = VariantInfo::new_static(Self::CASES);
728    const PAYLOAD_OFFSET32: usize = Self::INFO.payload_offset32 as usize;
729}
730
731/// Host types which can be passed to WebAssembly components.
732///
733/// This trait is implemented for all types that can be passed to components
734/// either as parameters of component exports or returns of component imports.
735/// This trait represents the ability to convert from the native host
736/// representation to the canonical ABI.
737///
738/// Built-in types to Rust such as `Option<T>` implement this trait as
739/// appropriate. For a mapping of component model to Rust types see
740/// [`ComponentType`].
741///
742/// For user-defined types, for example `record` types mapped to Rust `struct`s,
743/// this crate additionally has
744/// [`#[derive(Lower)]`](macro@crate::component::Lower).
745///
746/// Note that like [`ComponentType`] the definition of this trait is intended to
747/// be an internal implementation detail of Wasmtime at this time. It's
748/// recommended to use the `#[derive(Lower)]` implementation instead.
749pub unsafe trait Lower: ComponentType {
750    /// Performs the "lower" function in the linear memory version of the
751    /// canonical ABI.
752    ///
753    /// This method will lower the current value into a component. The `lower`
754    /// function performs a "flat" lowering into the `dst` specified which is
755    /// allowed to be uninitialized entering this method but is guaranteed to be
756    /// fully initialized if the method returns `Ok(())`.
757    ///
758    /// The `cx` context provided is the context within which this lowering is
759    /// happening. This contains information such as canonical options specified
760    /// (e.g. string encodings, memories, etc), the store itself, along with
761    /// type information.
762    ///
763    /// The `ty` parameter is the destination type that is being lowered into.
764    /// For example this is the component's "view" of the type that is being
765    /// lowered. This is guaranteed to have passed a `typecheck` earlier.
766    ///
767    /// This will only be called if `typecheck` passes for `Op::Lower`.
768    #[doc(hidden)]
769    fn linear_lower_to_flat<T>(
770        &self,
771        cx: &mut LowerContext<'_, T>,
772        ty: InterfaceType,
773        dst: &mut MaybeUninit<Self::Lower>,
774    ) -> Result<()>;
775
776    /// Performs the "store" operation in the linear memory version of the
777    /// canonical ABI.
778    ///
779    /// This function will store `self` into the linear memory described by
780    /// `cx` at the `offset` provided.
781    ///
782    /// It is expected that `offset` is a valid offset in memory for
783    /// `Self::SIZE32` bytes. At this time that's not an unsafe contract as it's
784    /// always re-checked on all stores, but this is something that will need to
785    /// be improved in the future to remove extra bounds checks. For now this
786    /// function will panic if there's a bug and `offset` isn't valid within
787    /// memory.
788    ///
789    /// The `ty` type information passed here is the same as the type
790    /// information passed to `lower` above, and is the component's own view of
791    /// what the resulting type should be.
792    ///
793    /// This will only be called if `typecheck` passes for `Op::Lower`.
794    #[doc(hidden)]
795    fn linear_lower_to_memory<T>(
796        &self,
797        cx: &mut LowerContext<'_, T>,
798        ty: InterfaceType,
799        offset: usize,
800    ) -> Result<()>;
801
802    /// Provided method to lower a list of `Self` into memory.
803    ///
804    /// Requires that `offset` has already been checked for alignment and
805    /// validity in terms of being in-bounds, otherwise this may panic.
806    ///
807    /// This is primarily here to get overridden for implementations of integers
808    /// which can avoid some extra fluff and use a pattern that's more easily
809    /// optimizable by LLVM.
810    #[doc(hidden)]
811    fn linear_store_list_to_memory<T>(
812        cx: &mut LowerContext<'_, T>,
813        ty: InterfaceType,
814        mut offset: usize,
815        items: &[Self],
816    ) -> Result<()>
817    where
818        Self: Sized,
819    {
820        for item in items {
821            item.linear_lower_to_memory(cx, ty, offset)?;
822            offset += Self::SIZE32;
823        }
824        Ok(())
825    }
826}
827
828/// Host types which can be created from the canonical ABI.
829///
830/// This is the mirror of the [`Lower`] trait where it represents the capability
831/// of acquiring items from WebAssembly and passing them to the host.
832///
833/// Built-in types to Rust such as `Option<T>` implement this trait as
834/// appropriate. For a mapping of component model to Rust types see
835/// [`ComponentType`].
836///
837/// For user-defined types, for example `record` types mapped to Rust `struct`s,
838/// this crate additionally has
839/// [`#[derive(Lift)]`](macro@crate::component::Lift).
840///
841/// Note that like [`ComponentType`] the definition of this trait is intended to
842/// be an internal implementation detail of Wasmtime at this time. It's
843/// recommended to use the `#[derive(Lift)]` implementation instead.
844pub unsafe trait Lift: Sized + ComponentType {
845    /// Performs the "lift" operation in the linear memory version of the
846    /// canonical ABI.
847    ///
848    /// This function performs a "flat" lift operation from the `src` specified
849    /// which is a sequence of core wasm values. The lifting operation will
850    /// validate core wasm values and produce a `Self` on success.
851    ///
852    /// The `cx` provided contains contextual information such as the store
853    /// that's being loaded from, canonical options, and type information.
854    ///
855    /// The `ty` parameter is the origin component's specification for what the
856    /// type that is being lifted is. For example this is the record type or the
857    /// resource type that is being lifted.
858    ///
859    /// Note that this has a default implementation but if `typecheck` passes
860    /// for `Op::Lift` this needs to be overridden.
861    #[doc(hidden)]
862    fn linear_lift_from_flat(
863        cx: &mut LiftContext<'_>,
864        ty: InterfaceType,
865        src: &Self::Lower,
866    ) -> Result<Self>;
867
868    /// Performs the "load" operation in the linear memory version of the
869    /// canonical ABI.
870    ///
871    /// This will read the `bytes` provided, which are a sub-slice into the
872    /// linear memory described by `cx`. The `bytes` array provided is
873    /// guaranteed to be `Self::SIZE32` bytes large. All of memory is then also
874    /// available through `cx` for bounds-checks and such as necessary for
875    /// strings/lists.
876    ///
877    /// The `ty` argument is the type that's being loaded, as described by the
878    /// original component.
879    ///
880    /// Note that this has a default implementation but if `typecheck` passes
881    /// for `Op::Lift` this needs to be overridden.
882    #[doc(hidden)]
883    fn linear_lift_from_memory(
884        cx: &mut LiftContext<'_>,
885        ty: InterfaceType,
886        bytes: &[u8],
887    ) -> Result<Self>;
888
889    /// Converts `list` into a `Vec<T>`, used in `Lift for Vec<T>`.
890    #[doc(hidden)]
891    fn linear_lift_list_from_memory(
892        cx: &mut LiftContext<'_>,
893        list: &WasmList<Self>,
894    ) -> Result<Vec<Self>>
895    where
896        Self: Sized,
897    {
898        let mut dst = Vec::with_capacity(list.len);
899        Self::linear_lift_into_from_memory(cx, list, &mut dst)?;
900        Ok(dst)
901    }
902
903    /// Load no more than `max_count` items from `list` into `dst`.
904    ///
905    /// This is primarily here to get overridden for implementations of integers
906    /// which can avoid some extra fluff and use a pattern that's more easily
907    /// optimizable by LLVM.
908    #[doc(hidden)]
909    fn linear_lift_into_from_memory(
910        cx: &mut LiftContext<'_>,
911        list: &WasmList<Self>,
912        dst: &mut impl Extend<Self>,
913    ) -> Result<()>
914    where
915        Self: Sized,
916    {
917        for i in 0..list.len {
918            dst.extend(Some(list.get_from_store(cx, i).unwrap()?));
919        }
920        Ok(())
921    }
922}
923
924// Macro to help generate "forwarding implementations" of `ComponentType` to
925// another type, used for wrappers in Rust like `&T`, `Box<T>`, etc. Note that
926// these wrappers only implement lowering because lifting native Rust types
927// cannot be done.
928macro_rules! forward_type_impls {
929    ($(($($generics:tt)*) $a:ty => $b:ty,)*) => ($(
930        unsafe impl <$($generics)*> ComponentType for $a {
931            type Lower = <$b as ComponentType>::Lower;
932
933            const ABI: CanonicalAbiInfo = <$b as ComponentType>::ABI;
934
935            #[inline]
936            fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
937                <$b as ComponentType>::typecheck(ty, types)
938            }
939        }
940    )*)
941}
942
943forward_type_impls! {
944    (T: ComponentType + ?Sized) &'_ T => T,
945    (T: ComponentType + ?Sized) Box<T> => T,
946    (T: ComponentType + ?Sized) alloc::sync::Arc<T> => T,
947    () String => str,
948    (T: ComponentType) Vec<T> => [T],
949}
950
951macro_rules! forward_lowers {
952    ($(($($generics:tt)*) $a:ty => $b:ty,)*) => ($(
953        unsafe impl <$($generics)*> Lower for $a {
954            fn linear_lower_to_flat<U>(
955                &self,
956                cx: &mut LowerContext<'_, U>,
957                ty: InterfaceType,
958                dst: &mut MaybeUninit<Self::Lower>,
959            ) -> Result<()> {
960                <$b as Lower>::linear_lower_to_flat(self, cx, ty, dst)
961            }
962
963            fn linear_lower_to_memory<U>(
964                &self,
965                cx: &mut LowerContext<'_, U>,
966                ty: InterfaceType,
967                offset: usize,
968            ) -> Result<()> {
969                <$b as Lower>::linear_lower_to_memory(self, cx, ty, offset)
970            }
971        }
972    )*)
973}
974
975forward_lowers! {
976    (T: Lower + ?Sized) &'_ T => T,
977    (T: Lower + ?Sized) Box<T> => T,
978    (T: Lower + ?Sized) alloc::sync::Arc<T> => T,
979    () String => str,
980    (T: Lower) Vec<T> => [T],
981}
982
983macro_rules! forward_string_lifts {
984    ($($a:ty,)*) => ($(
985        unsafe impl Lift for $a {
986            #[inline]
987            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
988                Ok(<WasmStr as Lift>::linear_lift_from_flat(cx, ty, src)?.to_str_from_memory(cx.memory())?.into())
989            }
990
991            #[inline]
992            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
993                Ok(<WasmStr as Lift>::linear_lift_from_memory(cx, ty, bytes)?.to_str_from_memory(cx.memory())?.into())
994            }
995        }
996    )*)
997}
998
999forward_string_lifts! {
1000    Box<str>,
1001    alloc::sync::Arc<str>,
1002    String,
1003}
1004
1005macro_rules! forward_list_lifts {
1006    ($($a:ty,)*) => ($(
1007        unsafe impl <T: Lift> Lift for $a {
1008            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1009                let list = <WasmList::<T> as Lift>::linear_lift_from_flat(cx, ty, src)?;
1010                Ok(T::linear_lift_list_from_memory(cx, &list)?.into())
1011            }
1012
1013            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1014                let list = <WasmList::<T> as Lift>::linear_lift_from_memory(cx, ty, bytes)?;
1015                Ok(T::linear_lift_list_from_memory(cx, &list)?.into())
1016            }
1017        }
1018    )*)
1019}
1020
1021forward_list_lifts! {
1022    Box<[T]>,
1023    alloc::sync::Arc<[T]>,
1024    Vec<T>,
1025}
1026
1027// Macro to help generate `ComponentType` implementations for primitive types
1028// such as integers, char, bool, etc.
1029macro_rules! integers {
1030    ($($primitive:ident = $ty:ident in $field:ident/$get:ident with abi:$abi:ident,)*) => ($(
1031        unsafe impl ComponentType for $primitive {
1032            type Lower = ValRaw;
1033
1034            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::$abi;
1035
1036            const MAY_REQUIRE_REALLOC: bool = false;
1037
1038            fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1039                match ty {
1040                    InterfaceType::$ty => Ok(()),
1041                    other => bail!("expected `{}` found `{}`", desc(&InterfaceType::$ty), desc(other))
1042                }
1043            }
1044        }
1045
1046        unsafe impl Lower for $primitive {
1047            #[inline]
1048            #[allow(trivial_numeric_casts, reason = "macro-generated code")]
1049            fn linear_lower_to_flat<T>(
1050                &self,
1051                _cx: &mut LowerContext<'_, T>,
1052                ty: InterfaceType,
1053                dst: &mut MaybeUninit<Self::Lower>,
1054            ) -> Result<()> {
1055                debug_assert!(matches!(ty, InterfaceType::$ty));
1056                dst.write(ValRaw::$field(*self as $field));
1057                Ok(())
1058            }
1059
1060            #[inline]
1061            fn linear_lower_to_memory<T>(
1062                &self,
1063                cx: &mut LowerContext<'_, T>,
1064                ty: InterfaceType,
1065                offset: usize,
1066            ) -> Result<()> {
1067                debug_assert!(matches!(ty, InterfaceType::$ty));
1068                debug_assert!(offset % Self::SIZE32 == 0);
1069                *cx.get(offset) = self.to_le_bytes();
1070                Ok(())
1071            }
1072
1073            fn linear_store_list_to_memory<T>(
1074                cx: &mut LowerContext<'_, T>,
1075                ty: InterfaceType,
1076                offset: usize,
1077                items: &[Self],
1078            ) -> Result<()> {
1079                debug_assert!(matches!(ty, InterfaceType::$ty));
1080
1081                // Double-check that the CM alignment is at least the host's
1082                // alignment for this type which should be true for all
1083                // platforms.
1084                assert!((Self::ALIGN32 as usize) >= mem::align_of::<Self>());
1085
1086                // Slice `cx`'s memory to the window that we'll be modifying.
1087                // This should all have already been verified in terms of
1088                // alignment and sizing meaning that these assertions here are
1089                // not truly necessary but are instead double-checks.
1090                //
1091                // Note that we're casting a `[u8]` slice to `[Self]` with
1092                // `align_to_mut` which is not safe in general but is safe in
1093                // our specific case as all `u8` patterns are valid `Self`
1094                // patterns since `Self` is an integral type.
1095                let dst = &mut cx.as_slice_mut()[offset..][..items.len() * Self::SIZE32];
1096                let (before, middle, end) = unsafe { dst.align_to_mut::<Self>() };
1097                assert!(before.is_empty() && end.is_empty());
1098                assert_eq!(middle.len(), items.len());
1099
1100                // And with all that out of the way perform the copying loop.
1101                // This is not a `copy_from_slice` because endianness needs to
1102                // be handled here, but LLVM should pretty easily transform this
1103                // into a memcpy on little-endian platforms.
1104                for (dst, src) in middle.iter_mut().zip(items) {
1105                    *dst = src.to_le();
1106                }
1107                Ok(())
1108            }
1109        }
1110
1111        unsafe impl Lift for $primitive {
1112            #[inline]
1113            #[allow(
1114                trivial_numeric_casts,
1115                clippy::cast_possible_truncation,
1116                reason = "macro-generated code"
1117            )]
1118            fn linear_lift_from_flat(_cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1119                debug_assert!(matches!(ty, InterfaceType::$ty));
1120                Ok(src.$get() as $primitive)
1121            }
1122
1123            #[inline]
1124            fn linear_lift_from_memory(_cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1125                debug_assert!(matches!(ty, InterfaceType::$ty));
1126                debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1127                Ok($primitive::from_le_bytes(bytes.try_into().unwrap()))
1128            }
1129
1130            fn linear_lift_into_from_memory(
1131                cx: &mut LiftContext<'_>,
1132                list: &WasmList<Self>,
1133                dst: &mut impl Extend<Self>,
1134            ) -> Result<()>
1135            where
1136                Self: Sized,
1137            {
1138                dst.extend(list._as_le_slice(cx.memory())
1139                           .iter()
1140                           .map(|i| Self::from_le(*i)));
1141                Ok(())
1142            }
1143        }
1144    )*)
1145}
1146
1147integers! {
1148    i8 = S8 in i32/get_i32 with abi:SCALAR1,
1149    u8 = U8 in u32/get_u32 with abi:SCALAR1,
1150    i16 = S16 in i32/get_i32 with abi:SCALAR2,
1151    u16 = U16 in u32/get_u32 with abi:SCALAR2,
1152    i32 = S32 in i32/get_i32 with abi:SCALAR4,
1153    u32 = U32 in u32/get_u32 with abi:SCALAR4,
1154    i64 = S64 in i64/get_i64 with abi:SCALAR8,
1155    u64 = U64 in u64/get_u64 with abi:SCALAR8,
1156}
1157
1158macro_rules! floats {
1159    ($($float:ident/$get_float:ident = $ty:ident with abi:$abi:ident)*) => ($(const _: () = {
1160        unsafe impl ComponentType for $float {
1161            type Lower = ValRaw;
1162
1163            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::$abi;
1164
1165            fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1166                match ty {
1167                    InterfaceType::$ty => Ok(()),
1168                    other => bail!("expected `{}` found `{}`", desc(&InterfaceType::$ty), desc(other))
1169                }
1170            }
1171        }
1172
1173        unsafe impl Lower for $float {
1174            #[inline]
1175            fn linear_lower_to_flat<T>(
1176                &self,
1177                _cx: &mut LowerContext<'_, T>,
1178                ty: InterfaceType,
1179                dst: &mut MaybeUninit<Self::Lower>,
1180            ) -> Result<()> {
1181                debug_assert!(matches!(ty, InterfaceType::$ty));
1182                dst.write(ValRaw::$float(self.to_bits()));
1183                Ok(())
1184            }
1185
1186            #[inline]
1187            fn linear_lower_to_memory<T>(
1188                &self,
1189                cx: &mut LowerContext<'_, T>,
1190                ty: InterfaceType,
1191                offset: usize,
1192            ) -> Result<()> {
1193                debug_assert!(matches!(ty, InterfaceType::$ty));
1194                debug_assert!(offset % Self::SIZE32 == 0);
1195                let ptr = cx.get(offset);
1196                *ptr = self.to_bits().to_le_bytes();
1197                Ok(())
1198            }
1199
1200            fn linear_store_list_to_memory<T>(
1201                cx: &mut LowerContext<'_, T>,
1202                ty: InterfaceType,
1203                offset: usize,
1204                items: &[Self],
1205            ) -> Result<()> {
1206                debug_assert!(matches!(ty, InterfaceType::$ty));
1207
1208                // Double-check that the CM alignment is at least the host's
1209                // alignment for this type which should be true for all
1210                // platforms.
1211                assert!((Self::ALIGN32 as usize) >= mem::align_of::<Self>());
1212
1213                // Slice `cx`'s memory to the window that we'll be modifying.
1214                // This should all have already been verified in terms of
1215                // alignment and sizing meaning that these assertions here are
1216                // not truly necessary but are instead double-checks.
1217                let dst = &mut cx.as_slice_mut()[offset..][..items.len() * Self::SIZE32];
1218                assert!(dst.as_ptr().cast::<Self>().is_aligned());
1219
1220                // And with all that out of the way perform the copying loop.
1221                // This is not a `copy_from_slice` because endianness needs to
1222                // be handled here, but LLVM should pretty easily transform this
1223                // into a memcpy on little-endian platforms.
1224                // TODO use `as_chunks` when https://github.com/rust-lang/rust/issues/74985
1225                // is stabilized
1226                for (dst, src) in iter::zip(dst.chunks_exact_mut(Self::SIZE32), items) {
1227                    let dst: &mut [u8; Self::SIZE32] = dst.try_into().unwrap();
1228                    *dst = src.to_le_bytes();
1229                }
1230                Ok(())
1231            }
1232        }
1233
1234        unsafe impl Lift for $float {
1235            #[inline]
1236            fn linear_lift_from_flat(_cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1237                debug_assert!(matches!(ty, InterfaceType::$ty));
1238                Ok($float::from_bits(src.$get_float()))
1239            }
1240
1241            #[inline]
1242            fn linear_lift_from_memory(_cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1243                debug_assert!(matches!(ty, InterfaceType::$ty));
1244                debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1245                Ok($float::from_le_bytes(bytes.try_into().unwrap()))
1246            }
1247
1248            fn linear_lift_list_from_memory(cx: &mut LiftContext<'_>, list: &WasmList<Self>) -> Result<Vec<Self>> where Self: Sized {
1249                // See comments in `WasmList::get` for the panicking indexing
1250                let byte_size = list.len * mem::size_of::<Self>();
1251                let bytes = &cx.memory()[list.ptr..][..byte_size];
1252
1253                // The canonical ABI requires that everything is aligned to its
1254                // own size, so this should be an aligned array.
1255                assert!(bytes.as_ptr().cast::<Self>().is_aligned());
1256
1257                // Copy the resulting slice to a new Vec, handling endianness
1258                // in the process
1259                // TODO use `as_chunks` when https://github.com/rust-lang/rust/issues/74985
1260                // is stabilized
1261                Ok(
1262                    bytes
1263                        .chunks_exact(Self::SIZE32)
1264                        .map(|i| $float::from_le_bytes(i.try_into().unwrap()))
1265                        .collect()
1266                )
1267            }
1268        }
1269    };)*)
1270}
1271
1272floats! {
1273    f32/get_f32 = Float32 with abi:SCALAR4
1274    f64/get_f64 = Float64 with abi:SCALAR8
1275}
1276
1277unsafe impl ComponentType for bool {
1278    type Lower = ValRaw;
1279
1280    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR1;
1281
1282    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1283        match ty {
1284            InterfaceType::Bool => Ok(()),
1285            other => bail!("expected `bool` found `{}`", desc(other)),
1286        }
1287    }
1288}
1289
1290unsafe impl Lower for bool {
1291    fn linear_lower_to_flat<T>(
1292        &self,
1293        _cx: &mut LowerContext<'_, T>,
1294        ty: InterfaceType,
1295        dst: &mut MaybeUninit<Self::Lower>,
1296    ) -> Result<()> {
1297        debug_assert!(matches!(ty, InterfaceType::Bool));
1298        dst.write(ValRaw::i32(*self as i32));
1299        Ok(())
1300    }
1301
1302    fn linear_lower_to_memory<T>(
1303        &self,
1304        cx: &mut LowerContext<'_, T>,
1305        ty: InterfaceType,
1306        offset: usize,
1307    ) -> Result<()> {
1308        debug_assert!(matches!(ty, InterfaceType::Bool));
1309        debug_assert!(offset % Self::SIZE32 == 0);
1310        cx.get::<1>(offset)[0] = *self as u8;
1311        Ok(())
1312    }
1313}
1314
1315unsafe impl Lift for bool {
1316    #[inline]
1317    fn linear_lift_from_flat(
1318        _cx: &mut LiftContext<'_>,
1319        ty: InterfaceType,
1320        src: &Self::Lower,
1321    ) -> Result<Self> {
1322        debug_assert!(matches!(ty, InterfaceType::Bool));
1323        match src.get_i32() {
1324            0 => Ok(false),
1325            _ => Ok(true),
1326        }
1327    }
1328
1329    #[inline]
1330    fn linear_lift_from_memory(
1331        _cx: &mut LiftContext<'_>,
1332        ty: InterfaceType,
1333        bytes: &[u8],
1334    ) -> Result<Self> {
1335        debug_assert!(matches!(ty, InterfaceType::Bool));
1336        match bytes[0] {
1337            0 => Ok(false),
1338            _ => Ok(true),
1339        }
1340    }
1341}
1342
1343unsafe impl ComponentType for char {
1344    type Lower = ValRaw;
1345
1346    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
1347
1348    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1349        match ty {
1350            InterfaceType::Char => Ok(()),
1351            other => bail!("expected `char` found `{}`", desc(other)),
1352        }
1353    }
1354}
1355
1356unsafe impl Lower for char {
1357    #[inline]
1358    fn linear_lower_to_flat<T>(
1359        &self,
1360        _cx: &mut LowerContext<'_, T>,
1361        ty: InterfaceType,
1362        dst: &mut MaybeUninit<Self::Lower>,
1363    ) -> Result<()> {
1364        debug_assert!(matches!(ty, InterfaceType::Char));
1365        dst.write(ValRaw::u32(u32::from(*self)));
1366        Ok(())
1367    }
1368
1369    #[inline]
1370    fn linear_lower_to_memory<T>(
1371        &self,
1372        cx: &mut LowerContext<'_, T>,
1373        ty: InterfaceType,
1374        offset: usize,
1375    ) -> Result<()> {
1376        debug_assert!(matches!(ty, InterfaceType::Char));
1377        debug_assert!(offset % Self::SIZE32 == 0);
1378        *cx.get::<4>(offset) = u32::from(*self).to_le_bytes();
1379        Ok(())
1380    }
1381}
1382
1383unsafe impl Lift for char {
1384    #[inline]
1385    fn linear_lift_from_flat(
1386        _cx: &mut LiftContext<'_>,
1387        ty: InterfaceType,
1388        src: &Self::Lower,
1389    ) -> Result<Self> {
1390        debug_assert!(matches!(ty, InterfaceType::Char));
1391        Ok(char::try_from(src.get_u32())?)
1392    }
1393
1394    #[inline]
1395    fn linear_lift_from_memory(
1396        _cx: &mut LiftContext<'_>,
1397        ty: InterfaceType,
1398        bytes: &[u8],
1399    ) -> Result<Self> {
1400        debug_assert!(matches!(ty, InterfaceType::Char));
1401        debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1402        let bits = u32::from_le_bytes(bytes.try_into().unwrap());
1403        Ok(char::try_from(bits)?)
1404    }
1405}
1406
1407// FIXME(#4311): these probably need different constants for memory64
1408const UTF16_TAG: usize = 1 << 31;
1409const MAX_STRING_BYTE_LENGTH: usize = (1 << 31) - 1;
1410
1411// Note that this is similar to `ComponentType for WasmStr` except it can only
1412// be used for lowering, not lifting.
1413unsafe impl ComponentType for str {
1414    type Lower = [ValRaw; 2];
1415
1416    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1417
1418    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1419        match ty {
1420            InterfaceType::String => Ok(()),
1421            other => bail!("expected `string` found `{}`", desc(other)),
1422        }
1423    }
1424}
1425
1426unsafe impl Lower for str {
1427    fn linear_lower_to_flat<T>(
1428        &self,
1429        cx: &mut LowerContext<'_, T>,
1430        ty: InterfaceType,
1431        dst: &mut MaybeUninit<[ValRaw; 2]>,
1432    ) -> Result<()> {
1433        debug_assert!(matches!(ty, InterfaceType::String));
1434        let (ptr, len) = lower_string(cx, self)?;
1435        // See "WRITEPTR64" above for why this is always storing a 64-bit
1436        // integer.
1437        map_maybe_uninit!(dst[0]).write(ValRaw::i64(ptr as i64));
1438        map_maybe_uninit!(dst[1]).write(ValRaw::i64(len as i64));
1439        Ok(())
1440    }
1441
1442    fn linear_lower_to_memory<T>(
1443        &self,
1444        cx: &mut LowerContext<'_, T>,
1445        ty: InterfaceType,
1446        offset: usize,
1447    ) -> Result<()> {
1448        debug_assert!(matches!(ty, InterfaceType::String));
1449        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
1450        let (ptr, len) = lower_string(cx, self)?;
1451        // FIXME(#4311): needs memory64 handling
1452        *cx.get(offset + 0) = u32::try_from(ptr).unwrap().to_le_bytes();
1453        *cx.get(offset + 4) = u32::try_from(len).unwrap().to_le_bytes();
1454        Ok(())
1455    }
1456}
1457
1458fn lower_string<T>(cx: &mut LowerContext<'_, T>, string: &str) -> Result<(usize, usize)> {
1459    // Note that in general the wasm module can't assume anything about what the
1460    // host strings are encoded as. Additionally hosts are allowed to have
1461    // differently-encoded strings at runtime. Finally when copying a string
1462    // into wasm it's somewhat strict in the sense that the various patterns of
1463    // allocation and such are already dictated for us.
1464    //
1465    // In general what this means is that when copying a string from the host
1466    // into the destination we need to follow one of the cases of copying into
1467    // WebAssembly. It doesn't particularly matter which case as long as it ends
1468    // up in the right encoding. For example a destination encoding of
1469    // latin1+utf16 has a number of ways to get copied into and we do something
1470    // here that isn't the default "utf8 to latin1+utf16" since we have access
1471    // to simd-accelerated helpers in the `encoding_rs` crate. This is ok though
1472    // because we can fake that the host string was already stored in latin1
1473    // format and follow that copy pattern instead.
1474    match cx.options.string_encoding() {
1475        // This corresponds to `store_string_copy` in the canonical ABI where
1476        // the host's representation is utf-8 and the wasm module wants utf-8 so
1477        // a copy is all that's needed (and the `realloc` can be precise for the
1478        // initial memory allocation).
1479        StringEncoding::Utf8 => {
1480            if string.len() > MAX_STRING_BYTE_LENGTH {
1481                bail!(
1482                    "string length of {} too large to copy into wasm",
1483                    string.len()
1484                );
1485            }
1486            let ptr = cx.realloc(0, 0, 1, string.len())?;
1487            cx.as_slice_mut()[ptr..][..string.len()].copy_from_slice(string.as_bytes());
1488            Ok((ptr, string.len()))
1489        }
1490
1491        // This corresponds to `store_utf8_to_utf16` in the canonical ABI. Here
1492        // an over-large allocation is performed and then shrunk afterwards if
1493        // necessary.
1494        StringEncoding::Utf16 => {
1495            let size = string.len() * 2;
1496            if size > MAX_STRING_BYTE_LENGTH {
1497                bail!(
1498                    "string length of {} too large to copy into wasm",
1499                    string.len()
1500                );
1501            }
1502            let mut ptr = cx.realloc(0, 0, 2, size)?;
1503            let mut copied = 0;
1504            let bytes = &mut cx.as_slice_mut()[ptr..][..size];
1505            for (u, bytes) in string.encode_utf16().zip(bytes.chunks_mut(2)) {
1506                let u_bytes = u.to_le_bytes();
1507                bytes[0] = u_bytes[0];
1508                bytes[1] = u_bytes[1];
1509                copied += 1;
1510            }
1511            if (copied * 2) < size {
1512                ptr = cx.realloc(ptr, size, 2, copied * 2)?;
1513            }
1514            Ok((ptr, copied))
1515        }
1516
1517        StringEncoding::CompactUtf16 => {
1518            // This corresponds to `store_string_to_latin1_or_utf16`
1519            let bytes = string.as_bytes();
1520            let mut iter = string.char_indices();
1521            let mut ptr = cx.realloc(0, 0, 2, bytes.len())?;
1522            let mut dst = &mut cx.as_slice_mut()[ptr..][..bytes.len()];
1523            let mut result = 0;
1524            while let Some((i, ch)) = iter.next() {
1525                // Test if this `char` fits into the latin1 encoding.
1526                if let Ok(byte) = u8::try_from(u32::from(ch)) {
1527                    dst[result] = byte;
1528                    result += 1;
1529                    continue;
1530                }
1531
1532                // .. if utf16 is forced to be used then the allocation is
1533                // bumped up to the maximum size.
1534                let worst_case = bytes
1535                    .len()
1536                    .checked_mul(2)
1537                    .ok_or_else(|| anyhow!("byte length overflow"))?;
1538                if worst_case > MAX_STRING_BYTE_LENGTH {
1539                    bail!("byte length too large");
1540                }
1541                ptr = cx.realloc(ptr, bytes.len(), 2, worst_case)?;
1542                dst = &mut cx.as_slice_mut()[ptr..][..worst_case];
1543
1544                // Previously encoded latin1 bytes are inflated to their 16-bit
1545                // size for utf16
1546                for i in (0..result).rev() {
1547                    dst[2 * i] = dst[i];
1548                    dst[2 * i + 1] = 0;
1549                }
1550
1551                // and then the remainder of the string is encoded.
1552                for (u, bytes) in string[i..]
1553                    .encode_utf16()
1554                    .zip(dst[2 * result..].chunks_mut(2))
1555                {
1556                    let u_bytes = u.to_le_bytes();
1557                    bytes[0] = u_bytes[0];
1558                    bytes[1] = u_bytes[1];
1559                    result += 1;
1560                }
1561                if worst_case > 2 * result {
1562                    ptr = cx.realloc(ptr, worst_case, 2, 2 * result)?;
1563                }
1564                return Ok((ptr, result | UTF16_TAG));
1565            }
1566            if result < bytes.len() {
1567                ptr = cx.realloc(ptr, bytes.len(), 2, result)?;
1568            }
1569            Ok((ptr, result))
1570        }
1571    }
1572}
1573
1574/// Representation of a string located in linear memory in a WebAssembly
1575/// instance.
1576///
1577/// This type can be used in place of `String` and `str` for string-taking APIs
1578/// in some situations. The purpose of this type is to represent a range of
1579/// validated bytes within a component but does not actually copy the bytes. The
1580/// primary method, [`WasmStr::to_str`], attempts to return a reference to the
1581/// string directly located in the component's memory, avoiding a copy into the
1582/// host if possible.
1583///
1584/// The downside of this type, however, is that accessing a string requires a
1585/// [`Store`](crate::Store) pointer (via [`StoreContext`]). Bindings generated
1586/// by [`bindgen!`](crate::component::bindgen), for example, do not have access
1587/// to [`StoreContext`] and thus can't use this type.
1588///
1589/// This is intended for more advanced use cases such as defining functions
1590/// directly in a [`Linker`](crate::component::Linker). It's expected that in
1591/// the future [`bindgen!`](crate::component::bindgen) will also have a way to
1592/// use this type.
1593///
1594/// This type is used with [`TypedFunc`], for example, when WebAssembly returns
1595/// a string. This type cannot be used to give a string to WebAssembly, instead
1596/// `&str` should be used for that (since it's coming from the host).
1597///
1598/// Note that this type represents an in-bounds string in linear memory, but it
1599/// does not represent a valid string (e.g. valid utf-8). Validation happens
1600/// when [`WasmStr::to_str`] is called.
1601///
1602/// Also note that this type does not implement [`Lower`], it only implements
1603/// [`Lift`].
1604pub struct WasmStr {
1605    ptr: usize,
1606    len: usize,
1607    options: Options,
1608}
1609
1610impl WasmStr {
1611    pub(crate) fn new(ptr: usize, len: usize, cx: &mut LiftContext<'_>) -> Result<WasmStr> {
1612        let byte_len = match cx.options.string_encoding() {
1613            StringEncoding::Utf8 => Some(len),
1614            StringEncoding::Utf16 => len.checked_mul(2),
1615            StringEncoding::CompactUtf16 => {
1616                if len & UTF16_TAG == 0 {
1617                    Some(len)
1618                } else {
1619                    (len ^ UTF16_TAG).checked_mul(2)
1620                }
1621            }
1622        };
1623        match byte_len.and_then(|len| ptr.checked_add(len)) {
1624            Some(n) if n <= cx.memory().len() => {}
1625            _ => bail!("string pointer/length out of bounds of memory"),
1626        }
1627        Ok(WasmStr {
1628            ptr,
1629            len,
1630            options: *cx.options,
1631        })
1632    }
1633
1634    /// Returns the underlying string that this cursor points to.
1635    ///
1636    /// Note that this will internally decode the string from the wasm's
1637    /// encoding to utf-8 and additionally perform validation.
1638    ///
1639    /// The `store` provided must be the store where this string lives to
1640    /// access the correct memory.
1641    ///
1642    /// # Errors
1643    ///
1644    /// Returns an error if the string wasn't encoded correctly (e.g. invalid
1645    /// utf-8).
1646    ///
1647    /// # Panics
1648    ///
1649    /// Panics if this string is not owned by `store`.
1650    //
1651    // TODO: should add accessors for specifically utf-8 and utf-16 that perhaps
1652    // in an opt-in basis don't do validation. Additionally there should be some
1653    // method that returns `[u16]` after validating to avoid the utf16-to-utf8
1654    // transcode.
1655    pub fn to_str<'a, T: 'static>(
1656        &self,
1657        store: impl Into<StoreContext<'a, T>>,
1658    ) -> Result<Cow<'a, str>> {
1659        let store = store.into().0;
1660        let memory = self.options.memory(store);
1661        self.to_str_from_memory(memory)
1662    }
1663
1664    pub(crate) fn to_str_from_memory<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1665        match self.options.string_encoding() {
1666            StringEncoding::Utf8 => self.decode_utf8(memory),
1667            StringEncoding::Utf16 => self.decode_utf16(memory, self.len),
1668            StringEncoding::CompactUtf16 => {
1669                if self.len & UTF16_TAG == 0 {
1670                    self.decode_latin1(memory)
1671                } else {
1672                    self.decode_utf16(memory, self.len ^ UTF16_TAG)
1673                }
1674            }
1675        }
1676    }
1677
1678    fn decode_utf8<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1679        // Note that bounds-checking already happen in construction of `WasmStr`
1680        // so this is never expected to panic. This could theoretically be
1681        // unchecked indexing if we're feeling wild enough.
1682        Ok(str::from_utf8(&memory[self.ptr..][..self.len])?.into())
1683    }
1684
1685    fn decode_utf16<'a>(&self, memory: &'a [u8], len: usize) -> Result<Cow<'a, str>> {
1686        // See notes in `decode_utf8` for why this is panicking indexing.
1687        let memory = &memory[self.ptr..][..len * 2];
1688        Ok(core::char::decode_utf16(
1689            memory
1690                .chunks(2)
1691                .map(|chunk| u16::from_le_bytes(chunk.try_into().unwrap())),
1692        )
1693        .collect::<Result<String, _>>()?
1694        .into())
1695    }
1696
1697    fn decode_latin1<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1698        // See notes in `decode_utf8` for why this is panicking indexing.
1699        Ok(encoding_rs::mem::decode_latin1(
1700            &memory[self.ptr..][..self.len],
1701        ))
1702    }
1703}
1704
1705// Note that this is similar to `ComponentType for str` except it can only be
1706// used for lifting, not lowering.
1707unsafe impl ComponentType for WasmStr {
1708    type Lower = <str as ComponentType>::Lower;
1709
1710    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1711
1712    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1713        match ty {
1714            InterfaceType::String => Ok(()),
1715            other => bail!("expected `string` found `{}`", desc(other)),
1716        }
1717    }
1718}
1719
1720unsafe impl Lift for WasmStr {
1721    #[inline]
1722    fn linear_lift_from_flat(
1723        cx: &mut LiftContext<'_>,
1724        ty: InterfaceType,
1725        src: &Self::Lower,
1726    ) -> Result<Self> {
1727        debug_assert!(matches!(ty, InterfaceType::String));
1728        // FIXME(#4311): needs memory64 treatment
1729        let ptr = src[0].get_u32();
1730        let len = src[1].get_u32();
1731        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
1732        WasmStr::new(ptr, len, cx)
1733    }
1734
1735    #[inline]
1736    fn linear_lift_from_memory(
1737        cx: &mut LiftContext<'_>,
1738        ty: InterfaceType,
1739        bytes: &[u8],
1740    ) -> Result<Self> {
1741        debug_assert!(matches!(ty, InterfaceType::String));
1742        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
1743        // FIXME(#4311): needs memory64 treatment
1744        let ptr = u32::from_le_bytes(bytes[..4].try_into().unwrap());
1745        let len = u32::from_le_bytes(bytes[4..].try_into().unwrap());
1746        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
1747        WasmStr::new(ptr, len, cx)
1748    }
1749}
1750
1751unsafe impl<T> ComponentType for [T]
1752where
1753    T: ComponentType,
1754{
1755    type Lower = [ValRaw; 2];
1756
1757    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1758
1759    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
1760        match ty {
1761            InterfaceType::List(t) => T::typecheck(&types.types[*t].element, types),
1762            other => bail!("expected `list` found `{}`", desc(other)),
1763        }
1764    }
1765}
1766
1767unsafe impl<T> Lower for [T]
1768where
1769    T: Lower,
1770{
1771    fn linear_lower_to_flat<U>(
1772        &self,
1773        cx: &mut LowerContext<'_, U>,
1774        ty: InterfaceType,
1775        dst: &mut MaybeUninit<[ValRaw; 2]>,
1776    ) -> Result<()> {
1777        let elem = match ty {
1778            InterfaceType::List(i) => cx.types[i].element,
1779            _ => bad_type_info(),
1780        };
1781        let (ptr, len) = lower_list(cx, elem, self)?;
1782        // See "WRITEPTR64" above for why this is always storing a 64-bit
1783        // integer.
1784        map_maybe_uninit!(dst[0]).write(ValRaw::i64(ptr as i64));
1785        map_maybe_uninit!(dst[1]).write(ValRaw::i64(len as i64));
1786        Ok(())
1787    }
1788
1789    fn linear_lower_to_memory<U>(
1790        &self,
1791        cx: &mut LowerContext<'_, U>,
1792        ty: InterfaceType,
1793        offset: usize,
1794    ) -> Result<()> {
1795        let elem = match ty {
1796            InterfaceType::List(i) => cx.types[i].element,
1797            _ => bad_type_info(),
1798        };
1799        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
1800        let (ptr, len) = lower_list(cx, elem, self)?;
1801        *cx.get(offset + 0) = u32::try_from(ptr).unwrap().to_le_bytes();
1802        *cx.get(offset + 4) = u32::try_from(len).unwrap().to_le_bytes();
1803        Ok(())
1804    }
1805}
1806
1807// FIXME: this is not a memcpy for `T` where `T` is something like `u8`.
1808//
1809// Some attempts to fix this have proved not fruitful. In isolation an attempt
1810// was made where:
1811//
1812// * `MemoryMut` stored a `*mut [u8]` as its "last view" of memory to avoid
1813//   reloading the base pointer constantly. This view is reset on `realloc`.
1814// * The bounds-checks in `MemoryMut::get` were removed (replaced with unsafe
1815//   indexing)
1816//
1817// Even then though this didn't correctly vectorized for `Vec<u8>`. It's not
1818// entirely clear why but it appeared that it's related to reloading the base
1819// pointer to memory (I guess from `MemoryMut` itself?). Overall I'm not really
1820// clear on what's happening there, but this is surely going to be a performance
1821// bottleneck in the future.
1822fn lower_list<T, U>(
1823    cx: &mut LowerContext<'_, U>,
1824    ty: InterfaceType,
1825    list: &[T],
1826) -> Result<(usize, usize)>
1827where
1828    T: Lower,
1829{
1830    let elem_size = T::SIZE32;
1831    let size = list
1832        .len()
1833        .checked_mul(elem_size)
1834        .ok_or_else(|| anyhow!("size overflow copying a list"))?;
1835    let ptr = cx.realloc(0, 0, T::ALIGN32, size)?;
1836    T::linear_store_list_to_memory(cx, ty, ptr, list)?;
1837    Ok((ptr, list.len()))
1838}
1839
1840/// Representation of a list of values that are owned by a WebAssembly instance.
1841///
1842/// For some more commentary about the rationale for this type see the
1843/// documentation of [`WasmStr`]. In summary this type can avoid a copy when
1844/// passing data to the host in some situations but is additionally more
1845/// cumbersome to use by requiring a [`Store`](crate::Store) to be provided.
1846///
1847/// This type is used whenever a `(list T)` is returned from a [`TypedFunc`],
1848/// for example. This type represents a list of values that are stored in linear
1849/// memory which are waiting to be read.
1850///
1851/// Note that this type represents only a valid range of bytes for the list
1852/// itself, it does not represent validity of the elements themselves and that's
1853/// performed when they're iterated.
1854///
1855/// Note that this type does not implement the [`Lower`] trait, only [`Lift`].
1856pub struct WasmList<T> {
1857    ptr: usize,
1858    len: usize,
1859    options: Options,
1860    elem: InterfaceType,
1861    instance: Instance,
1862    _marker: marker::PhantomData<T>,
1863}
1864
1865impl<T: Lift> WasmList<T> {
1866    pub(crate) fn new(
1867        ptr: usize,
1868        len: usize,
1869        cx: &mut LiftContext<'_>,
1870        elem: InterfaceType,
1871    ) -> Result<WasmList<T>> {
1872        match len
1873            .checked_mul(T::SIZE32)
1874            .and_then(|len| ptr.checked_add(len))
1875        {
1876            Some(n) if n <= cx.memory().len() => {}
1877            _ => bail!("list pointer/length out of bounds of memory"),
1878        }
1879        if ptr % usize::try_from(T::ALIGN32)? != 0 {
1880            bail!("list pointer is not aligned")
1881        }
1882        Ok(WasmList {
1883            ptr,
1884            len,
1885            options: *cx.options,
1886            elem,
1887            instance: cx.instance_handle(),
1888            _marker: marker::PhantomData,
1889        })
1890    }
1891
1892    /// Returns the item length of this vector
1893    #[inline]
1894    pub fn len(&self) -> usize {
1895        self.len
1896    }
1897
1898    /// Gets the `n`th element of this list.
1899    ///
1900    /// Returns `None` if `index` is out of bounds. Returns `Some(Err(..))` if
1901    /// the value couldn't be decoded (it was invalid). Returns `Some(Ok(..))`
1902    /// if the value is valid.
1903    ///
1904    /// # Panics
1905    ///
1906    /// This function will panic if the string did not originally come from the
1907    /// `store` specified.
1908    //
1909    // TODO: given that interface values are intended to be consumed in one go
1910    // should we even expose a random access iteration API? In theory all
1911    // consumers should be validating through the iterator.
1912    pub fn get(&self, mut store: impl AsContextMut, index: usize) -> Option<Result<T>> {
1913        let store = store.as_context_mut().0;
1914        self.options.store_id().assert_belongs_to(store.id());
1915        let mut cx = LiftContext::new(store, &self.options, self.instance);
1916        self.get_from_store(&mut cx, index)
1917    }
1918
1919    fn get_from_store(&self, cx: &mut LiftContext<'_>, index: usize) -> Option<Result<T>> {
1920        if index >= self.len {
1921            return None;
1922        }
1923        // Note that this is using panicking indexing and this is expected to
1924        // never fail. The bounds-checking here happened during the construction
1925        // of the `WasmList` itself which means these should always be in-bounds
1926        // (and wasm memory can only grow). This could theoretically be
1927        // unchecked indexing if we're confident enough and it's actually a perf
1928        // issue one day.
1929        let bytes = &cx.memory()[self.ptr + index * T::SIZE32..][..T::SIZE32];
1930        Some(T::linear_lift_from_memory(cx, self.elem, bytes))
1931    }
1932
1933    /// Returns an iterator over the elements of this list.
1934    ///
1935    /// Each item of the list may fail to decode and is represented through the
1936    /// `Result` value of the iterator.
1937    pub fn iter<'a, U: 'static>(
1938        &'a self,
1939        store: impl Into<StoreContextMut<'a, U>>,
1940    ) -> impl ExactSizeIterator<Item = Result<T>> + 'a {
1941        let store = store.into().0;
1942        self.options.store_id().assert_belongs_to(store.id());
1943        let mut cx = LiftContext::new(store, &self.options, self.instance);
1944        (0..self.len).map(move |i| self.get_from_store(&mut cx, i).unwrap())
1945    }
1946}
1947
1948macro_rules! raw_wasm_list_accessors {
1949    ($($i:ident)*) => ($(
1950        impl WasmList<$i> {
1951            /// Get access to the raw underlying memory for this list.
1952            ///
1953            /// This method will return a direct slice into the original wasm
1954            /// module's linear memory where the data for this slice is stored.
1955            /// This allows the embedder to have efficient access to the
1956            /// underlying memory if needed and avoid copies and such if
1957            /// desired.
1958            ///
1959            /// Note that multi-byte integers are stored in little-endian format
1960            /// so portable processing of this slice must be aware of the host's
1961            /// byte-endianness. The `from_le` constructors in the Rust standard
1962            /// library should be suitable for converting from little-endian.
1963            ///
1964            /// # Panics
1965            ///
1966            /// Panics if the `store` provided is not the one from which this
1967            /// slice originated.
1968            pub fn as_le_slice<'a, T: 'static>(&self, store: impl Into<StoreContext<'a, T>>) -> &'a [$i] {
1969                let memory = self.options.memory(store.into().0);
1970                self._as_le_slice(memory)
1971            }
1972
1973            fn _as_le_slice<'a>(&self, all_of_memory: &'a [u8]) -> &'a [$i] {
1974                // See comments in `WasmList::get` for the panicking indexing
1975                let byte_size = self.len * mem::size_of::<$i>();
1976                let bytes = &all_of_memory[self.ptr..][..byte_size];
1977
1978                // The canonical ABI requires that everything is aligned to its
1979                // own size, so this should be an aligned array. Furthermore the
1980                // alignment of primitive integers for hosts should be smaller
1981                // than or equal to the size of the primitive itself, meaning
1982                // that a wasm canonical-abi-aligned list is also aligned for
1983                // the host. That should mean that the head/tail slices here are
1984                // empty.
1985                //
1986                // Also note that the `unsafe` here is needed since the type
1987                // we're aligning to isn't guaranteed to be valid, but in our
1988                // case it's just integers and bytes so this should be safe.
1989                unsafe {
1990                    let (head, body, tail) = bytes.align_to::<$i>();
1991                    assert!(head.is_empty() && tail.is_empty());
1992                    body
1993                }
1994            }
1995        }
1996    )*)
1997}
1998
1999raw_wasm_list_accessors! {
2000    i8 i16 i32 i64
2001    u8 u16 u32 u64
2002}
2003
2004// Note that this is similar to `ComponentType for str` except it can only be
2005// used for lifting, not lowering.
2006unsafe impl<T: ComponentType> ComponentType for WasmList<T> {
2007    type Lower = <[T] as ComponentType>::Lower;
2008
2009    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
2010
2011    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2012        <[T] as ComponentType>::typecheck(ty, types)
2013    }
2014}
2015
2016unsafe impl<T: Lift> Lift for WasmList<T> {
2017    fn linear_lift_from_flat(
2018        cx: &mut LiftContext<'_>,
2019        ty: InterfaceType,
2020        src: &Self::Lower,
2021    ) -> Result<Self> {
2022        let elem = match ty {
2023            InterfaceType::List(i) => cx.types[i].element,
2024            _ => bad_type_info(),
2025        };
2026        // FIXME(#4311): needs memory64 treatment
2027        let ptr = src[0].get_u32();
2028        let len = src[1].get_u32();
2029        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
2030        WasmList::new(ptr, len, cx, elem)
2031    }
2032
2033    fn linear_lift_from_memory(
2034        cx: &mut LiftContext<'_>,
2035        ty: InterfaceType,
2036        bytes: &[u8],
2037    ) -> Result<Self> {
2038        let elem = match ty {
2039            InterfaceType::List(i) => cx.types[i].element,
2040            _ => bad_type_info(),
2041        };
2042        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2043        // FIXME(#4311): needs memory64 treatment
2044        let ptr = u32::from_le_bytes(bytes[..4].try_into().unwrap());
2045        let len = u32::from_le_bytes(bytes[4..].try_into().unwrap());
2046        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
2047        WasmList::new(ptr, len, cx, elem)
2048    }
2049}
2050
2051/// Verify that the given wasm type is a tuple with the expected fields in the right order.
2052fn typecheck_tuple(
2053    ty: &InterfaceType,
2054    types: &InstanceType<'_>,
2055    expected: &[fn(&InterfaceType, &InstanceType<'_>) -> Result<()>],
2056) -> Result<()> {
2057    match ty {
2058        InterfaceType::Tuple(t) => {
2059            let tuple = &types.types[*t];
2060            if tuple.types.len() != expected.len() {
2061                bail!(
2062                    "expected {}-tuple, found {}-tuple",
2063                    expected.len(),
2064                    tuple.types.len()
2065                );
2066            }
2067            for (ty, check) in tuple.types.iter().zip(expected) {
2068                check(ty, types)?;
2069            }
2070            Ok(())
2071        }
2072        other => bail!("expected `tuple` found `{}`", desc(other)),
2073    }
2074}
2075
2076/// Verify that the given wasm type is a record with the expected fields in the right order and with the right
2077/// names.
2078pub fn typecheck_record(
2079    ty: &InterfaceType,
2080    types: &InstanceType<'_>,
2081    expected: &[(&str, fn(&InterfaceType, &InstanceType<'_>) -> Result<()>)],
2082) -> Result<()> {
2083    match ty {
2084        InterfaceType::Record(index) => {
2085            let fields = &types.types[*index].fields;
2086
2087            if fields.len() != expected.len() {
2088                bail!(
2089                    "expected record of {} fields, found {} fields",
2090                    expected.len(),
2091                    fields.len()
2092                );
2093            }
2094
2095            for (field, &(name, check)) in fields.iter().zip(expected) {
2096                check(&field.ty, types)
2097                    .with_context(|| format!("type mismatch for field {name}"))?;
2098
2099                if field.name != name {
2100                    bail!("expected record field named {}, found {}", name, field.name);
2101                }
2102            }
2103
2104            Ok(())
2105        }
2106        other => bail!("expected `record` found `{}`", desc(other)),
2107    }
2108}
2109
2110/// Verify that the given wasm type is a variant with the expected cases in the right order and with the right
2111/// names.
2112pub fn typecheck_variant(
2113    ty: &InterfaceType,
2114    types: &InstanceType<'_>,
2115    expected: &[(
2116        &str,
2117        Option<fn(&InterfaceType, &InstanceType<'_>) -> Result<()>>,
2118    )],
2119) -> Result<()> {
2120    match ty {
2121        InterfaceType::Variant(index) => {
2122            let cases = &types.types[*index].cases;
2123
2124            if cases.len() != expected.len() {
2125                bail!(
2126                    "expected variant of {} cases, found {} cases",
2127                    expected.len(),
2128                    cases.len()
2129                );
2130            }
2131
2132            for ((case_name, case_ty), &(name, check)) in cases.iter().zip(expected) {
2133                if *case_name != name {
2134                    bail!("expected variant case named {name}, found {case_name}");
2135                }
2136
2137                match (check, case_ty) {
2138                    (Some(check), Some(ty)) => check(ty, types)
2139                        .with_context(|| format!("type mismatch for case {name}"))?,
2140                    (None, None) => {}
2141                    (Some(_), None) => {
2142                        bail!("case `{name}` has no type but one was expected")
2143                    }
2144                    (None, Some(_)) => {
2145                        bail!("case `{name}` has a type but none was expected")
2146                    }
2147                }
2148            }
2149
2150            Ok(())
2151        }
2152        other => bail!("expected `variant` found `{}`", desc(other)),
2153    }
2154}
2155
2156/// Verify that the given wasm type is a enum with the expected cases in the right order and with the right
2157/// names.
2158pub fn typecheck_enum(
2159    ty: &InterfaceType,
2160    types: &InstanceType<'_>,
2161    expected: &[&str],
2162) -> Result<()> {
2163    match ty {
2164        InterfaceType::Enum(index) => {
2165            let names = &types.types[*index].names;
2166
2167            if names.len() != expected.len() {
2168                bail!(
2169                    "expected enum of {} names, found {} names",
2170                    expected.len(),
2171                    names.len()
2172                );
2173            }
2174
2175            for (name, expected) in names.iter().zip(expected) {
2176                if name != expected {
2177                    bail!("expected enum case named {}, found {}", expected, name);
2178                }
2179            }
2180
2181            Ok(())
2182        }
2183        other => bail!("expected `enum` found `{}`", desc(other)),
2184    }
2185}
2186
2187/// Verify that the given wasm type is a flags type with the expected flags in the right order and with the right
2188/// names.
2189pub fn typecheck_flags(
2190    ty: &InterfaceType,
2191    types: &InstanceType<'_>,
2192    expected: &[&str],
2193) -> Result<()> {
2194    match ty {
2195        InterfaceType::Flags(index) => {
2196            let names = &types.types[*index].names;
2197
2198            if names.len() != expected.len() {
2199                bail!(
2200                    "expected flags type with {} names, found {} names",
2201                    expected.len(),
2202                    names.len()
2203                );
2204            }
2205
2206            for (name, expected) in names.iter().zip(expected) {
2207                if name != expected {
2208                    bail!("expected flag named {}, found {}", expected, name);
2209                }
2210            }
2211
2212            Ok(())
2213        }
2214        other => bail!("expected `flags` found `{}`", desc(other)),
2215    }
2216}
2217
2218/// Format the specified bitflags using the specified names for debugging
2219pub fn format_flags(bits: &[u32], names: &[&str], f: &mut fmt::Formatter) -> fmt::Result {
2220    f.write_str("(")?;
2221    let mut wrote = false;
2222    for (index, name) in names.iter().enumerate() {
2223        if ((bits[index / 32] >> (index % 32)) & 1) != 0 {
2224            if wrote {
2225                f.write_str("|")?;
2226            } else {
2227                wrote = true;
2228            }
2229
2230            f.write_str(name)?;
2231        }
2232    }
2233    f.write_str(")")
2234}
2235
2236unsafe impl<T> ComponentType for Option<T>
2237where
2238    T: ComponentType,
2239{
2240    type Lower = TupleLower<<u32 as ComponentType>::Lower, T::Lower>;
2241
2242    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::variant_static(&[None, Some(T::ABI)]);
2243
2244    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2245        match ty {
2246            InterfaceType::Option(t) => T::typecheck(&types.types[*t].ty, types),
2247            other => bail!("expected `option` found `{}`", desc(other)),
2248        }
2249    }
2250}
2251
2252unsafe impl<T> ComponentVariant for Option<T>
2253where
2254    T: ComponentType,
2255{
2256    const CASES: &'static [Option<CanonicalAbiInfo>] = &[None, Some(T::ABI)];
2257}
2258
2259unsafe impl<T> Lower for Option<T>
2260where
2261    T: Lower,
2262{
2263    fn linear_lower_to_flat<U>(
2264        &self,
2265        cx: &mut LowerContext<'_, U>,
2266        ty: InterfaceType,
2267        dst: &mut MaybeUninit<Self::Lower>,
2268    ) -> Result<()> {
2269        let payload = match ty {
2270            InterfaceType::Option(ty) => cx.types[ty].ty,
2271            _ => bad_type_info(),
2272        };
2273        match self {
2274            None => {
2275                map_maybe_uninit!(dst.A1).write(ValRaw::i32(0));
2276                // Note that this is unsafe as we're writing an arbitrary
2277                // bit-pattern to an arbitrary type, but part of the unsafe
2278                // contract of the `ComponentType` trait is that we can assign
2279                // any bit-pattern. By writing all zeros here we're ensuring
2280                // that the core wasm arguments this translates to will all be
2281                // zeros (as the canonical ABI requires).
2282                unsafe {
2283                    map_maybe_uninit!(dst.A2).as_mut_ptr().write_bytes(0u8, 1);
2284                }
2285            }
2286            Some(val) => {
2287                map_maybe_uninit!(dst.A1).write(ValRaw::i32(1));
2288                val.linear_lower_to_flat(cx, payload, map_maybe_uninit!(dst.A2))?;
2289            }
2290        }
2291        Ok(())
2292    }
2293
2294    fn linear_lower_to_memory<U>(
2295        &self,
2296        cx: &mut LowerContext<'_, U>,
2297        ty: InterfaceType,
2298        offset: usize,
2299    ) -> Result<()> {
2300        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
2301        let payload = match ty {
2302            InterfaceType::Option(ty) => cx.types[ty].ty,
2303            _ => bad_type_info(),
2304        };
2305        match self {
2306            None => {
2307                cx.get::<1>(offset)[0] = 0;
2308            }
2309            Some(val) => {
2310                cx.get::<1>(offset)[0] = 1;
2311                val.linear_lower_to_memory(
2312                    cx,
2313                    payload,
2314                    offset + (Self::INFO.payload_offset32 as usize),
2315                )?;
2316            }
2317        }
2318        Ok(())
2319    }
2320}
2321
2322unsafe impl<T> Lift for Option<T>
2323where
2324    T: Lift,
2325{
2326    fn linear_lift_from_flat(
2327        cx: &mut LiftContext<'_>,
2328        ty: InterfaceType,
2329        src: &Self::Lower,
2330    ) -> Result<Self> {
2331        let payload = match ty {
2332            InterfaceType::Option(ty) => cx.types[ty].ty,
2333            _ => bad_type_info(),
2334        };
2335        Ok(match src.A1.get_i32() {
2336            0 => None,
2337            1 => Some(T::linear_lift_from_flat(cx, payload, &src.A2)?),
2338            _ => bail!("invalid option discriminant"),
2339        })
2340    }
2341
2342    fn linear_lift_from_memory(
2343        cx: &mut LiftContext<'_>,
2344        ty: InterfaceType,
2345        bytes: &[u8],
2346    ) -> Result<Self> {
2347        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2348        let payload_ty = match ty {
2349            InterfaceType::Option(ty) => cx.types[ty].ty,
2350            _ => bad_type_info(),
2351        };
2352        let discrim = bytes[0];
2353        let payload = &bytes[Self::INFO.payload_offset32 as usize..];
2354        match discrim {
2355            0 => Ok(None),
2356            1 => Ok(Some(T::linear_lift_from_memory(cx, payload_ty, payload)?)),
2357            _ => bail!("invalid option discriminant"),
2358        }
2359    }
2360}
2361
2362#[derive(Clone, Copy)]
2363#[repr(C)]
2364pub struct ResultLower<T: Copy, E: Copy> {
2365    tag: ValRaw,
2366    payload: ResultLowerPayload<T, E>,
2367}
2368
2369#[derive(Clone, Copy)]
2370#[repr(C)]
2371union ResultLowerPayload<T: Copy, E: Copy> {
2372    ok: T,
2373    err: E,
2374}
2375
2376unsafe impl<T, E> ComponentType for Result<T, E>
2377where
2378    T: ComponentType,
2379    E: ComponentType,
2380{
2381    type Lower = ResultLower<T::Lower, E::Lower>;
2382
2383    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::variant_static(&[Some(T::ABI), Some(E::ABI)]);
2384
2385    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2386        match ty {
2387            InterfaceType::Result(r) => {
2388                let result = &types.types[*r];
2389                match &result.ok {
2390                    Some(ty) => T::typecheck(ty, types)?,
2391                    None if T::IS_RUST_UNIT_TYPE => {}
2392                    None => bail!("expected no `ok` type"),
2393                }
2394                match &result.err {
2395                    Some(ty) => E::typecheck(ty, types)?,
2396                    None if E::IS_RUST_UNIT_TYPE => {}
2397                    None => bail!("expected no `err` type"),
2398                }
2399                Ok(())
2400            }
2401            other => bail!("expected `result` found `{}`", desc(other)),
2402        }
2403    }
2404}
2405
2406/// Lowers the payload of a variant into the storage for the entire payload,
2407/// handling writing zeros at the end of the representation if this payload is
2408/// smaller than the entire flat representation.
2409///
2410/// * `payload` - the flat storage space for the entire payload of the variant
2411/// * `typed_payload` - projection from the payload storage space to the
2412///   individual storage space for this variant.
2413/// * `lower` - lowering operation used to initialize the `typed_payload` return
2414///   value.
2415///
2416/// For more information on this se the comments in the `Lower for Result`
2417/// implementation below.
2418pub unsafe fn lower_payload<P, T>(
2419    payload: &mut MaybeUninit<P>,
2420    typed_payload: impl FnOnce(&mut MaybeUninit<P>) -> &mut MaybeUninit<T>,
2421    lower: impl FnOnce(&mut MaybeUninit<T>) -> Result<()>,
2422) -> Result<()> {
2423    let typed = typed_payload(payload);
2424    lower(typed)?;
2425
2426    let typed_len = unsafe { storage_as_slice(typed).len() };
2427    let payload = unsafe { storage_as_slice_mut(payload) };
2428    for slot in payload[typed_len..].iter_mut() {
2429        slot.write(ValRaw::u64(0));
2430    }
2431    Ok(())
2432}
2433
2434unsafe impl<T, E> ComponentVariant for Result<T, E>
2435where
2436    T: ComponentType,
2437    E: ComponentType,
2438{
2439    const CASES: &'static [Option<CanonicalAbiInfo>] = &[Some(T::ABI), Some(E::ABI)];
2440}
2441
2442unsafe impl<T, E> Lower for Result<T, E>
2443where
2444    T: Lower,
2445    E: Lower,
2446{
2447    fn linear_lower_to_flat<U>(
2448        &self,
2449        cx: &mut LowerContext<'_, U>,
2450        ty: InterfaceType,
2451        dst: &mut MaybeUninit<Self::Lower>,
2452    ) -> Result<()> {
2453        let (ok, err) = match ty {
2454            InterfaceType::Result(ty) => {
2455                let ty = &cx.types[ty];
2456                (ty.ok, ty.err)
2457            }
2458            _ => bad_type_info(),
2459        };
2460
2461        // This implementation of `Lower::lower`, if you're reading these from
2462        // the top of this file, is the first location that the "join" logic of
2463        // the component model's canonical ABI encountered. The rough problem is
2464        // that let's say we have a component model type of the form:
2465        //
2466        //      (result u64 (error (tuple f32 u16)))
2467        //
2468        // The flat representation of this is actually pretty tricky. Currently
2469        // it is:
2470        //
2471        //      i32 i64 i32
2472        //
2473        // The first `i32` is the discriminant for the `result`, and the payload
2474        // is represented by `i64 i32`. The "ok" variant will only use the `i64`
2475        // and the "err" variant will use both `i64` and `i32`.
2476        //
2477        // In the "ok" variant the first issue is encountered. The size of one
2478        // variant may not match the size of the other variants. All variants
2479        // start at the "front" but when lowering a type we need to be sure to
2480        // initialize the later variants (lest we leak random host memory into
2481        // the guest module). Due to how the `Lower` type is represented as a
2482        // `union` of all the variants what ends up happening here is that
2483        // internally within the `lower_payload` after the typed payload is
2484        // lowered the remaining bits of the payload that weren't initialized
2485        // are all set to zero. This will guarantee that we'll write to all the
2486        // slots for each variant.
2487        //
2488        // The "err" variant encounters the second issue, however, which is that
2489        // the flat representation for each type may differ between payloads. In
2490        // the "ok" arm an `i64` is written, but the `lower` implementation for
2491        // the "err" arm will write an `f32` and then an `i32`. For this
2492        // implementation of `lower` to be valid the `f32` needs to get inflated
2493        // to an `i64` with zero-padding in the upper bits. What may be
2494        // surprising, however, is that none of this is handled in this file.
2495        // This implementation looks like it's blindly deferring to `E::lower`
2496        // and hoping it does the right thing.
2497        //
2498        // In reality, however, the correctness of variant lowering relies on
2499        // two subtle details of the `ValRaw` implementation in Wasmtime:
2500        //
2501        // 1. First the `ValRaw` value always contains little-endian values.
2502        //    This means that if a `u32` is written, a `u64` is read, and then
2503        //    the `u64` has its upper bits truncated the original value will
2504        //    always be retained. This is primarily here for big-endian
2505        //    platforms where if it weren't little endian then the opposite
2506        //    would occur and the wrong value would be read.
2507        //
2508        // 2. Second, and perhaps even more subtly, the `ValRaw` constructors
2509        //    for 32-bit types actually always initialize 64-bits of the
2510        //    `ValRaw`. In the component model flat ABI only 32 and 64-bit types
2511        //    are used so 64-bits is big enough to contain everything. This
2512        //    means that when a `ValRaw` is written into the destination it will
2513        //    always, whether it's needed or not, be "ready" to get extended up
2514        //    to 64-bits.
2515        //
2516        // Put together these two subtle guarantees means that all `Lower`
2517        // implementations can be written "naturally" as one might naively
2518        // expect. Variants will, on each arm, zero out remaining fields and all
2519        // writes to the flat representation will automatically be 64-bit writes
2520        // meaning that if the value is read as a 64-bit value, which isn't
2521        // known at the time of the write, it'll still be correct.
2522        match self {
2523            Ok(e) => {
2524                map_maybe_uninit!(dst.tag).write(ValRaw::i32(0));
2525                unsafe {
2526                    lower_payload(
2527                        map_maybe_uninit!(dst.payload),
2528                        |payload| map_maybe_uninit!(payload.ok),
2529                        |dst| match ok {
2530                            Some(ok) => e.linear_lower_to_flat(cx, ok, dst),
2531                            None => Ok(()),
2532                        },
2533                    )
2534                }
2535            }
2536            Err(e) => {
2537                map_maybe_uninit!(dst.tag).write(ValRaw::i32(1));
2538                unsafe {
2539                    lower_payload(
2540                        map_maybe_uninit!(dst.payload),
2541                        |payload| map_maybe_uninit!(payload.err),
2542                        |dst| match err {
2543                            Some(err) => e.linear_lower_to_flat(cx, err, dst),
2544                            None => Ok(()),
2545                        },
2546                    )
2547                }
2548            }
2549        }
2550    }
2551
2552    fn linear_lower_to_memory<U>(
2553        &self,
2554        cx: &mut LowerContext<'_, U>,
2555        ty: InterfaceType,
2556        offset: usize,
2557    ) -> Result<()> {
2558        let (ok, err) = match ty {
2559            InterfaceType::Result(ty) => {
2560                let ty = &cx.types[ty];
2561                (ty.ok, ty.err)
2562            }
2563            _ => bad_type_info(),
2564        };
2565        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
2566        let payload_offset = Self::INFO.payload_offset32 as usize;
2567        match self {
2568            Ok(e) => {
2569                cx.get::<1>(offset)[0] = 0;
2570                if let Some(ok) = ok {
2571                    e.linear_lower_to_memory(cx, ok, offset + payload_offset)?;
2572                }
2573            }
2574            Err(e) => {
2575                cx.get::<1>(offset)[0] = 1;
2576                if let Some(err) = err {
2577                    e.linear_lower_to_memory(cx, err, offset + payload_offset)?;
2578                }
2579            }
2580        }
2581        Ok(())
2582    }
2583}
2584
2585unsafe impl<T, E> Lift for Result<T, E>
2586where
2587    T: Lift,
2588    E: Lift,
2589{
2590    #[inline]
2591    fn linear_lift_from_flat(
2592        cx: &mut LiftContext<'_>,
2593        ty: InterfaceType,
2594        src: &Self::Lower,
2595    ) -> Result<Self> {
2596        let (ok, err) = match ty {
2597            InterfaceType::Result(ty) => {
2598                let ty = &cx.types[ty];
2599                (ty.ok, ty.err)
2600            }
2601            _ => bad_type_info(),
2602        };
2603        // Note that this implementation specifically isn't trying to actually
2604        // reinterpret or alter the bits of `lower` depending on which variant
2605        // we're lifting. This ends up all working out because the value is
2606        // stored in little-endian format.
2607        //
2608        // When stored in little-endian format the `{T,E}::Lower`, when each
2609        // individual `ValRaw` is read, means that if an i64 value, extended
2610        // from an i32 value, was stored then when the i32 value is read it'll
2611        // automatically ignore the upper bits.
2612        //
2613        // This "trick" allows us to seamlessly pass through the `Self::Lower`
2614        // representation into the lifting/lowering without trying to handle
2615        // "join"ed types as per the canonical ABI. It just so happens that i64
2616        // bits will naturally be reinterpreted as f64. Additionally if the
2617        // joined type is i64 but only the lower bits are read that's ok and we
2618        // don't need to validate the upper bits.
2619        //
2620        // This is largely enabled by WebAssembly/component-model#35 where no
2621        // validation needs to be performed for ignored bits and bytes here.
2622        Ok(match src.tag.get_i32() {
2623            0 => Ok(unsafe { lift_option(cx, ok, &src.payload.ok)? }),
2624            1 => Err(unsafe { lift_option(cx, err, &src.payload.err)? }),
2625            _ => bail!("invalid expected discriminant"),
2626        })
2627    }
2628
2629    #[inline]
2630    fn linear_lift_from_memory(
2631        cx: &mut LiftContext<'_>,
2632        ty: InterfaceType,
2633        bytes: &[u8],
2634    ) -> Result<Self> {
2635        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2636        let discrim = bytes[0];
2637        let payload = &bytes[Self::INFO.payload_offset32 as usize..];
2638        let (ok, err) = match ty {
2639            InterfaceType::Result(ty) => {
2640                let ty = &cx.types[ty];
2641                (ty.ok, ty.err)
2642            }
2643            _ => bad_type_info(),
2644        };
2645        match discrim {
2646            0 => Ok(Ok(load_option(cx, ok, &payload[..T::SIZE32])?)),
2647            1 => Ok(Err(load_option(cx, err, &payload[..E::SIZE32])?)),
2648            _ => bail!("invalid expected discriminant"),
2649        }
2650    }
2651}
2652
2653fn lift_option<T>(cx: &mut LiftContext<'_>, ty: Option<InterfaceType>, src: &T::Lower) -> Result<T>
2654where
2655    T: Lift,
2656{
2657    match ty {
2658        Some(ty) => T::linear_lift_from_flat(cx, ty, src),
2659        None => Ok(empty_lift()),
2660    }
2661}
2662
2663fn load_option<T>(cx: &mut LiftContext<'_>, ty: Option<InterfaceType>, bytes: &[u8]) -> Result<T>
2664where
2665    T: Lift,
2666{
2667    match ty {
2668        Some(ty) => T::linear_lift_from_memory(cx, ty, bytes),
2669        None => Ok(empty_lift()),
2670    }
2671}
2672
2673fn empty_lift<T>() -> T
2674where
2675    T: Lift,
2676{
2677    assert!(T::IS_RUST_UNIT_TYPE);
2678    assert_eq!(mem::size_of::<T>(), 0);
2679    unsafe { MaybeUninit::uninit().assume_init() }
2680}
2681
2682/// Helper structure to define `Lower` for tuples below.
2683///
2684/// Uses default type parameters to have fields be zero-sized and not present
2685/// in memory for smaller tuple values.
2686#[expect(non_snake_case, reason = "more amenable to macro-generated code")]
2687#[doc(hidden)]
2688#[derive(Clone, Copy)]
2689#[repr(C)]
2690pub struct TupleLower<
2691    T1 = (),
2692    T2 = (),
2693    T3 = (),
2694    T4 = (),
2695    T5 = (),
2696    T6 = (),
2697    T7 = (),
2698    T8 = (),
2699    T9 = (),
2700    T10 = (),
2701    T11 = (),
2702    T12 = (),
2703    T13 = (),
2704    T14 = (),
2705    T15 = (),
2706    T16 = (),
2707    T17 = (),
2708> {
2709    // NB: these names match the names in `for_each_function_signature!`
2710    A1: T1,
2711    A2: T2,
2712    A3: T3,
2713    A4: T4,
2714    A5: T5,
2715    A6: T6,
2716    A7: T7,
2717    A8: T8,
2718    A9: T9,
2719    A10: T10,
2720    A11: T11,
2721    A12: T12,
2722    A13: T13,
2723    A14: T14,
2724    A15: T15,
2725    A16: T16,
2726    A17: T17,
2727    _align_tuple_lower0_correctly: [ValRaw; 0],
2728}
2729
2730macro_rules! impl_component_ty_for_tuples {
2731    ($n:tt $($t:ident)*) => {
2732        #[allow(non_snake_case, reason = "macro-generated code")]
2733        unsafe impl<$($t,)*> ComponentType for ($($t,)*)
2734            where $($t: ComponentType),*
2735        {
2736            type Lower = TupleLower<$($t::Lower),*>;
2737
2738            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::record_static(&[
2739                $($t::ABI),*
2740            ]);
2741
2742            const IS_RUST_UNIT_TYPE: bool = {
2743                let mut _is_unit = true;
2744                $(
2745                    let _anything_to_bind_the_macro_variable = $t::IS_RUST_UNIT_TYPE;
2746                    _is_unit = false;
2747                )*
2748                _is_unit
2749            };
2750
2751            fn typecheck(
2752                ty: &InterfaceType,
2753                types: &InstanceType<'_>,
2754            ) -> Result<()> {
2755                typecheck_tuple(ty, types, &[$($t::typecheck),*])
2756            }
2757        }
2758
2759        #[allow(non_snake_case, reason = "macro-generated code")]
2760        unsafe impl<$($t,)*> Lower for ($($t,)*)
2761            where $($t: Lower),*
2762        {
2763            fn linear_lower_to_flat<U>(
2764                &self,
2765                cx: &mut LowerContext<'_, U>,
2766                ty: InterfaceType,
2767                _dst: &mut MaybeUninit<Self::Lower>,
2768            ) -> Result<()> {
2769                let types = match ty {
2770                    InterfaceType::Tuple(t) => &cx.types[t].types,
2771                    _ => bad_type_info(),
2772                };
2773                let ($($t,)*) = self;
2774                let mut _types = types.iter();
2775                $(
2776                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2777                    $t.linear_lower_to_flat(cx, ty, map_maybe_uninit!(_dst.$t))?;
2778                )*
2779                Ok(())
2780            }
2781
2782            fn linear_lower_to_memory<U>(
2783                &self,
2784                cx: &mut LowerContext<'_, U>,
2785                ty: InterfaceType,
2786                mut _offset: usize,
2787            ) -> Result<()> {
2788                debug_assert!(_offset % (Self::ALIGN32 as usize) == 0);
2789                let types = match ty {
2790                    InterfaceType::Tuple(t) => &cx.types[t].types,
2791                    _ => bad_type_info(),
2792                };
2793                let ($($t,)*) = self;
2794                let mut _types = types.iter();
2795                $(
2796                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2797                    $t.linear_lower_to_memory(cx, ty, $t::ABI.next_field32_size(&mut _offset))?;
2798                )*
2799                Ok(())
2800            }
2801        }
2802
2803        #[allow(non_snake_case, reason = "macro-generated code")]
2804        unsafe impl<$($t,)*> Lift for ($($t,)*)
2805            where $($t: Lift),*
2806        {
2807            #[inline]
2808            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, _src: &Self::Lower) -> Result<Self> {
2809                let types = match ty {
2810                    InterfaceType::Tuple(t) => &cx.types[t].types,
2811                    _ => bad_type_info(),
2812                };
2813                let mut _types = types.iter();
2814                Ok(($(
2815                    $t::linear_lift_from_flat(
2816                        cx,
2817                        *_types.next().unwrap_or_else(bad_type_info),
2818                        &_src.$t,
2819                    )?,
2820                )*))
2821            }
2822
2823            #[inline]
2824            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
2825                debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2826                let types = match ty {
2827                    InterfaceType::Tuple(t) => &cx.types[t].types,
2828                    _ => bad_type_info(),
2829                };
2830                let mut _types = types.iter();
2831                let mut _offset = 0;
2832                $(
2833                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2834                    let $t = $t::linear_lift_from_memory(cx, ty, &bytes[$t::ABI.next_field32_size(&mut _offset)..][..$t::SIZE32])?;
2835                )*
2836                Ok(($($t,)*))
2837            }
2838        }
2839
2840        #[allow(non_snake_case, reason = "macro-generated code")]
2841        unsafe impl<$($t,)*> ComponentNamedList for ($($t,)*)
2842            where $($t: ComponentType),*
2843        {}
2844    };
2845}
2846
2847for_each_function_signature!(impl_component_ty_for_tuples);
2848
2849pub fn desc(ty: &InterfaceType) -> &'static str {
2850    match ty {
2851        InterfaceType::U8 => "u8",
2852        InterfaceType::S8 => "s8",
2853        InterfaceType::U16 => "u16",
2854        InterfaceType::S16 => "s16",
2855        InterfaceType::U32 => "u32",
2856        InterfaceType::S32 => "s32",
2857        InterfaceType::U64 => "u64",
2858        InterfaceType::S64 => "s64",
2859        InterfaceType::Float32 => "f32",
2860        InterfaceType::Float64 => "f64",
2861        InterfaceType::Bool => "bool",
2862        InterfaceType::Char => "char",
2863        InterfaceType::String => "string",
2864        InterfaceType::List(_) => "list",
2865        InterfaceType::Tuple(_) => "tuple",
2866        InterfaceType::Option(_) => "option",
2867        InterfaceType::Result(_) => "result",
2868
2869        InterfaceType::Record(_) => "record",
2870        InterfaceType::Variant(_) => "variant",
2871        InterfaceType::Flags(_) => "flags",
2872        InterfaceType::Enum(_) => "enum",
2873        InterfaceType::Own(_) => "owned resource",
2874        InterfaceType::Borrow(_) => "borrowed resource",
2875        InterfaceType::Future(_) => "future",
2876        InterfaceType::Stream(_) => "stream",
2877        InterfaceType::ErrorContext(_) => "error-context",
2878    }
2879}
2880
2881#[cold]
2882#[doc(hidden)]
2883pub fn bad_type_info<T>() -> T {
2884    // NB: should consider something like `unreachable_unchecked` here if this
2885    // becomes a performance bottleneck at some point, but that also comes with
2886    // a tradeoff of propagating a lot of unsafety, so it may not be worth it.
2887    panic!("bad type information detected");
2888}