wasmtime/runtime/component/func/
typed.rs

1use crate::component::Instance;
2use crate::component::func::{Func, LiftContext, LowerContext, Options};
3use crate::component::matching::InstanceType;
4use crate::component::storage::{storage_as_slice, storage_as_slice_mut};
5use crate::prelude::*;
6use crate::{AsContextMut, StoreContext, StoreContextMut, ValRaw};
7use alloc::borrow::Cow;
8use core::fmt;
9use core::iter;
10use core::marker;
11use core::mem::{self, MaybeUninit};
12use core::str;
13use wasmtime_environ::component::{
14    CanonicalAbiInfo, InterfaceType, MAX_FLAT_PARAMS, MAX_FLAT_RESULTS, StringEncoding, VariantInfo,
15};
16
17#[cfg(feature = "component-model-async")]
18use crate::component::concurrent::{self, AsAccessor, PreparedCall};
19
20/// A statically-typed version of [`Func`] which takes `Params` as input and
21/// returns `Return`.
22///
23/// This is an efficient way to invoke a WebAssembly component where if the
24/// inputs and output are statically known this can eschew the vast majority of
25/// machinery and checks when calling WebAssembly. This is the most optimized
26/// way to call a WebAssembly component.
27///
28/// Note that like [`Func`] this is a pointer within a [`Store`](crate::Store)
29/// and usage will panic if used with the wrong store.
30///
31/// This type is primarily created with the [`Func::typed`] API.
32///
33/// See [`ComponentType`] for more information about supported types.
34pub struct TypedFunc<Params, Return> {
35    func: Func,
36
37    // The definition of this field is somewhat subtle and may be surprising.
38    // Naively one might expect something like
39    //
40    //      _marker: marker::PhantomData<fn(Params) -> Return>,
41    //
42    // Since this is a function pointer after all. The problem with this
43    // definition though is that it imposes the wrong variance on `Params` from
44    // what we want. Abstractly a `fn(Params)` is able to store `Params` within
45    // it meaning you can only give it `Params` that live longer than the
46    // function pointer.
47    //
48    // With a component model function, however, we're always copying data from
49    // the host into the guest, so we are never storing pointers to `Params`
50    // into the guest outside the duration of a `call`, meaning we can actually
51    // accept values in `TypedFunc::call` which live for a shorter duration
52    // than the `Params` argument on the struct.
53    //
54    // This all means that we don't use a phantom function pointer, but instead
55    // feign phantom storage here to get the variance desired.
56    _marker: marker::PhantomData<(Params, Return)>,
57}
58
59impl<Params, Return> Copy for TypedFunc<Params, Return> {}
60
61impl<Params, Return> Clone for TypedFunc<Params, Return> {
62    fn clone(&self) -> TypedFunc<Params, Return> {
63        *self
64    }
65}
66
67impl<Params, Return> TypedFunc<Params, Return>
68where
69    Params: ComponentNamedList + Lower,
70    Return: ComponentNamedList + Lift,
71{
72    /// Creates a new [`TypedFunc`] from the provided component [`Func`],
73    /// unsafely asserting that the underlying function takes `Params` as
74    /// input and returns `Return`.
75    ///
76    /// # Unsafety
77    ///
78    /// This is an unsafe function because it does not verify that the [`Func`]
79    /// provided actually implements this signature. It's up to the caller to
80    /// have performed some other sort of check to ensure that the signature is
81    /// correct.
82    pub unsafe fn new_unchecked(func: Func) -> TypedFunc<Params, Return> {
83        TypedFunc {
84            _marker: marker::PhantomData,
85            func,
86        }
87    }
88
89    /// Returns the underlying un-typed [`Func`] that this [`TypedFunc`]
90    /// references.
91    pub fn func(&self) -> &Func {
92        &self.func
93    }
94
95    /// Calls the underlying WebAssembly component function using the provided
96    /// `params` as input.
97    ///
98    /// This method is used to enter into a component. Execution happens within
99    /// the `store` provided. The `params` are copied into WebAssembly memory
100    /// as appropriate and a core wasm function is invoked.
101    ///
102    /// # Post-return
103    ///
104    /// In the component model each function can have a "post return" specified
105    /// which allows cleaning up the arguments returned to the host. For example
106    /// if WebAssembly returns a string to the host then it might be a uniquely
107    /// allocated string which, after the host finishes processing it, needs to
108    /// be deallocated in the wasm instance's own linear memory to prevent
109    /// memory leaks in wasm itself. The `post-return` canonical abi option is
110    /// used to configured this.
111    ///
112    /// To accommodate this feature of the component model after invoking a
113    /// function via [`TypedFunc::call`] you must next invoke
114    /// [`TypedFunc::post_return`]. Note that the return value of the function
115    /// should be processed between these two function calls. The return value
116    /// continues to be usable from an embedder's perspective after
117    /// `post_return` is called, but after `post_return` is invoked it may no
118    /// longer retain the same value that the wasm module originally returned.
119    ///
120    /// Also note that [`TypedFunc::post_return`] must be invoked irrespective
121    /// of whether the canonical ABI option `post-return` was configured or not.
122    /// This means that embedders must unconditionally call
123    /// [`TypedFunc::post_return`] when a function returns. If this function
124    /// call returns an error, however, then [`TypedFunc::post_return`] is not
125    /// required.
126    ///
127    /// # Errors
128    ///
129    /// This function can return an error for a number of reasons:
130    ///
131    /// * If the wasm itself traps during execution.
132    /// * If the wasm traps while copying arguments into memory.
133    /// * If the wasm provides bad allocation pointers when copying arguments
134    ///   into memory.
135    /// * If the wasm returns a value which violates the canonical ABI.
136    /// * If this function's instances cannot be entered, for example if the
137    ///   instance is currently calling a host function.
138    /// * If a previous function call occurred and the corresponding
139    ///   `post_return` hasn't been invoked yet.
140    ///
141    /// In general there are many ways that things could go wrong when copying
142    /// types in and out of a wasm module with the canonical ABI, and certain
143    /// error conditions are specific to certain types. For example a
144    /// WebAssembly module can't return an invalid `char`. When allocating space
145    /// for this host to copy a string into the returned pointer must be
146    /// in-bounds in memory.
147    ///
148    /// If an error happens then the error should contain detailed enough
149    /// information to understand which part of the canonical ABI went wrong
150    /// and what to inspect.
151    ///
152    /// # Panics
153    ///
154    /// Panics if this is called on a function in an asynchronous store. This
155    /// only works with functions defined within a synchronous store. Also
156    /// panics if `store` does not own this function.
157    pub fn call(&self, store: impl AsContextMut, params: Params) -> Result<Return> {
158        assert!(
159            !store.as_context().async_support(),
160            "must use `call_async` when async support is enabled on the config"
161        );
162        self.call_impl(store, params)
163    }
164
165    /// Exactly like [`Self::call`], except for use on asynchronous stores.
166    ///
167    /// # Panics
168    ///
169    /// Panics if this is called on a function in a synchronous store. This
170    /// only works with functions defined within an asynchronous store. Also
171    /// panics if `store` does not own this function.
172    #[cfg(feature = "async")]
173    pub async fn call_async(
174        &self,
175        mut store: impl AsContextMut<Data: Send>,
176        params: Params,
177    ) -> Result<Return>
178    where
179        Return: 'static,
180    {
181        let mut store = store.as_context_mut();
182        assert!(
183            store.0.async_support(),
184            "cannot use `call_async` when async support is not enabled on the config"
185        );
186        #[cfg(feature = "component-model-async")]
187        {
188            use crate::component::concurrent::TaskId;
189            use crate::runtime::vm::SendSyncPtr;
190            use core::ptr::NonNull;
191
192            let ptr = SendSyncPtr::from(NonNull::from(&params).cast::<u8>());
193            let prepared =
194                self.prepare_call(store.as_context_mut(), false, false, move |cx, ty, dst| {
195                    // SAFETY: The goal here is to get `Params`, a non-`'static`
196                    // value, to live long enough to the lowering of the
197                    // parameters. We're guaranteed that `Params` lives in the
198                    // future of the outer function (we're in an `async fn`) so it'll
199                    // stay alive as long as the future itself. That is distinct,
200                    // for example, from the signature of `call_concurrent` below.
201                    //
202                    // Here a pointer to `Params` is smuggled to this location
203                    // through a `SendSyncPtr<u8>` to thwart the `'static` check
204                    // of rustc and the signature of `prepare_call`.
205                    //
206                    // Note the use of `RemoveOnDrop` in the code that follows
207                    // this closure, which ensures that the task will be removed
208                    // from the concurrent state to which it belongs when the
209                    // containing `Future` is dropped, thereby ensuring that
210                    // this closure will never be called if it hasn't already,
211                    // meaning it will never see a dangling pointer.
212                    let params = unsafe { ptr.cast::<Params>().as_ref() };
213                    Self::lower_args(cx, ty, dst, params)
214                })?;
215
216            struct RemoveOnDrop<'a, T: 'static> {
217                store: StoreContextMut<'a, T>,
218                task: TaskId,
219            }
220
221            impl<'a, T> Drop for RemoveOnDrop<'a, T> {
222                fn drop(&mut self) {
223                    self.task.remove(self.store.as_context_mut()).unwrap();
224                }
225            }
226
227            let mut wrapper = RemoveOnDrop {
228                store,
229                task: prepared.task_id(),
230            };
231
232            let result = concurrent::queue_call(wrapper.store.as_context_mut(), prepared)?;
233            self.func
234                .instance
235                .run_concurrent(wrapper.store.as_context_mut(), async |_| result.await)
236                .await?
237        }
238        #[cfg(not(feature = "component-model-async"))]
239        {
240            store
241                .on_fiber(|store| self.call_impl(store, params))
242                .await?
243        }
244    }
245
246    /// Start a concurrent call to this function.
247    ///
248    /// Unlike [`Self::call`] and [`Self::call_async`] (both of which require
249    /// exclusive access to the store until the completion of the call), calls
250    /// made using this method may run concurrently with other calls to the same
251    /// instance.  In addition, the runtime will call the `post-return` function
252    /// (if any) automatically when the guest task completes -- no need to
253    /// explicitly call `Func::post_return` afterward.
254    ///
255    /// # Panics
256    ///
257    /// Panics if the store that the [`Accessor`] is derived from does not own
258    /// this function.
259    #[cfg(feature = "component-model-async")]
260    pub async fn call_concurrent(
261        self,
262        accessor: impl AsAccessor<Data: Send>,
263        params: Params,
264    ) -> Result<Return>
265    where
266        Params: 'static,
267        Return: 'static,
268    {
269        let accessor = accessor.as_accessor();
270        let result = accessor.with(|mut store| {
271            let mut store = store.as_context_mut();
272            assert!(
273                store.0.async_support(),
274                "cannot use `call_concurrent` when async support is not enabled on the config"
275            );
276
277            let prepared =
278                self.prepare_call(store.as_context_mut(), true, true, move |cx, ty, dst| {
279                    Self::lower_args(cx, ty, dst, &params)
280                })?;
281            concurrent::queue_call(store, prepared)
282        });
283
284        result?.await
285    }
286
287    fn lower_args<T>(
288        cx: &mut LowerContext<T>,
289        ty: InterfaceType,
290        dst: &mut [MaybeUninit<ValRaw>],
291        params: &Params,
292    ) -> Result<()> {
293        use crate::component::storage::slice_to_storage_mut;
294
295        if Params::flatten_count() <= MAX_FLAT_PARAMS {
296            // SAFETY: the safety of `slice_to_storage_mut` relies on
297            // `Params::Lower` being represented by a sequence of
298            // `ValRaw`, and that's a guarantee upheld by the `Lower`
299            // trait itself.
300            let dst: &mut MaybeUninit<Params::Lower> = unsafe { slice_to_storage_mut(dst) };
301            Self::lower_stack_args(cx, &params, ty, dst)
302        } else {
303            Self::lower_heap_args(cx, &params, ty, &mut dst[0])
304        }
305    }
306
307    /// Calls `concurrent::prepare_call` with monomorphized functions for
308    /// lowering the parameters and lifting the result according to the number
309    /// of core Wasm parameters and results in the signature of the function to
310    /// be called.
311    #[cfg(feature = "component-model-async")]
312    fn prepare_call<T>(
313        self,
314        store: StoreContextMut<'_, T>,
315        remove_task_automatically: bool,
316        call_post_return_automatically: bool,
317        lower: impl FnOnce(
318            &mut LowerContext<T>,
319            InterfaceType,
320            &mut [MaybeUninit<ValRaw>],
321        ) -> Result<()>
322        + Send
323        + Sync
324        + 'static,
325    ) -> Result<PreparedCall<Return>>
326    where
327        Return: 'static,
328    {
329        use crate::component::storage::slice_to_storage;
330
331        let param_count = if Params::flatten_count() <= MAX_FLAT_PARAMS {
332            Params::flatten_count()
333        } else {
334            1
335        };
336        let max_results = if self.func.abi_async(store.0) {
337            MAX_FLAT_PARAMS
338        } else {
339            MAX_FLAT_RESULTS
340        };
341        concurrent::prepare_call(
342            store,
343            self.func,
344            param_count,
345            remove_task_automatically,
346            call_post_return_automatically,
347            move |func, store, params_out| {
348                func.with_lower_context(store, call_post_return_automatically, |cx, ty| {
349                    lower(cx, ty, params_out)
350                })
351            },
352            move |func, store, results| {
353                let result = if Return::flatten_count() <= max_results {
354                    func.with_lift_context(store, |cx, ty| {
355                        // SAFETY: Per the safety requiments documented for the
356                        // `ComponentType` trait, `Return::Lower` must be
357                        // compatible at the binary level with a `[ValRaw; N]`,
358                        // where `N` is `mem::size_of::<Return::Lower>() /
359                        // mem::size_of::<ValRaw>()`.  And since this function
360                        // is only used when `Return::flatten_count() <=
361                        // MAX_FLAT_RESULTS` and `MAX_FLAT_RESULTS == 1`, `N`
362                        // can only either be 0 or 1.
363                        //
364                        // See `ComponentInstance::exit_call` for where we use
365                        // the result count passed from
366                        // `wasmtime_environ::fact::trampoline`-generated code
367                        // to ensure the slice has the correct length, and also
368                        // `concurrent::start_call` for where we conservatively
369                        // use a slice length of 1 unconditionally.  Also note
370                        // that, as of this writing `slice_to_storage`
371                        // double-checks the slice length is sufficient.
372                        let results: &Return::Lower = unsafe { slice_to_storage(results) };
373                        Self::lift_stack_result(cx, ty, results)
374                    })?
375                } else {
376                    func.with_lift_context(store, |cx, ty| {
377                        Self::lift_heap_result(cx, ty, &results[0])
378                    })?
379                };
380                Ok(Box::new(result))
381            },
382        )
383    }
384
385    fn call_impl(&self, mut store: impl AsContextMut, params: Params) -> Result<Return> {
386        let store = store.as_context_mut();
387
388        if self.func.abi_async(store.0) {
389            bail!("must enable the `component-model-async` feature to call async-lifted exports")
390        }
391
392        // Note that this is in theory simpler than it might read at this time.
393        // Here we're doing a runtime dispatch on the `flatten_count` for the
394        // params/results to see whether they're inbounds. This creates 4 cases
395        // to handle. In reality this is a highly optimizable branch where LLVM
396        // will easily figure out that only one branch here is taken.
397        //
398        // Otherwise this current construction is done to ensure that the stack
399        // space reserved for the params/results is always of the appropriate
400        // size (as the params/results needed differ depending on the "flatten"
401        // count)
402        //
403        // SAFETY: the safety of these invocations of `call_raw` depends on the
404        // correctness of the ascription of the `LowerParams` and `LowerReturn`
405        // types on the `call_raw` function. That's upheld here through the
406        // safety requirements of `Lift` and `Lower` on `Params` and `Return` in
407        // combination with checking the various possible branches here and
408        // dispatching to appropriately typed functions.
409        unsafe {
410            // This type is used as `LowerParams` for `call_raw` which is either
411            // `Params::Lower` or `ValRaw` representing it's either on the stack
412            // or it's on the heap. This allocates 1 extra `ValRaw` on the stack
413            // if `Params` is empty and `Return` is also empty, but that's a
414            // reasonable enough price to pay for now given the current code
415            // organization.
416            #[derive(Copy, Clone)]
417            union Union<T: Copy, U: Copy> {
418                _a: T,
419                _b: U,
420            }
421
422            if Return::flatten_count() <= MAX_FLAT_RESULTS {
423                self.func.call_raw(
424                    store,
425                    |cx, ty, dst: &mut MaybeUninit<Union<Params::Lower, ValRaw>>| {
426                        let dst = storage_as_slice_mut(dst);
427                        Self::lower_args(cx, ty, dst, &params)
428                    },
429                    Self::lift_stack_result,
430                )
431            } else {
432                self.func.call_raw(
433                    store,
434                    |cx, ty, dst: &mut MaybeUninit<Union<Params::Lower, ValRaw>>| {
435                        let dst = storage_as_slice_mut(dst);
436                        Self::lower_args(cx, ty, dst, &params)
437                    },
438                    Self::lift_heap_result,
439                )
440            }
441        }
442    }
443
444    /// Lower parameters directly onto the stack specified by the `dst`
445    /// location.
446    ///
447    /// This is only valid to call when the "flatten count" is small enough, or
448    /// when the canonical ABI says arguments go through the stack rather than
449    /// the heap.
450    fn lower_stack_args<T>(
451        cx: &mut LowerContext<'_, T>,
452        params: &Params,
453        ty: InterfaceType,
454        dst: &mut MaybeUninit<Params::Lower>,
455    ) -> Result<()> {
456        assert!(Params::flatten_count() <= MAX_FLAT_PARAMS);
457        params.linear_lower_to_flat(cx, ty, dst)?;
458        Ok(())
459    }
460
461    /// Lower parameters onto a heap-allocated location.
462    ///
463    /// This is used when the stack space to be used for the arguments is above
464    /// the `MAX_FLAT_PARAMS` threshold. Here the wasm's `realloc` function is
465    /// invoked to allocate space and then parameters are stored at that heap
466    /// pointer location.
467    fn lower_heap_args<T>(
468        cx: &mut LowerContext<'_, T>,
469        params: &Params,
470        ty: InterfaceType,
471        dst: &mut MaybeUninit<ValRaw>,
472    ) -> Result<()> {
473        // Memory must exist via validation if the arguments are stored on the
474        // heap, so we can create a `MemoryMut` at this point. Afterwards
475        // `realloc` is used to allocate space for all the arguments and then
476        // they're all stored in linear memory.
477        //
478        // Note that `realloc` will bake in a check that the returned pointer is
479        // in-bounds.
480        let ptr = cx.realloc(0, 0, Params::ALIGN32, Params::SIZE32)?;
481        params.linear_lower_to_memory(cx, ty, ptr)?;
482
483        // Note that the pointer here is stored as a 64-bit integer. This allows
484        // this to work with either 32 or 64-bit memories. For a 32-bit memory
485        // it'll just ignore the upper 32 zero bits, and for 64-bit memories
486        // this'll have the full 64-bits. Note that for 32-bit memories the call
487        // to `realloc` above guarantees that the `ptr` is in-bounds meaning
488        // that we will know that the zero-extended upper bits of `ptr` are
489        // guaranteed to be zero.
490        //
491        // This comment about 64-bit integers is also referred to below with
492        // "WRITEPTR64".
493        dst.write(ValRaw::i64(ptr as i64));
494
495        Ok(())
496    }
497
498    /// Lift the result of a function directly from the stack result.
499    ///
500    /// This is only used when the result fits in the maximum number of stack
501    /// slots.
502    fn lift_stack_result(
503        cx: &mut LiftContext<'_>,
504        ty: InterfaceType,
505        dst: &Return::Lower,
506    ) -> Result<Return> {
507        Return::linear_lift_from_flat(cx, ty, dst)
508    }
509
510    /// Lift the result of a function where the result is stored indirectly on
511    /// the heap.
512    fn lift_heap_result(
513        cx: &mut LiftContext<'_>,
514        ty: InterfaceType,
515        dst: &ValRaw,
516    ) -> Result<Return> {
517        assert!(Return::flatten_count() > MAX_FLAT_RESULTS);
518        // FIXME(#4311): needs to read an i64 for memory64
519        let ptr = usize::try_from(dst.get_u32())?;
520        if ptr % usize::try_from(Return::ALIGN32)? != 0 {
521            bail!("return pointer not aligned");
522        }
523
524        let bytes = cx
525            .memory()
526            .get(ptr..)
527            .and_then(|b| b.get(..Return::SIZE32))
528            .ok_or_else(|| anyhow::anyhow!("pointer out of bounds of memory"))?;
529        Return::linear_lift_from_memory(cx, ty, bytes)
530    }
531
532    /// See [`Func::post_return`]
533    pub fn post_return(&self, store: impl AsContextMut) -> Result<()> {
534        self.func.post_return(store)
535    }
536
537    /// See [`Func::post_return_async`]
538    #[cfg(feature = "async")]
539    pub async fn post_return_async<T: Send>(
540        &self,
541        store: impl AsContextMut<Data = T>,
542    ) -> Result<()> {
543        self.func.post_return_async(store).await
544    }
545}
546
547/// A trait representing a static list of named types that can be passed to or
548/// returned from a [`TypedFunc`].
549///
550/// This trait is implemented for a number of tuple types and is not expected
551/// to be implemented externally. The contents of this trait are hidden as it's
552/// intended to be an implementation detail of Wasmtime. The contents of this
553/// trait are not covered by Wasmtime's stability guarantees.
554///
555/// For more information about this trait see [`Func::typed`] and
556/// [`TypedFunc`].
557//
558// Note that this is an `unsafe` trait, and the unsafety means that
559// implementations of this trait must be correct or otherwise [`TypedFunc`]
560// would not be memory safe. The main reason this is `unsafe` is the
561// `typecheck` function which must operate correctly relative to the `AsTuple`
562// interpretation of the implementor.
563pub unsafe trait ComponentNamedList: ComponentType {}
564
565/// A trait representing types which can be passed to and read from components
566/// with the canonical ABI.
567///
568/// This trait is implemented for Rust types which can be communicated to
569/// components. The [`Func::typed`] and [`TypedFunc`] Rust items are the main
570/// consumers of this trait.
571///
572/// Supported Rust types include:
573///
574/// | Component Model Type              | Rust Type                            |
575/// |-----------------------------------|--------------------------------------|
576/// | `{s,u}{8,16,32,64}`               | `{i,u}{8,16,32,64}`                  |
577/// | `f{32,64}`                        | `f{32,64}`                           |
578/// | `bool`                            | `bool`                               |
579/// | `char`                            | `char`                               |
580/// | `tuple<A, B>`                     | `(A, B)`                             |
581/// | `option<T>`                       | `Option<T>`                          |
582/// | `result`                          | `Result<(), ()>`                     |
583/// | `result<T>`                       | `Result<T, ()>`                      |
584/// | `result<_, E>`                    | `Result<(), E>`                      |
585/// | `result<T, E>`                    | `Result<T, E>`                       |
586/// | `string`                          | `String`, `&str`, or [`WasmStr`]     |
587/// | `list<T>`                         | `Vec<T>`, `&[T]`, or [`WasmList`]    |
588/// | `own<T>`, `borrow<T>`             | [`Resource<T>`] or [`ResourceAny`]   |
589/// | `record`                          | [`#[derive(ComponentType)]`][d-cm]   |
590/// | `variant`                         | [`#[derive(ComponentType)]`][d-cm]   |
591/// | `enum`                            | [`#[derive(ComponentType)]`][d-cm]   |
592/// | `flags`                           | [`flags!`][f-m]                      |
593///
594/// [`Resource<T>`]: crate::component::Resource
595/// [`ResourceAny`]: crate::component::ResourceAny
596/// [d-cm]: macro@crate::component::ComponentType
597/// [f-m]: crate::component::flags
598///
599/// Rust standard library pointers such as `&T`, `Box<T>`, and `Arc<T>`
600/// additionally represent whatever type `T` represents in the component model.
601/// Note that types such as `record`, `variant`, `enum`, and `flags` are
602/// generated by the embedder at compile time. These macros derive
603/// implementation of this trait for custom types to map to custom types in the
604/// component model. Note that for `record`, `variant`, `enum`, and `flags`
605/// those types are often generated by the
606/// [`bindgen!`](crate::component::bindgen) macro from WIT definitions.
607///
608/// Types that implement [`ComponentType`] are used for `Params` and `Return`
609/// in [`TypedFunc`] and [`Func::typed`].
610///
611/// The contents of this trait are hidden as it's intended to be an
612/// implementation detail of Wasmtime. The contents of this trait are not
613/// covered by Wasmtime's stability guarantees.
614///
615/// # Safety
616///
617/// Note that this is an `unsafe` trait as `TypedFunc`'s safety heavily relies on
618/// the correctness of the implementations of this trait. Some ways in which this
619/// trait must be correct to be safe are:
620///
621/// * The `Lower` associated type must be a `ValRaw` sequence. It doesn't have to
622///   literally be `[ValRaw; N]` but when laid out in memory it must be adjacent
623///   `ValRaw` values and have a multiple of the size of `ValRaw` and the same
624///   alignment.
625///
626/// * The `lower` function must initialize the bits within `Lower` that are going
627///   to be read by the trampoline that's used to enter core wasm. A trampoline
628///   is passed `*mut Lower` and will read the canonical abi arguments in
629///   sequence, so all of the bits must be correctly initialized.
630///
631/// * The `size` and `align` functions must be correct for this value stored in
632///   the canonical ABI. The `Cursor<T>` iteration of these bytes rely on this
633///   for correctness as they otherwise eschew bounds-checking.
634///
635/// There are likely some other correctness issues which aren't documented as
636/// well, this isn't currently an exhaustive list. It suffices to say, though,
637/// that correctness bugs in this trait implementation are highly likely to
638/// lead to security bugs, which again leads to the `unsafe` in the trait.
639///
640/// Note that this trait specifically is not sealed because `bindgen!`-generated
641/// types must be able to implement this trait using a `#[derive]` macro. For
642/// users it's recommended to not implement this trait manually given the
643/// non-exhaustive list of safety requirements that must be upheld. This trait
644/// is implemented at your own risk if you do so.
645///
646/// # Send and Sync
647///
648/// While on the topic of safety it's worth discussing the `Send` and `Sync`
649/// bounds here as well. These bounds might naively seem like they shouldn't be
650/// required for all component types as they're host-level types not guest-level
651/// types persisted anywhere. Various subtleties lead to these bounds, however:
652///
653/// * Fibers require that all stack-local variables are `Send` and `Sync` for
654///   fibers themselves to be send/sync. Unfortunately we have no help from the
655///   compiler on this one so it's up to Wasmtime's discipline to maintain this.
656///   One instance of this is that return values are placed on the stack as
657///   they're lowered into guest memory. This lowering operation can involve
658///   malloc and context switches, so return values must be Send/Sync.
659///
660/// * In the implementation of component model async it's not uncommon for types
661///   to be "buffered" in the store temporarily. For example parameters might
662///   reside in a store temporarily while wasm has backpressure turned on.
663///
664/// Overall it's generally easiest to require `Send` and `Sync` for all
665/// component types. There additionally aren't known use case for non-`Send` or
666/// non-`Sync` types at this time.
667pub unsafe trait ComponentType: Send + Sync {
668    /// Representation of the "lowered" form of this component value.
669    ///
670    /// Lowerings lower into core wasm values which are represented by `ValRaw`.
671    /// This `Lower` type must be a list of `ValRaw` as either a literal array
672    /// or a struct where every field is a `ValRaw`. This must be `Copy` (as
673    /// `ValRaw` is `Copy`) and support all byte patterns. This being correct is
674    /// one reason why the trait is unsafe.
675    #[doc(hidden)]
676    type Lower: Copy;
677
678    /// The information about this type's canonical ABI (size/align/etc).
679    #[doc(hidden)]
680    const ABI: CanonicalAbiInfo;
681
682    #[doc(hidden)]
683    const SIZE32: usize = Self::ABI.size32 as usize;
684    #[doc(hidden)]
685    const ALIGN32: u32 = Self::ABI.align32;
686
687    #[doc(hidden)]
688    const IS_RUST_UNIT_TYPE: bool = false;
689
690    /// Whether this type might require a call to the guest's realloc function
691    /// to allocate linear memory when lowering (e.g. a non-empty `string`).
692    ///
693    /// If this is `false`, Wasmtime may optimize lowering by using
694    /// `LowerContext::new_without_realloc` and lowering values outside of any
695    /// fiber.  That will panic if the lowering process ends up needing realloc
696    /// after all, so `true` is a conservative default.
697    #[doc(hidden)]
698    const MAY_REQUIRE_REALLOC: bool = true;
699
700    /// Returns the number of core wasm abi values will be used to represent
701    /// this type in its lowered form.
702    ///
703    /// This divides the size of `Self::Lower` by the size of `ValRaw`.
704    #[doc(hidden)]
705    fn flatten_count() -> usize {
706        assert!(mem::size_of::<Self::Lower>() % mem::size_of::<ValRaw>() == 0);
707        assert!(mem::align_of::<Self::Lower>() == mem::align_of::<ValRaw>());
708        mem::size_of::<Self::Lower>() / mem::size_of::<ValRaw>()
709    }
710
711    /// Performs a type-check to see whether this component value type matches
712    /// the interface type `ty` provided.
713    #[doc(hidden)]
714    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()>;
715}
716
717#[doc(hidden)]
718pub unsafe trait ComponentVariant: ComponentType {
719    const CASES: &'static [Option<CanonicalAbiInfo>];
720    const INFO: VariantInfo = VariantInfo::new_static(Self::CASES);
721    const PAYLOAD_OFFSET32: usize = Self::INFO.payload_offset32 as usize;
722}
723
724/// Host types which can be passed to WebAssembly components.
725///
726/// This trait is implemented for all types that can be passed to components
727/// either as parameters of component exports or returns of component imports.
728/// This trait represents the ability to convert from the native host
729/// representation to the canonical ABI.
730///
731/// Built-in types to Rust such as `Option<T>` implement this trait as
732/// appropriate. For a mapping of component model to Rust types see
733/// [`ComponentType`].
734///
735/// For user-defined types, for example `record` types mapped to Rust `struct`s,
736/// this crate additionally has
737/// [`#[derive(Lower)]`](macro@crate::component::Lower).
738///
739/// Note that like [`ComponentType`] the definition of this trait is intended to
740/// be an internal implementation detail of Wasmtime at this time. It's
741/// recommended to use the `#[derive(Lower)]` implementation instead.
742pub unsafe trait Lower: ComponentType {
743    /// Performs the "lower" function in the linear memory version of the
744    /// canonical ABI.
745    ///
746    /// This method will lower the current value into a component. The `lower`
747    /// function performs a "flat" lowering into the `dst` specified which is
748    /// allowed to be uninitialized entering this method but is guaranteed to be
749    /// fully initialized if the method returns `Ok(())`.
750    ///
751    /// The `cx` context provided is the context within which this lowering is
752    /// happening. This contains information such as canonical options specified
753    /// (e.g. string encodings, memories, etc), the store itself, along with
754    /// type information.
755    ///
756    /// The `ty` parameter is the destination type that is being lowered into.
757    /// For example this is the component's "view" of the type that is being
758    /// lowered. This is guaranteed to have passed a `typecheck` earlier.
759    ///
760    /// This will only be called if `typecheck` passes for `Op::Lower`.
761    #[doc(hidden)]
762    fn linear_lower_to_flat<T>(
763        &self,
764        cx: &mut LowerContext<'_, T>,
765        ty: InterfaceType,
766        dst: &mut MaybeUninit<Self::Lower>,
767    ) -> Result<()>;
768
769    /// Performs the "store" operation in the linear memory version of the
770    /// canonical ABI.
771    ///
772    /// This function will store `self` into the linear memory described by
773    /// `cx` at the `offset` provided.
774    ///
775    /// It is expected that `offset` is a valid offset in memory for
776    /// `Self::SIZE32` bytes. At this time that's not an unsafe contract as it's
777    /// always re-checked on all stores, but this is something that will need to
778    /// be improved in the future to remove extra bounds checks. For now this
779    /// function will panic if there's a bug and `offset` isn't valid within
780    /// memory.
781    ///
782    /// The `ty` type information passed here is the same as the type
783    /// information passed to `lower` above, and is the component's own view of
784    /// what the resulting type should be.
785    ///
786    /// This will only be called if `typecheck` passes for `Op::Lower`.
787    #[doc(hidden)]
788    fn linear_lower_to_memory<T>(
789        &self,
790        cx: &mut LowerContext<'_, T>,
791        ty: InterfaceType,
792        offset: usize,
793    ) -> Result<()>;
794
795    /// Provided method to lower a list of `Self` into memory.
796    ///
797    /// Requires that `offset` has already been checked for alignment and
798    /// validity in terms of being in-bounds, otherwise this may panic.
799    ///
800    /// This is primarily here to get overridden for implementations of integers
801    /// which can avoid some extra fluff and use a pattern that's more easily
802    /// optimizable by LLVM.
803    #[doc(hidden)]
804    fn linear_store_list_to_memory<T>(
805        cx: &mut LowerContext<'_, T>,
806        ty: InterfaceType,
807        mut offset: usize,
808        items: &[Self],
809    ) -> Result<()>
810    where
811        Self: Sized,
812    {
813        for item in items {
814            item.linear_lower_to_memory(cx, ty, offset)?;
815            offset += Self::SIZE32;
816        }
817        Ok(())
818    }
819}
820
821/// Host types which can be created from the canonical ABI.
822///
823/// This is the mirror of the [`Lower`] trait where it represents the capability
824/// of acquiring items from WebAssembly and passing them to the host.
825///
826/// Built-in types to Rust such as `Option<T>` implement this trait as
827/// appropriate. For a mapping of component model to Rust types see
828/// [`ComponentType`].
829///
830/// For user-defined types, for example `record` types mapped to Rust `struct`s,
831/// this crate additionally has
832/// [`#[derive(Lift)]`](macro@crate::component::Lift).
833///
834/// Note that like [`ComponentType`] the definition of this trait is intended to
835/// be an internal implementation detail of Wasmtime at this time. It's
836/// recommended to use the `#[derive(Lift)]` implementation instead.
837pub unsafe trait Lift: Sized + ComponentType {
838    /// Performs the "lift" operation in the linear memory version of the
839    /// canonical ABI.
840    ///
841    /// This function performs a "flat" lift operation from the `src` specified
842    /// which is a sequence of core wasm values. The lifting operation will
843    /// validate core wasm values and produce a `Self` on success.
844    ///
845    /// The `cx` provided contains contextual information such as the store
846    /// that's being loaded from, canonical options, and type information.
847    ///
848    /// The `ty` parameter is the origin component's specification for what the
849    /// type that is being lifted is. For example this is the record type or the
850    /// resource type that is being lifted.
851    ///
852    /// Note that this has a default implementation but if `typecheck` passes
853    /// for `Op::Lift` this needs to be overridden.
854    #[doc(hidden)]
855    fn linear_lift_from_flat(
856        cx: &mut LiftContext<'_>,
857        ty: InterfaceType,
858        src: &Self::Lower,
859    ) -> Result<Self>;
860
861    /// Performs the "load" operation in the linear memory version of the
862    /// canonical ABI.
863    ///
864    /// This will read the `bytes` provided, which are a sub-slice into the
865    /// linear memory described by `cx`. The `bytes` array provided is
866    /// guaranteed to be `Self::SIZE32` bytes large. All of memory is then also
867    /// available through `cx` for bounds-checks and such as necessary for
868    /// strings/lists.
869    ///
870    /// The `ty` argument is the type that's being loaded, as described by the
871    /// original component.
872    ///
873    /// Note that this has a default implementation but if `typecheck` passes
874    /// for `Op::Lift` this needs to be overridden.
875    #[doc(hidden)]
876    fn linear_lift_from_memory(
877        cx: &mut LiftContext<'_>,
878        ty: InterfaceType,
879        bytes: &[u8],
880    ) -> Result<Self>;
881
882    /// Converts `list` into a `Vec<T>`, used in `Lift for Vec<T>`.
883    #[doc(hidden)]
884    fn linear_lift_list_from_memory(
885        cx: &mut LiftContext<'_>,
886        list: &WasmList<Self>,
887    ) -> Result<Vec<Self>>
888    where
889        Self: Sized,
890    {
891        let mut dst = Vec::with_capacity(list.len);
892        Self::linear_lift_into_from_memory(cx, list, &mut dst)?;
893        Ok(dst)
894    }
895
896    /// Load no more than `max_count` items from `list` into `dst`.
897    ///
898    /// This is primarily here to get overridden for implementations of integers
899    /// which can avoid some extra fluff and use a pattern that's more easily
900    /// optimizable by LLVM.
901    #[doc(hidden)]
902    fn linear_lift_into_from_memory(
903        cx: &mut LiftContext<'_>,
904        list: &WasmList<Self>,
905        dst: &mut impl Extend<Self>,
906    ) -> Result<()>
907    where
908        Self: Sized,
909    {
910        for i in 0..list.len {
911            dst.extend(Some(list.get_from_store(cx, i).unwrap()?));
912        }
913        Ok(())
914    }
915}
916
917// Macro to help generate "forwarding implementations" of `ComponentType` to
918// another type, used for wrappers in Rust like `&T`, `Box<T>`, etc. Note that
919// these wrappers only implement lowering because lifting native Rust types
920// cannot be done.
921macro_rules! forward_type_impls {
922    ($(($($generics:tt)*) $a:ty => $b:ty,)*) => ($(
923        unsafe impl <$($generics)*> ComponentType for $a {
924            type Lower = <$b as ComponentType>::Lower;
925
926            const ABI: CanonicalAbiInfo = <$b as ComponentType>::ABI;
927
928            #[inline]
929            fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
930                <$b as ComponentType>::typecheck(ty, types)
931            }
932        }
933    )*)
934}
935
936forward_type_impls! {
937    (T: ComponentType + ?Sized) &'_ T => T,
938    (T: ComponentType + ?Sized) Box<T> => T,
939    (T: ComponentType + ?Sized) alloc::sync::Arc<T> => T,
940    () String => str,
941    (T: ComponentType) Vec<T> => [T],
942}
943
944macro_rules! forward_lowers {
945    ($(($($generics:tt)*) $a:ty => $b:ty,)*) => ($(
946        unsafe impl <$($generics)*> Lower for $a {
947            fn linear_lower_to_flat<U>(
948                &self,
949                cx: &mut LowerContext<'_, U>,
950                ty: InterfaceType,
951                dst: &mut MaybeUninit<Self::Lower>,
952            ) -> Result<()> {
953                <$b as Lower>::linear_lower_to_flat(self, cx, ty, dst)
954            }
955
956            fn linear_lower_to_memory<U>(
957                &self,
958                cx: &mut LowerContext<'_, U>,
959                ty: InterfaceType,
960                offset: usize,
961            ) -> Result<()> {
962                <$b as Lower>::linear_lower_to_memory(self, cx, ty, offset)
963            }
964        }
965    )*)
966}
967
968forward_lowers! {
969    (T: Lower + ?Sized) &'_ T => T,
970    (T: Lower + ?Sized) Box<T> => T,
971    (T: Lower + ?Sized) alloc::sync::Arc<T> => T,
972    () String => str,
973    (T: Lower) Vec<T> => [T],
974}
975
976macro_rules! forward_string_lifts {
977    ($($a:ty,)*) => ($(
978        unsafe impl Lift for $a {
979            #[inline]
980            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
981                Ok(<WasmStr as Lift>::linear_lift_from_flat(cx, ty, src)?.to_str_from_memory(cx.memory())?.into())
982            }
983
984            #[inline]
985            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
986                Ok(<WasmStr as Lift>::linear_lift_from_memory(cx, ty, bytes)?.to_str_from_memory(cx.memory())?.into())
987            }
988        }
989    )*)
990}
991
992forward_string_lifts! {
993    Box<str>,
994    alloc::sync::Arc<str>,
995    String,
996}
997
998macro_rules! forward_list_lifts {
999    ($($a:ty,)*) => ($(
1000        unsafe impl <T: Lift> Lift for $a {
1001            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1002                let list = <WasmList::<T> as Lift>::linear_lift_from_flat(cx, ty, src)?;
1003                Ok(T::linear_lift_list_from_memory(cx, &list)?.into())
1004            }
1005
1006            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1007                let list = <WasmList::<T> as Lift>::linear_lift_from_memory(cx, ty, bytes)?;
1008                Ok(T::linear_lift_list_from_memory(cx, &list)?.into())
1009            }
1010        }
1011    )*)
1012}
1013
1014forward_list_lifts! {
1015    Box<[T]>,
1016    alloc::sync::Arc<[T]>,
1017    Vec<T>,
1018}
1019
1020// Macro to help generate `ComponentType` implementations for primitive types
1021// such as integers, char, bool, etc.
1022macro_rules! integers {
1023    ($($primitive:ident = $ty:ident in $field:ident/$get:ident with abi:$abi:ident,)*) => ($(
1024        unsafe impl ComponentType for $primitive {
1025            type Lower = ValRaw;
1026
1027            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::$abi;
1028
1029            const MAY_REQUIRE_REALLOC: bool = false;
1030
1031            fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1032                match ty {
1033                    InterfaceType::$ty => Ok(()),
1034                    other => bail!("expected `{}` found `{}`", desc(&InterfaceType::$ty), desc(other))
1035                }
1036            }
1037        }
1038
1039        unsafe impl Lower for $primitive {
1040            #[inline]
1041            #[allow(trivial_numeric_casts, reason = "macro-generated code")]
1042            fn linear_lower_to_flat<T>(
1043                &self,
1044                _cx: &mut LowerContext<'_, T>,
1045                ty: InterfaceType,
1046                dst: &mut MaybeUninit<Self::Lower>,
1047            ) -> Result<()> {
1048                debug_assert!(matches!(ty, InterfaceType::$ty));
1049                dst.write(ValRaw::$field(*self as $field));
1050                Ok(())
1051            }
1052
1053            #[inline]
1054            fn linear_lower_to_memory<T>(
1055                &self,
1056                cx: &mut LowerContext<'_, T>,
1057                ty: InterfaceType,
1058                offset: usize,
1059            ) -> Result<()> {
1060                debug_assert!(matches!(ty, InterfaceType::$ty));
1061                debug_assert!(offset % Self::SIZE32 == 0);
1062                *cx.get(offset) = self.to_le_bytes();
1063                Ok(())
1064            }
1065
1066            fn linear_store_list_to_memory<T>(
1067                cx: &mut LowerContext<'_, T>,
1068                ty: InterfaceType,
1069                offset: usize,
1070                items: &[Self],
1071            ) -> Result<()> {
1072                debug_assert!(matches!(ty, InterfaceType::$ty));
1073
1074                // Double-check that the CM alignment is at least the host's
1075                // alignment for this type which should be true for all
1076                // platforms.
1077                assert!((Self::ALIGN32 as usize) >= mem::align_of::<Self>());
1078
1079                // Slice `cx`'s memory to the window that we'll be modifying.
1080                // This should all have already been verified in terms of
1081                // alignment and sizing meaning that these assertions here are
1082                // not truly necessary but are instead double-checks.
1083                //
1084                // Note that we're casting a `[u8]` slice to `[Self]` with
1085                // `align_to_mut` which is not safe in general but is safe in
1086                // our specific case as all `u8` patterns are valid `Self`
1087                // patterns since `Self` is an integral type.
1088                let dst = &mut cx.as_slice_mut()[offset..][..items.len() * Self::SIZE32];
1089                let (before, middle, end) = unsafe { dst.align_to_mut::<Self>() };
1090                assert!(before.is_empty() && end.is_empty());
1091                assert_eq!(middle.len(), items.len());
1092
1093                // And with all that out of the way perform the copying loop.
1094                // This is not a `copy_from_slice` because endianness needs to
1095                // be handled here, but LLVM should pretty easily transform this
1096                // into a memcpy on little-endian platforms.
1097                for (dst, src) in middle.iter_mut().zip(items) {
1098                    *dst = src.to_le();
1099                }
1100                Ok(())
1101            }
1102        }
1103
1104        unsafe impl Lift for $primitive {
1105            #[inline]
1106            #[allow(
1107                trivial_numeric_casts,
1108                clippy::cast_possible_truncation,
1109                reason = "macro-generated code"
1110            )]
1111            fn linear_lift_from_flat(_cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1112                debug_assert!(matches!(ty, InterfaceType::$ty));
1113                Ok(src.$get() as $primitive)
1114            }
1115
1116            #[inline]
1117            fn linear_lift_from_memory(_cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1118                debug_assert!(matches!(ty, InterfaceType::$ty));
1119                debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1120                Ok($primitive::from_le_bytes(bytes.try_into().unwrap()))
1121            }
1122
1123            fn linear_lift_into_from_memory(
1124                cx: &mut LiftContext<'_>,
1125                list: &WasmList<Self>,
1126                dst: &mut impl Extend<Self>,
1127            ) -> Result<()>
1128            where
1129                Self: Sized,
1130            {
1131                dst.extend(list._as_le_slice(cx.memory())
1132                           .iter()
1133                           .map(|i| Self::from_le(*i)));
1134                Ok(())
1135            }
1136        }
1137    )*)
1138}
1139
1140integers! {
1141    i8 = S8 in i32/get_i32 with abi:SCALAR1,
1142    u8 = U8 in u32/get_u32 with abi:SCALAR1,
1143    i16 = S16 in i32/get_i32 with abi:SCALAR2,
1144    u16 = U16 in u32/get_u32 with abi:SCALAR2,
1145    i32 = S32 in i32/get_i32 with abi:SCALAR4,
1146    u32 = U32 in u32/get_u32 with abi:SCALAR4,
1147    i64 = S64 in i64/get_i64 with abi:SCALAR8,
1148    u64 = U64 in u64/get_u64 with abi:SCALAR8,
1149}
1150
1151macro_rules! floats {
1152    ($($float:ident/$get_float:ident = $ty:ident with abi:$abi:ident)*) => ($(const _: () = {
1153        unsafe impl ComponentType for $float {
1154            type Lower = ValRaw;
1155
1156            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::$abi;
1157
1158            fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1159                match ty {
1160                    InterfaceType::$ty => Ok(()),
1161                    other => bail!("expected `{}` found `{}`", desc(&InterfaceType::$ty), desc(other))
1162                }
1163            }
1164        }
1165
1166        unsafe impl Lower for $float {
1167            #[inline]
1168            fn linear_lower_to_flat<T>(
1169                &self,
1170                _cx: &mut LowerContext<'_, T>,
1171                ty: InterfaceType,
1172                dst: &mut MaybeUninit<Self::Lower>,
1173            ) -> Result<()> {
1174                debug_assert!(matches!(ty, InterfaceType::$ty));
1175                dst.write(ValRaw::$float(self.to_bits()));
1176                Ok(())
1177            }
1178
1179            #[inline]
1180            fn linear_lower_to_memory<T>(
1181                &self,
1182                cx: &mut LowerContext<'_, T>,
1183                ty: InterfaceType,
1184                offset: usize,
1185            ) -> Result<()> {
1186                debug_assert!(matches!(ty, InterfaceType::$ty));
1187                debug_assert!(offset % Self::SIZE32 == 0);
1188                let ptr = cx.get(offset);
1189                *ptr = self.to_bits().to_le_bytes();
1190                Ok(())
1191            }
1192
1193            fn linear_store_list_to_memory<T>(
1194                cx: &mut LowerContext<'_, T>,
1195                ty: InterfaceType,
1196                offset: usize,
1197                items: &[Self],
1198            ) -> Result<()> {
1199                debug_assert!(matches!(ty, InterfaceType::$ty));
1200
1201                // Double-check that the CM alignment is at least the host's
1202                // alignment for this type which should be true for all
1203                // platforms.
1204                assert!((Self::ALIGN32 as usize) >= mem::align_of::<Self>());
1205
1206                // Slice `cx`'s memory to the window that we'll be modifying.
1207                // This should all have already been verified in terms of
1208                // alignment and sizing meaning that these assertions here are
1209                // not truly necessary but are instead double-checks.
1210                let dst = &mut cx.as_slice_mut()[offset..][..items.len() * Self::SIZE32];
1211                assert!(dst.as_ptr().cast::<Self>().is_aligned());
1212
1213                // And with all that out of the way perform the copying loop.
1214                // This is not a `copy_from_slice` because endianness needs to
1215                // be handled here, but LLVM should pretty easily transform this
1216                // into a memcpy on little-endian platforms.
1217                // TODO use `as_chunks` when https://github.com/rust-lang/rust/issues/74985
1218                // is stabilized
1219                for (dst, src) in iter::zip(dst.chunks_exact_mut(Self::SIZE32), items) {
1220                    let dst: &mut [u8; Self::SIZE32] = dst.try_into().unwrap();
1221                    *dst = src.to_le_bytes();
1222                }
1223                Ok(())
1224            }
1225        }
1226
1227        unsafe impl Lift for $float {
1228            #[inline]
1229            fn linear_lift_from_flat(_cx: &mut LiftContext<'_>, ty: InterfaceType, src: &Self::Lower) -> Result<Self> {
1230                debug_assert!(matches!(ty, InterfaceType::$ty));
1231                Ok($float::from_bits(src.$get_float()))
1232            }
1233
1234            #[inline]
1235            fn linear_lift_from_memory(_cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
1236                debug_assert!(matches!(ty, InterfaceType::$ty));
1237                debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1238                Ok($float::from_le_bytes(bytes.try_into().unwrap()))
1239            }
1240
1241            fn linear_lift_list_from_memory(cx: &mut LiftContext<'_>, list: &WasmList<Self>) -> Result<Vec<Self>> where Self: Sized {
1242                // See comments in `WasmList::get` for the panicking indexing
1243                let byte_size = list.len * mem::size_of::<Self>();
1244                let bytes = &cx.memory()[list.ptr..][..byte_size];
1245
1246                // The canonical ABI requires that everything is aligned to its
1247                // own size, so this should be an aligned array.
1248                assert!(bytes.as_ptr().cast::<Self>().is_aligned());
1249
1250                // Copy the resulting slice to a new Vec, handling endianness
1251                // in the process
1252                // TODO use `as_chunks` when https://github.com/rust-lang/rust/issues/74985
1253                // is stabilized
1254                Ok(
1255                    bytes
1256                        .chunks_exact(Self::SIZE32)
1257                        .map(|i| $float::from_le_bytes(i.try_into().unwrap()))
1258                        .collect()
1259                )
1260            }
1261        }
1262    };)*)
1263}
1264
1265floats! {
1266    f32/get_f32 = Float32 with abi:SCALAR4
1267    f64/get_f64 = Float64 with abi:SCALAR8
1268}
1269
1270unsafe impl ComponentType for bool {
1271    type Lower = ValRaw;
1272
1273    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR1;
1274
1275    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1276        match ty {
1277            InterfaceType::Bool => Ok(()),
1278            other => bail!("expected `bool` found `{}`", desc(other)),
1279        }
1280    }
1281}
1282
1283unsafe impl Lower for bool {
1284    fn linear_lower_to_flat<T>(
1285        &self,
1286        _cx: &mut LowerContext<'_, T>,
1287        ty: InterfaceType,
1288        dst: &mut MaybeUninit<Self::Lower>,
1289    ) -> Result<()> {
1290        debug_assert!(matches!(ty, InterfaceType::Bool));
1291        dst.write(ValRaw::i32(*self as i32));
1292        Ok(())
1293    }
1294
1295    fn linear_lower_to_memory<T>(
1296        &self,
1297        cx: &mut LowerContext<'_, T>,
1298        ty: InterfaceType,
1299        offset: usize,
1300    ) -> Result<()> {
1301        debug_assert!(matches!(ty, InterfaceType::Bool));
1302        debug_assert!(offset % Self::SIZE32 == 0);
1303        cx.get::<1>(offset)[0] = *self as u8;
1304        Ok(())
1305    }
1306}
1307
1308unsafe impl Lift for bool {
1309    #[inline]
1310    fn linear_lift_from_flat(
1311        _cx: &mut LiftContext<'_>,
1312        ty: InterfaceType,
1313        src: &Self::Lower,
1314    ) -> Result<Self> {
1315        debug_assert!(matches!(ty, InterfaceType::Bool));
1316        match src.get_i32() {
1317            0 => Ok(false),
1318            _ => Ok(true),
1319        }
1320    }
1321
1322    #[inline]
1323    fn linear_lift_from_memory(
1324        _cx: &mut LiftContext<'_>,
1325        ty: InterfaceType,
1326        bytes: &[u8],
1327    ) -> Result<Self> {
1328        debug_assert!(matches!(ty, InterfaceType::Bool));
1329        match bytes[0] {
1330            0 => Ok(false),
1331            _ => Ok(true),
1332        }
1333    }
1334}
1335
1336unsafe impl ComponentType for char {
1337    type Lower = ValRaw;
1338
1339    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::SCALAR4;
1340
1341    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1342        match ty {
1343            InterfaceType::Char => Ok(()),
1344            other => bail!("expected `char` found `{}`", desc(other)),
1345        }
1346    }
1347}
1348
1349unsafe impl Lower for char {
1350    #[inline]
1351    fn linear_lower_to_flat<T>(
1352        &self,
1353        _cx: &mut LowerContext<'_, T>,
1354        ty: InterfaceType,
1355        dst: &mut MaybeUninit<Self::Lower>,
1356    ) -> Result<()> {
1357        debug_assert!(matches!(ty, InterfaceType::Char));
1358        dst.write(ValRaw::u32(u32::from(*self)));
1359        Ok(())
1360    }
1361
1362    #[inline]
1363    fn linear_lower_to_memory<T>(
1364        &self,
1365        cx: &mut LowerContext<'_, T>,
1366        ty: InterfaceType,
1367        offset: usize,
1368    ) -> Result<()> {
1369        debug_assert!(matches!(ty, InterfaceType::Char));
1370        debug_assert!(offset % Self::SIZE32 == 0);
1371        *cx.get::<4>(offset) = u32::from(*self).to_le_bytes();
1372        Ok(())
1373    }
1374}
1375
1376unsafe impl Lift for char {
1377    #[inline]
1378    fn linear_lift_from_flat(
1379        _cx: &mut LiftContext<'_>,
1380        ty: InterfaceType,
1381        src: &Self::Lower,
1382    ) -> Result<Self> {
1383        debug_assert!(matches!(ty, InterfaceType::Char));
1384        Ok(char::try_from(src.get_u32())?)
1385    }
1386
1387    #[inline]
1388    fn linear_lift_from_memory(
1389        _cx: &mut LiftContext<'_>,
1390        ty: InterfaceType,
1391        bytes: &[u8],
1392    ) -> Result<Self> {
1393        debug_assert!(matches!(ty, InterfaceType::Char));
1394        debug_assert!((bytes.as_ptr() as usize) % Self::SIZE32 == 0);
1395        let bits = u32::from_le_bytes(bytes.try_into().unwrap());
1396        Ok(char::try_from(bits)?)
1397    }
1398}
1399
1400// FIXME(#4311): these probably need different constants for memory64
1401const UTF16_TAG: usize = 1 << 31;
1402const MAX_STRING_BYTE_LENGTH: usize = (1 << 31) - 1;
1403
1404// Note that this is similar to `ComponentType for WasmStr` except it can only
1405// be used for lowering, not lifting.
1406unsafe impl ComponentType for str {
1407    type Lower = [ValRaw; 2];
1408
1409    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1410
1411    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1412        match ty {
1413            InterfaceType::String => Ok(()),
1414            other => bail!("expected `string` found `{}`", desc(other)),
1415        }
1416    }
1417}
1418
1419unsafe impl Lower for str {
1420    fn linear_lower_to_flat<T>(
1421        &self,
1422        cx: &mut LowerContext<'_, T>,
1423        ty: InterfaceType,
1424        dst: &mut MaybeUninit<[ValRaw; 2]>,
1425    ) -> Result<()> {
1426        debug_assert!(matches!(ty, InterfaceType::String));
1427        let (ptr, len) = lower_string(cx, self)?;
1428        // See "WRITEPTR64" above for why this is always storing a 64-bit
1429        // integer.
1430        map_maybe_uninit!(dst[0]).write(ValRaw::i64(ptr as i64));
1431        map_maybe_uninit!(dst[1]).write(ValRaw::i64(len as i64));
1432        Ok(())
1433    }
1434
1435    fn linear_lower_to_memory<T>(
1436        &self,
1437        cx: &mut LowerContext<'_, T>,
1438        ty: InterfaceType,
1439        offset: usize,
1440    ) -> Result<()> {
1441        debug_assert!(matches!(ty, InterfaceType::String));
1442        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
1443        let (ptr, len) = lower_string(cx, self)?;
1444        // FIXME(#4311): needs memory64 handling
1445        *cx.get(offset + 0) = u32::try_from(ptr).unwrap().to_le_bytes();
1446        *cx.get(offset + 4) = u32::try_from(len).unwrap().to_le_bytes();
1447        Ok(())
1448    }
1449}
1450
1451fn lower_string<T>(cx: &mut LowerContext<'_, T>, string: &str) -> Result<(usize, usize)> {
1452    // Note that in general the wasm module can't assume anything about what the
1453    // host strings are encoded as. Additionally hosts are allowed to have
1454    // differently-encoded strings at runtime. Finally when copying a string
1455    // into wasm it's somewhat strict in the sense that the various patterns of
1456    // allocation and such are already dictated for us.
1457    //
1458    // In general what this means is that when copying a string from the host
1459    // into the destination we need to follow one of the cases of copying into
1460    // WebAssembly. It doesn't particularly matter which case as long as it ends
1461    // up in the right encoding. For example a destination encoding of
1462    // latin1+utf16 has a number of ways to get copied into and we do something
1463    // here that isn't the default "utf8 to latin1+utf16" since we have access
1464    // to simd-accelerated helpers in the `encoding_rs` crate. This is ok though
1465    // because we can fake that the host string was already stored in latin1
1466    // format and follow that copy pattern instead.
1467    match cx.options.string_encoding() {
1468        // This corresponds to `store_string_copy` in the canonical ABI where
1469        // the host's representation is utf-8 and the wasm module wants utf-8 so
1470        // a copy is all that's needed (and the `realloc` can be precise for the
1471        // initial memory allocation).
1472        StringEncoding::Utf8 => {
1473            if string.len() > MAX_STRING_BYTE_LENGTH {
1474                bail!(
1475                    "string length of {} too large to copy into wasm",
1476                    string.len()
1477                );
1478            }
1479            let ptr = cx.realloc(0, 0, 1, string.len())?;
1480            cx.as_slice_mut()[ptr..][..string.len()].copy_from_slice(string.as_bytes());
1481            Ok((ptr, string.len()))
1482        }
1483
1484        // This corresponds to `store_utf8_to_utf16` in the canonical ABI. Here
1485        // an over-large allocation is performed and then shrunk afterwards if
1486        // necessary.
1487        StringEncoding::Utf16 => {
1488            let size = string.len() * 2;
1489            if size > MAX_STRING_BYTE_LENGTH {
1490                bail!(
1491                    "string length of {} too large to copy into wasm",
1492                    string.len()
1493                );
1494            }
1495            let mut ptr = cx.realloc(0, 0, 2, size)?;
1496            let mut copied = 0;
1497            let bytes = &mut cx.as_slice_mut()[ptr..][..size];
1498            for (u, bytes) in string.encode_utf16().zip(bytes.chunks_mut(2)) {
1499                let u_bytes = u.to_le_bytes();
1500                bytes[0] = u_bytes[0];
1501                bytes[1] = u_bytes[1];
1502                copied += 1;
1503            }
1504            if (copied * 2) < size {
1505                ptr = cx.realloc(ptr, size, 2, copied * 2)?;
1506            }
1507            Ok((ptr, copied))
1508        }
1509
1510        StringEncoding::CompactUtf16 => {
1511            // This corresponds to `store_string_to_latin1_or_utf16`
1512            let bytes = string.as_bytes();
1513            let mut iter = string.char_indices();
1514            let mut ptr = cx.realloc(0, 0, 2, bytes.len())?;
1515            let mut dst = &mut cx.as_slice_mut()[ptr..][..bytes.len()];
1516            let mut result = 0;
1517            while let Some((i, ch)) = iter.next() {
1518                // Test if this `char` fits into the latin1 encoding.
1519                if let Ok(byte) = u8::try_from(u32::from(ch)) {
1520                    dst[result] = byte;
1521                    result += 1;
1522                    continue;
1523                }
1524
1525                // .. if utf16 is forced to be used then the allocation is
1526                // bumped up to the maximum size.
1527                let worst_case = bytes
1528                    .len()
1529                    .checked_mul(2)
1530                    .ok_or_else(|| anyhow!("byte length overflow"))?;
1531                if worst_case > MAX_STRING_BYTE_LENGTH {
1532                    bail!("byte length too large");
1533                }
1534                ptr = cx.realloc(ptr, bytes.len(), 2, worst_case)?;
1535                dst = &mut cx.as_slice_mut()[ptr..][..worst_case];
1536
1537                // Previously encoded latin1 bytes are inflated to their 16-bit
1538                // size for utf16
1539                for i in (0..result).rev() {
1540                    dst[2 * i] = dst[i];
1541                    dst[2 * i + 1] = 0;
1542                }
1543
1544                // and then the remainder of the string is encoded.
1545                for (u, bytes) in string[i..]
1546                    .encode_utf16()
1547                    .zip(dst[2 * result..].chunks_mut(2))
1548                {
1549                    let u_bytes = u.to_le_bytes();
1550                    bytes[0] = u_bytes[0];
1551                    bytes[1] = u_bytes[1];
1552                    result += 1;
1553                }
1554                if worst_case > 2 * result {
1555                    ptr = cx.realloc(ptr, worst_case, 2, 2 * result)?;
1556                }
1557                return Ok((ptr, result | UTF16_TAG));
1558            }
1559            if result < bytes.len() {
1560                ptr = cx.realloc(ptr, bytes.len(), 2, result)?;
1561            }
1562            Ok((ptr, result))
1563        }
1564    }
1565}
1566
1567/// Representation of a string located in linear memory in a WebAssembly
1568/// instance.
1569///
1570/// This type can be used in place of `String` and `str` for string-taking APIs
1571/// in some situations. The purpose of this type is to represent a range of
1572/// validated bytes within a component but does not actually copy the bytes. The
1573/// primary method, [`WasmStr::to_str`], attempts to return a reference to the
1574/// string directly located in the component's memory, avoiding a copy into the
1575/// host if possible.
1576///
1577/// The downside of this type, however, is that accessing a string requires a
1578/// [`Store`](crate::Store) pointer (via [`StoreContext`]). Bindings generated
1579/// by [`bindgen!`](crate::component::bindgen), for example, do not have access
1580/// to [`StoreContext`] and thus can't use this type.
1581///
1582/// This is intended for more advanced use cases such as defining functions
1583/// directly in a [`Linker`](crate::component::Linker). It's expected that in
1584/// the future [`bindgen!`](crate::component::bindgen) will also have a way to
1585/// use this type.
1586///
1587/// This type is used with [`TypedFunc`], for example, when WebAssembly returns
1588/// a string. This type cannot be used to give a string to WebAssembly, instead
1589/// `&str` should be used for that (since it's coming from the host).
1590///
1591/// Note that this type represents an in-bounds string in linear memory, but it
1592/// does not represent a valid string (e.g. valid utf-8). Validation happens
1593/// when [`WasmStr::to_str`] is called.
1594///
1595/// Also note that this type does not implement [`Lower`], it only implements
1596/// [`Lift`].
1597pub struct WasmStr {
1598    ptr: usize,
1599    len: usize,
1600    options: Options,
1601}
1602
1603impl WasmStr {
1604    pub(crate) fn new(ptr: usize, len: usize, cx: &mut LiftContext<'_>) -> Result<WasmStr> {
1605        let byte_len = match cx.options.string_encoding() {
1606            StringEncoding::Utf8 => Some(len),
1607            StringEncoding::Utf16 => len.checked_mul(2),
1608            StringEncoding::CompactUtf16 => {
1609                if len & UTF16_TAG == 0 {
1610                    Some(len)
1611                } else {
1612                    (len ^ UTF16_TAG).checked_mul(2)
1613                }
1614            }
1615        };
1616        match byte_len.and_then(|len| ptr.checked_add(len)) {
1617            Some(n) if n <= cx.memory().len() => {}
1618            _ => bail!("string pointer/length out of bounds of memory"),
1619        }
1620        Ok(WasmStr {
1621            ptr,
1622            len,
1623            options: *cx.options,
1624        })
1625    }
1626
1627    /// Returns the underlying string that this cursor points to.
1628    ///
1629    /// Note that this will internally decode the string from the wasm's
1630    /// encoding to utf-8 and additionally perform validation.
1631    ///
1632    /// The `store` provided must be the store where this string lives to
1633    /// access the correct memory.
1634    ///
1635    /// # Errors
1636    ///
1637    /// Returns an error if the string wasn't encoded correctly (e.g. invalid
1638    /// utf-8).
1639    ///
1640    /// # Panics
1641    ///
1642    /// Panics if this string is not owned by `store`.
1643    //
1644    // TODO: should add accessors for specifically utf-8 and utf-16 that perhaps
1645    // in an opt-in basis don't do validation. Additionally there should be some
1646    // method that returns `[u16]` after validating to avoid the utf16-to-utf8
1647    // transcode.
1648    pub fn to_str<'a, T: 'static>(
1649        &self,
1650        store: impl Into<StoreContext<'a, T>>,
1651    ) -> Result<Cow<'a, str>> {
1652        let store = store.into().0;
1653        let memory = self.options.memory(store);
1654        self.to_str_from_memory(memory)
1655    }
1656
1657    pub(crate) fn to_str_from_memory<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1658        match self.options.string_encoding() {
1659            StringEncoding::Utf8 => self.decode_utf8(memory),
1660            StringEncoding::Utf16 => self.decode_utf16(memory, self.len),
1661            StringEncoding::CompactUtf16 => {
1662                if self.len & UTF16_TAG == 0 {
1663                    self.decode_latin1(memory)
1664                } else {
1665                    self.decode_utf16(memory, self.len ^ UTF16_TAG)
1666                }
1667            }
1668        }
1669    }
1670
1671    fn decode_utf8<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1672        // Note that bounds-checking already happen in construction of `WasmStr`
1673        // so this is never expected to panic. This could theoretically be
1674        // unchecked indexing if we're feeling wild enough.
1675        Ok(str::from_utf8(&memory[self.ptr..][..self.len])?.into())
1676    }
1677
1678    fn decode_utf16<'a>(&self, memory: &'a [u8], len: usize) -> Result<Cow<'a, str>> {
1679        // See notes in `decode_utf8` for why this is panicking indexing.
1680        let memory = &memory[self.ptr..][..len * 2];
1681        Ok(core::char::decode_utf16(
1682            memory
1683                .chunks(2)
1684                .map(|chunk| u16::from_le_bytes(chunk.try_into().unwrap())),
1685        )
1686        .collect::<Result<String, _>>()?
1687        .into())
1688    }
1689
1690    fn decode_latin1<'a>(&self, memory: &'a [u8]) -> Result<Cow<'a, str>> {
1691        // See notes in `decode_utf8` for why this is panicking indexing.
1692        Ok(encoding_rs::mem::decode_latin1(
1693            &memory[self.ptr..][..self.len],
1694        ))
1695    }
1696}
1697
1698// Note that this is similar to `ComponentType for str` except it can only be
1699// used for lifting, not lowering.
1700unsafe impl ComponentType for WasmStr {
1701    type Lower = <str as ComponentType>::Lower;
1702
1703    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1704
1705    fn typecheck(ty: &InterfaceType, _types: &InstanceType<'_>) -> Result<()> {
1706        match ty {
1707            InterfaceType::String => Ok(()),
1708            other => bail!("expected `string` found `{}`", desc(other)),
1709        }
1710    }
1711}
1712
1713unsafe impl Lift for WasmStr {
1714    #[inline]
1715    fn linear_lift_from_flat(
1716        cx: &mut LiftContext<'_>,
1717        ty: InterfaceType,
1718        src: &Self::Lower,
1719    ) -> Result<Self> {
1720        debug_assert!(matches!(ty, InterfaceType::String));
1721        // FIXME(#4311): needs memory64 treatment
1722        let ptr = src[0].get_u32();
1723        let len = src[1].get_u32();
1724        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
1725        WasmStr::new(ptr, len, cx)
1726    }
1727
1728    #[inline]
1729    fn linear_lift_from_memory(
1730        cx: &mut LiftContext<'_>,
1731        ty: InterfaceType,
1732        bytes: &[u8],
1733    ) -> Result<Self> {
1734        debug_assert!(matches!(ty, InterfaceType::String));
1735        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
1736        // FIXME(#4311): needs memory64 treatment
1737        let ptr = u32::from_le_bytes(bytes[..4].try_into().unwrap());
1738        let len = u32::from_le_bytes(bytes[4..].try_into().unwrap());
1739        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
1740        WasmStr::new(ptr, len, cx)
1741    }
1742}
1743
1744unsafe impl<T> ComponentType for [T]
1745where
1746    T: ComponentType,
1747{
1748    type Lower = [ValRaw; 2];
1749
1750    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
1751
1752    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
1753        match ty {
1754            InterfaceType::List(t) => T::typecheck(&types.types[*t].element, types),
1755            other => bail!("expected `list` found `{}`", desc(other)),
1756        }
1757    }
1758}
1759
1760unsafe impl<T> Lower for [T]
1761where
1762    T: Lower,
1763{
1764    fn linear_lower_to_flat<U>(
1765        &self,
1766        cx: &mut LowerContext<'_, U>,
1767        ty: InterfaceType,
1768        dst: &mut MaybeUninit<[ValRaw; 2]>,
1769    ) -> Result<()> {
1770        let elem = match ty {
1771            InterfaceType::List(i) => cx.types[i].element,
1772            _ => bad_type_info(),
1773        };
1774        let (ptr, len) = lower_list(cx, elem, self)?;
1775        // See "WRITEPTR64" above for why this is always storing a 64-bit
1776        // integer.
1777        map_maybe_uninit!(dst[0]).write(ValRaw::i64(ptr as i64));
1778        map_maybe_uninit!(dst[1]).write(ValRaw::i64(len as i64));
1779        Ok(())
1780    }
1781
1782    fn linear_lower_to_memory<U>(
1783        &self,
1784        cx: &mut LowerContext<'_, U>,
1785        ty: InterfaceType,
1786        offset: usize,
1787    ) -> Result<()> {
1788        let elem = match ty {
1789            InterfaceType::List(i) => cx.types[i].element,
1790            _ => bad_type_info(),
1791        };
1792        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
1793        let (ptr, len) = lower_list(cx, elem, self)?;
1794        *cx.get(offset + 0) = u32::try_from(ptr).unwrap().to_le_bytes();
1795        *cx.get(offset + 4) = u32::try_from(len).unwrap().to_le_bytes();
1796        Ok(())
1797    }
1798}
1799
1800// FIXME: this is not a memcpy for `T` where `T` is something like `u8`.
1801//
1802// Some attempts to fix this have proved not fruitful. In isolation an attempt
1803// was made where:
1804//
1805// * `MemoryMut` stored a `*mut [u8]` as its "last view" of memory to avoid
1806//   reloading the base pointer constantly. This view is reset on `realloc`.
1807// * The bounds-checks in `MemoryMut::get` were removed (replaced with unsafe
1808//   indexing)
1809//
1810// Even then though this didn't correctly vectorized for `Vec<u8>`. It's not
1811// entirely clear why but it appeared that it's related to reloading the base
1812// pointer to memory (I guess from `MemoryMut` itself?). Overall I'm not really
1813// clear on what's happening there, but this is surely going to be a performance
1814// bottleneck in the future.
1815fn lower_list<T, U>(
1816    cx: &mut LowerContext<'_, U>,
1817    ty: InterfaceType,
1818    list: &[T],
1819) -> Result<(usize, usize)>
1820where
1821    T: Lower,
1822{
1823    let elem_size = T::SIZE32;
1824    let size = list
1825        .len()
1826        .checked_mul(elem_size)
1827        .ok_or_else(|| anyhow!("size overflow copying a list"))?;
1828    let ptr = cx.realloc(0, 0, T::ALIGN32, size)?;
1829    T::linear_store_list_to_memory(cx, ty, ptr, list)?;
1830    Ok((ptr, list.len()))
1831}
1832
1833/// Representation of a list of values that are owned by a WebAssembly instance.
1834///
1835/// For some more commentary about the rationale for this type see the
1836/// documentation of [`WasmStr`]. In summary this type can avoid a copy when
1837/// passing data to the host in some situations but is additionally more
1838/// cumbersome to use by requiring a [`Store`](crate::Store) to be provided.
1839///
1840/// This type is used whenever a `(list T)` is returned from a [`TypedFunc`],
1841/// for example. This type represents a list of values that are stored in linear
1842/// memory which are waiting to be read.
1843///
1844/// Note that this type represents only a valid range of bytes for the list
1845/// itself, it does not represent validity of the elements themselves and that's
1846/// performed when they're iterated.
1847///
1848/// Note that this type does not implement the [`Lower`] trait, only [`Lift`].
1849pub struct WasmList<T> {
1850    ptr: usize,
1851    len: usize,
1852    options: Options,
1853    elem: InterfaceType,
1854    instance: Instance,
1855    _marker: marker::PhantomData<T>,
1856}
1857
1858impl<T: Lift> WasmList<T> {
1859    pub(crate) fn new(
1860        ptr: usize,
1861        len: usize,
1862        cx: &mut LiftContext<'_>,
1863        elem: InterfaceType,
1864    ) -> Result<WasmList<T>> {
1865        match len
1866            .checked_mul(T::SIZE32)
1867            .and_then(|len| ptr.checked_add(len))
1868        {
1869            Some(n) if n <= cx.memory().len() => {}
1870            _ => bail!("list pointer/length out of bounds of memory"),
1871        }
1872        if ptr % usize::try_from(T::ALIGN32)? != 0 {
1873            bail!("list pointer is not aligned")
1874        }
1875        Ok(WasmList {
1876            ptr,
1877            len,
1878            options: *cx.options,
1879            elem,
1880            instance: cx.instance_handle(),
1881            _marker: marker::PhantomData,
1882        })
1883    }
1884
1885    /// Returns the item length of this vector
1886    #[inline]
1887    pub fn len(&self) -> usize {
1888        self.len
1889    }
1890
1891    /// Gets the `n`th element of this list.
1892    ///
1893    /// Returns `None` if `index` is out of bounds. Returns `Some(Err(..))` if
1894    /// the value couldn't be decoded (it was invalid). Returns `Some(Ok(..))`
1895    /// if the value is valid.
1896    ///
1897    /// # Panics
1898    ///
1899    /// This function will panic if the string did not originally come from the
1900    /// `store` specified.
1901    //
1902    // TODO: given that interface values are intended to be consumed in one go
1903    // should we even expose a random access iteration API? In theory all
1904    // consumers should be validating through the iterator.
1905    pub fn get(&self, mut store: impl AsContextMut, index: usize) -> Option<Result<T>> {
1906        let store = store.as_context_mut().0;
1907        self.options.store_id().assert_belongs_to(store.id());
1908        let mut cx = LiftContext::new(store, &self.options, self.instance);
1909        self.get_from_store(&mut cx, index)
1910    }
1911
1912    fn get_from_store(&self, cx: &mut LiftContext<'_>, index: usize) -> Option<Result<T>> {
1913        if index >= self.len {
1914            return None;
1915        }
1916        // Note that this is using panicking indexing and this is expected to
1917        // never fail. The bounds-checking here happened during the construction
1918        // of the `WasmList` itself which means these should always be in-bounds
1919        // (and wasm memory can only grow). This could theoretically be
1920        // unchecked indexing if we're confident enough and it's actually a perf
1921        // issue one day.
1922        let bytes = &cx.memory()[self.ptr + index * T::SIZE32..][..T::SIZE32];
1923        Some(T::linear_lift_from_memory(cx, self.elem, bytes))
1924    }
1925
1926    /// Returns an iterator over the elements of this list.
1927    ///
1928    /// Each item of the list may fail to decode and is represented through the
1929    /// `Result` value of the iterator.
1930    pub fn iter<'a, U: 'static>(
1931        &'a self,
1932        store: impl Into<StoreContextMut<'a, U>>,
1933    ) -> impl ExactSizeIterator<Item = Result<T>> + 'a {
1934        let store = store.into().0;
1935        self.options.store_id().assert_belongs_to(store.id());
1936        let mut cx = LiftContext::new(store, &self.options, self.instance);
1937        (0..self.len).map(move |i| self.get_from_store(&mut cx, i).unwrap())
1938    }
1939}
1940
1941macro_rules! raw_wasm_list_accessors {
1942    ($($i:ident)*) => ($(
1943        impl WasmList<$i> {
1944            /// Get access to the raw underlying memory for this list.
1945            ///
1946            /// This method will return a direct slice into the original wasm
1947            /// module's linear memory where the data for this slice is stored.
1948            /// This allows the embedder to have efficient access to the
1949            /// underlying memory if needed and avoid copies and such if
1950            /// desired.
1951            ///
1952            /// Note that multi-byte integers are stored in little-endian format
1953            /// so portable processing of this slice must be aware of the host's
1954            /// byte-endianness. The `from_le` constructors in the Rust standard
1955            /// library should be suitable for converting from little-endian.
1956            ///
1957            /// # Panics
1958            ///
1959            /// Panics if the `store` provided is not the one from which this
1960            /// slice originated.
1961            pub fn as_le_slice<'a, T: 'static>(&self, store: impl Into<StoreContext<'a, T>>) -> &'a [$i] {
1962                let memory = self.options.memory(store.into().0);
1963                self._as_le_slice(memory)
1964            }
1965
1966            fn _as_le_slice<'a>(&self, all_of_memory: &'a [u8]) -> &'a [$i] {
1967                // See comments in `WasmList::get` for the panicking indexing
1968                let byte_size = self.len * mem::size_of::<$i>();
1969                let bytes = &all_of_memory[self.ptr..][..byte_size];
1970
1971                // The canonical ABI requires that everything is aligned to its
1972                // own size, so this should be an aligned array. Furthermore the
1973                // alignment of primitive integers for hosts should be smaller
1974                // than or equal to the size of the primitive itself, meaning
1975                // that a wasm canonical-abi-aligned list is also aligned for
1976                // the host. That should mean that the head/tail slices here are
1977                // empty.
1978                //
1979                // Also note that the `unsafe` here is needed since the type
1980                // we're aligning to isn't guaranteed to be valid, but in our
1981                // case it's just integers and bytes so this should be safe.
1982                unsafe {
1983                    let (head, body, tail) = bytes.align_to::<$i>();
1984                    assert!(head.is_empty() && tail.is_empty());
1985                    body
1986                }
1987            }
1988        }
1989    )*)
1990}
1991
1992raw_wasm_list_accessors! {
1993    i8 i16 i32 i64
1994    u8 u16 u32 u64
1995}
1996
1997// Note that this is similar to `ComponentType for str` except it can only be
1998// used for lifting, not lowering.
1999unsafe impl<T: ComponentType> ComponentType for WasmList<T> {
2000    type Lower = <[T] as ComponentType>::Lower;
2001
2002    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::POINTER_PAIR;
2003
2004    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2005        <[T] as ComponentType>::typecheck(ty, types)
2006    }
2007}
2008
2009unsafe impl<T: Lift> Lift for WasmList<T> {
2010    fn linear_lift_from_flat(
2011        cx: &mut LiftContext<'_>,
2012        ty: InterfaceType,
2013        src: &Self::Lower,
2014    ) -> Result<Self> {
2015        let elem = match ty {
2016            InterfaceType::List(i) => cx.types[i].element,
2017            _ => bad_type_info(),
2018        };
2019        // FIXME(#4311): needs memory64 treatment
2020        let ptr = src[0].get_u32();
2021        let len = src[1].get_u32();
2022        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
2023        WasmList::new(ptr, len, cx, elem)
2024    }
2025
2026    fn linear_lift_from_memory(
2027        cx: &mut LiftContext<'_>,
2028        ty: InterfaceType,
2029        bytes: &[u8],
2030    ) -> Result<Self> {
2031        let elem = match ty {
2032            InterfaceType::List(i) => cx.types[i].element,
2033            _ => bad_type_info(),
2034        };
2035        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2036        // FIXME(#4311): needs memory64 treatment
2037        let ptr = u32::from_le_bytes(bytes[..4].try_into().unwrap());
2038        let len = u32::from_le_bytes(bytes[4..].try_into().unwrap());
2039        let (ptr, len) = (usize::try_from(ptr)?, usize::try_from(len)?);
2040        WasmList::new(ptr, len, cx, elem)
2041    }
2042}
2043
2044/// Verify that the given wasm type is a tuple with the expected fields in the right order.
2045fn typecheck_tuple(
2046    ty: &InterfaceType,
2047    types: &InstanceType<'_>,
2048    expected: &[fn(&InterfaceType, &InstanceType<'_>) -> Result<()>],
2049) -> Result<()> {
2050    match ty {
2051        InterfaceType::Tuple(t) => {
2052            let tuple = &types.types[*t];
2053            if tuple.types.len() != expected.len() {
2054                bail!(
2055                    "expected {}-tuple, found {}-tuple",
2056                    expected.len(),
2057                    tuple.types.len()
2058                );
2059            }
2060            for (ty, check) in tuple.types.iter().zip(expected) {
2061                check(ty, types)?;
2062            }
2063            Ok(())
2064        }
2065        other => bail!("expected `tuple` found `{}`", desc(other)),
2066    }
2067}
2068
2069/// Verify that the given wasm type is a record with the expected fields in the right order and with the right
2070/// names.
2071pub fn typecheck_record(
2072    ty: &InterfaceType,
2073    types: &InstanceType<'_>,
2074    expected: &[(&str, fn(&InterfaceType, &InstanceType<'_>) -> Result<()>)],
2075) -> Result<()> {
2076    match ty {
2077        InterfaceType::Record(index) => {
2078            let fields = &types.types[*index].fields;
2079
2080            if fields.len() != expected.len() {
2081                bail!(
2082                    "expected record of {} fields, found {} fields",
2083                    expected.len(),
2084                    fields.len()
2085                );
2086            }
2087
2088            for (field, &(name, check)) in fields.iter().zip(expected) {
2089                check(&field.ty, types)
2090                    .with_context(|| format!("type mismatch for field {name}"))?;
2091
2092                if field.name != name {
2093                    bail!("expected record field named {}, found {}", name, field.name);
2094                }
2095            }
2096
2097            Ok(())
2098        }
2099        other => bail!("expected `record` found `{}`", desc(other)),
2100    }
2101}
2102
2103/// Verify that the given wasm type is a variant with the expected cases in the right order and with the right
2104/// names.
2105pub fn typecheck_variant(
2106    ty: &InterfaceType,
2107    types: &InstanceType<'_>,
2108    expected: &[(
2109        &str,
2110        Option<fn(&InterfaceType, &InstanceType<'_>) -> Result<()>>,
2111    )],
2112) -> Result<()> {
2113    match ty {
2114        InterfaceType::Variant(index) => {
2115            let cases = &types.types[*index].cases;
2116
2117            if cases.len() != expected.len() {
2118                bail!(
2119                    "expected variant of {} cases, found {} cases",
2120                    expected.len(),
2121                    cases.len()
2122                );
2123            }
2124
2125            for ((case_name, case_ty), &(name, check)) in cases.iter().zip(expected) {
2126                if *case_name != name {
2127                    bail!("expected variant case named {name}, found {case_name}");
2128                }
2129
2130                match (check, case_ty) {
2131                    (Some(check), Some(ty)) => check(ty, types)
2132                        .with_context(|| format!("type mismatch for case {name}"))?,
2133                    (None, None) => {}
2134                    (Some(_), None) => {
2135                        bail!("case `{name}` has no type but one was expected")
2136                    }
2137                    (None, Some(_)) => {
2138                        bail!("case `{name}` has a type but none was expected")
2139                    }
2140                }
2141            }
2142
2143            Ok(())
2144        }
2145        other => bail!("expected `variant` found `{}`", desc(other)),
2146    }
2147}
2148
2149/// Verify that the given wasm type is a enum with the expected cases in the right order and with the right
2150/// names.
2151pub fn typecheck_enum(
2152    ty: &InterfaceType,
2153    types: &InstanceType<'_>,
2154    expected: &[&str],
2155) -> Result<()> {
2156    match ty {
2157        InterfaceType::Enum(index) => {
2158            let names = &types.types[*index].names;
2159
2160            if names.len() != expected.len() {
2161                bail!(
2162                    "expected enum of {} names, found {} names",
2163                    expected.len(),
2164                    names.len()
2165                );
2166            }
2167
2168            for (name, expected) in names.iter().zip(expected) {
2169                if name != expected {
2170                    bail!("expected enum case named {}, found {}", expected, name);
2171                }
2172            }
2173
2174            Ok(())
2175        }
2176        other => bail!("expected `enum` found `{}`", desc(other)),
2177    }
2178}
2179
2180/// Verify that the given wasm type is a flags type with the expected flags in the right order and with the right
2181/// names.
2182pub fn typecheck_flags(
2183    ty: &InterfaceType,
2184    types: &InstanceType<'_>,
2185    expected: &[&str],
2186) -> Result<()> {
2187    match ty {
2188        InterfaceType::Flags(index) => {
2189            let names = &types.types[*index].names;
2190
2191            if names.len() != expected.len() {
2192                bail!(
2193                    "expected flags type with {} names, found {} names",
2194                    expected.len(),
2195                    names.len()
2196                );
2197            }
2198
2199            for (name, expected) in names.iter().zip(expected) {
2200                if name != expected {
2201                    bail!("expected flag named {}, found {}", expected, name);
2202                }
2203            }
2204
2205            Ok(())
2206        }
2207        other => bail!("expected `flags` found `{}`", desc(other)),
2208    }
2209}
2210
2211/// Format the specified bitflags using the specified names for debugging
2212pub fn format_flags(bits: &[u32], names: &[&str], f: &mut fmt::Formatter) -> fmt::Result {
2213    f.write_str("(")?;
2214    let mut wrote = false;
2215    for (index, name) in names.iter().enumerate() {
2216        if ((bits[index / 32] >> (index % 32)) & 1) != 0 {
2217            if wrote {
2218                f.write_str("|")?;
2219            } else {
2220                wrote = true;
2221            }
2222
2223            f.write_str(name)?;
2224        }
2225    }
2226    f.write_str(")")
2227}
2228
2229unsafe impl<T> ComponentType for Option<T>
2230where
2231    T: ComponentType,
2232{
2233    type Lower = TupleLower<<u32 as ComponentType>::Lower, T::Lower>;
2234
2235    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::variant_static(&[None, Some(T::ABI)]);
2236
2237    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2238        match ty {
2239            InterfaceType::Option(t) => T::typecheck(&types.types[*t].ty, types),
2240            other => bail!("expected `option` found `{}`", desc(other)),
2241        }
2242    }
2243}
2244
2245unsafe impl<T> ComponentVariant for Option<T>
2246where
2247    T: ComponentType,
2248{
2249    const CASES: &'static [Option<CanonicalAbiInfo>] = &[None, Some(T::ABI)];
2250}
2251
2252unsafe impl<T> Lower for Option<T>
2253where
2254    T: Lower,
2255{
2256    fn linear_lower_to_flat<U>(
2257        &self,
2258        cx: &mut LowerContext<'_, U>,
2259        ty: InterfaceType,
2260        dst: &mut MaybeUninit<Self::Lower>,
2261    ) -> Result<()> {
2262        let payload = match ty {
2263            InterfaceType::Option(ty) => cx.types[ty].ty,
2264            _ => bad_type_info(),
2265        };
2266        match self {
2267            None => {
2268                map_maybe_uninit!(dst.A1).write(ValRaw::i32(0));
2269                // Note that this is unsafe as we're writing an arbitrary
2270                // bit-pattern to an arbitrary type, but part of the unsafe
2271                // contract of the `ComponentType` trait is that we can assign
2272                // any bit-pattern. By writing all zeros here we're ensuring
2273                // that the core wasm arguments this translates to will all be
2274                // zeros (as the canonical ABI requires).
2275                unsafe {
2276                    map_maybe_uninit!(dst.A2).as_mut_ptr().write_bytes(0u8, 1);
2277                }
2278            }
2279            Some(val) => {
2280                map_maybe_uninit!(dst.A1).write(ValRaw::i32(1));
2281                val.linear_lower_to_flat(cx, payload, map_maybe_uninit!(dst.A2))?;
2282            }
2283        }
2284        Ok(())
2285    }
2286
2287    fn linear_lower_to_memory<U>(
2288        &self,
2289        cx: &mut LowerContext<'_, U>,
2290        ty: InterfaceType,
2291        offset: usize,
2292    ) -> Result<()> {
2293        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
2294        let payload = match ty {
2295            InterfaceType::Option(ty) => cx.types[ty].ty,
2296            _ => bad_type_info(),
2297        };
2298        match self {
2299            None => {
2300                cx.get::<1>(offset)[0] = 0;
2301            }
2302            Some(val) => {
2303                cx.get::<1>(offset)[0] = 1;
2304                val.linear_lower_to_memory(
2305                    cx,
2306                    payload,
2307                    offset + (Self::INFO.payload_offset32 as usize),
2308                )?;
2309            }
2310        }
2311        Ok(())
2312    }
2313}
2314
2315unsafe impl<T> Lift for Option<T>
2316where
2317    T: Lift,
2318{
2319    fn linear_lift_from_flat(
2320        cx: &mut LiftContext<'_>,
2321        ty: InterfaceType,
2322        src: &Self::Lower,
2323    ) -> Result<Self> {
2324        let payload = match ty {
2325            InterfaceType::Option(ty) => cx.types[ty].ty,
2326            _ => bad_type_info(),
2327        };
2328        Ok(match src.A1.get_i32() {
2329            0 => None,
2330            1 => Some(T::linear_lift_from_flat(cx, payload, &src.A2)?),
2331            _ => bail!("invalid option discriminant"),
2332        })
2333    }
2334
2335    fn linear_lift_from_memory(
2336        cx: &mut LiftContext<'_>,
2337        ty: InterfaceType,
2338        bytes: &[u8],
2339    ) -> Result<Self> {
2340        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2341        let payload_ty = match ty {
2342            InterfaceType::Option(ty) => cx.types[ty].ty,
2343            _ => bad_type_info(),
2344        };
2345        let discrim = bytes[0];
2346        let payload = &bytes[Self::INFO.payload_offset32 as usize..];
2347        match discrim {
2348            0 => Ok(None),
2349            1 => Ok(Some(T::linear_lift_from_memory(cx, payload_ty, payload)?)),
2350            _ => bail!("invalid option discriminant"),
2351        }
2352    }
2353}
2354
2355#[derive(Clone, Copy)]
2356#[repr(C)]
2357pub struct ResultLower<T: Copy, E: Copy> {
2358    tag: ValRaw,
2359    payload: ResultLowerPayload<T, E>,
2360}
2361
2362#[derive(Clone, Copy)]
2363#[repr(C)]
2364union ResultLowerPayload<T: Copy, E: Copy> {
2365    ok: T,
2366    err: E,
2367}
2368
2369unsafe impl<T, E> ComponentType for Result<T, E>
2370where
2371    T: ComponentType,
2372    E: ComponentType,
2373{
2374    type Lower = ResultLower<T::Lower, E::Lower>;
2375
2376    const ABI: CanonicalAbiInfo = CanonicalAbiInfo::variant_static(&[Some(T::ABI), Some(E::ABI)]);
2377
2378    fn typecheck(ty: &InterfaceType, types: &InstanceType<'_>) -> Result<()> {
2379        match ty {
2380            InterfaceType::Result(r) => {
2381                let result = &types.types[*r];
2382                match &result.ok {
2383                    Some(ty) => T::typecheck(ty, types)?,
2384                    None if T::IS_RUST_UNIT_TYPE => {}
2385                    None => bail!("expected no `ok` type"),
2386                }
2387                match &result.err {
2388                    Some(ty) => E::typecheck(ty, types)?,
2389                    None if E::IS_RUST_UNIT_TYPE => {}
2390                    None => bail!("expected no `err` type"),
2391                }
2392                Ok(())
2393            }
2394            other => bail!("expected `result` found `{}`", desc(other)),
2395        }
2396    }
2397}
2398
2399/// Lowers the payload of a variant into the storage for the entire payload,
2400/// handling writing zeros at the end of the representation if this payload is
2401/// smaller than the entire flat representation.
2402///
2403/// * `payload` - the flat storage space for the entire payload of the variant
2404/// * `typed_payload` - projection from the payload storage space to the
2405///   individual storage space for this variant.
2406/// * `lower` - lowering operation used to initialize the `typed_payload` return
2407///   value.
2408///
2409/// For more information on this se the comments in the `Lower for Result`
2410/// implementation below.
2411pub unsafe fn lower_payload<P, T>(
2412    payload: &mut MaybeUninit<P>,
2413    typed_payload: impl FnOnce(&mut MaybeUninit<P>) -> &mut MaybeUninit<T>,
2414    lower: impl FnOnce(&mut MaybeUninit<T>) -> Result<()>,
2415) -> Result<()> {
2416    let typed = typed_payload(payload);
2417    lower(typed)?;
2418
2419    let typed_len = unsafe { storage_as_slice(typed).len() };
2420    let payload = unsafe { storage_as_slice_mut(payload) };
2421    for slot in payload[typed_len..].iter_mut() {
2422        slot.write(ValRaw::u64(0));
2423    }
2424    Ok(())
2425}
2426
2427unsafe impl<T, E> ComponentVariant for Result<T, E>
2428where
2429    T: ComponentType,
2430    E: ComponentType,
2431{
2432    const CASES: &'static [Option<CanonicalAbiInfo>] = &[Some(T::ABI), Some(E::ABI)];
2433}
2434
2435unsafe impl<T, E> Lower for Result<T, E>
2436where
2437    T: Lower,
2438    E: Lower,
2439{
2440    fn linear_lower_to_flat<U>(
2441        &self,
2442        cx: &mut LowerContext<'_, U>,
2443        ty: InterfaceType,
2444        dst: &mut MaybeUninit<Self::Lower>,
2445    ) -> Result<()> {
2446        let (ok, err) = match ty {
2447            InterfaceType::Result(ty) => {
2448                let ty = &cx.types[ty];
2449                (ty.ok, ty.err)
2450            }
2451            _ => bad_type_info(),
2452        };
2453
2454        // This implementation of `Lower::lower`, if you're reading these from
2455        // the top of this file, is the first location that the "join" logic of
2456        // the component model's canonical ABI encountered. The rough problem is
2457        // that let's say we have a component model type of the form:
2458        //
2459        //      (result u64 (error (tuple f32 u16)))
2460        //
2461        // The flat representation of this is actually pretty tricky. Currently
2462        // it is:
2463        //
2464        //      i32 i64 i32
2465        //
2466        // The first `i32` is the discriminant for the `result`, and the payload
2467        // is represented by `i64 i32`. The "ok" variant will only use the `i64`
2468        // and the "err" variant will use both `i64` and `i32`.
2469        //
2470        // In the "ok" variant the first issue is encountered. The size of one
2471        // variant may not match the size of the other variants. All variants
2472        // start at the "front" but when lowering a type we need to be sure to
2473        // initialize the later variants (lest we leak random host memory into
2474        // the guest module). Due to how the `Lower` type is represented as a
2475        // `union` of all the variants what ends up happening here is that
2476        // internally within the `lower_payload` after the typed payload is
2477        // lowered the remaining bits of the payload that weren't initialized
2478        // are all set to zero. This will guarantee that we'll write to all the
2479        // slots for each variant.
2480        //
2481        // The "err" variant encounters the second issue, however, which is that
2482        // the flat representation for each type may differ between payloads. In
2483        // the "ok" arm an `i64` is written, but the `lower` implementation for
2484        // the "err" arm will write an `f32` and then an `i32`. For this
2485        // implementation of `lower` to be valid the `f32` needs to get inflated
2486        // to an `i64` with zero-padding in the upper bits. What may be
2487        // surprising, however, is that none of this is handled in this file.
2488        // This implementation looks like it's blindly deferring to `E::lower`
2489        // and hoping it does the right thing.
2490        //
2491        // In reality, however, the correctness of variant lowering relies on
2492        // two subtle details of the `ValRaw` implementation in Wasmtime:
2493        //
2494        // 1. First the `ValRaw` value always contains little-endian values.
2495        //    This means that if a `u32` is written, a `u64` is read, and then
2496        //    the `u64` has its upper bits truncated the original value will
2497        //    always be retained. This is primarily here for big-endian
2498        //    platforms where if it weren't little endian then the opposite
2499        //    would occur and the wrong value would be read.
2500        //
2501        // 2. Second, and perhaps even more subtly, the `ValRaw` constructors
2502        //    for 32-bit types actually always initialize 64-bits of the
2503        //    `ValRaw`. In the component model flat ABI only 32 and 64-bit types
2504        //    are used so 64-bits is big enough to contain everything. This
2505        //    means that when a `ValRaw` is written into the destination it will
2506        //    always, whether it's needed or not, be "ready" to get extended up
2507        //    to 64-bits.
2508        //
2509        // Put together these two subtle guarantees means that all `Lower`
2510        // implementations can be written "naturally" as one might naively
2511        // expect. Variants will, on each arm, zero out remaining fields and all
2512        // writes to the flat representation will automatically be 64-bit writes
2513        // meaning that if the value is read as a 64-bit value, which isn't
2514        // known at the time of the write, it'll still be correct.
2515        match self {
2516            Ok(e) => {
2517                map_maybe_uninit!(dst.tag).write(ValRaw::i32(0));
2518                unsafe {
2519                    lower_payload(
2520                        map_maybe_uninit!(dst.payload),
2521                        |payload| map_maybe_uninit!(payload.ok),
2522                        |dst| match ok {
2523                            Some(ok) => e.linear_lower_to_flat(cx, ok, dst),
2524                            None => Ok(()),
2525                        },
2526                    )
2527                }
2528            }
2529            Err(e) => {
2530                map_maybe_uninit!(dst.tag).write(ValRaw::i32(1));
2531                unsafe {
2532                    lower_payload(
2533                        map_maybe_uninit!(dst.payload),
2534                        |payload| map_maybe_uninit!(payload.err),
2535                        |dst| match err {
2536                            Some(err) => e.linear_lower_to_flat(cx, err, dst),
2537                            None => Ok(()),
2538                        },
2539                    )
2540                }
2541            }
2542        }
2543    }
2544
2545    fn linear_lower_to_memory<U>(
2546        &self,
2547        cx: &mut LowerContext<'_, U>,
2548        ty: InterfaceType,
2549        offset: usize,
2550    ) -> Result<()> {
2551        let (ok, err) = match ty {
2552            InterfaceType::Result(ty) => {
2553                let ty = &cx.types[ty];
2554                (ty.ok, ty.err)
2555            }
2556            _ => bad_type_info(),
2557        };
2558        debug_assert!(offset % (Self::ALIGN32 as usize) == 0);
2559        let payload_offset = Self::INFO.payload_offset32 as usize;
2560        match self {
2561            Ok(e) => {
2562                cx.get::<1>(offset)[0] = 0;
2563                if let Some(ok) = ok {
2564                    e.linear_lower_to_memory(cx, ok, offset + payload_offset)?;
2565                }
2566            }
2567            Err(e) => {
2568                cx.get::<1>(offset)[0] = 1;
2569                if let Some(err) = err {
2570                    e.linear_lower_to_memory(cx, err, offset + payload_offset)?;
2571                }
2572            }
2573        }
2574        Ok(())
2575    }
2576}
2577
2578unsafe impl<T, E> Lift for Result<T, E>
2579where
2580    T: Lift,
2581    E: Lift,
2582{
2583    #[inline]
2584    fn linear_lift_from_flat(
2585        cx: &mut LiftContext<'_>,
2586        ty: InterfaceType,
2587        src: &Self::Lower,
2588    ) -> Result<Self> {
2589        let (ok, err) = match ty {
2590            InterfaceType::Result(ty) => {
2591                let ty = &cx.types[ty];
2592                (ty.ok, ty.err)
2593            }
2594            _ => bad_type_info(),
2595        };
2596        // Note that this implementation specifically isn't trying to actually
2597        // reinterpret or alter the bits of `lower` depending on which variant
2598        // we're lifting. This ends up all working out because the value is
2599        // stored in little-endian format.
2600        //
2601        // When stored in little-endian format the `{T,E}::Lower`, when each
2602        // individual `ValRaw` is read, means that if an i64 value, extended
2603        // from an i32 value, was stored then when the i32 value is read it'll
2604        // automatically ignore the upper bits.
2605        //
2606        // This "trick" allows us to seamlessly pass through the `Self::Lower`
2607        // representation into the lifting/lowering without trying to handle
2608        // "join"ed types as per the canonical ABI. It just so happens that i64
2609        // bits will naturally be reinterpreted as f64. Additionally if the
2610        // joined type is i64 but only the lower bits are read that's ok and we
2611        // don't need to validate the upper bits.
2612        //
2613        // This is largely enabled by WebAssembly/component-model#35 where no
2614        // validation needs to be performed for ignored bits and bytes here.
2615        Ok(match src.tag.get_i32() {
2616            0 => Ok(unsafe { lift_option(cx, ok, &src.payload.ok)? }),
2617            1 => Err(unsafe { lift_option(cx, err, &src.payload.err)? }),
2618            _ => bail!("invalid expected discriminant"),
2619        })
2620    }
2621
2622    #[inline]
2623    fn linear_lift_from_memory(
2624        cx: &mut LiftContext<'_>,
2625        ty: InterfaceType,
2626        bytes: &[u8],
2627    ) -> Result<Self> {
2628        debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2629        let discrim = bytes[0];
2630        let payload = &bytes[Self::INFO.payload_offset32 as usize..];
2631        let (ok, err) = match ty {
2632            InterfaceType::Result(ty) => {
2633                let ty = &cx.types[ty];
2634                (ty.ok, ty.err)
2635            }
2636            _ => bad_type_info(),
2637        };
2638        match discrim {
2639            0 => Ok(Ok(load_option(cx, ok, &payload[..T::SIZE32])?)),
2640            1 => Ok(Err(load_option(cx, err, &payload[..E::SIZE32])?)),
2641            _ => bail!("invalid expected discriminant"),
2642        }
2643    }
2644}
2645
2646fn lift_option<T>(cx: &mut LiftContext<'_>, ty: Option<InterfaceType>, src: &T::Lower) -> Result<T>
2647where
2648    T: Lift,
2649{
2650    match ty {
2651        Some(ty) => T::linear_lift_from_flat(cx, ty, src),
2652        None => Ok(empty_lift()),
2653    }
2654}
2655
2656fn load_option<T>(cx: &mut LiftContext<'_>, ty: Option<InterfaceType>, bytes: &[u8]) -> Result<T>
2657where
2658    T: Lift,
2659{
2660    match ty {
2661        Some(ty) => T::linear_lift_from_memory(cx, ty, bytes),
2662        None => Ok(empty_lift()),
2663    }
2664}
2665
2666fn empty_lift<T>() -> T
2667where
2668    T: Lift,
2669{
2670    assert!(T::IS_RUST_UNIT_TYPE);
2671    assert_eq!(mem::size_of::<T>(), 0);
2672    unsafe { MaybeUninit::uninit().assume_init() }
2673}
2674
2675/// Helper structure to define `Lower` for tuples below.
2676///
2677/// Uses default type parameters to have fields be zero-sized and not present
2678/// in memory for smaller tuple values.
2679#[expect(non_snake_case, reason = "more amenable to macro-generated code")]
2680#[doc(hidden)]
2681#[derive(Clone, Copy)]
2682#[repr(C)]
2683pub struct TupleLower<
2684    T1 = (),
2685    T2 = (),
2686    T3 = (),
2687    T4 = (),
2688    T5 = (),
2689    T6 = (),
2690    T7 = (),
2691    T8 = (),
2692    T9 = (),
2693    T10 = (),
2694    T11 = (),
2695    T12 = (),
2696    T13 = (),
2697    T14 = (),
2698    T15 = (),
2699    T16 = (),
2700    T17 = (),
2701> {
2702    // NB: these names match the names in `for_each_function_signature!`
2703    A1: T1,
2704    A2: T2,
2705    A3: T3,
2706    A4: T4,
2707    A5: T5,
2708    A6: T6,
2709    A7: T7,
2710    A8: T8,
2711    A9: T9,
2712    A10: T10,
2713    A11: T11,
2714    A12: T12,
2715    A13: T13,
2716    A14: T14,
2717    A15: T15,
2718    A16: T16,
2719    A17: T17,
2720    _align_tuple_lower0_correctly: [ValRaw; 0],
2721}
2722
2723macro_rules! impl_component_ty_for_tuples {
2724    ($n:tt $($t:ident)*) => {
2725        #[allow(non_snake_case, reason = "macro-generated code")]
2726        unsafe impl<$($t,)*> ComponentType for ($($t,)*)
2727            where $($t: ComponentType),*
2728        {
2729            type Lower = TupleLower<$($t::Lower),*>;
2730
2731            const ABI: CanonicalAbiInfo = CanonicalAbiInfo::record_static(&[
2732                $($t::ABI),*
2733            ]);
2734
2735            const IS_RUST_UNIT_TYPE: bool = {
2736                let mut _is_unit = true;
2737                $(
2738                    let _anything_to_bind_the_macro_variable = $t::IS_RUST_UNIT_TYPE;
2739                    _is_unit = false;
2740                )*
2741                _is_unit
2742            };
2743
2744            fn typecheck(
2745                ty: &InterfaceType,
2746                types: &InstanceType<'_>,
2747            ) -> Result<()> {
2748                typecheck_tuple(ty, types, &[$($t::typecheck),*])
2749            }
2750        }
2751
2752        #[allow(non_snake_case, reason = "macro-generated code")]
2753        unsafe impl<$($t,)*> Lower for ($($t,)*)
2754            where $($t: Lower),*
2755        {
2756            fn linear_lower_to_flat<U>(
2757                &self,
2758                cx: &mut LowerContext<'_, U>,
2759                ty: InterfaceType,
2760                _dst: &mut MaybeUninit<Self::Lower>,
2761            ) -> Result<()> {
2762                let types = match ty {
2763                    InterfaceType::Tuple(t) => &cx.types[t].types,
2764                    _ => bad_type_info(),
2765                };
2766                let ($($t,)*) = self;
2767                let mut _types = types.iter();
2768                $(
2769                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2770                    $t.linear_lower_to_flat(cx, ty, map_maybe_uninit!(_dst.$t))?;
2771                )*
2772                Ok(())
2773            }
2774
2775            fn linear_lower_to_memory<U>(
2776                &self,
2777                cx: &mut LowerContext<'_, U>,
2778                ty: InterfaceType,
2779                mut _offset: usize,
2780            ) -> Result<()> {
2781                debug_assert!(_offset % (Self::ALIGN32 as usize) == 0);
2782                let types = match ty {
2783                    InterfaceType::Tuple(t) => &cx.types[t].types,
2784                    _ => bad_type_info(),
2785                };
2786                let ($($t,)*) = self;
2787                let mut _types = types.iter();
2788                $(
2789                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2790                    $t.linear_lower_to_memory(cx, ty, $t::ABI.next_field32_size(&mut _offset))?;
2791                )*
2792                Ok(())
2793            }
2794        }
2795
2796        #[allow(non_snake_case, reason = "macro-generated code")]
2797        unsafe impl<$($t,)*> Lift for ($($t,)*)
2798            where $($t: Lift),*
2799        {
2800            #[inline]
2801            fn linear_lift_from_flat(cx: &mut LiftContext<'_>, ty: InterfaceType, _src: &Self::Lower) -> Result<Self> {
2802                let types = match ty {
2803                    InterfaceType::Tuple(t) => &cx.types[t].types,
2804                    _ => bad_type_info(),
2805                };
2806                let mut _types = types.iter();
2807                Ok(($(
2808                    $t::linear_lift_from_flat(
2809                        cx,
2810                        *_types.next().unwrap_or_else(bad_type_info),
2811                        &_src.$t,
2812                    )?,
2813                )*))
2814            }
2815
2816            #[inline]
2817            fn linear_lift_from_memory(cx: &mut LiftContext<'_>, ty: InterfaceType, bytes: &[u8]) -> Result<Self> {
2818                debug_assert!((bytes.as_ptr() as usize) % (Self::ALIGN32 as usize) == 0);
2819                let types = match ty {
2820                    InterfaceType::Tuple(t) => &cx.types[t].types,
2821                    _ => bad_type_info(),
2822                };
2823                let mut _types = types.iter();
2824                let mut _offset = 0;
2825                $(
2826                    let ty = *_types.next().unwrap_or_else(bad_type_info);
2827                    let $t = $t::linear_lift_from_memory(cx, ty, &bytes[$t::ABI.next_field32_size(&mut _offset)..][..$t::SIZE32])?;
2828                )*
2829                Ok(($($t,)*))
2830            }
2831        }
2832
2833        #[allow(non_snake_case, reason = "macro-generated code")]
2834        unsafe impl<$($t,)*> ComponentNamedList for ($($t,)*)
2835            where $($t: ComponentType),*
2836        {}
2837    };
2838}
2839
2840for_each_function_signature!(impl_component_ty_for_tuples);
2841
2842pub fn desc(ty: &InterfaceType) -> &'static str {
2843    match ty {
2844        InterfaceType::U8 => "u8",
2845        InterfaceType::S8 => "s8",
2846        InterfaceType::U16 => "u16",
2847        InterfaceType::S16 => "s16",
2848        InterfaceType::U32 => "u32",
2849        InterfaceType::S32 => "s32",
2850        InterfaceType::U64 => "u64",
2851        InterfaceType::S64 => "s64",
2852        InterfaceType::Float32 => "f32",
2853        InterfaceType::Float64 => "f64",
2854        InterfaceType::Bool => "bool",
2855        InterfaceType::Char => "char",
2856        InterfaceType::String => "string",
2857        InterfaceType::List(_) => "list",
2858        InterfaceType::Tuple(_) => "tuple",
2859        InterfaceType::Option(_) => "option",
2860        InterfaceType::Result(_) => "result",
2861
2862        InterfaceType::Record(_) => "record",
2863        InterfaceType::Variant(_) => "variant",
2864        InterfaceType::Flags(_) => "flags",
2865        InterfaceType::Enum(_) => "enum",
2866        InterfaceType::Own(_) => "owned resource",
2867        InterfaceType::Borrow(_) => "borrowed resource",
2868        InterfaceType::Future(_) => "future",
2869        InterfaceType::Stream(_) => "stream",
2870        InterfaceType::ErrorContext(_) => "error-context",
2871    }
2872}
2873
2874#[cold]
2875#[doc(hidden)]
2876pub fn bad_type_info<T>() -> T {
2877    // NB: should consider something like `unreachable_unchecked` here if this
2878    // becomes a performance bottleneck at some point, but that also comes with
2879    // a tradeoff of propagating a lot of unsafety, so it may not be worth it.
2880    panic!("bad type information detected");
2881}