Skip to main content

wasmtime/runtime/component/
concurrent.rs

1//! Runtime support for the Component Model Async ABI.
2//!
3//! This module and its submodules provide host runtime support for Component
4//! Model Async features such as async-lifted exports, async-lowered imports,
5//! streams, futures, and related intrinsics.  See [the Async
6//! Explainer](https://github.com/WebAssembly/component-model/blob/main/design/mvp/Concurrency.md)
7//! for a high-level overview.
8//!
9//! At the core of this support is an event loop which schedules and switches
10//! between guest tasks and any host tasks they create.  Each
11//! `Store` will have at most one event loop running at any given
12//! time, and that loop may be suspended and resumed by the host embedder using
13//! e.g. `StoreContextMut::run_concurrent`.  The `StoreContextMut::poll_until`
14//! function contains the loop itself, while the
15//! `StoreOpaque::concurrent_state` field holds its state.
16//!
17//! # Public API Overview
18//!
19//! ## Top-level API (e.g. kicking off host->guest calls and driving the event loop)
20//!
21//! - `[Typed]Func::call_concurrent`: Start a host->guest call to an
22//! async-lifted or sync-lifted import, creating a guest task.
23//!
24//! - `StoreContextMut::run_concurrent`: Run the event loop for the specified
25//! instance, allowing any and all tasks belonging to that instance to make
26//! progress.
27//!
28//! - `StoreContextMut::spawn`: Run a background task as part of the event loop
29//! for the specified instance.
30//!
31//! - `{Future,Stream}Reader::new`: Create a new Component Model `future` or
32//! `stream` which may be passed to the guest.  This takes a
33//! `{Future,Stream}Producer` implementation which will be polled for items when
34//! the consumer requests them.
35//!
36//! - `{Future,Stream}Reader::pipe`: Consume a `future` or `stream` by
37//! connecting it to a `{Future,Stream}Consumer` which will consume any items
38//! produced by the write end.
39//!
40//! ## Host Task API (e.g. implementing concurrent host functions and background tasks)
41//!
42//! - `LinkerInstance::func_wrap_concurrent`: Register a concurrent host
43//! function with the linker.  That function will take an `Accessor` as its
44//! first parameter, which provides access to the store between (but not across)
45//! await points.
46//!
47//! - `Accessor::with`: Access the store and its associated data.
48//!
49//! - `Accessor::spawn`: Run a background task as part of the event loop for the
50//! store.  This is equivalent to `StoreContextMut::spawn` but more convenient to use
51//! in host functions.
52
53use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57    HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70    CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71    InstanceState,
72};
73use crate::vm::{
74    AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98    CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99    MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100    RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101    TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102    TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113    Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114    FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115    StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136/// Constant defined in the Component Model spec to indicate that the async
137/// intrinsic (e.g. `future.write`) has not yet completed.
138const BLOCKED: u32 = 0xffff_ffff;
139
140/// Corresponds to `CallState` in the upstream spec.
141#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143    Starting = 0,
144    Started = 1,
145    Returned = 2,
146    StartCancelled = 3,
147    ReturnCancelled = 4,
148}
149
150impl Status {
151    /// Packs this status and the optional `waitable` provided into a 32-bit
152    /// result that the canonical ABI requires.
153    ///
154    /// The low 4 bits are reserved for the status while the upper 28 bits are
155    /// the waitable, if present.
156    pub fn pack(self, waitable: Option<u32>) -> u32 {
157        assert!(matches!(self, Status::Returned) == waitable.is_none());
158        let waitable = waitable.unwrap_or(0);
159        assert!(waitable < (1 << 28));
160        (waitable << 4) | (self as u32)
161    }
162}
163
164/// Corresponds to `EventCode` in the Component Model spec, plus related payload
165/// data.
166#[derive(Clone, Copy, Debug)]
167enum Event {
168    None,
169    Subtask {
170        status: Status,
171    },
172    StreamRead {
173        code: ReturnCode,
174        pending: Option<(TypeStreamTableIndex, u32)>,
175    },
176    StreamWrite {
177        code: ReturnCode,
178        pending: Option<(TypeStreamTableIndex, u32)>,
179    },
180    FutureRead {
181        code: ReturnCode,
182        pending: Option<(TypeFutureTableIndex, u32)>,
183    },
184    FutureWrite {
185        code: ReturnCode,
186        pending: Option<(TypeFutureTableIndex, u32)>,
187    },
188    Cancelled,
189}
190
191impl Event {
192    /// Lower this event to core Wasm integers for delivery to the guest.
193    ///
194    /// Note that the waitable handle, if any, is assumed to be lowered
195    /// separately.
196    fn parts(self) -> (u32, u32) {
197        const EVENT_NONE: u32 = 0;
198        const EVENT_SUBTASK: u32 = 1;
199        const EVENT_STREAM_READ: u32 = 2;
200        const EVENT_STREAM_WRITE: u32 = 3;
201        const EVENT_FUTURE_READ: u32 = 4;
202        const EVENT_FUTURE_WRITE: u32 = 5;
203        const EVENT_CANCELLED: u32 = 6;
204        match self {
205            Event::None => (EVENT_NONE, 0),
206            Event::Cancelled => (EVENT_CANCELLED, 0),
207            Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208            Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209            Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210            Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211            Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212        }
213    }
214}
215
216/// Corresponds to `CallbackCode` in the spec.
217mod callback_code {
218    pub const EXIT: u32 = 0;
219    pub const YIELD: u32 = 1;
220    pub const WAIT: u32 = 2;
221}
222
223// Flags passed to the `start-call` intrinsic.
224const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227/// Provides access to either store data (via the `get` method) or the store
228/// itself (via [`AsContext`]/[`AsContextMut`]), as well as the component
229/// instance to which the current host task belongs.
230///
231/// See [`Accessor::with`] for details.
232pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233    store: StoreContextMut<'a, T>,
234    get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239    D: HasData + ?Sized,
240    T: 'static,
241{
242    /// Creates a new [`Access`] from its component parts.
243    pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244        Self { store, get_data }
245    }
246
247    /// Get mutable access to the store data.
248    pub fn data_mut(&mut self) -> &mut T {
249        self.store.data_mut()
250    }
251
252    /// Get mutable access to the store data.
253    pub fn get(&mut self) -> D::Data<'_> {
254        (self.get_data)(self.data_mut())
255    }
256
257    /// Spawn a background task.
258    ///
259    /// See [`Accessor::spawn`] for details.
260    pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261    where
262        T: 'static,
263    {
264        let accessor = Accessor {
265            get_data: self.get_data,
266            token: StoreToken::new(self.store.as_context_mut()),
267        };
268        self.store
269            .as_context_mut()
270            .spawn_with_accessor(accessor, task)
271    }
272
273    /// Returns the getter this accessor is using to project from `T` into
274    /// `D::Data`.
275    pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276        self.get_data
277    }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282    D: HasData + ?Sized,
283    T: 'static,
284{
285    type Data = T;
286
287    fn as_context(&self) -> StoreContext<'_, T> {
288        self.store.as_context()
289    }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294    D: HasData + ?Sized,
295    T: 'static,
296{
297    fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298        self.store.as_context_mut()
299    }
300}
301
302/// Provides scoped mutable access to store data in the context of a concurrent
303/// host task future.
304///
305/// This allows multiple host task futures to execute concurrently and access
306/// the store between (but not across) `await` points.
307///
308/// # Rationale
309///
310/// This structure is sort of like `&mut T` plus a projection from `&mut T` to
311/// `D::Data<'_>`. The problem this is solving, however, is that it does not
312/// literally store these values. The basic problem is that when a concurrent
313/// host future is being polled it has access to `&mut T` (and the whole
314/// `Store`) but when it's not being polled it does not have access to these
315/// values. This reflects how the store is only ever polling one future at a
316/// time so the store is effectively being passed between futures.
317///
318/// Rust's `Future` trait, however, has no means of passing a `Store`
319/// temporarily between futures. The [`Context`](core::task::Context) type does
320/// not have the ability to attach arbitrary information to it at this time.
321/// This type, [`Accessor`], is used to bridge this expressivity gap.
322///
323/// The [`Accessor`] type here represents the ability to acquire, temporarily in
324/// a synchronous manner, the current store. The [`Accessor::with`] function
325/// yields an [`Access`] which can be used to access [`StoreContextMut`], `&mut
326/// T`, or `D::Data<'_>`. Note though that [`Accessor::with`] intentionally does
327/// not take an `async` closure as its argument, instead it's a synchronous
328/// closure which must complete during on run of `Future::poll`. This reflects
329/// how the store is temporarily made available while a host future is being
330/// polled.
331///
332/// # Implementation
333///
334/// This type does not actually store `&mut T` nor `StoreContextMut<T>`, and
335/// this type additionally doesn't even have a lifetime parameter. This is
336/// instead a representation of proof of the ability to acquire these while a
337/// future is being polled. Wasmtime will, when it polls a host future,
338/// configure ambient state such that the `Accessor` that a future closes over
339/// will work and be able to access the store.
340///
341/// This has a number of implications for users such as:
342///
343/// * It's intentional that `Accessor` cannot be cloned, it needs to stay within
344///   the lifetime of a single future.
345/// * A future is expected to, however, close over an `Accessor` and keep it
346///   alive probably for the duration of the entire future.
347/// * Different host futures will be given different `Accessor`s, and that's
348///   intentional.
349/// * The `Accessor` type is `Send` and `Sync` irrespective of `T` which
350///   alleviates some otherwise required bounds to be written down.
351///
352/// # Using `Accessor` in `Drop`
353///
354/// The methods on `Accessor` are only expected to work in the context of
355/// `Future::poll` and are not guaranteed to work in `Drop`. This is because a
356/// host future can be dropped at any time throughout the system and Wasmtime
357/// store context is not necessarily available at that time. It's recommended to
358/// not use `Accessor` methods in anything connected to a `Drop` implementation
359/// as they will panic and have unintended results. If you run into this though
360/// feel free to file an issue on the Wasmtime repository.
361pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363    D: HasData + ?Sized,
364{
365    token: StoreToken<T>,
366    get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369/// A helper trait to take any type of accessor-with-data in functions.
370///
371/// This trait is similar to [`AsContextMut`] except that it's used when
372/// working with an [`Accessor`] instead of a [`StoreContextMut`]. The
373/// [`Accessor`] is the main type used in concurrent settings and is passed to
374/// functions such as [`Func::call_concurrent`].
375///
376/// This trait is implemented for [`Accessor`] and `&T` where `T` implements
377/// this trait. This effectively means that regardless of the `D` in
378/// `Accessor<T, D>` it can still be passed to a function which just needs a
379/// store accessor.
380///
381/// Acquiring an [`Accessor`] can be done through
382/// [`StoreContextMut::run_concurrent`] for example or in a host function
383/// through
384/// [`Linker::func_wrap_concurrent`](crate::component::LinkerInstance::func_wrap_concurrent).
385pub trait AsAccessor {
386    /// The `T` in `Store<T>` that this accessor refers to.
387    type Data: 'static;
388
389    /// The `D` in `Accessor<T, D>`, or the projection out of
390    /// `Self::Data`.
391    type AccessorData: HasData + ?Sized;
392
393    /// Returns the accessor that this is referring to.
394    fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398    type Data = T::Data;
399    type AccessorData = T::AccessorData;
400
401    fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402        T::as_accessor(self)
403    }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407    type Data = T;
408    type AccessorData = D;
409
410    fn as_accessor(&self) -> &Accessor<T, D> {
411        self
412    }
413}
414
415// Note that it is intentional at this time that `Accessor` does not actually
416// store `&mut T` or anything similar. This distinctly enables the `Accessor`
417// structure to be both `Send` and `Sync` regardless of what `T` is (or `D` for
418// that matter). This is used to ergonomically simplify bindings where the
419// majority of the time `Accessor` is closed over in a future which then needs
420// to be `Send` and `Sync`. To avoid needing to write `T: Send` everywhere (as
421// you already have to write `T: 'static`...) it helps to avoid this.
422//
423// Note as well that `Accessor` doesn't actually store its data at all. Instead
424// it's more of a "proof" of what can be accessed from TLS. API design around
425// `Accessor` and functions like `Linker::func_wrap_concurrent` are
426// intentionally made to ensure that `Accessor` is ideally only used in the
427// context that TLS variables are actually set. For example host functions are
428// given `&Accessor`, not `Accessor`, and this prevents them from persisting
429// the value outside of a future. Within the future the TLS variables are all
430// guaranteed to be set while the future is being polled.
431//
432// Finally though this is not an ironclad guarantee, but nor does it need to be.
433// The TLS APIs are designed to panic or otherwise model usage where they're
434// called recursively or similar. It's hoped that code cannot be constructed to
435// actually hit this at runtime but this is not a safety requirement at this
436// time.
437const _: () = {
438    const fn assert<T: Send + Sync>() {}
439    assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443    /// Creates a new `Accessor` backed by the specified functions.
444    ///
445    /// - `get`: used to retrieve the store
446    ///
447    /// - `get_data`: used to "project" from the store's associated data to
448    /// another type (e.g. a field of that data or a wrapper around it).
449    ///
450    /// - `spawn`: used to queue spawned background tasks to be run later
451    pub(crate) fn new(token: StoreToken<T>) -> Self {
452        Self {
453            token,
454            get_data: |x| x,
455        }
456    }
457}
458
459impl<T, D> Accessor<T, D>
460where
461    D: HasData + ?Sized,
462{
463    /// Run the specified closure, passing it mutable access to the store.
464    ///
465    /// This function is one of the main building blocks of the [`Accessor`]
466    /// type. This yields synchronous, blocking, access to the store via an
467    /// [`Access`]. The [`Access`] implements [`AsContextMut`] in addition to
468    /// providing the ability to access `D` via [`Access::get`]. Note that the
469    /// `fun` here is given only temporary access to the store and `T`/`D`
470    /// meaning that the return value `R` here is not allowed to capture borrows
471    /// into the two. If access is needed to data within `T` or `D` outside of
472    /// this closure then it must be `clone`d out, for example.
473    ///
474    /// # Panics
475    ///
476    /// This function will panic if it is call recursively with any other
477    /// accessor already in scope. For example if `with` is called within `fun`,
478    /// then this function will panic. It is up to the embedder to ensure that
479    /// this does not happen.
480    pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481        tls::get(|vmstore| {
482            fun(Access {
483                store: self.token.as_context_mut(vmstore),
484                get_data: self.get_data,
485            })
486        })
487    }
488
489    /// Returns the getter this accessor is using to project from `T` into
490    /// `D::Data`.
491    pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492        self.get_data
493    }
494
495    /// Changes this accessor to access `D2` instead of the current type
496    /// parameter `D`.
497    ///
498    /// This changes the underlying data access from `T` to `D2::Data<'_>`.
499    ///
500    /// # Panics
501    ///
502    /// When using this API the returned value is disconnected from `&self` and
503    /// the lifetime binding the `self` argument. An `Accessor` only works
504    /// within the context of the closure or async closure that it was
505    /// originally given to, however. This means that due to the fact that the
506    /// returned value has no lifetime connection it's possible to use the
507    /// accessor outside of `&self`, the original accessor, and panic.
508    ///
509    /// The returned value should only be used within the scope of the original
510    /// `Accessor` that `self` refers to.
511    pub fn with_getter<D2: HasData>(
512        &self,
513        get_data: fn(&mut T) -> D2::Data<'_>,
514    ) -> Accessor<T, D2> {
515        Accessor {
516            token: self.token,
517            get_data,
518        }
519    }
520
521    /// Spawn a background task which will receive an `&Accessor<T, D>` and
522    /// run concurrently with any other tasks in progress for the current
523    /// store.
524    ///
525    /// This is particularly useful for host functions which return a `stream`
526    /// or `future` such that the code to write to the write end of that
527    /// `stream` or `future` must run after the function returns.
528    ///
529    /// The returned [`JoinHandle`] may be used to cancel the task.
530    ///
531    /// # Panics
532    ///
533    /// Panics if called within a closure provided to the [`Accessor::with`]
534    /// function. This can only be called outside an active invocation of
535    /// [`Accessor::with`].
536    pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537    where
538        T: 'static,
539    {
540        let accessor = self.clone_for_spawn();
541        self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542    }
543
544    fn clone_for_spawn(&self) -> Self {
545        Self {
546            token: self.token,
547            get_data: self.get_data,
548        }
549    }
550
551    /// Polls to see if this store contains any "interesting" tasks still within
552    /// it.
553    ///
554    /// Returns `Poll::Ready(())` if there are no more interesting tasks, and
555    /// otherwise returns `Poll::Pending`. If pending is returned then whenever
556    /// the last remaining "interesting" task has exited the provided context's
557    /// waker will be notified. Note that only the waker passed to the last call
558    /// to `poll_no_interesting_tasks` for the store will be notified, so this
559    /// is only appropriate to use once-at-a-time per store.
560    ///
561    /// The component model specification, as of this current date, does not
562    /// have a distinction between "interesting" tasks and not. The current
563    /// intention is that in a future revision of the component model this will
564    /// be distinguished at the component ABI level where tasks will be able to
565    /// flag themselves as "interesting" optionally. Additionally extra work can
566    /// be opted-in to being "interesting".
567    ///
568    /// For now what this means is that all component model tasks within this
569    /// store are considered interesting. This specifically includes the entire
570    /// duration of a task, so even all of the time after a task has returned
571    /// but before it has exited. This means that this function is, today,
572    /// effectively a proxy for "are there any more tasks still running in this
573    /// store". This can be used by embedders to determine whether there's any
574    /// more work going on, even in the background, for a particular guest.
575    /// Hosts can use this as a signal that the guest wants to stay alive a
576    /// little longer, even after a task has returned.
577    ///
578    /// In the future this predicate won't include all tasks in this store. Some
579    /// tasks will be able to flag themselves as not interesting, meaning that
580    /// when this returns ready it'd be possible that there are still tasks
581    /// remaining in the store.
582    ///
583    /// Note that at this time spawned threads within a task are always
584    /// considered uninteresting. If this function returns ready, then spawned
585    /// threads may still be in the store.
586    pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587        self.with(|mut access| {
588            let store = access.as_context_mut().0;
589            let state = store.concurrent_state_mut_without_forcing_current_thread();
590            if state.interesting_tasks == 0 {
591                Poll::Ready(())
592            } else {
593                state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594                Poll::Pending
595            }
596        })
597    }
598
599    /// Poll to see if the component instance corresponding to the specified
600    /// function is ready to run a concurrent call without queuing it.
601    ///
602    /// This will return `Poll::Ready(())` only if all of the following are true:
603    ///
604    /// - The store to which the instance belongs has no work items queued
605    ///   (e.g. events to deliver to the guest, etc.)
606    ///
607    /// - The instance does not have backpressure enabled.
608    ///
609    /// - The instance does not have a sync call in progress (which implicitly
610    ///   enables backpressure).
611    ///
612    /// Otherwise, it will return `Poll::Pending`.  If pending is returned then
613    /// whenever _any_ instance becomes ready for a call the provided context's
614    /// waker will be notified.  Note that only the waker passed to the last
615    /// call to `poll_ready_for_concurrent_call` for the store will be notified
616    /// (regardless of whether the same or different `Func` is specified
617    /// relative to earlier calls), so this is only appropriate to use
618    /// once-at-a-time per store.  Again, note that the waker may be notified
619    /// when _any_ instance becomes callable (i.e. not necessarily the last one
620    /// polled), so this function must be called again to determine if the
621    /// instance of interest is ready.
622    pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623        self.with(|mut access| {
624            let store = access.as_context_mut().0;
625
626            {
627                let state = store.concurrent_state_mut_without_forcing_current_thread();
628                if state.switch_item.is_some()
629                    || !state.high_priority.is_empty()
630                    || !state.low_priority.is_empty()
631                {
632                    state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633                    return Poll::Pending;
634                }
635            }
636
637            let (_, _, _, raw_options) = func.abi_info(store);
638            let instance = func.instance().runtime_instance(raw_options.instance);
639            let state = store.instance_state(instance).concurrent_state();
640            if state.backpressure == 0 {
641                Poll::Ready(())
642            } else {
643                store
644                    .concurrent_state_mut_without_forcing_current_thread()
645                    .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646                Poll::Pending
647            }
648        })
649    }
650}
651
652/// Represents an async closure which may be provided to `Accessor::spawn`,
653/// `Accessor::forward`, or `StoreContextMut::spawn`.
654// TODO: Replace this with `core::ops::AsyncFnOnce` when we are able to put `Send`
655// bound on the unnamed `Future` directly.
656//
657// As of this writing, it's not possible to specify e.g. `Send` and `Sync`
658// bounds on the `Future` type returned by an `AsyncFnOnce`.  Also, using `F:
659// Future<Output = Result<()>> + Send + Sync, FN: FnOnce(&Accessor<T>) -> F +
660// Send + Sync + 'static` fails with a type mismatch error as we cannot describe
661// that `F` should have `&Accessor<T>`'s unnamed lifetime
662//
663// Instead, this trait is used as a workaround for this limitation, the bound on `Self`
664// implementing `AsyncFnOnce()` is required for Rust to automatically infer that an async
665// closure is to be provided wherever we are accepting `impl for<'fut> AccessorTask<'fut, T, D>`
666// as argument. Otherwise, users will have to fully qualify the closure types before it
667// is accepted as a valid value. This also means that it is not intended for a user
668// to manually implement this trait for any arbitrary type, since they would first
669// have to implement `AsyncFnOnce`, which is unstable.
670//
671// The blanket implementation for this trait will ensure that `Self` is an async closure
672// that returns a `Future` that is `Send`, and lives as long as `&Accessor<T, D>`
673pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674    AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676    D: HasData + ?Sized,
677{
678    /// Run the task.
679    fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684    T: 'static,
685    F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686    F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687    Fut: Future<Output = Result<()>> + Send + 'fut,
688    D: HasData,
689{
690    fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691        (self)(accessor)
692    }
693}
694
695/// Represents parameter and result metadata for the caller side of a
696/// guest->guest call orchestrated by a fused adapter.
697enum CallerInfo {
698    /// Metadata for a call to an async-lowered import
699    Async {
700        params: Vec<ValRaw>,
701        has_result: bool,
702    },
703    /// Metadata for a call to an sync-lowered import
704    Sync {
705        params: Vec<ValRaw>,
706        result_count: u32,
707    },
708}
709
710/// Indicates how a guest task is waiting on a waitable set.
711enum WaitMode {
712    /// The guest task is waiting using `task.wait`
713    Fiber(StoreFiber<'static>),
714    /// The guest task is waiting via a callback declared as part of an
715    /// async-lifted export.
716    Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721        match self {
722            Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723            Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724        }
725    }
726}
727
728/// Represents the reason a fiber is suspending itself.
729#[derive(Debug)]
730enum SuspendReason {
731    /// The fiber is waiting for an event to be delivered to the specified
732    /// waitable set or task.
733    Waiting {
734        set: TableId<WaitableSet>,
735        thread: QualifiedThreadId,
736    },
737    /// The fiber is waiting for a subtask to suspend or exit, e.g. for a
738    /// guest-to-guest call or a `subtask.cancel`.
739    YieldingToSubtask { thread: QualifiedThreadId },
740    /// The fiber has finished handling its most recent work item and is waiting
741    /// for another (or to be dropped if it is no longer needed).
742    NeedWork,
743    /// The fiber is yielding and should be resumed once other tasks have had a
744    /// chance to run.
745    Yielding { thread: QualifiedThreadId },
746    /// The fiber was explicitly suspended with a call to `thread.suspend` or
747    /// `thread.switch-to`.
748    ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751/// Represents a pending call into guest code for a given guest task.
752enum GuestCallKind {
753    /// Indicates there's an event to deliver to the task, possibly related to a
754    /// waitable set the task has been waiting on or polling.
755    DeliverEvent {
756        /// The instance to which the task belongs.
757        instance: Instance,
758        /// The waitable set the event belongs to, if any.
759        ///
760        /// If this is `None` the event will be waiting in the
761        /// `GuestTask::event` field for the task.
762        set: Option<TableId<WaitableSet>>,
763    },
764    /// Indicates that a new guest task call is pending and may be executed
765    /// using the specified closure.
766    StartImplicit {
767        /// The closure to call.
768        closure: Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>,
769        /// Whether the task was previously pending due to e.g. backpressure.
770        was_pending: bool,
771    },
772    StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
773}
774
775impl fmt::Debug for GuestCallKind {
776    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
777        match self {
778            Self::DeliverEvent { instance, set } => f
779                .debug_struct("DeliverEvent")
780                .field("instance", instance)
781                .field("set", set)
782                .finish(),
783            Self::StartImplicit { was_pending, .. } => f
784                .debug_struct("StartImplicit")
785                .field("was_pending", was_pending)
786                .finish(),
787            Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
788        }
789    }
790}
791
792/// The target of a suspension intrinsic.
793#[derive(Copy, Clone, Debug)]
794pub enum SuspensionTarget {
795    Resume(u32),
796    Promote(u32),
797    None,
798}
799
800/// Behavior for `resume_thread`.
801#[derive(Copy, Clone, Debug)]
802pub enum ResumeThread {
803    Promote,
804    Resume,
805    ResumeLater,
806}
807
808/// Represents a pending call into guest code for a given guest thread.
809#[derive(Debug)]
810struct GuestCall {
811    thread: QualifiedThreadId,
812    kind: GuestCallKind,
813}
814
815impl GuestCall {
816    /// Returns whether or not the call is ready to run.
817    ///
818    /// A call will not be ready to run if either:
819    ///
820    /// - the (sub-)component instance to be called has already been entered and
821    ///   cannot be reentered until an in-progress call completes, and the task
822    ///   requires the exclusive lock
823    ///
824    /// - the call is for a not-yet started task and the (sub-)component
825    ///   instance to be called has backpressure enabled _or_ there are pending
826    ///   tasks waiting in front of this task
827    fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
828        let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
829        let async_typed = task.async_typed;
830        let async_lifted = task.async_lifted;
831        let has_callback = task.callback.is_some();
832        let instance = task.instance;
833        let state = store.instance_state(instance).concurrent_state();
834
835        let ready = match &self.kind {
836            GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
837            GuestCallKind::StartImplicit { was_pending, .. } => {
838                if async_typed {
839                    // Sync- and callback-lifted exports need the instance's
840                    // exclusive lock to start, but stackful ones don't.
841                    let needs_exclusive = !async_lifted || has_callback;
842
843                    // This task may enter the instance if (a) there's no
844                    // backpressure enabled, and (b) either the exclusive lock
845                    // is free or we can bypass the lock per the
846                    // `needs_exclusive` logic above.
847                    let may_enter =
848                        !((needs_exclusive && state.do_not_enter) || state.backpressure > 0);
849
850                    // If this task is already waiting in line (or was
851                    // previously in line before getting the go-ahead) or there
852                    // is no line to wait in, we can skip that.  Otherwise, we
853                    // need to get in line to avoid barging ahead of others that
854                    // have been waiting for backpressure to clear, even if
855                    // backpressure is no longer enabled.
856                    let skip_wait = *was_pending || state.pending_start_count == 0;
857
858                    // This task is ready if it may enter the instance _and_
859                    // there's no need to get in line first.
860                    may_enter && skip_wait
861                } else {
862                    true
863                }
864            }
865            GuestCallKind::StartExplicit(_) => true,
866        };
867        log::trace!(
868            "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {}; pending_start_count: {})",
869            state.do_not_enter,
870            state.backpressure,
871            state.pending_start_count,
872        );
873        Ok(ready)
874    }
875}
876
877/// Job to be run on a worker fiber.
878enum WorkerItem {
879    GuestCall(GuestCall),
880    Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
881}
882
883/// Represents a pending work item to be handled by the event loop for a given
884/// component instance.
885enum WorkItem {
886    /// A host task to be pushed to `ConcurrentState::futures`.
887    PushFuture(AlwaysMut<HostTaskFuture>),
888    /// A fiber to resume.
889    ResumeFiber {
890        instance: RuntimeInstance,
891        thread: QualifiedThreadId,
892        fiber: StoreFiber<'static>,
893    },
894    /// A thread to resume.
895    ResumeThread {
896        instance: RuntimeInstance,
897        thread: QualifiedThreadId,
898    },
899    /// A pending call into guest code for a given guest task.
900    GuestCall {
901        instance: RuntimeInstance,
902        call: GuestCall,
903    },
904    /// A job to run on a worker fiber.
905    WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
906}
907
908/// A fiber temporarily outside the store's scheduler state.
909///
910/// Construct this before returning a future so cancellation before its first
911/// poll also disposes of the fiber. Successful handoffs take the fiber only
912/// after all fallible destination checks have completed.
913struct ResumingFiber<'a> {
914    store: &'a mut StoreOpaque,
915    fiber: Option<StoreFiber<'static>>,
916}
917
918impl Drop for ResumingFiber<'_> {
919    fn drop(&mut self) {
920        if let Some(mut fiber) = self.fiber.take() {
921            fiber.dispose(self.store);
922        }
923    }
924}
925
926impl fmt::Debug for WorkItem {
927    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
928        match self {
929            Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
930            Self::ResumeFiber {
931                instance, thread, ..
932            } => f
933                .debug_struct("ResumeFiber")
934                .field("instance", instance)
935                .field("thread", thread)
936                .finish(),
937            Self::ResumeThread { instance, thread } => f
938                .debug_struct("ResumeThread")
939                .field("instance", instance)
940                .field("thread", thread)
941                .finish(),
942            Self::GuestCall { instance, call } => f
943                .debug_struct("GuestCall")
944                .field("instance", instance)
945                .field("call", call)
946                .finish(),
947            Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
948        }
949    }
950}
951
952/// Whether a suspension intrinsic was cancelled or completed
953#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
954pub(crate) enum WaitResult {
955    Cancelled,
956    Completed,
957}
958
959/// Poll the specified future until it completes on behalf of a guest->host call
960/// using a sync-lowered import.
961///
962/// This is similar to `Instance::first_poll` except it's for sync-lowered
963/// imports, meaning we don't need to handle cancellation and we can block the
964/// caller until the task completes, at which point the caller can handle
965/// lowering the result to the guest's stack and linear memory.
966pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
967    store: &mut dyn VMStore,
968    host_task: EnteredHostTask,
969    future: impl Future<Output = Result<R>> + Send + 'static,
970) -> Result<R> {
971    // Poll the future once before creating a host task. The host task will be
972    // created lazily if it's needed during the poll and otherwise will be
973    // created if the future suspends.  We can use a dummy `Waker` here because
974    // we'll add the future to `ConcurrentState::futures` and poll it
975    // automatically from the event loop if it doesn't complete immediately
976    // here.
977    let mut future = Box::pin(future);
978    let poll = tls::set(store, || {
979        future
980            .as_mut()
981            .poll(&mut Context::from_waker(&Waker::noop()))
982    });
983
984    let caller = match host_task {
985        Some(caller) => caller,
986        None => bail_bug!("host task wasn't created but should have been"),
987    };
988
989    let task = match poll {
990        // It completed immediately, so no persistent host task is needed.
991        Poll::Ready(result) => return result,
992
993        // It did not complete immediately; create the host task and add it to
994        // `ConcurrentState::futures` so it will be polled via the event loop;
995        // then use `GuestThread::sync_call_set` to wait for the task to
996        // complete, suspending the current fiber until it does so.
997        Poll::Pending => {
998            let Some(task) = store.materialize_host_task_id()? else {
999                bail_bug!("current thread is not a host thread")
1000            };
1001
1002            // Wrap the future in a closure which will stash its result in the
1003            // host task and resume this fiber when it completes.
1004            let future = Box::pin(async move {
1005                let result = run_with_host_task_set(task, future).await??;
1006                tls::get(move |store| {
1007                    let state = store.concurrent_state_mut()?;
1008                    let host_state = &mut state.get_mut(task)?.state;
1009                    assert!(matches!(host_state, HostTaskState::CalleeStarted));
1010                    *host_state = HostTaskState::CalleeFinished(Box::new(result));
1011
1012                    Waitable::Host(task).set_event(
1013                        state,
1014                        Some(Event::Subtask {
1015                            status: Status::Returned,
1016                        }),
1017                    )?;
1018
1019                    Ok(())
1020                })
1021            }) as HostTaskFuture;
1022
1023            let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
1024            store.switch_or_trap_if_may_not_suspend(caller_instance)?;
1025
1026            let state = store.concurrent_state_mut()?;
1027            state.push_future(future);
1028
1029            let set = state.get_mut(caller.thread)?.sync_call_set;
1030            Waitable::Host(task).join(state, Some(set))?;
1031
1032            store.suspend(SuspendReason::Waiting {
1033                set,
1034                thread: caller,
1035            })?;
1036
1037            // Remove the `task` from the `sync_call_set` to ensure that when
1038            // this function returns and the task is deleted that there are no
1039            // more lingering references to this host task.
1040            Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1041            task
1042        }
1043    };
1044
1045    // Retrieve and return the result.
1046    let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1047    match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1048        HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1049            Ok(result) => *result,
1050            Err(_) => bail_bug!("host task finished with wrong type of result"),
1051        }),
1052        _ => bail_bug!("unexpected host task state after completion"),
1053    }
1054}
1055
1056/// Execute the specified guest call.
1057fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1058    match call.kind {
1059        GuestCallKind::DeliverEvent { instance, set } => {
1060            // This thread's wait on `set`, if any, which started in
1061            // `Instance::wait_with_callback`, ends here.
1062            if let Some(set) = set {
1063                store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1064            }
1065            let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1066                Some(pair) => pair,
1067                None => match set {
1068                    // The event which woke this thread up was taken by another
1069                    // thread (e.g. via `waitable-set.poll`) or its waitable was
1070                    // removed from the set before this thread had a chance to
1071                    // run. Regardless, keep waiting.
1072                    Some(set) => {
1073                        log::trace!(
1074                            "event for {:?} on {set:?} no longer present; waiting again",
1075                            call.thread
1076                        );
1077                        return instance.wait_with_callback(&mut **store, call.thread, set);
1078                    }
1079                    // When a thread yields it doesn't wait on a set and the
1080                    // event to be delivered isn't stored anywhere, and this is
1081                    // just handling that case so indicate that the yield is
1082                    // complete.
1083                    None => (Event::None, None),
1084                },
1085            };
1086            let state = store.concurrent_state_mut()?;
1087            // If this thread was woken up by an event in `set` but is instead
1088            // receiving its own task's event (e.g. a cancellation), then the
1089            // set's event is left for another thread waiting on it, so wake
1090            // the next waiter up if there is one.
1091            if let Some(set) = set
1092                && waitable.is_none()
1093                && !state.get_mut(set)?.ready.is_empty()
1094            {
1095                state.wake_waiter(set)?;
1096            }
1097            let task = state.get_mut(call.thread.task)?;
1098            let runtime_instance = task.instance;
1099            let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1100
1101            log::trace!(
1102                "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1103                call.thread,
1104            );
1105
1106            instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1107                invoke_callback(store, call.thread.task, event, handle)
1108            })?;
1109        }
1110        GuestCallKind::StartImplicit { closure, .. } => {
1111            closure(store)?;
1112        }
1113        GuestCallKind::StartExplicit(fun) => {
1114            fun(store)?;
1115        }
1116    }
1117
1118    Ok(())
1119}
1120
1121/// Invoke the callback of the callback-lifted `task` with `event` and
1122/// `handle`, returning the callback code it produces.
1123fn invoke_callback(
1124    store: &mut dyn VMStore,
1125    task: TableId<GuestTask>,
1126    event: Event,
1127    handle: u32,
1128) -> Result<u32> {
1129    let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1130        bail_bug!("guest task callback field not present")
1131    };
1132
1133    let code = callback(store, event, handle)?;
1134
1135    store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1136
1137    Ok(code)
1138}
1139
1140impl<T> Store<T> {
1141    /// Convenience wrapper for [`StoreContextMut::run_concurrent`].
1142    pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1143    where
1144        T: Send + 'static,
1145    {
1146        ensure!(
1147            self.as_context().0.concurrency_support(),
1148            "cannot use `run_concurrent` when Config::concurrency_support disabled",
1149        );
1150        self.as_context_mut().run_concurrent(fun).await
1151    }
1152
1153    #[doc(hidden)]
1154    pub fn assert_concurrent_state_empty(&mut self) {
1155        self.as_context_mut().assert_concurrent_state_empty();
1156    }
1157
1158    #[doc(hidden)]
1159    pub fn concurrent_state_table_size(&mut self) -> usize {
1160        self.as_context_mut().concurrent_state_table_size()
1161    }
1162
1163    #[doc(hidden)]
1164    pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1165        self.as_context_mut()
1166            .set_max_concurrent_state_table_capacity(max)
1167    }
1168
1169    /// Convenience wrapper for [`StoreContextMut::spawn`].
1170    pub fn spawn(
1171        &mut self,
1172        task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1173    ) -> Result<JoinHandle>
1174    where
1175        T: 'static,
1176    {
1177        self.as_context_mut().spawn(task)
1178    }
1179}
1180
1181impl<T> StoreContextMut<'_, T> {
1182    /// Assert that all the relevant tables and queues in the concurrent state
1183    /// for this store are empty.
1184    ///
1185    /// This is for sanity checking in integration tests
1186    /// (e.g. `component-async-tests`) that the relevant state has been cleared
1187    /// after each test concludes.  This should help us catch leaks, e.g. guest
1188    /// tasks which haven't been deleted despite having completed and having
1189    /// been dropped by their supertasks.
1190    ///
1191    /// Only intended for use in Wasmtime's own testing.
1192    #[doc(hidden)]
1193    pub fn assert_concurrent_state_empty(self) {
1194        let store = self.0;
1195        store
1196            .store_data_mut()
1197            .components
1198            .assert_instance_states_empty();
1199        let state = store.concurrent_state_mut().unwrap();
1200        assert!(
1201            state.table.get_mut().is_empty(),
1202            "non-empty table: {:?}",
1203            state.table.get_mut()
1204        );
1205        assert!(state.switch_item.is_none());
1206        assert!(state.next_switch_item.is_none());
1207        assert!(state.high_priority.is_empty());
1208        assert!(state.low_priority.is_empty());
1209        assert!(state.saved_next_switch_items.is_empty());
1210        assert!(state.unforced_current_thread.is_none());
1211        assert!(state.deferred_host_call_context.is_none());
1212        assert!(state.futures_mut().unwrap().is_empty());
1213        assert!(state.global_error_context_ref_counts.is_empty());
1214    }
1215
1216    /// Helper function to perform tests over the size of the concurrent state
1217    /// table which can be useful for detecting leaks.
1218    ///
1219    /// Only intended for use in Wasmtime's own testing.
1220    #[doc(hidden)]
1221    pub fn concurrent_state_table_size(&mut self) -> usize {
1222        self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1223    }
1224
1225    /// Helper function to set the maximum size of the table used to store
1226    /// e.g. tasks, threads, waitables, etc.
1227    ///
1228    /// This is currently only intended for use in Wasmtime's own testing, but
1229    /// could potentially be made part of the official public API if desired.
1230    #[doc(hidden)]
1231    pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1232        self.0
1233            .concurrent_state_mut_without_forcing_current_thread()
1234            .table
1235            .get_mut()
1236            .set_max_capacity(max);
1237    }
1238
1239    /// Spawn a background task to run as part of this instance's event loop.
1240    ///
1241    /// The task will receive an `&Accessor<U>` and run concurrently with
1242    /// any other tasks in progress for the instance.
1243    ///
1244    /// Note that the task will only make progress if and when the event loop
1245    /// for this instance is run.
1246    ///
1247    /// The returned [`JoinHandle`] may be used to cancel the task.
1248    pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1249    where
1250        T: 'static,
1251    {
1252        let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1253        self.spawn_with_accessor(accessor, task)
1254    }
1255
1256    /// Internal implementation of `spawn` functions where a `store` is
1257    /// available along with an `Accessor`.
1258    fn spawn_with_accessor<D>(
1259        self,
1260        accessor: Accessor<T, D>,
1261        task: impl for<'fut> AccessorTask<'fut, T, D>,
1262    ) -> Result<JoinHandle>
1263    where
1264        T: 'static,
1265        D: HasData + ?Sized,
1266    {
1267        // Create an "abortable future" here where internally the future will
1268        // hook calls to poll and possibly spawn more background tasks on each
1269        // iteration.
1270        let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1271        self.0
1272            .concurrent_state_mut()?
1273            .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1274        Ok(handle)
1275    }
1276
1277    /// Run the specified closure `fun` to completion as part of this store's
1278    /// event loop.
1279    ///
1280    /// This will run `fun` as part of this store's event loop until it
1281    /// yields a result.  `fun` is provided an [`Accessor`], which provides
1282    /// controlled access to the store and its data.
1283    ///
1284    /// This function can be used to invoke [`Func::call_concurrent`] for
1285    /// example within the async closure provided here.
1286    ///
1287    /// This function will unconditionally return an error if
1288    /// [`Config::concurrency_support`] is disabled.
1289    ///
1290    /// [`Config::concurrency_support`]: crate::Config::concurrency_support
1291    ///
1292    /// # Store-blocking behavior
1293    ///
1294    /// At this time there are certain situations in which the `Future` returned
1295    /// by the `AsyncFnOnce` passed to this function will not be polled for an
1296    /// extended period of time, despite one or more `Waker::wake` events having
1297    /// occurred for the task to which it belongs.  This can manifest as the
1298    /// `Future` seeming to be "blocked" or "locked up", but is actually due to
1299    /// the `Store` being held by e.g. a blocking host function, preventing the
1300    /// `Future` from being polled. A canonical example of this is when the
1301    /// `fun` provided to this function attempts to set a timeout for an
1302    /// invocation of a wasm function. In this situation the async closure is
1303    /// waiting both on (a) the wasm computation to finish, and (b) the timeout
1304    /// to elapse. At this time this setup will not always work and the timeout
1305    /// may not reliably fire.
1306    ///
1307    /// This function will not block the current thread and as such is always
1308    /// suitable to run in an `async` context, but the current implementation of
1309    /// Wasmtime can lead to situations where a certain wasm computation is
1310    /// required to make progress the closure to make progress. This is an
1311    /// artifact of Wasmtime's historical implementation of `async` functions
1312    /// and is the topic of [#11869] and [#11870]. In the timeout example from
1313    /// above it means that Wasmtime can get "wedged" for a bit where (a) must
1314    /// progress for a readiness notification of (b) to get delivered.
1315    ///
1316    /// This effectively means that it's not possible to reliably perform a
1317    /// "select" operation within the `fun` closure, which timeouts for example
1318    /// are based on. Fixing this requires some relatively major refactoring
1319    /// work within Wasmtime itself. This is a known pitfall otherwise and one
1320    /// that is intended to be fixed one day. In the meantime it's recommended
1321    /// to apply timeouts or such to the entire `run_concurrent` call itself
1322    /// rather than internally.
1323    ///
1324    /// [#11869]: https://github.com/bytecodealliance/wasmtime/issues/11869
1325    /// [#11870]: https://github.com/bytecodealliance/wasmtime/issues/11870
1326    ///
1327    /// # Example
1328    ///
1329    /// ```
1330    /// # use {
1331    /// #   wasmtime::{
1332    /// #     error::{Result},
1333    /// #     component::{ Component, Linker, Resource, ResourceTable},
1334    /// #     Config, Engine, Store
1335    /// #   },
1336    /// # };
1337    /// #
1338    /// # struct MyResource(u32);
1339    /// # struct Ctx { table: ResourceTable }
1340    /// #
1341    /// # async fn foo() -> Result<()> {
1342    /// # let mut config = Config::new();
1343    /// # let engine = Engine::new(&config)?;
1344    /// # let mut store = Store::new(&engine, Ctx { table: ResourceTable::new() });
1345    /// # let mut linker = Linker::new(&engine);
1346    /// # let component = Component::new(&engine, "")?;
1347    /// # let instance = linker.instantiate_async(&mut store, &component).await?;
1348    /// # let foo = instance.get_typed_func::<(Resource<MyResource>,), (Resource<MyResource>,)>(&mut store, "foo")?;
1349    /// # let bar = instance.get_typed_func::<(u32,), ()>(&mut store, "bar")?;
1350    /// store.run_concurrent(async |accessor| -> wasmtime::Result<_> {
1351    ///    let resource = accessor.with(|mut access| access.get().table.push(MyResource(42)))?;
1352    ///    let (another_resource,) = foo.call_concurrent(accessor, (resource,)).await?;
1353    ///    let value = accessor.with(|mut access| access.get().table.delete(another_resource))?;
1354    ///    bar.call_concurrent(accessor, (value.0,)).await?;
1355    ///    Ok(())
1356    /// }).await??;
1357    /// # Ok(())
1358    /// # }
1359    /// ```
1360    pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1361    where
1362        T: Send + 'static,
1363    {
1364        ensure!(
1365            self.0.concurrency_support(),
1366            "cannot use `run_concurrent` when Config::concurrency_support disabled",
1367        );
1368        self.do_run_concurrent(fun, false).await
1369    }
1370
1371    pub(super) async fn run_concurrent_trap_on_idle<R>(
1372        self,
1373        fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1374    ) -> Result<R> {
1375        self.do_run_concurrent(fun, true).await
1376    }
1377
1378    async fn do_run_concurrent<R>(
1379        mut self,
1380        fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1381        trap_on_idle: bool,
1382    ) -> Result<R> {
1383        debug_assert!(self.0.concurrency_support());
1384        self.0.check_event_loop_not_running()?;
1385
1386        // Deferred host tasks are generally not expected when running an event
1387        // loop itself. This only happens in odd situations like manually
1388        // calling `poll` in a sync host function, but nevertheless just put
1389        // things into a defined state for the event loop itself.
1390        self.0.concurrent_state_mut()?.materialize_host_task()?;
1391
1392        let token = StoreToken::new(self.as_context_mut());
1393
1394        struct Dropper<'a, T: 'static, V> {
1395            store: StoreContextMut<'a, T>,
1396            value: ManuallyDrop<V>,
1397        }
1398
1399        impl<'a, T, V> Drop for Dropper<'a, T, V> {
1400            fn drop(&mut self) {
1401                self.store
1402                    .0
1403                    .concurrent_state_mut_already_forced_current_thread()
1404                    .event_loop_running = false;
1405
1406                tls::set(self.store.0, || {
1407                    // SAFETY: Here we drop the value without moving it for the
1408                    // first and only time -- per the contract for `Drop::drop`,
1409                    // this code won't run again, and the `value` field will no
1410                    // longer be accessible.
1411                    unsafe { ManuallyDrop::drop(&mut self.value) }
1412                });
1413            }
1414        }
1415
1416        let accessor = &Accessor::new(token);
1417        self.0
1418            .concurrent_state_mut_already_forced_current_thread()
1419            .event_loop_running = true;
1420        let dropper = &mut Dropper {
1421            store: self,
1422            value: ManuallyDrop::new(fun(accessor)),
1423        };
1424        // SAFETY: We never move `dropper` nor its `value` field.
1425        let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1426
1427        let result = dropper
1428            .store
1429            .as_context_mut()
1430            .poll_until(future, trap_on_idle)
1431            .await;
1432
1433        if result.is_err() {
1434            dropper.store.0.set_trapped();
1435        }
1436
1437        result
1438    }
1439
1440    /// Run this store's event loop.
1441    ///
1442    /// The returned future will resolve when the specified future completes or,
1443    /// if `trap_on_idle` is true, when the event loop can't make further
1444    /// progress.
1445    async fn poll_until<R>(
1446        mut self,
1447        mut future: Pin<&mut impl Future<Output = R>>,
1448        trap_on_idle: bool,
1449    ) -> Result<R> {
1450        struct Reset<'a, T: 'static> {
1451            store: StoreContextMut<'a, T>,
1452            futures: Option<FuturesUnordered<HostTaskFuture>>,
1453        }
1454
1455        impl<'a, T> Drop for Reset<'a, T> {
1456            fn drop(&mut self) {
1457                if let Some(futures) = self.futures.take() {
1458                    *self
1459                        .store
1460                        .0
1461                        .concurrent_state_mut_already_forced_current_thread()
1462                        .futures
1463                        .get_mut() = Some(futures);
1464                }
1465            }
1466        }
1467
1468        // Keep a continuously busy event loop from holding one executor poll
1469        // indefinitely.  In particular, Tokio's cooperative budget is only
1470        // replenished when the future returns `Pending` to the executor.
1471        const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1472        let mut turns_without_yield = 0;
1473
1474        loop {
1475            // Take `ConcurrentState::futures` out of the store so we can poll
1476            // it while also safely giving any of the futures inside access to
1477            // `self`.
1478            let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1479            let mut reset = Reset {
1480                store: self.as_context_mut(),
1481                futures,
1482            };
1483            let mut next = match reset.futures.as_mut() {
1484                Some(f) => pin!(f.next()),
1485                None => bail_bug!("concurrent state missing futures field"),
1486            };
1487
1488            enum PollResult<R> {
1489                Complete(R),
1490                ProcessWork {
1491                    ready: Option<WorkItem>,
1492                    low_priority: bool,
1493                },
1494            }
1495
1496            let result = future::poll_fn(|cx| {
1497                // First, poll the future we were passed as an argument and
1498                // return immediately if it's ready.
1499                if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1500                    return Poll::Ready(Ok(PollResult::Complete(value)));
1501                }
1502
1503                // If the store has been poisoned by a trap, either before this
1504                // event loop started or while it was running, then refuse to
1505                // make any further progress. Any work items, suspended fibers,
1506                // or pending futures left over from the trap might otherwise
1507                // resume guest code or observe scheduler state which was never
1508                // unwound. Note that this is checked after polling `future`
1509                // above so that it can observe the original error, e.g. a
1510                // trap delivered over a oneshot channel by a work item.
1511                if reset.store.0.component_trapped() {
1512                    return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1513                }
1514
1515                // Next, poll `ConcurrentState::futures` (which includes any
1516                // pending host tasks and/or background tasks), returning
1517                // immediately if one of them fails.
1518                let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1519                    Poll::Ready(Some(output)) => {
1520                        match output {
1521                            Err(e) => return Poll::Ready(Err(e)),
1522                            Ok(()) => {}
1523                        }
1524                        Poll::Ready(true)
1525                    }
1526                    Poll::Ready(None) => Poll::Ready(false),
1527                    Poll::Pending => Poll::Pending,
1528                };
1529
1530                // Next, identify the next work item to process, if any, using
1531                // the following priority order:
1532                //
1533                // - `switch_item`: Represents the guest thread we _must_ switch
1534                // to before any other thread runs per the determinism
1535                // requirements in the Component Model spec.
1536                //
1537                // - `high_priority`: "Urgent" work items, e.g. async calls have
1538                // become freshly unblocked due to backpressure clearing or
1539                // similar, stream or future state updates, etc.
1540                //
1541                // - `low_priority`: Work items such as resuming a fiber after
1542                // it yields, in which case the point is to let other items run
1543                // first.
1544                let state = reset.store.0.concurrent_state_mut()?;
1545                let mut ready = state.switch_item.take();
1546                let mut low_priority = false;
1547                if ready.is_none() {
1548                    ready = state.high_priority.pop_back();
1549                    if ready.is_none() {
1550                        ready = state.low_priority.pop_back();
1551                        low_priority = true;
1552                    }
1553                }
1554                if ready.is_some() {
1555                    return Poll::Ready(Ok(PollResult::ProcessWork {
1556                        ready,
1557                        low_priority,
1558                    }));
1559                }
1560
1561                if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1562                    waker.wake();
1563                }
1564
1565                // Finally, if we have nothing else to do right now, determine what to do
1566                // based on whether there are any pending futures in
1567                // `ConcurrentState::futures`.
1568                return match next {
1569                    Poll::Ready(true) => {
1570                        // In this case, one of the futures in
1571                        // `ConcurrentState::futures` completed
1572                        // successfully, so we return now and continue
1573                        // the outer loop in case there is another one
1574                        // ready to complete.
1575                        Poll::Ready(Ok(PollResult::ProcessWork {
1576                            ready: None,
1577                            low_priority: false,
1578                        }))
1579                    }
1580                    Poll::Ready(false) => {
1581                        // Poll the future we were passed one last time
1582                        // in case one of `ConcurrentState::futures` had
1583                        // the side effect of unblocking it.
1584                        if let Poll::Ready(value) =
1585                            tls::set(reset.store.0, || future.as_mut().poll(cx))
1586                        {
1587                            Poll::Ready(Ok(PollResult::Complete(value)))
1588                        } else {
1589                            // In this case, there are no more pending
1590                            // futures in `ConcurrentState::futures`,
1591                            // there are no remaining work items, _and_
1592                            // the future we were passed as an argument
1593                            // still hasn't completed.
1594                            if trap_on_idle {
1595                                // `trap_on_idle` is true, so we exit
1596                                // immediately.
1597
1598                                // If there are any tasks belonging to
1599                                // an instance which may not suspend, trap with
1600                                // `CannotBlockSyncTask`:
1601                                Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1602                                    Trap::CannotBlockSyncTask.into()
1603                                } else {
1604                                    // Otherwise, trap with `AsyncDeadlock`:
1605                                    Trap::AsyncDeadlock.into()
1606                                }))
1607                            } else {
1608                                // `trap_on_idle` is false, so we assume
1609                                // that future will wake up and give us
1610                                // more work to do when it's ready to.
1611                                Poll::Pending
1612                            }
1613                        }
1614                    }
1615                    // There is at least one pending future in
1616                    // `ConcurrentState::futures` and we have nothing
1617                    // else to do but wait for now, so we return
1618                    // `Pending`.
1619                    Poll::Pending => Poll::Pending,
1620                };
1621            })
1622            .await;
1623
1624            // Put the `ConcurrentState::futures` back into the store before we
1625            // return or handle any work items since one or more of those items
1626            // might append more futures.
1627            drop(reset);
1628
1629            match result? {
1630                // The future we were passed as an argument completed, so we
1631                // return the result.
1632                PollResult::Complete(value) => break Ok(value),
1633                // The future we were passed has not yet completed, so handle
1634                // any work items and then loop again.
1635                PollResult::ProcessWork {
1636                    mut ready,
1637                    low_priority,
1638                } => {
1639                    // If we're about to run a low-priority task, first yield to
1640                    // the executor.  This ensures that it won't be starved of
1641                    // the ability to e.g. update the readiness of sockets,
1642                    // etc. which the guest may be using `thread.yield` along
1643                    // with `waitable-set.poll` to monitor in a CPU-heavy loop.
1644                    //
1645                    // This works for e.g. `thread.yield` and callbacks which
1646                    // return `CALLBACK_CODE_YIELD` because we queue a low
1647                    // priority item to resume the task (i.e. resume the thread
1648                    // or call the callback, respectively) just prior to
1649                    // suspending it.  Indeed, as of this writing those are the
1650                    // _only_ situations we queue low-priority tasks.
1651                    // Therefore, we interpret the guest's request to yield as
1652                    // meaning "yield to other guest tasks _and_/_or_ host
1653                    // operations such as updating socket readiness", the latter
1654                    // being the async runtime's responsibility.
1655                    //
1656                    // In the future, if this ends up causing measurable
1657                    // performance issues, this could be optimized such that we
1658                    // only yield periodically (e.g. for batches of low priority
1659                    // items) and not for each and every individual item.
1660                    //
1661                    // While we're yielded put the item back on the queue so it
1662                    // stays live within the store, and then if/when we come
1663                    // back take it back out. Note that the `await` point here
1664                    // owns the entire `StoreContextMut` so what we push should
1665                    // be what we pop.
1666                    if low_priority {
1667                        if let Some(item) = ready.take() {
1668                            self.0.concurrent_state_mut()?.low_priority.push_back(item);
1669                        }
1670                        self.0.yield_now().await;
1671                        turns_without_yield = 0;
1672                        ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1673                    }
1674
1675                    if let Some(item) = ready {
1676                        self.as_context_mut().handle_work_item(item).await?;
1677                    }
1678
1679                    turns_without_yield += 1;
1680                    if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1681                        turns_without_yield = 0;
1682                        self.0.yield_now().await;
1683                    }
1684                }
1685            }
1686        }
1687    }
1688
1689    /// Handle the specified work item, possibly resuming a fiber if applicable.
1690    fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1691        // Construct the guarded future before returning, even when the caller
1692        // never polls this work item. Other variants keep their fibers in the
1693        // store until their first poll.
1694        let item = match item {
1695            WorkItem::ResumeFiber {
1696                instance,
1697                thread,
1698                fiber,
1699            } => {
1700                let future = self.0.resume_fiber(fiber);
1701                log::trace!(
1702                    "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1703                );
1704                return futures::future::Either::Left(future);
1705            }
1706            item => item,
1707        };
1708        futures::future::Either::Right(async move {
1709            log::trace!("handle work item {item:?}");
1710            match item {
1711                WorkItem::PushFuture(future) => {
1712                    self.0
1713                        .concurrent_state_mut()?
1714                        .futures_mut()?
1715                        .push(future.into_inner());
1716                }
1717                WorkItem::ResumeFiber { .. } => unreachable!(),
1718                WorkItem::ResumeThread { thread, .. } => {
1719                    let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1720                    if !matches!(slot, GuestThreadState::Ready { .. }) {
1721                        bail_bug!("cannot resume non-pending thread {thread:?}");
1722                    }
1723                    let GuestThreadState::Ready { fiber } =
1724                        mem::replace(slot, GuestThreadState::Running)
1725                    else {
1726                        unreachable!()
1727                    };
1728                    self.0.resume_fiber(fiber).await?;
1729                }
1730                WorkItem::GuestCall { mut call, .. } => {
1731                    if call.is_ready(self.0)? {
1732                        if let GuestCallKind::StartImplicit {
1733                            was_pending: true, ..
1734                        } = &call.kind
1735                        {
1736                            let instance = self
1737                                .0
1738                                .concurrent_state_mut()?
1739                                .get_mut(call.thread.task)?
1740                                .instance;
1741
1742                            self.0.decrement_pending_start_count(instance, 1)?;
1743                        }
1744
1745                        self.0
1746                            .concurrent_state_mut()?
1747                            .get_mut(call.thread.thread)?
1748                            .wake_on_cancel = WakeOnCancel::None;
1749
1750                        self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1751                    } else {
1752                        let state = self.0.concurrent_state_mut()?;
1753                        let task = state.get_mut(call.thread.task)?;
1754                        if !task.starting_sent {
1755                            task.starting_sent = true;
1756                            if let GuestCallKind::StartImplicit { .. } = &call.kind {
1757                                Waitable::Guest(call.thread.task).set_event(
1758                                    state,
1759                                    Some(Event::Subtask {
1760                                        status: Status::Starting,
1761                                    }),
1762                                )?;
1763                            }
1764                        }
1765
1766                        let increment = if let GuestCallKind::StartImplicit {
1767                            was_pending, ..
1768                        } = &mut call.kind
1769                        {
1770                            let increment = !*was_pending;
1771                            *was_pending = true;
1772                            increment
1773                        } else {
1774                            false
1775                        };
1776
1777                        let instance = state.get_mut(call.thread.task)?.instance;
1778                        // If this call is for a callback-lifted task waiting on a
1779                        // waitable set, it can't run until `instance` is enterable
1780                        // again. Meanwhile, the set may have an event which another
1781                        // thread waiting on it could take, and that thread may
1782                        // even be the reason `instance` isn't enterable, so wake
1783                        // the next waiter up as well if there's still an event
1784                        // for it. Whichever thread runs first takes the event, and
1785                        // the other goes back to waiting if none is left.
1786                        if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1787                            if !state.get_mut(*set)?.ready.is_empty() {
1788                                state.wake_waiter(*set)?;
1789                            }
1790                        }
1791
1792                        // Switch back to the caller (or canceller) immediately if
1793                        // applicable since we aren't yet able to run the subtask it
1794                        // yielded to.
1795                        state.take_next_switch_item()?;
1796
1797                        let state = self.0.instance_state(instance).concurrent_state();
1798
1799                        state.pending.insert(call.thread, call.kind);
1800
1801                        if increment {
1802                            state.pending_start_count += 1;
1803                        }
1804                    }
1805                }
1806                WorkItem::WorkerFunction(fun) => {
1807                    self.run_on_worker(WorkerItem::Function(fun)).await?;
1808                }
1809            }
1810
1811            Ok(())
1812        })
1813    }
1814
1815    /// Execute the specified guest call on a worker fiber.
1816    async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1817        let state = self.0.concurrent_state_mut()?;
1818        if state.worker_item.is_some() {
1819            bail_bug!("worker item already set when taking worker fiber");
1820        }
1821        let worker = if let Some(fiber) = state.worker.take() {
1822            fiber
1823        } else {
1824            // SAFETY: the `make_fiber_unchecked` function is unsafe because the
1825            // returned fiber is unconditionally `Send` as opposed to being
1826            // conditionally send depending on the argument (in this case
1827            // `self.0`). This `async` function, however, is conditionally
1828            // `Send` depending on `self`, in this case `StoreContextMut<T>`,
1829            // which is already going to be conditionally `Send` depending on
1830            // `T`.
1831            //
1832            // The returned fiber is possibly stored within the `Store<T>` as
1833            // well. If `T: Send` then that's fine and everything's dandy. If
1834            // `T: !Send`, however, then the store is already not-`Send` meaning
1835            // that putting more actually-not-`Send` things inside of it isn't
1836            // an issue.
1837            //
1838            // The main issue here is that the returned fiber effectively can't
1839            // get transferred outside the context of the store. That's an
1840            // implementation detail we'll have to rely on, but is currently
1841            // true.
1842            unsafe {
1843                fiber::make_fiber_unchecked(self.0, move |store| {
1844                    loop {
1845                        let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1846                            bail_bug!("worker_item not present when resuming fiber")
1847                        };
1848                        match item {
1849                            WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1850                            WorkerItem::Function(fun) => fun.into_inner()(store)?,
1851                        }
1852
1853                        store.suspend(SuspendReason::NeedWork)?;
1854                    }
1855                })?
1856            }
1857        };
1858
1859        // No fallible operation between taking the fiber and guarding it in
1860        // resume_fiber; the current thread was forced before the take.
1861        self.0
1862            .concurrent_state_mut_already_forced_current_thread()
1863            .worker_item = Some(item);
1864
1865        self.0.resume_fiber(worker).await
1866    }
1867
1868    /// Wrap the specified host function in a future which will call it, passing
1869    /// it an `&Accessor<T>`.
1870    ///
1871    /// See the `Accessor` documentation for details.
1872    pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1873    where
1874        T: 'static,
1875        F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1876            + Send
1877            + Sync
1878            + 'static,
1879        R: Send + Sync + 'static,
1880    {
1881        let token = StoreToken::new(self);
1882        async move {
1883            let mut accessor = Accessor::new(token);
1884            closure(&mut accessor).await
1885        }
1886    }
1887
1888    pub(crate) async fn start_instance(
1889        &mut self,
1890        instance: ModuleInstance,
1891        callee: Option<RuntimeInstance>,
1892    ) -> Result<ModuleInstance> {
1893        // See `Func::call_async` for why this is checked up front.
1894        self.0.check_event_loop_not_running()?;
1895        let (tx, rx) = oneshot::channel();
1896        let token = StoreToken::new(self.as_context_mut());
1897        self.0.queue_task(move |store| {
1898            _ = tx.send(
1899                super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1900                    .map(|()| instance),
1901            );
1902            Ok(())
1903        })?;
1904        self.as_context_mut()
1905            .run_concurrent_trap_on_idle(async |_| {
1906                rx.await
1907                    .map_err(|_| format_err!("oneshot channel canceled"))
1908            })
1909            .await??
1910    }
1911}
1912
1913/// Return value of [`StoreOpaque::host_task_create`].
1914///
1915/// This is an `Option` to handle the dynamic `store.concurrency_support()`
1916/// property. When present this records the guest thread to restore when the
1917/// host call exits. The corresponding [`HostTask`] will need to be lazily
1918/// created if needed via [`StoreOpaque::materialize_host_task_id`].
1919pub type EnteredHostTask = Option<QualifiedThreadId>;
1920
1921impl StoreOpaque {
1922    pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1923        if self
1924            .concurrent_state_mut_already_forced_current_thread()
1925            .event_loop_running
1926        {
1927            bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1928        }
1929        Ok(())
1930    }
1931
1932    /// Returns the currently-running thread, promoting any deferred lazy guest
1933    /// thread into a fully-materialized `CurrentThread`. Deferred [`HostTask`]s
1934    /// are not materialized.
1935    #[inline]
1936    pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1937        // Without concurrency support there is nothing to force.
1938        if !self.concurrency_support() {
1939            return Ok(CurrentThread::None);
1940        }
1941
1942        // If the JIT-visible current thread isn't a deferred thread then
1943        // `ConcurrentState` is already up to date.
1944        if !self
1945            .vm_store_context_mut()
1946            .current_thread_mut()
1947            .is_deferred()
1948        {
1949            return Ok(self
1950                .concurrent_state_mut_already_forced_current_thread()
1951                .unforced_current_thread);
1952        }
1953
1954        self.force_deferred_current_thread()
1955    }
1956
1957    /// Slow path of [`Self::current_thread`]: promote the deferred lazy
1958    /// thread into a fully-materialized `CurrentThread`.
1959    #[cold]
1960    fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1961        // The component instance whose adapters pushed the deferred frames; all
1962        // frames in a guest-to-guest, sync-to-sync call chain of fused adapters
1963        // live within a single `wasmtime::component::Instance` (because
1964        // cross-`wasmtime::component::Instance` calls don't go through fused
1965        // adapters), and guest code only ever runs as a guest thread, so the
1966        // chain's base thread is already materialized in `ConcurrentState` and
1967        // we can get the `ComponentInstanceId` shared by the whole chain from
1968        // here.
1969        let state = self.concurrent_state_mut_without_forcing_current_thread();
1970        let id = match state.unforced_current_thread.guest_task() {
1971            Some(task) => state.get_mut(task)?.instance.instance,
1972            None => bail_bug!("deferred component-model thread with non-guest base"),
1973        };
1974
1975        // Collect the deferred frames pushed inline by fused adapters, walking
1976        // the `parent` chain from innermost to the base.
1977        let mut frames = SmallVec::<[_; 8]>::new();
1978        let mut cur = *self.vm_store_context_mut().current_thread_mut();
1979        while let Some(ptr) = cur.as_deferred() {
1980            // SAFETY: `ptr` points at a `VMDeferredThread` living in a fused
1981            // adapter's stack frame that is suspended below us on the stack
1982            // (mid-call, waiting for this nested call to return), so the
1983            // referent is still valid and exclusively ours to read.
1984            let deferred = unsafe { ptr.as_non_null().as_ref() };
1985            frames.push((
1986                deferred.callee_async != 0,
1987                deferred.callee_instance,
1988                deferred.saved_context,
1989                deferred.lazy_lenders_start,
1990            ));
1991            cur = deferred.parent;
1992        }
1993
1994        // Mark the current thread forced *before* replaying so that any
1995        // reentrant `force_current_thread` call short-circuits via the
1996        // non-deferred path above.
1997        *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1998
1999        // Save the current context, as we need to overwrite it while replaying
2000        // below.
2001        let current_context = *self.vm_store_context_mut().component_context_mut();
2002
2003        // Replay the deferred `enter_guest_sync_call`s outermost-first so that
2004        // the resulting `ConcurrentState` matches what the non-deferred path
2005        // would have otherwise produced.
2006        //
2007        // A frame's own lazy lends run from its start to that of the frame
2008        // pushed after it, or to the end of the buffer for the innermost.
2009        let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
2010        frames.reverse();
2011        let lazy_lenders_base = match frames.first() {
2012            Some((_, _, _, start)) => *start,
2013            None => bail_bug!("forcing a deferred thread with no deferred frames"),
2014        };
2015        let lazy_lender_ends = frames
2016            .iter()
2017            .skip(1)
2018            .map(|(_, _, _, start)| *start)
2019            .chain([lazy_lenders_end])
2020            .collect::<SmallVec<[_; 8]>>();
2021        for ((callee_async, callee_instance, saved_context, start), lenders_end) in
2022            frames.into_iter().zip(lazy_lender_ends)
2023        {
2024            // Restore the caller's context slots so that we save the correct
2025            // values into the caller's thread, exactly as the non-deferred path
2026            // would have on entry.
2027            *self.vm_store_context_mut().component_context_mut() = saved_context;
2028            let callee = RuntimeInstance {
2029                instance: id,
2030                index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
2031            };
2032            self.enter_guest_sync_call(callee_async, callee)?;
2033
2034            // Now that this frame has a real `GuestTask`, promote any lazy
2035            // lends it recorded while deferred.
2036            self.promote_current_lazy_lends(id, start, lenders_end)?;
2037        }
2038
2039        // Every lazy-lend record was promoted, so the length resets back to
2040        // where the outermost frame found it.
2041        debug_assert_eq!(
2042            *self.vm_store_context_mut().lazy_lenders_len_mut(),
2043            lazy_lenders_end
2044        );
2045        *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
2046
2047        // Replaying done; restore the current context.
2048        *self.vm_store_context_mut().component_context_mut() = current_context;
2049
2050        Ok(self
2051            .concurrent_state_mut_without_forcing_current_thread()
2052            .unforced_current_thread)
2053    }
2054
2055    /// Promotes the `lazy_lenders` entries in `start..end` into real lends
2056    /// owned by the current thread's `CallContext`.
2057    ///
2058    /// The task's `validate_scope_exit` then undoes them as if they had been
2059    /// lent there all along.
2060    fn promote_current_lazy_lends(
2061        &mut self,
2062        id: ComponentInstanceId,
2063        start: usize,
2064        end: usize,
2065    ) -> Result<()> {
2066        if start == end {
2067            return Ok(());
2068        }
2069
2070        let scope = match self
2071            .concurrent_state_mut_without_forcing_current_thread()
2072            .unforced_current_thread
2073            .guest_task()
2074        {
2075            Some(task) => VMScope::id(task.rep()),
2076            None => bail_bug!("replayed deferred frame is not a guest thread"),
2077        };
2078        let instance = Instance::from_wasmtime(self, id);
2079        self.promote_lazy_lends(instance, scope, start, end)
2080    }
2081
2082    fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2083        match self.current_thread()?.guest() {
2084            Some(id) => Ok(*id),
2085            None => bail_bug!("current thread is not a guest thread"),
2086        }
2087    }
2088
2089    // A result of `None` may indicate that this is either the top-level event
2090    // loop, a deferred host task, or concurrency support is disabled. In all
2091    // cases we don't have an ID for the task.
2092    pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2093        match self.current_thread()? {
2094            CurrentThread::Host(id) => Ok(Some(id)),
2095            CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2096            _ => bail_bug!("current thread is not a host thread"),
2097        }
2098    }
2099
2100    /// Returns the current host task ID, materializing a deferred host task if
2101    /// one is active. `None` represents a call from the top-level host.
2102    fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2103        Ok(self
2104            .concurrent_state_mut()?
2105            .materialize_current_host_task_id()?)
2106    }
2107
2108    fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2109        log::trace!("enter sync-typed call {callee:?}");
2110        let state = self.instance_state(callee).concurrent_state();
2111        let old_do_not_suspend = state.do_not_suspend;
2112        state.do_not_suspend = true;
2113
2114        let thread = self.current_guest_thread()?;
2115        let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2116        if thread.old_do_not_suspend.is_some() {
2117            bail_bug!("current thread already has `old_do_not_suspend` value");
2118        }
2119
2120        thread.old_do_not_suspend = Some(old_do_not_suspend);
2121
2122        Ok(())
2123    }
2124
2125    fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2126        log::trace!("exit sync-typed call {callee:?}");
2127        let thread = self.current_guest_thread()?;
2128        let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2129        let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2130            bail_bug!("current thread missing `old_do_not_suspend` value");
2131        };
2132        let state = self.instance_state(callee).concurrent_state();
2133        state.do_not_suspend = old_do_not_suspend;
2134        Ok(())
2135    }
2136
2137    /// Push a `GuestTask` onto the task stack for either a sync-to-sync,
2138    /// guest-to-guest call or a sync host-to-guest call.
2139    ///
2140    /// This task will only be used for the purpose of handling calls to
2141    /// intrinsic functions; both parameter lowering and result lifting are
2142    /// assumed to be taken care of elsewhere.
2143    ///
2144    /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
2145    /// fused adapters, see `StoreOpaque::current_thread`, `VMDeferredThread`,
2146    /// and `lower_fact_enter_sync_call`. Make sure all this stuff stays in
2147    /// sync!
2148    pub(crate) fn enter_guest_sync_call(
2149        &mut self,
2150        callee_async_typed: bool,
2151        callee: RuntimeInstance,
2152    ) -> Result<()> {
2153        log::trace!("enter sync-lifted call {callee:?}");
2154        if !self.concurrency_support() {
2155            return self.enter_call_not_concurrent();
2156        }
2157
2158        // Stash the next switch item (if any) for the duration of this call
2159        // since it's for the caller's caller, not for us.  We'll restore this
2160        // in `exit_guest_sync_call`.
2161        let state = self.concurrent_state_mut()?;
2162        let item = state.next_switch_item.take();
2163        state.saved_next_switch_items.push(item);
2164
2165        let thread = self.current_thread()?;
2166        let caller = if let Some(thread) = thread.guest() {
2167            Caller::Guest { thread: *thread }
2168        } else {
2169            Caller::Host {
2170                tx: None,
2171                caller: self.materialize_host_task_id()?,
2172            }
2173        };
2174
2175        let state = self.concurrent_state_mut()?;
2176        let guest_thread = GuestTask::new(
2177            state,
2178            Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2179            LiftResult {
2180                lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2181                ty: TypeTupleIndex::reserved_value(),
2182                memory: None,
2183                string_encoding: StringEncoding::Utf8,
2184            },
2185            caller,
2186            None,
2187            callee,
2188            callee_async_typed,
2189            false,
2190        )?;
2191
2192        Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2193            guest_thread.thread,
2194            self,
2195            callee.index,
2196        )?;
2197        self.set_thread(guest_thread)?;
2198
2199        if !callee_async_typed {
2200            self.enter_sync_call(callee)?;
2201        }
2202
2203        Ok(())
2204    }
2205
2206    /// Pop a `GuestTask` previously pushed using `enter_guest_sync_call`.
2207    ///
2208    /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
2209    /// fused adapters and then when the call returns we check to see if the
2210    /// task's contruction was forced and if not avoid calling out of the JIT
2211    /// code to this function. See `lower_fact_exit_sync_call`. Make sure all
2212    /// this stuff stays in sync!
2213    pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2214        if !self.concurrency_support() {
2215            return Ok(self.exit_call_not_concurrent());
2216        }
2217
2218        let thread = match self.current_thread()?.guest() {
2219            Some(t) => *t,
2220            None => bail_bug!("expected task when exiting"),
2221        };
2222        let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2223        let instance = task.instance;
2224
2225        let caller = match &task.caller {
2226            &Caller::Guest { thread } => thread.into(),
2227            &Caller::Host { caller, .. } => caller
2228                .map(CurrentThread::Host)
2229                .unwrap_or(CurrentThread::None),
2230        };
2231        task.lift_result = None;
2232        task.exited = true;
2233        let async_typed = task.async_typed;
2234
2235        if !async_typed {
2236            self.exit_sync_call(instance)?;
2237        }
2238
2239        self.set_thread(caller)?;
2240
2241        log::trace!("exit sync-lifted call {instance:?}");
2242
2243        if async_typed {
2244            // If we're async-typed, returning control to our caller won't help
2245            // resolve any outstanding sync-typed call which might be in
2246            // progress, so we may need to switch or trap before exiting this
2247            // thread:
2248            self.switch_or_trap_if_may_not_suspend(instance)?;
2249        }
2250
2251        self.cleanup_thread(thread, instance)?;
2252
2253        self.concurrent_state_mut()?
2254            .decrement_task_ref_count(thread.task)?;
2255
2256        let state = self.concurrent_state_mut()?;
2257        let Some(item) = state.saved_next_switch_items.pop() else {
2258            bail_bug!("unable to pop from `saved_next_switch_items`");
2259        };
2260        if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2261            // Stash it back in the store to ensure it's cleaned up on store
2262            // drop:
2263            state.push_high_priority(item);
2264            bail_bug!("`next_switch_item` unexpectedly already set");
2265        }
2266
2267        Ok(())
2268    }
2269
2270    /// Similar to `enter_guest_sync_call` except for when the guest makes a
2271    /// transition to the host.
2272    ///
2273    /// This initially records a deferred host call. A full [`HostTask`] should
2274    /// be allocated later if needed via
2275    /// [`StoreOpaque::materialize_host_task_id`].
2276    pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2277        if !self.concurrency_support() {
2278            self.enter_call_not_concurrent()?;
2279            return Ok(None);
2280        }
2281        let caller = self.current_guest_thread()?;
2282        log::trace!("new deferred host task with caller {caller:?}");
2283
2284        self.set_thread(CurrentThread::DeferredHost(caller))?;
2285        let state = self.concurrent_state_mut()?;
2286        debug_assert!(state.deferred_host_call_context.is_none());
2287        state.deferred_host_call_context = Some(CallContext::default());
2288        state.debug_assert_deferred_host_invariant();
2289        Ok(Some(caller))
2290    }
2291
2292    /// Dual of `host_task_create` and signifies that the host has finished and
2293    /// will be cleaned up.
2294    ///
2295    /// Note that this isn't invoked when the host is invoked asynchronously and
2296    /// the host isn't complete yet. In that situation the host task persists
2297    /// and will be cleaned up separately in `subtask_drop`
2298    pub(crate) fn host_task_delete(
2299        &mut self,
2300        original_task: EnteredHostTask,
2301        materialized_task: Option<TableId<HostTask>>,
2302    ) -> Result<()> {
2303        match original_task {
2304            Some(caller) => {
2305                self.set_thread(caller)?;
2306                if materialized_task.is_none() {
2307                    let state = self.concurrent_state_mut()?;
2308                    let context = state
2309                        .deferred_host_call_context
2310                        .take()
2311                        .expect("deferred host call context should be present");
2312                    debug_assert!(context.is_empty());
2313                    state.debug_assert_deferred_host_invariant();
2314                }
2315                log::trace!(
2316                    "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2317                );
2318                if let Some(task) = materialized_task {
2319                    self.concurrent_state_mut()?.delete_host_task(task)?;
2320                }
2321            }
2322            None => {
2323                debug_assert!(materialized_task.is_none());
2324                self.exit_call_not_concurrent();
2325            }
2326        }
2327        Ok(())
2328    }
2329
2330    /// Helper function to retrieve the `InstanceState` for the
2331    /// specified instance.
2332    fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2333        self.component_instance_mut(instance.instance)
2334            .instance_state(instance.index)
2335    }
2336
2337    /// Same as `instance_state`, but also returns the store's `HandleCounter`
2338    /// for inserting into the instance's handle tables.
2339    fn instance_state_and_handle_counter(
2340        &mut self,
2341        instance: RuntimeInstance,
2342    ) -> (&mut InstanceState, &mut HandleCounter) {
2343        let (component, counter) = self
2344            .store_data_mut()
2345            .component_instance_and_handle_counter_mut(instance.instance);
2346        (component.instance_state(instance.index), counter)
2347    }
2348
2349    /// Begins inserting into the handle table of the specified instance.
2350    fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2351        let (component, counter) = self
2352            .store_data_mut()
2353            .component_instance_and_handle_counter_mut(instance.instance);
2354        component.handle_table_insertion(instance.index, counter)
2355    }
2356
2357    /// Configure the currently running `thread`.
2358    ///
2359    /// This will save off any state necessary for the previous thread, if
2360    /// applicable, and then it'll additionally update state for `thread` if
2361    /// needed too.
2362    pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2363        let thread = thread.into();
2364        let state = self.concurrent_state_mut()?;
2365        state.debug_assert_deferred_host_invariant();
2366        let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2367
2368        state.handle_thread_switch(old_thread, thread)?;
2369
2370        // First thing to do after swapping threads is updating the context
2371        // slots for this thread within the store. This restores the behavior of
2372        // `context.{get,set}`. This involves taking the old state out of the
2373        // store, saving it in the thread that's being swapped from, and doing
2374        // the inverse for the new thread. When debug assertions are enabled
2375        // this also leaves behind sentinel values to try to uncover bugs where
2376        // this may be forgotten.
2377        if let Some(old_thread) = old_thread.guest() {
2378            let old_context = *self.vm_store_context_mut().component_context_mut();
2379            self.concurrent_state_mut()?
2380                .get_mut(old_thread.thread)?
2381                .context = old_context;
2382        }
2383        if cfg!(debug_assertions) {
2384            *self.vm_store_context_mut().component_context_mut() =
2385                [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2386        }
2387        if let Some(thread) = thread.guest() {
2388            let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2389            let context = thread.context;
2390            if cfg!(debug_assertions) {
2391                thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2392            }
2393            *self.vm_store_context_mut().component_context_mut() = context;
2394        }
2395
2396        // Keep the JIT-visible current-thread pointer in sync.
2397        *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2398            VMLazyThread::none()
2399        } else {
2400            VMLazyThread::forced()
2401        };
2402
2403        Ok(old_thread)
2404    }
2405
2406    /// Call `switch_if_may_not_suspend` and trap if it returns `false`.
2407    fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2408        if self.switch_if_may_not_suspend(instance)? {
2409            Ok(())
2410        } else {
2411            Err(Trap::CannotBlockSyncTask.into())
2412        }
2413    }
2414
2415    /// Check if the specified instance has a sync-typed call in progress; if so
2416    /// attempt to switch to another ready thread for that instance, and if no
2417    /// such thread exists, return false.
2418    fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2419        // Call this for the side effect of forcing any deferred task creation,
2420        // which may influence the value of `ConcurrentState::do_not_suspend`
2421        // below:
2422        self.concurrent_state_mut()?;
2423
2424        Ok(!self.concurrency_support()
2425            || !self
2426                .instance_state(instance)
2427                .concurrent_state()
2428                .do_not_suspend
2429            || self
2430                .concurrent_state_mut()?
2431                .promote_instance_local_thread_work_item(instance)?)
2432    }
2433
2434    /// Record that we're about to enter a (sub-)component instance which does
2435    /// not support more than one concurrent, stackful activation, meaning it
2436    /// cannot be entered again until the next call returns.
2437    fn enter_instance(&mut self, instance: RuntimeInstance) {
2438        log::trace!("enter {instance:?}");
2439        self.instance_state(instance)
2440            .concurrent_state()
2441            .do_not_enter = true;
2442    }
2443
2444    /// Record that we've exited a (sub-)component instance previously entered
2445    /// with `Self::enter_instance` and then calls `Self::partition_pending`.
2446    /// See the documentation for the latter for details.
2447    fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2448        log::trace!("exit {instance:?}");
2449        self.instance_state(instance)
2450            .concurrent_state()
2451            .do_not_enter = false;
2452        self.partition_pending(instance)
2453    }
2454
2455    /// Iterate over `InstanceState::pending`, moving any ready items into the
2456    /// "high priority" work item queue.
2457    ///
2458    /// Also, notify `ConcurrentState::ready_for_concurrent_call_waker` if
2459    /// present.
2460    ///
2461    /// See `GuestCall::is_ready` for details.
2462    fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2463        let mut any_ready = false;
2464        for (thread, kind) in
2465            mem::take(&mut self.instance_state(instance).concurrent_state().pending)
2466        {
2467            let call = GuestCall { thread, kind };
2468            if call.is_ready(self)? {
2469                any_ready = true;
2470                self.concurrent_state_mut()?
2471                    .push_high_priority(WorkItem::GuestCall { instance, call });
2472            } else {
2473                self.instance_state(instance)
2474                    .concurrent_state()
2475                    .pending
2476                    .insert(call.thread, call.kind);
2477            }
2478        }
2479
2480        if any_ready {
2481            if let Some(waker) = self
2482                .concurrent_state_mut()?
2483                .ready_for_concurrent_call_waker
2484                .take()
2485            {
2486                waker.wake();
2487            }
2488        }
2489
2490        Ok(())
2491    }
2492
2493    /// Implements the `backpressure.{inc,dec}` intrinsics.
2494    pub(crate) fn backpressure_modify(
2495        &mut self,
2496        caller_instance: RuntimeInstance,
2497        modify: impl FnOnce(u16) -> Option<u16>,
2498    ) -> Result<()> {
2499        let state = self.instance_state(caller_instance).concurrent_state();
2500        let old = state.backpressure;
2501        let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2502        state.backpressure = new;
2503
2504        log::trace!("backpressure update for {caller_instance:?} from {old} to {new}");
2505
2506        if old > 0 && new == 0 {
2507            // Backpressure was previously enabled and is now disabled; move any
2508            // newly-eligible guest calls to the "high priority" queue.
2509            self.partition_pending(caller_instance)?;
2510        }
2511
2512        Ok(())
2513    }
2514
2515    /// Resume the specified fiber, giving it exclusive access to the specified
2516    /// store.
2517    fn resume_fiber(
2518        &mut self,
2519        fiber: StoreFiber<'static>,
2520    ) -> impl Future<Output = Result<()>> + '_ {
2521        let mut resuming = ResumingFiber {
2522            store: self,
2523            fiber: Some(fiber),
2524        };
2525        async move {
2526            let old_thread = resuming.store.current_thread()?;
2527            log::trace!("resume_fiber: save current thread {old_thread:?}");
2528
2529            let fiber = resuming.fiber.take().unwrap();
2530            resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2531
2532            resuming.store.set_thread(old_thread)?;
2533
2534            let state = resuming.store.concurrent_state_mut()?;
2535
2536            if let Some(ot) = old_thread.guest() {
2537                state.set_thread_running(ot.thread)?;
2538            }
2539            log::trace!("resume_fiber: restore current thread {old_thread:?}");
2540
2541            if resuming.fiber.is_some() {
2542                log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2543                // See the `SuspendReason` documentation for what each case means.
2544                let reason = match state.suspend_reason.take() {
2545                    Some(r) => r,
2546                    None => bail_bug!("suspend reason missing when resuming fiber"),
2547                };
2548                match reason {
2549                    SuspendReason::NeedWork => {
2550                        if state.worker.is_none() {
2551                            state.worker = resuming.fiber.take();
2552                        }
2553                    }
2554                    SuspendReason::Yielding { thread } => {
2555                        let slot = &mut state.get_mut(thread.thread)?.state;
2556                        slot.check_no_fiber()?;
2557                        *slot = GuestThreadState::Ready {
2558                            fiber: resuming.fiber.take().unwrap(),
2559                        };
2560                        let instance = state.get_mut(thread.task)?.instance;
2561                        state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2562                    }
2563                    SuspendReason::ExplicitlySuspending { thread } => {
2564                        let slot = &mut state.get_mut(thread.thread)?.state;
2565                        slot.check_no_fiber()?;
2566                        *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2567                    }
2568                    SuspendReason::Waiting { set, thread } => {
2569                        match state.get_mut(set)?.waiting.entry(thread) {
2570                            Entry::Vacant(entry) => {
2571                                entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2572                            }
2573                            Entry::Occupied(_) => {
2574                                bail_bug!("entry unexpectedly already exists for {thread:?}");
2575                            }
2576                        }
2577                    }
2578                    SuspendReason::YieldingToSubtask { thread } => {
2579                        // In this case, the thread has either invoked or sent a
2580                        // cancel request to a subtask, and is now yielding to that
2581                        // subtask.  According to the CM spec, that subtask may only
2582                        // yield back to the original thread the first time it
2583                        // suspends or exits (or a thread that it has resumed
2584                        // suspends or exits, etc.), which we ensure by setting
2585                        // `ConcurrentState::next_switch_item` here.
2586
2587                        let instance = state.get_mut(thread.task)?.instance;
2588                        if state.next_switch_item.is_some() {
2589                            // This should be unreachable per the save/restore code
2590                            // in `Self::suspend`.
2591                            bail_bug!(
2592                                "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2593                             a thread wanted to wait on a subtask"
2594                            );
2595                        }
2596                        let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2597                            instance,
2598                            thread,
2599                            fiber: resuming.fiber.take().unwrap(),
2600                        });
2601                        log::trace!("set next switch item to {item:?}");
2602                    }
2603                };
2604            } else {
2605                log::trace!("resume_fiber: fiber has exited");
2606            }
2607
2608            Ok(())
2609        }
2610    }
2611
2612    /// Suspend the current fiber, storing the reason in
2613    /// `ConcurrentState::suspend_reason` to indicate the conditions under which
2614    /// it should be resumed.
2615    ///
2616    /// See the `SuspendReason` documentation for details.
2617    fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2618        log::trace!("suspend fiber: {reason:?}");
2619
2620        // We'll panic if we call `Self::with_blocking` when the fiber is being
2621        // disposed, so check for that and exit ASAP if appropriate.
2622        if !self.fiber_async_state_mut().can_block() {
2623            return Err(format_err!("future dropped"));
2624        }
2625
2626        let state = self.concurrent_state_mut()?;
2627
2628        // If we're yielding or waiting on behalf of a guest thread, we'll be
2629        // overwriting the current thread, so save it now and restore it once
2630        // we've resumed.
2631        //
2632        // Also, if we're yielding to a subtask, we're about to overwrite
2633        // `ConcurrentState::next_switch_item`, so also save and restore that.
2634        let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2635            SuspendReason::Yielding { .. }
2636            | SuspendReason::Waiting { .. }
2637            | SuspendReason::ExplicitlySuspending { .. } => {
2638                // If there's a thread waiting for this subtask to suspend, this
2639                // is a good time to switch back to it.
2640                if state.switch_item.is_none() {
2641                    state.take_next_switch_item()?;
2642                }
2643
2644                (true, false)
2645            }
2646            SuspendReason::YieldingToSubtask { .. } => (true, true),
2647            SuspendReason::NeedWork => (false, false),
2648        };
2649
2650        let old_next_switch_item = if save_and_restore_next_switch_item {
2651            Some(state.save_next_switch_item()?)
2652        } else {
2653            None
2654        };
2655
2656        let old_guest_thread = if save_and_restore_thread {
2657            self.current_thread()?
2658        } else {
2659            CurrentThread::None
2660        };
2661
2662        let waiting_set = match &reason {
2663            SuspendReason::Waiting { set, .. } => Some(*set),
2664            _ => None,
2665        };
2666
2667        let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2668        if suspend_reason.is_some() {
2669            bail_bug!("suspend reason unexpectedly already set for store");
2670        }
2671        *suspend_reason = Some(reason);
2672
2673        // Count this thread as waiting on `waiting_set`, if any, for as long as
2674        // it's suspended.
2675        if let Some(set) = waiting_set {
2676            self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2677        }
2678
2679        self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2680
2681        if let Some(set) = waiting_set {
2682            self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2683        }
2684
2685        if save_and_restore_thread {
2686            self.set_thread(old_guest_thread)?;
2687        }
2688
2689        if let Some(item) = old_next_switch_item {
2690            let state = self.concurrent_state_mut()?;
2691            state.restore_next_switch_item(item)?;
2692        }
2693
2694        Ok(())
2695    }
2696
2697    fn wait_for_event(
2698        &mut self,
2699        caller_instance: RuntimeInstance,
2700        waitable: Waitable,
2701    ) -> Result<()> {
2702        let caller = self.current_guest_thread()?;
2703        let state = self.concurrent_state_mut()?;
2704
2705        waitable.trap_if_in_waitable_set(state)?;
2706
2707        let set = state.get_mut(caller.thread)?.sync_call_set;
2708        waitable.join(state, Some(set))?;
2709
2710        self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2711
2712        self.suspend(SuspendReason::Waiting {
2713            set,
2714            thread: caller,
2715        })?;
2716        let state = self.concurrent_state_mut()?;
2717
2718        waitable.join(state, None)
2719    }
2720
2721    /// Cleans up the data structures backing the `guest_thread` specified,
2722    /// removing it from the internal tables of `runtime_instance` as well.
2723    ///
2724    /// This function is used whenever a guest thread has fully exited and
2725    /// completed. This'll clean up the associated `GuestThread` structure and
2726    /// related resources it contains.
2727    ///
2728    /// Other functionality that this implements is:
2729    ///
2730    /// * This will perform conditional cleanup of the `GuestTask` that owns
2731    ///   this thread if `cleanup_task` is `CleanupTask::Yes`.
2732    /// * If there are no more threads in the `GuestTask` that this thread is
2733    ///   associated with, and if the task hasn't produced a result (e.g. it's not
2734    ///   returned or cancelled), then a trap will be raised that a result
2735    ///   wasn't ever produced.
2736    /// * If this task is finished, meaning the top-level thread exited and
2737    ///   additionally it's been returned or cancelled, then this will handle
2738    ///   management of the store's "active interesting tasks" counter.
2739    ///
2740    /// Effectively this is intended to be a "narrow waist" through which many
2741    /// destruction operations are funneled through.
2742    fn cleanup_thread(
2743        &mut self,
2744        guest_thread: QualifiedThreadId,
2745        runtime_instance: RuntimeInstance,
2746    ) -> Result<()> {
2747        let state = self.concurrent_state_mut()?;
2748        // If we never suspended, we never had a chance to deliver a subtask
2749        // status update, if any, to our caller, so we do that here:
2750        state.take_next_switch_item()?;
2751        let thread_data = state.get_mut(guest_thread.thread)?;
2752        let sync_call_set = thread_data.sync_call_set;
2753        if let Some(guest_id) = thread_data.instance_rep {
2754            self.instance_state(runtime_instance)
2755                .thread_handle_table()
2756                .guest_thread_remove(guest_id)?;
2757        }
2758        let state = self.concurrent_state_mut()?;
2759
2760        // Clean up any pending subtasks in the sync_call_set
2761        for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2762            if let Some(Event::Subtask {
2763                status: Status::Returned | Status::ReturnCancelled,
2764            }) = waitable.common(state)?.event
2765            {
2766                waitable.delete_from(state)?;
2767            }
2768        }
2769
2770        state.delete(guest_thread.thread)?;
2771        state.delete(sync_call_set)?;
2772        let task = state.get_mut(guest_thread.task)?;
2773        task.threads.remove(&guest_thread.thread);
2774
2775        if task.threads.is_empty() && !task.returned_or_cancelled() {
2776            bail!(Trap::NoAsyncResult);
2777        }
2778
2779        if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2780            task.decremented_interesting_task_count = true;
2781
2782            debug_assert!(state.interesting_tasks > 0);
2783            state.interesting_tasks -= 1;
2784            if state.interesting_tasks == 0
2785                && let Some(waker) = state.interesting_tasks_empty_waker.take()
2786            {
2787                waker.wake();
2788            }
2789        }
2790
2791        state.decrement_task_ref_count(guest_thread.task)?;
2792
2793        Ok(())
2794    }
2795
2796    /// Performs cancellation of the `guest_task` specified with the
2797    /// precondition that the task hasn't lowered its parameters.
2798    ///
2799    /// In this situation the task hasn't ever been started meaning it hasn't
2800    /// actually run any wasm code yet. This requires cleaning up metadata such
2801    /// as thread information attached to the task.
2802    ///
2803    /// This is used for task cancellation via `subtask.cancel`, the intrinsic.
2804    fn cancel_guest_subtask_without_lowered_parameters(
2805        &mut self,
2806        caller_instance: RuntimeInstance,
2807        guest_task: TableId<GuestTask>,
2808    ) -> Result<()> {
2809        let concurrent_state = self.concurrent_state_mut()?;
2810        let task = concurrent_state.get_mut(guest_task)?;
2811        assert!(!task.already_lowered_parameters());
2812        // The task is in a `starting` state, meaning it hasn't run at
2813        // all yet.  Here we update its fields to indicate that it is
2814        // ready to delete immediately once `subtask.drop` is called.
2815        task.lower_params = None;
2816        task.lift_result = None;
2817        task.exited = true;
2818        let instance = task.instance;
2819
2820        // Clean up the thread within this task as it's now never going
2821        // to run.
2822        //
2823        // Note that `next_switch_item`, if any, belongs to the canceller and
2824        // not this thread since this thread never ran, so it's saved and
2825        // restored here to prevent `cleanup_thread` from taking it.
2826        assert_eq!(1, task.threads.len());
2827        let thread = *task.threads.iter().next().unwrap();
2828        let next_switch_item = concurrent_state.save_next_switch_item()?;
2829        self.cleanup_thread(
2830            QualifiedThreadId {
2831                task: guest_task,
2832                thread,
2833            },
2834            caller_instance,
2835        )?;
2836        self.concurrent_state_mut()?
2837            .restore_next_switch_item(next_switch_item)?;
2838
2839        // Not yet started; cancel and remove from pending
2840        let pending = &mut self.instance_state(instance).concurrent_state().pending;
2841        let pending_count = pending.len();
2842        let mut decrement = 0;
2843        let mut consume = |kind: &GuestCallKind| {
2844            if let GuestCallKind::StartImplicit {
2845                was_pending: true, ..
2846            } = kind
2847            {
2848                decrement += 1;
2849            }
2850        };
2851        pending.retain(|thread, kind| {
2852            if thread.task == guest_task {
2853                consume(kind);
2854                false
2855            } else {
2856                true
2857            }
2858        });
2859
2860        if pending.len() == pending_count {
2861            // If it wasn't found in the pending set, it might be waiting in one
2862            // of the task queues or switch items:
2863            let mut matches = |item: &WorkItem| match item {
2864                WorkItem::ResumeThread { thread, .. } => thread.task == guest_task,
2865                WorkItem::GuestCall {
2866                    call: GuestCall { thread, kind },
2867                    ..
2868                } => {
2869                    if thread.task == guest_task {
2870                        consume(kind);
2871                        true
2872                    } else {
2873                        false
2874                    }
2875                }
2876                _ => false,
2877            };
2878
2879            let mut remove = |queue: &mut VecDeque<WorkItem>| {
2880                let count = queue.len();
2881                queue.retain(|item| !matches(item));
2882                count != queue.len()
2883            };
2884
2885            let state = self.concurrent_state_mut()?;
2886            let mut found = remove(&mut state.high_priority);
2887            found |= remove(&mut state.low_priority);
2888
2889            for item in [&mut state.switch_item, &mut state.next_switch_item] {
2890                if matches!(item, Some(item) if matches(item)) {
2891                    found = true;
2892                    *item = None;
2893                }
2894            }
2895
2896            if !found {
2897                bail!(Trap::SubtaskCancelAfterTerminal);
2898            }
2899        }
2900
2901        self.decrement_pending_start_count(instance, decrement)?;
2902
2903        Ok(())
2904    }
2905
2906    fn decrement_pending_start_count(
2907        &mut self,
2908        instance: RuntimeInstance,
2909        decrement: usize,
2910    ) -> Result<()> {
2911        if decrement > 0 {
2912            let count = &mut self
2913                .instance_state(instance)
2914                .concurrent_state()
2915                .pending_start_count;
2916
2917            if *count < decrement {
2918                bail_bug!("`ConcurrentInstantState::pending_start_count` underflow");
2919            }
2920
2921            *count -= decrement;
2922
2923            // Give the next tasks in line, if any, a chance to start:
2924            self.partition_pending(instance)?;
2925        }
2926
2927        Ok(())
2928    }
2929
2930    /// Used by `ResourceTables` to record the scope of a borrow to get undone
2931    /// in the future.
2932    pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2933        if !self.concurrency_support() {
2934            return Ok(self
2935                .current_scope_id_not_concurrent()?
2936                .map(|id| CurrentScope::Id(VMScope::id(id))));
2937        }
2938
2939        Ok(match self.current_thread()? {
2940            CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2941            CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2942            CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2943            CurrentThread::None => return Ok(None),
2944        })
2945    }
2946
2947    pub(crate) fn queue_task(
2948        &mut self,
2949        task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2950    ) -> Result<()> {
2951        self.concurrent_state_mut()?
2952            .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2953        Ok(())
2954    }
2955
2956    /// Used in `poll_until` just prior to trapping due to a "deadlock"
2957    /// condition.
2958    ///
2959    /// This helps us distinguish between a simple deadlock condition (where no
2960    /// work is available for the event loop to do, nor is there any way for new
2961    /// work to be added) and a "cannot block sync task" condition where at
2962    /// least once instance has an outstanding sync-typed task running, in which
2963    /// case we'll trap with a different error message.
2964    fn any_may_not_suspend(&mut self) -> Result<bool> {
2965        // Note that this currently requires a linear search across the whole
2966        // `ConcurrentState::table`.  We _could_ optimize that, but since (1)
2967        // this function is only used when trapping, (2) the only thing you can
2968        // really do with a store that's been poisoned by a trap is drop it, and
2969        // (3) we must do a linear search through the table when dropping a
2970        // store anyway to dispose of fibers, it's reasonable for us to also do
2971        // a linear search here.
2972        Ok(self
2973            .concurrent_state_mut()?
2974            .table
2975            .get_mut()
2976            .iter_mut()
2977            .filter_map(|(_, entry)| {
2978                if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2979                    Some(task.instance)
2980                } else {
2981                    None
2982                }
2983            })
2984            .collect::<Vec<_>>()
2985            .into_iter()
2986            .any(|instance| {
2987                self.instance_state(instance)
2988                    .concurrent_state()
2989                    .do_not_suspend
2990            }))
2991    }
2992}
2993
2994impl Instance {
2995    /// Get the next pending event for the specified task and (optional)
2996    /// waitable set, along with the waitable handle if applicable.
2997    fn get_event(
2998        self,
2999        store: &mut StoreOpaque,
3000        guest_task: TableId<GuestTask>,
3001        set: Option<TableId<WaitableSet>>,
3002        cancellable: bool,
3003    ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
3004        let state = store.concurrent_state_mut()?;
3005
3006        let task = state.get_mut(guest_task)?;
3007        let event = &mut task.event;
3008        if let Some(ev) = event
3009            && (cancellable || !matches!(ev, Event::Cancelled))
3010        {
3011            log::trace!("deliver event {ev:?} to {guest_task:?}");
3012
3013            if matches!(ev, Event::Cancelled) {
3014                task.cancel_request_delivered = true;
3015            }
3016
3017            let ev = *ev;
3018            *event = None;
3019            return Ok(Some((ev, None)));
3020        }
3021
3022        let set = match set {
3023            Some(set) => set,
3024            None => return Ok(None),
3025        };
3026        let waitable = match state.get_mut(set)?.ready.pop_first() {
3027            Some(v) => v,
3028            None => return Ok(None),
3029        };
3030
3031        let common = waitable.common(state)?;
3032        let handle = match common.handle {
3033            Some(h) => h,
3034            None => bail_bug!("handle not set when delivering event"),
3035        };
3036        let event = match common.event.take() {
3037            Some(e) => e,
3038            None => bail_bug!("event not set when delivering event"),
3039        };
3040
3041        log::trace!(
3042            "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
3043        );
3044
3045        waitable.on_delivery(store, self, event)?;
3046
3047        Ok(Some((event, Some((waitable, handle)))))
3048    }
3049
3050    /// Run `call` for the callback-lifted `guest_thread` and then handle the
3051    /// `CallbackCode` it returns.
3052    ///
3053    /// `call` runs with `guest_thread` as the current thread and with
3054    /// `runtime_instance` entered; both are restored before the final code is
3055    /// handled.
3056    fn handle_callback_code(
3057        self,
3058        store: &mut dyn VMStore,
3059        guest_thread: QualifiedThreadId,
3060        runtime_instance: RuntimeInstance,
3061        call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
3062    ) -> Result<()> {
3063        let old_thread = store.set_thread(guest_thread)?;
3064        log::trace!(
3065            "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
3066        );
3067
3068        store.enter_instance(runtime_instance);
3069
3070        let mut code = call(store)?;
3071
3072        // Implement the spec behavior here of effectively ignoring the return
3073        // code if there's a pending cancellation for this task available.
3074        //
3075        // TODO: the spec doesn't validate anything here, but seems like the
3076        // waitable set may want to be validated?
3077        if matches!(
3078            unpack_callback_code(code).0,
3079            callback_code::YIELD | callback_code::WAIT
3080        ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
3081        {
3082            let (unpacked, set) = unpack_callback_code(code);
3083            log::trace!(
3084                "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
3085                 cancellation; delivering it via callback"
3086            );
3087            code = invoke_callback(store, guest_thread.task, event, 0)?;
3088        }
3089
3090        store.exit_instance(runtime_instance)?;
3091
3092        store.set_thread(old_thread)?;
3093        log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
3094
3095        let store = &mut **store;
3096        let (code, set) = unpack_callback_code(code);
3097
3098        log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
3099
3100        let state = store.concurrent_state_mut()?;
3101
3102        state.take_next_switch_item()?;
3103
3104        let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
3105            let set = store
3106                .instance_state(runtime_instance)
3107                .handle_table()
3108                .waitable_set_rep(handle)?;
3109
3110            Ok(TableId::<WaitableSet>::new(set))
3111        };
3112
3113        match code {
3114            callback_code::EXIT => {
3115                log::trace!("implicit thread {guest_thread:?} completed");
3116                let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3117                task.exited = true;
3118                task.callback = None;
3119
3120                // Since we're async-typed, returning control to our caller
3121                // won't help resolve any outstanding sync-typed call which
3122                // might be in progress, so we may need to switch or trap before
3123                // exiting this thread:
3124                store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
3125
3126                store.cleanup_thread(guest_thread, runtime_instance)?;
3127            }
3128            callback_code::YIELD => {
3129                // Set `GuestTask::wake_on_cancel` to allow `subtask.cancel` to
3130                // promote this thread if appropriate.
3131                let old = state
3132                    .get_mut(guest_thread.thread)?
3133                    .wake_on_cancel
3134                    .replace(WakeOnCancel::Yielding);
3135                if !old.is_none() {
3136                    bail_bug!("thread unexpectedly had wake_on_cancel set");
3137                }
3138
3139                // If an `Event::Cancelled` is pending in `GuestTask::event`
3140                // when this call is delivered, we'll deliver that; otherwise,
3141                // we'll deliver `Event::None`.  Note that the `Event::None` is
3142                // deliberately not stored in `GuestTask::event` since that's
3143                // shared by all threads of the task and the `Event::None` is
3144                // only for this thread.
3145                let call = GuestCall {
3146                    thread: guest_thread,
3147                    kind: GuestCallKind::DeliverEvent {
3148                        instance: self,
3149                        set: None,
3150                    },
3151                };
3152                // Push this thread onto the "low priority" queue so it runs
3153                // after any other threads have had a chance to run.
3154                state.push_low_priority(WorkItem::GuestCall {
3155                    instance: runtime_instance,
3156                    call,
3157                });
3158            }
3159            callback_code::WAIT => {
3160                let set = get_set(store, set)?;
3161                self.wait_with_callback(store, guest_thread, set)?;
3162            }
3163            _ => bail!(Trap::UnsupportedCallbackCode),
3164        }
3165
3166        Ok(())
3167    }
3168
3169    /// Start waiting on `set` with `guest_thread`, which uses a callback.
3170    ///
3171    /// The thread will receive an event from `set` via a
3172    /// `GuestCallKind::DeliverEvent` work item, either immediately if one is
3173    /// already available, or once one is published.
3174    fn wait_with_callback(
3175        self,
3176        store: &mut StoreOpaque,
3177        guest_thread: QualifiedThreadId,
3178        set: TableId<WaitableSet>,
3179    ) -> Result<()> {
3180        let state = store.concurrent_state_mut()?;
3181        // This thread's wait ends when that work item is handled in
3182        // `handle_guest_call`.
3183        state.get_mut(set)?.num_waiting += 1;
3184
3185        if state.get_mut(guest_thread.task)?.event.is_some()
3186            || !state.get_mut(set)?.ready.is_empty()
3187        {
3188            // An event is immediately available; deliver it ASAP.
3189            let instance = state.get_mut(guest_thread.task)?.instance;
3190            state.push_high_priority(WorkItem::GuestCall {
3191                instance,
3192                call: GuestCall {
3193                    thread: guest_thread,
3194                    kind: GuestCallKind::DeliverEvent {
3195                        instance: self,
3196                        set: Some(set),
3197                    },
3198                },
3199            });
3200            return Ok(());
3201        }
3202
3203        // No event is immediately available, so this thread is about to block.
3204        // Like any other suspension point (e.g. the callback `EXIT` path), if
3205        // a sync-typed call is in progress for this instance we must first
3206        // switch to another ready thread of that instance, or trap if there is
3207        // none.
3208        let instance = store
3209            .concurrent_state_mut()?
3210            .get_mut(guest_thread.task)?
3211            .instance;
3212        store.switch_or_trap_if_may_not_suspend(instance)?;
3213
3214        // Register to be woken up when an event is published for this waitable
3215        // set.
3216        //
3217        // Here we also set `GuestTask::wake_on_cancel` which allows
3218        // `subtask.cancel` to interrupt the wait.
3219        let state = store.concurrent_state_mut()?;
3220        let old = state
3221            .get_mut(guest_thread.thread)?
3222            .wake_on_cancel
3223            .replace(WakeOnCancel::Waiting(set));
3224        if !old.is_none() {
3225            bail_bug!("thread unexpectedly had wake_on_cancel set");
3226        }
3227        let old = state
3228            .get_mut(set)?
3229            .waiting
3230            .insert(guest_thread, WaitMode::Callback(self));
3231        if !old.is_none() {
3232            bail_bug!("set's waiting set already had this thread registered");
3233        }
3234        Ok(())
3235    }
3236
3237    /// Stage the specified guest call as the work item to run next, to be
3238    /// started as soon as backpressure and/or reentrance rules allow.
3239    ///
3240    /// SAFETY: The raw pointer arguments must be valid references to guest
3241    /// functions (with the appropriate signatures) when the closures staged by
3242    /// this function are called.
3243    unsafe fn stage_call<T: 'static>(
3244        self,
3245        mut store: StoreContextMut<T>,
3246        guest_thread: QualifiedThreadId,
3247        callee: SendSyncPtr<VMFuncRef>,
3248        param_count: usize,
3249        result_count: usize,
3250        async_: bool,
3251        callback: Option<SendSyncPtr<VMFuncRef>>,
3252        post_return: Option<SendSyncPtr<VMFuncRef>>,
3253        host_caller: bool,
3254    ) -> Result<()> {
3255        /// Return a closure which will call the specified function in the scope
3256        /// of the specified task.
3257        ///
3258        /// This will use `GuestTask::lower_params` to lower the parameters, but
3259        /// will not lift the result; instead, it returns a
3260        /// `[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]` from which the result, if
3261        /// any, may be lifted.  Note that an async-lifted export will have
3262        /// returned its result using the `task.return` intrinsic (or not
3263        /// returned a result at all, in the case of `task.cancel`), in which
3264        /// case the "result" of this call will either be a callback code or
3265        /// nothing.
3266        ///
3267        /// SAFETY: `callee` must be a valid `*mut VMFuncRef` at the time when
3268        /// the returned closure is called.
3269        unsafe fn make_call<T: 'static>(
3270            store: StoreContextMut<T>,
3271            guest_thread: QualifiedThreadId,
3272            callee: SendSyncPtr<VMFuncRef>,
3273            param_count: usize,
3274            result_count: usize,
3275        ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3276        + Send
3277        + Sync
3278        + 'static
3279        + use<T> {
3280            let token = StoreToken::new(store);
3281            move |store: &mut dyn VMStore| {
3282                let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3283
3284                store
3285                    .concurrent_state_mut()?
3286                    .set_thread_running(guest_thread.thread)?;
3287                let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3288                let lower = match task.lower_params.take() {
3289                    Some(l) => l,
3290                    None => bail_bug!("lower_params missing"),
3291                };
3292
3293                lower(store, &mut storage[..param_count])?;
3294
3295                let mut store = token.as_context_mut(store);
3296
3297                // SAFETY: Per the contract documented in `make_call's`
3298                // documentation, `callee` must be a valid pointer.
3299                unsafe {
3300                    crate::Func::call_unchecked_raw(
3301                        &mut store,
3302                        callee.as_non_null(),
3303                        NonNull::new(
3304                            &mut storage[..param_count.max(result_count)]
3305                                as *mut [MaybeUninit<ValRaw>] as _,
3306                        )
3307                        .unwrap(),
3308                        UncaughtException::Trap,
3309                    )?;
3310                }
3311
3312                Ok(storage)
3313            }
3314        }
3315
3316        // SAFETY: Per the contract described in this function documentation,
3317        // the `callee` pointer which `call` closes over must be valid when
3318        // called by the closure we queue below.
3319        let call = unsafe {
3320            make_call(
3321                store.as_context_mut(),
3322                guest_thread,
3323                callee,
3324                param_count,
3325                result_count,
3326            )
3327        };
3328
3329        let callee_instance = store
3330            .0
3331            .concurrent_state_mut()?
3332            .get_mut(guest_thread.task)?
3333            .instance;
3334
3335        let fun = if callback.is_some() {
3336            assert!(async_);
3337
3338            Box::new(move |store: &mut dyn VMStore| {
3339                self.add_guest_thread_to_instance_table(
3340                    guest_thread.thread,
3341                    store,
3342                    callee_instance.index,
3343                )?;
3344                self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3345                    let storage = call(store)?;
3346
3347                    // SAFETY: `wasmparser` will have validated that the
3348                    // callback function returns a `i32` result.
3349                    Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3350                })
3351            }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3352        } else {
3353            let token = StoreToken::new(store.as_context_mut());
3354            Box::new(move |store: &mut dyn VMStore| {
3355                self.add_guest_thread_to_instance_table(
3356                    guest_thread.thread,
3357                    store,
3358                    callee_instance.index,
3359                )?;
3360                let old_thread = store.set_thread(guest_thread)?;
3361                log::trace!(
3362                    "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3363                );
3364                let flags = self.id().get(store).instance_flags(callee_instance.index);
3365
3366                let callee_async_typed = store
3367                    .concurrent_state_mut()?
3368                    .get_mut(guest_thread.task)?
3369                    .async_typed;
3370
3371                // Unless this is a callback-less (i.e. stackful) async-lifted
3372                // or sync-typed export, we need to record that the instance
3373                // cannot be entered until the call returns.
3374                if !async_ && callee_async_typed {
3375                    store.enter_instance(callee_instance);
3376                }
3377
3378                if !callee_async_typed {
3379                    store.enter_sync_call(callee_instance)?;
3380                }
3381
3382                // SAFETY: See the documentation for `make_call` to review the
3383                // contract we must uphold for `call` here.
3384                //
3385                // Per the contract described in the `stage_call`
3386                // documentation, the `callee` pointer which `call` closes
3387                // over must be valid.
3388                let storage = call(store)?;
3389
3390                if !callee_async_typed {
3391                    store.exit_sync_call(callee_instance)?;
3392                }
3393
3394                if !async_ {
3395                    // This is a sync-lifted export, so now is when we lift the
3396                    // result, optionally call the post-return function, if any,
3397                    // and finally notify any current or future waiters that the
3398                    // subtask has returned.
3399
3400                    if callee_async_typed {
3401                        store.exit_instance(callee_instance)?;
3402                    }
3403
3404                    let lift = {
3405                        let state = store.concurrent_state_mut()?;
3406                        if !state.get_mut(guest_thread.task)?.result.is_none() {
3407                            bail_bug!("task has already produced a result");
3408                        }
3409
3410                        match state.get_mut(guest_thread.task)?.lift_result.take() {
3411                            Some(lift) => lift,
3412                            None => bail_bug!("lift_result field is missing"),
3413                        }
3414                    };
3415
3416                    // SAFETY: `result_count` represents the number of core Wasm
3417                    // results returned, per `wasmparser`.
3418                    let result = (lift.lift)(store, unsafe {
3419                        mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3420                            &storage[..result_count],
3421                        )
3422                    })?;
3423
3424                    let post_return_arg = match result_count {
3425                        0 => ValRaw::i32(0),
3426                        // SAFETY: `result_count` represents the number of
3427                        // core Wasm results returned, per `wasmparser`.
3428                        1 => unsafe { storage[0].assume_init() },
3429                        _ => unreachable!(),
3430                    };
3431
3432                    unsafe {
3433                        call_post_return(
3434                            token.as_context_mut(store),
3435                            post_return.map(|v| v.as_non_null()),
3436                            post_return_arg,
3437                            flags,
3438                        )?;
3439                    }
3440
3441                    self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3442                }
3443
3444                store.set_thread(old_thread)?;
3445
3446                store
3447                    .concurrent_state_mut()?
3448                    .get_mut(guest_thread.task)?
3449                    .exited = true;
3450
3451                log::trace!(
3452                    "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3453                );
3454
3455                if callee_async_typed {
3456                    // If we're async-typed, returning control to our caller
3457                    // won't help resolve any outstanding sync-typed call which
3458                    // might be in progress, so we may need to switch or trap
3459                    // before exiting this thread:
3460                    store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3461                }
3462
3463                // This is a callback-less call, so the implicit thread has now completed
3464                store.cleanup_thread(guest_thread, callee_instance)?;
3465                Ok(())
3466            })
3467        };
3468
3469        store.0.concurrent_state_mut()?.push_work_item(
3470            WorkItem::GuestCall {
3471                instance: callee_instance,
3472                call: GuestCall {
3473                    thread: guest_thread,
3474                    kind: GuestCallKind::StartImplicit {
3475                        closure: fun,
3476                        was_pending: false,
3477                    },
3478                },
3479            },
3480            if host_caller {
3481                // We put host-initiated calls in the low priority queue so that
3482                // they're in line behind any existing, yielding tasks.  This
3483                // helps avoid starvation situations when the host is making new
3484                // calls faster than old ones can complete.  See issue #14769
3485                // for an example.
3486                Priority::Low
3487            } else {
3488                Priority::Switch
3489            },
3490        )?;
3491
3492        Ok(())
3493    }
3494
3495    /// Prepare (but do not start) a guest->guest call.
3496    ///
3497    /// This is called from fused adapter code generated in
3498    /// `wasmtime_environ::fact::trampoline::Compiler`.  `start` and `return_`
3499    /// are synthesized Wasm functions which move the parameters from the caller
3500    /// to the callee and the result from the callee to the caller,
3501    /// respectively.  The adapter will call `Self::start_call` immediately
3502    /// after calling this function.
3503    ///
3504    /// SAFETY: All the pointer arguments must be valid pointers to guest
3505    /// entities (and with the expected signatures for the function references
3506    /// -- see `wasmtime_environ::fact::trampoline::Compiler` for details).
3507    unsafe fn prepare_call<T: 'static>(
3508        self,
3509        mut store: StoreContextMut<T>,
3510        start: NonNull<VMFuncRef>,
3511        return_: NonNull<VMFuncRef>,
3512        caller_instance: RuntimeComponentInstanceIndex,
3513        callee_instance: RuntimeComponentInstanceIndex,
3514        task_return_type: TypeTupleIndex,
3515        callee_async_typed: bool,
3516        memory: *mut VMMemoryDefinition,
3517        string_encoding: StringEncoding,
3518        caller_info: CallerInfo,
3519    ) -> Result<()> {
3520        enum ResultInfo {
3521            Heap { results: u32 },
3522            Stack { result_count: u32 },
3523        }
3524
3525        let result_info = match &caller_info {
3526            CallerInfo::Async {
3527                has_result: true,
3528                params,
3529            } => ResultInfo::Heap {
3530                // FIXME(#4311): needs to read an i64 for memory64
3531                results: match params.last() {
3532                    Some(r) => r.get_u32(),
3533                    None => bail_bug!("retptr missing"),
3534                },
3535            },
3536            CallerInfo::Async {
3537                has_result: false, ..
3538            } => ResultInfo::Stack { result_count: 0 },
3539            CallerInfo::Sync {
3540                result_count,
3541                params,
3542            } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3543                // FIXME(#4311): needs to read an i64 for memory64
3544                results: match params.last() {
3545                    Some(r) => r.get_u32(),
3546                    None => bail_bug!("arg ptr missing"),
3547                },
3548            },
3549            CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3550                result_count: *result_count,
3551            },
3552        };
3553
3554        let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3555
3556        // Create a new guest task for the call, closing over the `start` and
3557        // `return_` functions to lift the parameters and lower the result,
3558        // respectively.
3559        let start = SendSyncPtr::new(start);
3560        let return_ = SendSyncPtr::new(return_);
3561        let token = StoreToken::new(store.as_context_mut());
3562        let old_thread = store.0.current_guest_thread()?;
3563
3564        let state = store.0.concurrent_state_mut()?;
3565
3566        debug_assert_eq!(
3567            state.get_mut(old_thread.task)?.instance,
3568            self.runtime_instance(caller_instance)
3569        );
3570
3571        let guest_thread = GuestTask::new(
3572            state,
3573            Box::new(move |store, dst| {
3574                let mut store = token.as_context_mut(store);
3575                assert!(dst.len() <= MAX_FLAT_PARAMS);
3576                // The `+ 1` here accounts for the return pointer, if any:
3577                let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3578                let count = match caller_info {
3579                    // Async callers, if they have a result, use the last
3580                    // parameter as a return pointer so chop that off if
3581                    // relevant here.
3582                    CallerInfo::Async { params, has_result } => {
3583                        let params = &params[..params.len() - usize::from(has_result)];
3584                        for (param, src) in params.iter().zip(&mut src) {
3585                            src.write(*param);
3586                        }
3587                        params.len()
3588                    }
3589
3590                    // Sync callers forward everything directly.
3591                    CallerInfo::Sync { params, .. } => {
3592                        for (param, src) in params.iter().zip(&mut src) {
3593                            src.write(*param);
3594                        }
3595                        params.len()
3596                    }
3597                };
3598                // SAFETY: `start` is a valid `*mut VMFuncRef` from
3599                // `wasmtime-cranelift`-generated fused adapter code.  Based on
3600                // how it was constructed (see
3601                // `wasmtime_environ::fact::trampoline::Compiler::compile_async_start_adapter`
3602                // for details) we know it takes count parameters and returns
3603                // `dst.len()` results.
3604                unsafe {
3605                    crate::Func::call_unchecked_raw(
3606                        &mut store,
3607                        start.as_non_null(),
3608                        NonNull::new(
3609                            &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3610                        )
3611                        .unwrap(),
3612                        UncaughtException::Trap,
3613                    )?;
3614                }
3615                dst.copy_from_slice(&src[..dst.len()]);
3616                let task = store.0.current_guest_thread()?.task;
3617                let state = store.0.concurrent_state_mut()?;
3618                Waitable::Guest(task).set_event(
3619                    state,
3620                    Some(Event::Subtask {
3621                        status: Status::Started,
3622                    }),
3623                )?;
3624                Ok(())
3625            }),
3626            LiftResult {
3627                lift: Box::new(move |store, src| {
3628                    // SAFETY: See comment in closure passed as `lower_params`
3629                    // parameter above.
3630                    let mut store = token.as_context_mut(store);
3631                    let mut my_src = src.to_owned(); // TODO: use stack to avoid allocation?
3632                    if let ResultInfo::Heap { results } = &result_info {
3633                        my_src.push(ValRaw::u32(*results));
3634                    }
3635
3636                    // SAFETY: `return_` is a valid `*mut VMFuncRef` from
3637                    // `wasmtime-cranelift`-generated fused adapter code.  Based
3638                    // on how it was constructed (see
3639                    // `wasmtime_environ::fact::trampoline::Compiler::compile_async_return_adapter`
3640                    // for details) we know it takes `src.len()` parameters and
3641                    // returns up to 1 result.
3642                    unsafe {
3643                        crate::Func::call_unchecked_raw(
3644                            &mut store,
3645                            return_.as_non_null(),
3646                            my_src.as_mut_slice().into(),
3647                            UncaughtException::Trap,
3648                        )?;
3649                    }
3650
3651                    let thread = store.0.current_guest_thread()?;
3652                    let state = store.0.concurrent_state_mut()?;
3653                    if sync_caller {
3654                        state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3655                            if let ResultInfo::Stack { result_count } = &result_info {
3656                                match result_count {
3657                                    0 => None,
3658                                    1 => Some(my_src[0]),
3659                                    _ => unreachable!(),
3660                                }
3661                            } else {
3662                                None
3663                            },
3664                        );
3665                    }
3666                    Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3667                }),
3668                ty: task_return_type,
3669                memory: NonNull::new(memory).map(SendSyncPtr::new),
3670                string_encoding,
3671            },
3672            Caller::Guest { thread: old_thread },
3673            None,
3674            self.runtime_instance(callee_instance),
3675            callee_async_typed,
3676            // We don't know whether the callee export was lifted sync or async
3677            // yet, but we'll update this in `start_call`:
3678            false,
3679        )?;
3680
3681        // Make the new thread the current one so that `Self::start_call` knows
3682        // which one to start.
3683        store.0.set_thread(guest_thread)?;
3684        log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3685
3686        Ok(())
3687    }
3688
3689    /// Call the specified callback function for an async-lifted export.
3690    ///
3691    /// SAFETY: `function` must be a valid reference to a guest function of the
3692    /// correct signature for a callback.
3693    unsafe fn call_callback<T>(
3694        self,
3695        mut store: StoreContextMut<T>,
3696        function: SendSyncPtr<VMFuncRef>,
3697        event: Event,
3698        handle: u32,
3699    ) -> Result<u32> {
3700        let (ordinal, result) = event.parts();
3701        let params = &mut [
3702            ValRaw::u32(ordinal),
3703            ValRaw::u32(handle),
3704            ValRaw::u32(result),
3705        ];
3706        // SAFETY: `func` is a valid `*mut VMFuncRef` from either
3707        // `wasmtime-cranelift`-generated fused adapter code or
3708        // `component::Options`.  Per `wasmparser` callback signature
3709        // validation, we know it takes three parameters and returns one.
3710        unsafe {
3711            crate::Func::call_unchecked_raw(
3712                &mut store,
3713                function.as_non_null(),
3714                params.as_mut_slice().into(),
3715                UncaughtException::Trap,
3716            )?;
3717        }
3718        Ok(params[0].get_u32())
3719    }
3720
3721    /// Start a guest->guest call previously prepared using
3722    /// `Self::prepare_call`.
3723    ///
3724    /// This is called from fused adapter code generated in
3725    /// `wasmtime_environ::fact::trampoline::Compiler`.  The adapter will call
3726    /// this function immediately after calling `Self::prepare_call`.
3727    ///
3728    /// If `flags` contains `START_FLAG_ASYNC_CALLER` then the caller used an
3729    /// async-lowered import and the call's packed status is written to
3730    /// `storage[0]`. Otherwise this blocks until the callee has produced its
3731    /// result, if any, which is written to `storage[0]`.
3732    ///
3733    /// SAFETY: The `*mut VMFuncRef` arguments must be valid pointers to guest
3734    /// functions with the appropriate signatures for the current guest task.
3735    /// If this is a call to an async-lowered import, the actual call may be
3736    /// deferred and run after this function returns, in which case the pointer
3737    /// arguments must also be valid when the call happens.
3738    unsafe fn start_call<T: 'static>(
3739        self,
3740        mut store: StoreContextMut<T>,
3741        callback: *mut VMFuncRef,
3742        post_return: *mut VMFuncRef,
3743        callee: NonNull<VMFuncRef>,
3744        param_count: u32,
3745        result_count: u32,
3746        flags: u32,
3747        storage: &mut [MaybeUninit<ValRaw>],
3748    ) -> Result<()> {
3749        let token = StoreToken::new(store.as_context_mut());
3750        let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3751        let guest_thread = store.0.current_guest_thread()?;
3752        let state = store.0.concurrent_state_mut()?;
3753
3754        // FIXME(#14950): this in theory should be a `bail_bug!`, but this can
3755        // be hit through `Func::call` for example. This isn't great because it
3756        // means that `Func::call` is sort of fundamentally incompatible with
3757        // component-model-async...
3758        if !state.event_loop_running {
3759            bail!(
3760                "async function started during an original synchronous \
3761                 entrypoint; please use an async-style entrypoint instead"
3762            );
3763        }
3764
3765        let callee = SendSyncPtr::new(callee);
3766        let param_count = usize::try_from(param_count)?;
3767        assert!(param_count <= MAX_FLAT_PARAMS);
3768        let result_count = usize::try_from(result_count)?;
3769        assert!(result_count <= MAX_FLAT_RESULTS);
3770
3771        let task = state.get_mut(guest_thread.task)?;
3772        let callee_async_typed = task.async_typed;
3773        let callee_instance = task.instance;
3774
3775        task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3776
3777        if let Some(callback) = NonNull::new(callback) {
3778            // We're calling an async-lifted export with a callback, so store
3779            // the callback and related context as part of the task so we can
3780            // call it later when needed.
3781            let callback = SendSyncPtr::new(callback);
3782            task.callback = Some(Box::new(move |store, event, handle| {
3783                let store = token.as_context_mut(store);
3784                unsafe { self.call_callback::<T>(store, callback, event, handle) }
3785            }));
3786        }
3787
3788        let Caller::Guest { thread: caller } = &task.caller else {
3789            // As of this writing, `start_call` is only used for guest->guest
3790            // calls.
3791            bail_bug!("start_call unexpectedly invoked for host->guest call");
3792        };
3793        let caller = *caller;
3794        let caller_instance = state.get_mut(caller.task)?.instance;
3795
3796        // Stage the call as the work item to run next, modulo backpressure, etc.
3797        unsafe {
3798            self.stage_call(
3799                store.as_context_mut(),
3800                guest_thread,
3801                callee,
3802                param_count,
3803                result_count,
3804                (flags & START_FLAG_ASYNC_CALLEE) != 0,
3805                NonNull::new(callback).map(SendSyncPtr::new),
3806                NonNull::new(post_return).map(SendSyncPtr::new),
3807                false,
3808            )?;
3809        }
3810
3811        let old_do_not_suspend = if callee_async_typed {
3812            // If we're starting an async-typed call, it is permitted to suspend
3813            // since that will just return control back to the caller, which may
3814            // be able to avoid blocking if needed.
3815            //
3816            // We'll restore the old value after the call either returns or
3817            // suspends.
3818            let state = store.0.instance_state(callee_instance).concurrent_state();
3819            let old_do_not_suspend = state.do_not_suspend;
3820            state.do_not_suspend = false;
3821            Some(old_do_not_suspend)
3822        } else {
3823            None
3824        };
3825
3826        let state = store.0.concurrent_state_mut()?;
3827
3828        // Use the caller's `GuestThread::sync_call_set` to register interest in
3829        // the subtask...
3830        let guest_waitable = Waitable::Guest(guest_thread.task);
3831        let old_set = guest_waitable.common(state)?.set;
3832        let set = state.get_mut(caller.thread)?.sync_call_set;
3833        guest_waitable.join(state, Some(set))?;
3834
3835        store.0.set_thread(CurrentThread::None)?;
3836
3837        // ... and suspend this fiber temporarily while we wait for it to start.
3838        //
3839        // Note that we _could_ call the callee directly using the current fiber
3840        // rather than suspend this one, but that would make reasoning about the
3841        // event loop more complicated and is probably only worth doing if
3842        // there's a measurable performance benefit.  In addition, it would mean
3843        // blocking the caller if the callee calls a blocking sync-lowered
3844        // import, and as of this writing the spec says we must not do that.
3845        //
3846        // Alternatively, the fused adapter code could be modified to call the
3847        // callee directly without calling a host-provided intrinsic at all (in
3848        // which case it would need to do its own, inline backpressure checks,
3849        // etc.).  Again, we'd want to see a measurable performance benefit
3850        // before committing to such an optimization.  And again, we'd need to
3851        // update the spec to allow that.
3852        let mut yielded = false;
3853        let (status, waitable) = loop {
3854            store.0.suspend(if yielded {
3855                SuspendReason::Waiting {
3856                    set,
3857                    thread: caller,
3858                }
3859            } else {
3860                yielded = true;
3861                SuspendReason::YieldingToSubtask { thread: caller }
3862            })?;
3863
3864            if let Some(old_do_not_suspend) = old_do_not_suspend {
3865                store
3866                    .0
3867                    .instance_state(callee_instance)
3868                    .concurrent_state()
3869                    .do_not_suspend = old_do_not_suspend;
3870            }
3871
3872            let state = store.0.concurrent_state_mut()?;
3873
3874            log::trace!("taking event for {:?}", guest_thread.task);
3875            let event = guest_waitable.take_event(state)?;
3876            let Some(Event::Subtask { status }) = event else {
3877                bail_bug!("subtasks should only get subtask events, got {event:?}")
3878            };
3879
3880            log::trace!("status {status:?} for {:?}", guest_thread.task);
3881
3882            if status == Status::Returned {
3883                // It returned, so we can stop waiting.
3884                break (status, None);
3885            } else if async_caller {
3886                // It hasn't returned yet, but the caller is calling via an
3887                // async-lowered import, so we generate a handle for the task
3888                // waitable and return the status.
3889                let handle = store
3890                    .0
3891                    .handle_table_insertion(caller_instance)
3892                    .subtask_guest(guest_thread.task.rep())?;
3893                store
3894                    .0
3895                    .concurrent_state_mut()?
3896                    .get_mut(guest_thread.task)?
3897                    .common
3898                    .handle = Some(handle);
3899                break (status, Some(handle));
3900            } else {
3901                // The callee hasn't returned yet, and the caller is calling via
3902                // a sync-lowered import, so we loop and keep waiting until the
3903                // callee returns.
3904                store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3905            }
3906        };
3907
3908        guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3909
3910        // Reset the current thread to point to the caller as it resumes control.
3911        store.0.set_thread(caller)?;
3912
3913        store
3914            .0
3915            .concurrent_state_mut()?
3916            .set_thread_running(caller.thread)?;
3917
3918        log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3919
3920        if async_caller {
3921            let Some(slot) = storage.first_mut() else {
3922                bail_bug!("no storage for async call status");
3923            };
3924            *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3925        } else {
3926            // The caller used a sync-lowered import, in which case the result,
3927            // if any, has been stashed in `GuestTask::sync_result`.
3928            let state = store.0.concurrent_state_mut()?;
3929            let task = state.get_mut(guest_thread.task)?;
3930            if let Some(Some(result)) = task.sync_result.take()? {
3931                storage[0] = MaybeUninit::new(result);
3932            }
3933        }
3934
3935        if status == Status::Returned {
3936            store
3937                .0
3938                .concurrent_state_mut()?
3939                .decrement_task_ref_count(guest_thread.task)?;
3940        }
3941
3942        Ok(())
3943    }
3944
3945    /// Poll the specified future once on behalf of a guest->host call using an
3946    /// async-lowered import.
3947    ///
3948    /// If it returns `Ready`, return `Ok(None)`.  Otherwise, if it returns
3949    /// `Pending`, add it to the set of futures to be polled as part of this
3950    /// instance's event loop until it completes, and then return
3951    /// `Ok(Some(handle))` where `handle` is the waitable handle to return.
3952    ///
3953    /// Whether the future returns `Ready` immediately or later, the `lower`
3954    /// function will be used to lower the result, if any, into the guest caller's
3955    /// stack and linear memory. The `lower` function is invoked with the
3956    /// `Option<R>` param `None` if the future is cancelled. The
3957    /// `Option<TableId<HostTask>>` is passed as `Some` if the host task was
3958    /// materialized during execution and allows `lower` to delete the task if
3959    /// needed.
3960    pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3961        self,
3962        mut store: StoreContextMut<'_, T>,
3963        host_task: EnteredHostTask,
3964        result_may_require_realloc: bool,
3965        future: impl Future<Output = Result<R>> + Send + 'static,
3966        lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3967        + Send
3968        + 'static,
3969    ) -> Result<u32> {
3970        let token = StoreToken::new(store.as_context_mut());
3971
3972        // Create an abortable future which hooks calls to poll and manages call
3973        // context state for the future.
3974        let (join_handle, future) = JoinHandle::run(future);
3975        let mut future = Box::pin(future);
3976
3977        // Finally, poll the future.  We can use a dummy `Waker` here because
3978        // we'll add the future to `ConcurrentState::futures` and poll it
3979        // automatically from the event loop if it doesn't complete immediately
3980        // here.
3981        let poll = tls::set(store.0, || {
3982            future
3983                .as_mut()
3984                .poll(&mut Context::from_waker(&Waker::noop()))
3985        });
3986
3987        match poll {
3988            // It finished immediately; lower the result and delete the task.
3989            Poll::Ready(result) => {
3990                let result = result.transpose()?;
3991                // Check if the host task was materialized so that it can be
3992                // deleted in `lower`.
3993                let task = store.0.current_materialized_host_task()?;
3994                lower(store.as_context_mut(), result, true, task)?;
3995                return Ok(Status::Returned.pack(None));
3996            }
3997
3998            // Future isn't ready yet, so fall through.
3999            Poll::Pending => {}
4000        }
4001
4002        // The future will outlive this call frame, so materialize the deferred
4003        // host task and attach its cancellation handle before publishing it to
4004        // the event loop.
4005        let Some(task) = store.0.materialize_host_task_id()? else {
4006            bail_bug!("current thread is not a host thread")
4007        };
4008        {
4009            let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
4010            assert!(matches!(state, HostTaskState::CalleeStarted));
4011            *state = HostTaskState::CalleeRunning(join_handle);
4012        }
4013
4014        // It hasn't finished yet; add the future to
4015        // `ConcurrentState::futures` so it will be polled by the event
4016        // loop and allocate a waitable handle to return to the guest.
4017
4018        // Wrap the future in a closure responsible for lowering the result into
4019        // the guest's stack and memory, as well as notifying any waiters that
4020        // the task returned.
4021        let future = Box::pin(async move {
4022            let result = match run_with_host_task_set(task, future).await? {
4023                Some(result) => Some(result?),
4024                None => None,
4025            };
4026            let on_complete = move |store: &mut dyn VMStore| {
4027                // Restore the `current_thread` to be the host so `lower` knows
4028                // how to manipulate borrows and knows which scope of borrows
4029                // to check.
4030                let mut store = token.as_context_mut(store);
4031                let old = store.0.set_thread(task)?;
4032
4033                let status = if result.is_some() {
4034                    Status::Returned
4035                } else {
4036                    Status::ReturnCancelled
4037                };
4038
4039                lower(store.as_context_mut(), result, false, Some(task))?;
4040                let state = store.0.concurrent_state_mut()?;
4041                match &mut state.get_mut(task)?.state {
4042                    // Cancellation is only complete once lowering has finished
4043                    // and the terminal event can be published.
4044                    pending @ HostTaskState::CalleeCancelling => {
4045                        *pending = HostTaskState::CalleeDone { cancelled: true };
4046                    }
4047
4048                    // Otherwise transition this task to the done state.
4049                    other => *other = HostTaskState::CalleeDone { cancelled: false },
4050                }
4051                Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
4052
4053                store.0.set_thread(old)?;
4054                Ok(())
4055            };
4056
4057            tls::get(move |store| {
4058                if result_may_require_realloc {
4059                    // Lowering may call the guest's realloc function, so run
4060                    // it on a worker fiber. This is required because the guest
4061                    // might try to suspend the fiber which is unsound when it's
4062                    // not actually on a fiber.
4063                    store
4064                        .concurrent_state_mut()?
4065                        .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
4066                            on_complete,
4067                        ))));
4068                    Ok(())
4069                } else {
4070                    // No guest code can be called while lowering this result,
4071                    // so avoid scheduling a worker fiber.
4072                    on_complete(store)
4073                }
4074            })
4075        });
4076
4077        // Make this task visible to the guest and then record what it
4078        // was made visible as.
4079        let caller = match host_task {
4080            Some(caller) => caller,
4081            None => bail_bug!("host task wasn't created but should have been"),
4082        };
4083        let state = store.0.concurrent_state_mut()?;
4084        state.push_future(future);
4085        let instance = state.get_mut(caller.task)?.instance;
4086        let handle = store
4087            .0
4088            .handle_table_insertion(instance)
4089            .subtask_host(task.rep())?;
4090        store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
4091        log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
4092
4093        // Restore the currently running thread to this host task's
4094        // caller. Note that the host task isn't deallocated as it's
4095        // within the store and will get deallocated later.
4096        store.0.set_thread(caller)?;
4097        Ok(Status::Started.pack(Some(handle)))
4098    }
4099
4100    /// Implements the `task.return` intrinsic, lifting the result for the
4101    /// current guest task.
4102    pub(crate) fn task_return(
4103        self,
4104        store: &mut dyn VMStore,
4105        ty: TypeTupleIndex,
4106        options: OptionsIndex,
4107        storage: &[ValRaw],
4108    ) -> Result<()> {
4109        let guest_thread = store.current_guest_thread()?;
4110        let state = store.concurrent_state_mut()?;
4111        if !state.get_mut(guest_thread.task)?.async_lifted {
4112            bail!(Trap::TaskReturnOrCancelSyncLifted);
4113        }
4114        let lift = state
4115            .get_mut(guest_thread.task)?
4116            .lift_result
4117            .take()
4118            .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4119        if !state.get_mut(guest_thread.task)?.result.is_none() {
4120            bail_bug!("task result unexpectedly already set");
4121        }
4122
4123        let CanonicalOptions {
4124            string_encoding,
4125            data_model,
4126            ..
4127        } = &self.id().get(store).component().env_component().options[options];
4128
4129        let invalid = ty != lift.ty
4130            || string_encoding != &lift.string_encoding
4131            || match data_model {
4132                CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
4133                    Some(memory) => {
4134                        let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
4135                        let actual = self.id().get(store).runtime_memory(memory);
4136                        expected != actual.as_ptr()
4137                    }
4138                    // Memory not specified, meaning it didn't need to be
4139                    // specified per validation, so not invalid.
4140                    None => false,
4141                },
4142                // Always invalid as this isn't supported.
4143                CanonicalOptionsDataModel::Gc { .. } => true,
4144            };
4145
4146        if invalid {
4147            bail!(Trap::TaskReturnInvalid);
4148        }
4149
4150        log::trace!("task.return for {guest_thread:?}");
4151
4152        let result = (lift.lift)(store, storage)?;
4153        self.task_complete(store, guest_thread.task, result, Status::Returned)
4154    }
4155
4156    /// Implements the `task.cancel` intrinsic.
4157    pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4158        let guest_thread = store.current_guest_thread()?;
4159        let state = store.concurrent_state_mut()?;
4160        let task = state.get_mut(guest_thread.task)?;
4161        if !task.async_lifted {
4162            bail!(Trap::TaskReturnOrCancelSyncLifted);
4163        }
4164        if !task.cancel_request_delivered {
4165            bail!(Trap::TaskCancelNotCancelled);
4166        }
4167        _ = task
4168            .lift_result
4169            .take()
4170            .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4171
4172        if !task.result.is_none() {
4173            bail_bug!("task result should not bet set yet");
4174        }
4175
4176        log::trace!("task.cancel for {guest_thread:?}");
4177
4178        self.task_complete(
4179            store,
4180            guest_thread.task,
4181            Box::new(DummyResult),
4182            Status::ReturnCancelled,
4183        )
4184    }
4185
4186    /// Complete the specified guest task (i.e. indicate that it has either
4187    /// returned a (possibly empty) result or cancelled itself).
4188    ///
4189    /// This will return any resource borrows and notify any current or future
4190    /// waiters that the task has completed.
4191    fn task_complete(
4192        self,
4193        store: &mut StoreOpaque,
4194        guest_task: TableId<GuestTask>,
4195        result: Box<dyn Any + Send + Sync>,
4196        status: Status,
4197    ) -> Result<()> {
4198        store
4199            .component_resource_tables(Some(self))?
4200            .validate_scope_exit()?;
4201
4202        let state = store.concurrent_state_mut()?;
4203        let task = state.get_mut(guest_task)?;
4204
4205        // A cancellation request is only delivered to a task which hasn't yet
4206        // resolved, so drop any pending, undelivered `Event::Cancelled` now
4207        // that this task has resolved.
4208        task.event = None;
4209
4210        if let Caller::Host { tx, .. } = &mut task.caller {
4211            if let Some(tx) = tx.take() {
4212                _ = tx.send(result);
4213            }
4214        } else {
4215            task.result = Some(result);
4216            Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4217        }
4218
4219        Ok(())
4220    }
4221
4222    /// Implements the `waitable-set.new` intrinsic.
4223    pub(crate) fn waitable_set_new(
4224        self,
4225        store: &mut StoreOpaque,
4226        caller_instance: RuntimeComponentInstanceIndex,
4227    ) -> Result<u32> {
4228        let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4229        let handle = store
4230            .handle_table_insertion(self.runtime_instance(caller_instance))
4231            .waitable_set(set.rep())?;
4232        log::trace!("new waitable set {set:?} (handle {handle})");
4233        Ok(handle)
4234    }
4235
4236    /// Implements the `waitable-set.drop` intrinsic.
4237    pub(crate) fn waitable_set_drop(
4238        self,
4239        store: &mut StoreOpaque,
4240        caller_instance: RuntimeComponentInstanceIndex,
4241        set: u32,
4242    ) -> Result<()> {
4243        let rep = store
4244            .instance_state(self.runtime_instance(caller_instance))
4245            .handle_table()
4246            .waitable_set_remove(set)?;
4247
4248        log::trace!("drop waitable set {rep} (handle {set})");
4249
4250        // Note that we're careful to check for waiters _before_ deleting the
4251        // set to avoid dropping any waiters in `WaitMode::Fiber(_)`, which
4252        // would panic.  See `drop-waitable-set-with-waiters.wast` for details.
4253        let set = store
4254            .concurrent_state_mut()?
4255            .get_mut(TableId::<WaitableSet>::new(rep))?;
4256        if set.num_waiting > 0 {
4257            bail!(Trap::WaitableSetDropHasWaiters);
4258        }
4259
4260        store
4261            .concurrent_state_mut()?
4262            .delete(TableId::<WaitableSet>::new(rep))?;
4263
4264        Ok(())
4265    }
4266
4267    /// Implements the `waitable.join` intrinsic.
4268    pub(crate) fn waitable_join(
4269        self,
4270        store: &mut StoreOpaque,
4271        caller_instance: RuntimeComponentInstanceIndex,
4272        waitable_handle: u32,
4273        set_handle: u32,
4274    ) -> Result<()> {
4275        let mut instance = self.id().get_mut(store);
4276        let waitable =
4277            Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4278
4279        let set = if set_handle == 0 {
4280            None
4281        } else {
4282            let set = instance.instance_states().0[caller_instance]
4283                .handle_table()
4284                .waitable_set_rep(set_handle)?;
4285
4286            Some(TableId::<WaitableSet>::new(set))
4287        };
4288
4289        waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4290
4291        log::trace!(
4292            "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4293        );
4294
4295        waitable.join(store.concurrent_state_mut()?, set)
4296    }
4297
4298    /// Implements the `subtask.drop` intrinsic.
4299    pub(crate) fn subtask_drop(
4300        self,
4301        store: &mut StoreOpaque,
4302        caller_instance: RuntimeComponentInstanceIndex,
4303        task_id: u32,
4304    ) -> Result<()> {
4305        self.waitable_join(store, caller_instance, task_id, 0)?;
4306
4307        let (rep, is_host) = store
4308            .instance_state(self.runtime_instance(caller_instance))
4309            .handle_table()
4310            .subtask_remove(task_id)?;
4311
4312        let concurrent_state = store.concurrent_state_mut()?;
4313        let waitable = if is_host {
4314            let id = TableId::<HostTask>::new(rep);
4315            let task = concurrent_state.get_mut(id)?;
4316            match &task.state {
4317                HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4318                    bail!(Trap::SubtaskDropNotResolved)
4319                }
4320                HostTaskState::CalleeDone { .. } => {}
4321                HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4322                    bail_bug!("invalid state for callee in `subtask.drop`")
4323                }
4324            }
4325
4326            Waitable::Host(id)
4327        } else {
4328            let id = TableId::<GuestTask>::new(rep);
4329            let task = concurrent_state.get_mut(id)?;
4330            if task.lift_result.is_some() {
4331                bail!(Trap::SubtaskDropNotResolved);
4332            }
4333            Waitable::Guest(id)
4334        };
4335
4336        waitable.common(concurrent_state)?.handle = None;
4337
4338        // If this subtask has an event that means that the terminal status of
4339        // this subtask wasn't yet received so it can't be dropped yet.
4340        if waitable.take_event(concurrent_state)?.is_some() {
4341            bail!(Trap::SubtaskDropNotResolved);
4342        }
4343
4344        waitable.delete_from(concurrent_state)?;
4345
4346        log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4347        Ok(())
4348    }
4349
4350    /// Implements the `waitable-set.wait` intrinsic.
4351    pub(crate) fn waitable_set_wait(
4352        self,
4353        store: &mut StoreOpaque,
4354        options: OptionsIndex,
4355        set: u32,
4356        payload: u32,
4357    ) -> Result<u32> {
4358        let &CanonicalOptions {
4359            instance: caller_instance,
4360            ..
4361        } = &self.id().get(store).component().env_component().options[options];
4362        let caller = self.runtime_instance(caller_instance);
4363        let rep = store
4364            .instance_state(self.runtime_instance(caller_instance))
4365            .handle_table()
4366            .waitable_set_rep(set)?;
4367
4368        self.waitable_check(
4369            store,
4370            caller,
4371            WaitableCheck::Wait,
4372            WaitableCheckParams {
4373                set: TableId::new(rep),
4374                options,
4375                payload,
4376            },
4377        )
4378    }
4379
4380    /// Implements the `waitable-set.poll` intrinsic.
4381    pub(crate) fn waitable_set_poll(
4382        self,
4383        store: &mut StoreOpaque,
4384        options: OptionsIndex,
4385        set: u32,
4386        payload: u32,
4387    ) -> Result<u32> {
4388        let &CanonicalOptions {
4389            instance: caller_instance,
4390            ..
4391        } = &self.id().get(store).component().env_component().options[options];
4392        let caller = self.runtime_instance(caller_instance);
4393        let rep = store
4394            .instance_state(caller)
4395            .handle_table()
4396            .waitable_set_rep(set)?;
4397
4398        self.waitable_check(
4399            store,
4400            caller,
4401            WaitableCheck::Poll,
4402            WaitableCheckParams {
4403                set: TableId::new(rep),
4404                options,
4405                payload,
4406            },
4407        )
4408    }
4409
4410    /// Implements the `thread.index` intrinsic.
4411    pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4412        let thread_id = store.current_guest_thread()?.thread;
4413        match store
4414            .concurrent_state_mut()?
4415            .get_mut(thread_id)?
4416            .instance_rep
4417        {
4418            Some(r) => Ok(r),
4419            None => bail_bug!("thread should have instance_rep by now"),
4420        }
4421    }
4422
4423    /// Implements the `thread.new-indirect` intrinsic.
4424    pub(crate) fn thread_new_indirect<T: 'static>(
4425        self,
4426        mut store: StoreContextMut<T>,
4427        runtime_instance: RuntimeComponentInstanceIndex,
4428        start_func_ty: ModuleInternedTypeIndex,
4429        start_func_table_idx: RuntimeTableIndex,
4430        start_func_idx: u32,
4431        context: i32,
4432    ) -> Result<u32> {
4433        log::trace!("creating new thread");
4434
4435        let (instance, registry) = self.id().get_mut_and_registry(store.0);
4436        let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4437        else {
4438            bail_bug!("thread.new-indirect start function type should be registered");
4439        };
4440        let callee = instance
4441            .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4442            .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4443        if callee.type_index(store.0) != start_func_ty {
4444            bail!(Trap::ThreadNewIndirectInvalidType);
4445        }
4446
4447        let token = StoreToken::new(store.as_context_mut());
4448        let start_func = Box::new(
4449            move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4450                let old_thread = store.set_thread(guest_thread)?;
4451                log::trace!(
4452                    "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4453                );
4454
4455                let mut store = token.as_context_mut(store);
4456                let mut params = [ValRaw::i32(context)];
4457                // Use call_unchecked rather than call or call_async, as we don't want to run the function
4458                // on a separate fiber if we're running in an async store.
4459                unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4460
4461                store.0.set_thread(old_thread)?;
4462
4463                let runtime_instance = self.runtime_instance(runtime_instance);
4464
4465                // We're not returning to any caller, so we may need to switch
4466                // or trap if the instance has an outstanding sync-typed call:
4467                store
4468                    .0
4469                    .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4470
4471                store.0.cleanup_thread(guest_thread, runtime_instance)?;
4472
4473                log::trace!("explicit thread {guest_thread:?} completed");
4474                let state = store.0.concurrent_state_mut()?;
4475                if let Some(t) = old_thread.guest() {
4476                    state.set_thread_running(t.thread)?;
4477                }
4478                log::trace!("thread start: restored {old_thread:?} as current thread");
4479
4480                Ok(())
4481            },
4482        );
4483
4484        let current_thread = store.0.current_guest_thread()?;
4485        let state = store.0.concurrent_state_mut()?;
4486        let parent_task = current_thread.task;
4487
4488        let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4489        let thread_id = state.push(new_thread)?;
4490        state.get_mut(parent_task)?.threads.insert(thread_id);
4491
4492        log::trace!("new thread with id {thread_id:?} created");
4493
4494        self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4495    }
4496
4497    pub(crate) fn resume_thread(
4498        self,
4499        store: &mut StoreOpaque,
4500        runtime_instance: RuntimeComponentInstanceIndex,
4501        thread_idx: u32,
4502        how: ResumeThread,
4503    ) -> Result<bool> {
4504        let thread_id =
4505            GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4506        let state = store.concurrent_state_mut()?;
4507        let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4508
4509        if store.current_guest_thread()? == guest_thread {
4510            bail!(Trap::CannotResumeThread);
4511        }
4512
4513        let priority = match how {
4514            ResumeThread::Resume => Priority::Switch,
4515            ResumeThread::ResumeLater => Priority::Low,
4516
4517            // Promotion is a noop unless the thread is ready, so only search
4518            // through pending work items to see if something matches.
4519            ResumeThread::Promote => {
4520                let instance = self.runtime_instance(runtime_instance);
4521                let do_not_enter = store
4522                    .instance_state(instance)
4523                    .concurrent_state()
4524                    .do_not_enter;
4525                return store.concurrent_state_mut()?.promote_work_item_matching(
4526                    |item: &WorkItem| match item {
4527                        WorkItem::ResumeThread { thread, .. }
4528                        | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4529
4530                        WorkItem::GuestCall {
4531                            call: GuestCall { thread, kind },
4532                            ..
4533                        } => {
4534                            *thread == guest_thread
4535                                && match kind {
4536                                    // Events can only be delivered if this
4537                                    // instance's exclusive lock isn't held, so
4538                                    // gate on that.
4539                                    GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4540
4541                                    // A not-yet-started thread made ready via
4542                                    // `thread.resume-later`.
4543                                    GuestCallKind::StartExplicit(_) => true,
4544
4545                                    // Technically shouldn't be possible since
4546                                    // if this hasn't started nothing could get
4547                                    // its id...
4548                                    GuestCallKind::StartImplicit { .. } => false,
4549                                }
4550                        }
4551
4552                        WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4553                    },
4554                );
4555            }
4556        };
4557
4558        let state = store.concurrent_state_mut()?;
4559        let thread = state.get_mut(guest_thread.thread)?;
4560
4561        // When resuming a thread it must be in a suspended state otherwise
4562        // this operation is a trap.
4563        match &thread.state {
4564            GuestThreadState::NotStartedExplicit(_) => {}
4565            GuestThreadState::Suspended(_) => {
4566                log::trace!("resuming thread {thread_id:?} that was suspended");
4567                state.schedule_suspended_thread(
4568                    guest_thread,
4569                    self.runtime_instance(runtime_instance),
4570                    priority,
4571                )?;
4572                return Ok(true);
4573            }
4574            _ => bail!(Trap::CannotResumeThread),
4575        }
4576        match mem::replace(&mut thread.state, GuestThreadState::Running) {
4577            GuestThreadState::NotStartedExplicit(start_func) => {
4578                log::trace!("starting thread {guest_thread:?}");
4579                let guest_call = WorkItem::GuestCall {
4580                    instance: self.runtime_instance(runtime_instance),
4581                    call: GuestCall {
4582                        thread: guest_thread,
4583                        kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4584                            start_func(store, guest_thread)
4585                        })),
4586                    },
4587                };
4588                state.push_work_item(guest_call, priority)?;
4589            }
4590            other => {
4591                thread.state = other;
4592                bail_bug!("thread state checked to be resumable above");
4593            }
4594        }
4595        Ok(true)
4596    }
4597
4598    fn add_guest_thread_to_instance_table(
4599        self,
4600        thread_id: TableId<GuestThread>,
4601        store: &mut StoreOpaque,
4602        runtime_instance: RuntimeComponentInstanceIndex,
4603    ) -> Result<u32> {
4604        let (state, counter) =
4605            store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4606        let guest_id = state
4607            .thread_handle_table()
4608            .guest_thread_insert(counter, thread_id.rep())?;
4609        store
4610            .concurrent_state_mut()?
4611            .get_mut(thread_id)?
4612            .instance_rep = Some(guest_id);
4613        Ok(guest_id)
4614    }
4615
4616    /// Helper function for the `thread.yield`, thread.suspend`,
4617    /// `thread.suspend-then-resume`, `thread.suspend-then-promote`,
4618    /// `thread.yield-then-resume`, and `thread.yield-then-promote` intrinsics.
4619    pub(crate) fn suspension_intrinsic(
4620        self,
4621        store: &mut StoreOpaque,
4622        caller: RuntimeComponentInstanceIndex,
4623        yielding: bool,
4624        to_thread: SuspensionTarget,
4625    ) -> Result<WaitResult> {
4626        let check_suspend = match to_thread {
4627            SuspensionTarget::Promote(thread) => {
4628                !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4629            }
4630            SuspensionTarget::Resume(thread) => {
4631                if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4632                    bail_bug!(
4633                        "`resume_thread` should only ever return false \
4634                         when `ResumeThread::Promote` is passed to it"
4635                    );
4636                }
4637                false
4638            }
4639            SuspensionTarget::None => true,
4640        };
4641
4642        if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4643            return if yielding {
4644                Ok(WaitResult::Completed)
4645            } else {
4646                Err(Trap::CannotBlockSyncTask.into())
4647            };
4648        }
4649
4650        let guest_thread = store.current_guest_thread()?;
4651
4652        let reason = if yielding {
4653            SuspendReason::Yielding {
4654                thread: guest_thread,
4655            }
4656        } else {
4657            SuspendReason::ExplicitlySuspending {
4658                thread: guest_thread,
4659            }
4660        };
4661
4662        store.suspend(reason)?;
4663
4664        Ok(WaitResult::Completed)
4665    }
4666
4667    /// Helper function for the `waitable-set.wait` and `waitable-set.poll` intrinsics.
4668    fn waitable_check(
4669        self,
4670        store: &mut StoreOpaque,
4671        caller: RuntimeInstance,
4672        check: WaitableCheck,
4673        params: WaitableCheckParams,
4674    ) -> Result<u32> {
4675        let guest_thread = store.current_guest_thread()?;
4676
4677        log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4678
4679        // If we're waiting, and there are no events immediately available,
4680        // suspend the fiber until that changes.
4681        match &check {
4682            WaitableCheck::Wait => {
4683                let set = params.set;
4684
4685                // Note that this is a loop since, by the time this fiber is
4686                // resumed after being woken up, another thread may have already
4687                // taken the event (e.g. via `waitable-set.poll`), in which case
4688                // we keep waiting.
4689                loop {
4690                    let state = store.concurrent_state_mut()?;
4691                    let task = state.get_mut(guest_thread.task)?;
4692                    if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4693                        || !state.get_mut(set)?.ready.is_empty()
4694                    {
4695                        break;
4696                    }
4697
4698                    store.switch_or_trap_if_may_not_suspend(caller)?;
4699
4700                    store.suspend(SuspendReason::Waiting {
4701                        set,
4702                        thread: guest_thread,
4703                    })?;
4704                }
4705            }
4706            WaitableCheck::Poll => {}
4707        }
4708
4709        log::trace!(
4710            "waitable check for {guest_thread:?}; set {:?}, part two",
4711            params.set
4712        );
4713
4714        // Deliver any pending events to the guest and return.
4715        let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4716
4717        let (ordinal, handle, result) = match &check {
4718            WaitableCheck::Wait => {
4719                let (event, waitable) = match event {
4720                    Some(p) => p,
4721                    None => bail_bug!("event expected to be present"),
4722                };
4723                let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4724                let (ordinal, result) = event.parts();
4725                (ordinal, handle, result)
4726            }
4727            WaitableCheck::Poll => {
4728                if let Some((event, waitable)) = event {
4729                    let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4730                    let (ordinal, result) = event.parts();
4731                    (ordinal, handle, result)
4732                } else {
4733                    log::trace!(
4734                        "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4735                        guest_thread.task,
4736                        params.set
4737                    );
4738                    let (ordinal, result) = Event::None.parts();
4739                    (ordinal, 0, result)
4740                }
4741            }
4742        };
4743        let ptr = crate::component::func::validate_inbounds_dynamic(
4744            &CanonicalAbiInfo::POINTER_PAIR,
4745            self.options_memory(store, params.options),
4746            &ValRaw::u32(params.payload),
4747        )?;
4748        let memory = self
4749            .options_memory_range_mut(store, params.options, ptr, 8)
4750            .unwrap();
4751        memory[0..4].copy_from_slice(&handle.to_le_bytes());
4752        memory[4..8].copy_from_slice(&result.to_le_bytes());
4753        Ok(ordinal)
4754    }
4755
4756    /// Implements the `subtask.cancel` intrinsic.
4757    pub(crate) fn subtask_cancel(
4758        self,
4759        store: &mut StoreOpaque,
4760        caller_instance: RuntimeComponentInstanceIndex,
4761        async_: bool,
4762        task_id: u32,
4763    ) -> Result<u32> {
4764        let (rep, is_host) = store
4765            .instance_state(self.runtime_instance(caller_instance))
4766            .handle_table()
4767            .subtask_rep(task_id)?;
4768        let waitable = if is_host {
4769            Waitable::Host(TableId::<HostTask>::new(rep))
4770        } else {
4771            Waitable::Guest(TableId::<GuestTask>::new(rep))
4772        };
4773        let concurrent_state = store.concurrent_state_mut()?;
4774
4775        log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4776
4777        waitable.trap_if_in_waitable_set(concurrent_state)?;
4778
4779        let needs_block;
4780        if let Waitable::Host(host_task) = waitable {
4781            let state = &mut concurrent_state.get_mut(host_task)?.state;
4782            match state {
4783                // If the callee is still running, signal an abort is requested.
4784                //
4785                // After cancelling this falls through to block waiting for the
4786                // host task to actually finish assuming that `async_` is false.
4787                // This blocking behavior resolves the race of `handle.abort()`
4788                // with the task actually getting cancelled or finishing.
4789                HostTaskState::CalleeRunning(handle) => {
4790                    handle.abort();
4791                    *state = HostTaskState::CalleeCancelling;
4792                    needs_block = true;
4793                }
4794
4795                // Cancellation was already requested, so fail as the task can't
4796                // be cancelled twice.
4797                HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4798                    bail!(Trap::SubtaskCancelAfterTerminal);
4799                }
4800                HostTaskState::CalleeDone { cancelled: false } => {
4801                    // The callee is already done so there's no need to
4802                    // block further for an event.
4803                    *state = HostTaskState::CalleeDone { cancelled: true };
4804                    needs_block = false;
4805                }
4806
4807                // These states should not be possible for a subtask that's
4808                // visible from the guest, so trap here.
4809                HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4810                    bail_bug!("invalid states for host callee")
4811                }
4812            }
4813        } else {
4814            let guest_task = TableId::<GuestTask>::new(rep);
4815            let task = concurrent_state.get_mut(guest_task)?;
4816
4817            // Cancellation was already requested, so fail as the task can't be
4818            // cancelled twice.
4819            if task.cancel_requested {
4820                bail!(Trap::SubtaskCancelAfterTerminal);
4821            }
4822
4823            if !task.already_lowered_parameters() {
4824                store.cancel_guest_subtask_without_lowered_parameters(
4825                    self.runtime_instance(caller_instance),
4826                    guest_task,
4827                )?;
4828                return Ok(Status::StartCancelled as u32);
4829            } else if !task.returned_or_cancelled() {
4830                // Started, but not yet returned or cancelled; send the
4831                // `CANCELLED` event
4832                //
4833                // Note that this might overwrite an event that was set earlier
4834                // (e.g. `Event::None` if the task is yielding), but that's
4835                // okay -- this should supersede the previous state.
4836                task.cancel_requested = true;
4837                task.event = Some(Event::Cancelled);
4838                let runtime_instance = task.instance;
4839                for thread in task.threads.clone() {
4840                    let thread = QualifiedThreadId {
4841                        task: guest_task,
4842                        thread,
4843                    };
4844                    let concurrent_state = store.concurrent_state_mut()?;
4845                    let thread_mut = concurrent_state.get_mut(thread.thread)?;
4846
4847                    let yield_ = |store: &mut StoreOpaque| {
4848                        // While we're yielding, temporarily set
4849                        // `do_not_suspend` to false on the subtask's instance
4850                        // since we'll be getting control back if it does
4851                        // suspend.
4852                        let state = store.instance_state(runtime_instance).concurrent_state();
4853                        let old_do_not_suspend = state.do_not_suspend;
4854                        state.do_not_suspend = false;
4855
4856                        let caller = store.current_guest_thread()?;
4857
4858                        // Temporarily add the waitable to the caller's
4859                        // `sync_call_set` to ensure that (1) it isn't already
4860                        // part of a different set and (2) it can't be added to
4861                        // a different set while we yield to the subtask.
4862                        let state = store.concurrent_state_mut()?;
4863                        let set = state.get_mut(caller.thread)?.sync_call_set;
4864                        waitable.join(state, Some(set))?;
4865
4866                        store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4867
4868                        let state = store.concurrent_state_mut()?;
4869                        waitable.join(state, None)?;
4870
4871                        store
4872                            .instance_state(runtime_instance)
4873                            .concurrent_state()
4874                            .do_not_suspend = old_do_not_suspend;
4875
4876                        Ok::<(), crate::Error>(())
4877                    };
4878
4879                    match thread_mut.wake_on_cancel.take() {
4880                        WakeOnCancel::Waiting(set) => {
4881                            // The thread is in a cancellable wait, so wake it up:
4882                            let instance = concurrent_state.take_callback_waiter(set, thread)?;
4883                            let item = WorkItem::GuestCall {
4884                                instance: runtime_instance,
4885                                call: GuestCall {
4886                                    thread,
4887                                    kind: GuestCallKind::DeliverEvent {
4888                                        instance,
4889                                        set: Some(set),
4890                                    },
4891                                },
4892                            };
4893                            concurrent_state.set_switch_item(item)?;
4894
4895                            yield_(store)?;
4896
4897                            break;
4898                        }
4899                        WakeOnCancel::Yielding => {
4900                            if concurrent_state.promote_thread_work_item(thread)? {
4901                                yield_(store)?;
4902                                break;
4903                            } else if store
4904                                .instance_state(runtime_instance)
4905                                .concurrent_state()
4906                                .pending
4907                                .contains_key(&thread)
4908                            {
4909                                // The thread's work item was parked in
4910                                // `pending` because its instance can't be
4911                                // entered right now, so the cancellation
4912                                // can't be delivered yet. It'll be delivered
4913                                // via `GuestTask::event` once the instance
4914                                // becomes enterable and the work item runs.
4915                                // Until then the thread is still yielding, so
4916                                // restore `wake_on_cancel` and move on to the
4917                                // next thread.
4918                                store
4919                                    .concurrent_state_mut()?
4920                                    .get_mut(thread.thread)?
4921                                    .wake_on_cancel = WakeOnCancel::Yielding;
4922                            } else {
4923                                bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4924                            }
4925                        }
4926                        WakeOnCancel::None => {}
4927                    }
4928                }
4929
4930                // Guest tasks need to block if they have not yet returned or
4931                // cancelled, even as a result of the event delivery above.
4932                needs_block = !store
4933                    .concurrent_state_mut()?
4934                    .get_mut(guest_task)?
4935                    .returned_or_cancelled()
4936            } else {
4937                needs_block = false;
4938            }
4939        };
4940
4941        // If we need to block waiting on the terminal status of this subtask
4942        // then return immediately in `async` mode, or otherwise wait for the
4943        // event to get signaled through the store.
4944        if needs_block {
4945            if async_ {
4946                return Ok(BLOCKED);
4947            }
4948
4949            // For cancelling host tasks save and later restore
4950            // `next_switch_item` so that our caller (or canceller) isn't
4951            // resumed in the meantime. This is our prerogative as a runtime
4952            // where we get to decide whether cancellation of host things blocks
4953            // or doesn't, and here we just need to get the final status from
4954            // the host side so it shouldn't block.
4955            //
4956            // A guest subtask, however, may genuinely block, so skip this in
4957            // that case.
4958            let old_next_switch_item = if is_host {
4959                Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4960            } else {
4961                None
4962            };
4963
4964            // Wait for this waitable to get signaled with its terminal
4965            // status. Once that's done fall through to the shared code.
4966            store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4967
4968            if let Some(item) = old_next_switch_item {
4969                let state = store.concurrent_state_mut()?;
4970                state.restore_next_switch_item(item)?;
4971            }
4972
4973            // .. fall through to determine what event's in store for us.
4974        }
4975
4976        let event = waitable.take_event(store.concurrent_state_mut()?)?;
4977        if let Some(Event::Subtask {
4978            status: status @ (Status::Returned | Status::ReturnCancelled),
4979        }) = event
4980        {
4981            Ok(status as u32)
4982        } else {
4983            bail!(Trap::SubtaskCancelAfterTerminal);
4984        }
4985    }
4986}
4987
4988/// Trait representing component model ABI async intrinsics and fused adapter
4989/// helper functions.
4990///
4991/// SAFETY (callers): Most of the methods in this trait accept raw pointers,
4992/// which must be valid for at least the duration of the call (and possibly for
4993/// as long as the relevant guest task exists, in the case of `*mut VMFuncRef`
4994/// pointers used for async calls).
4995pub trait VMComponentAsyncStore {
4996    /// A helper function for fused adapter modules involving calls where the
4997    /// one of the caller or callee is async.
4998    ///
4999    /// This helper is not used when the caller and callee both use the sync
5000    /// ABI, only when at least one is async is this used.
5001    unsafe fn prepare_call(
5002        &mut self,
5003        instance: Instance,
5004        memory: *mut VMMemoryDefinition,
5005        start: NonNull<VMFuncRef>,
5006        return_: NonNull<VMFuncRef>,
5007        caller_instance: RuntimeComponentInstanceIndex,
5008        callee_instance: RuntimeComponentInstanceIndex,
5009        task_return_type: TypeTupleIndex,
5010        callee_async: bool,
5011        string_encoding: StringEncoding,
5012        result_count: u32,
5013        storage: *mut ValRaw,
5014        storage_len: usize,
5015    ) -> Result<()>;
5016
5017    /// A helper function for fused adapter modules to start a call previously
5018    /// set up with `prepare_call`.
5019    unsafe fn start_call(
5020        &mut self,
5021        instance: Instance,
5022        callback: *mut VMFuncRef,
5023        post_return: *mut VMFuncRef,
5024        callee: NonNull<VMFuncRef>,
5025        param_count: u32,
5026        result_count: u32,
5027        flags: u32,
5028        storage: *mut MaybeUninit<ValRaw>,
5029        storage_len: usize,
5030    ) -> Result<()>;
5031
5032    /// The `future.write` intrinsic.
5033    fn future_write(
5034        &mut self,
5035        instance: Instance,
5036        caller: RuntimeComponentInstanceIndex,
5037        ty: TypeFutureTableIndex,
5038        options: OptionsIndex,
5039        future: u32,
5040        address: u32,
5041    ) -> Result<u32>;
5042
5043    /// The `future.read` intrinsic.
5044    fn future_read(
5045        &mut self,
5046        instance: Instance,
5047        caller: RuntimeComponentInstanceIndex,
5048        ty: TypeFutureTableIndex,
5049        options: OptionsIndex,
5050        future: u32,
5051        address: u32,
5052    ) -> Result<u32>;
5053
5054    /// The `future.drop-writable` intrinsic.
5055    fn future_drop_writable(
5056        &mut self,
5057        instance: Instance,
5058        ty: TypeFutureTableIndex,
5059        writer: u32,
5060    ) -> Result<()>;
5061
5062    /// The `stream.write` intrinsic.
5063    fn stream_write(
5064        &mut self,
5065        instance: Instance,
5066        caller: RuntimeComponentInstanceIndex,
5067        ty: TypeStreamTableIndex,
5068        options: OptionsIndex,
5069        stream: u32,
5070        address: u32,
5071        count: u32,
5072    ) -> Result<u32>;
5073
5074    /// The `stream.read` intrinsic.
5075    fn stream_read(
5076        &mut self,
5077        instance: Instance,
5078        caller: RuntimeComponentInstanceIndex,
5079        ty: TypeStreamTableIndex,
5080        options: OptionsIndex,
5081        stream: u32,
5082        address: u32,
5083        count: u32,
5084    ) -> Result<u32>;
5085
5086    /// The "fast-path" implementation of the `stream.write` intrinsic for
5087    /// "flat" (i.e. memcpy-able) payloads.
5088    fn flat_stream_write(
5089        &mut self,
5090        instance: Instance,
5091        caller: RuntimeComponentInstanceIndex,
5092        ty: TypeStreamTableIndex,
5093        options: OptionsIndex,
5094        payload_size: u32,
5095        payload_align: u32,
5096        stream: u32,
5097        address: u32,
5098        count: u32,
5099    ) -> Result<u32>;
5100
5101    /// The "fast-path" implementation of the `stream.read` intrinsic for "flat"
5102    /// (i.e. memcpy-able) payloads.
5103    fn flat_stream_read(
5104        &mut self,
5105        instance: Instance,
5106        caller: RuntimeComponentInstanceIndex,
5107        ty: TypeStreamTableIndex,
5108        options: OptionsIndex,
5109        payload_size: u32,
5110        payload_align: u32,
5111        stream: u32,
5112        address: u32,
5113        count: u32,
5114    ) -> Result<u32>;
5115
5116    /// The `stream.drop-writable` intrinsic.
5117    fn stream_drop_writable(
5118        &mut self,
5119        instance: Instance,
5120        ty: TypeStreamTableIndex,
5121        writer: u32,
5122    ) -> Result<()>;
5123
5124    /// The `error-context.debug-message` intrinsic.
5125    fn error_context_debug_message(
5126        &mut self,
5127        instance: Instance,
5128        ty: TypeComponentLocalErrorContextTableIndex,
5129        options: OptionsIndex,
5130        err_ctx_handle: u32,
5131        debug_msg_address: u32,
5132    ) -> Result<()>;
5133
5134    /// The `thread.new-indirect` intrinsic
5135    fn thread_new_indirect(
5136        &mut self,
5137        instance: Instance,
5138        caller: RuntimeComponentInstanceIndex,
5139        func_ty_idx: ModuleInternedTypeIndex,
5140        start_func_table_idx: RuntimeTableIndex,
5141        start_func_idx: u32,
5142        context: i32,
5143    ) -> Result<u32>;
5144}
5145
5146/// SAFETY: See trait docs.
5147impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5148    unsafe fn prepare_call(
5149        &mut self,
5150        instance: Instance,
5151        memory: *mut VMMemoryDefinition,
5152        start: NonNull<VMFuncRef>,
5153        return_: NonNull<VMFuncRef>,
5154        caller_instance: RuntimeComponentInstanceIndex,
5155        callee_instance: RuntimeComponentInstanceIndex,
5156        task_return_type: TypeTupleIndex,
5157        callee_async: bool,
5158        string_encoding: StringEncoding,
5159        result_count_or_max_if_async: u32,
5160        storage: *mut ValRaw,
5161        storage_len: usize,
5162    ) -> Result<()> {
5163        // SAFETY: The `wasmtime_cranelift`-generated code that calls
5164        // this method will have ensured that `storage` is a valid
5165        // pointer containing at least `storage_len` items.
5166        let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5167
5168        unsafe {
5169            instance.prepare_call(
5170                StoreContextMut(self),
5171                start,
5172                return_,
5173                caller_instance,
5174                callee_instance,
5175                task_return_type,
5176                callee_async,
5177                memory,
5178                string_encoding,
5179                match result_count_or_max_if_async {
5180                    PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5181                        params,
5182                        has_result: false,
5183                    },
5184                    PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5185                        params,
5186                        has_result: true,
5187                    },
5188                    result_count => CallerInfo::Sync {
5189                        params,
5190                        result_count,
5191                    },
5192                },
5193            )
5194        }
5195    }
5196
5197    unsafe fn start_call(
5198        &mut self,
5199        instance: Instance,
5200        callback: *mut VMFuncRef,
5201        post_return: *mut VMFuncRef,
5202        callee: NonNull<VMFuncRef>,
5203        param_count: u32,
5204        result_count: u32,
5205        flags: u32,
5206        storage: *mut MaybeUninit<ValRaw>,
5207        storage_len: usize,
5208    ) -> Result<()> {
5209        unsafe {
5210            instance.start_call(
5211                StoreContextMut(self),
5212                callback,
5213                post_return,
5214                callee,
5215                param_count,
5216                result_count,
5217                flags,
5218                // SAFETY: The `wasmtime_cranelift`-generated code that calls
5219                // this method will have ensured that `storage` is a valid
5220                // pointer containing at least `storage_len` items.
5221                core::slice::from_raw_parts_mut(storage, storage_len),
5222            )
5223        }
5224    }
5225
5226    fn future_write(
5227        &mut self,
5228        instance: Instance,
5229        caller: RuntimeComponentInstanceIndex,
5230        ty: TypeFutureTableIndex,
5231        options: OptionsIndex,
5232        future: u32,
5233        address: u32,
5234    ) -> Result<u32> {
5235        instance
5236            .guest_write(
5237                StoreContextMut(self),
5238                caller,
5239                TransmitIndex::Future(ty),
5240                options,
5241                None,
5242                future,
5243                address,
5244                1,
5245            )
5246            .map(|result| result.encode())
5247    }
5248
5249    fn future_read(
5250        &mut self,
5251        instance: Instance,
5252        caller: RuntimeComponentInstanceIndex,
5253        ty: TypeFutureTableIndex,
5254        options: OptionsIndex,
5255        future: u32,
5256        address: u32,
5257    ) -> Result<u32> {
5258        instance
5259            .guest_read(
5260                StoreContextMut(self),
5261                caller,
5262                TransmitIndex::Future(ty),
5263                options,
5264                None,
5265                future,
5266                address,
5267                1,
5268            )
5269            .map(|result| result.encode())
5270    }
5271
5272    fn stream_write(
5273        &mut self,
5274        instance: Instance,
5275        caller: RuntimeComponentInstanceIndex,
5276        ty: TypeStreamTableIndex,
5277        options: OptionsIndex,
5278        stream: u32,
5279        address: u32,
5280        count: u32,
5281    ) -> Result<u32> {
5282        instance
5283            .guest_write(
5284                StoreContextMut(self),
5285                caller,
5286                TransmitIndex::Stream(ty),
5287                options,
5288                None,
5289                stream,
5290                address,
5291                count,
5292            )
5293            .map(|result| result.encode())
5294    }
5295
5296    fn stream_read(
5297        &mut self,
5298        instance: Instance,
5299        caller: RuntimeComponentInstanceIndex,
5300        ty: TypeStreamTableIndex,
5301        options: OptionsIndex,
5302        stream: u32,
5303        address: u32,
5304        count: u32,
5305    ) -> Result<u32> {
5306        instance
5307            .guest_read(
5308                StoreContextMut(self),
5309                caller,
5310                TransmitIndex::Stream(ty),
5311                options,
5312                None,
5313                stream,
5314                address,
5315                count,
5316            )
5317            .map(|result| result.encode())
5318    }
5319
5320    fn future_drop_writable(
5321        &mut self,
5322        instance: Instance,
5323        ty: TypeFutureTableIndex,
5324        writer: u32,
5325    ) -> Result<()> {
5326        instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5327    }
5328
5329    fn flat_stream_write(
5330        &mut self,
5331        instance: Instance,
5332        caller: RuntimeComponentInstanceIndex,
5333        ty: TypeStreamTableIndex,
5334        options: OptionsIndex,
5335        payload_size: u32,
5336        payload_align: u32,
5337        stream: u32,
5338        address: u32,
5339        count: u32,
5340    ) -> Result<u32> {
5341        instance
5342            .guest_write(
5343                StoreContextMut(self),
5344                caller,
5345                TransmitIndex::Stream(ty),
5346                options,
5347                Some(FlatAbi {
5348                    size: payload_size,
5349                    align: payload_align,
5350                }),
5351                stream,
5352                address,
5353                count,
5354            )
5355            .map(|result| result.encode())
5356    }
5357
5358    fn flat_stream_read(
5359        &mut self,
5360        instance: Instance,
5361        caller: RuntimeComponentInstanceIndex,
5362        ty: TypeStreamTableIndex,
5363        options: OptionsIndex,
5364        payload_size: u32,
5365        payload_align: u32,
5366        stream: u32,
5367        address: u32,
5368        count: u32,
5369    ) -> Result<u32> {
5370        instance
5371            .guest_read(
5372                StoreContextMut(self),
5373                caller,
5374                TransmitIndex::Stream(ty),
5375                options,
5376                Some(FlatAbi {
5377                    size: payload_size,
5378                    align: payload_align,
5379                }),
5380                stream,
5381                address,
5382                count,
5383            )
5384            .map(|result| result.encode())
5385    }
5386
5387    fn stream_drop_writable(
5388        &mut self,
5389        instance: Instance,
5390        ty: TypeStreamTableIndex,
5391        writer: u32,
5392    ) -> Result<()> {
5393        instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5394    }
5395
5396    fn error_context_debug_message(
5397        &mut self,
5398        instance: Instance,
5399        ty: TypeComponentLocalErrorContextTableIndex,
5400        options: OptionsIndex,
5401        err_ctx_handle: u32,
5402        debug_msg_address: u32,
5403    ) -> Result<()> {
5404        instance.error_context_debug_message(
5405            StoreContextMut(self),
5406            ty,
5407            options,
5408            err_ctx_handle,
5409            debug_msg_address,
5410        )
5411    }
5412
5413    fn thread_new_indirect(
5414        &mut self,
5415        instance: Instance,
5416        caller: RuntimeComponentInstanceIndex,
5417        func_ty_idx: ModuleInternedTypeIndex,
5418        start_func_table_idx: RuntimeTableIndex,
5419        start_func_idx: u32,
5420        context: i32,
5421    ) -> Result<u32> {
5422        instance.thread_new_indirect(
5423            StoreContextMut(self),
5424            caller,
5425            func_ty_idx,
5426            start_func_table_idx,
5427            start_func_idx,
5428            context,
5429        )
5430    }
5431}
5432
5433type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5434
5435/// Runs the given future with the current thread set to `task` each time it is
5436/// polled.
5437async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5438where
5439    F: Future,
5440{
5441    let mut future = pin!(future);
5442    future::poll_fn(|cx| {
5443        let old_thread = match tls::get(|store| store.set_thread(task)) {
5444            Ok(thread) => thread,
5445            Err(error) => return Poll::Ready(Err(error)),
5446        };
5447        let result = future.as_mut().poll(cx);
5448        match tls::get(|store| store.set_thread(old_thread)) {
5449            Ok(_) => result.map(Ok),
5450            Err(error) => Poll::Ready(Err(error)),
5451        }
5452    })
5453    .await
5454}
5455
5456/// Represents the state of a pending host task.
5457///
5458/// This is used to represent tasks when the guest calls into the host.
5459pub(crate) struct HostTask {
5460    common: WaitableCommon,
5461
5462    /// State of borrows/etc the host needs to track. Used when the guest passes
5463    /// borrows to the host, for example.
5464    call_context: CallContext,
5465
5466    state: HostTaskState,
5467
5468    group: TaskGroupId,
5469}
5470
5471enum HostTaskState {
5472    /// A host task has been created and it's considered "started".
5473    ///
5474    /// The host task has yet to enter `first_poll` or `poll_and_block` which
5475    /// is where this will get updated further.
5476    CalleeStarted,
5477
5478    /// State used for tasks in `first_poll` meaning that the guest did an async
5479    /// lower of a host async function which is blocked. The specified handle is
5480    /// linked to the future in the main `FuturesUnordered` of a store which is
5481    /// used to cancel it if the guest requests cancellation.
5482    CalleeRunning(JoinHandle),
5483
5484    /// Cancellation was requested, but the completion worker has not finished
5485    /// lowering the result and publishing the terminal event. The task must
5486    /// remain in the table until that worker completes.
5487    CalleeCancelling,
5488
5489    /// Terminal state used for tasks in `poll_and_block` to store the result of
5490    /// their computation. Note that this state is not used for tasks in
5491    /// `first_poll`.
5492    CalleeFinished(LiftedResult),
5493
5494    /// Terminal state for host tasks meaning that the task was cancelled or the
5495    /// result was taken.
5496    CalleeDone { cancelled: bool },
5497}
5498
5499impl HostTask {
5500    fn new(
5501        concurrent_state: &mut ConcurrentState,
5502        state: HostTaskState,
5503        caller: QualifiedThreadId,
5504    ) -> Result<Self> {
5505        let group = concurrent_state.get_mut(caller.task)?.group;
5506        concurrent_state.increment_group_ref_count(group)?;
5507
5508        Ok(Self {
5509            common: WaitableCommon::default(),
5510            call_context: CallContext::default(),
5511            state,
5512            group,
5513        })
5514    }
5515}
5516
5517impl TableDebug for HostTask {
5518    fn type_name() -> &'static str {
5519        "HostTask"
5520    }
5521}
5522
5523type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5524
5525/// Represents the caller of a given guest task.
5526enum Caller {
5527    /// The host called the guest task.
5528    Host {
5529        /// If present, may be used to deliver the result.
5530        tx: Option<oneshot::Sender<LiftedResult>>,
5531        /// The host task which called into the guest, or `None` for a call from
5532        /// the top-level host. The task may belong to an entirely unrelated
5533        /// top-level component instance than the one the host called into.
5534        caller: Option<TableId<HostTask>>,
5535    },
5536    /// Another guest thread called the guest task
5537    Guest {
5538        /// The id of the caller
5539        thread: QualifiedThreadId,
5540    },
5541}
5542
5543/// Represents a closure and related canonical ABI parameters required to
5544/// validate a `task.return` call at runtime and lift the result.
5545struct LiftResult {
5546    lift: RawLift,
5547    ty: TypeTupleIndex,
5548    memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5549    string_encoding: StringEncoding,
5550}
5551
5552/// The table ID for a guest thread, qualified by the task to which it belongs.
5553///
5554/// This exists to minimize table lookups and the necessity to pass stores around mutably
5555/// for the common case of identifying the task to which a thread belongs.
5556#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5557pub(crate) struct QualifiedThreadId {
5558    task: TableId<GuestTask>,
5559    thread: TableId<GuestThread>,
5560}
5561
5562impl QualifiedThreadId {
5563    fn qualify(
5564        state: &mut ConcurrentState,
5565        thread: TableId<GuestThread>,
5566    ) -> Result<QualifiedThreadId> {
5567        Ok(QualifiedThreadId {
5568            task: state.get_mut(thread)?.parent_task,
5569            thread,
5570        })
5571    }
5572}
5573
5574impl fmt::Debug for QualifiedThreadId {
5575    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5576        f.debug_tuple("QualifiedThreadId")
5577            .field(&self.task.rep())
5578            .field(&self.thread.rep())
5579            .finish()
5580    }
5581}
5582
5583enum GuestThreadState {
5584    NotStartedImplicit,
5585    NotStartedExplicit(
5586        Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5587    ),
5588    Running,
5589    Suspended(StoreFiber<'static>),
5590    Ready {
5591        fiber: StoreFiber<'static>,
5592    },
5593    Completed,
5594}
5595
5596impl GuestThreadState {
5597    fn check_no_fiber(&self) -> Result<()> {
5598        match self {
5599            Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5600            Self::NotStartedImplicit
5601            | Self::NotStartedExplicit(_)
5602            | Self::Running
5603            | Self::Completed => Ok(()),
5604        }
5605    }
5606}
5607
5608impl fmt::Debug for GuestThreadState {
5609    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5610        match self {
5611            Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5612            Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5613            Self::Running => f.debug_tuple("Running").finish(),
5614            Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5615            Self::Ready { .. } => f.debug_struct("Ready").finish(),
5616            Self::Completed => f.debug_tuple("Completed").finish(),
5617        }
5618    }
5619}
5620
5621#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5622enum WakeOnCancel {
5623    None,
5624    Waiting(TableId<WaitableSet>),
5625    Yielding,
5626}
5627
5628impl WakeOnCancel {
5629    fn is_none(self) -> bool {
5630        matches!(self, WakeOnCancel::None)
5631    }
5632
5633    fn replace(&mut self, other: WakeOnCancel) -> Self {
5634        let old = *self;
5635        *self = other;
5636        old
5637    }
5638
5639    fn take(&mut self) -> Self {
5640        self.replace(WakeOnCancel::None)
5641    }
5642}
5643
5644pub struct GuestThread {
5645    /// Context-local state used to implement the `context.{get,set}`
5646    /// intrinsics.
5647    context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5648    /// The owning guest task.
5649    parent_task: TableId<GuestTask>,
5650    /// If non-`None`, indicates that the thread is currently either waiting a
5651    /// waitable set or yielding but may be cancelled and woken immediately.
5652    wake_on_cancel: WakeOnCancel,
5653    /// The execution state of this guest thread
5654    state: GuestThreadState,
5655    /// The index of this thread in the component instance's handle table.
5656    /// This must always be `Some` after initialization.
5657    instance_rep: Option<u32>,
5658    /// Scratch waitable set used to watch subtasks during synchronous calls.
5659    sync_call_set: TableId<WaitableSet>,
5660    /// The old value of `do_not_suspend` prior to the sync-typed task for which
5661    /// this thread was created, if relevant.
5662    old_do_not_suspend: Option<bool>,
5663}
5664
5665impl GuestThread {
5666    /// Retrieve the `GuestThread` corresponding to the specified guest-visible
5667    /// handle.
5668    fn from_instance(
5669        state: Pin<&mut ComponentInstance>,
5670        caller_instance: RuntimeComponentInstanceIndex,
5671        guest_thread: u32,
5672    ) -> Result<TableId<Self>> {
5673        let rep = state.instance_states().0[caller_instance]
5674            .thread_handle_table()
5675            .guest_thread_rep(guest_thread)?;
5676        Ok(TableId::new(rep))
5677    }
5678
5679    fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5680        let sync_call_set = state.push(WaitableSet {
5681            is_sync_call_set: true,
5682            ..WaitableSet::default()
5683        })?;
5684        Ok(Self {
5685            context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5686            parent_task,
5687            wake_on_cancel: WakeOnCancel::None,
5688            state: GuestThreadState::NotStartedImplicit,
5689            instance_rep: None,
5690            sync_call_set,
5691            old_do_not_suspend: None,
5692        })
5693    }
5694
5695    fn new_explicit(
5696        state: &mut ConcurrentState,
5697        parent_task: TableId<GuestTask>,
5698        start_func: Box<
5699            dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5700        >,
5701    ) -> Result<Self> {
5702        state.increment_task_ref_count(parent_task)?;
5703        let sync_call_set = state.push(WaitableSet {
5704            is_sync_call_set: true,
5705            ..WaitableSet::default()
5706        })?;
5707        Ok(Self {
5708            context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5709            parent_task,
5710            wake_on_cancel: WakeOnCancel::None,
5711            state: GuestThreadState::NotStartedExplicit(start_func),
5712            instance_rep: None,
5713            sync_call_set,
5714            old_do_not_suspend: None,
5715        })
5716    }
5717}
5718
5719impl TableDebug for GuestThread {
5720    fn type_name() -> &'static str {
5721        "GuestThread"
5722    }
5723}
5724
5725enum SyncResult {
5726    NotProduced,
5727    Produced(Option<ValRaw>),
5728    Taken,
5729}
5730
5731impl SyncResult {
5732    fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5733        Ok(match mem::replace(self, SyncResult::Taken) {
5734            SyncResult::NotProduced => None,
5735            SyncResult::Produced(val) => Some(val),
5736            SyncResult::Taken => {
5737                bail_bug!("attempted to take a synchronous result that was already taken")
5738            }
5739        })
5740    }
5741}
5742
5743/// Represents a pending guest task.
5744pub(crate) struct GuestTask {
5745    /// See `WaitableCommon`
5746    common: WaitableCommon,
5747    /// Closure to lower the parameters passed to this task.
5748    lower_params: Option<RawLower>,
5749    /// See `LiftResult`
5750    lift_result: Option<LiftResult>,
5751    /// A place to stash the type-erased lifted result if it can't be delivered
5752    /// immediately.
5753    result: Option<LiftedResult>,
5754    /// Closure to call the callback function for an async-lifted export, if
5755    /// provided.
5756    callback: Option<CallbackFn>,
5757    /// See `Caller`
5758    caller: Caller,
5759    /// Borrow state for this task.
5760    ///
5761    /// Keeps track of `borrow<T>` received to this task to ensure that
5762    /// everything is dropped by the time it exits.
5763    call_context: CallContext,
5764    /// A place to stash the lowered result for a sync-to-async call until it
5765    /// can be returned to the caller.
5766    sync_result: SyncResult,
5767    /// Whether or not the task has been cancelled (i.e. whether the
5768    /// cancellation request has been delivered to the task, and thus whether
5769    /// the task is permitted to call `task.cancel`).
5770    cancel_request_delivered: bool,
5771    /// Whether or not the caller has requested cancellation of this task via
5772    /// `subtask.cancel` while it was still unresolved. A subsequent
5773    /// `subtask.cancel` of the same task traps. This matches the spec's
5774    /// `Subtask.cancellation_requested`.
5775    cancel_requested: bool,
5776    /// Whether or not we've sent a `Status::Starting` event to any current or
5777    /// future waiters for this waitable.
5778    starting_sent: bool,
5779    /// The runtime instance to which the exported function for this guest task
5780    /// belongs.
5781    ///
5782    /// Note that the task may do a sync->sync call via a fused adapter which
5783    /// results in that task executing code in a different instance, and it may
5784    /// call host functions and intrinsics from that other instance.
5785    instance: RuntimeInstance,
5786    /// If present, a pending `Event::Cancelled` to be delivered to this task.
5787    event: Option<Event>,
5788    /// Whether or not the task has exited.
5789    exited: bool,
5790    /// Threads belonging to this task
5791    threads: HashSet<TableId<GuestThread>>,
5792    /// Indicates whether this task was created for a call to an async-typed
5793    /// export.
5794    async_typed: bool,
5795    /// Indicates whether this task was created for a call to an async-lifted
5796    /// export.
5797    async_lifted: bool,
5798    /// Whether `ConcurrentState::interesting_tasks` has been decremented for
5799    /// this task yet.
5800    decremented_interesting_task_count: bool,
5801    /// The `TaskGroup` to which this task belongs.
5802    group: TaskGroupId,
5803    /// The number of outstanding references to this task.
5804    ///
5805    /// When this goes to zero, the task will be removed from
5806    /// `ConcurrentState::table`.
5807    ref_count: usize,
5808}
5809
5810impl GuestTask {
5811    fn already_lowered_parameters(&self) -> bool {
5812        // We reset `lower_params` after we lower the parameters
5813        self.lower_params.is_none()
5814    }
5815
5816    fn returned_or_cancelled(&self) -> bool {
5817        // We reset `lift_result` after we return or exit
5818        self.lift_result.is_none()
5819    }
5820
5821    fn new(
5822        state: &mut ConcurrentState,
5823        lower_params: RawLower,
5824        lift_result: LiftResult,
5825        caller: Caller,
5826        callback: Option<CallbackFn>,
5827        instance: RuntimeInstance,
5828        async_typed: bool,
5829        async_lifted: bool,
5830    ) -> Result<QualifiedThreadId> {
5831        let group = match caller {
5832            Caller::Guest { thread } => {
5833                let group = state.get_mut(thread.task)?.group;
5834                state.increment_group_ref_count(group)?;
5835                group
5836            }
5837            Caller::Host { .. } => state.make_task_group()?,
5838        };
5839
5840        let task = state.push(Self {
5841            common: WaitableCommon::default(),
5842            lower_params: Some(lower_params),
5843            lift_result: Some(lift_result),
5844            result: None,
5845            callback,
5846            caller,
5847            call_context: CallContext::default(),
5848            sync_result: SyncResult::NotProduced,
5849            cancel_request_delivered: false,
5850            cancel_requested: false,
5851            starting_sent: false,
5852            instance,
5853            event: None,
5854            exited: false,
5855            threads: HashSet::new(),
5856            async_typed,
5857            async_lifted,
5858            decremented_interesting_task_count: false,
5859            group,
5860            // One for the caller and one for the implicit guest thread we're
5861            // about to create:
5862            ref_count: 2,
5863        })?;
5864        let new_thread = GuestThread::new_implicit(state, task)?;
5865        let thread = state.push(new_thread)?;
5866        state.get_mut(task)?.threads.insert(thread);
5867        state.interesting_tasks += 1;
5868        let thread = QualifiedThreadId { task, thread };
5869        log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5870        Ok(thread)
5871    }
5872}
5873
5874impl TableDebug for GuestTask {
5875    fn type_name() -> &'static str {
5876        "GuestTask"
5877    }
5878}
5879
5880/// Represents state common to all kinds of waitables.
5881#[derive(Default)]
5882struct WaitableCommon {
5883    /// The currently pending event for this waitable, if any.
5884    event: Option<Event>,
5885    /// The set to which this waitable belongs, if any.
5886    set: Option<TableId<WaitableSet>>,
5887    /// The handle with which the guest refers to this waitable, if any.
5888    handle: Option<u32>,
5889}
5890
5891/// Represents a Component Model Async `waitable`.
5892#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5893enum Waitable {
5894    /// A host task
5895    Host(TableId<HostTask>),
5896    /// A guest task
5897    Guest(TableId<GuestTask>),
5898    /// The read or write end of a stream or future
5899    Transmit(TableId<TransmitHandle>),
5900}
5901
5902impl Waitable {
5903    /// Retrieve the `Waitable` corresponding to the specified guest-visible
5904    /// handle.
5905    fn from_instance(
5906        state: Pin<&mut ComponentInstance>,
5907        caller_instance: RuntimeComponentInstanceIndex,
5908        waitable: u32,
5909    ) -> Result<Self> {
5910        use crate::runtime::vm::component::Waitable;
5911
5912        let (waitable, kind) = state.instance_states().0[caller_instance]
5913            .handle_table()
5914            .waitable_rep(waitable)?;
5915
5916        Ok(match kind {
5917            Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5918            Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5919            Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5920        })
5921    }
5922
5923    /// Retrieve the host-visible identifier for this `Waitable`.
5924    fn rep(&self) -> u32 {
5925        match self {
5926            Self::Host(id) => id.rep(),
5927            Self::Guest(id) => id.rep(),
5928            Self::Transmit(id) => id.rep(),
5929        }
5930    }
5931
5932    /// Move this `Waitable` to the specified set (when `set` is `Some(_)`) or
5933    /// remove it from any set it may currently belong to (when `set` is
5934    /// `None`).
5935    fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5936        log::trace!("waitable {self:?} join set {set:?}");
5937
5938        let old = mem::replace(&mut self.common(state)?.set, set);
5939
5940        if let Some(old) = old {
5941            match *self {
5942                Waitable::Host(id) => state.remove_child(id, old),
5943                Waitable::Guest(id) => state.remove_child(id, old),
5944                Waitable::Transmit(id) => state.remove_child(id, old),
5945            }?;
5946
5947            state.get_mut(old)?.ready.remove(self);
5948        }
5949
5950        if let Some(set) = set {
5951            match *self {
5952                Waitable::Host(id) => state.add_child(id, set),
5953                Waitable::Guest(id) => state.add_child(id, set),
5954                Waitable::Transmit(id) => state.add_child(id, set),
5955            }?;
5956
5957            if self.common(state)?.event.is_some() {
5958                self.mark_ready(state)?;
5959            }
5960        }
5961
5962        Ok(())
5963    }
5964
5965    /// Retrieve mutable access to the `WaitableCommon` for this `Waitable`.
5966    fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5967        Ok(match self {
5968            Self::Host(id) => &mut state.get_mut(*id)?.common,
5969            Self::Guest(id) => &mut state.get_mut(*id)?.common,
5970            Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5971        })
5972    }
5973
5974    /// Trap if this waitable is currently a member of a waitable set.
5975    ///
5976    /// A synchronous stream/future/subtask operation may end up blocking on
5977    /// this waitable, so it is not allowed to run while the waitable is also
5978    /// being watched by a waitable set.
5979    fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5980        if self.common(state)?.set.is_some() {
5981            bail!(Trap::WaitableSyncAndAsync);
5982        }
5983        Ok(())
5984    }
5985
5986    /// Trap if a thread is currently synchronously waiting on this waitable.
5987    ///
5988    /// This corresponds to the spec's `trap_if(w.has_sync_waiter)` and is
5989    /// used to prevent other threads from stealing or otherwise interfering
5990    /// with the event the sync waiter is expecting.
5991    fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5992        if let Some(set) = self.common(state)?.set
5993            && state.get_mut(set)?.is_sync_call_set
5994        {
5995            bail!(Trap::WaitableSyncAndAsync);
5996        }
5997        Ok(())
5998    }
5999
6000    /// Set or clear the pending event for this waitable and either deliver it
6001    /// to the first waiter, if any, or mark it as ready to be delivered to the
6002    /// next waiter that arrives.
6003    fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
6004        log::trace!("set event for {self:?}: {event:?}");
6005        self.common(state)?.event = event;
6006        self.mark_ready(state)
6007    }
6008
6009    /// Take the pending event from this waitable, leaving `None` in its place.
6010    fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
6011        let common = self.common(state)?;
6012        let event = common.event.take();
6013        if let Some(set) = self.common(state)?.set {
6014            state.get_mut(set)?.ready.remove(self);
6015        }
6016
6017        Ok(event)
6018    }
6019
6020    /// Deliver the current event for this waitable to the first waiter, if any,
6021    /// or else mark it as ready to be delivered to the next waiter that
6022    /// arrives.
6023    fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
6024        if let Some(set) = self.common(state)?.set {
6025            state.get_mut(set)?.ready.insert(*self);
6026            state.wake_waiter(set)?;
6027        }
6028        Ok(())
6029    }
6030
6031    /// Remove this waitable from the store's rep table (or decrement the ref
6032    /// count if it's a guest task).
6033    fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
6034        match self {
6035            Self::Host(task) => state.delete_host_task(*task),
6036            Self::Guest(task) => state.decrement_task_ref_count(*task),
6037            Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
6038        }
6039    }
6040}
6041
6042impl fmt::Debug for Waitable {
6043    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6044        match self {
6045            Self::Host(id) => write!(f, "{id:?}"),
6046            Self::Guest(id) => write!(f, "{id:?}"),
6047            Self::Transmit(id) => write!(f, "{id:?}"),
6048        }
6049    }
6050}
6051
6052/// Represents a Component Model Async `waitable-set`.
6053#[derive(Default)]
6054struct WaitableSet {
6055    /// Which waitables in this set have pending events, if any.
6056    ready: BTreeSet<Waitable>,
6057    /// Which guest threads are currently waiting on this set, if any.
6058    waiting: BTreeMap<QualifiedThreadId, WaitMode>,
6059    /// How many guest threads are waiting on this set, mirroring the spec's
6060    /// `WaitableSet.num_waiting`.
6061    num_waiting: usize,
6062    /// Whether this set is a synthetic, internal one meant for handling
6063    /// synchronous calls.
6064    is_sync_call_set: bool,
6065}
6066
6067impl WaitableSet {
6068    /// Records that a thread counted in `num_waiting` is no longer waiting.
6069    fn stop_waiting(&mut self) -> Result<()> {
6070        self.num_waiting = match self.num_waiting.checked_sub(1) {
6071            Some(n) => n,
6072            None => bail_bug!("waiter not accounted for in waitable set"),
6073        };
6074        Ok(())
6075    }
6076}
6077
6078impl TableDebug for WaitableSet {
6079    fn type_name() -> &'static str {
6080        "WaitableSet"
6081    }
6082}
6083
6084/// Type-erased closure to lower the parameters for a guest task.
6085type RawLower =
6086    Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
6087
6088/// Type-erased closure to lift the result for a guest task.
6089type RawLift = Box<
6090    dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6091>;
6092
6093/// Type erased result of a guest task which may be downcast to the expected
6094/// type by a host caller (or simply ignored in the case of a guest caller; see
6095/// `DummyResult`).
6096type LiftedResult = Box<dyn Any + Send + Sync>;
6097
6098/// Used to return a result from a `LiftFn` when the actual result has already
6099/// been lowered to a guest task's stack and linear memory.
6100struct DummyResult;
6101
6102/// Represents the Component Model Async state of a (sub-)component instance.
6103#[derive(Default)]
6104pub struct ConcurrentInstanceState {
6105    /// Whether backpressure is set for this instance (enabled if >0)
6106    backpressure: u16,
6107    /// Whether this instance can be entered
6108    do_not_enter: bool,
6109    /// Whether this instance may suspend (i.e. whether this instance is
6110    /// currently running a sync-typed function).
6111    do_not_suspend: bool,
6112    /// Pending calls for this instance which require `Self::backpressure` to be
6113    /// zero and/or `Self::do_not_enter` to be false before they can proceed.
6114    pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
6115    /// Number of pending `GuestCallKind::StartImplicit` calls which have not
6116    /// yet been started.
6117    pending_start_count: usize,
6118}
6119
6120impl ConcurrentInstanceState {
6121    pub fn pending_is_empty(&self) -> bool {
6122        self.pending.is_empty()
6123    }
6124}
6125
6126#[derive(Debug, Copy, Clone)]
6127pub(crate) enum CurrentThread {
6128    /// The currently running thread is a guest, identified here with its
6129    /// task/thread id combo.
6130    Guest(QualifiedThreadId),
6131    /// The currently running thread is a host task.
6132    Host(TableId<HostTask>),
6133    /// The currently running thread is a host call whose task has not yet been
6134    /// materialized. The contained ID identifies its guest caller.
6135    DeferredHost(QualifiedThreadId),
6136    /// There is no currently running thread because we are in the main event
6137    /// loop or concurrency is disabled.
6138    None,
6139}
6140
6141impl CurrentThread {
6142    fn guest(&self) -> Option<&QualifiedThreadId> {
6143        match self {
6144            Self::Guest(id) => Some(id),
6145            _ => None,
6146        }
6147    }
6148
6149    fn guest_task(&self) -> Option<TableId<GuestTask>> {
6150        match self {
6151            Self::Guest(id) => Some(id.task),
6152            _ => None,
6153        }
6154    }
6155
6156    fn is_none(&self) -> bool {
6157        matches!(self, Self::None)
6158    }
6159}
6160
6161impl From<QualifiedThreadId> for CurrentThread {
6162    fn from(id: QualifiedThreadId) -> Self {
6163        Self::Guest(id)
6164    }
6165}
6166
6167impl From<TableId<HostTask>> for CurrentThread {
6168    fn from(id: TableId<HostTask>) -> Self {
6169        Self::Host(id)
6170    }
6171}
6172
6173enum Priority {
6174    Switch,
6175    #[expect(
6176        dead_code,
6177        reason = "Here for completeness and in case we need it later"
6178    )]
6179    High,
6180    Low,
6181}
6182
6183/// Represents the Component Model Async state of a store.
6184pub struct ConcurrentState {
6185    /// The currently running thread, if any.
6186    ///
6187    /// Note that we lazily materialize threads on-demand and this field is not
6188    /// necessarily up-to-date. The `StoreOpaque::current_thread` method should
6189    /// be preferred over directly accessing this field.
6190    unforced_current_thread: CurrentThread,
6191
6192    /// Borrow state for the deferred host call, if any.
6193    ///
6194    /// This is `Some` if and only if [`Self::unforced_current_thread`] is
6195    /// [`CurrentThread::DeferredHost`]. Materializing the host task moves this
6196    /// context into that task.
6197    deferred_host_call_context: Option<CallContext>,
6198
6199    /// The set of pending host and background tasks, if any.
6200    ///
6201    /// See `ComponentInstance::poll_until` for where we temporarily take this
6202    /// out, poll it, then put it back to avoid any mutable aliasing hazards.
6203    futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6204    /// The table of waitables, waitable sets, etc.
6205    table: AlwaysMut<ResourceTable>,
6206    /// The next item to switch to if any.
6207    ///
6208    /// This takes precedence over items in the `high_priority` queue below and
6209    /// should be used in cases such as subtask calls and thread resume/promote
6210    /// operations where we must switch to a specific thread at the next turn of
6211    /// the event loop regardless of what happens to be present in the
6212    /// `high_priority` queue.
6213    switch_item: Option<WorkItem>,
6214    /// The item to set `switch_item` to when the current thread suspends.
6215    ///
6216    /// This is used when ever an async-lowered import or `subtask.cancel` is
6217    /// called in order to track the thread to switch back to once the current
6218    /// subtask suspends, if any.
6219    next_switch_item: Option<WorkItem>,
6220    /// The "high priority" work queue for this store's event loop.
6221    high_priority: VecDeque<WorkItem>,
6222    /// The "low priority" work queue for this store's event loop.
6223    low_priority: VecDeque<WorkItem>,
6224    /// A place to stash the reason a fiber is suspending so that the code which
6225    /// resumed it will know under what conditions the fiber should be resumed
6226    /// again.
6227    suspend_reason: Option<SuspendReason>,
6228    /// A cached fiber which is waiting for work to do.
6229    ///
6230    /// This helps us avoid creating a new fiber for each `GuestCall` work item.
6231    worker: Option<StoreFiber<'static>>,
6232    /// A place to stash the work item for which we're resuming a worker fiber.
6233    worker_item: Option<WorkerItem>,
6234
6235    /// Reference counts for all component error contexts
6236    ///
6237    /// NOTE: it is possible the global ref count to be *greater* than the sum of
6238    /// (sub)component ref counts as tracked by `error_context_tables`, for
6239    /// example when the host holds one or more references to error contexts.
6240    ///
6241    /// The key of this primary map is often referred to as the "rep" (i.e. host-side
6242    /// component-wide representation) of the index into concurrent state for a given
6243    /// stored `ErrorContext`.
6244    ///
6245    /// Stated another way, `TypeComponentGlobalErrorContextTableIndex` is essentially the same
6246    /// as a `TableId<ErrorContextState>`.
6247    global_error_context_ref_counts:
6248        BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6249
6250    /// The number of "interesting tasks" currently executing in the store.
6251    ///
6252    /// This tracks the concept of a component instance lifetime as defined in
6253    /// https://github.com/WebAssembly/component-model/pull/643. Specifically
6254    /// all tasks currently increment this counter which then gets decremented
6255    /// when they exit. In the future some tasks might not increment this
6256    /// counter, but for now all do.
6257    ///
6258    /// This is used to implement `Accessor::poll_no_interesting_tasks` to
6259    /// inform the embedder when all tasks have completed. This is then
6260    /// used in wasmtime-wasi-http, for example, to know when an instance is
6261    /// idle.
6262    interesting_tasks: usize,
6263
6264    /// Single waker to notify when `interesting_tasks` reaches 0.
6265    ///
6266    /// Used in the implementation of `Accessor::poll_no_interesting_tasks`.
6267    interesting_tasks_empty_waker: Option<Waker>,
6268
6269    /// Single waker to notify when a component instance goes from
6270    /// not-concurrently-callable to concurrently-callable.
6271    ///
6272    /// Used in the implementation of `Accessor::poll_ready_for_concurrent_call`.
6273    ready_for_concurrent_call_waker: Option<Waker>,
6274
6275    /// Whether the `StoreContextMut::poll_until` event loop is running.
6276    event_loop_running: bool,
6277
6278    /// Stack of switch items to push to and pop from when entering and exiting
6279    /// sync-to-sync calls.
6280    saved_next_switch_items: Vec<Option<WorkItem>>,
6281
6282    /// See [TaskGroupHook].
6283    #[cfg(feature = "task-group-hook")]
6284    task_group_hook: Option<Box<dyn TaskGroupHook>>,
6285}
6286
6287impl Default for ConcurrentState {
6288    fn default() -> Self {
6289        Self {
6290            unforced_current_thread: CurrentThread::None,
6291            deferred_host_call_context: None,
6292            table: AlwaysMut::new(ResourceTable::new()),
6293            futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6294            switch_item: None,
6295            next_switch_item: None,
6296            high_priority: VecDeque::new(),
6297            low_priority: VecDeque::new(),
6298            suspend_reason: None,
6299            worker: None,
6300            worker_item: None,
6301            global_error_context_ref_counts: BTreeMap::new(),
6302            interesting_tasks: 0,
6303            interesting_tasks_empty_waker: None,
6304            ready_for_concurrent_call_waker: None,
6305            event_loop_running: false,
6306            saved_next_switch_items: Vec::new(),
6307            #[cfg(feature = "task-group-hook")]
6308            task_group_hook: None,
6309        }
6310    }
6311}
6312
6313impl ConcurrentState {
6314    /// Take ownership of any fibers and futures owned by this object.
6315    ///
6316    /// This should be used when disposing of the `Store` containing this object
6317    /// in order to gracefully resolve any and all fibers using
6318    /// `StoreFiber::dispose`.  This is necessary to avoid possible
6319    /// use-after-free bugs due to fibers which may still have access to the
6320    /// `Store`.
6321    ///
6322    /// Additionally, the futures collected with this function should be dropped
6323    /// within a `tls::set` call, which will ensure than any futures closing
6324    /// over an `&Accessor` will have access to the store when dropped, allowing
6325    /// e.g. `WithAccessor[AndValue]` instances to be disposed of without
6326    /// panicking.
6327    ///
6328    /// Note that this will leave the object in an inconsistent and unusable
6329    /// state, so it should only be used just prior to dropping it.
6330    pub(crate) fn take_fibers_and_futures(
6331        &mut self,
6332        fibers: &mut Vec<StoreFiber<'static>>,
6333        futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6334    ) {
6335        let ConcurrentState {
6336            table,
6337            worker,
6338            switch_item,
6339            next_switch_item,
6340            high_priority,
6341            low_priority,
6342            saved_next_switch_items,
6343            futures: pending_futures,
6344            worker_item: _,
6345            unforced_current_thread: _,
6346            deferred_host_call_context: _,
6347            suspend_reason: _,
6348            global_error_context_ref_counts: _,
6349            interesting_tasks: _,
6350            interesting_tasks_empty_waker: _,
6351            ready_for_concurrent_call_waker: _,
6352            event_loop_running: _,
6353            #[cfg(feature = "task-group-hook")]
6354                task_group_hook: _,
6355        } = self;
6356
6357        let mut items = Vec::new();
6358        for (_, entry) in table.get_mut().iter_mut() {
6359            let entry = entry.into_entry_mut();
6360            if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6361                for mode in mem::take(&mut set.waiting).into_values() {
6362                    match mode {
6363                        WaitMode::Fiber(fiber) => {
6364                            fibers.push(fiber);
6365                        }
6366                        WaitMode::Callback(_) => {}
6367                    }
6368                }
6369            } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6370                match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6371                    GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6372                        fibers.push(fiber);
6373                    }
6374                    GuestThreadState::NotStartedImplicit
6375                    | GuestThreadState::NotStartedExplicit(_)
6376                    | GuestThreadState::Running
6377                    | GuestThreadState::Completed => {}
6378                }
6379            } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6380                if let Some(item) = item.take() {
6381                    items.push(item);
6382                }
6383            }
6384        }
6385
6386        if let Some(fiber) = worker.take() {
6387            fibers.push(fiber);
6388        }
6389
6390        let mut handle_item = |item| match item {
6391            WorkItem::ResumeFiber { fiber, .. } => {
6392                fibers.push(fiber);
6393            }
6394            WorkItem::PushFuture(future) => {
6395                pending_futures
6396                    .get_mut()
6397                    .as_mut()
6398                    .unwrap()
6399                    .push(future.into_inner());
6400            }
6401            WorkItem::ResumeThread { .. }
6402            | WorkItem::GuestCall { .. }
6403            | WorkItem::WorkerFunction(_) => {}
6404        };
6405
6406        for item in items {
6407            handle_item(item);
6408        }
6409        if let Some(item) = switch_item.take() {
6410            handle_item(item);
6411        }
6412        if let Some(item) = next_switch_item.take() {
6413            handle_item(item);
6414        }
6415        for item in mem::take(high_priority) {
6416            handle_item(item);
6417        }
6418        for item in mem::take(low_priority) {
6419            handle_item(item);
6420        }
6421        for item in mem::take(saved_next_switch_items)
6422            .into_iter()
6423            .filter_map(|v| v)
6424        {
6425            handle_item(item);
6426        }
6427
6428        if let Some(them) = pending_futures.get_mut().take() {
6429            futures.push(them);
6430        }
6431    }
6432
6433    #[cfg(feature = "gc")]
6434    pub(crate) fn trace_fiber_roots(
6435        &mut self,
6436        modules: &ModuleRegistry,
6437        unwind: &dyn Unwind,
6438        gc_roots_list: &mut GcRootsList,
6439    ) {
6440        let ConcurrentState {
6441            table,
6442            worker,
6443            switch_item,
6444            next_switch_item,
6445            high_priority,
6446            low_priority,
6447            saved_next_switch_items,
6448
6449            // TODO(cm-gc): This field contains `ValRaw`s, but they are never GC
6450            // references because the component model doesn't support GC yet. We
6451            // will need to trace these somehow when it does.
6452            futures: _,
6453
6454            // These fields do not contain GC references.
6455            worker_item: _,
6456            unforced_current_thread: _,
6457            deferred_host_call_context: _,
6458            suspend_reason: _,
6459            global_error_context_ref_counts: _,
6460            interesting_tasks: _,
6461            interesting_tasks_empty_waker: _,
6462            ready_for_concurrent_call_waker: _,
6463            event_loop_running: _,
6464            #[cfg(feature = "task-group-hook")]
6465                task_group_hook: _,
6466        } = self;
6467
6468        for (_, entry) in table.get_mut().iter_mut() {
6469            let entry = entry.into_entry_mut();
6470            if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6471                for mode in set.waiting.values_mut() {
6472                    match mode {
6473                        WaitMode::Fiber(fiber) => {
6474                            fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6475                        }
6476                        WaitMode::Callback(_) => {}
6477                    }
6478                }
6479            } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6480                match &mut thread.state {
6481                    GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6482                        fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6483                    }
6484                    GuestThreadState::NotStartedImplicit
6485                    | GuestThreadState::NotStartedExplicit(_)
6486                    | GuestThreadState::Running
6487                    | GuestThreadState::Completed => {}
6488                }
6489            } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6490                entry.downcast_mut::<Option<WorkItem>>()
6491            {
6492                fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6493            }
6494        }
6495
6496        if let Some(fiber) = worker {
6497            fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6498        }
6499
6500        let mut handle_item = |item: &mut WorkItem| match item {
6501            WorkItem::ResumeFiber { fiber, .. } => {
6502                fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6503            }
6504            WorkItem::PushFuture(_future) => {
6505                // TODO(cm-gc): once futures can contain GC roots, we will need
6506                // to trace them.
6507            }
6508            WorkItem::ResumeThread { .. }
6509            | WorkItem::GuestCall { .. }
6510            | WorkItem::WorkerFunction(_) => {}
6511        };
6512
6513        if let Some(item) = switch_item {
6514            handle_item(item);
6515        }
6516        if let Some(item) = next_switch_item {
6517            handle_item(item);
6518        }
6519        for item in high_priority {
6520            handle_item(item);
6521        }
6522        for item in low_priority {
6523            handle_item(item);
6524        }
6525        for item in saved_next_switch_items
6526            .iter_mut()
6527            .filter_map(|v| v.as_mut())
6528        {
6529            handle_item(item);
6530        }
6531    }
6532
6533    fn schedule_suspended_thread(
6534        &mut self,
6535        thread: QualifiedThreadId,
6536        instance: RuntimeInstance,
6537        priority: Priority,
6538    ) -> Result<()> {
6539        // Validate the destination while the source still owns the fiber.
6540        if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6541            bail_bug!("switch item already set");
6542        }
6543        let slot = &mut self.get_mut(thread.thread)?.state;
6544        if !matches!(slot, GuestThreadState::Suspended(_)) {
6545            bail!(Trap::CannotResumeThread);
6546        }
6547        let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6548        else {
6549            unreachable!()
6550        };
6551        let item = WorkItem::ResumeFiber {
6552            instance,
6553            thread,
6554            fiber,
6555        };
6556        match priority {
6557            Priority::Switch => self.switch_item = Some(item),
6558            Priority::High => self.push_high_priority(item),
6559            Priority::Low => self.push_low_priority(item),
6560        }
6561        Ok(())
6562    }
6563
6564    fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6565        let slot = &mut self.get_mut(thread)?.state;
6566        slot.check_no_fiber()?;
6567        *slot = GuestThreadState::Running;
6568        Ok(())
6569    }
6570
6571    fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6572        // push can consume and drop its argument when the table is full.
6573        // Reserve with an empty item before taking the fiber-owning source.
6574        let saved = self.push(None::<WorkItem>)?;
6575        let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6576        *slot = self.next_switch_item.take();
6577        Ok(saved)
6578    }
6579
6580    fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6581        if self.next_switch_item.is_some() {
6582            bail_bug!("next switch item already set when restoring");
6583        }
6584        self.next_switch_item = self.delete(saved)?;
6585        Ok(())
6586    }
6587
6588    fn take_callback_waiter(
6589        &mut self,
6590        set: TableId<WaitableSet>,
6591        thread: QualifiedThreadId,
6592    ) -> Result<Instance> {
6593        let waiting = &mut self.get_mut(set)?.waiting;
6594        let instance = match waiting.get(&thread) {
6595            Some(WaitMode::Callback(instance)) => *instance,
6596            other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6597        };
6598        waiting.remove(&thread);
6599        Ok(instance)
6600    }
6601
6602    fn push<V: Send + Sync + 'static>(
6603        &mut self,
6604        value: V,
6605    ) -> Result<TableId<V>, ResourceTableError> {
6606        self.table.get_mut().push(value).map(TableId::from)
6607    }
6608
6609    fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6610        self.table.get_mut().get_mut(&Resource::from(id))
6611    }
6612
6613    pub fn add_child<T: 'static, U: 'static>(
6614        &mut self,
6615        child: TableId<T>,
6616        parent: TableId<U>,
6617    ) -> Result<(), ResourceTableError> {
6618        self.table
6619            .get_mut()
6620            .add_child(Resource::from(child), Resource::from(parent))
6621    }
6622
6623    pub fn remove_child<T: 'static, U: 'static>(
6624        &mut self,
6625        child: TableId<T>,
6626        parent: TableId<U>,
6627    ) -> Result<(), ResourceTableError> {
6628        self.table
6629            .get_mut()
6630            .remove_child(Resource::from(child), Resource::from(parent))
6631    }
6632
6633    fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6634        self.table.get_mut().delete(Resource::from(id))
6635    }
6636
6637    fn push_future(&mut self, future: HostTaskFuture) {
6638        // Note that we can't directly push to `ConcurrentState::futures` here
6639        // since this may be called from a future that's being polled inside
6640        // `Self::poll_until`, which temporarily removes the `FuturesUnordered`
6641        // so it has exclusive access while polling it.  Therefore, we push a
6642        // work item to the "high priority" queue, which will actually push to
6643        // `ConcurrentState::futures` later.
6644        self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6645    }
6646
6647    fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6648        if self.switch_item.is_some() {
6649            // Preserve ownership on this internal-error path. The caller
6650            // must stop scheduling after the error; teardown disposes the item.
6651            self.push_high_priority(item);
6652            bail_bug!("switch item already set");
6653        }
6654
6655        self.switch_item = Some(item);
6656        log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6657
6658        Ok(())
6659    }
6660
6661    fn take_next_switch_item(&mut self) -> Result<()> {
6662        if let Some(item) = self.next_switch_item.take() {
6663            self.set_switch_item(item)?;
6664        }
6665        Ok(())
6666    }
6667
6668    /// Wake the first thread waiting on `set`, if any, so that it may receive
6669    /// one of the set's pending events.
6670    fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6671        // Finish every fallible lookup and check before taking the waiter out
6672        // of the set, so that an error leaves its fiber owned by the set.
6673        let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6674            return Ok(());
6675        };
6676        let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6677        if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6678            bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6679        }
6680        let instance = self.get_mut(thread.task)?.instance;
6681        self.get_mut(thread.thread)?.wake_on_cancel.take();
6682        let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6683
6684        let item = match mode {
6685            WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6686                instance,
6687                thread,
6688                fiber,
6689            },
6690            WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6691                instance,
6692                call: GuestCall {
6693                    thread,
6694                    kind: GuestCallKind::DeliverEvent {
6695                        instance: callback_instance,
6696                        set: Some(set),
6697                    },
6698                },
6699            },
6700        };
6701        self.push_high_priority(item);
6702        Ok(())
6703    }
6704
6705    fn push_high_priority(&mut self, item: WorkItem) {
6706        self.high_priority.push_front(item);
6707        log::trace!(
6708            "push high priority: {:?}",
6709            self.high_priority.front().unwrap()
6710        );
6711    }
6712
6713    fn push_low_priority(&mut self, item: WorkItem) {
6714        self.low_priority.push_front(item);
6715        log::trace!(
6716            "push low priority: {:?}",
6717            self.low_priority.front().unwrap()
6718        );
6719    }
6720
6721    fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6722        match priority {
6723            Priority::Switch => self.set_switch_item(item)?,
6724            Priority::High => self.push_high_priority(item),
6725            Priority::Low => self.push_low_priority(item),
6726        }
6727
6728        Ok(())
6729    }
6730
6731    fn promote_instance_local_thread_work_item(
6732        &mut self,
6733        current_instance: RuntimeInstance,
6734    ) -> Result<bool> {
6735        log::trace!("promote thread work items for {current_instance:?}");
6736
6737        self.promote_work_item_matching(|item: &WorkItem| {
6738            let result = match item {
6739                WorkItem::ResumeThread { instance, .. }
6740                | WorkItem::ResumeFiber { instance, .. }
6741                | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6742                _ => false,
6743            };
6744
6745            log::trace!("candidate {item:?}: {result}");
6746            result
6747        })
6748    }
6749
6750    fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6751        self.promote_work_item_matching(|item: &WorkItem| match item {
6752            WorkItem::ResumeThread {
6753                thread: item_thread,
6754                ..
6755            }
6756            | WorkItem::GuestCall {
6757                call:
6758                    GuestCall {
6759                        thread: item_thread,
6760                        ..
6761                    },
6762                ..
6763            } => *item_thread == thread,
6764            _ => false,
6765        })
6766    }
6767
6768    fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6769    where
6770        F: FnMut(&WorkItem) -> bool,
6771    {
6772        // Scan from the back to preserve the ordering of items
6773        // are popped from the back of the `VecDeque`s by `poll_until` and
6774        // pushed to the front by `push_{high,low}_priority`.
6775
6776        // Inspect items while they are still owned by the store. A predicate
6777        // (including tracing) may panic; it must not drop an extracted queue.
6778        if self.switch_item.is_none() {
6779            if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6780                self.switch_item = self.high_priority.remove(index);
6781            }
6782        }
6783        if self.switch_item.is_none() {
6784            if let Some(index) = self.low_priority.iter().rposition(predicate) {
6785                self.switch_item = self.low_priority.remove(index);
6786            }
6787        }
6788        Ok(self.switch_item.is_some())
6789    }
6790
6791    /// Used by `ResourceTables` to acquire the current `CallContext` for the
6792    /// specified task.
6793    pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6794        match task.kind() {
6795            ScopeKind::HostId(task) => {
6796                let task: TableId<HostTask> = TableId::new(task);
6797                Ok(&mut self.get_mut(task)?.call_context)
6798            }
6799            ScopeKind::Id(task) => {
6800                let task: TableId<GuestTask> = TableId::new(task);
6801                Ok(&mut self.get_mut(task)?.call_context)
6802            }
6803        }
6804    }
6805
6806    pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6807        self.deferred_host_call_context.as_mut()
6808    }
6809
6810    fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6811        match self.futures.get_mut().as_mut() {
6812            Some(f) => Ok(f),
6813            None => bail_bug!("futures field of concurrent state is currently taken"),
6814        }
6815    }
6816
6817    pub(crate) fn table(&mut self) -> &mut ResourceTable {
6818        self.table.get_mut()
6819    }
6820
6821    fn debug_assert_deferred_host_invariant(&self) {
6822        debug_assert_eq!(
6823            self.deferred_host_call_context.is_some(),
6824            matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6825            "a deferred host thread and call context must exist together",
6826        );
6827    }
6828
6829    fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6830        self.debug_assert_deferred_host_invariant();
6831        let caller = match self.unforced_current_thread {
6832            CurrentThread::DeferredHost(caller) => caller,
6833            thread => return Ok(thread),
6834        };
6835
6836        // Push first so allocation failure leaves the deferred state intact.
6837        let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6838        let task = self.push(task)?;
6839        let call_context = self
6840            .deferred_host_call_context
6841            .take()
6842            .expect("deferred host call context should be present");
6843        self.get_mut(task)
6844            .expect("newly inserted host task should be present")
6845            .call_context = call_context;
6846        self.unforced_current_thread = CurrentThread::Host(task);
6847        self.debug_assert_deferred_host_invariant();
6848        log::trace!("new host task materialized {task:?}");
6849        Ok(CurrentThread::Host(task))
6850    }
6851
6852    fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6853        match self.materialize_host_task()? {
6854            CurrentThread::Host(id) => Ok(Some(id)),
6855            CurrentThread::None => Ok(None),
6856            CurrentThread::Guest(_) => {
6857                bail_bug!("tried to materialize a host task id from a guest thread")
6858            }
6859            CurrentThread::DeferredHost(_) => {
6860                bail_bug!(
6861                    "current thread is a deferred host thread which should have been materialized"
6862                )
6863            }
6864        }
6865    }
6866
6867    pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6868        match self.materialize_host_task()? {
6869            CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6870            _ => bail_bug!("current scope is not a deferred host scope"),
6871        }
6872    }
6873
6874    fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6875        let count = &mut self.get_mut(task)?.ref_count;
6876        *count += 1;
6877        log::trace!("increment {task:?} to {count}");
6878        Ok(())
6879    }
6880
6881    fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6882        let count = &mut self.get_mut(task)?.ref_count;
6883        if *count == 0 {
6884            bail_bug!("can't decrement task ref count that's already zero");
6885        }
6886        *count -= 1;
6887        log::trace!("decrement {task:?} to {count}");
6888        if *count == 0 {
6889            let task = self.delete(task)?;
6890
6891            self.decrement_group_ref_count(task.group)?;
6892
6893            // When a guest task is created it increments the
6894            // `ConcurrentState::interesting_tasks` counter, and that needs
6895            // to be paired with a decrement. There are a few situations in
6896            // which the decrement needs to happen which don't all funnel
6897            // through here, so in lieu of that at least try to catch issues
6898            // where we forgot to do a decrement.
6899            debug_assert!(task.decremented_interesting_task_count);
6900        }
6901        Ok(())
6902    }
6903
6904    fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6905        log::trace!("delete host task {task:?}");
6906        let task = self.delete(task)?;
6907
6908        self.decrement_group_ref_count(task.group)
6909    }
6910}
6911
6912/// Provide a type hint to compiler about the shape of a parameter lower
6913/// closure.
6914fn for_any_lower<
6915    F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6916>(
6917    fun: F,
6918) -> F {
6919    fun
6920}
6921
6922/// Provide a type hint to compiler about the shape of a result lift closure.
6923fn for_any_lift<
6924    F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6925>(
6926    fun: F,
6927) -> F {
6928    fun
6929}
6930
6931fn check_ambient_store(id: StoreId) {
6932    let message = "\
6933        `Future`s which depend on asynchronous component tasks, streams, or \
6934        futures to complete may only be polled from the event loop of the \
6935        store to which they belong.  Please use \
6936        `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6937    ";
6938    tls::try_get(|store| {
6939        let matched = match store {
6940            tls::TryGet::Some(store) => store.id() == id,
6941            tls::TryGet::Taken | tls::TryGet::None => false,
6942        };
6943
6944        if !matched {
6945            panic!("{message}")
6946        }
6947    });
6948}
6949
6950fn unpack_callback_code(code: u32) -> (u32, u32) {
6951    (code & 0xF, code >> 4)
6952}
6953
6954/// Helper struct for packaging parameters to be passed to
6955/// `ComponentInstance::waitable_check` for calls to `waitable-set.wait` or
6956/// `waitable-set.poll`.
6957struct WaitableCheckParams {
6958    set: TableId<WaitableSet>,
6959    options: OptionsIndex,
6960    payload: u32,
6961}
6962
6963/// Indicates whether `ComponentInstance::waitable_check` is being called for
6964/// `waitable-set.wait` or `waitable-set.poll`.
6965enum WaitableCheck {
6966    Wait,
6967    Poll,
6968}
6969
6970/// Represents a guest task called from the host, prepared using `prepare_call`.
6971pub(crate) struct PreparedCall<R> {
6972    /// The guest export to be called
6973    handle: Func,
6974    /// The guest thread created by `prepare_call`
6975    thread: QualifiedThreadId,
6976    /// The number of lowered core Wasm parameters to pass to the call.
6977    param_count: usize,
6978    /// The `oneshot::Receiver` to which the result of the call will be
6979    /// delivered when it is available.
6980    rx: oneshot::Receiver<LiftedResult>,
6981    _phantom: PhantomData<R>,
6982}
6983
6984impl<R> PreparedCall<R> {
6985    /// Acquire a reference to the `TaskId` for this `PreparedCall`.
6986    ///
6987    /// Note that this increments the reference count for the task, and
6988    /// `TaskId::host_future_dropped` decrements it.  Hence,
6989    /// `TaskId::host_future_dropped` _must_ be called before the `TaskId` is
6990    /// dropped to avoid leaking the task.
6991    pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6992        store
6993            .concurrent_state_mut()?
6994            .increment_task_ref_count(self.thread.task)?;
6995
6996        Ok(TaskId {
6997            task: self.thread.task,
6998        })
6999    }
7000}
7001
7002/// Represents a task created by `prepare_call`.
7003pub(crate) struct TaskId {
7004    task: TableId<GuestTask>,
7005}
7006
7007impl TaskId {
7008    /// The host future for an async task was dropped.
7009    ///
7010    /// If the parameters have not been lowered yet, it is no longer valid to
7011    /// do so, as the lowering closure would see a dangling pointer. In this
7012    /// case the closure is replaced with one that fails, so the call fails if
7013    /// it's ever started. This can't fail, since the task can't be deleted
7014    /// while its host future is live.
7015    ///
7016    /// Otherwise, there may be running threads, or ones that are suspended and
7017    /// can be resumed by other tasks for this component, so we mark the future
7018    /// as dropped and delete the task when all threads are done.
7019    pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
7020        let task = store.concurrent_state_mut()?.get_mut(self.task)?;
7021        if !task.already_lowered_parameters() {
7022            // TODO: avoid leaking the task and trapping in this case, e.g. by
7023            // allowing the replacement callback to return a value that tells
7024            // the event loop "nevermind, the host gave up on this call; just
7025            // skip it, clean up the task, and carry on with other work".
7026            task.lower_params = Some(Box::new(|store, _| {
7027                store.set_trapped();
7028                bail!("the host future for this call was dropped before it started")
7029            }));
7030        } else {
7031            // TODO: we're temporarily ignoring any error here to avoid a
7032            // panic-on-unwrap in `call_async_concurrent`.  Instead, we should
7033            // determine a safe way to defer the decrement (and any error it
7034            // might produce) outside of a `drop` call.
7035            _ = store
7036                .concurrent_state_mut()?
7037                .decrement_task_ref_count(self.task);
7038        }
7039
7040        Ok(())
7041    }
7042}
7043
7044/// Prepare a call to the specified exported Wasm function, providing functions
7045/// for lowering the parameters and lifting the result.
7046///
7047/// To enqueue the returned `PreparedCall` in the `ComponentInstance`'s event
7048/// loop, use `stage_call`.
7049pub(crate) fn prepare_call<T, R>(
7050    mut store: StoreContextMut<T>,
7051    handle: Func,
7052    param_count: usize,
7053    lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
7054    + Send
7055    + Sync
7056    + 'static,
7057    lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
7058    + Send
7059    + Sync
7060    + 'static,
7061) -> Result<PreparedCall<R>> {
7062    // This check should be redundant, but leave it here so that there is a
7063    // clear error when trying to execute wasm in a poisoned store instead of
7064    // possibly hitting some other `bail_bug!` because of inconsistent state.
7065    if !store.0.may_enter() {
7066        bail!(Trap::CannotEnterComponent);
7067    }
7068
7069    let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
7070
7071    let instance = handle.instance().id().get(store.0);
7072    let options = &instance.component().env_component().options[options];
7073    let ty = &instance.component().types()[ty];
7074    let async_typed = ty.async_;
7075    let async_lifted = raw_options.async_;
7076    let task_return_type = ty.results;
7077    let component_instance = raw_options.instance;
7078    let callback = options.callback.map(|i| instance.runtime_callback(i));
7079    let memory = options
7080        .memory()
7081        .map(|i| instance.runtime_memory(i))
7082        .map(SendSyncPtr::new);
7083    let string_encoding = options.string_encoding;
7084    let token = StoreToken::new(store.as_context_mut());
7085    let caller = store.0.materialize_host_task_id()?;
7086    let state = store.0.concurrent_state_mut()?;
7087
7088    let (tx, rx) = oneshot::channel();
7089
7090    let instance = handle.instance().runtime_instance(component_instance);
7091    let thread = GuestTask::new(
7092        state,
7093        Box::new(for_any_lower(move |store, params| {
7094            lower_params(token.as_context_mut(store), params)
7095        })),
7096        LiftResult {
7097            lift: Box::new(for_any_lift(move |store, result| {
7098                lift_result(store, result)
7099            })),
7100            ty: task_return_type,
7101            memory,
7102            string_encoding,
7103        },
7104        Caller::Host {
7105            tx: Some(tx),
7106            caller,
7107        },
7108        callback.map(|callback| {
7109            let callback = SendSyncPtr::new(callback);
7110            let instance = handle.instance();
7111            Box::new(move |store: &mut dyn VMStore, event, handle| {
7112                let store = token.as_context_mut(store);
7113                // SAFETY: Per the contract of `prepare_call`, the callback
7114                // will remain valid at least as long is this task exists.
7115                unsafe { instance.call_callback(store, callback, event, handle) }
7116            }) as CallbackFn
7117        }),
7118        instance,
7119        async_typed,
7120        async_lifted,
7121    )?;
7122
7123    Ok(PreparedCall {
7124        handle,
7125        thread,
7126        param_count,
7127        rx,
7128        _phantom: PhantomData,
7129    })
7130}
7131
7132pub(crate) struct StagedCall<R> {
7133    store: StoreId,
7134    rx: oneshot::Receiver<LiftedResult>,
7135    _marker: PhantomData<fn() -> R>,
7136    /// The task group to which the underlying task belongs.
7137    ///
7138    /// Note that this must be treated as opaque and not used to look up the
7139    /// corresponding `TaskGroup` since we decrement the task refcount in
7140    /// `StagedCall::new` which may cause the group to be disposed.  This field
7141    /// is only present to enable `[Typed]FuncCallConcurrent::group`, allowing
7142    /// the embedder to correlate the task group with `TaskGroupHook` events.
7143    group: TaskGroupId,
7144}
7145
7146impl<R> StagedCall<R> {
7147    /// Queue a call previously prepared using `prepare_call` to be run as part of
7148    /// the associated `ComponentInstance`'s event loop.
7149    ///
7150    /// Note that this has the side-effect of disposing the specified
7151    /// `PrepareCall` and decrementing the task reference count since
7152    /// `StagedCall` has no further need to access the task directly.
7153    ///
7154    /// The returned future will resolve to the result once it is available, but
7155    /// must only be polled via the instance's event loop. See
7156    /// `StoreContextMut::run_concurrent` for details.
7157    pub(crate) fn new<T: 'static>(
7158        mut store: StoreContextMut<T>,
7159        prepared: PreparedCall<R>,
7160    ) -> Result<StagedCall<R>> {
7161        let PreparedCall {
7162            handle,
7163            thread,
7164            param_count,
7165            rx,
7166            ..
7167        } = prepared;
7168
7169        stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7170
7171        let state = store.0.concurrent_state_mut()?;
7172        let group = state.get_mut(thread.task)?.group;
7173        state.decrement_task_ref_count(thread.task)?;
7174
7175        Ok(StagedCall {
7176            store: store.0.id(),
7177            rx,
7178            _marker: PhantomData,
7179            group,
7180        })
7181    }
7182}
7183
7184impl<R> Future for StagedCall<R>
7185where
7186    R: 'static,
7187{
7188    type Output = Result<R>;
7189
7190    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7191        check_ambient_store(self.store);
7192        Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7193            Ok(r) => match r.downcast() {
7194                Ok(r) => Ok(*r),
7195                Err(_) => bail_bug!("wrong type of value produced"),
7196            },
7197            Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7198        })
7199    }
7200}
7201
7202/// Queue a call previously prepared using `prepare_call` to be run as part of
7203/// the associated `ComponentInstance`'s event loop.
7204fn stage_call0<T: 'static>(
7205    store: StoreContextMut<T>,
7206    handle: Func,
7207    guest_thread: QualifiedThreadId,
7208    param_count: usize,
7209) -> Result<()> {
7210    let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7211    let is_concurrent = raw_options.async_;
7212    let callback = raw_options.callback;
7213    let instance = handle.instance();
7214    let callee = handle.lifted_core_func(store.0);
7215    let post_return = raw_options
7216        .post_return
7217        .map(|i| instance.id().get(store.0).runtime_post_return(i));
7218    let callback = callback.map(|i| {
7219        let instance = instance.id().get(store.0);
7220        SendSyncPtr::new(instance.runtime_callback(i))
7221    });
7222
7223    log::trace!("queueing call {guest_thread:?}");
7224
7225    // SAFETY: `callee`, `callback`, and `post_return` are valid pointers
7226    // (with signatures appropriate for this call) and will remain valid as
7227    // long as this instance is valid.
7228    unsafe {
7229        instance.stage_call(
7230            store,
7231            guest_thread,
7232            SendSyncPtr::new(callee),
7233            param_count,
7234            1,
7235            is_concurrent,
7236            callback,
7237            post_return.map(SendSyncPtr::new),
7238            true,
7239        )
7240    }
7241}
7242
7243#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7244mod tests {
7245    use super::*;
7246    use crate::component::{Component, Linker};
7247    use crate::store::AsStoreOpaque;
7248    use crate::{Config, Engine};
7249
7250    fn host_subtask(
7251        state: HostTaskState,
7252        event: Option<Event>,
7253    ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7254        let mut config = Config::new();
7255        config.wasm_component_model_async(true);
7256        let engine = Engine::new(&config)?;
7257        let component = Component::new(&engine, "(component)")?;
7258        let mut store = Store::new(&engine, ());
7259        let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7260        let store_opaque = store.as_store_opaque();
7261        let concurrent_state = store_opaque.concurrent_state_mut()?;
7262        // These tests create an isolated host task without a guest caller, so
7263        // give it its own task group instead of inheriting the caller's group.
7264        let group = concurrent_state.make_task_group()?;
7265        let task = concurrent_state.push(HostTask {
7266            common: WaitableCommon::default(),
7267            call_context: CallContext::default(),
7268            state,
7269            group,
7270        })?;
7271        let handle = store_opaque
7272            .handle_table_insertion(
7273                instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7274            )
7275            .subtask_host(task.rep())?;
7276        let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7277        common.handle = Some(handle);
7278        common.event = event;
7279        Ok((store, instance, task, handle))
7280    }
7281
7282    #[test]
7283    fn host_subtask_drop_during_cancellation() -> Result<()> {
7284        for abort_completed in [false, true] {
7285            let (handle, future) = JoinHandle::run(future::pending::<()>());
7286            let mut future = pin!(future);
7287            let (mut store, instance, task, handle) =
7288                host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7289            let store = store.as_store_opaque();
7290            let caller = RuntimeComponentInstanceIndex::from_u32(0);
7291            assert_eq!(
7292                instance.subtask_cancel(store, caller, true, handle)?,
7293                BLOCKED
7294            );
7295            if abort_completed {
7296                // Even after the abort resolves, the completion worker still
7297                // needs the task entry to lower the result and publish an event.
7298                assert!(matches!(
7299                    future
7300                        .as_mut()
7301                        .poll(&mut Context::from_waker(Waker::noop())),
7302                    Poll::Ready(None),
7303                ));
7304            }
7305            for async_ in [false, true] {
7306                let err = instance
7307                    .subtask_cancel(store, caller, async_, handle)
7308                    .unwrap_err();
7309                assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7310            }
7311            let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7312            assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7313            assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7314        }
7315        Ok(())
7316    }
7317
7318    #[test]
7319    fn host_subtask_cancel_after_completion() -> Result<()> {
7320        for async_ in [false, true] {
7321            let (mut store, instance, task, handle) = host_subtask(
7322                HostTaskState::CalleeDone { cancelled: false },
7323                Some(Event::Subtask {
7324                    status: Status::Returned,
7325                }),
7326            )?;
7327            let store = store.as_store_opaque();
7328            let caller = RuntimeComponentInstanceIndex::from_u32(0);
7329            assert_eq!(
7330                instance.subtask_cancel(store, caller, async_, handle)?,
7331                Status::Returned as u32,
7332            );
7333            let err = instance
7334                .subtask_cancel(store, caller, async_, handle)
7335                .unwrap_err();
7336            assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7337            instance.subtask_drop(store, caller, handle)?;
7338            assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7339        }
7340        Ok(())
7341    }
7342
7343    #[test]
7344    fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7345        for (cancelled, status) in [
7346            (false, Status::Returned),
7347            (true, Status::Returned),
7348            (true, Status::ReturnCancelled),
7349        ] {
7350            for delivered in [false, true] {
7351                let event = if delivered {
7352                    None
7353                } else {
7354                    Some(Event::Subtask { status })
7355                };
7356                let (mut store, instance, task, handle) =
7357                    host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7358                let store = store.as_store_opaque();
7359                let result = instance.subtask_drop(
7360                    store,
7361                    RuntimeComponentInstanceIndex::from_u32(0),
7362                    handle,
7363                );
7364                if delivered {
7365                    result?;
7366                    assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7367                } else {
7368                    let err = result.unwrap_err();
7369                    assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7370                    assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7371                }
7372            }
7373        }
7374        Ok(())
7375    }
7376}