Skip to main content

wasmtime/runtime/component/
concurrent.rs

1//! Runtime support for the Component Model Async ABI.
2//!
3//! This module and its submodules provide host runtime support for Component
4//! Model Async features such as async-lifted exports, async-lowered imports,
5//! streams, futures, and related intrinsics.  See [the Async
6//! Explainer](https://github.com/WebAssembly/component-model/blob/main/design/mvp/Concurrency.md)
7//! for a high-level overview.
8//!
9//! At the core of this support is an event loop which schedules and switches
10//! between guest tasks and any host tasks they create.  Each
11//! `Store` will have at most one event loop running at any given
12//! time, and that loop may be suspended and resumed by the host embedder using
13//! e.g. `StoreContextMut::run_concurrent`.  The `StoreContextMut::poll_until`
14//! function contains the loop itself, while the
15//! `StoreOpaque::concurrent_state` field holds its state.
16//!
17//! # Public API Overview
18//!
19//! ## Top-level API (e.g. kicking off host->guest calls and driving the event loop)
20//!
21//! - `[Typed]Func::call_concurrent`: Start a host->guest call to an
22//! async-lifted or sync-lifted import, creating a guest task.
23//!
24//! - `StoreContextMut::run_concurrent`: Run the event loop for the specified
25//! instance, allowing any and all tasks belonging to that instance to make
26//! progress.
27//!
28//! - `StoreContextMut::spawn`: Run a background task as part of the event loop
29//! for the specified instance.
30//!
31//! - `{Future,Stream}Reader::new`: Create a new Component Model `future` or
32//! `stream` which may be passed to the guest.  This takes a
33//! `{Future,Stream}Producer` implementation which will be polled for items when
34//! the consumer requests them.
35//!
36//! - `{Future,Stream}Reader::pipe`: Consume a `future` or `stream` by
37//! connecting it to a `{Future,Stream}Consumer` which will consume any items
38//! produced by the write end.
39//!
40//! ## Host Task API (e.g. implementing concurrent host functions and background tasks)
41//!
42//! - `LinkerInstance::func_wrap_concurrent`: Register a concurrent host
43//! function with the linker.  That function will take an `Accessor` as its
44//! first parameter, which provides access to the store between (but not across)
45//! await points.
46//!
47//! - `Accessor::with`: Access the store and its associated data.
48//!
49//! - `Accessor::spawn`: Run a background task as part of the event loop for the
50//! store.  This is equivalent to `StoreContextMut::spawn` but more convenient to use
51//! in host functions.
52
53use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57    HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70    CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71    InstanceState,
72};
73use crate::vm::{
74    AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98    CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99    MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100    RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101    TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102    TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113    Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114    FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115    StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136/// Constant defined in the Component Model spec to indicate that the async
137/// intrinsic (e.g. `future.write`) has not yet completed.
138const BLOCKED: u32 = 0xffff_ffff;
139
140/// Corresponds to `CallState` in the upstream spec.
141#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143    Starting = 0,
144    Started = 1,
145    Returned = 2,
146    StartCancelled = 3,
147    ReturnCancelled = 4,
148}
149
150impl Status {
151    /// Packs this status and the optional `waitable` provided into a 32-bit
152    /// result that the canonical ABI requires.
153    ///
154    /// The low 4 bits are reserved for the status while the upper 28 bits are
155    /// the waitable, if present.
156    pub fn pack(self, waitable: Option<u32>) -> u32 {
157        assert!(matches!(self, Status::Returned) == waitable.is_none());
158        let waitable = waitable.unwrap_or(0);
159        assert!(waitable < (1 << 28));
160        (waitable << 4) | (self as u32)
161    }
162}
163
164/// Corresponds to `EventCode` in the Component Model spec, plus related payload
165/// data.
166#[derive(Clone, Copy, Debug)]
167enum Event {
168    None,
169    Subtask {
170        status: Status,
171    },
172    StreamRead {
173        code: ReturnCode,
174        pending: Option<(TypeStreamTableIndex, u32)>,
175    },
176    StreamWrite {
177        code: ReturnCode,
178        pending: Option<(TypeStreamTableIndex, u32)>,
179    },
180    FutureRead {
181        code: ReturnCode,
182        pending: Option<(TypeFutureTableIndex, u32)>,
183    },
184    FutureWrite {
185        code: ReturnCode,
186        pending: Option<(TypeFutureTableIndex, u32)>,
187    },
188    Cancelled,
189}
190
191impl Event {
192    /// Lower this event to core Wasm integers for delivery to the guest.
193    ///
194    /// Note that the waitable handle, if any, is assumed to be lowered
195    /// separately.
196    fn parts(self) -> (u32, u32) {
197        const EVENT_NONE: u32 = 0;
198        const EVENT_SUBTASK: u32 = 1;
199        const EVENT_STREAM_READ: u32 = 2;
200        const EVENT_STREAM_WRITE: u32 = 3;
201        const EVENT_FUTURE_READ: u32 = 4;
202        const EVENT_FUTURE_WRITE: u32 = 5;
203        const EVENT_CANCELLED: u32 = 6;
204        match self {
205            Event::None => (EVENT_NONE, 0),
206            Event::Cancelled => (EVENT_CANCELLED, 0),
207            Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208            Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209            Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210            Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211            Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212        }
213    }
214}
215
216/// Corresponds to `CallbackCode` in the spec.
217mod callback_code {
218    pub const EXIT: u32 = 0;
219    pub const YIELD: u32 = 1;
220    pub const WAIT: u32 = 2;
221}
222
223// Flags passed to the `start-call` intrinsic.
224const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227/// Provides access to either store data (via the `get` method) or the store
228/// itself (via [`AsContext`]/[`AsContextMut`]), as well as the component
229/// instance to which the current host task belongs.
230///
231/// See [`Accessor::with`] for details.
232pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233    store: StoreContextMut<'a, T>,
234    get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239    D: HasData + ?Sized,
240    T: 'static,
241{
242    /// Creates a new [`Access`] from its component parts.
243    pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244        Self { store, get_data }
245    }
246
247    /// Get mutable access to the store data.
248    pub fn data_mut(&mut self) -> &mut T {
249        self.store.data_mut()
250    }
251
252    /// Get mutable access to the store data.
253    pub fn get(&mut self) -> D::Data<'_> {
254        (self.get_data)(self.data_mut())
255    }
256
257    /// Spawn a background task.
258    ///
259    /// See [`Accessor::spawn`] for details.
260    pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261    where
262        T: 'static,
263    {
264        let accessor = Accessor {
265            get_data: self.get_data,
266            token: StoreToken::new(self.store.as_context_mut()),
267        };
268        self.store
269            .as_context_mut()
270            .spawn_with_accessor(accessor, task)
271    }
272
273    /// Returns the getter this accessor is using to project from `T` into
274    /// `D::Data`.
275    pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276        self.get_data
277    }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282    D: HasData + ?Sized,
283    T: 'static,
284{
285    type Data = T;
286
287    fn as_context(&self) -> StoreContext<'_, T> {
288        self.store.as_context()
289    }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294    D: HasData + ?Sized,
295    T: 'static,
296{
297    fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298        self.store.as_context_mut()
299    }
300}
301
302/// Provides scoped mutable access to store data in the context of a concurrent
303/// host task future.
304///
305/// This allows multiple host task futures to execute concurrently and access
306/// the store between (but not across) `await` points.
307///
308/// # Rationale
309///
310/// This structure is sort of like `&mut T` plus a projection from `&mut T` to
311/// `D::Data<'_>`. The problem this is solving, however, is that it does not
312/// literally store these values. The basic problem is that when a concurrent
313/// host future is being polled it has access to `&mut T` (and the whole
314/// `Store`) but when it's not being polled it does not have access to these
315/// values. This reflects how the store is only ever polling one future at a
316/// time so the store is effectively being passed between futures.
317///
318/// Rust's `Future` trait, however, has no means of passing a `Store`
319/// temporarily between futures. The [`Context`](core::task::Context) type does
320/// not have the ability to attach arbitrary information to it at this time.
321/// This type, [`Accessor`], is used to bridge this expressivity gap.
322///
323/// The [`Accessor`] type here represents the ability to acquire, temporarily in
324/// a synchronous manner, the current store. The [`Accessor::with`] function
325/// yields an [`Access`] which can be used to access [`StoreContextMut`], `&mut
326/// T`, or `D::Data<'_>`. Note though that [`Accessor::with`] intentionally does
327/// not take an `async` closure as its argument, instead it's a synchronous
328/// closure which must complete during on run of `Future::poll`. This reflects
329/// how the store is temporarily made available while a host future is being
330/// polled.
331///
332/// # Implementation
333///
334/// This type does not actually store `&mut T` nor `StoreContextMut<T>`, and
335/// this type additionally doesn't even have a lifetime parameter. This is
336/// instead a representation of proof of the ability to acquire these while a
337/// future is being polled. Wasmtime will, when it polls a host future,
338/// configure ambient state such that the `Accessor` that a future closes over
339/// will work and be able to access the store.
340///
341/// This has a number of implications for users such as:
342///
343/// * It's intentional that `Accessor` cannot be cloned, it needs to stay within
344///   the lifetime of a single future.
345/// * A future is expected to, however, close over an `Accessor` and keep it
346///   alive probably for the duration of the entire future.
347/// * Different host futures will be given different `Accessor`s, and that's
348///   intentional.
349/// * The `Accessor` type is `Send` and `Sync` irrespective of `T` which
350///   alleviates some otherwise required bounds to be written down.
351///
352/// # Using `Accessor` in `Drop`
353///
354/// The methods on `Accessor` are only expected to work in the context of
355/// `Future::poll` and are not guaranteed to work in `Drop`. This is because a
356/// host future can be dropped at any time throughout the system and Wasmtime
357/// store context is not necessarily available at that time. It's recommended to
358/// not use `Accessor` methods in anything connected to a `Drop` implementation
359/// as they will panic and have unintended results. If you run into this though
360/// feel free to file an issue on the Wasmtime repository.
361pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363    D: HasData + ?Sized,
364{
365    token: StoreToken<T>,
366    get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369/// A helper trait to take any type of accessor-with-data in functions.
370///
371/// This trait is similar to [`AsContextMut`] except that it's used when
372/// working with an [`Accessor`] instead of a [`StoreContextMut`]. The
373/// [`Accessor`] is the main type used in concurrent settings and is passed to
374/// functions such as [`Func::call_concurrent`].
375///
376/// This trait is implemented for [`Accessor`] and `&T` where `T` implements
377/// this trait. This effectively means that regardless of the `D` in
378/// `Accessor<T, D>` it can still be passed to a function which just needs a
379/// store accessor.
380///
381/// Acquiring an [`Accessor`] can be done through
382/// [`StoreContextMut::run_concurrent`] for example or in a host function
383/// through
384/// [`Linker::func_wrap_concurrent`](crate::component::LinkerInstance::func_wrap_concurrent).
385pub trait AsAccessor {
386    /// The `T` in `Store<T>` that this accessor refers to.
387    type Data: 'static;
388
389    /// The `D` in `Accessor<T, D>`, or the projection out of
390    /// `Self::Data`.
391    type AccessorData: HasData + ?Sized;
392
393    /// Returns the accessor that this is referring to.
394    fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398    type Data = T::Data;
399    type AccessorData = T::AccessorData;
400
401    fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402        T::as_accessor(self)
403    }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407    type Data = T;
408    type AccessorData = D;
409
410    fn as_accessor(&self) -> &Accessor<T, D> {
411        self
412    }
413}
414
415// Note that it is intentional at this time that `Accessor` does not actually
416// store `&mut T` or anything similar. This distinctly enables the `Accessor`
417// structure to be both `Send` and `Sync` regardless of what `T` is (or `D` for
418// that matter). This is used to ergonomically simplify bindings where the
419// majority of the time `Accessor` is closed over in a future which then needs
420// to be `Send` and `Sync`. To avoid needing to write `T: Send` everywhere (as
421// you already have to write `T: 'static`...) it helps to avoid this.
422//
423// Note as well that `Accessor` doesn't actually store its data at all. Instead
424// it's more of a "proof" of what can be accessed from TLS. API design around
425// `Accessor` and functions like `Linker::func_wrap_concurrent` are
426// intentionally made to ensure that `Accessor` is ideally only used in the
427// context that TLS variables are actually set. For example host functions are
428// given `&Accessor`, not `Accessor`, and this prevents them from persisting
429// the value outside of a future. Within the future the TLS variables are all
430// guaranteed to be set while the future is being polled.
431//
432// Finally though this is not an ironclad guarantee, but nor does it need to be.
433// The TLS APIs are designed to panic or otherwise model usage where they're
434// called recursively or similar. It's hoped that code cannot be constructed to
435// actually hit this at runtime but this is not a safety requirement at this
436// time.
437const _: () = {
438    const fn assert<T: Send + Sync>() {}
439    assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443    /// Creates a new `Accessor` backed by the specified functions.
444    ///
445    /// - `get`: used to retrieve the store
446    ///
447    /// - `get_data`: used to "project" from the store's associated data to
448    /// another type (e.g. a field of that data or a wrapper around it).
449    ///
450    /// - `spawn`: used to queue spawned background tasks to be run later
451    pub(crate) fn new(token: StoreToken<T>) -> Self {
452        Self {
453            token,
454            get_data: |x| x,
455        }
456    }
457}
458
459impl<T, D> Accessor<T, D>
460where
461    D: HasData + ?Sized,
462{
463    /// Run the specified closure, passing it mutable access to the store.
464    ///
465    /// This function is one of the main building blocks of the [`Accessor`]
466    /// type. This yields synchronous, blocking, access to the store via an
467    /// [`Access`]. The [`Access`] implements [`AsContextMut`] in addition to
468    /// providing the ability to access `D` via [`Access::get`]. Note that the
469    /// `fun` here is given only temporary access to the store and `T`/`D`
470    /// meaning that the return value `R` here is not allowed to capture borrows
471    /// into the two. If access is needed to data within `T` or `D` outside of
472    /// this closure then it must be `clone`d out, for example.
473    ///
474    /// # Panics
475    ///
476    /// This function will panic if it is call recursively with any other
477    /// accessor already in scope. For example if `with` is called within `fun`,
478    /// then this function will panic. It is up to the embedder to ensure that
479    /// this does not happen.
480    pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481        tls::get(|vmstore| {
482            fun(Access {
483                store: self.token.as_context_mut(vmstore),
484                get_data: self.get_data,
485            })
486        })
487    }
488
489    /// Returns the getter this accessor is using to project from `T` into
490    /// `D::Data`.
491    pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492        self.get_data
493    }
494
495    /// Changes this accessor to access `D2` instead of the current type
496    /// parameter `D`.
497    ///
498    /// This changes the underlying data access from `T` to `D2::Data<'_>`.
499    ///
500    /// # Panics
501    ///
502    /// When using this API the returned value is disconnected from `&self` and
503    /// the lifetime binding the `self` argument. An `Accessor` only works
504    /// within the context of the closure or async closure that it was
505    /// originally given to, however. This means that due to the fact that the
506    /// returned value has no lifetime connection it's possible to use the
507    /// accessor outside of `&self`, the original accessor, and panic.
508    ///
509    /// The returned value should only be used within the scope of the original
510    /// `Accessor` that `self` refers to.
511    pub fn with_getter<D2: HasData>(
512        &self,
513        get_data: fn(&mut T) -> D2::Data<'_>,
514    ) -> Accessor<T, D2> {
515        Accessor {
516            token: self.token,
517            get_data,
518        }
519    }
520
521    /// Spawn a background task which will receive an `&Accessor<T, D>` and
522    /// run concurrently with any other tasks in progress for the current
523    /// store.
524    ///
525    /// This is particularly useful for host functions which return a `stream`
526    /// or `future` such that the code to write to the write end of that
527    /// `stream` or `future` must run after the function returns.
528    ///
529    /// The returned [`JoinHandle`] may be used to cancel the task.
530    ///
531    /// # Panics
532    ///
533    /// Panics if called within a closure provided to the [`Accessor::with`]
534    /// function. This can only be called outside an active invocation of
535    /// [`Accessor::with`].
536    pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537    where
538        T: 'static,
539    {
540        let accessor = self.clone_for_spawn();
541        self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542    }
543
544    fn clone_for_spawn(&self) -> Self {
545        Self {
546            token: self.token,
547            get_data: self.get_data,
548        }
549    }
550
551    /// Polls to see if this store contains any "interesting" tasks still within
552    /// it.
553    ///
554    /// Returns `Poll::Ready(())` if there are no more interesting tasks, and
555    /// otherwise returns `Poll::Pending`. If pending is returned then whenever
556    /// the last remaining "interesting" task has exited the provided context's
557    /// waker will be notified. Note that only the waker passed to the last call
558    /// to `poll_no_interesting_tasks` for the store will be notified, so this
559    /// is only appropriate to use once-at-a-time per store.
560    ///
561    /// The component model specification, as of this current date, does not
562    /// have a distinction between "interesting" tasks and not. The current
563    /// intention is that in a future revision of the component model this will
564    /// be distinguished at the component ABI level where tasks will be able to
565    /// flag themselves as "interesting" optionally. Additionally extra work can
566    /// be opted-in to being "interesting".
567    ///
568    /// For now what this means is that all component model tasks within this
569    /// store are considered interesting. This specifically includes the entire
570    /// duration of a task, so even all of the time after a task has returned
571    /// but before it has exited. This means that this function is, today,
572    /// effectively a proxy for "are there any more tasks still running in this
573    /// store". This can be used by embedders to determine whether there's any
574    /// more work going on, even in the background, for a particular guest.
575    /// Hosts can use this as a signal that the guest wants to stay alive a
576    /// little longer, even after a task has returned.
577    ///
578    /// In the future this predicate won't include all tasks in this store. Some
579    /// tasks will be able to flag themselves as not interesting, meaning that
580    /// when this returns ready it'd be possible that there are still tasks
581    /// remaining in the store.
582    ///
583    /// Note that at this time spawned threads within a task are always
584    /// considered uninteresting. If this function returns ready, then spawned
585    /// threads may still be in the store.
586    pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587        self.with(|mut access| {
588            let store = access.as_context_mut().0;
589            let state = store.concurrent_state_mut_without_forcing_current_thread();
590            if state.interesting_tasks == 0 {
591                Poll::Ready(())
592            } else {
593                state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594                Poll::Pending
595            }
596        })
597    }
598
599    /// Poll to see if the component instance corresponding to the specified
600    /// function is ready to run a concurrent call without queuing it.
601    ///
602    /// This will return `Poll::Ready(())` only if all of the following are true:
603    ///
604    /// - The store to which the instance belongs has no work items queued
605    ///   (e.g. events to deliver to the guest, etc.)
606    ///
607    /// - The instance does not have backpressure enabled.
608    ///
609    /// - The instance does not have a sync call in progress (which implicitly
610    ///   enables backpressure).
611    ///
612    /// Otherwise, it will return `Poll::Pending`.  If pending is returned then
613    /// whenever _any_ instance becomes ready for a call the provided context's
614    /// waker will be notified.  Note that only the waker passed to the last
615    /// call to `poll_ready_for_concurrent_call` for the store will be notified
616    /// (regardless of whether the same or different `Func` is specified
617    /// relative to earlier calls), so this is only appropriate to use
618    /// once-at-a-time per store.  Again, note that the waker may be notified
619    /// when _any_ instance becomes callable (i.e. not necessarily the last one
620    /// polled), so this function must be called again to determine if the
621    /// instance of interest is ready.
622    pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623        self.with(|mut access| {
624            let store = access.as_context_mut().0;
625
626            {
627                let state = store.concurrent_state_mut_without_forcing_current_thread();
628                if state.switch_item.is_some()
629                    || !state.high_priority.is_empty()
630                    || !state.low_priority.is_empty()
631                {
632                    state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633                    return Poll::Pending;
634                }
635            }
636
637            let (_, _, _, raw_options) = func.abi_info(store);
638            let instance = func.instance().runtime_instance(raw_options.instance);
639            let state = store.instance_state(instance).concurrent_state();
640            if state.backpressure == 0 {
641                Poll::Ready(())
642            } else {
643                store
644                    .concurrent_state_mut_without_forcing_current_thread()
645                    .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646                Poll::Pending
647            }
648        })
649    }
650}
651
652/// Represents an async closure which may be provided to `Accessor::spawn`,
653/// `Accessor::forward`, or `StoreContextMut::spawn`.
654// TODO: Replace this with `core::ops::AsyncFnOnce` when we are able to put `Send`
655// bound on the unnamed `Future` directly.
656//
657// As of this writing, it's not possible to specify e.g. `Send` and `Sync`
658// bounds on the `Future` type returned by an `AsyncFnOnce`.  Also, using `F:
659// Future<Output = Result<()>> + Send + Sync, FN: FnOnce(&Accessor<T>) -> F +
660// Send + Sync + 'static` fails with a type mismatch error as we cannot describe
661// that `F` should have `&Accessor<T>`'s unnamed lifetime
662//
663// Instead, this trait is used as a workaround for this limitation, the bound on `Self`
664// implementing `AsyncFnOnce()` is required for Rust to automatically infer that an async
665// closure is to be provided wherever we are accepting `impl for<'fut> AccessorTask<'fut, T, D>`
666// as argument. Otherwise, users will have to fully qualify the closure types before it
667// is accepted as a valid value. This also means that it is not intended for a user
668// to manually implement this trait for any arbitrary type, since they would first
669// have to implement `AsyncFnOnce`, which is unstable.
670//
671// The blanket implementation for this trait will ensure that `Self` is an async closure
672// that returns a `Future` that is `Send`, and lives as long as `&Accessor<T, D>`
673pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674    AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676    D: HasData + ?Sized,
677{
678    /// Run the task.
679    fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684    T: 'static,
685    F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686    F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687    Fut: Future<Output = Result<()>> + Send + 'fut,
688    D: HasData,
689{
690    fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691        (self)(accessor)
692    }
693}
694
695/// Represents parameter and result metadata for the caller side of a
696/// guest->guest call orchestrated by a fused adapter.
697enum CallerInfo {
698    /// Metadata for a call to an async-lowered import
699    Async {
700        params: Vec<ValRaw>,
701        has_result: bool,
702    },
703    /// Metadata for a call to an sync-lowered import
704    Sync {
705        params: Vec<ValRaw>,
706        result_count: u32,
707    },
708}
709
710/// Indicates how a guest task is waiting on a waitable set.
711enum WaitMode {
712    /// The guest task is waiting using `task.wait`
713    Fiber(StoreFiber<'static>),
714    /// The guest task is waiting via a callback declared as part of an
715    /// async-lifted export.
716    Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721        match self {
722            Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723            Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724        }
725    }
726}
727
728/// Represents the reason a fiber is suspending itself.
729#[derive(Debug)]
730enum SuspendReason {
731    /// The fiber is waiting for an event to be delivered to the specified
732    /// waitable set or task.
733    Waiting {
734        set: TableId<WaitableSet>,
735        thread: QualifiedThreadId,
736    },
737    /// The fiber is waiting for a subtask to suspend or exit, e.g. for a
738    /// guest-to-guest call or a `subtask.cancel`.
739    YieldingToSubtask { thread: QualifiedThreadId },
740    /// The fiber has finished handling its most recent work item and is waiting
741    /// for another (or to be dropped if it is no longer needed).
742    NeedWork,
743    /// The fiber is yielding and should be resumed once other tasks have had a
744    /// chance to run.
745    Yielding { thread: QualifiedThreadId },
746    /// The fiber was explicitly suspended with a call to `thread.suspend` or
747    /// `thread.switch-to`.
748    ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751/// Represents a pending call into guest code for a given guest task.
752enum GuestCallKind {
753    /// Indicates there's an event to deliver to the task, possibly related to a
754    /// waitable set the task has been waiting on or polling.
755    DeliverEvent {
756        /// The instance to which the task belongs.
757        instance: Instance,
758        /// The waitable set the event belongs to, if any.
759        ///
760        /// If this is `None` the event will be waiting in the
761        /// `GuestTask::event` field for the task.
762        set: Option<TableId<WaitableSet>>,
763    },
764    /// Indicates that a new guest task call is pending and may be executed
765    /// using the specified closure.
766    StartImplicit {
767        /// The closure to call.
768        closure: Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>,
769        /// Whether the task was previously pending due to e.g. backpressure.
770        was_pending: bool,
771    },
772    StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
773}
774
775impl fmt::Debug for GuestCallKind {
776    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
777        match self {
778            Self::DeliverEvent { instance, set } => f
779                .debug_struct("DeliverEvent")
780                .field("instance", instance)
781                .field("set", set)
782                .finish(),
783            Self::StartImplicit { was_pending, .. } => f
784                .debug_struct("StartImplicit")
785                .field("was_pending", was_pending)
786                .finish(),
787            Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
788        }
789    }
790}
791
792/// The target of a suspension intrinsic.
793#[derive(Copy, Clone, Debug)]
794pub enum SuspensionTarget {
795    Resume(u32),
796    Promote(u32),
797    None,
798}
799
800/// Behavior for `resume_thread`.
801#[derive(Copy, Clone, Debug)]
802pub enum ResumeThread {
803    Promote,
804    Resume,
805    ResumeLater,
806}
807
808/// Represents a pending call into guest code for a given guest thread.
809#[derive(Debug)]
810struct GuestCall {
811    thread: QualifiedThreadId,
812    kind: GuestCallKind,
813}
814
815impl GuestCall {
816    /// Returns whether or not the call is ready to run.
817    ///
818    /// A call will not be ready to run if either:
819    ///
820    /// - the (sub-)component instance to be called has already been entered and
821    ///   cannot be reentered until an in-progress call completes, and the task
822    ///   requires the exclusive lock
823    ///
824    /// - the call is for a not-yet started task and the (sub-)component
825    ///   instance to be called has backpressure enabled _or_ there are pending
826    ///   tasks waiting in front of this task
827    fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
828        let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
829        let async_typed = task.async_typed;
830        let async_lifted = task.async_lifted;
831        let has_callback = task.callback.is_some();
832        let instance = task.instance;
833        let state = store.instance_state(instance).concurrent_state();
834
835        let ready = match &self.kind {
836            GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
837            GuestCallKind::StartImplicit { was_pending, .. } => {
838                if async_typed {
839                    // Sync- and callback-lifted exports need the instance's
840                    // exclusive lock to start, but stackful ones don't.
841                    let needs_exclusive = !async_lifted || has_callback;
842
843                    // This task may enter the instance if (a) there's no
844                    // backpressure enabled, and (b) either the exclusive lock
845                    // is free or we can bypass the lock per the
846                    // `needs_exclusive` logic above.
847                    let may_enter =
848                        !((needs_exclusive && state.do_not_enter) || state.backpressure > 0);
849
850                    // If this task is already waiting in line (or was
851                    // previously in line before getting the go-ahead) or there
852                    // is no line to wait in, we can skip that.  Otherwise, we
853                    // need to get in line to avoid barging ahead of others that
854                    // have been waiting for backpressure to clear, even if
855                    // backpressure is no longer enabled.
856                    let skip_wait = *was_pending || state.pending_start_count == 0;
857
858                    // This task is ready if it may enter the instance _and_
859                    // there's no need to get in line first.
860                    may_enter && skip_wait
861                } else {
862                    true
863                }
864            }
865            GuestCallKind::StartExplicit(_) => true,
866        };
867        log::trace!(
868            "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {}; pending_start_count: {})",
869            state.do_not_enter,
870            state.backpressure,
871            state.pending_start_count,
872        );
873        Ok(ready)
874    }
875}
876
877/// Job to be run on a worker fiber.
878enum WorkerItem {
879    GuestCall(GuestCall),
880    Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
881}
882
883/// Represents a pending work item to be handled by the event loop for a given
884/// component instance.
885enum WorkItem {
886    /// A host task to be pushed to `ConcurrentState::futures`.
887    PushFuture(AlwaysMut<HostTaskFuture>),
888    /// A fiber to resume.
889    ResumeFiber {
890        instance: RuntimeInstance,
891        thread: QualifiedThreadId,
892        fiber: StoreFiber<'static>,
893    },
894    /// A thread to resume.
895    ResumeThread {
896        instance: RuntimeInstance,
897        thread: QualifiedThreadId,
898    },
899    /// A pending call into guest code for a given guest task.
900    GuestCall {
901        instance: RuntimeInstance,
902        call: GuestCall,
903    },
904    /// A job to run on a worker fiber.
905    WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
906}
907
908/// A fiber temporarily outside the store's scheduler state.
909///
910/// Construct this before returning a future so cancellation before its first
911/// poll also disposes of the fiber. Successful handoffs take the fiber only
912/// after all fallible destination checks have completed.
913struct ResumingFiber<'a> {
914    store: &'a mut StoreOpaque,
915    fiber: Option<StoreFiber<'static>>,
916}
917
918impl Drop for ResumingFiber<'_> {
919    fn drop(&mut self) {
920        if let Some(mut fiber) = self.fiber.take() {
921            fiber.dispose(self.store);
922        }
923    }
924}
925
926impl fmt::Debug for WorkItem {
927    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
928        match self {
929            Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
930            Self::ResumeFiber {
931                instance, thread, ..
932            } => f
933                .debug_struct("ResumeFiber")
934                .field("instance", instance)
935                .field("thread", thread)
936                .finish(),
937            Self::ResumeThread { instance, thread } => f
938                .debug_struct("ResumeThread")
939                .field("instance", instance)
940                .field("thread", thread)
941                .finish(),
942            Self::GuestCall { instance, call } => f
943                .debug_struct("GuestCall")
944                .field("instance", instance)
945                .field("call", call)
946                .finish(),
947            Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
948        }
949    }
950}
951
952/// Whether a suspension intrinsic was cancelled or completed
953#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
954pub(crate) enum WaitResult {
955    Cancelled,
956    Completed,
957}
958
959/// Poll the specified future until it completes on behalf of a guest->host call
960/// using a sync-lowered import.
961///
962/// This is similar to `Instance::first_poll` except it's for sync-lowered
963/// imports, meaning we don't need to handle cancellation and we can block the
964/// caller until the task completes, at which point the caller can handle
965/// lowering the result to the guest's stack and linear memory.
966pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
967    store: &mut dyn VMStore,
968    host_task: EnteredHostTask,
969    future: impl Future<Output = Result<R>> + Send + 'static,
970) -> Result<R> {
971    // Poll the future once before creating a host task. The host task will be
972    // created lazily if it's needed during the poll and otherwise will be
973    // created if the future suspends.  We can use a dummy `Waker` here because
974    // we'll add the future to `ConcurrentState::futures` and poll it
975    // automatically from the event loop if it doesn't complete immediately
976    // here.
977    let mut future = Box::pin(future);
978    let poll = tls::set(store, || {
979        future
980            .as_mut()
981            .poll(&mut Context::from_waker(&Waker::noop()))
982    });
983
984    let caller = match host_task {
985        Some(caller) => caller,
986        None => bail_bug!("host task wasn't created but should have been"),
987    };
988
989    let task = match poll {
990        // It completed immediately, so no persistent host task is needed.
991        Poll::Ready(result) => return result,
992
993        // It did not complete immediately; create the host task and add it to
994        // `ConcurrentState::futures` so it will be polled via the event loop;
995        // then use `GuestThread::sync_call_set` to wait for the task to
996        // complete, suspending the current fiber until it does so.
997        Poll::Pending => {
998            let Some(task) = store.materialize_host_task_id()? else {
999                bail_bug!("current thread is not a host thread")
1000            };
1001
1002            // Wrap the future in a closure which will stash its result in the
1003            // host task and resume this fiber when it completes.
1004            let future = Box::pin(async move {
1005                let result = run_with_host_task_set(task, future).await??;
1006                tls::get(move |store| {
1007                    let state = store.concurrent_state_mut()?;
1008                    let host_state = &mut state.get_mut(task)?.state;
1009                    assert!(matches!(host_state, HostTaskState::CalleeStarted));
1010                    *host_state = HostTaskState::CalleeFinished(Box::new(result));
1011
1012                    Waitable::Host(task).set_event(
1013                        state,
1014                        Some(Event::Subtask {
1015                            status: Status::Returned,
1016                        }),
1017                    )?;
1018
1019                    Ok(())
1020                })
1021            }) as HostTaskFuture;
1022
1023            let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
1024            store.switch_or_trap_if_may_not_suspend(caller_instance)?;
1025
1026            let state = store.concurrent_state_mut()?;
1027            state.push_future(future);
1028
1029            let set = state.get_mut(caller.thread)?.sync_call_set;
1030            Waitable::Host(task).join(state, Some(set))?;
1031
1032            store.suspend(SuspendReason::Waiting {
1033                set,
1034                thread: caller,
1035            })?;
1036
1037            // Remove the `task` from the `sync_call_set` to ensure that when
1038            // this function returns and the task is deleted that there are no
1039            // more lingering references to this host task.
1040            Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1041            task
1042        }
1043    };
1044
1045    // Retrieve and return the result.
1046    let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1047    match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1048        HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1049            Ok(result) => *result,
1050            Err(_) => bail_bug!("host task finished with wrong type of result"),
1051        }),
1052        _ => bail_bug!("unexpected host task state after completion"),
1053    }
1054}
1055
1056/// Execute the specified guest call.
1057fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1058    match call.kind {
1059        GuestCallKind::DeliverEvent { instance, set } => {
1060            // This thread's wait on `set`, if any, which started in
1061            // `Instance::wait_with_callback`, ends here.
1062            if let Some(set) = set {
1063                store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1064            }
1065            let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1066                Some(pair) => pair,
1067                None => match set {
1068                    // The event which woke this thread up was taken by another
1069                    // thread (e.g. via `waitable-set.poll`) or its waitable was
1070                    // removed from the set before this thread had a chance to
1071                    // run. Regardless, keep waiting.
1072                    Some(set) => {
1073                        log::trace!(
1074                            "event for {:?} on {set:?} no longer present; waiting again",
1075                            call.thread
1076                        );
1077                        return instance.wait_with_callback(
1078                            store.store_opaque_mut(),
1079                            call.thread,
1080                            set,
1081                        );
1082                    }
1083                    // When a thread yields it doesn't wait on a set and the
1084                    // event to be delivered isn't stored anywhere, and this is
1085                    // just handling that case so indicate that the yield is
1086                    // complete.
1087                    None => (Event::None, None),
1088                },
1089            };
1090            let state = store.concurrent_state_mut()?;
1091            // If this thread was woken up by an event in `set` but is instead
1092            // receiving its own task's event (e.g. a cancellation), then the
1093            // set's event is left for another thread waiting on it, so wake
1094            // the next waiter up if there is one.
1095            if let Some(set) = set
1096                && waitable.is_none()
1097                && !state.get_mut(set)?.ready.is_empty()
1098            {
1099                state.wake_waiter(set)?;
1100            }
1101            let task = state.get_mut(call.thread.task)?;
1102            let runtime_instance = task.instance;
1103            let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1104
1105            log::trace!(
1106                "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1107                call.thread,
1108            );
1109
1110            instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1111                invoke_callback(store, call.thread.task, event, handle)
1112            })?;
1113        }
1114        GuestCallKind::StartImplicit { closure, .. } => {
1115            closure(store)?;
1116        }
1117        GuestCallKind::StartExplicit(fun) => {
1118            fun(store)?;
1119        }
1120    }
1121
1122    Ok(())
1123}
1124
1125/// Invoke the callback of the callback-lifted `task` with `event` and
1126/// `handle`, returning the callback code it produces.
1127fn invoke_callback(
1128    store: &mut dyn VMStore,
1129    task: TableId<GuestTask>,
1130    event: Event,
1131    handle: u32,
1132) -> Result<u32> {
1133    let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1134        bail_bug!("guest task callback field not present")
1135    };
1136
1137    let code = callback(store, event, handle)?;
1138
1139    store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1140
1141    Ok(code)
1142}
1143
1144impl<T> Store<T> {
1145    /// Convenience wrapper for [`StoreContextMut::run_concurrent`].
1146    pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1147    where
1148        T: Send + 'static,
1149    {
1150        ensure!(
1151            self.as_context().0.concurrency_support(),
1152            "cannot use `run_concurrent` when Config::concurrency_support disabled",
1153        );
1154        self.as_context_mut().run_concurrent(fun).await
1155    }
1156
1157    #[doc(hidden)]
1158    pub fn assert_concurrent_state_empty(&mut self) {
1159        self.as_context_mut().assert_concurrent_state_empty();
1160    }
1161
1162    #[doc(hidden)]
1163    pub fn concurrent_state_table_size(&mut self) -> usize {
1164        self.as_context_mut().concurrent_state_table_size()
1165    }
1166
1167    /// Convenience wrapper for [`StoreContextMut::spawn`].
1168    pub fn spawn(
1169        &mut self,
1170        task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1171    ) -> Result<JoinHandle>
1172    where
1173        T: 'static,
1174    {
1175        self.as_context_mut().spawn(task)
1176    }
1177}
1178
1179impl<T> StoreContextMut<'_, T> {
1180    /// Assert that all the relevant tables and queues in the concurrent state
1181    /// for this store are empty.
1182    ///
1183    /// This is for sanity checking in integration tests
1184    /// (e.g. `component-async-tests`) that the relevant state has been cleared
1185    /// after each test concludes.  This should help us catch leaks, e.g. guest
1186    /// tasks which haven't been deleted despite having completed and having
1187    /// been dropped by their supertasks.
1188    ///
1189    /// Only intended for use in Wasmtime's own testing.
1190    #[doc(hidden)]
1191    pub fn assert_concurrent_state_empty(self) {
1192        let store = self.0;
1193        store
1194            .store_data_mut()
1195            .components
1196            .assert_instance_states_empty();
1197        let state = store.concurrent_state_mut().unwrap();
1198        assert!(
1199            state.table.get_mut().is_empty(),
1200            "non-empty table: {:?}",
1201            state.table.get_mut()
1202        );
1203        assert!(state.switch_item.is_none());
1204        assert!(state.next_switch_item.is_none());
1205        assert!(state.high_priority.is_empty());
1206        assert!(state.low_priority.is_empty());
1207        assert!(state.saved_next_switch_items.is_empty());
1208        assert!(state.unforced_current_thread.is_none());
1209        assert!(state.deferred_host_call_context.is_none());
1210        assert!(state.futures_mut().unwrap().is_empty());
1211        assert!(state.global_error_context_ref_counts.is_empty());
1212    }
1213
1214    /// Helper function to perform tests over the size of the concurrent state
1215    /// table which can be useful for detecting leaks.
1216    ///
1217    /// Only intended for use in Wasmtime's own testing.
1218    #[doc(hidden)]
1219    pub fn concurrent_state_table_size(&mut self) -> usize {
1220        self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1221    }
1222
1223    /// Spawn a background task to run as part of this instance's event loop.
1224    ///
1225    /// The task will receive an `&Accessor<U>` and run concurrently with
1226    /// any other tasks in progress for the instance.
1227    ///
1228    /// Note that the task will only make progress if and when the event loop
1229    /// for this instance is run.
1230    ///
1231    /// The returned [`JoinHandle`] may be used to cancel the task.
1232    pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1233    where
1234        T: 'static,
1235    {
1236        let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1237        self.spawn_with_accessor(accessor, task)
1238    }
1239
1240    /// Internal implementation of `spawn` functions where a `store` is
1241    /// available along with an `Accessor`.
1242    fn spawn_with_accessor<D>(
1243        self,
1244        accessor: Accessor<T, D>,
1245        task: impl for<'fut> AccessorTask<'fut, T, D>,
1246    ) -> Result<JoinHandle>
1247    where
1248        T: 'static,
1249        D: HasData + ?Sized,
1250    {
1251        // Create an "abortable future" here where internally the future will
1252        // hook calls to poll and possibly spawn more background tasks on each
1253        // iteration.
1254        let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1255        self.0
1256            .concurrent_state_mut()?
1257            .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1258        Ok(handle)
1259    }
1260
1261    /// Run the specified closure `fun` to completion as part of this store's
1262    /// event loop.
1263    ///
1264    /// This will run `fun` as part of this store's event loop until it
1265    /// yields a result.  `fun` is provided an [`Accessor`], which provides
1266    /// controlled access to the store and its data.
1267    ///
1268    /// This function can be used to invoke [`Func::call_concurrent`] for
1269    /// example within the async closure provided here.
1270    ///
1271    /// This function will unconditionally return an error if
1272    /// [`Config::concurrency_support`] is disabled.
1273    ///
1274    /// [`Config::concurrency_support`]: crate::Config::concurrency_support
1275    ///
1276    /// # Store-blocking behavior
1277    ///
1278    /// At this time there are certain situations in which the `Future` returned
1279    /// by the `AsyncFnOnce` passed to this function will not be polled for an
1280    /// extended period of time, despite one or more `Waker::wake` events having
1281    /// occurred for the task to which it belongs.  This can manifest as the
1282    /// `Future` seeming to be "blocked" or "locked up", but is actually due to
1283    /// the `Store` being held by e.g. a blocking host function, preventing the
1284    /// `Future` from being polled. A canonical example of this is when the
1285    /// `fun` provided to this function attempts to set a timeout for an
1286    /// invocation of a wasm function. In this situation the async closure is
1287    /// waiting both on (a) the wasm computation to finish, and (b) the timeout
1288    /// to elapse. At this time this setup will not always work and the timeout
1289    /// may not reliably fire.
1290    ///
1291    /// This function will not block the current thread and as such is always
1292    /// suitable to run in an `async` context, but the current implementation of
1293    /// Wasmtime can lead to situations where a certain wasm computation is
1294    /// required to make progress the closure to make progress. This is an
1295    /// artifact of Wasmtime's historical implementation of `async` functions
1296    /// and is the topic of [#11869] and [#11870]. In the timeout example from
1297    /// above it means that Wasmtime can get "wedged" for a bit where (a) must
1298    /// progress for a readiness notification of (b) to get delivered.
1299    ///
1300    /// This effectively means that it's not possible to reliably perform a
1301    /// "select" operation within the `fun` closure, which timeouts for example
1302    /// are based on. Fixing this requires some relatively major refactoring
1303    /// work within Wasmtime itself. This is a known pitfall otherwise and one
1304    /// that is intended to be fixed one day. In the meantime it's recommended
1305    /// to apply timeouts or such to the entire `run_concurrent` call itself
1306    /// rather than internally.
1307    ///
1308    /// [#11869]: https://github.com/bytecodealliance/wasmtime/issues/11869
1309    /// [#11870]: https://github.com/bytecodealliance/wasmtime/issues/11870
1310    ///
1311    /// # Example
1312    ///
1313    /// ```
1314    /// # use {
1315    /// #   wasmtime::{
1316    /// #     error::{Result},
1317    /// #     component::{ Component, Linker, Resource, ResourceTable},
1318    /// #     Config, Engine, Store
1319    /// #   },
1320    /// # };
1321    /// #
1322    /// # struct MyResource(u32);
1323    /// # struct Ctx { table: ResourceTable }
1324    /// #
1325    /// # async fn foo() -> Result<()> {
1326    /// # let mut config = Config::new();
1327    /// # let engine = Engine::new(&config)?;
1328    /// # let mut store = Store::new(&engine, Ctx { table: ResourceTable::new() });
1329    /// # let mut linker = Linker::new(&engine);
1330    /// # let component = Component::new(&engine, "")?;
1331    /// # let instance = linker.instantiate_async(&mut store, &component).await?;
1332    /// # let foo = instance.get_typed_func::<(Resource<MyResource>,), (Resource<MyResource>,)>(&mut store, "foo")?;
1333    /// # let bar = instance.get_typed_func::<(u32,), ()>(&mut store, "bar")?;
1334    /// store.run_concurrent(async |accessor| -> wasmtime::Result<_> {
1335    ///    let resource = accessor.with(|mut access| access.get().table.push(MyResource(42)))?;
1336    ///    let (another_resource,) = foo.call_concurrent(accessor, (resource,)).await?;
1337    ///    let value = accessor.with(|mut access| access.get().table.delete(another_resource))?;
1338    ///    bar.call_concurrent(accessor, (value.0,)).await?;
1339    ///    Ok(())
1340    /// }).await??;
1341    /// # Ok(())
1342    /// # }
1343    /// ```
1344    pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1345    where
1346        T: Send + 'static,
1347    {
1348        ensure!(
1349            self.0.concurrency_support(),
1350            "cannot use `run_concurrent` when Config::concurrency_support disabled",
1351        );
1352        self.do_run_concurrent(fun, false).await
1353    }
1354
1355    pub(super) async fn run_concurrent_trap_on_idle<R>(
1356        self,
1357        fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1358    ) -> Result<R> {
1359        self.do_run_concurrent(fun, true).await
1360    }
1361
1362    async fn do_run_concurrent<R>(
1363        mut self,
1364        fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1365        trap_on_idle: bool,
1366    ) -> Result<R> {
1367        debug_assert!(self.0.concurrency_support());
1368        self.0.check_event_loop_not_running()?;
1369        let token = StoreToken::new(self.as_context_mut());
1370
1371        struct Dropper<'a, T: 'static, V> {
1372            store: StoreContextMut<'a, T>,
1373            value: ManuallyDrop<V>,
1374        }
1375
1376        impl<'a, T, V> Drop for Dropper<'a, T, V> {
1377            fn drop(&mut self) {
1378                self.store
1379                    .0
1380                    .concurrent_state_mut_already_forced_current_thread()
1381                    .event_loop_running = false;
1382
1383                tls::set(self.store.0, || {
1384                    // SAFETY: Here we drop the value without moving it for the
1385                    // first and only time -- per the contract for `Drop::drop`,
1386                    // this code won't run again, and the `value` field will no
1387                    // longer be accessible.
1388                    unsafe { ManuallyDrop::drop(&mut self.value) }
1389                });
1390            }
1391        }
1392
1393        let accessor = &Accessor::new(token);
1394        self.0
1395            .concurrent_state_mut_already_forced_current_thread()
1396            .event_loop_running = true;
1397        let dropper = &mut Dropper {
1398            store: self,
1399            value: ManuallyDrop::new(fun(accessor)),
1400        };
1401        // SAFETY: We never move `dropper` nor its `value` field.
1402        let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1403
1404        let result = dropper
1405            .store
1406            .as_context_mut()
1407            .poll_until(future, trap_on_idle)
1408            .await;
1409
1410        if result.is_err() {
1411            dropper.store.0.set_trapped();
1412        }
1413
1414        result
1415    }
1416
1417    /// Run this store's event loop.
1418    ///
1419    /// The returned future will resolve when the specified future completes or,
1420    /// if `trap_on_idle` is true, when the event loop can't make further
1421    /// progress.
1422    async fn poll_until<R>(
1423        mut self,
1424        mut future: Pin<&mut impl Future<Output = R>>,
1425        trap_on_idle: bool,
1426    ) -> Result<R> {
1427        struct Reset<'a, T: 'static> {
1428            store: StoreContextMut<'a, T>,
1429            futures: Option<FuturesUnordered<HostTaskFuture>>,
1430        }
1431
1432        impl<'a, T> Drop for Reset<'a, T> {
1433            fn drop(&mut self) {
1434                if let Some(futures) = self.futures.take() {
1435                    *self
1436                        .store
1437                        .0
1438                        .concurrent_state_mut_already_forced_current_thread()
1439                        .futures
1440                        .get_mut() = Some(futures);
1441                }
1442            }
1443        }
1444
1445        // Keep a continuously busy event loop from holding one executor poll
1446        // indefinitely.  In particular, Tokio's cooperative budget is only
1447        // replenished when the future returns `Pending` to the executor.
1448        const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1449        let mut turns_without_yield = 0;
1450
1451        loop {
1452            // Take `ConcurrentState::futures` out of the store so we can poll
1453            // it while also safely giving any of the futures inside access to
1454            // `self`.
1455            let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1456            let mut reset = Reset {
1457                store: self.as_context_mut(),
1458                futures,
1459            };
1460            let mut next = match reset.futures.as_mut() {
1461                Some(f) => pin!(f.next()),
1462                None => bail_bug!("concurrent state missing futures field"),
1463            };
1464
1465            enum PollResult<R> {
1466                Complete(R),
1467                ProcessWork {
1468                    ready: Option<WorkItem>,
1469                    low_priority: bool,
1470                },
1471            }
1472
1473            let result = future::poll_fn(|cx| {
1474                // First, poll the future we were passed as an argument and
1475                // return immediately if it's ready.
1476                if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1477                    return Poll::Ready(Ok(PollResult::Complete(value)));
1478                }
1479
1480                // If the store has been poisoned by a trap, either before this
1481                // event loop started or while it was running, then refuse to
1482                // make any further progress. Any work items, suspended fibers,
1483                // or pending futures left over from the trap might otherwise
1484                // resume guest code or observe scheduler state which was never
1485                // unwound. Note that this is checked after polling `future`
1486                // above so that it can observe the original error, e.g. a
1487                // trap delivered over a oneshot channel by a work item.
1488                if reset.store.0.component_trapped() {
1489                    return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1490                }
1491
1492                // Next, poll `ConcurrentState::futures` (which includes any
1493                // pending host tasks and/or background tasks), returning
1494                // immediately if one of them fails.
1495                let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1496                    Poll::Ready(Some(output)) => {
1497                        match output {
1498                            Err(e) => return Poll::Ready(Err(e)),
1499                            Ok(()) => {}
1500                        }
1501                        Poll::Ready(true)
1502                    }
1503                    Poll::Ready(None) => Poll::Ready(false),
1504                    Poll::Pending => Poll::Pending,
1505                };
1506
1507                // Next, identify the next work item to process, if any, using
1508                // the following priority order:
1509                //
1510                // - `switch_item`: Represents the guest thread we _must_ switch
1511                // to before any other thread runs per the determinism
1512                // requirements in the Component Model spec.
1513                //
1514                // - `high_priority`: "Urgent" work items, e.g. async calls have
1515                // become freshly unblocked due to backpressure clearing or
1516                // similar, stream or future state updates, etc.
1517                //
1518                // - `low_priority`: Work items such as resuming a fiber after
1519                // it yields, in which case the point is to let other items run
1520                // first.
1521                let state = reset.store.0.concurrent_state_mut()?;
1522                let mut ready = state.switch_item.take();
1523                let mut low_priority = false;
1524                if ready.is_none() {
1525                    ready = state.high_priority.pop_back();
1526                    if ready.is_none() {
1527                        ready = state.low_priority.pop_back();
1528                        low_priority = true;
1529                    }
1530                }
1531                if ready.is_some() {
1532                    return Poll::Ready(Ok(PollResult::ProcessWork {
1533                        ready,
1534                        low_priority,
1535                    }));
1536                }
1537
1538                if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1539                    waker.wake();
1540                }
1541
1542                // Finally, if we have nothing else to do right now, determine what to do
1543                // based on whether there are any pending futures in
1544                // `ConcurrentState::futures`.
1545                return match next {
1546                    Poll::Ready(true) => {
1547                        // In this case, one of the futures in
1548                        // `ConcurrentState::futures` completed
1549                        // successfully, so we return now and continue
1550                        // the outer loop in case there is another one
1551                        // ready to complete.
1552                        Poll::Ready(Ok(PollResult::ProcessWork {
1553                            ready: None,
1554                            low_priority: false,
1555                        }))
1556                    }
1557                    Poll::Ready(false) => {
1558                        // Poll the future we were passed one last time
1559                        // in case one of `ConcurrentState::futures` had
1560                        // the side effect of unblocking it.
1561                        if let Poll::Ready(value) =
1562                            tls::set(reset.store.0, || future.as_mut().poll(cx))
1563                        {
1564                            Poll::Ready(Ok(PollResult::Complete(value)))
1565                        } else {
1566                            // In this case, there are no more pending
1567                            // futures in `ConcurrentState::futures`,
1568                            // there are no remaining work items, _and_
1569                            // the future we were passed as an argument
1570                            // still hasn't completed.
1571                            if trap_on_idle {
1572                                // `trap_on_idle` is true, so we exit
1573                                // immediately.
1574
1575                                // If there are any tasks belonging to
1576                                // an instance which may not suspend, trap with
1577                                // `CannotBlockSyncTask`:
1578                                Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1579                                    Trap::CannotBlockSyncTask.into()
1580                                } else {
1581                                    // Otherwise, trap with `AsyncDeadlock`:
1582                                    Trap::AsyncDeadlock.into()
1583                                }))
1584                            } else {
1585                                // `trap_on_idle` is false, so we assume
1586                                // that future will wake up and give us
1587                                // more work to do when it's ready to.
1588                                Poll::Pending
1589                            }
1590                        }
1591                    }
1592                    // There is at least one pending future in
1593                    // `ConcurrentState::futures` and we have nothing
1594                    // else to do but wait for now, so we return
1595                    // `Pending`.
1596                    Poll::Pending => Poll::Pending,
1597                };
1598            })
1599            .await;
1600
1601            // Put the `ConcurrentState::futures` back into the store before we
1602            // return or handle any work items since one or more of those items
1603            // might append more futures.
1604            drop(reset);
1605
1606            match result? {
1607                // The future we were passed as an argument completed, so we
1608                // return the result.
1609                PollResult::Complete(value) => break Ok(value),
1610                // The future we were passed has not yet completed, so handle
1611                // any work items and then loop again.
1612                PollResult::ProcessWork {
1613                    mut ready,
1614                    low_priority,
1615                } => {
1616                    // If we're about to run a low-priority task, first yield to
1617                    // the executor.  This ensures that it won't be starved of
1618                    // the ability to e.g. update the readiness of sockets,
1619                    // etc. which the guest may be using `thread.yield` along
1620                    // with `waitable-set.poll` to monitor in a CPU-heavy loop.
1621                    //
1622                    // This works for e.g. `thread.yield` and callbacks which
1623                    // return `CALLBACK_CODE_YIELD` because we queue a low
1624                    // priority item to resume the task (i.e. resume the thread
1625                    // or call the callback, respectively) just prior to
1626                    // suspending it.  Indeed, as of this writing those are the
1627                    // _only_ situations we queue low-priority tasks.
1628                    // Therefore, we interpret the guest's request to yield as
1629                    // meaning "yield to other guest tasks _and_/_or_ host
1630                    // operations such as updating socket readiness", the latter
1631                    // being the async runtime's responsibility.
1632                    //
1633                    // In the future, if this ends up causing measurable
1634                    // performance issues, this could be optimized such that we
1635                    // only yield periodically (e.g. for batches of low priority
1636                    // items) and not for each and every individual item.
1637                    //
1638                    // While we're yielded put the item back on the queue so it
1639                    // stays live within the store, and then if/when we come
1640                    // back take it back out. Note that the `await` point here
1641                    // owns the entire `StoreContextMut` so what we push should
1642                    // be what we pop.
1643                    if low_priority {
1644                        if let Some(item) = ready.take() {
1645                            self.0.concurrent_state_mut()?.low_priority.push_back(item);
1646                        }
1647                        self.0.yield_now().await;
1648                        turns_without_yield = 0;
1649                        ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1650                    }
1651
1652                    if let Some(item) = ready {
1653                        self.as_context_mut().handle_work_item(item).await?;
1654                    }
1655
1656                    turns_without_yield += 1;
1657                    if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1658                        turns_without_yield = 0;
1659                        self.0.yield_now().await;
1660                    }
1661                }
1662            }
1663        }
1664    }
1665
1666    /// Handle the specified work item, possibly resuming a fiber if applicable.
1667    fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1668        // Construct the guarded future before returning, even when the caller
1669        // never polls this work item. Other variants keep their fibers in the
1670        // store until their first poll.
1671        let item = match item {
1672            WorkItem::ResumeFiber {
1673                instance,
1674                thread,
1675                fiber,
1676            } => {
1677                let future = self.0.resume_fiber(fiber);
1678                log::trace!(
1679                    "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1680                );
1681                return futures::future::Either::Left(future);
1682            }
1683            item => item,
1684        };
1685        futures::future::Either::Right(async move {
1686            log::trace!("handle work item {item:?}");
1687            match item {
1688                WorkItem::PushFuture(future) => {
1689                    self.0
1690                        .concurrent_state_mut()?
1691                        .futures_mut()?
1692                        .push(future.into_inner());
1693                }
1694                WorkItem::ResumeFiber { .. } => unreachable!(),
1695                WorkItem::ResumeThread { thread, .. } => {
1696                    let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1697                    if !matches!(slot, GuestThreadState::Ready { .. }) {
1698                        bail_bug!("cannot resume non-pending thread {thread:?}");
1699                    }
1700                    let GuestThreadState::Ready { fiber } =
1701                        mem::replace(slot, GuestThreadState::Running)
1702                    else {
1703                        unreachable!()
1704                    };
1705                    self.0.resume_fiber(fiber).await?;
1706                }
1707                WorkItem::GuestCall { mut call, .. } => {
1708                    if call.is_ready(self.0)? {
1709                        if let GuestCallKind::StartImplicit {
1710                            was_pending: true, ..
1711                        } = &call.kind
1712                        {
1713                            let instance = self
1714                                .0
1715                                .concurrent_state_mut()?
1716                                .get_mut(call.thread.task)?
1717                                .instance;
1718
1719                            self.0.decrement_pending_start_count(instance, 1)?;
1720                        }
1721
1722                        self.0
1723                            .concurrent_state_mut()?
1724                            .get_mut(call.thread.thread)?
1725                            .wake_on_cancel = WakeOnCancel::None;
1726
1727                        self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1728                    } else {
1729                        let state = self.0.concurrent_state_mut()?;
1730                        let task = state.get_mut(call.thread.task)?;
1731                        if !task.starting_sent {
1732                            task.starting_sent = true;
1733                            if let GuestCallKind::StartImplicit { .. } = &call.kind {
1734                                Waitable::Guest(call.thread.task).set_event(
1735                                    state,
1736                                    Some(Event::Subtask {
1737                                        status: Status::Starting,
1738                                    }),
1739                                )?;
1740                            }
1741                        }
1742
1743                        let increment = if let GuestCallKind::StartImplicit {
1744                            was_pending, ..
1745                        } = &mut call.kind
1746                        {
1747                            let increment = !*was_pending;
1748                            *was_pending = true;
1749                            increment
1750                        } else {
1751                            false
1752                        };
1753
1754                        let instance = state.get_mut(call.thread.task)?.instance;
1755                        // If this call is for a callback-lifted task waiting on a
1756                        // waitable set, it can't run until `instance` is enterable
1757                        // again. Meanwhile, the set may have an event which another
1758                        // thread waiting on it could take, and that thread may
1759                        // even be the reason `instance` isn't enterable, so wake
1760                        // the next waiter up as well if there's still an event
1761                        // for it. Whichever thread runs first takes the event, and
1762                        // the other goes back to waiting if none is left.
1763                        if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1764                            if !state.get_mut(*set)?.ready.is_empty() {
1765                                state.wake_waiter(*set)?;
1766                            }
1767                        }
1768
1769                        // Switch back to the caller (or canceller) immediately if
1770                        // applicable since we aren't yet able to run the subtask it
1771                        // yielded to.
1772                        state.take_next_switch_item()?;
1773
1774                        let state = self.0.instance_state(instance).concurrent_state();
1775
1776                        state.pending.insert(call.thread, call.kind);
1777
1778                        if increment {
1779                            state.pending_start_count += 1;
1780                        }
1781                    }
1782                }
1783                WorkItem::WorkerFunction(fun) => {
1784                    self.run_on_worker(WorkerItem::Function(fun)).await?;
1785                }
1786            }
1787
1788            Ok(())
1789        })
1790    }
1791
1792    /// Execute the specified guest call on a worker fiber.
1793    async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1794        let state = self.0.concurrent_state_mut()?;
1795        if state.worker_item.is_some() {
1796            bail_bug!("worker item already set when taking worker fiber");
1797        }
1798        let worker = if let Some(fiber) = state.worker.take() {
1799            fiber
1800        } else {
1801            // SAFETY: the `make_fiber_unchecked` function is unsafe because the
1802            // returned fiber is unconditionally `Send` as opposed to being
1803            // conditionally send depending on the argument (in this case
1804            // `self.0`). This `async` function, however, is conditionally
1805            // `Send` depending on `self`, in this case `StoreContextMut<T>`,
1806            // which is already going to be conditionally `Send` depending on
1807            // `T`.
1808            //
1809            // The returned fiber is possibly stored within the `Store<T>` as
1810            // well. If `T: Send` then that's fine and everything's dandy. If
1811            // `T: !Send`, however, then the store is already not-`Send` meaning
1812            // that putting more actually-not-`Send` things inside of it isn't
1813            // an issue.
1814            //
1815            // The main issue here is that the returned fiber effectively can't
1816            // get transferred outside the context of the store. That's an
1817            // implementation detail we'll have to rely on, but is currently
1818            // true.
1819            unsafe {
1820                fiber::make_fiber_unchecked(self.0, move |store| {
1821                    loop {
1822                        let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1823                            bail_bug!("worker_item not present when resuming fiber")
1824                        };
1825                        match item {
1826                            WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1827                            WorkerItem::Function(fun) => fun.into_inner()(store)?,
1828                        }
1829
1830                        store.suspend(SuspendReason::NeedWork)?;
1831                    }
1832                })?
1833            }
1834        };
1835
1836        // No fallible operation between taking the fiber and guarding it in
1837        // resume_fiber; the current thread was forced before the take.
1838        self.0
1839            .concurrent_state_mut_already_forced_current_thread()
1840            .worker_item = Some(item);
1841
1842        self.0.resume_fiber(worker).await
1843    }
1844
1845    /// Wrap the specified host function in a future which will call it, passing
1846    /// it an `&Accessor<T>`.
1847    ///
1848    /// See the `Accessor` documentation for details.
1849    pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1850    where
1851        T: 'static,
1852        F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1853            + Send
1854            + Sync
1855            + 'static,
1856        R: Send + Sync + 'static,
1857    {
1858        let token = StoreToken::new(self);
1859        async move {
1860            let mut accessor = Accessor::new(token);
1861            closure(&mut accessor).await
1862        }
1863    }
1864
1865    pub(crate) async fn start_instance(
1866        &mut self,
1867        instance: ModuleInstance,
1868        callee: Option<RuntimeInstance>,
1869    ) -> Result<ModuleInstance> {
1870        // See `Func::call_async` for why this is checked up front.
1871        self.0.check_event_loop_not_running()?;
1872        let (tx, rx) = oneshot::channel();
1873        let token = StoreToken::new(self.as_context_mut());
1874        self.0.queue_task(move |store| {
1875            _ = tx.send(
1876                super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1877                    .map(|()| instance),
1878            );
1879            Ok(())
1880        })?;
1881        self.as_context_mut()
1882            .run_concurrent_trap_on_idle(async |_| {
1883                rx.await
1884                    .map_err(|_| format_err!("oneshot channel canceled"))
1885            })
1886            .await??
1887    }
1888}
1889
1890/// Return value of [`StoreOpaque::host_task_create`].
1891///
1892/// This is an `Option` to handle the dynamic `store.concurrency_support()`
1893/// property. When present this records the guest thread to restore when the
1894/// host call exits. The corresponding [`HostTask`] will need to be lazily
1895/// created if needed via [`StoreOpaque::materialize_host_task_id`].
1896pub type EnteredHostTask = Option<QualifiedThreadId>;
1897
1898impl StoreOpaque {
1899    pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1900        if self
1901            .concurrent_state_mut_already_forced_current_thread()
1902            .event_loop_running
1903        {
1904            bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1905        }
1906        Ok(())
1907    }
1908
1909    /// Returns the currently-running thread, promoting any deferred lazy guest
1910    /// thread into a fully-materialized `CurrentThread`. Deferred [`HostTask`]s
1911    /// are not materialized.
1912    #[inline]
1913    pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1914        // Without concurrency support there is nothing to force.
1915        if !self.concurrency_support() {
1916            return Ok(CurrentThread::None);
1917        }
1918
1919        // If the JIT-visible current thread isn't a deferred thread then
1920        // `ConcurrentState` is already up to date.
1921        if !self
1922            .vm_store_context_mut()
1923            .current_thread_mut()
1924            .is_deferred()
1925        {
1926            return Ok(self
1927                .concurrent_state_mut_already_forced_current_thread()
1928                .unforced_current_thread);
1929        }
1930
1931        self.force_deferred_current_thread()
1932    }
1933
1934    /// Slow path of [`Self::current_thread`]: promote the deferred lazy
1935    /// thread into a fully-materialized `CurrentThread`.
1936    #[cold]
1937    fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1938        // The component instance whose adapters pushed the deferred frames; all
1939        // frames in a guest-to-guest, sync-to-sync call chain of fused adapters
1940        // live within a single `wasmtime::component::Instance` (because
1941        // cross-`wasmtime::component::Instance` calls don't go through fused
1942        // adapters), and guest code only ever runs as a guest thread, so the
1943        // chain's base thread is already materialized in `ConcurrentState` and
1944        // we can get the `ComponentInstanceId` shared by the whole chain from
1945        // here.
1946        let state = self.concurrent_state_mut_without_forcing_current_thread();
1947        let id = match state.unforced_current_thread.guest_task() {
1948            Some(task) => state.get_mut(task)?.instance.instance,
1949            None => bail_bug!("deferred component-model thread with non-guest base"),
1950        };
1951
1952        // Collect the deferred frames pushed inline by fused adapters, walking
1953        // the `parent` chain from innermost to the base.
1954        let mut frames = SmallVec::<[_; 8]>::new();
1955        let mut cur = *self.vm_store_context_mut().current_thread_mut();
1956        while let Some(ptr) = cur.as_deferred() {
1957            // SAFETY: `ptr` points at a `VMDeferredThread` living in a fused
1958            // adapter's stack frame that is suspended below us on the stack
1959            // (mid-call, waiting for this nested call to return), so the
1960            // referent is still valid and exclusively ours to read.
1961            let deferred = unsafe { ptr.as_non_null().as_ref() };
1962            frames.push((
1963                deferred.callee_async != 0,
1964                deferred.callee_instance,
1965                deferred.saved_context,
1966                deferred.lazy_lenders_start,
1967            ));
1968            cur = deferred.parent;
1969        }
1970
1971        // Mark the current thread forced *before* replaying so that any
1972        // reentrant `force_current_thread` call short-circuits via the
1973        // non-deferred path above.
1974        *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1975
1976        // Save the current context, as we need to overwrite it while replaying
1977        // below.
1978        let current_context = *self.vm_store_context_mut().component_context_mut();
1979
1980        // Replay the deferred `enter_guest_sync_call`s outermost-first so that
1981        // the resulting `ConcurrentState` matches what the non-deferred path
1982        // would have otherwise produced.
1983        //
1984        // A frame's own lazy lends run from its start to that of the frame
1985        // pushed after it, or to the end of the buffer for the innermost.
1986        let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
1987        frames.reverse();
1988        let lazy_lenders_base = match frames.first() {
1989            Some((_, _, _, start)) => *start,
1990            None => bail_bug!("forcing a deferred thread with no deferred frames"),
1991        };
1992        let lazy_lender_ends = frames
1993            .iter()
1994            .skip(1)
1995            .map(|(_, _, _, start)| *start)
1996            .chain([lazy_lenders_end])
1997            .collect::<SmallVec<[_; 8]>>();
1998        for ((callee_async, callee_instance, saved_context, start), lenders_end) in
1999            frames.into_iter().zip(lazy_lender_ends)
2000        {
2001            // Restore the caller's context slots so that we save the correct
2002            // values into the caller's thread, exactly as the non-deferred path
2003            // would have on entry.
2004            *self.vm_store_context_mut().component_context_mut() = saved_context;
2005            let callee = RuntimeInstance {
2006                instance: id,
2007                index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
2008            };
2009            self.enter_guest_sync_call(callee_async, callee)?;
2010
2011            // Now that this frame has a real `GuestTask`, promote any lazy
2012            // lends it recorded while deferred.
2013            self.promote_current_lazy_lends(id, start, lenders_end)?;
2014        }
2015
2016        // Every lazy-lend record was promoted, so the length resets back to
2017        // where the outermost frame found it.
2018        debug_assert_eq!(
2019            *self.vm_store_context_mut().lazy_lenders_len_mut(),
2020            lazy_lenders_end
2021        );
2022        *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
2023
2024        // Replaying done; restore the current context.
2025        *self.vm_store_context_mut().component_context_mut() = current_context;
2026
2027        Ok(self
2028            .concurrent_state_mut_without_forcing_current_thread()
2029            .unforced_current_thread)
2030    }
2031
2032    /// Promotes the `lazy_lenders` entries in `start..end` into real lends
2033    /// owned by the current thread's `CallContext`.
2034    ///
2035    /// The task's `validate_scope_exit` then undoes them as if they had been
2036    /// lent there all along.
2037    fn promote_current_lazy_lends(
2038        &mut self,
2039        id: ComponentInstanceId,
2040        start: usize,
2041        end: usize,
2042    ) -> Result<()> {
2043        if start == end {
2044            return Ok(());
2045        }
2046
2047        let scope = match self
2048            .concurrent_state_mut_without_forcing_current_thread()
2049            .unforced_current_thread
2050            .guest_task()
2051        {
2052            Some(task) => VMScope::id(task.rep()),
2053            None => bail_bug!("replayed deferred frame is not a guest thread"),
2054        };
2055        let instance = Instance::from_wasmtime(self, id);
2056        self.promote_lazy_lends(instance, scope, start, end)
2057    }
2058
2059    fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2060        match self.current_thread()?.guest() {
2061            Some(id) => Ok(*id),
2062            None => bail_bug!("current thread is not a guest thread"),
2063        }
2064    }
2065
2066    // A result of `None` may indicate that this is either the top-level event
2067    // loop, a deferred host task, or concurrency support is disabled. In all
2068    // cases we don't have an ID for the task.
2069    pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2070        match self.current_thread()? {
2071            CurrentThread::Host(id) => Ok(Some(id)),
2072            CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2073            _ => bail_bug!("current thread is not a host thread"),
2074        }
2075    }
2076
2077    /// Returns the current host task ID, materializing a deferred host task if
2078    /// one is active. `None` represents a call from the top-level host.
2079    fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2080        Ok(self
2081            .concurrent_state_mut()?
2082            .materialize_current_host_task_id()?)
2083    }
2084
2085    fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2086        log::trace!("enter sync-typed call {callee:?}");
2087        let state = self.instance_state(callee).concurrent_state();
2088        let old_do_not_suspend = state.do_not_suspend;
2089        state.do_not_suspend = true;
2090
2091        let thread = self.current_guest_thread()?;
2092        let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2093        if thread.old_do_not_suspend.is_some() {
2094            bail_bug!("current thread already has `old_do_not_suspend` value");
2095        }
2096
2097        thread.old_do_not_suspend = Some(old_do_not_suspend);
2098
2099        Ok(())
2100    }
2101
2102    fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2103        log::trace!("exit sync-typed call {callee:?}");
2104        let thread = self.current_guest_thread()?;
2105        let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2106        let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2107            bail_bug!("current thread missing `old_do_not_suspend` value");
2108        };
2109        let state = self.instance_state(callee).concurrent_state();
2110        state.do_not_suspend = old_do_not_suspend;
2111        Ok(())
2112    }
2113
2114    /// Push a `GuestTask` onto the task stack for either a sync-to-sync,
2115    /// guest-to-guest call or a sync host-to-guest call.
2116    ///
2117    /// This task will only be used for the purpose of handling calls to
2118    /// intrinsic functions; both parameter lowering and result lifting are
2119    /// assumed to be taken care of elsewhere.
2120    ///
2121    /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
2122    /// fused adapters, see `StoreOpaque::current_thread`, `VMDeferredThread`,
2123    /// and `lower_fact_enter_sync_call`. Make sure all this stuff stays in
2124    /// sync!
2125    pub(crate) fn enter_guest_sync_call(
2126        &mut self,
2127        callee_async_typed: bool,
2128        callee: RuntimeInstance,
2129    ) -> Result<()> {
2130        log::trace!("enter sync-lifted call {callee:?}");
2131        if !self.concurrency_support() {
2132            return self.enter_call_not_concurrent();
2133        }
2134
2135        // Stash the next switch item (if any) for the duration of this call
2136        // since it's for the caller's caller, not for us.  We'll restore this
2137        // in `exit_guest_sync_call`.
2138        let state = self.concurrent_state_mut()?;
2139        let item = state.next_switch_item.take();
2140        state.saved_next_switch_items.push(item);
2141
2142        let thread = self.current_thread()?;
2143        let caller = if let Some(thread) = thread.guest() {
2144            Caller::Guest { thread: *thread }
2145        } else {
2146            Caller::Host {
2147                tx: None,
2148                caller: self.materialize_host_task_id()?,
2149            }
2150        };
2151
2152        let state = self.concurrent_state_mut()?;
2153        let guest_thread = GuestTask::new(
2154            state,
2155            Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2156            LiftResult {
2157                lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2158                ty: TypeTupleIndex::reserved_value(),
2159                memory: None,
2160                string_encoding: StringEncoding::Utf8,
2161            },
2162            caller,
2163            None,
2164            callee,
2165            callee_async_typed,
2166            false,
2167        )?;
2168
2169        Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2170            guest_thread.thread,
2171            self,
2172            callee.index,
2173        )?;
2174        self.set_thread(guest_thread)?;
2175
2176        if !callee_async_typed {
2177            self.enter_sync_call(callee)?;
2178        }
2179
2180        Ok(())
2181    }
2182
2183    /// Pop a `GuestTask` previously pushed using `enter_guest_sync_call`.
2184    ///
2185    /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
2186    /// fused adapters and then when the call returns we check to see if the
2187    /// task's contruction was forced and if not avoid calling out of the JIT
2188    /// code to this function. See `lower_fact_exit_sync_call`. Make sure all
2189    /// this stuff stays in sync!
2190    pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2191        if !self.concurrency_support() {
2192            return Ok(self.exit_call_not_concurrent());
2193        }
2194
2195        let thread = match self.current_thread()?.guest() {
2196            Some(t) => *t,
2197            None => bail_bug!("expected task when exiting"),
2198        };
2199        let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2200        let instance = task.instance;
2201
2202        let caller = match &task.caller {
2203            &Caller::Guest { thread } => thread.into(),
2204            &Caller::Host { caller, .. } => caller
2205                .map(CurrentThread::Host)
2206                .unwrap_or(CurrentThread::None),
2207        };
2208        task.lift_result = None;
2209        task.exited = true;
2210        let async_typed = task.async_typed;
2211
2212        if !async_typed {
2213            self.exit_sync_call(instance)?;
2214        }
2215
2216        self.set_thread(caller)?;
2217
2218        log::trace!("exit sync-lifted call {instance:?}");
2219
2220        if async_typed {
2221            // If we're async-typed, returning control to our caller won't help
2222            // resolve any outstanding sync-typed call which might be in
2223            // progress, so we may need to switch or trap before exiting this
2224            // thread:
2225            self.switch_or_trap_if_may_not_suspend(instance)?;
2226        }
2227
2228        self.cleanup_thread(thread, instance)?;
2229
2230        self.concurrent_state_mut()?
2231            .decrement_task_ref_count(thread.task)?;
2232
2233        let state = self.concurrent_state_mut()?;
2234        let Some(item) = state.saved_next_switch_items.pop() else {
2235            bail_bug!("unable to pop from `saved_next_switch_items`");
2236        };
2237        if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2238            // Stash it back in the store to ensure it's cleaned up on store
2239            // drop:
2240            state.push_high_priority(item);
2241            bail_bug!("`next_switch_item` unexpectedly already set");
2242        }
2243
2244        Ok(())
2245    }
2246
2247    /// Similar to `enter_guest_sync_call` except for when the guest makes a
2248    /// transition to the host.
2249    ///
2250    /// This initially records a deferred host call. A full [`HostTask`] should
2251    /// be allocated later if needed via
2252    /// [`StoreOpaque::materialize_host_task_id`].
2253    pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2254        if !self.concurrency_support() {
2255            self.enter_call_not_concurrent()?;
2256            return Ok(None);
2257        }
2258        let caller = self.current_guest_thread()?;
2259        log::trace!("new deferred host task with caller {caller:?}");
2260
2261        self.set_thread(CurrentThread::DeferredHost(caller))?;
2262        let state = self.concurrent_state_mut()?;
2263        debug_assert!(state.deferred_host_call_context.is_none());
2264        state.deferred_host_call_context = Some(CallContext::default());
2265        state.debug_assert_deferred_host_invariant();
2266        Ok(Some(caller))
2267    }
2268
2269    /// Dual of `host_task_create` and signifies that the host has finished and
2270    /// will be cleaned up.
2271    ///
2272    /// Note that this isn't invoked when the host is invoked asynchronously and
2273    /// the host isn't complete yet. In that situation the host task persists
2274    /// and will be cleaned up separately in `subtask_drop`
2275    pub(crate) fn host_task_delete(
2276        &mut self,
2277        original_task: EnteredHostTask,
2278        materialized_task: Option<TableId<HostTask>>,
2279    ) -> Result<()> {
2280        match original_task {
2281            Some(caller) => {
2282                self.set_thread(caller)?;
2283                if materialized_task.is_none() {
2284                    let state = self.concurrent_state_mut()?;
2285                    let context = state
2286                        .deferred_host_call_context
2287                        .take()
2288                        .expect("deferred host call context should be present");
2289                    debug_assert!(context.is_empty());
2290                    state.debug_assert_deferred_host_invariant();
2291                }
2292                log::trace!(
2293                    "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2294                );
2295                if let Some(task) = materialized_task {
2296                    self.concurrent_state_mut()?.delete_host_task(task)?;
2297                }
2298            }
2299            None => {
2300                debug_assert!(materialized_task.is_none());
2301                self.exit_call_not_concurrent();
2302            }
2303        }
2304        Ok(())
2305    }
2306
2307    /// Helper function to retrieve the `InstanceState` for the
2308    /// specified instance.
2309    fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2310        self.component_instance_mut(instance.instance)
2311            .instance_state(instance.index)
2312    }
2313
2314    /// Same as `instance_state`, but also returns the store's `HandleCounter`
2315    /// for inserting into the instance's handle tables.
2316    fn instance_state_and_handle_counter(
2317        &mut self,
2318        instance: RuntimeInstance,
2319    ) -> (&mut InstanceState, &mut HandleCounter) {
2320        let (component, counter) = self
2321            .store_data_mut()
2322            .component_instance_and_handle_counter_mut(instance.instance);
2323        (component.instance_state(instance.index), counter)
2324    }
2325
2326    /// Begins inserting into the handle table of the specified instance.
2327    fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2328        let (component, counter) = self
2329            .store_data_mut()
2330            .component_instance_and_handle_counter_mut(instance.instance);
2331        component.handle_table_insertion(instance.index, counter)
2332    }
2333
2334    /// Configure the currently running `thread`.
2335    ///
2336    /// This will save off any state necessary for the previous thread, if
2337    /// applicable, and then it'll additionally update state for `thread` if
2338    /// needed too.
2339    pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2340        let thread = thread.into();
2341        let state = self.concurrent_state_mut()?;
2342        state.debug_assert_deferred_host_invariant();
2343        let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2344
2345        state.handle_thread_switch(old_thread, thread)?;
2346
2347        // First thing to do after swapping threads is updating the context
2348        // slots for this thread within the store. This restores the behavior of
2349        // `context.{get,set}`. This involves taking the old state out of the
2350        // store, saving it in the thread that's being swapped from, and doing
2351        // the inverse for the new thread. When debug assertions are enabled
2352        // this also leaves behind sentinel values to try to uncover bugs where
2353        // this may be forgotten.
2354        if let Some(old_thread) = old_thread.guest() {
2355            let old_context = *self.vm_store_context_mut().component_context_mut();
2356            self.concurrent_state_mut()?
2357                .get_mut(old_thread.thread)?
2358                .context = old_context;
2359        }
2360        if cfg!(debug_assertions) {
2361            *self.vm_store_context_mut().component_context_mut() =
2362                [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2363        }
2364        if let Some(thread) = thread.guest() {
2365            let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2366            let context = thread.context;
2367            if cfg!(debug_assertions) {
2368                thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2369            }
2370            *self.vm_store_context_mut().component_context_mut() = context;
2371        }
2372
2373        // Keep the JIT-visible current-thread pointer in sync.
2374        *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2375            VMLazyThread::none()
2376        } else {
2377            VMLazyThread::forced()
2378        };
2379
2380        Ok(old_thread)
2381    }
2382
2383    /// Call `switch_if_may_not_suspend` and trap if it returns `false`.
2384    fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2385        if self.switch_if_may_not_suspend(instance)? {
2386            Ok(())
2387        } else {
2388            Err(Trap::CannotBlockSyncTask.into())
2389        }
2390    }
2391
2392    /// Check if the specified instance has a sync-typed call in progress; if so
2393    /// attempt to switch to another ready thread for that instance, and if no
2394    /// such thread exists, return false.
2395    fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2396        // Call this for the side effect of forcing any deferred task creation,
2397        // which may influence the value of `ConcurrentState::do_not_suspend`
2398        // below:
2399        self.concurrent_state_mut()?;
2400
2401        Ok(!self.concurrency_support()
2402            || !self
2403                .instance_state(instance)
2404                .concurrent_state()
2405                .do_not_suspend
2406            || self
2407                .concurrent_state_mut()?
2408                .promote_instance_local_thread_work_item(instance)?)
2409    }
2410
2411    /// Record that we're about to enter a (sub-)component instance which does
2412    /// not support more than one concurrent, stackful activation, meaning it
2413    /// cannot be entered again until the next call returns.
2414    fn enter_instance(&mut self, instance: RuntimeInstance) {
2415        log::trace!("enter {instance:?}");
2416        self.instance_state(instance)
2417            .concurrent_state()
2418            .do_not_enter = true;
2419    }
2420
2421    /// Record that we've exited a (sub-)component instance previously entered
2422    /// with `Self::enter_instance` and then calls `Self::partition_pending`.
2423    /// See the documentation for the latter for details.
2424    fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2425        log::trace!("exit {instance:?}");
2426        self.instance_state(instance)
2427            .concurrent_state()
2428            .do_not_enter = false;
2429        self.partition_pending(instance)
2430    }
2431
2432    /// Iterate over `InstanceState::pending`, moving any ready items into the
2433    /// "high priority" work item queue.
2434    ///
2435    /// Also, notify `ConcurrentState::ready_for_concurrent_call_waker` if
2436    /// present.
2437    ///
2438    /// See `GuestCall::is_ready` for details.
2439    fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2440        let mut any_ready = false;
2441        for (thread, kind) in
2442            mem::take(&mut self.instance_state(instance).concurrent_state().pending)
2443        {
2444            let call = GuestCall { thread, kind };
2445            if call.is_ready(self)? {
2446                any_ready = true;
2447                self.concurrent_state_mut()?
2448                    .push_high_priority(WorkItem::GuestCall { instance, call });
2449            } else {
2450                self.instance_state(instance)
2451                    .concurrent_state()
2452                    .pending
2453                    .insert(call.thread, call.kind);
2454            }
2455        }
2456
2457        if any_ready {
2458            if let Some(waker) = self
2459                .concurrent_state_mut()?
2460                .ready_for_concurrent_call_waker
2461                .take()
2462            {
2463                waker.wake();
2464            }
2465        }
2466
2467        Ok(())
2468    }
2469
2470    /// Implements the `backpressure.{inc,dec}` intrinsics.
2471    pub(crate) fn backpressure_modify(
2472        &mut self,
2473        caller_instance: RuntimeInstance,
2474        modify: impl FnOnce(u16) -> Option<u16>,
2475    ) -> Result<()> {
2476        let state = self.instance_state(caller_instance).concurrent_state();
2477        let old = state.backpressure;
2478        let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2479        state.backpressure = new;
2480
2481        log::trace!("backpressure update for {caller_instance:?} from {old} to {new}");
2482
2483        if old > 0 && new == 0 {
2484            // Backpressure was previously enabled and is now disabled; move any
2485            // newly-eligible guest calls to the "high priority" queue.
2486            self.partition_pending(caller_instance)?;
2487        }
2488
2489        Ok(())
2490    }
2491
2492    /// Resume the specified fiber, giving it exclusive access to the specified
2493    /// store.
2494    fn resume_fiber(
2495        &mut self,
2496        fiber: StoreFiber<'static>,
2497    ) -> impl Future<Output = Result<()>> + '_ {
2498        let mut resuming = ResumingFiber {
2499            store: self,
2500            fiber: Some(fiber),
2501        };
2502        async move {
2503            let old_thread = resuming.store.current_thread()?;
2504            log::trace!("resume_fiber: save current thread {old_thread:?}");
2505
2506            let fiber = resuming.fiber.take().unwrap();
2507            resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2508
2509            resuming.store.set_thread(old_thread)?;
2510
2511            let state = resuming.store.concurrent_state_mut()?;
2512
2513            if let Some(ot) = old_thread.guest() {
2514                state.set_thread_running(ot.thread)?;
2515            }
2516            log::trace!("resume_fiber: restore current thread {old_thread:?}");
2517
2518            if resuming.fiber.is_some() {
2519                log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2520                // See the `SuspendReason` documentation for what each case means.
2521                let reason = match state.suspend_reason.take() {
2522                    Some(r) => r,
2523                    None => bail_bug!("suspend reason missing when resuming fiber"),
2524                };
2525                match reason {
2526                    SuspendReason::NeedWork => {
2527                        if state.worker.is_none() {
2528                            state.worker = resuming.fiber.take();
2529                        }
2530                    }
2531                    SuspendReason::Yielding { thread } => {
2532                        let slot = &mut state.get_mut(thread.thread)?.state;
2533                        slot.check_no_fiber()?;
2534                        *slot = GuestThreadState::Ready {
2535                            fiber: resuming.fiber.take().unwrap(),
2536                        };
2537                        let instance = state.get_mut(thread.task)?.instance;
2538                        state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2539                    }
2540                    SuspendReason::ExplicitlySuspending { thread } => {
2541                        let slot = &mut state.get_mut(thread.thread)?.state;
2542                        slot.check_no_fiber()?;
2543                        *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2544                    }
2545                    SuspendReason::Waiting { set, thread } => {
2546                        match state.get_mut(set)?.waiting.entry(thread) {
2547                            Entry::Vacant(entry) => {
2548                                entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2549                            }
2550                            Entry::Occupied(_) => {
2551                                bail_bug!("entry unexpectedly already exists for {thread:?}");
2552                            }
2553                        }
2554                    }
2555                    SuspendReason::YieldingToSubtask { thread } => {
2556                        // In this case, the thread has either invoked or sent a
2557                        // cancel request to a subtask, and is now yielding to that
2558                        // subtask.  According to the CM spec, that subtask may only
2559                        // yield back to the original thread the first time it
2560                        // suspends or exits (or a thread that it has resumed
2561                        // suspends or exits, etc.), which we ensure by setting
2562                        // `ConcurrentState::next_switch_item` here.
2563
2564                        let instance = state.get_mut(thread.task)?.instance;
2565                        if state.next_switch_item.is_some() {
2566                            // This should be unreachable per the save/restore code
2567                            // in `Self::suspend`.
2568                            bail_bug!(
2569                                "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2570                             a thread wanted to wait on a subtask"
2571                            );
2572                        }
2573                        let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2574                            instance,
2575                            thread,
2576                            fiber: resuming.fiber.take().unwrap(),
2577                        });
2578                        log::trace!("set next switch item to {item:?}");
2579                    }
2580                };
2581            } else {
2582                log::trace!("resume_fiber: fiber has exited");
2583            }
2584
2585            Ok(())
2586        }
2587    }
2588
2589    /// Suspend the current fiber, storing the reason in
2590    /// `ConcurrentState::suspend_reason` to indicate the conditions under which
2591    /// it should be resumed.
2592    ///
2593    /// See the `SuspendReason` documentation for details.
2594    fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2595        log::trace!("suspend fiber: {reason:?}");
2596
2597        // We'll panic if we call `Self::with_blocking` when the fiber is being
2598        // disposed, so check for that and exit ASAP if appropriate.
2599        if !self.fiber_async_state_mut().can_block() {
2600            return Err(format_err!("future dropped"));
2601        }
2602
2603        let state = self.concurrent_state_mut()?;
2604
2605        // If we're yielding or waiting on behalf of a guest thread, we'll be
2606        // overwriting the current thread, so save it now and restore it once
2607        // we've resumed.
2608        //
2609        // Also, if we're yielding to a subtask, we're about to overwrite
2610        // `ConcurrentState::next_switch_item`, so also save and restore that.
2611        let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2612            SuspendReason::Yielding { .. }
2613            | SuspendReason::Waiting { .. }
2614            | SuspendReason::ExplicitlySuspending { .. } => {
2615                // If there's a thread waiting for this subtask to suspend, this
2616                // is a good time to switch back to it.
2617                if state.switch_item.is_none() {
2618                    state.take_next_switch_item()?;
2619                }
2620
2621                (true, false)
2622            }
2623            SuspendReason::YieldingToSubtask { .. } => (true, true),
2624            SuspendReason::NeedWork => (false, false),
2625        };
2626
2627        let old_next_switch_item = if save_and_restore_next_switch_item {
2628            Some(state.save_next_switch_item()?)
2629        } else {
2630            None
2631        };
2632
2633        let old_guest_thread = if save_and_restore_thread {
2634            self.current_thread()?
2635        } else {
2636            CurrentThread::None
2637        };
2638
2639        let waiting_set = match &reason {
2640            SuspendReason::Waiting { set, .. } => Some(*set),
2641            _ => None,
2642        };
2643
2644        let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2645        if suspend_reason.is_some() {
2646            bail_bug!("suspend reason unexpectedly already set for store");
2647        }
2648        *suspend_reason = Some(reason);
2649
2650        // Count this thread as waiting on `waiting_set`, if any, for as long as
2651        // it's suspended.
2652        if let Some(set) = waiting_set {
2653            self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2654        }
2655
2656        self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2657
2658        if let Some(set) = waiting_set {
2659            self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2660        }
2661
2662        if save_and_restore_thread {
2663            self.set_thread(old_guest_thread)?;
2664        }
2665
2666        if let Some(item) = old_next_switch_item {
2667            let state = self.concurrent_state_mut()?;
2668            state.restore_next_switch_item(item)?;
2669        }
2670
2671        Ok(())
2672    }
2673
2674    fn wait_for_event(
2675        &mut self,
2676        caller_instance: RuntimeInstance,
2677        waitable: Waitable,
2678    ) -> Result<()> {
2679        let caller = self.current_guest_thread()?;
2680        let state = self.concurrent_state_mut()?;
2681
2682        waitable.trap_if_in_waitable_set(state)?;
2683
2684        let set = state.get_mut(caller.thread)?.sync_call_set;
2685        waitable.join(state, Some(set))?;
2686
2687        self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2688
2689        self.suspend(SuspendReason::Waiting {
2690            set,
2691            thread: caller,
2692        })?;
2693        let state = self.concurrent_state_mut()?;
2694
2695        waitable.join(state, None)
2696    }
2697
2698    /// Cleans up the data structures backing the `guest_thread` specified,
2699    /// removing it from the internal tables of `runtime_instance` as well.
2700    ///
2701    /// This function is used whenever a guest thread has fully exited and
2702    /// completed. This'll clean up the associated `GuestThread` structure and
2703    /// related resources it contains.
2704    ///
2705    /// Other functionality that this implements is:
2706    ///
2707    /// * This will perform conditional cleanup of the `GuestTask` that owns
2708    ///   this thread if `cleanup_task` is `CleanupTask::Yes`.
2709    /// * If there are no more threads in the `GuestTask` that this thread is
2710    ///   associated with, and if the task hasn't produced a result (e.g. it's not
2711    ///   returned or cancelled), then a trap will be raised that a result
2712    ///   wasn't ever produced.
2713    /// * If this task is finished, meaning the top-level thread exited and
2714    ///   additionally it's been returned or cancelled, then this will handle
2715    ///   management of the store's "active interesting tasks" counter.
2716    ///
2717    /// Effectively this is intended to be a "narrow waist" through which many
2718    /// destruction operations are funneled through.
2719    fn cleanup_thread(
2720        &mut self,
2721        guest_thread: QualifiedThreadId,
2722        runtime_instance: RuntimeInstance,
2723    ) -> Result<()> {
2724        let state = self.concurrent_state_mut()?;
2725        // If we never suspended, we never had a chance to deliver a subtask
2726        // status update, if any, to our caller, so we do that here:
2727        state.take_next_switch_item()?;
2728        let thread_data = state.get_mut(guest_thread.thread)?;
2729        let sync_call_set = thread_data.sync_call_set;
2730        if let Some(guest_id) = thread_data.instance_rep {
2731            self.instance_state(runtime_instance)
2732                .thread_handle_table()
2733                .guest_thread_remove(guest_id)?;
2734        }
2735        let state = self.concurrent_state_mut()?;
2736
2737        // Clean up any pending subtasks in the sync_call_set
2738        for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2739            if let Some(Event::Subtask {
2740                status: Status::Returned | Status::ReturnCancelled,
2741            }) = waitable.common(state)?.event
2742            {
2743                waitable.delete_from(state)?;
2744            }
2745        }
2746
2747        state.delete(guest_thread.thread)?;
2748        state.delete(sync_call_set)?;
2749        let task = state.get_mut(guest_thread.task)?;
2750        task.threads.remove(&guest_thread.thread);
2751
2752        if task.threads.is_empty() && !task.returned_or_cancelled() {
2753            bail!(Trap::NoAsyncResult);
2754        }
2755
2756        if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2757            task.decremented_interesting_task_count = true;
2758
2759            debug_assert!(state.interesting_tasks > 0);
2760            state.interesting_tasks -= 1;
2761            if state.interesting_tasks == 0
2762                && let Some(waker) = state.interesting_tasks_empty_waker.take()
2763            {
2764                waker.wake();
2765            }
2766        }
2767
2768        state.decrement_task_ref_count(guest_thread.task)?;
2769
2770        Ok(())
2771    }
2772
2773    /// Performs cancellation of the `guest_task` specified with the
2774    /// precondition that the task hasn't lowered its parameters.
2775    ///
2776    /// In this situation the task hasn't ever been started meaning it hasn't
2777    /// actually run any wasm code yet. This requires cleaning up metadata such
2778    /// as thread information attached to the task.
2779    ///
2780    /// This is used for task cancellation via `subtask.cancel`, the intrinsic.
2781    fn cancel_guest_subtask_without_lowered_parameters(
2782        &mut self,
2783        caller_instance: RuntimeInstance,
2784        guest_task: TableId<GuestTask>,
2785    ) -> Result<()> {
2786        let concurrent_state = self.concurrent_state_mut()?;
2787        let task = concurrent_state.get_mut(guest_task)?;
2788        assert!(!task.already_lowered_parameters());
2789        // The task is in a `starting` state, meaning it hasn't run at
2790        // all yet.  Here we update its fields to indicate that it is
2791        // ready to delete immediately once `subtask.drop` is called.
2792        task.lower_params = None;
2793        task.lift_result = None;
2794        task.exited = true;
2795        let instance = task.instance;
2796
2797        // Clean up the thread within this task as it's now never going
2798        // to run.
2799        //
2800        // Note that `next_switch_item`, if any, belongs to the canceller and
2801        // not this thread since this thread never ran, so it's saved and
2802        // restored here to prevent `cleanup_thread` from taking it.
2803        assert_eq!(1, task.threads.len());
2804        let thread = *task.threads.iter().next().unwrap();
2805        let next_switch_item = concurrent_state.save_next_switch_item()?;
2806        self.cleanup_thread(
2807            QualifiedThreadId {
2808                task: guest_task,
2809                thread,
2810            },
2811            caller_instance,
2812        )?;
2813        self.concurrent_state_mut()?
2814            .restore_next_switch_item(next_switch_item)?;
2815
2816        // Not yet started; cancel and remove from pending
2817        let pending = &mut self.instance_state(instance).concurrent_state().pending;
2818        let pending_count = pending.len();
2819        let mut decrement = 0;
2820        let mut consume = |kind: &GuestCallKind| {
2821            if let GuestCallKind::StartImplicit {
2822                was_pending: true, ..
2823            } = kind
2824            {
2825                decrement += 1;
2826            }
2827        };
2828        pending.retain(|thread, kind| {
2829            if thread.task == guest_task {
2830                consume(kind);
2831                false
2832            } else {
2833                true
2834            }
2835        });
2836
2837        if pending.len() == pending_count {
2838            // If it wasn't found in the pending set, it might be waiting in one
2839            // of the task queues or switch items:
2840            let mut matches = |item: &WorkItem| match item {
2841                WorkItem::ResumeThread { thread, .. } => thread.task == guest_task,
2842                WorkItem::GuestCall {
2843                    call: GuestCall { thread, kind },
2844                    ..
2845                } => {
2846                    if thread.task == guest_task {
2847                        consume(kind);
2848                        true
2849                    } else {
2850                        false
2851                    }
2852                }
2853                _ => false,
2854            };
2855
2856            let mut remove = |queue: &mut VecDeque<WorkItem>| {
2857                let count = queue.len();
2858                queue.retain(|item| !matches(item));
2859                count != queue.len()
2860            };
2861
2862            let state = self.concurrent_state_mut()?;
2863            let mut found = remove(&mut state.high_priority);
2864            found |= remove(&mut state.low_priority);
2865
2866            for item in [&mut state.switch_item, &mut state.next_switch_item] {
2867                if matches!(item, Some(item) if matches(item)) {
2868                    found = true;
2869                    *item = None;
2870                }
2871            }
2872
2873            if !found {
2874                bail!(Trap::SubtaskCancelAfterTerminal);
2875            }
2876        }
2877
2878        self.decrement_pending_start_count(instance, decrement)?;
2879
2880        Ok(())
2881    }
2882
2883    fn decrement_pending_start_count(
2884        &mut self,
2885        instance: RuntimeInstance,
2886        decrement: usize,
2887    ) -> Result<()> {
2888        if decrement > 0 {
2889            let count = &mut self
2890                .instance_state(instance)
2891                .concurrent_state()
2892                .pending_start_count;
2893
2894            if *count < decrement {
2895                bail_bug!("`ConcurrentInstantState::pending_start_count` underflow");
2896            }
2897
2898            *count -= decrement;
2899
2900            // Give the next tasks in line, if any, a chance to start:
2901            self.partition_pending(instance)?;
2902        }
2903
2904        Ok(())
2905    }
2906
2907    /// Used by `ResourceTables` to record the scope of a borrow to get undone
2908    /// in the future.
2909    pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2910        if !self.concurrency_support() {
2911            return Ok(self
2912                .current_scope_id_not_concurrent()?
2913                .map(|id| CurrentScope::Id(VMScope::id(id))));
2914        }
2915
2916        Ok(match self.current_thread()? {
2917            CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2918            CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2919            CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2920            CurrentThread::None => return Ok(None),
2921        })
2922    }
2923
2924    pub(crate) fn queue_task(
2925        &mut self,
2926        task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2927    ) -> Result<()> {
2928        self.concurrent_state_mut()?
2929            .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2930        Ok(())
2931    }
2932
2933    /// Used in `poll_until` just prior to trapping due to a "deadlock"
2934    /// condition.
2935    ///
2936    /// This helps us distinguish between a simple deadlock condition (where no
2937    /// work is available for the event loop to do, nor is there any way for new
2938    /// work to be added) and a "cannot block sync task" condition where at
2939    /// least once instance has an outstanding sync-typed task running, in which
2940    /// case we'll trap with a different error message.
2941    fn any_may_not_suspend(&mut self) -> Result<bool> {
2942        // Note that this currently requires a linear search across the whole
2943        // `ConcurrentState::table`.  We _could_ optimize that, but since (1)
2944        // this function is only used when trapping, (2) the only thing you can
2945        // really do with a store that's been poisoned by a trap is drop it, and
2946        // (3) we must do a linear search through the table when dropping a
2947        // store anyway to dispose of fibers, it's reasonable for us to also do
2948        // a linear search here.
2949        Ok(self
2950            .concurrent_state_mut()?
2951            .table
2952            .get_mut()
2953            .iter_mut()
2954            .filter_map(|(_, entry)| {
2955                if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2956                    Some(task.instance)
2957                } else {
2958                    None
2959                }
2960            })
2961            .collect::<Vec<_>>()
2962            .into_iter()
2963            .any(|instance| {
2964                self.instance_state(instance)
2965                    .concurrent_state()
2966                    .do_not_suspend
2967            }))
2968    }
2969}
2970
2971impl Instance {
2972    /// Get the next pending event for the specified task and (optional)
2973    /// waitable set, along with the waitable handle if applicable.
2974    fn get_event(
2975        self,
2976        store: &mut StoreOpaque,
2977        guest_task: TableId<GuestTask>,
2978        set: Option<TableId<WaitableSet>>,
2979        cancellable: bool,
2980    ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
2981        let state = store.concurrent_state_mut()?;
2982
2983        let task = state.get_mut(guest_task)?;
2984        let event = &mut task.event;
2985        if let Some(ev) = event
2986            && (cancellable || !matches!(ev, Event::Cancelled))
2987        {
2988            log::trace!("deliver event {ev:?} to {guest_task:?}");
2989
2990            if matches!(ev, Event::Cancelled) {
2991                task.cancel_request_delivered = true;
2992            }
2993
2994            let ev = *ev;
2995            *event = None;
2996            return Ok(Some((ev, None)));
2997        }
2998
2999        let set = match set {
3000            Some(set) => set,
3001            None => return Ok(None),
3002        };
3003        let waitable = match state.get_mut(set)?.ready.pop_first() {
3004            Some(v) => v,
3005            None => return Ok(None),
3006        };
3007
3008        let common = waitable.common(state)?;
3009        let handle = match common.handle {
3010            Some(h) => h,
3011            None => bail_bug!("handle not set when delivering event"),
3012        };
3013        let event = match common.event.take() {
3014            Some(e) => e,
3015            None => bail_bug!("event not set when delivering event"),
3016        };
3017
3018        log::trace!(
3019            "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
3020        );
3021
3022        waitable.on_delivery(store, self, event)?;
3023
3024        Ok(Some((event, Some((waitable, handle)))))
3025    }
3026
3027    /// Run `call` for the callback-lifted `guest_thread` and then handle the
3028    /// `CallbackCode` it returns.
3029    ///
3030    /// `call` runs with `guest_thread` as the current thread and with
3031    /// `runtime_instance` entered; both are restored before the final code is
3032    /// handled.
3033    fn handle_callback_code(
3034        self,
3035        store: &mut dyn VMStore,
3036        guest_thread: QualifiedThreadId,
3037        runtime_instance: RuntimeInstance,
3038        call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
3039    ) -> Result<()> {
3040        let old_thread = store.set_thread(guest_thread)?;
3041        log::trace!(
3042            "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
3043        );
3044
3045        store.enter_instance(runtime_instance);
3046
3047        let mut code = call(store)?;
3048
3049        // Implement the spec behavior here of effectively ignoring the return
3050        // code if there's a pending cancellation for this task available.
3051        //
3052        // TODO: the spec doesn't validate anything here, but seems like the
3053        // waitable set may want to be validated?
3054        if matches!(
3055            unpack_callback_code(code).0,
3056            callback_code::YIELD | callback_code::WAIT
3057        ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
3058        {
3059            let (unpacked, set) = unpack_callback_code(code);
3060            log::trace!(
3061                "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
3062                 cancellation; delivering it via callback"
3063            );
3064            code = invoke_callback(store, guest_thread.task, event, 0)?;
3065        }
3066
3067        store.exit_instance(runtime_instance)?;
3068
3069        store.set_thread(old_thread)?;
3070        log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
3071
3072        let store = store.store_opaque_mut();
3073        let (code, set) = unpack_callback_code(code);
3074
3075        log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
3076
3077        let state = store.concurrent_state_mut()?;
3078
3079        state.take_next_switch_item()?;
3080
3081        let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
3082            let set = store
3083                .instance_state(runtime_instance)
3084                .handle_table()
3085                .waitable_set_rep(handle)?;
3086
3087            Ok(TableId::<WaitableSet>::new(set))
3088        };
3089
3090        match code {
3091            callback_code::EXIT => {
3092                log::trace!("implicit thread {guest_thread:?} completed");
3093                let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3094                task.exited = true;
3095                task.callback = None;
3096
3097                // Since we're async-typed, returning control to our caller
3098                // won't help resolve any outstanding sync-typed call which
3099                // might be in progress, so we may need to switch or trap before
3100                // exiting this thread:
3101                store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
3102
3103                store.cleanup_thread(guest_thread, runtime_instance)?;
3104            }
3105            callback_code::YIELD => {
3106                // Set `GuestTask::wake_on_cancel` to allow `subtask.cancel` to
3107                // promote this thread if appropriate.
3108                let old = state
3109                    .get_mut(guest_thread.thread)?
3110                    .wake_on_cancel
3111                    .replace(WakeOnCancel::Yielding);
3112                if !old.is_none() {
3113                    bail_bug!("thread unexpectedly had wake_on_cancel set");
3114                }
3115
3116                // If an `Event::Cancelled` is pending in `GuestTask::event`
3117                // when this call is delivered, we'll deliver that; otherwise,
3118                // we'll deliver `Event::None`.  Note that the `Event::None` is
3119                // deliberately not stored in `GuestTask::event` since that's
3120                // shared by all threads of the task and the `Event::None` is
3121                // only for this thread.
3122                let call = GuestCall {
3123                    thread: guest_thread,
3124                    kind: GuestCallKind::DeliverEvent {
3125                        instance: self,
3126                        set: None,
3127                    },
3128                };
3129                // Push this thread onto the "low priority" queue so it runs
3130                // after any other threads have had a chance to run.
3131                state.push_low_priority(WorkItem::GuestCall {
3132                    instance: runtime_instance,
3133                    call,
3134                });
3135            }
3136            callback_code::WAIT => {
3137                let set = get_set(store, set)?;
3138                self.wait_with_callback(store, guest_thread, set)?;
3139            }
3140            _ => bail!(Trap::UnsupportedCallbackCode),
3141        }
3142
3143        Ok(())
3144    }
3145
3146    /// Start waiting on `set` with `guest_thread`, which uses a callback.
3147    ///
3148    /// The thread will receive an event from `set` via a
3149    /// `GuestCallKind::DeliverEvent` work item, either immediately if one is
3150    /// already available, or once one is published.
3151    fn wait_with_callback(
3152        self,
3153        store: &mut StoreOpaque,
3154        guest_thread: QualifiedThreadId,
3155        set: TableId<WaitableSet>,
3156    ) -> Result<()> {
3157        let state = store.concurrent_state_mut()?;
3158        // This thread's wait ends when that work item is handled in
3159        // `handle_guest_call`.
3160        state.get_mut(set)?.num_waiting += 1;
3161
3162        if state.get_mut(guest_thread.task)?.event.is_some()
3163            || !state.get_mut(set)?.ready.is_empty()
3164        {
3165            // An event is immediately available; deliver it ASAP.
3166            let instance = state.get_mut(guest_thread.task)?.instance;
3167            state.push_high_priority(WorkItem::GuestCall {
3168                instance,
3169                call: GuestCall {
3170                    thread: guest_thread,
3171                    kind: GuestCallKind::DeliverEvent {
3172                        instance: self,
3173                        set: Some(set),
3174                    },
3175                },
3176            });
3177            return Ok(());
3178        }
3179
3180        // No event is immediately available, so this thread is about to block.
3181        // Like any other suspension point (e.g. the callback `EXIT` path), if
3182        // a sync-typed call is in progress for this instance we must first
3183        // switch to another ready thread of that instance, or trap if there is
3184        // none.
3185        let instance = store
3186            .concurrent_state_mut()?
3187            .get_mut(guest_thread.task)?
3188            .instance;
3189        store.switch_or_trap_if_may_not_suspend(instance)?;
3190
3191        // Register to be woken up when an event is published for this waitable
3192        // set.
3193        //
3194        // Here we also set `GuestTask::wake_on_cancel` which allows
3195        // `subtask.cancel` to interrupt the wait.
3196        let state = store.concurrent_state_mut()?;
3197        let old = state
3198            .get_mut(guest_thread.thread)?
3199            .wake_on_cancel
3200            .replace(WakeOnCancel::Waiting(set));
3201        if !old.is_none() {
3202            bail_bug!("thread unexpectedly had wake_on_cancel set");
3203        }
3204        let old = state
3205            .get_mut(set)?
3206            .waiting
3207            .insert(guest_thread, WaitMode::Callback(self));
3208        if !old.is_none() {
3209            bail_bug!("set's waiting set already had this thread registered");
3210        }
3211        Ok(())
3212    }
3213
3214    /// Stage the specified guest call as the work item to run next, to be
3215    /// started as soon as backpressure and/or reentrance rules allow.
3216    ///
3217    /// SAFETY: The raw pointer arguments must be valid references to guest
3218    /// functions (with the appropriate signatures) when the closures staged by
3219    /// this function are called.
3220    unsafe fn stage_call<T: 'static>(
3221        self,
3222        mut store: StoreContextMut<T>,
3223        guest_thread: QualifiedThreadId,
3224        callee: SendSyncPtr<VMFuncRef>,
3225        param_count: usize,
3226        result_count: usize,
3227        async_: bool,
3228        callback: Option<SendSyncPtr<VMFuncRef>>,
3229        post_return: Option<SendSyncPtr<VMFuncRef>>,
3230        host_caller: bool,
3231    ) -> Result<()> {
3232        /// Return a closure which will call the specified function in the scope
3233        /// of the specified task.
3234        ///
3235        /// This will use `GuestTask::lower_params` to lower the parameters, but
3236        /// will not lift the result; instead, it returns a
3237        /// `[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]` from which the result, if
3238        /// any, may be lifted.  Note that an async-lifted export will have
3239        /// returned its result using the `task.return` intrinsic (or not
3240        /// returned a result at all, in the case of `task.cancel`), in which
3241        /// case the "result" of this call will either be a callback code or
3242        /// nothing.
3243        ///
3244        /// SAFETY: `callee` must be a valid `*mut VMFuncRef` at the time when
3245        /// the returned closure is called.
3246        unsafe fn make_call<T: 'static>(
3247            store: StoreContextMut<T>,
3248            guest_thread: QualifiedThreadId,
3249            callee: SendSyncPtr<VMFuncRef>,
3250            param_count: usize,
3251            result_count: usize,
3252        ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3253        + Send
3254        + Sync
3255        + 'static
3256        + use<T> {
3257            let token = StoreToken::new(store);
3258            move |store: &mut dyn VMStore| {
3259                let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3260
3261                store
3262                    .concurrent_state_mut()?
3263                    .set_thread_running(guest_thread.thread)?;
3264                let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3265                let lower = match task.lower_params.take() {
3266                    Some(l) => l,
3267                    None => bail_bug!("lower_params missing"),
3268                };
3269
3270                lower(store, &mut storage[..param_count])?;
3271
3272                let mut store = token.as_context_mut(store);
3273
3274                // SAFETY: Per the contract documented in `make_call's`
3275                // documentation, `callee` must be a valid pointer.
3276                unsafe {
3277                    crate::Func::call_unchecked_raw(
3278                        &mut store,
3279                        callee.as_non_null(),
3280                        NonNull::new(
3281                            &mut storage[..param_count.max(result_count)]
3282                                as *mut [MaybeUninit<ValRaw>] as _,
3283                        )
3284                        .unwrap(),
3285                        UncaughtException::Trap,
3286                    )?;
3287                }
3288
3289                Ok(storage)
3290            }
3291        }
3292
3293        // SAFETY: Per the contract described in this function documentation,
3294        // the `callee` pointer which `call` closes over must be valid when
3295        // called by the closure we queue below.
3296        let call = unsafe {
3297            make_call(
3298                store.as_context_mut(),
3299                guest_thread,
3300                callee,
3301                param_count,
3302                result_count,
3303            )
3304        };
3305
3306        let callee_instance = store
3307            .0
3308            .concurrent_state_mut()?
3309            .get_mut(guest_thread.task)?
3310            .instance;
3311
3312        let fun = if callback.is_some() {
3313            assert!(async_);
3314
3315            Box::new(move |store: &mut dyn VMStore| {
3316                self.add_guest_thread_to_instance_table(
3317                    guest_thread.thread,
3318                    store,
3319                    callee_instance.index,
3320                )?;
3321                self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3322                    let storage = call(store)?;
3323
3324                    // SAFETY: `wasmparser` will have validated that the
3325                    // callback function returns a `i32` result.
3326                    Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3327                })
3328            }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3329        } else {
3330            let token = StoreToken::new(store.as_context_mut());
3331            Box::new(move |store: &mut dyn VMStore| {
3332                self.add_guest_thread_to_instance_table(
3333                    guest_thread.thread,
3334                    store,
3335                    callee_instance.index,
3336                )?;
3337                let old_thread = store.set_thread(guest_thread)?;
3338                log::trace!(
3339                    "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3340                );
3341                let flags = self.id().get(store).instance_flags(callee_instance.index);
3342
3343                let callee_async_typed = store
3344                    .concurrent_state_mut()?
3345                    .get_mut(guest_thread.task)?
3346                    .async_typed;
3347
3348                // Unless this is a callback-less (i.e. stackful) async-lifted
3349                // or sync-typed export, we need to record that the instance
3350                // cannot be entered until the call returns.
3351                if !async_ && callee_async_typed {
3352                    store.enter_instance(callee_instance);
3353                }
3354
3355                if !callee_async_typed {
3356                    store.enter_sync_call(callee_instance)?;
3357                }
3358
3359                // SAFETY: See the documentation for `make_call` to review the
3360                // contract we must uphold for `call` here.
3361                //
3362                // Per the contract described in the `stage_call`
3363                // documentation, the `callee` pointer which `call` closes
3364                // over must be valid.
3365                let storage = call(store)?;
3366
3367                if !callee_async_typed {
3368                    store.exit_sync_call(callee_instance)?;
3369                }
3370
3371                if !async_ {
3372                    // This is a sync-lifted export, so now is when we lift the
3373                    // result, optionally call the post-return function, if any,
3374                    // and finally notify any current or future waiters that the
3375                    // subtask has returned.
3376
3377                    if callee_async_typed {
3378                        store.exit_instance(callee_instance)?;
3379                    }
3380
3381                    let lift = {
3382                        let state = store.concurrent_state_mut()?;
3383                        if !state.get_mut(guest_thread.task)?.result.is_none() {
3384                            bail_bug!("task has already produced a result");
3385                        }
3386
3387                        match state.get_mut(guest_thread.task)?.lift_result.take() {
3388                            Some(lift) => lift,
3389                            None => bail_bug!("lift_result field is missing"),
3390                        }
3391                    };
3392
3393                    // SAFETY: `result_count` represents the number of core Wasm
3394                    // results returned, per `wasmparser`.
3395                    let result = (lift.lift)(store, unsafe {
3396                        mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3397                            &storage[..result_count],
3398                        )
3399                    })?;
3400
3401                    let post_return_arg = match result_count {
3402                        0 => ValRaw::i32(0),
3403                        // SAFETY: `result_count` represents the number of
3404                        // core Wasm results returned, per `wasmparser`.
3405                        1 => unsafe { storage[0].assume_init() },
3406                        _ => unreachable!(),
3407                    };
3408
3409                    unsafe {
3410                        call_post_return(
3411                            token.as_context_mut(store),
3412                            post_return.map(|v| v.as_non_null()),
3413                            post_return_arg,
3414                            flags,
3415                        )?;
3416                    }
3417
3418                    self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3419                }
3420
3421                store.set_thread(old_thread)?;
3422
3423                store
3424                    .concurrent_state_mut()?
3425                    .get_mut(guest_thread.task)?
3426                    .exited = true;
3427
3428                log::trace!(
3429                    "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3430                );
3431
3432                if callee_async_typed {
3433                    // If we're async-typed, returning control to our caller
3434                    // won't help resolve any outstanding sync-typed call which
3435                    // might be in progress, so we may need to switch or trap
3436                    // before exiting this thread:
3437                    store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3438                }
3439
3440                // This is a callback-less call, so the implicit thread has now completed
3441                store.cleanup_thread(guest_thread, callee_instance)?;
3442                Ok(())
3443            })
3444        };
3445
3446        store.0.concurrent_state_mut()?.push_work_item(
3447            WorkItem::GuestCall {
3448                instance: callee_instance,
3449                call: GuestCall {
3450                    thread: guest_thread,
3451                    kind: GuestCallKind::StartImplicit {
3452                        closure: fun,
3453                        was_pending: false,
3454                    },
3455                },
3456            },
3457            if host_caller {
3458                Priority::High
3459            } else {
3460                Priority::Switch
3461            },
3462        )?;
3463
3464        Ok(())
3465    }
3466
3467    /// Prepare (but do not start) a guest->guest call.
3468    ///
3469    /// This is called from fused adapter code generated in
3470    /// `wasmtime_environ::fact::trampoline::Compiler`.  `start` and `return_`
3471    /// are synthesized Wasm functions which move the parameters from the caller
3472    /// to the callee and the result from the callee to the caller,
3473    /// respectively.  The adapter will call `Self::start_call` immediately
3474    /// after calling this function.
3475    ///
3476    /// SAFETY: All the pointer arguments must be valid pointers to guest
3477    /// entities (and with the expected signatures for the function references
3478    /// -- see `wasmtime_environ::fact::trampoline::Compiler` for details).
3479    unsafe fn prepare_call<T: 'static>(
3480        self,
3481        mut store: StoreContextMut<T>,
3482        start: NonNull<VMFuncRef>,
3483        return_: NonNull<VMFuncRef>,
3484        caller_instance: RuntimeComponentInstanceIndex,
3485        callee_instance: RuntimeComponentInstanceIndex,
3486        task_return_type: TypeTupleIndex,
3487        callee_async_typed: bool,
3488        memory: *mut VMMemoryDefinition,
3489        string_encoding: StringEncoding,
3490        caller_info: CallerInfo,
3491    ) -> Result<()> {
3492        enum ResultInfo {
3493            Heap { results: u32 },
3494            Stack { result_count: u32 },
3495        }
3496
3497        let result_info = match &caller_info {
3498            CallerInfo::Async {
3499                has_result: true,
3500                params,
3501            } => ResultInfo::Heap {
3502                // FIXME(#4311): needs to read an i64 for memory64
3503                results: match params.last() {
3504                    Some(r) => r.get_u32(),
3505                    None => bail_bug!("retptr missing"),
3506                },
3507            },
3508            CallerInfo::Async {
3509                has_result: false, ..
3510            } => ResultInfo::Stack { result_count: 0 },
3511            CallerInfo::Sync {
3512                result_count,
3513                params,
3514            } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3515                // FIXME(#4311): needs to read an i64 for memory64
3516                results: match params.last() {
3517                    Some(r) => r.get_u32(),
3518                    None => bail_bug!("arg ptr missing"),
3519                },
3520            },
3521            CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3522                result_count: *result_count,
3523            },
3524        };
3525
3526        let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3527
3528        // Create a new guest task for the call, closing over the `start` and
3529        // `return_` functions to lift the parameters and lower the result,
3530        // respectively.
3531        let start = SendSyncPtr::new(start);
3532        let return_ = SendSyncPtr::new(return_);
3533        let token = StoreToken::new(store.as_context_mut());
3534        let old_thread = store.0.current_guest_thread()?;
3535
3536        let state = store.0.concurrent_state_mut()?;
3537
3538        debug_assert_eq!(
3539            state.get_mut(old_thread.task)?.instance,
3540            self.runtime_instance(caller_instance)
3541        );
3542
3543        let guest_thread = GuestTask::new(
3544            state,
3545            Box::new(move |store, dst| {
3546                let mut store = token.as_context_mut(store);
3547                assert!(dst.len() <= MAX_FLAT_PARAMS);
3548                // The `+ 1` here accounts for the return pointer, if any:
3549                let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3550                let count = match caller_info {
3551                    // Async callers, if they have a result, use the last
3552                    // parameter as a return pointer so chop that off if
3553                    // relevant here.
3554                    CallerInfo::Async { params, has_result } => {
3555                        let params = &params[..params.len() - usize::from(has_result)];
3556                        for (param, src) in params.iter().zip(&mut src) {
3557                            src.write(*param);
3558                        }
3559                        params.len()
3560                    }
3561
3562                    // Sync callers forward everything directly.
3563                    CallerInfo::Sync { params, .. } => {
3564                        for (param, src) in params.iter().zip(&mut src) {
3565                            src.write(*param);
3566                        }
3567                        params.len()
3568                    }
3569                };
3570                // SAFETY: `start` is a valid `*mut VMFuncRef` from
3571                // `wasmtime-cranelift`-generated fused adapter code.  Based on
3572                // how it was constructed (see
3573                // `wasmtime_environ::fact::trampoline::Compiler::compile_async_start_adapter`
3574                // for details) we know it takes count parameters and returns
3575                // `dst.len()` results.
3576                unsafe {
3577                    crate::Func::call_unchecked_raw(
3578                        &mut store,
3579                        start.as_non_null(),
3580                        NonNull::new(
3581                            &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3582                        )
3583                        .unwrap(),
3584                        UncaughtException::Trap,
3585                    )?;
3586                }
3587                dst.copy_from_slice(&src[..dst.len()]);
3588                let task = store.0.current_guest_thread()?.task;
3589                let state = store.0.concurrent_state_mut()?;
3590                Waitable::Guest(task).set_event(
3591                    state,
3592                    Some(Event::Subtask {
3593                        status: Status::Started,
3594                    }),
3595                )?;
3596                Ok(())
3597            }),
3598            LiftResult {
3599                lift: Box::new(move |store, src| {
3600                    // SAFETY: See comment in closure passed as `lower_params`
3601                    // parameter above.
3602                    let mut store = token.as_context_mut(store);
3603                    let mut my_src = src.to_owned(); // TODO: use stack to avoid allocation?
3604                    if let ResultInfo::Heap { results } = &result_info {
3605                        my_src.push(ValRaw::u32(*results));
3606                    }
3607
3608                    // SAFETY: `return_` is a valid `*mut VMFuncRef` from
3609                    // `wasmtime-cranelift`-generated fused adapter code.  Based
3610                    // on how it was constructed (see
3611                    // `wasmtime_environ::fact::trampoline::Compiler::compile_async_return_adapter`
3612                    // for details) we know it takes `src.len()` parameters and
3613                    // returns up to 1 result.
3614                    unsafe {
3615                        crate::Func::call_unchecked_raw(
3616                            &mut store,
3617                            return_.as_non_null(),
3618                            my_src.as_mut_slice().into(),
3619                            UncaughtException::Trap,
3620                        )?;
3621                    }
3622
3623                    let thread = store.0.current_guest_thread()?;
3624                    let state = store.0.concurrent_state_mut()?;
3625                    if sync_caller {
3626                        state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3627                            if let ResultInfo::Stack { result_count } = &result_info {
3628                                match result_count {
3629                                    0 => None,
3630                                    1 => Some(my_src[0]),
3631                                    _ => unreachable!(),
3632                                }
3633                            } else {
3634                                None
3635                            },
3636                        );
3637                    }
3638                    Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3639                }),
3640                ty: task_return_type,
3641                memory: NonNull::new(memory).map(SendSyncPtr::new),
3642                string_encoding,
3643            },
3644            Caller::Guest { thread: old_thread },
3645            None,
3646            self.runtime_instance(callee_instance),
3647            callee_async_typed,
3648            // We don't know whether the callee export was lifted sync or async
3649            // yet, but we'll update this in `start_call`:
3650            false,
3651        )?;
3652
3653        // Make the new thread the current one so that `Self::start_call` knows
3654        // which one to start.
3655        store.0.set_thread(guest_thread)?;
3656        log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3657
3658        Ok(())
3659    }
3660
3661    /// Call the specified callback function for an async-lifted export.
3662    ///
3663    /// SAFETY: `function` must be a valid reference to a guest function of the
3664    /// correct signature for a callback.
3665    unsafe fn call_callback<T>(
3666        self,
3667        mut store: StoreContextMut<T>,
3668        function: SendSyncPtr<VMFuncRef>,
3669        event: Event,
3670        handle: u32,
3671    ) -> Result<u32> {
3672        let (ordinal, result) = event.parts();
3673        let params = &mut [
3674            ValRaw::u32(ordinal),
3675            ValRaw::u32(handle),
3676            ValRaw::u32(result),
3677        ];
3678        // SAFETY: `func` is a valid `*mut VMFuncRef` from either
3679        // `wasmtime-cranelift`-generated fused adapter code or
3680        // `component::Options`.  Per `wasmparser` callback signature
3681        // validation, we know it takes three parameters and returns one.
3682        unsafe {
3683            crate::Func::call_unchecked_raw(
3684                &mut store,
3685                function.as_non_null(),
3686                params.as_mut_slice().into(),
3687                UncaughtException::Trap,
3688            )?;
3689        }
3690        Ok(params[0].get_u32())
3691    }
3692
3693    /// Start a guest->guest call previously prepared using
3694    /// `Self::prepare_call`.
3695    ///
3696    /// This is called from fused adapter code generated in
3697    /// `wasmtime_environ::fact::trampoline::Compiler`.  The adapter will call
3698    /// this function immediately after calling `Self::prepare_call`.
3699    ///
3700    /// If `flags` contains `START_FLAG_ASYNC_CALLER` then the caller used an
3701    /// async-lowered import and the call's packed status is written to
3702    /// `storage[0]`. Otherwise this blocks until the callee has produced its
3703    /// result, if any, which is written to `storage[0]`.
3704    ///
3705    /// SAFETY: The `*mut VMFuncRef` arguments must be valid pointers to guest
3706    /// functions with the appropriate signatures for the current guest task.
3707    /// If this is a call to an async-lowered import, the actual call may be
3708    /// deferred and run after this function returns, in which case the pointer
3709    /// arguments must also be valid when the call happens.
3710    unsafe fn start_call<T: 'static>(
3711        self,
3712        mut store: StoreContextMut<T>,
3713        callback: *mut VMFuncRef,
3714        post_return: *mut VMFuncRef,
3715        callee: NonNull<VMFuncRef>,
3716        param_count: u32,
3717        result_count: u32,
3718        flags: u32,
3719        storage: &mut [MaybeUninit<ValRaw>],
3720    ) -> Result<()> {
3721        let token = StoreToken::new(store.as_context_mut());
3722        let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3723        let guest_thread = store.0.current_guest_thread()?;
3724        let state = store.0.concurrent_state_mut()?;
3725
3726        // FIXME(#14950): this in theory should be a `bail_bug!`, but this can
3727        // be hit through `Func::call` for example. This isn't great because it
3728        // means that `Func::call` is sort of fundamentally incompatible with
3729        // component-model-async...
3730        if !state.event_loop_running {
3731            bail!(
3732                "async function started during an original synchronous \
3733                 entrypoint; please use an async-style entrypoint instead"
3734            );
3735        }
3736
3737        let callee = SendSyncPtr::new(callee);
3738        let param_count = usize::try_from(param_count)?;
3739        assert!(param_count <= MAX_FLAT_PARAMS);
3740        let result_count = usize::try_from(result_count)?;
3741        assert!(result_count <= MAX_FLAT_RESULTS);
3742
3743        let task = state.get_mut(guest_thread.task)?;
3744        let callee_async_typed = task.async_typed;
3745        let callee_instance = task.instance;
3746
3747        task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3748
3749        if let Some(callback) = NonNull::new(callback) {
3750            // We're calling an async-lifted export with a callback, so store
3751            // the callback and related context as part of the task so we can
3752            // call it later when needed.
3753            let callback = SendSyncPtr::new(callback);
3754            task.callback = Some(Box::new(move |store, event, handle| {
3755                let store = token.as_context_mut(store);
3756                unsafe { self.call_callback::<T>(store, callback, event, handle) }
3757            }));
3758        }
3759
3760        let Caller::Guest { thread: caller } = &task.caller else {
3761            // As of this writing, `start_call` is only used for guest->guest
3762            // calls.
3763            bail_bug!("start_call unexpectedly invoked for host->guest call");
3764        };
3765        let caller = *caller;
3766        let caller_instance = state.get_mut(caller.task)?.instance;
3767
3768        // Stage the call as the work item to run next, modulo backpressure, etc.
3769        unsafe {
3770            self.stage_call(
3771                store.as_context_mut(),
3772                guest_thread,
3773                callee,
3774                param_count,
3775                result_count,
3776                (flags & START_FLAG_ASYNC_CALLEE) != 0,
3777                NonNull::new(callback).map(SendSyncPtr::new),
3778                NonNull::new(post_return).map(SendSyncPtr::new),
3779                false,
3780            )?;
3781        }
3782
3783        let old_do_not_suspend = if callee_async_typed {
3784            // If we're starting an async-typed call, it is permitted to suspend
3785            // since that will just return control back to the caller, which may
3786            // be able to avoid blocking if needed.
3787            //
3788            // We'll restore the old value after the call either returns or
3789            // suspends.
3790            let state = store.0.instance_state(callee_instance).concurrent_state();
3791            let old_do_not_suspend = state.do_not_suspend;
3792            state.do_not_suspend = false;
3793            Some(old_do_not_suspend)
3794        } else {
3795            None
3796        };
3797
3798        let state = store.0.concurrent_state_mut()?;
3799
3800        // Use the caller's `GuestThread::sync_call_set` to register interest in
3801        // the subtask...
3802        let guest_waitable = Waitable::Guest(guest_thread.task);
3803        let old_set = guest_waitable.common(state)?.set;
3804        let set = state.get_mut(caller.thread)?.sync_call_set;
3805        guest_waitable.join(state, Some(set))?;
3806
3807        store.0.set_thread(CurrentThread::None)?;
3808
3809        // ... and suspend this fiber temporarily while we wait for it to start.
3810        //
3811        // Note that we _could_ call the callee directly using the current fiber
3812        // rather than suspend this one, but that would make reasoning about the
3813        // event loop more complicated and is probably only worth doing if
3814        // there's a measurable performance benefit.  In addition, it would mean
3815        // blocking the caller if the callee calls a blocking sync-lowered
3816        // import, and as of this writing the spec says we must not do that.
3817        //
3818        // Alternatively, the fused adapter code could be modified to call the
3819        // callee directly without calling a host-provided intrinsic at all (in
3820        // which case it would need to do its own, inline backpressure checks,
3821        // etc.).  Again, we'd want to see a measurable performance benefit
3822        // before committing to such an optimization.  And again, we'd need to
3823        // update the spec to allow that.
3824        let mut yielded = false;
3825        let (status, waitable) = loop {
3826            store.0.suspend(if yielded {
3827                SuspendReason::Waiting {
3828                    set,
3829                    thread: caller,
3830                }
3831            } else {
3832                yielded = true;
3833                SuspendReason::YieldingToSubtask { thread: caller }
3834            })?;
3835
3836            if let Some(old_do_not_suspend) = old_do_not_suspend {
3837                store
3838                    .0
3839                    .instance_state(callee_instance)
3840                    .concurrent_state()
3841                    .do_not_suspend = old_do_not_suspend;
3842            }
3843
3844            let state = store.0.concurrent_state_mut()?;
3845
3846            log::trace!("taking event for {:?}", guest_thread.task);
3847            let event = guest_waitable.take_event(state)?;
3848            let Some(Event::Subtask { status }) = event else {
3849                bail_bug!("subtasks should only get subtask events, got {event:?}")
3850            };
3851
3852            log::trace!("status {status:?} for {:?}", guest_thread.task);
3853
3854            if status == Status::Returned {
3855                // It returned, so we can stop waiting.
3856                break (status, None);
3857            } else if async_caller {
3858                // It hasn't returned yet, but the caller is calling via an
3859                // async-lowered import, so we generate a handle for the task
3860                // waitable and return the status.
3861                let handle = store
3862                    .0
3863                    .handle_table_insertion(caller_instance)
3864                    .subtask_guest(guest_thread.task.rep())?;
3865                store
3866                    .0
3867                    .concurrent_state_mut()?
3868                    .get_mut(guest_thread.task)?
3869                    .common
3870                    .handle = Some(handle);
3871                break (status, Some(handle));
3872            } else {
3873                // The callee hasn't returned yet, and the caller is calling via
3874                // a sync-lowered import, so we loop and keep waiting until the
3875                // callee returns.
3876                store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3877            }
3878        };
3879
3880        guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3881
3882        // Reset the current thread to point to the caller as it resumes control.
3883        store.0.set_thread(caller)?;
3884
3885        store
3886            .0
3887            .concurrent_state_mut()?
3888            .set_thread_running(caller.thread)?;
3889
3890        log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3891
3892        if async_caller {
3893            let Some(slot) = storage.first_mut() else {
3894                bail_bug!("no storage for async call status");
3895            };
3896            *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3897        } else {
3898            // The caller used a sync-lowered import, in which case the result,
3899            // if any, has been stashed in `GuestTask::sync_result`.
3900            let state = store.0.concurrent_state_mut()?;
3901            let task = state.get_mut(guest_thread.task)?;
3902            if let Some(Some(result)) = task.sync_result.take()? {
3903                storage[0] = MaybeUninit::new(result);
3904            }
3905        }
3906
3907        if status == Status::Returned {
3908            store
3909                .0
3910                .concurrent_state_mut()?
3911                .decrement_task_ref_count(guest_thread.task)?;
3912        }
3913
3914        Ok(())
3915    }
3916
3917    /// Poll the specified future once on behalf of a guest->host call using an
3918    /// async-lowered import.
3919    ///
3920    /// If it returns `Ready`, return `Ok(None)`.  Otherwise, if it returns
3921    /// `Pending`, add it to the set of futures to be polled as part of this
3922    /// instance's event loop until it completes, and then return
3923    /// `Ok(Some(handle))` where `handle` is the waitable handle to return.
3924    ///
3925    /// Whether the future returns `Ready` immediately or later, the `lower`
3926    /// function will be used to lower the result, if any, into the guest caller's
3927    /// stack and linear memory. The `lower` function is invoked with the
3928    /// `Option<R>` param `None` if the future is cancelled. The
3929    /// `Option<TableId<HostTask>>` is passed as `Some` if the host task was
3930    /// materialized during execution and allows `lower` to delete the task if
3931    /// needed.
3932    pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3933        self,
3934        mut store: StoreContextMut<'_, T>,
3935        host_task: EnteredHostTask,
3936        result_may_require_realloc: bool,
3937        future: impl Future<Output = Result<R>> + Send + 'static,
3938        lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3939        + Send
3940        + 'static,
3941    ) -> Result<u32> {
3942        let token = StoreToken::new(store.as_context_mut());
3943
3944        // Create an abortable future which hooks calls to poll and manages call
3945        // context state for the future.
3946        let (join_handle, future) = JoinHandle::run(future);
3947        let mut future = Box::pin(future);
3948
3949        // Finally, poll the future.  We can use a dummy `Waker` here because
3950        // we'll add the future to `ConcurrentState::futures` and poll it
3951        // automatically from the event loop if it doesn't complete immediately
3952        // here.
3953        let poll = tls::set(store.0, || {
3954            future
3955                .as_mut()
3956                .poll(&mut Context::from_waker(&Waker::noop()))
3957        });
3958
3959        match poll {
3960            // It finished immediately; lower the result and delete the task.
3961            Poll::Ready(result) => {
3962                let result = result.transpose()?;
3963                // Check if the host task was materialized so that it can be
3964                // deleted in `lower`.
3965                let task = store.0.current_materialized_host_task()?;
3966                lower(store.as_context_mut(), result, true, task)?;
3967                return Ok(Status::Returned.pack(None));
3968            }
3969
3970            // Future isn't ready yet, so fall through.
3971            Poll::Pending => {}
3972        }
3973
3974        // The future will outlive this call frame, so materialize the deferred
3975        // host task and attach its cancellation handle before publishing it to
3976        // the event loop.
3977        let Some(task) = store.0.materialize_host_task_id()? else {
3978            bail_bug!("current thread is not a host thread")
3979        };
3980        {
3981            let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
3982            assert!(matches!(state, HostTaskState::CalleeStarted));
3983            *state = HostTaskState::CalleeRunning(join_handle);
3984        }
3985
3986        // It hasn't finished yet; add the future to
3987        // `ConcurrentState::futures` so it will be polled by the event
3988        // loop and allocate a waitable handle to return to the guest.
3989
3990        // Wrap the future in a closure responsible for lowering the result into
3991        // the guest's stack and memory, as well as notifying any waiters that
3992        // the task returned.
3993        let future = Box::pin(async move {
3994            let result = match run_with_host_task_set(task, future).await? {
3995                Some(result) => Some(result?),
3996                None => None,
3997            };
3998            let on_complete = move |store: &mut dyn VMStore| {
3999                // Restore the `current_thread` to be the host so `lower` knows
4000                // how to manipulate borrows and knows which scope of borrows
4001                // to check.
4002                let mut store = token.as_context_mut(store);
4003                let old = store.0.set_thread(task)?;
4004
4005                let status = if result.is_some() {
4006                    Status::Returned
4007                } else {
4008                    Status::ReturnCancelled
4009                };
4010
4011                lower(store.as_context_mut(), result, false, Some(task))?;
4012                let state = store.0.concurrent_state_mut()?;
4013                match &mut state.get_mut(task)?.state {
4014                    // Cancellation is only complete once lowering has finished
4015                    // and the terminal event can be published.
4016                    pending @ HostTaskState::CalleeCancelling => {
4017                        *pending = HostTaskState::CalleeDone { cancelled: true };
4018                    }
4019
4020                    // Otherwise transition this task to the done state.
4021                    other => *other = HostTaskState::CalleeDone { cancelled: false },
4022                }
4023                Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
4024
4025                store.0.set_thread(old)?;
4026                Ok(())
4027            };
4028
4029            tls::get(move |store| {
4030                if result_may_require_realloc {
4031                    // Lowering may call the guest's realloc function, so run
4032                    // it on a worker fiber. This is required because the guest
4033                    // might try to suspend the fiber which is unsound when it's
4034                    // not actually on a fiber.
4035                    store
4036                        .concurrent_state_mut()?
4037                        .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
4038                            on_complete,
4039                        ))));
4040                    Ok(())
4041                } else {
4042                    // No guest code can be called while lowering this result,
4043                    // so avoid scheduling a worker fiber.
4044                    on_complete(store)
4045                }
4046            })
4047        });
4048
4049        // Make this task visible to the guest and then record what it
4050        // was made visible as.
4051        let caller = match host_task {
4052            Some(caller) => caller,
4053            None => bail_bug!("host task wasn't created but should have been"),
4054        };
4055        let state = store.0.concurrent_state_mut()?;
4056        state.push_future(future);
4057        let instance = state.get_mut(caller.task)?.instance;
4058        let handle = store
4059            .0
4060            .handle_table_insertion(instance)
4061            .subtask_host(task.rep())?;
4062        store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
4063        log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
4064
4065        // Restore the currently running thread to this host task's
4066        // caller. Note that the host task isn't deallocated as it's
4067        // within the store and will get deallocated later.
4068        store.0.set_thread(caller)?;
4069        Ok(Status::Started.pack(Some(handle)))
4070    }
4071
4072    /// Implements the `task.return` intrinsic, lifting the result for the
4073    /// current guest task.
4074    pub(crate) fn task_return(
4075        self,
4076        store: &mut dyn VMStore,
4077        ty: TypeTupleIndex,
4078        options: OptionsIndex,
4079        storage: &[ValRaw],
4080    ) -> Result<()> {
4081        let guest_thread = store.current_guest_thread()?;
4082        let state = store.concurrent_state_mut()?;
4083        if !state.get_mut(guest_thread.task)?.async_lifted {
4084            bail!(Trap::TaskReturnOrCancelSyncLifted);
4085        }
4086        let lift = state
4087            .get_mut(guest_thread.task)?
4088            .lift_result
4089            .take()
4090            .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4091        if !state.get_mut(guest_thread.task)?.result.is_none() {
4092            bail_bug!("task result unexpectedly already set");
4093        }
4094
4095        let CanonicalOptions {
4096            string_encoding,
4097            data_model,
4098            ..
4099        } = &self.id().get(store).component().env_component().options[options];
4100
4101        let invalid = ty != lift.ty
4102            || string_encoding != &lift.string_encoding
4103            || match data_model {
4104                CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
4105                    Some(memory) => {
4106                        let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
4107                        let actual = self.id().get(store).runtime_memory(memory);
4108                        expected != actual.as_ptr()
4109                    }
4110                    // Memory not specified, meaning it didn't need to be
4111                    // specified per validation, so not invalid.
4112                    None => false,
4113                },
4114                // Always invalid as this isn't supported.
4115                CanonicalOptionsDataModel::Gc { .. } => true,
4116            };
4117
4118        if invalid {
4119            bail!(Trap::TaskReturnInvalid);
4120        }
4121
4122        log::trace!("task.return for {guest_thread:?}");
4123
4124        let result = (lift.lift)(store, storage)?;
4125        self.task_complete(store, guest_thread.task, result, Status::Returned)
4126    }
4127
4128    /// Implements the `task.cancel` intrinsic.
4129    pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4130        let guest_thread = store.current_guest_thread()?;
4131        let state = store.concurrent_state_mut()?;
4132        let task = state.get_mut(guest_thread.task)?;
4133        if !task.async_lifted {
4134            bail!(Trap::TaskReturnOrCancelSyncLifted);
4135        }
4136        if !task.cancel_request_delivered {
4137            bail!(Trap::TaskCancelNotCancelled);
4138        }
4139        _ = task
4140            .lift_result
4141            .take()
4142            .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4143
4144        if !task.result.is_none() {
4145            bail_bug!("task result should not bet set yet");
4146        }
4147
4148        log::trace!("task.cancel for {guest_thread:?}");
4149
4150        self.task_complete(
4151            store,
4152            guest_thread.task,
4153            Box::new(DummyResult),
4154            Status::ReturnCancelled,
4155        )
4156    }
4157
4158    /// Complete the specified guest task (i.e. indicate that it has either
4159    /// returned a (possibly empty) result or cancelled itself).
4160    ///
4161    /// This will return any resource borrows and notify any current or future
4162    /// waiters that the task has completed.
4163    fn task_complete(
4164        self,
4165        store: &mut StoreOpaque,
4166        guest_task: TableId<GuestTask>,
4167        result: Box<dyn Any + Send + Sync>,
4168        status: Status,
4169    ) -> Result<()> {
4170        store
4171            .component_resource_tables(Some(self))?
4172            .validate_scope_exit()?;
4173
4174        let state = store.concurrent_state_mut()?;
4175        let task = state.get_mut(guest_task)?;
4176
4177        // A cancellation request is only delivered to a task which hasn't yet
4178        // resolved, so drop any pending, undelivered `Event::Cancelled` now
4179        // that this task has resolved.
4180        task.event = None;
4181
4182        if let Caller::Host { tx, .. } = &mut task.caller {
4183            if let Some(tx) = tx.take() {
4184                _ = tx.send(result);
4185            }
4186        } else {
4187            task.result = Some(result);
4188            Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4189        }
4190
4191        Ok(())
4192    }
4193
4194    /// Implements the `waitable-set.new` intrinsic.
4195    pub(crate) fn waitable_set_new(
4196        self,
4197        store: &mut StoreOpaque,
4198        caller_instance: RuntimeComponentInstanceIndex,
4199    ) -> Result<u32> {
4200        let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4201        let handle = store
4202            .handle_table_insertion(self.runtime_instance(caller_instance))
4203            .waitable_set(set.rep())?;
4204        log::trace!("new waitable set {set:?} (handle {handle})");
4205        Ok(handle)
4206    }
4207
4208    /// Implements the `waitable-set.drop` intrinsic.
4209    pub(crate) fn waitable_set_drop(
4210        self,
4211        store: &mut StoreOpaque,
4212        caller_instance: RuntimeComponentInstanceIndex,
4213        set: u32,
4214    ) -> Result<()> {
4215        let rep = store
4216            .instance_state(self.runtime_instance(caller_instance))
4217            .handle_table()
4218            .waitable_set_remove(set)?;
4219
4220        log::trace!("drop waitable set {rep} (handle {set})");
4221
4222        // Note that we're careful to check for waiters _before_ deleting the
4223        // set to avoid dropping any waiters in `WaitMode::Fiber(_)`, which
4224        // would panic.  See `drop-waitable-set-with-waiters.wast` for details.
4225        let set = store
4226            .concurrent_state_mut()?
4227            .get_mut(TableId::<WaitableSet>::new(rep))?;
4228        if set.num_waiting > 0 {
4229            bail!(Trap::WaitableSetDropHasWaiters);
4230        }
4231
4232        store
4233            .concurrent_state_mut()?
4234            .delete(TableId::<WaitableSet>::new(rep))?;
4235
4236        Ok(())
4237    }
4238
4239    /// Implements the `waitable.join` intrinsic.
4240    pub(crate) fn waitable_join(
4241        self,
4242        store: &mut StoreOpaque,
4243        caller_instance: RuntimeComponentInstanceIndex,
4244        waitable_handle: u32,
4245        set_handle: u32,
4246    ) -> Result<()> {
4247        let mut instance = self.id().get_mut(store);
4248        let waitable =
4249            Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4250
4251        let set = if set_handle == 0 {
4252            None
4253        } else {
4254            let set = instance.instance_states().0[caller_instance]
4255                .handle_table()
4256                .waitable_set_rep(set_handle)?;
4257
4258            Some(TableId::<WaitableSet>::new(set))
4259        };
4260
4261        waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4262
4263        log::trace!(
4264            "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4265        );
4266
4267        waitable.join(store.concurrent_state_mut()?, set)
4268    }
4269
4270    /// Implements the `subtask.drop` intrinsic.
4271    pub(crate) fn subtask_drop(
4272        self,
4273        store: &mut StoreOpaque,
4274        caller_instance: RuntimeComponentInstanceIndex,
4275        task_id: u32,
4276    ) -> Result<()> {
4277        self.waitable_join(store, caller_instance, task_id, 0)?;
4278
4279        let (rep, is_host) = store
4280            .instance_state(self.runtime_instance(caller_instance))
4281            .handle_table()
4282            .subtask_remove(task_id)?;
4283
4284        let concurrent_state = store.concurrent_state_mut()?;
4285        let waitable = if is_host {
4286            let id = TableId::<HostTask>::new(rep);
4287            let task = concurrent_state.get_mut(id)?;
4288            match &task.state {
4289                HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4290                    bail!(Trap::SubtaskDropNotResolved)
4291                }
4292                HostTaskState::CalleeDone { .. } => {}
4293                HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4294                    bail_bug!("invalid state for callee in `subtask.drop`")
4295                }
4296            }
4297
4298            Waitable::Host(id)
4299        } else {
4300            let id = TableId::<GuestTask>::new(rep);
4301            let task = concurrent_state.get_mut(id)?;
4302            if task.lift_result.is_some() {
4303                bail!(Trap::SubtaskDropNotResolved);
4304            }
4305            Waitable::Guest(id)
4306        };
4307
4308        waitable.common(concurrent_state)?.handle = None;
4309
4310        // If this subtask has an event that means that the terminal status of
4311        // this subtask wasn't yet received so it can't be dropped yet.
4312        if waitable.take_event(concurrent_state)?.is_some() {
4313            bail!(Trap::SubtaskDropNotResolved);
4314        }
4315
4316        waitable.delete_from(concurrent_state)?;
4317
4318        log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4319        Ok(())
4320    }
4321
4322    /// Implements the `waitable-set.wait` intrinsic.
4323    pub(crate) fn waitable_set_wait(
4324        self,
4325        store: &mut StoreOpaque,
4326        options: OptionsIndex,
4327        set: u32,
4328        payload: u32,
4329    ) -> Result<u32> {
4330        let &CanonicalOptions {
4331            instance: caller_instance,
4332            ..
4333        } = &self.id().get(store).component().env_component().options[options];
4334        let caller = self.runtime_instance(caller_instance);
4335        let rep = store
4336            .instance_state(self.runtime_instance(caller_instance))
4337            .handle_table()
4338            .waitable_set_rep(set)?;
4339
4340        self.waitable_check(
4341            store,
4342            caller,
4343            WaitableCheck::Wait,
4344            WaitableCheckParams {
4345                set: TableId::new(rep),
4346                options,
4347                payload,
4348            },
4349        )
4350    }
4351
4352    /// Implements the `waitable-set.poll` intrinsic.
4353    pub(crate) fn waitable_set_poll(
4354        self,
4355        store: &mut StoreOpaque,
4356        options: OptionsIndex,
4357        set: u32,
4358        payload: u32,
4359    ) -> Result<u32> {
4360        let &CanonicalOptions {
4361            instance: caller_instance,
4362            ..
4363        } = &self.id().get(store).component().env_component().options[options];
4364        let caller = self.runtime_instance(caller_instance);
4365        let rep = store
4366            .instance_state(caller)
4367            .handle_table()
4368            .waitable_set_rep(set)?;
4369
4370        self.waitable_check(
4371            store,
4372            caller,
4373            WaitableCheck::Poll,
4374            WaitableCheckParams {
4375                set: TableId::new(rep),
4376                options,
4377                payload,
4378            },
4379        )
4380    }
4381
4382    /// Implements the `thread.index` intrinsic.
4383    pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4384        let thread_id = store.current_guest_thread()?.thread;
4385        match store
4386            .concurrent_state_mut()?
4387            .get_mut(thread_id)?
4388            .instance_rep
4389        {
4390            Some(r) => Ok(r),
4391            None => bail_bug!("thread should have instance_rep by now"),
4392        }
4393    }
4394
4395    /// Implements the `thread.new-indirect` intrinsic.
4396    pub(crate) fn thread_new_indirect<T: 'static>(
4397        self,
4398        mut store: StoreContextMut<T>,
4399        runtime_instance: RuntimeComponentInstanceIndex,
4400        start_func_ty: ModuleInternedTypeIndex,
4401        start_func_table_idx: RuntimeTableIndex,
4402        start_func_idx: u32,
4403        context: i32,
4404    ) -> Result<u32> {
4405        log::trace!("creating new thread");
4406
4407        let (instance, registry) = self.id().get_mut_and_registry(store.0);
4408        let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4409        else {
4410            bail_bug!("thread.new-indirect start function type should be registered");
4411        };
4412        let callee = instance
4413            .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4414            .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4415        if callee.type_index(store.0) != start_func_ty {
4416            bail!(Trap::ThreadNewIndirectInvalidType);
4417        }
4418
4419        let token = StoreToken::new(store.as_context_mut());
4420        let start_func = Box::new(
4421            move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4422                let old_thread = store.set_thread(guest_thread)?;
4423                log::trace!(
4424                    "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4425                );
4426
4427                let mut store = token.as_context_mut(store);
4428                let mut params = [ValRaw::i32(context)];
4429                // Use call_unchecked rather than call or call_async, as we don't want to run the function
4430                // on a separate fiber if we're running in an async store.
4431                unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4432
4433                store.0.set_thread(old_thread)?;
4434
4435                let runtime_instance = self.runtime_instance(runtime_instance);
4436
4437                // We're not returning to any caller, so we may need to switch
4438                // or trap if the instance has an outstanding sync-typed call:
4439                store
4440                    .0
4441                    .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4442
4443                store.0.cleanup_thread(guest_thread, runtime_instance)?;
4444
4445                log::trace!("explicit thread {guest_thread:?} completed");
4446                let state = store.0.concurrent_state_mut()?;
4447                if let Some(t) = old_thread.guest() {
4448                    state.set_thread_running(t.thread)?;
4449                }
4450                log::trace!("thread start: restored {old_thread:?} as current thread");
4451
4452                Ok(())
4453            },
4454        );
4455
4456        let current_thread = store.0.current_guest_thread()?;
4457        let state = store.0.concurrent_state_mut()?;
4458        let parent_task = current_thread.task;
4459
4460        let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4461        let thread_id = state.push(new_thread)?;
4462        state.get_mut(parent_task)?.threads.insert(thread_id);
4463
4464        log::trace!("new thread with id {thread_id:?} created");
4465
4466        self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4467    }
4468
4469    pub(crate) fn resume_thread(
4470        self,
4471        store: &mut StoreOpaque,
4472        runtime_instance: RuntimeComponentInstanceIndex,
4473        thread_idx: u32,
4474        how: ResumeThread,
4475    ) -> Result<bool> {
4476        let thread_id =
4477            GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4478        let state = store.concurrent_state_mut()?;
4479        let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4480
4481        if store.current_guest_thread()? == guest_thread {
4482            bail!(Trap::CannotResumeThread);
4483        }
4484
4485        let priority = match how {
4486            ResumeThread::Resume => Priority::Switch,
4487            ResumeThread::ResumeLater => Priority::Low,
4488
4489            // Promotion is a noop unless the thread is ready, so only search
4490            // through pending work items to see if something matches.
4491            ResumeThread::Promote => {
4492                let instance = self.runtime_instance(runtime_instance);
4493                let do_not_enter = store
4494                    .instance_state(instance)
4495                    .concurrent_state()
4496                    .do_not_enter;
4497                return store.concurrent_state_mut()?.promote_work_item_matching(
4498                    |item: &WorkItem| match item {
4499                        WorkItem::ResumeThread { thread, .. }
4500                        | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4501
4502                        WorkItem::GuestCall {
4503                            call: GuestCall { thread, kind },
4504                            ..
4505                        } => {
4506                            *thread == guest_thread
4507                                && match kind {
4508                                    // Events can only be delivered if this
4509                                    // instance's exclusive lock isn't held, so
4510                                    // gate on that.
4511                                    GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4512
4513                                    // A not-yet-started thread made ready via
4514                                    // `thread.resume-later`.
4515                                    GuestCallKind::StartExplicit(_) => true,
4516
4517                                    // Technically shouldn't be possible since
4518                                    // if this hasn't started nothing could get
4519                                    // its id...
4520                                    GuestCallKind::StartImplicit { .. } => false,
4521                                }
4522                        }
4523
4524                        WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4525                    },
4526                );
4527            }
4528        };
4529
4530        let state = store.concurrent_state_mut()?;
4531        let thread = state.get_mut(guest_thread.thread)?;
4532
4533        // When resuming a thread it must be in a suspended state otherwise
4534        // this operation is a trap.
4535        match &thread.state {
4536            GuestThreadState::NotStartedExplicit(_) => {}
4537            GuestThreadState::Suspended(_) => {
4538                log::trace!("resuming thread {thread_id:?} that was suspended");
4539                state.schedule_suspended_thread(
4540                    guest_thread,
4541                    self.runtime_instance(runtime_instance),
4542                    priority,
4543                )?;
4544                return Ok(true);
4545            }
4546            _ => bail!(Trap::CannotResumeThread),
4547        }
4548        match mem::replace(&mut thread.state, GuestThreadState::Running) {
4549            GuestThreadState::NotStartedExplicit(start_func) => {
4550                log::trace!("starting thread {guest_thread:?}");
4551                let guest_call = WorkItem::GuestCall {
4552                    instance: self.runtime_instance(runtime_instance),
4553                    call: GuestCall {
4554                        thread: guest_thread,
4555                        kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4556                            start_func(store, guest_thread)
4557                        })),
4558                    },
4559                };
4560                state.push_work_item(guest_call, priority)?;
4561            }
4562            other => {
4563                thread.state = other;
4564                bail_bug!("thread state checked to be resumable above");
4565            }
4566        }
4567        Ok(true)
4568    }
4569
4570    fn add_guest_thread_to_instance_table(
4571        self,
4572        thread_id: TableId<GuestThread>,
4573        store: &mut StoreOpaque,
4574        runtime_instance: RuntimeComponentInstanceIndex,
4575    ) -> Result<u32> {
4576        let (state, counter) =
4577            store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4578        let guest_id = state
4579            .thread_handle_table()
4580            .guest_thread_insert(counter, thread_id.rep())?;
4581        store
4582            .concurrent_state_mut()?
4583            .get_mut(thread_id)?
4584            .instance_rep = Some(guest_id);
4585        Ok(guest_id)
4586    }
4587
4588    /// Helper function for the `thread.yield`, thread.suspend`,
4589    /// `thread.suspend-then-resume`, `thread.suspend-then-promote`,
4590    /// `thread.yield-then-resume`, and `thread.yield-then-promote` intrinsics.
4591    pub(crate) fn suspension_intrinsic(
4592        self,
4593        store: &mut StoreOpaque,
4594        caller: RuntimeComponentInstanceIndex,
4595        yielding: bool,
4596        to_thread: SuspensionTarget,
4597    ) -> Result<WaitResult> {
4598        let check_suspend = match to_thread {
4599            SuspensionTarget::Promote(thread) => {
4600                !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4601            }
4602            SuspensionTarget::Resume(thread) => {
4603                if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4604                    bail_bug!(
4605                        "`resume_thread` should only ever return false \
4606                         when `ResumeThread::Promote` is passed to it"
4607                    );
4608                }
4609                false
4610            }
4611            SuspensionTarget::None => true,
4612        };
4613
4614        if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4615            return if yielding {
4616                Ok(WaitResult::Completed)
4617            } else {
4618                Err(Trap::CannotBlockSyncTask.into())
4619            };
4620        }
4621
4622        let guest_thread = store.current_guest_thread()?;
4623
4624        let reason = if yielding {
4625            SuspendReason::Yielding {
4626                thread: guest_thread,
4627            }
4628        } else {
4629            SuspendReason::ExplicitlySuspending {
4630                thread: guest_thread,
4631            }
4632        };
4633
4634        store.suspend(reason)?;
4635
4636        Ok(WaitResult::Completed)
4637    }
4638
4639    /// Helper function for the `waitable-set.wait` and `waitable-set.poll` intrinsics.
4640    fn waitable_check(
4641        self,
4642        store: &mut StoreOpaque,
4643        caller: RuntimeInstance,
4644        check: WaitableCheck,
4645        params: WaitableCheckParams,
4646    ) -> Result<u32> {
4647        let guest_thread = store.current_guest_thread()?;
4648
4649        log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4650
4651        // If we're waiting, and there are no events immediately available,
4652        // suspend the fiber until that changes.
4653        match &check {
4654            WaitableCheck::Wait => {
4655                let set = params.set;
4656
4657                // Note that this is a loop since, by the time this fiber is
4658                // resumed after being woken up, another thread may have already
4659                // taken the event (e.g. via `waitable-set.poll`), in which case
4660                // we keep waiting.
4661                loop {
4662                    let state = store.concurrent_state_mut()?;
4663                    let task = state.get_mut(guest_thread.task)?;
4664                    if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4665                        || !state.get_mut(set)?.ready.is_empty()
4666                    {
4667                        break;
4668                    }
4669
4670                    store.switch_or_trap_if_may_not_suspend(caller)?;
4671
4672                    store.suspend(SuspendReason::Waiting {
4673                        set,
4674                        thread: guest_thread,
4675                    })?;
4676                }
4677            }
4678            WaitableCheck::Poll => {}
4679        }
4680
4681        log::trace!(
4682            "waitable check for {guest_thread:?}; set {:?}, part two",
4683            params.set
4684        );
4685
4686        // Deliver any pending events to the guest and return.
4687        let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4688
4689        let (ordinal, handle, result) = match &check {
4690            WaitableCheck::Wait => {
4691                let (event, waitable) = match event {
4692                    Some(p) => p,
4693                    None => bail_bug!("event expected to be present"),
4694                };
4695                let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4696                let (ordinal, result) = event.parts();
4697                (ordinal, handle, result)
4698            }
4699            WaitableCheck::Poll => {
4700                if let Some((event, waitable)) = event {
4701                    let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4702                    let (ordinal, result) = event.parts();
4703                    (ordinal, handle, result)
4704                } else {
4705                    log::trace!(
4706                        "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4707                        guest_thread.task,
4708                        params.set
4709                    );
4710                    let (ordinal, result) = Event::None.parts();
4711                    (ordinal, 0, result)
4712                }
4713            }
4714        };
4715        let ptr = crate::component::func::validate_inbounds_dynamic(
4716            &CanonicalAbiInfo::POINTER_PAIR,
4717            self.options_memory(store, params.options),
4718            &ValRaw::u32(params.payload),
4719        )?;
4720        let memory = self
4721            .options_memory_range_mut(store, params.options, ptr, 8)
4722            .unwrap();
4723        memory[0..4].copy_from_slice(&handle.to_le_bytes());
4724        memory[4..8].copy_from_slice(&result.to_le_bytes());
4725        Ok(ordinal)
4726    }
4727
4728    /// Implements the `subtask.cancel` intrinsic.
4729    pub(crate) fn subtask_cancel(
4730        self,
4731        store: &mut StoreOpaque,
4732        caller_instance: RuntimeComponentInstanceIndex,
4733        async_: bool,
4734        task_id: u32,
4735    ) -> Result<u32> {
4736        let (rep, is_host) = store
4737            .instance_state(self.runtime_instance(caller_instance))
4738            .handle_table()
4739            .subtask_rep(task_id)?;
4740        let waitable = if is_host {
4741            Waitable::Host(TableId::<HostTask>::new(rep))
4742        } else {
4743            Waitable::Guest(TableId::<GuestTask>::new(rep))
4744        };
4745        let concurrent_state = store.concurrent_state_mut()?;
4746
4747        log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4748
4749        waitable.trap_if_in_waitable_set(concurrent_state)?;
4750
4751        let needs_block;
4752        if let Waitable::Host(host_task) = waitable {
4753            let state = &mut concurrent_state.get_mut(host_task)?.state;
4754            match state {
4755                // If the callee is still running, signal an abort is requested.
4756                //
4757                // After cancelling this falls through to block waiting for the
4758                // host task to actually finish assuming that `async_` is false.
4759                // This blocking behavior resolves the race of `handle.abort()`
4760                // with the task actually getting cancelled or finishing.
4761                HostTaskState::CalleeRunning(handle) => {
4762                    handle.abort();
4763                    *state = HostTaskState::CalleeCancelling;
4764                    needs_block = true;
4765                }
4766
4767                // Cancellation was already requested, so fail as the task can't
4768                // be cancelled twice.
4769                HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4770                    bail!(Trap::SubtaskCancelAfterTerminal);
4771                }
4772                HostTaskState::CalleeDone { cancelled: false } => {
4773                    // The callee is already done so there's no need to
4774                    // block further for an event.
4775                    *state = HostTaskState::CalleeDone { cancelled: true };
4776                    needs_block = false;
4777                }
4778
4779                // These states should not be possible for a subtask that's
4780                // visible from the guest, so trap here.
4781                HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4782                    bail_bug!("invalid states for host callee")
4783                }
4784            }
4785        } else {
4786            let guest_task = TableId::<GuestTask>::new(rep);
4787            let task = concurrent_state.get_mut(guest_task)?;
4788
4789            // Cancellation was already requested, so fail as the task can't be
4790            // cancelled twice.
4791            if task.cancel_requested {
4792                bail!(Trap::SubtaskCancelAfterTerminal);
4793            }
4794
4795            if !task.already_lowered_parameters() {
4796                store.cancel_guest_subtask_without_lowered_parameters(
4797                    self.runtime_instance(caller_instance),
4798                    guest_task,
4799                )?;
4800                return Ok(Status::StartCancelled as u32);
4801            } else if !task.returned_or_cancelled() {
4802                // Started, but not yet returned or cancelled; send the
4803                // `CANCELLED` event
4804                //
4805                // Note that this might overwrite an event that was set earlier
4806                // (e.g. `Event::None` if the task is yielding), but that's
4807                // okay -- this should supersede the previous state.
4808                task.cancel_requested = true;
4809                task.event = Some(Event::Cancelled);
4810                let runtime_instance = task.instance;
4811                for thread in task.threads.clone() {
4812                    let thread = QualifiedThreadId {
4813                        task: guest_task,
4814                        thread,
4815                    };
4816                    let concurrent_state = store.concurrent_state_mut()?;
4817                    let thread_mut = concurrent_state.get_mut(thread.thread)?;
4818
4819                    let yield_ = |store: &mut StoreOpaque| {
4820                        // While we're yielding, temporarily set
4821                        // `do_not_suspend` to false on the subtask's instance
4822                        // since we'll be getting control back if it does
4823                        // suspend.
4824                        let state = store.instance_state(runtime_instance).concurrent_state();
4825                        let old_do_not_suspend = state.do_not_suspend;
4826                        state.do_not_suspend = false;
4827
4828                        let caller = store.current_guest_thread()?;
4829
4830                        // Temporarily add the waitable to the caller's
4831                        // `sync_call_set` to ensure that (1) it isn't already
4832                        // part of a different set and (2) it can't be added to
4833                        // a different set while we yield to the subtask.
4834                        let state = store.concurrent_state_mut()?;
4835                        let set = state.get_mut(caller.thread)?.sync_call_set;
4836                        waitable.join(state, Some(set))?;
4837
4838                        store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4839
4840                        let state = store.concurrent_state_mut()?;
4841                        waitable.join(state, None)?;
4842
4843                        store
4844                            .instance_state(runtime_instance)
4845                            .concurrent_state()
4846                            .do_not_suspend = old_do_not_suspend;
4847
4848                        Ok::<(), crate::Error>(())
4849                    };
4850
4851                    match thread_mut.wake_on_cancel.take() {
4852                        WakeOnCancel::Waiting(set) => {
4853                            // The thread is in a cancellable wait, so wake it up:
4854                            let instance = concurrent_state.take_callback_waiter(set, thread)?;
4855                            let item = WorkItem::GuestCall {
4856                                instance: runtime_instance,
4857                                call: GuestCall {
4858                                    thread,
4859                                    kind: GuestCallKind::DeliverEvent {
4860                                        instance,
4861                                        set: Some(set),
4862                                    },
4863                                },
4864                            };
4865                            concurrent_state.set_switch_item(item)?;
4866
4867                            yield_(store)?;
4868
4869                            break;
4870                        }
4871                        WakeOnCancel::Yielding => {
4872                            if concurrent_state.promote_thread_work_item(thread)? {
4873                                yield_(store)?;
4874                                break;
4875                            } else if store
4876                                .instance_state(runtime_instance)
4877                                .concurrent_state()
4878                                .pending
4879                                .contains_key(&thread)
4880                            {
4881                                // The thread's work item was parked in
4882                                // `pending` because its instance can't be
4883                                // entered right now, so the cancellation
4884                                // can't be delivered yet. It'll be delivered
4885                                // via `GuestTask::event` once the instance
4886                                // becomes enterable and the work item runs.
4887                                // Until then the thread is still yielding, so
4888                                // restore `wake_on_cancel` and move on to the
4889                                // next thread.
4890                                store
4891                                    .concurrent_state_mut()?
4892                                    .get_mut(thread.thread)?
4893                                    .wake_on_cancel = WakeOnCancel::Yielding;
4894                            } else {
4895                                bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4896                            }
4897                        }
4898                        WakeOnCancel::None => {}
4899                    }
4900                }
4901
4902                // Guest tasks need to block if they have not yet returned or
4903                // cancelled, even as a result of the event delivery above.
4904                needs_block = !store
4905                    .concurrent_state_mut()?
4906                    .get_mut(guest_task)?
4907                    .returned_or_cancelled()
4908            } else {
4909                needs_block = false;
4910            }
4911        };
4912
4913        // If we need to block waiting on the terminal status of this subtask
4914        // then return immediately in `async` mode, or otherwise wait for the
4915        // event to get signaled through the store.
4916        if needs_block {
4917            if async_ {
4918                return Ok(BLOCKED);
4919            }
4920
4921            // For cancelling host tasks save and later restore
4922            // `next_switch_item` so that our caller (or canceller) isn't
4923            // resumed in the meantime. This is our prerogative as a runtime
4924            // where we get to decide whether cancellation of host things blocks
4925            // or doesn't, and here we just need to get the final status from
4926            // the host side so it shouldn't block.
4927            //
4928            // A guest subtask, however, may genuinely block, so skip this in
4929            // that case.
4930            let old_next_switch_item = if is_host {
4931                Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4932            } else {
4933                None
4934            };
4935
4936            // Wait for this waitable to get signaled with its terminal
4937            // status. Once that's done fall through to the shared code.
4938            store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4939
4940            if let Some(item) = old_next_switch_item {
4941                let state = store.concurrent_state_mut()?;
4942                state.restore_next_switch_item(item)?;
4943            }
4944
4945            // .. fall through to determine what event's in store for us.
4946        }
4947
4948        let event = waitable.take_event(store.concurrent_state_mut()?)?;
4949        if let Some(Event::Subtask {
4950            status: status @ (Status::Returned | Status::ReturnCancelled),
4951        }) = event
4952        {
4953            Ok(status as u32)
4954        } else {
4955            bail!(Trap::SubtaskCancelAfterTerminal);
4956        }
4957    }
4958}
4959
4960/// Trait representing component model ABI async intrinsics and fused adapter
4961/// helper functions.
4962///
4963/// SAFETY (callers): Most of the methods in this trait accept raw pointers,
4964/// which must be valid for at least the duration of the call (and possibly for
4965/// as long as the relevant guest task exists, in the case of `*mut VMFuncRef`
4966/// pointers used for async calls).
4967pub trait VMComponentAsyncStore {
4968    /// A helper function for fused adapter modules involving calls where the
4969    /// one of the caller or callee is async.
4970    ///
4971    /// This helper is not used when the caller and callee both use the sync
4972    /// ABI, only when at least one is async is this used.
4973    unsafe fn prepare_call(
4974        &mut self,
4975        instance: Instance,
4976        memory: *mut VMMemoryDefinition,
4977        start: NonNull<VMFuncRef>,
4978        return_: NonNull<VMFuncRef>,
4979        caller_instance: RuntimeComponentInstanceIndex,
4980        callee_instance: RuntimeComponentInstanceIndex,
4981        task_return_type: TypeTupleIndex,
4982        callee_async: bool,
4983        string_encoding: StringEncoding,
4984        result_count: u32,
4985        storage: *mut ValRaw,
4986        storage_len: usize,
4987    ) -> Result<()>;
4988
4989    /// A helper function for fused adapter modules to start a call previously
4990    /// set up with `prepare_call`.
4991    unsafe fn start_call(
4992        &mut self,
4993        instance: Instance,
4994        callback: *mut VMFuncRef,
4995        post_return: *mut VMFuncRef,
4996        callee: NonNull<VMFuncRef>,
4997        param_count: u32,
4998        result_count: u32,
4999        flags: u32,
5000        storage: *mut MaybeUninit<ValRaw>,
5001        storage_len: usize,
5002    ) -> Result<()>;
5003
5004    /// The `future.write` intrinsic.
5005    fn future_write(
5006        &mut self,
5007        instance: Instance,
5008        caller: RuntimeComponentInstanceIndex,
5009        ty: TypeFutureTableIndex,
5010        options: OptionsIndex,
5011        future: u32,
5012        address: u32,
5013    ) -> Result<u32>;
5014
5015    /// The `future.read` intrinsic.
5016    fn future_read(
5017        &mut self,
5018        instance: Instance,
5019        caller: RuntimeComponentInstanceIndex,
5020        ty: TypeFutureTableIndex,
5021        options: OptionsIndex,
5022        future: u32,
5023        address: u32,
5024    ) -> Result<u32>;
5025
5026    /// The `future.drop-writable` intrinsic.
5027    fn future_drop_writable(
5028        &mut self,
5029        instance: Instance,
5030        ty: TypeFutureTableIndex,
5031        writer: u32,
5032    ) -> Result<()>;
5033
5034    /// The `stream.write` intrinsic.
5035    fn stream_write(
5036        &mut self,
5037        instance: Instance,
5038        caller: RuntimeComponentInstanceIndex,
5039        ty: TypeStreamTableIndex,
5040        options: OptionsIndex,
5041        stream: u32,
5042        address: u32,
5043        count: u32,
5044    ) -> Result<u32>;
5045
5046    /// The `stream.read` intrinsic.
5047    fn stream_read(
5048        &mut self,
5049        instance: Instance,
5050        caller: RuntimeComponentInstanceIndex,
5051        ty: TypeStreamTableIndex,
5052        options: OptionsIndex,
5053        stream: u32,
5054        address: u32,
5055        count: u32,
5056    ) -> Result<u32>;
5057
5058    /// The "fast-path" implementation of the `stream.write` intrinsic for
5059    /// "flat" (i.e. memcpy-able) payloads.
5060    fn flat_stream_write(
5061        &mut self,
5062        instance: Instance,
5063        caller: RuntimeComponentInstanceIndex,
5064        ty: TypeStreamTableIndex,
5065        options: OptionsIndex,
5066        payload_size: u32,
5067        payload_align: u32,
5068        stream: u32,
5069        address: u32,
5070        count: u32,
5071    ) -> Result<u32>;
5072
5073    /// The "fast-path" implementation of the `stream.read` intrinsic for "flat"
5074    /// (i.e. memcpy-able) payloads.
5075    fn flat_stream_read(
5076        &mut self,
5077        instance: Instance,
5078        caller: RuntimeComponentInstanceIndex,
5079        ty: TypeStreamTableIndex,
5080        options: OptionsIndex,
5081        payload_size: u32,
5082        payload_align: u32,
5083        stream: u32,
5084        address: u32,
5085        count: u32,
5086    ) -> Result<u32>;
5087
5088    /// The `stream.drop-writable` intrinsic.
5089    fn stream_drop_writable(
5090        &mut self,
5091        instance: Instance,
5092        ty: TypeStreamTableIndex,
5093        writer: u32,
5094    ) -> Result<()>;
5095
5096    /// The `error-context.debug-message` intrinsic.
5097    fn error_context_debug_message(
5098        &mut self,
5099        instance: Instance,
5100        ty: TypeComponentLocalErrorContextTableIndex,
5101        options: OptionsIndex,
5102        err_ctx_handle: u32,
5103        debug_msg_address: u32,
5104    ) -> Result<()>;
5105
5106    /// The `thread.new-indirect` intrinsic
5107    fn thread_new_indirect(
5108        &mut self,
5109        instance: Instance,
5110        caller: RuntimeComponentInstanceIndex,
5111        func_ty_idx: ModuleInternedTypeIndex,
5112        start_func_table_idx: RuntimeTableIndex,
5113        start_func_idx: u32,
5114        context: i32,
5115    ) -> Result<u32>;
5116}
5117
5118/// SAFETY: See trait docs.
5119impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5120    unsafe fn prepare_call(
5121        &mut self,
5122        instance: Instance,
5123        memory: *mut VMMemoryDefinition,
5124        start: NonNull<VMFuncRef>,
5125        return_: NonNull<VMFuncRef>,
5126        caller_instance: RuntimeComponentInstanceIndex,
5127        callee_instance: RuntimeComponentInstanceIndex,
5128        task_return_type: TypeTupleIndex,
5129        callee_async: bool,
5130        string_encoding: StringEncoding,
5131        result_count_or_max_if_async: u32,
5132        storage: *mut ValRaw,
5133        storage_len: usize,
5134    ) -> Result<()> {
5135        // SAFETY: The `wasmtime_cranelift`-generated code that calls
5136        // this method will have ensured that `storage` is a valid
5137        // pointer containing at least `storage_len` items.
5138        let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5139
5140        unsafe {
5141            instance.prepare_call(
5142                StoreContextMut(self),
5143                start,
5144                return_,
5145                caller_instance,
5146                callee_instance,
5147                task_return_type,
5148                callee_async,
5149                memory,
5150                string_encoding,
5151                match result_count_or_max_if_async {
5152                    PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5153                        params,
5154                        has_result: false,
5155                    },
5156                    PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5157                        params,
5158                        has_result: true,
5159                    },
5160                    result_count => CallerInfo::Sync {
5161                        params,
5162                        result_count,
5163                    },
5164                },
5165            )
5166        }
5167    }
5168
5169    unsafe fn start_call(
5170        &mut self,
5171        instance: Instance,
5172        callback: *mut VMFuncRef,
5173        post_return: *mut VMFuncRef,
5174        callee: NonNull<VMFuncRef>,
5175        param_count: u32,
5176        result_count: u32,
5177        flags: u32,
5178        storage: *mut MaybeUninit<ValRaw>,
5179        storage_len: usize,
5180    ) -> Result<()> {
5181        unsafe {
5182            instance.start_call(
5183                StoreContextMut(self),
5184                callback,
5185                post_return,
5186                callee,
5187                param_count,
5188                result_count,
5189                flags,
5190                // SAFETY: The `wasmtime_cranelift`-generated code that calls
5191                // this method will have ensured that `storage` is a valid
5192                // pointer containing at least `storage_len` items.
5193                core::slice::from_raw_parts_mut(storage, storage_len),
5194            )
5195        }
5196    }
5197
5198    fn future_write(
5199        &mut self,
5200        instance: Instance,
5201        caller: RuntimeComponentInstanceIndex,
5202        ty: TypeFutureTableIndex,
5203        options: OptionsIndex,
5204        future: u32,
5205        address: u32,
5206    ) -> Result<u32> {
5207        instance
5208            .guest_write(
5209                StoreContextMut(self),
5210                caller,
5211                TransmitIndex::Future(ty),
5212                options,
5213                None,
5214                future,
5215                address,
5216                1,
5217            )
5218            .map(|result| result.encode())
5219    }
5220
5221    fn future_read(
5222        &mut self,
5223        instance: Instance,
5224        caller: RuntimeComponentInstanceIndex,
5225        ty: TypeFutureTableIndex,
5226        options: OptionsIndex,
5227        future: u32,
5228        address: u32,
5229    ) -> Result<u32> {
5230        instance
5231            .guest_read(
5232                StoreContextMut(self),
5233                caller,
5234                TransmitIndex::Future(ty),
5235                options,
5236                None,
5237                future,
5238                address,
5239                1,
5240            )
5241            .map(|result| result.encode())
5242    }
5243
5244    fn stream_write(
5245        &mut self,
5246        instance: Instance,
5247        caller: RuntimeComponentInstanceIndex,
5248        ty: TypeStreamTableIndex,
5249        options: OptionsIndex,
5250        stream: u32,
5251        address: u32,
5252        count: u32,
5253    ) -> Result<u32> {
5254        instance
5255            .guest_write(
5256                StoreContextMut(self),
5257                caller,
5258                TransmitIndex::Stream(ty),
5259                options,
5260                None,
5261                stream,
5262                address,
5263                count,
5264            )
5265            .map(|result| result.encode())
5266    }
5267
5268    fn stream_read(
5269        &mut self,
5270        instance: Instance,
5271        caller: RuntimeComponentInstanceIndex,
5272        ty: TypeStreamTableIndex,
5273        options: OptionsIndex,
5274        stream: u32,
5275        address: u32,
5276        count: u32,
5277    ) -> Result<u32> {
5278        instance
5279            .guest_read(
5280                StoreContextMut(self),
5281                caller,
5282                TransmitIndex::Stream(ty),
5283                options,
5284                None,
5285                stream,
5286                address,
5287                count,
5288            )
5289            .map(|result| result.encode())
5290    }
5291
5292    fn future_drop_writable(
5293        &mut self,
5294        instance: Instance,
5295        ty: TypeFutureTableIndex,
5296        writer: u32,
5297    ) -> Result<()> {
5298        instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5299    }
5300
5301    fn flat_stream_write(
5302        &mut self,
5303        instance: Instance,
5304        caller: RuntimeComponentInstanceIndex,
5305        ty: TypeStreamTableIndex,
5306        options: OptionsIndex,
5307        payload_size: u32,
5308        payload_align: u32,
5309        stream: u32,
5310        address: u32,
5311        count: u32,
5312    ) -> Result<u32> {
5313        instance
5314            .guest_write(
5315                StoreContextMut(self),
5316                caller,
5317                TransmitIndex::Stream(ty),
5318                options,
5319                Some(FlatAbi {
5320                    size: payload_size,
5321                    align: payload_align,
5322                }),
5323                stream,
5324                address,
5325                count,
5326            )
5327            .map(|result| result.encode())
5328    }
5329
5330    fn flat_stream_read(
5331        &mut self,
5332        instance: Instance,
5333        caller: RuntimeComponentInstanceIndex,
5334        ty: TypeStreamTableIndex,
5335        options: OptionsIndex,
5336        payload_size: u32,
5337        payload_align: u32,
5338        stream: u32,
5339        address: u32,
5340        count: u32,
5341    ) -> Result<u32> {
5342        instance
5343            .guest_read(
5344                StoreContextMut(self),
5345                caller,
5346                TransmitIndex::Stream(ty),
5347                options,
5348                Some(FlatAbi {
5349                    size: payload_size,
5350                    align: payload_align,
5351                }),
5352                stream,
5353                address,
5354                count,
5355            )
5356            .map(|result| result.encode())
5357    }
5358
5359    fn stream_drop_writable(
5360        &mut self,
5361        instance: Instance,
5362        ty: TypeStreamTableIndex,
5363        writer: u32,
5364    ) -> Result<()> {
5365        instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5366    }
5367
5368    fn error_context_debug_message(
5369        &mut self,
5370        instance: Instance,
5371        ty: TypeComponentLocalErrorContextTableIndex,
5372        options: OptionsIndex,
5373        err_ctx_handle: u32,
5374        debug_msg_address: u32,
5375    ) -> Result<()> {
5376        instance.error_context_debug_message(
5377            StoreContextMut(self),
5378            ty,
5379            options,
5380            err_ctx_handle,
5381            debug_msg_address,
5382        )
5383    }
5384
5385    fn thread_new_indirect(
5386        &mut self,
5387        instance: Instance,
5388        caller: RuntimeComponentInstanceIndex,
5389        func_ty_idx: ModuleInternedTypeIndex,
5390        start_func_table_idx: RuntimeTableIndex,
5391        start_func_idx: u32,
5392        context: i32,
5393    ) -> Result<u32> {
5394        instance.thread_new_indirect(
5395            StoreContextMut(self),
5396            caller,
5397            func_ty_idx,
5398            start_func_table_idx,
5399            start_func_idx,
5400            context,
5401        )
5402    }
5403}
5404
5405type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5406
5407/// Runs the given future with the current thread set to `task` each time it is
5408/// polled.
5409async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5410where
5411    F: Future,
5412{
5413    let mut future = pin!(future);
5414    future::poll_fn(|cx| {
5415        let old_thread = match tls::get(|store| store.set_thread(task)) {
5416            Ok(thread) => thread,
5417            Err(error) => return Poll::Ready(Err(error)),
5418        };
5419        let result = future.as_mut().poll(cx);
5420        match tls::get(|store| store.set_thread(old_thread)) {
5421            Ok(_) => result.map(Ok),
5422            Err(error) => Poll::Ready(Err(error)),
5423        }
5424    })
5425    .await
5426}
5427
5428/// Represents the state of a pending host task.
5429///
5430/// This is used to represent tasks when the guest calls into the host.
5431pub(crate) struct HostTask {
5432    common: WaitableCommon,
5433
5434    /// State of borrows/etc the host needs to track. Used when the guest passes
5435    /// borrows to the host, for example.
5436    call_context: CallContext,
5437
5438    state: HostTaskState,
5439
5440    group: TaskGroupId,
5441}
5442
5443enum HostTaskState {
5444    /// A host task has been created and it's considered "started".
5445    ///
5446    /// The host task has yet to enter `first_poll` or `poll_and_block` which
5447    /// is where this will get updated further.
5448    CalleeStarted,
5449
5450    /// State used for tasks in `first_poll` meaning that the guest did an async
5451    /// lower of a host async function which is blocked. The specified handle is
5452    /// linked to the future in the main `FuturesUnordered` of a store which is
5453    /// used to cancel it if the guest requests cancellation.
5454    CalleeRunning(JoinHandle),
5455
5456    /// Cancellation was requested, but the completion worker has not finished
5457    /// lowering the result and publishing the terminal event. The task must
5458    /// remain in the table until that worker completes.
5459    CalleeCancelling,
5460
5461    /// Terminal state used for tasks in `poll_and_block` to store the result of
5462    /// their computation. Note that this state is not used for tasks in
5463    /// `first_poll`.
5464    CalleeFinished(LiftedResult),
5465
5466    /// Terminal state for host tasks meaning that the task was cancelled or the
5467    /// result was taken.
5468    CalleeDone { cancelled: bool },
5469}
5470
5471impl HostTask {
5472    fn new(
5473        concurrent_state: &mut ConcurrentState,
5474        state: HostTaskState,
5475        caller: QualifiedThreadId,
5476    ) -> Result<Self> {
5477        let group = concurrent_state.get_mut(caller.task)?.group;
5478        concurrent_state.increment_group_ref_count(group)?;
5479
5480        Ok(Self {
5481            common: WaitableCommon::default(),
5482            call_context: CallContext::default(),
5483            state,
5484            group,
5485        })
5486    }
5487}
5488
5489impl TableDebug for HostTask {
5490    fn type_name() -> &'static str {
5491        "HostTask"
5492    }
5493}
5494
5495type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5496
5497/// Represents the caller of a given guest task.
5498enum Caller {
5499    /// The host called the guest task.
5500    Host {
5501        /// If present, may be used to deliver the result.
5502        tx: Option<oneshot::Sender<LiftedResult>>,
5503        /// The host task which called into the guest, or `None` for a call from
5504        /// the top-level host. The task may belong to an entirely unrelated
5505        /// top-level component instance than the one the host called into.
5506        caller: Option<TableId<HostTask>>,
5507    },
5508    /// Another guest thread called the guest task
5509    Guest {
5510        /// The id of the caller
5511        thread: QualifiedThreadId,
5512    },
5513}
5514
5515/// Represents a closure and related canonical ABI parameters required to
5516/// validate a `task.return` call at runtime and lift the result.
5517struct LiftResult {
5518    lift: RawLift,
5519    ty: TypeTupleIndex,
5520    memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5521    string_encoding: StringEncoding,
5522}
5523
5524/// The table ID for a guest thread, qualified by the task to which it belongs.
5525///
5526/// This exists to minimize table lookups and the necessity to pass stores around mutably
5527/// for the common case of identifying the task to which a thread belongs.
5528#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5529pub(crate) struct QualifiedThreadId {
5530    task: TableId<GuestTask>,
5531    thread: TableId<GuestThread>,
5532}
5533
5534impl QualifiedThreadId {
5535    fn qualify(
5536        state: &mut ConcurrentState,
5537        thread: TableId<GuestThread>,
5538    ) -> Result<QualifiedThreadId> {
5539        Ok(QualifiedThreadId {
5540            task: state.get_mut(thread)?.parent_task,
5541            thread,
5542        })
5543    }
5544}
5545
5546impl fmt::Debug for QualifiedThreadId {
5547    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5548        f.debug_tuple("QualifiedThreadId")
5549            .field(&self.task.rep())
5550            .field(&self.thread.rep())
5551            .finish()
5552    }
5553}
5554
5555enum GuestThreadState {
5556    NotStartedImplicit,
5557    NotStartedExplicit(
5558        Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5559    ),
5560    Running,
5561    Suspended(StoreFiber<'static>),
5562    Ready {
5563        fiber: StoreFiber<'static>,
5564    },
5565    Completed,
5566}
5567
5568impl GuestThreadState {
5569    fn check_no_fiber(&self) -> Result<()> {
5570        match self {
5571            Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5572            Self::NotStartedImplicit
5573            | Self::NotStartedExplicit(_)
5574            | Self::Running
5575            | Self::Completed => Ok(()),
5576        }
5577    }
5578}
5579
5580impl fmt::Debug for GuestThreadState {
5581    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5582        match self {
5583            Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5584            Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5585            Self::Running => f.debug_tuple("Running").finish(),
5586            Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5587            Self::Ready { .. } => f.debug_struct("Ready").finish(),
5588            Self::Completed => f.debug_tuple("Completed").finish(),
5589        }
5590    }
5591}
5592
5593#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5594enum WakeOnCancel {
5595    None,
5596    Waiting(TableId<WaitableSet>),
5597    Yielding,
5598}
5599
5600impl WakeOnCancel {
5601    fn is_none(self) -> bool {
5602        matches!(self, WakeOnCancel::None)
5603    }
5604
5605    fn replace(&mut self, other: WakeOnCancel) -> Self {
5606        let old = *self;
5607        *self = other;
5608        old
5609    }
5610
5611    fn take(&mut self) -> Self {
5612        self.replace(WakeOnCancel::None)
5613    }
5614}
5615
5616pub struct GuestThread {
5617    /// Context-local state used to implement the `context.{get,set}`
5618    /// intrinsics.
5619    context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5620    /// The owning guest task.
5621    parent_task: TableId<GuestTask>,
5622    /// If non-`None`, indicates that the thread is currently either waiting a
5623    /// waitable set or yielding but may be cancelled and woken immediately.
5624    wake_on_cancel: WakeOnCancel,
5625    /// The execution state of this guest thread
5626    state: GuestThreadState,
5627    /// The index of this thread in the component instance's handle table.
5628    /// This must always be `Some` after initialization.
5629    instance_rep: Option<u32>,
5630    /// Scratch waitable set used to watch subtasks during synchronous calls.
5631    sync_call_set: TableId<WaitableSet>,
5632    /// The old value of `do_not_suspend` prior to the sync-typed task for which
5633    /// this thread was created, if relevant.
5634    old_do_not_suspend: Option<bool>,
5635}
5636
5637impl GuestThread {
5638    /// Retrieve the `GuestThread` corresponding to the specified guest-visible
5639    /// handle.
5640    fn from_instance(
5641        state: Pin<&mut ComponentInstance>,
5642        caller_instance: RuntimeComponentInstanceIndex,
5643        guest_thread: u32,
5644    ) -> Result<TableId<Self>> {
5645        let rep = state.instance_states().0[caller_instance]
5646            .thread_handle_table()
5647            .guest_thread_rep(guest_thread)?;
5648        Ok(TableId::new(rep))
5649    }
5650
5651    fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5652        let sync_call_set = state.push(WaitableSet {
5653            is_sync_call_set: true,
5654            ..WaitableSet::default()
5655        })?;
5656        Ok(Self {
5657            context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5658            parent_task,
5659            wake_on_cancel: WakeOnCancel::None,
5660            state: GuestThreadState::NotStartedImplicit,
5661            instance_rep: None,
5662            sync_call_set,
5663            old_do_not_suspend: None,
5664        })
5665    }
5666
5667    fn new_explicit(
5668        state: &mut ConcurrentState,
5669        parent_task: TableId<GuestTask>,
5670        start_func: Box<
5671            dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5672        >,
5673    ) -> Result<Self> {
5674        state.increment_task_ref_count(parent_task)?;
5675        let sync_call_set = state.push(WaitableSet {
5676            is_sync_call_set: true,
5677            ..WaitableSet::default()
5678        })?;
5679        Ok(Self {
5680            context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5681            parent_task,
5682            wake_on_cancel: WakeOnCancel::None,
5683            state: GuestThreadState::NotStartedExplicit(start_func),
5684            instance_rep: None,
5685            sync_call_set,
5686            old_do_not_suspend: None,
5687        })
5688    }
5689}
5690
5691impl TableDebug for GuestThread {
5692    fn type_name() -> &'static str {
5693        "GuestThread"
5694    }
5695}
5696
5697enum SyncResult {
5698    NotProduced,
5699    Produced(Option<ValRaw>),
5700    Taken,
5701}
5702
5703impl SyncResult {
5704    fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5705        Ok(match mem::replace(self, SyncResult::Taken) {
5706            SyncResult::NotProduced => None,
5707            SyncResult::Produced(val) => Some(val),
5708            SyncResult::Taken => {
5709                bail_bug!("attempted to take a synchronous result that was already taken")
5710            }
5711        })
5712    }
5713}
5714
5715/// Represents a pending guest task.
5716pub(crate) struct GuestTask {
5717    /// See `WaitableCommon`
5718    common: WaitableCommon,
5719    /// Closure to lower the parameters passed to this task.
5720    lower_params: Option<RawLower>,
5721    /// See `LiftResult`
5722    lift_result: Option<LiftResult>,
5723    /// A place to stash the type-erased lifted result if it can't be delivered
5724    /// immediately.
5725    result: Option<LiftedResult>,
5726    /// Closure to call the callback function for an async-lifted export, if
5727    /// provided.
5728    callback: Option<CallbackFn>,
5729    /// See `Caller`
5730    caller: Caller,
5731    /// Borrow state for this task.
5732    ///
5733    /// Keeps track of `borrow<T>` received to this task to ensure that
5734    /// everything is dropped by the time it exits.
5735    call_context: CallContext,
5736    /// A place to stash the lowered result for a sync-to-async call until it
5737    /// can be returned to the caller.
5738    sync_result: SyncResult,
5739    /// Whether or not the task has been cancelled (i.e. whether the
5740    /// cancellation request has been delivered to the task, and thus whether
5741    /// the task is permitted to call `task.cancel`).
5742    cancel_request_delivered: bool,
5743    /// Whether or not the caller has requested cancellation of this task via
5744    /// `subtask.cancel` while it was still unresolved. A subsequent
5745    /// `subtask.cancel` of the same task traps. This matches the spec's
5746    /// `Subtask.cancellation_requested`.
5747    cancel_requested: bool,
5748    /// Whether or not we've sent a `Status::Starting` event to any current or
5749    /// future waiters for this waitable.
5750    starting_sent: bool,
5751    /// The runtime instance to which the exported function for this guest task
5752    /// belongs.
5753    ///
5754    /// Note that the task may do a sync->sync call via a fused adapter which
5755    /// results in that task executing code in a different instance, and it may
5756    /// call host functions and intrinsics from that other instance.
5757    instance: RuntimeInstance,
5758    /// If present, a pending `Event::Cancelled` to be delivered to this task.
5759    event: Option<Event>,
5760    /// Whether or not the task has exited.
5761    exited: bool,
5762    /// Threads belonging to this task
5763    threads: HashSet<TableId<GuestThread>>,
5764    /// Indicates whether this task was created for a call to an async-typed
5765    /// export.
5766    async_typed: bool,
5767    /// Indicates whether this task was created for a call to an async-lifted
5768    /// export.
5769    async_lifted: bool,
5770    /// Whether `ConcurrentState::interesting_tasks` has been decremented for
5771    /// this task yet.
5772    decremented_interesting_task_count: bool,
5773    /// The `TaskGroup` to which this task belongs.
5774    group: TaskGroupId,
5775    /// The number of outstanding references to this task.
5776    ///
5777    /// When this goes to zero, the task will be removed from
5778    /// `ConcurrentState::table`.
5779    ref_count: usize,
5780}
5781
5782impl GuestTask {
5783    fn already_lowered_parameters(&self) -> bool {
5784        // We reset `lower_params` after we lower the parameters
5785        self.lower_params.is_none()
5786    }
5787
5788    fn returned_or_cancelled(&self) -> bool {
5789        // We reset `lift_result` after we return or exit
5790        self.lift_result.is_none()
5791    }
5792
5793    fn new(
5794        state: &mut ConcurrentState,
5795        lower_params: RawLower,
5796        lift_result: LiftResult,
5797        caller: Caller,
5798        callback: Option<CallbackFn>,
5799        instance: RuntimeInstance,
5800        async_typed: bool,
5801        async_lifted: bool,
5802    ) -> Result<QualifiedThreadId> {
5803        let group = match caller {
5804            Caller::Guest { thread } => {
5805                let group = state.get_mut(thread.task)?.group;
5806                state.increment_group_ref_count(group)?;
5807                group
5808            }
5809            Caller::Host { .. } => state.make_task_group()?,
5810        };
5811
5812        let task = state.push(Self {
5813            common: WaitableCommon::default(),
5814            lower_params: Some(lower_params),
5815            lift_result: Some(lift_result),
5816            result: None,
5817            callback,
5818            caller,
5819            call_context: CallContext::default(),
5820            sync_result: SyncResult::NotProduced,
5821            cancel_request_delivered: false,
5822            cancel_requested: false,
5823            starting_sent: false,
5824            instance,
5825            event: None,
5826            exited: false,
5827            threads: HashSet::new(),
5828            async_typed,
5829            async_lifted,
5830            decremented_interesting_task_count: false,
5831            group,
5832            // One for the caller and one for the implicit guest thread we're
5833            // about to create:
5834            ref_count: 2,
5835        })?;
5836        let new_thread = GuestThread::new_implicit(state, task)?;
5837        let thread = state.push(new_thread)?;
5838        state.get_mut(task)?.threads.insert(thread);
5839        state.interesting_tasks += 1;
5840        let thread = QualifiedThreadId { task, thread };
5841        log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5842        Ok(thread)
5843    }
5844}
5845
5846impl TableDebug for GuestTask {
5847    fn type_name() -> &'static str {
5848        "GuestTask"
5849    }
5850}
5851
5852/// Represents state common to all kinds of waitables.
5853#[derive(Default)]
5854struct WaitableCommon {
5855    /// The currently pending event for this waitable, if any.
5856    event: Option<Event>,
5857    /// The set to which this waitable belongs, if any.
5858    set: Option<TableId<WaitableSet>>,
5859    /// The handle with which the guest refers to this waitable, if any.
5860    handle: Option<u32>,
5861}
5862
5863/// Represents a Component Model Async `waitable`.
5864#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5865enum Waitable {
5866    /// A host task
5867    Host(TableId<HostTask>),
5868    /// A guest task
5869    Guest(TableId<GuestTask>),
5870    /// The read or write end of a stream or future
5871    Transmit(TableId<TransmitHandle>),
5872}
5873
5874impl Waitable {
5875    /// Retrieve the `Waitable` corresponding to the specified guest-visible
5876    /// handle.
5877    fn from_instance(
5878        state: Pin<&mut ComponentInstance>,
5879        caller_instance: RuntimeComponentInstanceIndex,
5880        waitable: u32,
5881    ) -> Result<Self> {
5882        use crate::runtime::vm::component::Waitable;
5883
5884        let (waitable, kind) = state.instance_states().0[caller_instance]
5885            .handle_table()
5886            .waitable_rep(waitable)?;
5887
5888        Ok(match kind {
5889            Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5890            Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5891            Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5892        })
5893    }
5894
5895    /// Retrieve the host-visible identifier for this `Waitable`.
5896    fn rep(&self) -> u32 {
5897        match self {
5898            Self::Host(id) => id.rep(),
5899            Self::Guest(id) => id.rep(),
5900            Self::Transmit(id) => id.rep(),
5901        }
5902    }
5903
5904    /// Move this `Waitable` to the specified set (when `set` is `Some(_)`) or
5905    /// remove it from any set it may currently belong to (when `set` is
5906    /// `None`).
5907    fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5908        log::trace!("waitable {self:?} join set {set:?}");
5909
5910        let old = mem::replace(&mut self.common(state)?.set, set);
5911
5912        if let Some(old) = old {
5913            match *self {
5914                Waitable::Host(id) => state.remove_child(id, old),
5915                Waitable::Guest(id) => state.remove_child(id, old),
5916                Waitable::Transmit(id) => state.remove_child(id, old),
5917            }?;
5918
5919            state.get_mut(old)?.ready.remove(self);
5920        }
5921
5922        if let Some(set) = set {
5923            match *self {
5924                Waitable::Host(id) => state.add_child(id, set),
5925                Waitable::Guest(id) => state.add_child(id, set),
5926                Waitable::Transmit(id) => state.add_child(id, set),
5927            }?;
5928
5929            if self.common(state)?.event.is_some() {
5930                self.mark_ready(state)?;
5931            }
5932        }
5933
5934        Ok(())
5935    }
5936
5937    /// Retrieve mutable access to the `WaitableCommon` for this `Waitable`.
5938    fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5939        Ok(match self {
5940            Self::Host(id) => &mut state.get_mut(*id)?.common,
5941            Self::Guest(id) => &mut state.get_mut(*id)?.common,
5942            Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5943        })
5944    }
5945
5946    /// Trap if this waitable is currently a member of a waitable set.
5947    ///
5948    /// A synchronous stream/future/subtask operation may end up blocking on
5949    /// this waitable, so it is not allowed to run while the waitable is also
5950    /// being watched by a waitable set.
5951    fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5952        if self.common(state)?.set.is_some() {
5953            bail!(Trap::WaitableSyncAndAsync);
5954        }
5955        Ok(())
5956    }
5957
5958    /// Trap if a thread is currently synchronously waiting on this waitable.
5959    ///
5960    /// This corresponds to the spec's `trap_if(w.has_sync_waiter)` and is
5961    /// used to prevent other threads from stealing or otherwise interfering
5962    /// with the event the sync waiter is expecting.
5963    fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5964        if let Some(set) = self.common(state)?.set
5965            && state.get_mut(set)?.is_sync_call_set
5966        {
5967            bail!(Trap::WaitableSyncAndAsync);
5968        }
5969        Ok(())
5970    }
5971
5972    /// Set or clear the pending event for this waitable and either deliver it
5973    /// to the first waiter, if any, or mark it as ready to be delivered to the
5974    /// next waiter that arrives.
5975    fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
5976        log::trace!("set event for {self:?}: {event:?}");
5977        self.common(state)?.event = event;
5978        self.mark_ready(state)
5979    }
5980
5981    /// Take the pending event from this waitable, leaving `None` in its place.
5982    fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
5983        let common = self.common(state)?;
5984        let event = common.event.take();
5985        if let Some(set) = self.common(state)?.set {
5986            state.get_mut(set)?.ready.remove(self);
5987        }
5988
5989        Ok(event)
5990    }
5991
5992    /// Deliver the current event for this waitable to the first waiter, if any,
5993    /// or else mark it as ready to be delivered to the next waiter that
5994    /// arrives.
5995    fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
5996        if let Some(set) = self.common(state)?.set {
5997            state.get_mut(set)?.ready.insert(*self);
5998            state.wake_waiter(set)?;
5999        }
6000        Ok(())
6001    }
6002
6003    /// Remove this waitable from the store's rep table (or decrement the ref
6004    /// count if it's a guest task).
6005    fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
6006        match self {
6007            Self::Host(task) => state.delete_host_task(*task),
6008            Self::Guest(task) => state.decrement_task_ref_count(*task),
6009            Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
6010        }
6011    }
6012}
6013
6014impl fmt::Debug for Waitable {
6015    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6016        match self {
6017            Self::Host(id) => write!(f, "{id:?}"),
6018            Self::Guest(id) => write!(f, "{id:?}"),
6019            Self::Transmit(id) => write!(f, "{id:?}"),
6020        }
6021    }
6022}
6023
6024/// Represents a Component Model Async `waitable-set`.
6025#[derive(Default)]
6026struct WaitableSet {
6027    /// Which waitables in this set have pending events, if any.
6028    ready: BTreeSet<Waitable>,
6029    /// Which guest threads are currently waiting on this set, if any.
6030    waiting: BTreeMap<QualifiedThreadId, WaitMode>,
6031    /// How many guest threads are waiting on this set, mirroring the spec's
6032    /// `WaitableSet.num_waiting`.
6033    num_waiting: usize,
6034    /// Whether this set is a synthetic, internal one meant for handling
6035    /// synchronous calls.
6036    is_sync_call_set: bool,
6037}
6038
6039impl WaitableSet {
6040    /// Records that a thread counted in `num_waiting` is no longer waiting.
6041    fn stop_waiting(&mut self) -> Result<()> {
6042        self.num_waiting = match self.num_waiting.checked_sub(1) {
6043            Some(n) => n,
6044            None => bail_bug!("waiter not accounted for in waitable set"),
6045        };
6046        Ok(())
6047    }
6048}
6049
6050impl TableDebug for WaitableSet {
6051    fn type_name() -> &'static str {
6052        "WaitableSet"
6053    }
6054}
6055
6056/// Type-erased closure to lower the parameters for a guest task.
6057type RawLower =
6058    Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
6059
6060/// Type-erased closure to lift the result for a guest task.
6061type RawLift = Box<
6062    dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6063>;
6064
6065/// Type erased result of a guest task which may be downcast to the expected
6066/// type by a host caller (or simply ignored in the case of a guest caller; see
6067/// `DummyResult`).
6068type LiftedResult = Box<dyn Any + Send + Sync>;
6069
6070/// Used to return a result from a `LiftFn` when the actual result has already
6071/// been lowered to a guest task's stack and linear memory.
6072struct DummyResult;
6073
6074/// Represents the Component Model Async state of a (sub-)component instance.
6075#[derive(Default)]
6076pub struct ConcurrentInstanceState {
6077    /// Whether backpressure is set for this instance (enabled if >0)
6078    backpressure: u16,
6079    /// Whether this instance can be entered
6080    do_not_enter: bool,
6081    /// Whether this instance may suspend (i.e. whether this instance is
6082    /// currently running a sync-typed function).
6083    do_not_suspend: bool,
6084    /// Pending calls for this instance which require `Self::backpressure` to be
6085    /// zero and/or `Self::do_not_enter` to be false before they can proceed.
6086    pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
6087    /// Number of pending `GuestCallKind::StartImplicit` calls which have not
6088    /// yet been started.
6089    pending_start_count: usize,
6090}
6091
6092impl ConcurrentInstanceState {
6093    pub fn pending_is_empty(&self) -> bool {
6094        self.pending.is_empty()
6095    }
6096}
6097
6098#[derive(Debug, Copy, Clone)]
6099pub(crate) enum CurrentThread {
6100    /// The currently running thread is a guest, identified here with its
6101    /// task/thread id combo.
6102    Guest(QualifiedThreadId),
6103    /// The currently running thread is a host task.
6104    Host(TableId<HostTask>),
6105    /// The currently running thread is a host call whose task has not yet been
6106    /// materialized. The contained ID identifies its guest caller.
6107    DeferredHost(QualifiedThreadId),
6108    /// There is no currently running thread because we are in the main event
6109    /// loop or concurrency is disabled.
6110    None,
6111}
6112
6113impl CurrentThread {
6114    fn guest(&self) -> Option<&QualifiedThreadId> {
6115        match self {
6116            Self::Guest(id) => Some(id),
6117            _ => None,
6118        }
6119    }
6120
6121    fn guest_task(&self) -> Option<TableId<GuestTask>> {
6122        match self {
6123            Self::Guest(id) => Some(id.task),
6124            _ => None,
6125        }
6126    }
6127
6128    fn is_none(&self) -> bool {
6129        matches!(self, Self::None)
6130    }
6131}
6132
6133impl From<QualifiedThreadId> for CurrentThread {
6134    fn from(id: QualifiedThreadId) -> Self {
6135        Self::Guest(id)
6136    }
6137}
6138
6139impl From<TableId<HostTask>> for CurrentThread {
6140    fn from(id: TableId<HostTask>) -> Self {
6141        Self::Host(id)
6142    }
6143}
6144
6145enum Priority {
6146    Switch,
6147    High,
6148    Low,
6149}
6150
6151/// Represents the Component Model Async state of a store.
6152pub struct ConcurrentState {
6153    /// The currently running thread, if any.
6154    ///
6155    /// Note that we lazily materialize threads on-demand and this field is not
6156    /// necessarily up-to-date. The `StoreOpaque::current_thread` method should
6157    /// be preferred over directly accessing this field.
6158    unforced_current_thread: CurrentThread,
6159
6160    /// Borrow state for the deferred host call, if any.
6161    ///
6162    /// This is `Some` if and only if [`Self::unforced_current_thread`] is
6163    /// [`CurrentThread::DeferredHost`]. Materializing the host task moves this
6164    /// context into that task.
6165    deferred_host_call_context: Option<CallContext>,
6166
6167    /// The set of pending host and background tasks, if any.
6168    ///
6169    /// See `ComponentInstance::poll_until` for where we temporarily take this
6170    /// out, poll it, then put it back to avoid any mutable aliasing hazards.
6171    futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6172    /// The table of waitables, waitable sets, etc.
6173    table: AlwaysMut<ResourceTable>,
6174    /// The next item to switch to if any.
6175    ///
6176    /// This takes precedence over items in the `high_priority` queue below and
6177    /// should be used in cases such as subtask calls and thread resume/promote
6178    /// operations where we must switch to a specific thread at the next turn of
6179    /// the event loop regardless of what happens to be present in the
6180    /// `high_priority` queue.
6181    switch_item: Option<WorkItem>,
6182    /// The item to set `switch_item` to when the current thread suspends.
6183    ///
6184    /// This is used when ever an async-lowered import or `subtask.cancel` is
6185    /// called in order to track the thread to switch back to once the current
6186    /// subtask suspends, if any.
6187    next_switch_item: Option<WorkItem>,
6188    /// The "high priority" work queue for this store's event loop.
6189    high_priority: VecDeque<WorkItem>,
6190    /// The "low priority" work queue for this store's event loop.
6191    low_priority: VecDeque<WorkItem>,
6192    /// A place to stash the reason a fiber is suspending so that the code which
6193    /// resumed it will know under what conditions the fiber should be resumed
6194    /// again.
6195    suspend_reason: Option<SuspendReason>,
6196    /// A cached fiber which is waiting for work to do.
6197    ///
6198    /// This helps us avoid creating a new fiber for each `GuestCall` work item.
6199    worker: Option<StoreFiber<'static>>,
6200    /// A place to stash the work item for which we're resuming a worker fiber.
6201    worker_item: Option<WorkerItem>,
6202
6203    /// Reference counts for all component error contexts
6204    ///
6205    /// NOTE: it is possible the global ref count to be *greater* than the sum of
6206    /// (sub)component ref counts as tracked by `error_context_tables`, for
6207    /// example when the host holds one or more references to error contexts.
6208    ///
6209    /// The key of this primary map is often referred to as the "rep" (i.e. host-side
6210    /// component-wide representation) of the index into concurrent state for a given
6211    /// stored `ErrorContext`.
6212    ///
6213    /// Stated another way, `TypeComponentGlobalErrorContextTableIndex` is essentially the same
6214    /// as a `TableId<ErrorContextState>`.
6215    global_error_context_ref_counts:
6216        BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6217
6218    /// The number of "interesting tasks" currently executing in the store.
6219    ///
6220    /// This tracks the concept of a component instance lifetime as defined in
6221    /// https://github.com/WebAssembly/component-model/pull/643. Specifically
6222    /// all tasks currently increment this counter which then gets decremented
6223    /// when they exit. In the future some tasks might not increment this
6224    /// counter, but for now all do.
6225    ///
6226    /// This is used to implement `Accessor::poll_no_interesting_tasks` to
6227    /// inform the embedder when all tasks have completed. This is then
6228    /// used in wasmtime-wasi-http, for example, to know when an instance is
6229    /// idle.
6230    interesting_tasks: usize,
6231
6232    /// Single waker to notify when `interesting_tasks` reaches 0.
6233    ///
6234    /// Used in the implementation of `Accessor::poll_no_interesting_tasks`.
6235    interesting_tasks_empty_waker: Option<Waker>,
6236
6237    /// Single waker to notify when a component instance goes from
6238    /// not-concurrently-callable to concurrently-callable.
6239    ///
6240    /// Used in the implementation of `Accessor::poll_ready_for_concurrent_call`.
6241    ready_for_concurrent_call_waker: Option<Waker>,
6242
6243    /// Whether the `StoreContextMut::poll_until` event loop is running.
6244    event_loop_running: bool,
6245
6246    /// Stack of switch items to push to and pop from when entering and exiting
6247    /// sync-to-sync calls.
6248    saved_next_switch_items: Vec<Option<WorkItem>>,
6249
6250    /// See [TaskGroupHook].
6251    #[cfg(feature = "task-group-hook")]
6252    task_group_hook: Option<Box<dyn TaskGroupHook>>,
6253}
6254
6255impl Default for ConcurrentState {
6256    fn default() -> Self {
6257        Self {
6258            unforced_current_thread: CurrentThread::None,
6259            deferred_host_call_context: None,
6260            table: AlwaysMut::new(ResourceTable::new()),
6261            futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6262            switch_item: None,
6263            next_switch_item: None,
6264            high_priority: VecDeque::new(),
6265            low_priority: VecDeque::new(),
6266            suspend_reason: None,
6267            worker: None,
6268            worker_item: None,
6269            global_error_context_ref_counts: BTreeMap::new(),
6270            interesting_tasks: 0,
6271            interesting_tasks_empty_waker: None,
6272            ready_for_concurrent_call_waker: None,
6273            event_loop_running: false,
6274            saved_next_switch_items: Vec::new(),
6275            #[cfg(feature = "task-group-hook")]
6276            task_group_hook: None,
6277        }
6278    }
6279}
6280
6281impl ConcurrentState {
6282    /// Take ownership of any fibers and futures owned by this object.
6283    ///
6284    /// This should be used when disposing of the `Store` containing this object
6285    /// in order to gracefully resolve any and all fibers using
6286    /// `StoreFiber::dispose`.  This is necessary to avoid possible
6287    /// use-after-free bugs due to fibers which may still have access to the
6288    /// `Store`.
6289    ///
6290    /// Additionally, the futures collected with this function should be dropped
6291    /// within a `tls::set` call, which will ensure than any futures closing
6292    /// over an `&Accessor` will have access to the store when dropped, allowing
6293    /// e.g. `WithAccessor[AndValue]` instances to be disposed of without
6294    /// panicking.
6295    ///
6296    /// Note that this will leave the object in an inconsistent and unusable
6297    /// state, so it should only be used just prior to dropping it.
6298    pub(crate) fn take_fibers_and_futures(
6299        &mut self,
6300        fibers: &mut Vec<StoreFiber<'static>>,
6301        futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6302    ) {
6303        let ConcurrentState {
6304            table,
6305            worker,
6306            switch_item,
6307            next_switch_item,
6308            high_priority,
6309            low_priority,
6310            saved_next_switch_items,
6311            futures: pending_futures,
6312            worker_item: _,
6313            unforced_current_thread: _,
6314            deferred_host_call_context: _,
6315            suspend_reason: _,
6316            global_error_context_ref_counts: _,
6317            interesting_tasks: _,
6318            interesting_tasks_empty_waker: _,
6319            ready_for_concurrent_call_waker: _,
6320            event_loop_running: _,
6321            #[cfg(feature = "task-group-hook")]
6322                task_group_hook: _,
6323        } = self;
6324
6325        let mut items = Vec::new();
6326        for (_, entry) in table.get_mut().iter_mut() {
6327            let entry = entry.into_entry_mut();
6328            if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6329                for mode in mem::take(&mut set.waiting).into_values() {
6330                    match mode {
6331                        WaitMode::Fiber(fiber) => {
6332                            fibers.push(fiber);
6333                        }
6334                        WaitMode::Callback(_) => {}
6335                    }
6336                }
6337            } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6338                match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6339                    GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6340                        fibers.push(fiber);
6341                    }
6342                    GuestThreadState::NotStartedImplicit
6343                    | GuestThreadState::NotStartedExplicit(_)
6344                    | GuestThreadState::Running
6345                    | GuestThreadState::Completed => {}
6346                }
6347            } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6348                if let Some(item) = item.take() {
6349                    items.push(item);
6350                }
6351            }
6352        }
6353
6354        if let Some(fiber) = worker.take() {
6355            fibers.push(fiber);
6356        }
6357
6358        let mut handle_item = |item| match item {
6359            WorkItem::ResumeFiber { fiber, .. } => {
6360                fibers.push(fiber);
6361            }
6362            WorkItem::PushFuture(future) => {
6363                pending_futures
6364                    .get_mut()
6365                    .as_mut()
6366                    .unwrap()
6367                    .push(future.into_inner());
6368            }
6369            WorkItem::ResumeThread { .. }
6370            | WorkItem::GuestCall { .. }
6371            | WorkItem::WorkerFunction(_) => {}
6372        };
6373
6374        for item in items {
6375            handle_item(item);
6376        }
6377        if let Some(item) = switch_item.take() {
6378            handle_item(item);
6379        }
6380        if let Some(item) = next_switch_item.take() {
6381            handle_item(item);
6382        }
6383        for item in mem::take(high_priority) {
6384            handle_item(item);
6385        }
6386        for item in mem::take(low_priority) {
6387            handle_item(item);
6388        }
6389        for item in mem::take(saved_next_switch_items)
6390            .into_iter()
6391            .filter_map(|v| v)
6392        {
6393            handle_item(item);
6394        }
6395
6396        if let Some(them) = pending_futures.get_mut().take() {
6397            futures.push(them);
6398        }
6399    }
6400
6401    #[cfg(feature = "gc")]
6402    pub(crate) fn trace_fiber_roots(
6403        &mut self,
6404        modules: &ModuleRegistry,
6405        unwind: &dyn Unwind,
6406        gc_roots_list: &mut GcRootsList,
6407    ) {
6408        let ConcurrentState {
6409            table,
6410            worker,
6411            switch_item,
6412            next_switch_item,
6413            high_priority,
6414            low_priority,
6415            saved_next_switch_items,
6416
6417            // TODO(cm-gc): This field contains `ValRaw`s, but they are never GC
6418            // references because the component model doesn't support GC yet. We
6419            // will need to trace these somehow when it does.
6420            futures: _,
6421
6422            // These fields do not contain GC references.
6423            worker_item: _,
6424            unforced_current_thread: _,
6425            deferred_host_call_context: _,
6426            suspend_reason: _,
6427            global_error_context_ref_counts: _,
6428            interesting_tasks: _,
6429            interesting_tasks_empty_waker: _,
6430            ready_for_concurrent_call_waker: _,
6431            event_loop_running: _,
6432            #[cfg(feature = "task-group-hook")]
6433                task_group_hook: _,
6434        } = self;
6435
6436        for (_, entry) in table.get_mut().iter_mut() {
6437            let entry = entry.into_entry_mut();
6438            if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6439                for mode in set.waiting.values_mut() {
6440                    match mode {
6441                        WaitMode::Fiber(fiber) => {
6442                            fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6443                        }
6444                        WaitMode::Callback(_) => {}
6445                    }
6446                }
6447            } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6448                match &mut thread.state {
6449                    GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6450                        fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6451                    }
6452                    GuestThreadState::NotStartedImplicit
6453                    | GuestThreadState::NotStartedExplicit(_)
6454                    | GuestThreadState::Running
6455                    | GuestThreadState::Completed => {}
6456                }
6457            } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6458                entry.downcast_mut::<Option<WorkItem>>()
6459            {
6460                fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6461            }
6462        }
6463
6464        if let Some(fiber) = worker {
6465            fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6466        }
6467
6468        let mut handle_item = |item: &mut WorkItem| match item {
6469            WorkItem::ResumeFiber { fiber, .. } => {
6470                fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6471            }
6472            WorkItem::PushFuture(_future) => {
6473                // TODO(cm-gc): once futures can contain GC roots, we will need
6474                // to trace them.
6475            }
6476            WorkItem::ResumeThread { .. }
6477            | WorkItem::GuestCall { .. }
6478            | WorkItem::WorkerFunction(_) => {}
6479        };
6480
6481        if let Some(item) = switch_item {
6482            handle_item(item);
6483        }
6484        if let Some(item) = next_switch_item {
6485            handle_item(item);
6486        }
6487        for item in high_priority {
6488            handle_item(item);
6489        }
6490        for item in low_priority {
6491            handle_item(item);
6492        }
6493        for item in saved_next_switch_items
6494            .iter_mut()
6495            .filter_map(|v| v.as_mut())
6496        {
6497            handle_item(item);
6498        }
6499    }
6500
6501    fn schedule_suspended_thread(
6502        &mut self,
6503        thread: QualifiedThreadId,
6504        instance: RuntimeInstance,
6505        priority: Priority,
6506    ) -> Result<()> {
6507        // Validate the destination while the source still owns the fiber.
6508        if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6509            bail_bug!("switch item already set");
6510        }
6511        let slot = &mut self.get_mut(thread.thread)?.state;
6512        if !matches!(slot, GuestThreadState::Suspended(_)) {
6513            bail!(Trap::CannotResumeThread);
6514        }
6515        let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6516        else {
6517            unreachable!()
6518        };
6519        let item = WorkItem::ResumeFiber {
6520            instance,
6521            thread,
6522            fiber,
6523        };
6524        match priority {
6525            Priority::Switch => self.switch_item = Some(item),
6526            Priority::High => self.push_high_priority(item),
6527            Priority::Low => self.push_low_priority(item),
6528        }
6529        Ok(())
6530    }
6531
6532    fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6533        let slot = &mut self.get_mut(thread)?.state;
6534        slot.check_no_fiber()?;
6535        *slot = GuestThreadState::Running;
6536        Ok(())
6537    }
6538
6539    fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6540        // push can consume and drop its argument when the table is full.
6541        // Reserve with an empty item before taking the fiber-owning source.
6542        let saved = self.push(None::<WorkItem>)?;
6543        let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6544        *slot = self.next_switch_item.take();
6545        Ok(saved)
6546    }
6547
6548    fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6549        if self.next_switch_item.is_some() {
6550            bail_bug!("next switch item already set when restoring");
6551        }
6552        self.next_switch_item = self.delete(saved)?;
6553        Ok(())
6554    }
6555
6556    fn take_callback_waiter(
6557        &mut self,
6558        set: TableId<WaitableSet>,
6559        thread: QualifiedThreadId,
6560    ) -> Result<Instance> {
6561        let waiting = &mut self.get_mut(set)?.waiting;
6562        let instance = match waiting.get(&thread) {
6563            Some(WaitMode::Callback(instance)) => *instance,
6564            other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6565        };
6566        waiting.remove(&thread);
6567        Ok(instance)
6568    }
6569
6570    fn push<V: Send + Sync + 'static>(
6571        &mut self,
6572        value: V,
6573    ) -> Result<TableId<V>, ResourceTableError> {
6574        self.table.get_mut().push(value).map(TableId::from)
6575    }
6576
6577    fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6578        self.table.get_mut().get_mut(&Resource::from(id))
6579    }
6580
6581    pub fn add_child<T: 'static, U: 'static>(
6582        &mut self,
6583        child: TableId<T>,
6584        parent: TableId<U>,
6585    ) -> Result<(), ResourceTableError> {
6586        self.table
6587            .get_mut()
6588            .add_child(Resource::from(child), Resource::from(parent))
6589    }
6590
6591    pub fn remove_child<T: 'static, U: 'static>(
6592        &mut self,
6593        child: TableId<T>,
6594        parent: TableId<U>,
6595    ) -> Result<(), ResourceTableError> {
6596        self.table
6597            .get_mut()
6598            .remove_child(Resource::from(child), Resource::from(parent))
6599    }
6600
6601    fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6602        self.table.get_mut().delete(Resource::from(id))
6603    }
6604
6605    fn push_future(&mut self, future: HostTaskFuture) {
6606        // Note that we can't directly push to `ConcurrentState::futures` here
6607        // since this may be called from a future that's being polled inside
6608        // `Self::poll_until`, which temporarily removes the `FuturesUnordered`
6609        // so it has exclusive access while polling it.  Therefore, we push a
6610        // work item to the "high priority" queue, which will actually push to
6611        // `ConcurrentState::futures` later.
6612        self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6613    }
6614
6615    fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6616        if self.switch_item.is_some() {
6617            // Preserve ownership on this internal-error path. The caller
6618            // must stop scheduling after the error; teardown disposes the item.
6619            self.push_high_priority(item);
6620            bail_bug!("switch item already set");
6621        }
6622
6623        self.switch_item = Some(item);
6624        log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6625
6626        Ok(())
6627    }
6628
6629    fn take_next_switch_item(&mut self) -> Result<()> {
6630        if let Some(item) = self.next_switch_item.take() {
6631            self.set_switch_item(item)?;
6632        }
6633        Ok(())
6634    }
6635
6636    /// Wake the first thread waiting on `set`, if any, so that it may receive
6637    /// one of the set's pending events.
6638    fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6639        // Finish every fallible lookup and check before taking the waiter out
6640        // of the set, so that an error leaves its fiber owned by the set.
6641        let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6642            return Ok(());
6643        };
6644        let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6645        if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6646            bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6647        }
6648        let instance = self.get_mut(thread.task)?.instance;
6649        self.get_mut(thread.thread)?.wake_on_cancel.take();
6650        let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6651
6652        let item = match mode {
6653            WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6654                instance,
6655                thread,
6656                fiber,
6657            },
6658            WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6659                instance,
6660                call: GuestCall {
6661                    thread,
6662                    kind: GuestCallKind::DeliverEvent {
6663                        instance: callback_instance,
6664                        set: Some(set),
6665                    },
6666                },
6667            },
6668        };
6669        self.push_high_priority(item);
6670        Ok(())
6671    }
6672
6673    fn push_high_priority(&mut self, item: WorkItem) {
6674        self.high_priority.push_front(item);
6675        log::trace!(
6676            "push high priority: {:?}",
6677            self.high_priority.front().unwrap()
6678        );
6679    }
6680
6681    fn push_low_priority(&mut self, item: WorkItem) {
6682        self.low_priority.push_front(item);
6683        log::trace!(
6684            "push low priority: {:?}",
6685            self.low_priority.front().unwrap()
6686        );
6687    }
6688
6689    fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6690        match priority {
6691            Priority::Switch => self.set_switch_item(item)?,
6692            Priority::High => self.push_high_priority(item),
6693            Priority::Low => self.push_low_priority(item),
6694        }
6695
6696        Ok(())
6697    }
6698
6699    fn promote_instance_local_thread_work_item(
6700        &mut self,
6701        current_instance: RuntimeInstance,
6702    ) -> Result<bool> {
6703        log::trace!("promote thread work items for {current_instance:?}");
6704
6705        self.promote_work_item_matching(|item: &WorkItem| {
6706            let result = match item {
6707                WorkItem::ResumeThread { instance, .. }
6708                | WorkItem::ResumeFiber { instance, .. }
6709                | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6710                _ => false,
6711            };
6712
6713            log::trace!("candidate {item:?}: {result}");
6714            result
6715        })
6716    }
6717
6718    fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6719        self.promote_work_item_matching(|item: &WorkItem| match item {
6720            WorkItem::ResumeThread {
6721                thread: item_thread,
6722                ..
6723            }
6724            | WorkItem::GuestCall {
6725                call:
6726                    GuestCall {
6727                        thread: item_thread,
6728                        ..
6729                    },
6730                ..
6731            } => *item_thread == thread,
6732            _ => false,
6733        })
6734    }
6735
6736    fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6737    where
6738        F: FnMut(&WorkItem) -> bool,
6739    {
6740        // Scan from the back to preserve the ordering of items
6741        // are popped from the back of the `VecDeque`s by `poll_until` and
6742        // pushed to the front by `push_{high,low}_priority`.
6743
6744        // Inspect items while they are still owned by the store. A predicate
6745        // (including tracing) may panic; it must not drop an extracted queue.
6746        if self.switch_item.is_none() {
6747            if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6748                self.switch_item = self.high_priority.remove(index);
6749            }
6750        }
6751        if self.switch_item.is_none() {
6752            if let Some(index) = self.low_priority.iter().rposition(predicate) {
6753                self.switch_item = self.low_priority.remove(index);
6754            }
6755        }
6756        Ok(self.switch_item.is_some())
6757    }
6758
6759    /// Used by `ResourceTables` to acquire the current `CallContext` for the
6760    /// specified task.
6761    pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6762        match task.kind() {
6763            ScopeKind::HostId(task) => {
6764                let task: TableId<HostTask> = TableId::new(task);
6765                Ok(&mut self.get_mut(task)?.call_context)
6766            }
6767            ScopeKind::Id(task) => {
6768                let task: TableId<GuestTask> = TableId::new(task);
6769                Ok(&mut self.get_mut(task)?.call_context)
6770            }
6771        }
6772    }
6773
6774    pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6775        self.deferred_host_call_context.as_mut()
6776    }
6777
6778    fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6779        match self.futures.get_mut().as_mut() {
6780            Some(f) => Ok(f),
6781            None => bail_bug!("futures field of concurrent state is currently taken"),
6782        }
6783    }
6784
6785    pub(crate) fn table(&mut self) -> &mut ResourceTable {
6786        self.table.get_mut()
6787    }
6788
6789    fn debug_assert_deferred_host_invariant(&self) {
6790        debug_assert_eq!(
6791            self.deferred_host_call_context.is_some(),
6792            matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6793            "a deferred host thread and call context must exist together",
6794        );
6795    }
6796
6797    fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6798        self.debug_assert_deferred_host_invariant();
6799        let caller = match self.unforced_current_thread {
6800            CurrentThread::DeferredHost(caller) => caller,
6801            thread => return Ok(thread),
6802        };
6803
6804        // Push first so allocation failure leaves the deferred state intact.
6805        let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6806        let task = self.push(task)?;
6807        let call_context = self
6808            .deferred_host_call_context
6809            .take()
6810            .expect("deferred host call context should be present");
6811        self.get_mut(task)
6812            .expect("newly inserted host task should be present")
6813            .call_context = call_context;
6814        self.unforced_current_thread = CurrentThread::Host(task);
6815        self.debug_assert_deferred_host_invariant();
6816        log::trace!("new host task materialized {task:?}");
6817        Ok(CurrentThread::Host(task))
6818    }
6819
6820    fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6821        match self.materialize_host_task()? {
6822            CurrentThread::Host(id) => Ok(Some(id)),
6823            CurrentThread::None => Ok(None),
6824            CurrentThread::Guest(_) => {
6825                bail_bug!("tried to materialize a host task id from a guest thread")
6826            }
6827            CurrentThread::DeferredHost(_) => {
6828                bail_bug!(
6829                    "current thread is a deferred host thread which should have been materialized"
6830                )
6831            }
6832        }
6833    }
6834
6835    pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6836        match self.materialize_host_task()? {
6837            CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6838            _ => bail_bug!("current scope is not a deferred host scope"),
6839        }
6840    }
6841
6842    fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6843        let count = &mut self.get_mut(task)?.ref_count;
6844        *count += 1;
6845        log::trace!("increment {task:?} to {count}");
6846        Ok(())
6847    }
6848
6849    fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6850        let count = &mut self.get_mut(task)?.ref_count;
6851        if *count == 0 {
6852            bail_bug!("can't decrement task ref count that's already zero");
6853        }
6854        *count -= 1;
6855        log::trace!("decrement {task:?} to {count}");
6856        if *count == 0 {
6857            let task = self.delete(task)?;
6858
6859            self.decrement_group_ref_count(task.group)?;
6860
6861            // When a guest task is created it increments the
6862            // `ConcurrentState::interesting_tasks` counter, and that needs
6863            // to be paired with a decrement. There are a few situations in
6864            // which the decrement needs to happen which don't all funnel
6865            // through here, so in lieu of that at least try to catch issues
6866            // where we forgot to do a decrement.
6867            debug_assert!(task.decremented_interesting_task_count);
6868        }
6869        Ok(())
6870    }
6871
6872    fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6873        log::trace!("delete host task {task:?}");
6874        let task = self.delete(task)?;
6875
6876        self.decrement_group_ref_count(task.group)
6877    }
6878}
6879
6880/// Provide a type hint to compiler about the shape of a parameter lower
6881/// closure.
6882fn for_any_lower<
6883    F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6884>(
6885    fun: F,
6886) -> F {
6887    fun
6888}
6889
6890/// Provide a type hint to compiler about the shape of a result lift closure.
6891fn for_any_lift<
6892    F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6893>(
6894    fun: F,
6895) -> F {
6896    fun
6897}
6898
6899fn check_ambient_store(id: StoreId) {
6900    let message = "\
6901        `Future`s which depend on asynchronous component tasks, streams, or \
6902        futures to complete may only be polled from the event loop of the \
6903        store to which they belong.  Please use \
6904        `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6905    ";
6906    tls::try_get(|store| {
6907        let matched = match store {
6908            tls::TryGet::Some(store) => store.id() == id,
6909            tls::TryGet::Taken | tls::TryGet::None => false,
6910        };
6911
6912        if !matched {
6913            panic!("{message}")
6914        }
6915    });
6916}
6917
6918fn unpack_callback_code(code: u32) -> (u32, u32) {
6919    (code & 0xF, code >> 4)
6920}
6921
6922/// Helper struct for packaging parameters to be passed to
6923/// `ComponentInstance::waitable_check` for calls to `waitable-set.wait` or
6924/// `waitable-set.poll`.
6925struct WaitableCheckParams {
6926    set: TableId<WaitableSet>,
6927    options: OptionsIndex,
6928    payload: u32,
6929}
6930
6931/// Indicates whether `ComponentInstance::waitable_check` is being called for
6932/// `waitable-set.wait` or `waitable-set.poll`.
6933enum WaitableCheck {
6934    Wait,
6935    Poll,
6936}
6937
6938/// Represents a guest task called from the host, prepared using `prepare_call`.
6939pub(crate) struct PreparedCall<R> {
6940    /// The guest export to be called
6941    handle: Func,
6942    /// The guest thread created by `prepare_call`
6943    thread: QualifiedThreadId,
6944    /// The number of lowered core Wasm parameters to pass to the call.
6945    param_count: usize,
6946    /// The `oneshot::Receiver` to which the result of the call will be
6947    /// delivered when it is available.
6948    rx: oneshot::Receiver<LiftedResult>,
6949    _phantom: PhantomData<R>,
6950}
6951
6952impl<R> PreparedCall<R> {
6953    /// Acquire a reference to the `TaskId` for this `PreparedCall`.
6954    ///
6955    /// Note that this increments the reference count for the task, and
6956    /// `TaskId::host_future_dropped` decrements it.  Hence,
6957    /// `TaskId::host_future_dropped` _must_ be called before the `TaskId` is
6958    /// dropped to avoid leaking the task.
6959    pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6960        store
6961            .concurrent_state_mut()?
6962            .increment_task_ref_count(self.thread.task)?;
6963
6964        Ok(TaskId {
6965            task: self.thread.task,
6966        })
6967    }
6968}
6969
6970/// Represents a task created by `prepare_call`.
6971pub(crate) struct TaskId {
6972    task: TableId<GuestTask>,
6973}
6974
6975impl TaskId {
6976    /// The host future for an async task was dropped.
6977    ///
6978    /// If the parameters have not been lowered yet, it is no longer valid to
6979    /// do so, as the lowering closure would see a dangling pointer. In this
6980    /// case the closure is replaced with one that fails, so the call fails if
6981    /// it's ever started. This can't fail, since the task can't be deleted
6982    /// while its host future is live.
6983    ///
6984    /// Otherwise, there may be running threads, or ones that are suspended and
6985    /// can be resumed by other tasks for this component, so we mark the future
6986    /// as dropped and delete the task when all threads are done.
6987    pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
6988        let task = store.concurrent_state_mut()?.get_mut(self.task)?;
6989        if !task.already_lowered_parameters() {
6990            // TODO: avoid leaking the task and trapping in this case, e.g. by
6991            // allowing the replacement callback to return a value that tells
6992            // the event loop "nevermind, the host gave up on this call; just
6993            // skip it, clean up the task, and carry on with other work".
6994            task.lower_params = Some(Box::new(|store, _| {
6995                store.set_trapped();
6996                bail!("the host future for this call was dropped before it started")
6997            }));
6998        } else {
6999            // TODO: we're temporarily ignoring any error here to avoid a
7000            // panic-on-unwrap in `call_async_concurrent`.  Instead, we should
7001            // determine a safe way to defer the decrement (and any error it
7002            // might produce) outside of a `drop` call.
7003            _ = store
7004                .concurrent_state_mut()?
7005                .decrement_task_ref_count(self.task);
7006        }
7007
7008        Ok(())
7009    }
7010}
7011
7012/// Prepare a call to the specified exported Wasm function, providing functions
7013/// for lowering the parameters and lifting the result.
7014///
7015/// To enqueue the returned `PreparedCall` in the `ComponentInstance`'s event
7016/// loop, use `stage_call`.
7017pub(crate) fn prepare_call<T, R>(
7018    mut store: StoreContextMut<T>,
7019    handle: Func,
7020    param_count: usize,
7021    lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
7022    + Send
7023    + Sync
7024    + 'static,
7025    lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
7026    + Send
7027    + Sync
7028    + 'static,
7029) -> Result<PreparedCall<R>> {
7030    // This check should be redundant, but leave it here so that there is a
7031    // clear error when trying to execute wasm in a poisoned store instead of
7032    // possibly hitting some other `bail_bug!` because of inconsistent state.
7033    if !store.0.may_enter() {
7034        bail!(Trap::CannotEnterComponent);
7035    }
7036
7037    let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
7038
7039    let instance = handle.instance().id().get(store.0);
7040    let options = &instance.component().env_component().options[options];
7041    let ty = &instance.component().types()[ty];
7042    let async_typed = ty.async_;
7043    let async_lifted = raw_options.async_;
7044    let task_return_type = ty.results;
7045    let component_instance = raw_options.instance;
7046    let callback = options.callback.map(|i| instance.runtime_callback(i));
7047    let memory = options
7048        .memory()
7049        .map(|i| instance.runtime_memory(i))
7050        .map(SendSyncPtr::new);
7051    let string_encoding = options.string_encoding;
7052    let token = StoreToken::new(store.as_context_mut());
7053    let caller = store.0.materialize_host_task_id()?;
7054    let state = store.0.concurrent_state_mut()?;
7055
7056    let (tx, rx) = oneshot::channel();
7057
7058    let instance = handle.instance().runtime_instance(component_instance);
7059    let thread = GuestTask::new(
7060        state,
7061        Box::new(for_any_lower(move |store, params| {
7062            lower_params(token.as_context_mut(store), params)
7063        })),
7064        LiftResult {
7065            lift: Box::new(for_any_lift(move |store, result| {
7066                lift_result(store, result)
7067            })),
7068            ty: task_return_type,
7069            memory,
7070            string_encoding,
7071        },
7072        Caller::Host {
7073            tx: Some(tx),
7074            caller,
7075        },
7076        callback.map(|callback| {
7077            let callback = SendSyncPtr::new(callback);
7078            let instance = handle.instance();
7079            Box::new(move |store: &mut dyn VMStore, event, handle| {
7080                let store = token.as_context_mut(store);
7081                // SAFETY: Per the contract of `prepare_call`, the callback
7082                // will remain valid at least as long is this task exists.
7083                unsafe { instance.call_callback(store, callback, event, handle) }
7084            }) as CallbackFn
7085        }),
7086        instance,
7087        async_typed,
7088        async_lifted,
7089    )?;
7090
7091    Ok(PreparedCall {
7092        handle,
7093        thread,
7094        param_count,
7095        rx,
7096        _phantom: PhantomData,
7097    })
7098}
7099
7100pub(crate) struct StagedCall<R> {
7101    store: StoreId,
7102    rx: oneshot::Receiver<LiftedResult>,
7103    _marker: PhantomData<fn() -> R>,
7104    /// The task group to which the underlying task belongs.
7105    ///
7106    /// Note that this must be treated as opaque and not used to look up the
7107    /// corresponding `TaskGroup` since we decrement the task refcount in
7108    /// `StagedCall::new` which may cause the group to be disposed.  This field
7109    /// is only present to enable `[Typed]FuncCallConcurrent::group`, allowing
7110    /// the embedder to correlate the task group with `TaskGroupHook` events.
7111    group: TaskGroupId,
7112}
7113
7114impl<R> StagedCall<R> {
7115    /// Queue a call previously prepared using `prepare_call` to be run as part of
7116    /// the associated `ComponentInstance`'s event loop.
7117    ///
7118    /// Note that this has the side-effect of disposing the specified
7119    /// `PrepareCall` and decrementing the task reference count since
7120    /// `StagedCall` has no further need to access the task directly.
7121    ///
7122    /// The returned future will resolve to the result once it is available, but
7123    /// must only be polled via the instance's event loop. See
7124    /// `StoreContextMut::run_concurrent` for details.
7125    pub(crate) fn new<T: 'static>(
7126        mut store: StoreContextMut<T>,
7127        prepared: PreparedCall<R>,
7128    ) -> Result<StagedCall<R>> {
7129        let PreparedCall {
7130            handle,
7131            thread,
7132            param_count,
7133            rx,
7134            ..
7135        } = prepared;
7136
7137        stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7138
7139        let state = store.0.concurrent_state_mut()?;
7140        let group = state.get_mut(thread.task)?.group;
7141        state.decrement_task_ref_count(thread.task)?;
7142
7143        Ok(StagedCall {
7144            store: store.0.id(),
7145            rx,
7146            _marker: PhantomData,
7147            group,
7148        })
7149    }
7150}
7151
7152impl<R> Future for StagedCall<R>
7153where
7154    R: 'static,
7155{
7156    type Output = Result<R>;
7157
7158    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7159        check_ambient_store(self.store);
7160        Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7161            Ok(r) => match r.downcast() {
7162                Ok(r) => Ok(*r),
7163                Err(_) => bail_bug!("wrong type of value produced"),
7164            },
7165            Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7166        })
7167    }
7168}
7169
7170/// Queue a call previously prepared using `prepare_call` to be run as part of
7171/// the associated `ComponentInstance`'s event loop.
7172fn stage_call0<T: 'static>(
7173    store: StoreContextMut<T>,
7174    handle: Func,
7175    guest_thread: QualifiedThreadId,
7176    param_count: usize,
7177) -> Result<()> {
7178    let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7179    let is_concurrent = raw_options.async_;
7180    let callback = raw_options.callback;
7181    let instance = handle.instance();
7182    let callee = handle.lifted_core_func(store.0);
7183    let post_return = raw_options
7184        .post_return
7185        .map(|i| instance.id().get(store.0).runtime_post_return(i));
7186    let callback = callback.map(|i| {
7187        let instance = instance.id().get(store.0);
7188        SendSyncPtr::new(instance.runtime_callback(i))
7189    });
7190
7191    log::trace!("queueing call {guest_thread:?}");
7192
7193    // SAFETY: `callee`, `callback`, and `post_return` are valid pointers
7194    // (with signatures appropriate for this call) and will remain valid as
7195    // long as this instance is valid.
7196    unsafe {
7197        instance.stage_call(
7198            store,
7199            guest_thread,
7200            SendSyncPtr::new(callee),
7201            param_count,
7202            1,
7203            is_concurrent,
7204            callback,
7205            post_return.map(SendSyncPtr::new),
7206            true,
7207        )
7208    }
7209}
7210
7211#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7212mod tests {
7213    use super::*;
7214    use crate::component::{Component, Linker};
7215    use crate::store::AsStoreOpaque;
7216    use crate::{Config, Engine};
7217
7218    fn host_subtask(
7219        state: HostTaskState,
7220        event: Option<Event>,
7221    ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7222        let mut config = Config::new();
7223        config.wasm_component_model_async(true);
7224        let engine = Engine::new(&config)?;
7225        let component = Component::new(&engine, "(component)")?;
7226        let mut store = Store::new(&engine, ());
7227        let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7228        let store_opaque = store.as_store_opaque();
7229        let concurrent_state = store_opaque.concurrent_state_mut()?;
7230        // These tests create an isolated host task without a guest caller, so
7231        // give it its own task group instead of inheriting the caller's group.
7232        let group = concurrent_state.make_task_group()?;
7233        let task = concurrent_state.push(HostTask {
7234            common: WaitableCommon::default(),
7235            call_context: CallContext::default(),
7236            state,
7237            group,
7238        })?;
7239        let handle = store_opaque
7240            .handle_table_insertion(
7241                instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7242            )
7243            .subtask_host(task.rep())?;
7244        let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7245        common.handle = Some(handle);
7246        common.event = event;
7247        Ok((store, instance, task, handle))
7248    }
7249
7250    #[test]
7251    fn host_subtask_drop_during_cancellation() -> Result<()> {
7252        for abort_completed in [false, true] {
7253            let (handle, future) = JoinHandle::run(future::pending::<()>());
7254            let mut future = pin!(future);
7255            let (mut store, instance, task, handle) =
7256                host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7257            let store = store.as_store_opaque();
7258            let caller = RuntimeComponentInstanceIndex::from_u32(0);
7259            assert_eq!(
7260                instance.subtask_cancel(store, caller, true, handle)?,
7261                BLOCKED
7262            );
7263            if abort_completed {
7264                // Even after the abort resolves, the completion worker still
7265                // needs the task entry to lower the result and publish an event.
7266                assert!(matches!(
7267                    future
7268                        .as_mut()
7269                        .poll(&mut Context::from_waker(Waker::noop())),
7270                    Poll::Ready(None),
7271                ));
7272            }
7273            for async_ in [false, true] {
7274                let err = instance
7275                    .subtask_cancel(store, caller, async_, handle)
7276                    .unwrap_err();
7277                assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7278            }
7279            let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7280            assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7281            assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7282        }
7283        Ok(())
7284    }
7285
7286    #[test]
7287    fn host_subtask_cancel_after_completion() -> Result<()> {
7288        for async_ in [false, true] {
7289            let (mut store, instance, task, handle) = host_subtask(
7290                HostTaskState::CalleeDone { cancelled: false },
7291                Some(Event::Subtask {
7292                    status: Status::Returned,
7293                }),
7294            )?;
7295            let store = store.as_store_opaque();
7296            let caller = RuntimeComponentInstanceIndex::from_u32(0);
7297            assert_eq!(
7298                instance.subtask_cancel(store, caller, async_, handle)?,
7299                Status::Returned as u32,
7300            );
7301            let err = instance
7302                .subtask_cancel(store, caller, async_, handle)
7303                .unwrap_err();
7304            assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7305            instance.subtask_drop(store, caller, handle)?;
7306            assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7307        }
7308        Ok(())
7309    }
7310
7311    #[test]
7312    fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7313        for (cancelled, status) in [
7314            (false, Status::Returned),
7315            (true, Status::Returned),
7316            (true, Status::ReturnCancelled),
7317        ] {
7318            for delivered in [false, true] {
7319                let event = if delivered {
7320                    None
7321                } else {
7322                    Some(Event::Subtask { status })
7323                };
7324                let (mut store, instance, task, handle) =
7325                    host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7326                let store = store.as_store_opaque();
7327                let result = instance.subtask_drop(
7328                    store,
7329                    RuntimeComponentInstanceIndex::from_u32(0),
7330                    handle,
7331                );
7332                if delivered {
7333                    result?;
7334                    assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7335                } else {
7336                    let err = result.unwrap_err();
7337                    assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7338                    assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7339                }
7340            }
7341        }
7342        Ok(())
7343    }
7344}