Skip to main content

wasmtime/runtime/component/
concurrent.rs

1//! Runtime support for the Component Model Async ABI.
2//!
3//! This module and its submodules provide host runtime support for Component
4//! Model Async features such as async-lifted exports, async-lowered imports,
5//! streams, futures, and related intrinsics.  See [the Async
6//! Explainer](https://github.com/WebAssembly/component-model/blob/main/design/mvp/Concurrency.md)
7//! for a high-level overview.
8//!
9//! At the core of this support is an event loop which schedules and switches
10//! between guest tasks and any host tasks they create.  Each
11//! `Store` will have at most one event loop running at any given
12//! time, and that loop may be suspended and resumed by the host embedder using
13//! e.g. `StoreContextMut::run_concurrent`.  The `StoreContextMut::poll_until`
14//! function contains the loop itself, while the
15//! `StoreOpaque::concurrent_state` field holds its state.
16//!
17//! # Public API Overview
18//!
19//! ## Top-level API (e.g. kicking off host->guest calls and driving the event loop)
20//!
21//! - `[Typed]Func::call_concurrent`: Start a host->guest call to an
22//! async-lifted or sync-lifted import, creating a guest task.
23//!
24//! - `StoreContextMut::run_concurrent`: Run the event loop for the specified
25//! instance, allowing any and all tasks belonging to that instance to make
26//! progress.
27//!
28//! - `StoreContextMut::spawn`: Run a background task as part of the event loop
29//! for the specified instance.
30//!
31//! - `{Future,Stream}Reader::new`: Create a new Component Model `future` or
32//! `stream` which may be passed to the guest.  This takes a
33//! `{Future,Stream}Producer` implementation which will be polled for items when
34//! the consumer requests them.
35//!
36//! - `{Future,Stream}Reader::pipe`: Consume a `future` or `stream` by
37//! connecting it to a `{Future,Stream}Consumer` which will consume any items
38//! produced by the write end.
39//!
40//! ## Host Task API (e.g. implementing concurrent host functions and background tasks)
41//!
42//! - `LinkerInstance::func_wrap_concurrent`: Register a concurrent host
43//! function with the linker.  That function will take an `Accessor` as its
44//! first parameter, which provides access to the store between (but not across)
45//! await points.
46//!
47//! - `Accessor::with`: Access the store and its associated data.
48//!
49//! - `Accessor::spawn`: Run a background task as part of the event loop for the
50//! store.  This is equivalent to `StoreContextMut::spawn` but more convenient to use
51//! in host functions.
52
53use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57    HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70    CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71    InstanceState,
72};
73use crate::vm::{
74    AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98    CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99    MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100    RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101    TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102    TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113    Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114    FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115    StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136/// Constant defined in the Component Model spec to indicate that the async
137/// intrinsic (e.g. `future.write`) has not yet completed.
138const BLOCKED: u32 = 0xffff_ffff;
139
140/// Corresponds to `CallState` in the upstream spec.
141#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143    Starting = 0,
144    Started = 1,
145    Returned = 2,
146    StartCancelled = 3,
147    ReturnCancelled = 4,
148}
149
150impl Status {
151    /// Packs this status and the optional `waitable` provided into a 32-bit
152    /// result that the canonical ABI requires.
153    ///
154    /// The low 4 bits are reserved for the status while the upper 28 bits are
155    /// the waitable, if present.
156    pub fn pack(self, waitable: Option<u32>) -> u32 {
157        assert!(matches!(self, Status::Returned) == waitable.is_none());
158        let waitable = waitable.unwrap_or(0);
159        assert!(waitable < (1 << 28));
160        (waitable << 4) | (self as u32)
161    }
162}
163
164/// Corresponds to `EventCode` in the Component Model spec, plus related payload
165/// data.
166#[derive(Clone, Copy, Debug)]
167enum Event {
168    None,
169    Subtask {
170        status: Status,
171    },
172    StreamRead {
173        code: ReturnCode,
174        pending: Option<(TypeStreamTableIndex, u32)>,
175    },
176    StreamWrite {
177        code: ReturnCode,
178        pending: Option<(TypeStreamTableIndex, u32)>,
179    },
180    FutureRead {
181        code: ReturnCode,
182        pending: Option<(TypeFutureTableIndex, u32)>,
183    },
184    FutureWrite {
185        code: ReturnCode,
186        pending: Option<(TypeFutureTableIndex, u32)>,
187    },
188    Cancelled,
189}
190
191impl Event {
192    /// Lower this event to core Wasm integers for delivery to the guest.
193    ///
194    /// Note that the waitable handle, if any, is assumed to be lowered
195    /// separately.
196    fn parts(self) -> (u32, u32) {
197        const EVENT_NONE: u32 = 0;
198        const EVENT_SUBTASK: u32 = 1;
199        const EVENT_STREAM_READ: u32 = 2;
200        const EVENT_STREAM_WRITE: u32 = 3;
201        const EVENT_FUTURE_READ: u32 = 4;
202        const EVENT_FUTURE_WRITE: u32 = 5;
203        const EVENT_CANCELLED: u32 = 6;
204        match self {
205            Event::None => (EVENT_NONE, 0),
206            Event::Cancelled => (EVENT_CANCELLED, 0),
207            Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208            Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209            Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210            Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211            Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212        }
213    }
214}
215
216/// Corresponds to `CallbackCode` in the spec.
217mod callback_code {
218    pub const EXIT: u32 = 0;
219    pub const YIELD: u32 = 1;
220    pub const WAIT: u32 = 2;
221}
222
223// Flags passed to the `start-call` intrinsic.
224const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227/// Provides access to either store data (via the `get` method) or the store
228/// itself (via [`AsContext`]/[`AsContextMut`]), as well as the component
229/// instance to which the current host task belongs.
230///
231/// See [`Accessor::with`] for details.
232pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233    store: StoreContextMut<'a, T>,
234    get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239    D: HasData + ?Sized,
240    T: 'static,
241{
242    /// Creates a new [`Access`] from its component parts.
243    pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244        Self { store, get_data }
245    }
246
247    /// Get mutable access to the store data.
248    pub fn data_mut(&mut self) -> &mut T {
249        self.store.data_mut()
250    }
251
252    /// Get mutable access to the store data.
253    pub fn get(&mut self) -> D::Data<'_> {
254        (self.get_data)(self.data_mut())
255    }
256
257    /// Spawn a background task.
258    ///
259    /// See [`Accessor::spawn`] for details.
260    pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261    where
262        T: 'static,
263    {
264        let accessor = Accessor {
265            get_data: self.get_data,
266            token: StoreToken::new(self.store.as_context_mut()),
267        };
268        self.store
269            .as_context_mut()
270            .spawn_with_accessor(accessor, task)
271    }
272
273    /// Returns the getter this accessor is using to project from `T` into
274    /// `D::Data`.
275    pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276        self.get_data
277    }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282    D: HasData + ?Sized,
283    T: 'static,
284{
285    type Data = T;
286
287    fn as_context(&self) -> StoreContext<'_, T> {
288        self.store.as_context()
289    }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294    D: HasData + ?Sized,
295    T: 'static,
296{
297    fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298        self.store.as_context_mut()
299    }
300}
301
302/// Provides scoped mutable access to store data in the context of a concurrent
303/// host task future.
304///
305/// This allows multiple host task futures to execute concurrently and access
306/// the store between (but not across) `await` points.
307///
308/// # Rationale
309///
310/// This structure is sort of like `&mut T` plus a projection from `&mut T` to
311/// `D::Data<'_>`. The problem this is solving, however, is that it does not
312/// literally store these values. The basic problem is that when a concurrent
313/// host future is being polled it has access to `&mut T` (and the whole
314/// `Store`) but when it's not being polled it does not have access to these
315/// values. This reflects how the store is only ever polling one future at a
316/// time so the store is effectively being passed between futures.
317///
318/// Rust's `Future` trait, however, has no means of passing a `Store`
319/// temporarily between futures. The [`Context`](core::task::Context) type does
320/// not have the ability to attach arbitrary information to it at this time.
321/// This type, [`Accessor`], is used to bridge this expressivity gap.
322///
323/// The [`Accessor`] type here represents the ability to acquire, temporarily in
324/// a synchronous manner, the current store. The [`Accessor::with`] function
325/// yields an [`Access`] which can be used to access [`StoreContextMut`], `&mut
326/// T`, or `D::Data<'_>`. Note though that [`Accessor::with`] intentionally does
327/// not take an `async` closure as its argument, instead it's a synchronous
328/// closure which must complete during on run of `Future::poll`. This reflects
329/// how the store is temporarily made available while a host future is being
330/// polled.
331///
332/// # Implementation
333///
334/// This type does not actually store `&mut T` nor `StoreContextMut<T>`, and
335/// this type additionally doesn't even have a lifetime parameter. This is
336/// instead a representation of proof of the ability to acquire these while a
337/// future is being polled. Wasmtime will, when it polls a host future,
338/// configure ambient state such that the `Accessor` that a future closes over
339/// will work and be able to access the store.
340///
341/// This has a number of implications for users such as:
342///
343/// * It's intentional that `Accessor` cannot be cloned, it needs to stay within
344///   the lifetime of a single future.
345/// * A future is expected to, however, close over an `Accessor` and keep it
346///   alive probably for the duration of the entire future.
347/// * Different host futures will be given different `Accessor`s, and that's
348///   intentional.
349/// * The `Accessor` type is `Send` and `Sync` irrespective of `T` which
350///   alleviates some otherwise required bounds to be written down.
351///
352/// # Using `Accessor` in `Drop`
353///
354/// The methods on `Accessor` are only expected to work in the context of
355/// `Future::poll` and are not guaranteed to work in `Drop`. This is because a
356/// host future can be dropped at any time throughout the system and Wasmtime
357/// store context is not necessarily available at that time. It's recommended to
358/// not use `Accessor` methods in anything connected to a `Drop` implementation
359/// as they will panic and have unintended results. If you run into this though
360/// feel free to file an issue on the Wasmtime repository.
361pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363    D: HasData + ?Sized,
364{
365    token: StoreToken<T>,
366    get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369/// A helper trait to take any type of accessor-with-data in functions.
370///
371/// This trait is similar to [`AsContextMut`] except that it's used when
372/// working with an [`Accessor`] instead of a [`StoreContextMut`]. The
373/// [`Accessor`] is the main type used in concurrent settings and is passed to
374/// functions such as [`Func::call_concurrent`].
375///
376/// This trait is implemented for [`Accessor`] and `&T` where `T` implements
377/// this trait. This effectively means that regardless of the `D` in
378/// `Accessor<T, D>` it can still be passed to a function which just needs a
379/// store accessor.
380///
381/// Acquiring an [`Accessor`] can be done through
382/// [`StoreContextMut::run_concurrent`] for example or in a host function
383/// through
384/// [`Linker::func_wrap_concurrent`](crate::component::LinkerInstance::func_wrap_concurrent).
385pub trait AsAccessor {
386    /// The `T` in `Store<T>` that this accessor refers to.
387    type Data: 'static;
388
389    /// The `D` in `Accessor<T, D>`, or the projection out of
390    /// `Self::Data`.
391    type AccessorData: HasData + ?Sized;
392
393    /// Returns the accessor that this is referring to.
394    fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398    type Data = T::Data;
399    type AccessorData = T::AccessorData;
400
401    fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402        T::as_accessor(self)
403    }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407    type Data = T;
408    type AccessorData = D;
409
410    fn as_accessor(&self) -> &Accessor<T, D> {
411        self
412    }
413}
414
415// Note that it is intentional at this time that `Accessor` does not actually
416// store `&mut T` or anything similar. This distinctly enables the `Accessor`
417// structure to be both `Send` and `Sync` regardless of what `T` is (or `D` for
418// that matter). This is used to ergonomically simplify bindings where the
419// majority of the time `Accessor` is closed over in a future which then needs
420// to be `Send` and `Sync`. To avoid needing to write `T: Send` everywhere (as
421// you already have to write `T: 'static`...) it helps to avoid this.
422//
423// Note as well that `Accessor` doesn't actually store its data at all. Instead
424// it's more of a "proof" of what can be accessed from TLS. API design around
425// `Accessor` and functions like `Linker::func_wrap_concurrent` are
426// intentionally made to ensure that `Accessor` is ideally only used in the
427// context that TLS variables are actually set. For example host functions are
428// given `&Accessor`, not `Accessor`, and this prevents them from persisting
429// the value outside of a future. Within the future the TLS variables are all
430// guaranteed to be set while the future is being polled.
431//
432// Finally though this is not an ironclad guarantee, but nor does it need to be.
433// The TLS APIs are designed to panic or otherwise model usage where they're
434// called recursively or similar. It's hoped that code cannot be constructed to
435// actually hit this at runtime but this is not a safety requirement at this
436// time.
437const _: () = {
438    const fn assert<T: Send + Sync>() {}
439    assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443    /// Creates a new `Accessor` backed by the specified functions.
444    ///
445    /// - `get`: used to retrieve the store
446    ///
447    /// - `get_data`: used to "project" from the store's associated data to
448    /// another type (e.g. a field of that data or a wrapper around it).
449    ///
450    /// - `spawn`: used to queue spawned background tasks to be run later
451    pub(crate) fn new(token: StoreToken<T>) -> Self {
452        Self {
453            token,
454            get_data: |x| x,
455        }
456    }
457}
458
459impl<T, D> Accessor<T, D>
460where
461    D: HasData + ?Sized,
462{
463    /// Run the specified closure, passing it mutable access to the store.
464    ///
465    /// This function is one of the main building blocks of the [`Accessor`]
466    /// type. This yields synchronous, blocking, access to the store via an
467    /// [`Access`]. The [`Access`] implements [`AsContextMut`] in addition to
468    /// providing the ability to access `D` via [`Access::get`]. Note that the
469    /// `fun` here is given only temporary access to the store and `T`/`D`
470    /// meaning that the return value `R` here is not allowed to capture borrows
471    /// into the two. If access is needed to data within `T` or `D` outside of
472    /// this closure then it must be `clone`d out, for example.
473    ///
474    /// # Panics
475    ///
476    /// This function will panic if it is call recursively with any other
477    /// accessor already in scope. For example if `with` is called within `fun`,
478    /// then this function will panic. It is up to the embedder to ensure that
479    /// this does not happen.
480    pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481        tls::get(|vmstore| {
482            fun(Access {
483                store: self.token.as_context_mut(vmstore),
484                get_data: self.get_data,
485            })
486        })
487    }
488
489    /// Returns the getter this accessor is using to project from `T` into
490    /// `D::Data`.
491    pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492        self.get_data
493    }
494
495    /// Changes this accessor to access `D2` instead of the current type
496    /// parameter `D`.
497    ///
498    /// This changes the underlying data access from `T` to `D2::Data<'_>`.
499    ///
500    /// # Panics
501    ///
502    /// When using this API the returned value is disconnected from `&self` and
503    /// the lifetime binding the `self` argument. An `Accessor` only works
504    /// within the context of the closure or async closure that it was
505    /// originally given to, however. This means that due to the fact that the
506    /// returned value has no lifetime connection it's possible to use the
507    /// accessor outside of `&self`, the original accessor, and panic.
508    ///
509    /// The returned value should only be used within the scope of the original
510    /// `Accessor` that `self` refers to.
511    pub fn with_getter<D2: HasData>(
512        &self,
513        get_data: fn(&mut T) -> D2::Data<'_>,
514    ) -> Accessor<T, D2> {
515        Accessor {
516            token: self.token,
517            get_data,
518        }
519    }
520
521    /// Spawn a background task which will receive an `&Accessor<T, D>` and
522    /// run concurrently with any other tasks in progress for the current
523    /// store.
524    ///
525    /// This is particularly useful for host functions which return a `stream`
526    /// or `future` such that the code to write to the write end of that
527    /// `stream` or `future` must run after the function returns.
528    ///
529    /// The returned [`JoinHandle`] may be used to cancel the task.
530    ///
531    /// # Panics
532    ///
533    /// Panics if called within a closure provided to the [`Accessor::with`]
534    /// function. This can only be called outside an active invocation of
535    /// [`Accessor::with`].
536    pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537    where
538        T: 'static,
539    {
540        let accessor = self.clone_for_spawn();
541        self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542    }
543
544    fn clone_for_spawn(&self) -> Self {
545        Self {
546            token: self.token,
547            get_data: self.get_data,
548        }
549    }
550
551    /// Polls to see if this store contains any "interesting" tasks still within
552    /// it.
553    ///
554    /// Returns `Poll::Ready(())` if there are no more interesting tasks, and
555    /// otherwise returns `Poll::Pending`. If pending is returned then whenever
556    /// the last remaining "interesting" task has exited the provided context's
557    /// waker will be notified. Note that only the waker passed to the last call
558    /// to `poll_no_interesting_tasks` for the store will be notified, so this
559    /// is only appropriate to use once-at-a-time per store.
560    ///
561    /// The component model specification, as of this current date, does not
562    /// have a distinction between "interesting" tasks and not. The current
563    /// intention is that in a future revision of the component model this will
564    /// be distinguished at the component ABI level where tasks will be able to
565    /// flag themselves as "interesting" optionally. Additionally extra work can
566    /// be opted-in to being "interesting".
567    ///
568    /// For now what this means is that all component model tasks within this
569    /// store are considered interesting. This specifically includes the entire
570    /// duration of a task, so even all of the time after a task has returned
571    /// but before it has exited. This means that this function is, today,
572    /// effectively a proxy for "are there any more tasks still running in this
573    /// store". This can be used by embedders to determine whether there's any
574    /// more work going on, even in the background, for a particular guest.
575    /// Hosts can use this as a signal that the guest wants to stay alive a
576    /// little longer, even after a task has returned.
577    ///
578    /// In the future this predicate won't include all tasks in this store. Some
579    /// tasks will be able to flag themselves as not interesting, meaning that
580    /// when this returns ready it'd be possible that there are still tasks
581    /// remaining in the store.
582    ///
583    /// Note that at this time spawned threads within a task are always
584    /// considered uninteresting. If this function returns ready, then spawned
585    /// threads may still be in the store.
586    pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587        self.with(|mut access| {
588            let store = access.as_context_mut().0;
589            let state = store.concurrent_state_mut_without_forcing_current_thread();
590            if state.interesting_tasks == 0 {
591                Poll::Ready(())
592            } else {
593                state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594                Poll::Pending
595            }
596        })
597    }
598
599    /// Poll to see if the component instance corresponding to the specified
600    /// function is ready to run a concurrent call without queuing it.
601    ///
602    /// This will return `Poll::Ready(())` only if all of the following are true:
603    ///
604    /// - The store to which the instance belongs has no work items queued
605    ///   (e.g. events to deliver to the guest, etc.)
606    ///
607    /// - The instance does not have backpressure enabled.
608    ///
609    /// - The instance does not have a sync call in progress (which implicitly
610    ///   enables backpressure).
611    ///
612    /// Otherwise, it will return `Poll::Pending`.  If pending is returned then
613    /// whenever _any_ instance becomes ready for a call the provided context's
614    /// waker will be notified.  Note that only the waker passed to the last
615    /// call to `poll_ready_for_concurrent_call` for the store will be notified
616    /// (regardless of whether the same or different `Func` is specified
617    /// relative to earlier calls), so this is only appropriate to use
618    /// once-at-a-time per store.  Again, note that the waker may be notified
619    /// when _any_ instance becomes callable (i.e. not necessarily the last one
620    /// polled), so this function must be called again to determine if the
621    /// instance of interest is ready.
622    pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623        self.with(|mut access| {
624            let store = access.as_context_mut().0;
625
626            {
627                let state = store.concurrent_state_mut_without_forcing_current_thread();
628                if state.switch_item.is_some()
629                    || !state.high_priority.is_empty()
630                    || !state.low_priority.is_empty()
631                {
632                    state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633                    return Poll::Pending;
634                }
635            }
636
637            let (_, _, _, raw_options) = func.abi_info(store);
638            let instance = func.instance().runtime_instance(raw_options.instance);
639            let state = store.instance_state(instance).concurrent_state();
640            if state.backpressure == 0 {
641                Poll::Ready(())
642            } else {
643                store
644                    .concurrent_state_mut_without_forcing_current_thread()
645                    .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646                Poll::Pending
647            }
648        })
649    }
650}
651
652/// Represents an async closure which may be provided to `Accessor::spawn`,
653/// `Accessor::forward`, or `StoreContextMut::spawn`.
654// TODO: Replace this with `core::ops::AsyncFnOnce` when we are able to put `Send`
655// bound on the unnamed `Future` directly.
656//
657// As of this writing, it's not possible to specify e.g. `Send` and `Sync`
658// bounds on the `Future` type returned by an `AsyncFnOnce`.  Also, using `F:
659// Future<Output = Result<()>> + Send + Sync, FN: FnOnce(&Accessor<T>) -> F +
660// Send + Sync + 'static` fails with a type mismatch error as we cannot describe
661// that `F` should have `&Accessor<T>`'s unnamed lifetime
662//
663// Instead, this trait is used as a workaround for this limitation, the bound on `Self`
664// implementing `AsyncFnOnce()` is required for Rust to automatically infer that an async
665// closure is to be provided wherever we are accepting `impl for<'fut> AccessorTask<'fut, T, D>`
666// as argument. Otherwise, users will have to fully qualify the closure types before it
667// is accepted as a valid value. This also means that it is not intended for a user
668// to manually implement this trait for any arbitrary type, since they would first
669// have to implement `AsyncFnOnce`, which is unstable.
670//
671// The blanket implementation for this trait will ensure that `Self` is an async closure
672// that returns a `Future` that is `Send`, and lives as long as `&Accessor<T, D>`
673pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674    AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676    D: HasData + ?Sized,
677{
678    /// Run the task.
679    fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684    T: 'static,
685    F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686    F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687    Fut: Future<Output = Result<()>> + Send + 'fut,
688    D: HasData,
689{
690    fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691        (self)(accessor)
692    }
693}
694
695/// Represents parameter and result metadata for the caller side of a
696/// guest->guest call orchestrated by a fused adapter.
697enum CallerInfo {
698    /// Metadata for a call to an async-lowered import
699    Async {
700        params: Vec<ValRaw>,
701        has_result: bool,
702    },
703    /// Metadata for a call to an sync-lowered import
704    Sync {
705        params: Vec<ValRaw>,
706        result_count: u32,
707    },
708}
709
710/// Indicates how a guest task is waiting on a waitable set.
711enum WaitMode {
712    /// The guest task is waiting using `task.wait`
713    Fiber(StoreFiber<'static>),
714    /// The guest task is waiting via a callback declared as part of an
715    /// async-lifted export.
716    Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721        match self {
722            Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723            Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724        }
725    }
726}
727
728/// Represents the reason a fiber is suspending itself.
729#[derive(Debug)]
730enum SuspendReason {
731    /// The fiber is waiting for an event to be delivered to the specified
732    /// waitable set or task.
733    Waiting {
734        set: TableId<WaitableSet>,
735        thread: QualifiedThreadId,
736    },
737    /// The fiber is waiting for a subtask to suspend or exit, e.g. for a
738    /// guest-to-guest call or a `subtask.cancel`.
739    YieldingToSubtask { thread: QualifiedThreadId },
740    /// The fiber has finished handling its most recent work item and is waiting
741    /// for another (or to be dropped if it is no longer needed).
742    NeedWork,
743    /// The fiber is yielding and should be resumed once other tasks have had a
744    /// chance to run.
745    Yielding { thread: QualifiedThreadId },
746    /// The fiber was explicitly suspended with a call to `thread.suspend` or
747    /// `thread.switch-to`.
748    ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751/// Represents a pending call into guest code for a given guest task.
752enum GuestCallKind {
753    /// Indicates there's an event to deliver to the task, possibly related to a
754    /// waitable set the task has been waiting on or polling.
755    DeliverEvent {
756        /// The instance to which the task belongs.
757        instance: Instance,
758        /// The waitable set the event belongs to, if any.
759        ///
760        /// If this is `None` the event will be waiting in the
761        /// `GuestTask::event` field for the task.
762        set: Option<TableId<WaitableSet>>,
763    },
764    /// Indicates that a new guest task call is pending and may be executed
765    /// using the specified closure.
766    StartImplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
767    StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
768}
769
770impl fmt::Debug for GuestCallKind {
771    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
772        match self {
773            Self::DeliverEvent { instance, set } => f
774                .debug_struct("DeliverEvent")
775                .field("instance", instance)
776                .field("set", set)
777                .finish(),
778            Self::StartImplicit(_) => f.debug_tuple("StartImplicit").finish(),
779            Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
780        }
781    }
782}
783
784/// The target of a suspension intrinsic.
785#[derive(Copy, Clone, Debug)]
786pub enum SuspensionTarget {
787    Resume(u32),
788    Promote(u32),
789    None,
790}
791
792/// Behavior for `resume_thread`.
793#[derive(Copy, Clone, Debug)]
794pub enum ResumeThread {
795    Promote,
796    Resume,
797    ResumeLater,
798}
799
800/// Represents a pending call into guest code for a given guest thread.
801#[derive(Debug)]
802struct GuestCall {
803    thread: QualifiedThreadId,
804    kind: GuestCallKind,
805}
806
807impl GuestCall {
808    /// Returns whether or not the call is ready to run.
809    ///
810    /// A call will not be ready to run if either:
811    ///
812    /// - the (sub-)component instance to be called has already been entered and
813    ///   cannot be reentered until an in-progress call completes, and the task
814    ///   requires the exclusive lock
815    ///
816    /// - the call is for a not-yet started task and the (sub-)component
817    ///   instance to be called has backpressure enabled
818    fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
819        let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
820        let async_typed = task.async_typed;
821        let async_lifted = task.async_lifted;
822        let has_callback = task.callback.is_some();
823        let instance = task.instance;
824        let state = store.instance_state(instance).concurrent_state();
825
826        let ready = match &self.kind {
827            GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
828            GuestCallKind::StartImplicit(_) => {
829                if async_typed {
830                    // Sync- and callback-lifted exports need the instance's
831                    // exclusive lock to start, but stackful ones don't.
832                    let needs_exclusive = !async_lifted || has_callback;
833                    !((needs_exclusive && state.do_not_enter) || state.backpressure > 0)
834                } else {
835                    true
836                }
837            }
838            GuestCallKind::StartExplicit(_) => true,
839        };
840        log::trace!(
841            "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {})",
842            state.do_not_enter,
843            state.backpressure
844        );
845        Ok(ready)
846    }
847}
848
849/// Job to be run on a worker fiber.
850enum WorkerItem {
851    GuestCall(GuestCall),
852    Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
853}
854
855/// Represents a pending work item to be handled by the event loop for a given
856/// component instance.
857enum WorkItem {
858    /// A host task to be pushed to `ConcurrentState::futures`.
859    PushFuture(AlwaysMut<HostTaskFuture>),
860    /// A fiber to resume.
861    ResumeFiber {
862        instance: RuntimeInstance,
863        thread: QualifiedThreadId,
864        fiber: StoreFiber<'static>,
865    },
866    /// A thread to resume.
867    ResumeThread {
868        instance: RuntimeInstance,
869        thread: QualifiedThreadId,
870    },
871    /// A pending call into guest code for a given guest task.
872    GuestCall {
873        instance: RuntimeInstance,
874        call: GuestCall,
875    },
876    /// A job to run on a worker fiber.
877    WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
878}
879
880/// A fiber temporarily outside the store's scheduler state.
881///
882/// Construct this before returning a future so cancellation before its first
883/// poll also disposes of the fiber. Successful handoffs take the fiber only
884/// after all fallible destination checks have completed.
885struct ResumingFiber<'a> {
886    store: &'a mut StoreOpaque,
887    fiber: Option<StoreFiber<'static>>,
888}
889
890impl Drop for ResumingFiber<'_> {
891    fn drop(&mut self) {
892        if let Some(mut fiber) = self.fiber.take() {
893            fiber.dispose(self.store);
894        }
895    }
896}
897
898impl fmt::Debug for WorkItem {
899    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
900        match self {
901            Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
902            Self::ResumeFiber {
903                instance, thread, ..
904            } => f
905                .debug_struct("ResumeFiber")
906                .field("instance", instance)
907                .field("thread", thread)
908                .finish(),
909            Self::ResumeThread { instance, thread } => f
910                .debug_struct("ResumeThread")
911                .field("instance", instance)
912                .field("thread", thread)
913                .finish(),
914            Self::GuestCall { instance, call } => f
915                .debug_struct("GuestCall")
916                .field("instance", instance)
917                .field("call", call)
918                .finish(),
919            Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
920        }
921    }
922}
923
924/// Whether a suspension intrinsic was cancelled or completed
925#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
926pub(crate) enum WaitResult {
927    Cancelled,
928    Completed,
929}
930
931/// Poll the specified future until it completes on behalf of a guest->host call
932/// using a sync-lowered import.
933///
934/// This is similar to `Instance::first_poll` except it's for sync-lowered
935/// imports, meaning we don't need to handle cancellation and we can block the
936/// caller until the task completes, at which point the caller can handle
937/// lowering the result to the guest's stack and linear memory.
938pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
939    store: &mut dyn VMStore,
940    host_task: EnteredHostTask,
941    future: impl Future<Output = Result<R>> + Send + 'static,
942) -> Result<R> {
943    // Poll the future once before creating a host task. The host task will be
944    // created lazily if it's needed during the poll and otherwise will be
945    // created if the future suspends.  We can use a dummy `Waker` here because
946    // we'll add the future to `ConcurrentState::futures` and poll it
947    // automatically from the event loop if it doesn't complete immediately
948    // here.
949    let mut future = Box::pin(future);
950    let poll = tls::set(store, || {
951        future
952            .as_mut()
953            .poll(&mut Context::from_waker(&Waker::noop()))
954    });
955
956    let caller = match host_task {
957        Some(caller) => caller,
958        None => bail_bug!("host task wasn't created but should have been"),
959    };
960
961    let task = match poll {
962        // It completed immediately, so no persistent host task is needed.
963        Poll::Ready(result) => return result,
964
965        // It did not complete immediately; create the host task and add it to
966        // `ConcurrentState::futures` so it will be polled via the event loop;
967        // then use `GuestThread::sync_call_set` to wait for the task to
968        // complete, suspending the current fiber until it does so.
969        Poll::Pending => {
970            let Some(task) = store.materialize_host_task_id()? else {
971                bail_bug!("current thread is not a host thread")
972            };
973
974            // Wrap the future in a closure which will stash its result in the
975            // host task and resume this fiber when it completes.
976            let future = Box::pin(async move {
977                let result = run_with_host_task_set(task, future).await??;
978                tls::get(move |store| {
979                    let state = store.concurrent_state_mut()?;
980                    let host_state = &mut state.get_mut(task)?.state;
981                    assert!(matches!(host_state, HostTaskState::CalleeStarted));
982                    *host_state = HostTaskState::CalleeFinished(Box::new(result));
983
984                    Waitable::Host(task).set_event(
985                        state,
986                        Some(Event::Subtask {
987                            status: Status::Returned,
988                        }),
989                    )?;
990
991                    Ok(())
992                })
993            }) as HostTaskFuture;
994
995            let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
996            store.switch_or_trap_if_may_not_suspend(caller_instance)?;
997
998            let state = store.concurrent_state_mut()?;
999            state.push_future(future);
1000
1001            let set = state.get_mut(caller.thread)?.sync_call_set;
1002            Waitable::Host(task).join(state, Some(set))?;
1003
1004            store.suspend(SuspendReason::Waiting {
1005                set,
1006                thread: caller,
1007            })?;
1008
1009            // Remove the `task` from the `sync_call_set` to ensure that when
1010            // this function returns and the task is deleted that there are no
1011            // more lingering references to this host task.
1012            Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1013            task
1014        }
1015    };
1016
1017    // Retrieve and return the result.
1018    let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1019    match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1020        HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1021            Ok(result) => *result,
1022            Err(_) => bail_bug!("host task finished with wrong type of result"),
1023        }),
1024        _ => bail_bug!("unexpected host task state after completion"),
1025    }
1026}
1027
1028/// Execute the specified guest call.
1029fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1030    match call.kind {
1031        GuestCallKind::DeliverEvent { instance, set } => {
1032            // This thread's wait on `set`, if any, which started in
1033            // `Instance::wait_with_callback`, ends here.
1034            if let Some(set) = set {
1035                store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1036            }
1037            let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1038                Some(pair) => pair,
1039                None => match set {
1040                    // The event which woke this thread up was taken by another
1041                    // thread (e.g. via `waitable-set.poll`) or its waitable was
1042                    // removed from the set before this thread had a chance to
1043                    // run. Regardless, keep waiting.
1044                    Some(set) => {
1045                        log::trace!(
1046                            "event for {:?} on {set:?} no longer present; waiting again",
1047                            call.thread
1048                        );
1049                        return instance.wait_with_callback(
1050                            store.store_opaque_mut(),
1051                            call.thread,
1052                            set,
1053                        );
1054                    }
1055                    // When a thread yields it doesn't wait on a set and the
1056                    // event to be delivered isn't stored anywhere, and this is
1057                    // just handling that case so indicate that the yield is
1058                    // complete.
1059                    None => (Event::None, None),
1060                },
1061            };
1062            let state = store.concurrent_state_mut()?;
1063            // If this thread was woken up by an event in `set` but is instead
1064            // receiving its own task's event (e.g. a cancellation), then the
1065            // set's event is left for another thread waiting on it, so wake
1066            // the next waiter up if there is one.
1067            if let Some(set) = set
1068                && waitable.is_none()
1069                && !state.get_mut(set)?.ready.is_empty()
1070            {
1071                state.wake_waiter(set)?;
1072            }
1073            let task = state.get_mut(call.thread.task)?;
1074            let runtime_instance = task.instance;
1075            let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1076
1077            log::trace!(
1078                "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1079                call.thread,
1080            );
1081
1082            instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1083                invoke_callback(store, call.thread.task, event, handle)
1084            })?;
1085        }
1086        GuestCallKind::StartImplicit(fun) => {
1087            fun(store)?;
1088        }
1089        GuestCallKind::StartExplicit(fun) => {
1090            fun(store)?;
1091        }
1092    }
1093
1094    Ok(())
1095}
1096
1097/// Invoke the callback of the callback-lifted `task` with `event` and
1098/// `handle`, returning the callback code it produces.
1099fn invoke_callback(
1100    store: &mut dyn VMStore,
1101    task: TableId<GuestTask>,
1102    event: Event,
1103    handle: u32,
1104) -> Result<u32> {
1105    let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1106        bail_bug!("guest task callback field not present")
1107    };
1108
1109    let code = callback(store, event, handle)?;
1110
1111    store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1112
1113    Ok(code)
1114}
1115
1116impl<T> Store<T> {
1117    /// Convenience wrapper for [`StoreContextMut::run_concurrent`].
1118    pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1119    where
1120        T: Send + 'static,
1121    {
1122        ensure!(
1123            self.as_context().0.concurrency_support(),
1124            "cannot use `run_concurrent` when Config::concurrency_support disabled",
1125        );
1126        self.as_context_mut().run_concurrent(fun).await
1127    }
1128
1129    #[doc(hidden)]
1130    pub fn assert_concurrent_state_empty(&mut self) {
1131        self.as_context_mut().assert_concurrent_state_empty();
1132    }
1133
1134    #[doc(hidden)]
1135    pub fn concurrent_state_table_size(&mut self) -> usize {
1136        self.as_context_mut().concurrent_state_table_size()
1137    }
1138
1139    /// Convenience wrapper for [`StoreContextMut::spawn`].
1140    pub fn spawn(
1141        &mut self,
1142        task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1143    ) -> Result<JoinHandle>
1144    where
1145        T: 'static,
1146    {
1147        self.as_context_mut().spawn(task)
1148    }
1149}
1150
1151impl<T> StoreContextMut<'_, T> {
1152    /// Assert that all the relevant tables and queues in the concurrent state
1153    /// for this store are empty.
1154    ///
1155    /// This is for sanity checking in integration tests
1156    /// (e.g. `component-async-tests`) that the relevant state has been cleared
1157    /// after each test concludes.  This should help us catch leaks, e.g. guest
1158    /// tasks which haven't been deleted despite having completed and having
1159    /// been dropped by their supertasks.
1160    ///
1161    /// Only intended for use in Wasmtime's own testing.
1162    #[doc(hidden)]
1163    pub fn assert_concurrent_state_empty(self) {
1164        let store = self.0;
1165        store
1166            .store_data_mut()
1167            .components
1168            .assert_instance_states_empty();
1169        let state = store.concurrent_state_mut().unwrap();
1170        assert!(
1171            state.table.get_mut().is_empty(),
1172            "non-empty table: {:?}",
1173            state.table.get_mut()
1174        );
1175        assert!(state.switch_item.is_none());
1176        assert!(state.next_switch_item.is_none());
1177        assert!(state.high_priority.is_empty());
1178        assert!(state.low_priority.is_empty());
1179        assert!(state.saved_next_switch_items.is_empty());
1180        assert!(state.unforced_current_thread.is_none());
1181        assert!(state.deferred_host_call_context.is_none());
1182        assert!(state.futures_mut().unwrap().is_empty());
1183        assert!(state.global_error_context_ref_counts.is_empty());
1184    }
1185
1186    /// Helper function to perform tests over the size of the concurrent state
1187    /// table which can be useful for detecting leaks.
1188    ///
1189    /// Only intended for use in Wasmtime's own testing.
1190    #[doc(hidden)]
1191    pub fn concurrent_state_table_size(&mut self) -> usize {
1192        self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1193    }
1194
1195    /// Spawn a background task to run as part of this instance's event loop.
1196    ///
1197    /// The task will receive an `&Accessor<U>` and run concurrently with
1198    /// any other tasks in progress for the instance.
1199    ///
1200    /// Note that the task will only make progress if and when the event loop
1201    /// for this instance is run.
1202    ///
1203    /// The returned [`JoinHandle`] may be used to cancel the task.
1204    pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1205    where
1206        T: 'static,
1207    {
1208        let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1209        self.spawn_with_accessor(accessor, task)
1210    }
1211
1212    /// Internal implementation of `spawn` functions where a `store` is
1213    /// available along with an `Accessor`.
1214    fn spawn_with_accessor<D>(
1215        self,
1216        accessor: Accessor<T, D>,
1217        task: impl for<'fut> AccessorTask<'fut, T, D>,
1218    ) -> Result<JoinHandle>
1219    where
1220        T: 'static,
1221        D: HasData + ?Sized,
1222    {
1223        // Create an "abortable future" here where internally the future will
1224        // hook calls to poll and possibly spawn more background tasks on each
1225        // iteration.
1226        let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1227        self.0
1228            .concurrent_state_mut()?
1229            .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1230        Ok(handle)
1231    }
1232
1233    /// Run the specified closure `fun` to completion as part of this store's
1234    /// event loop.
1235    ///
1236    /// This will run `fun` as part of this store's event loop until it
1237    /// yields a result.  `fun` is provided an [`Accessor`], which provides
1238    /// controlled access to the store and its data.
1239    ///
1240    /// This function can be used to invoke [`Func::call_concurrent`] for
1241    /// example within the async closure provided here.
1242    ///
1243    /// This function will unconditionally return an error if
1244    /// [`Config::concurrency_support`] is disabled.
1245    ///
1246    /// [`Config::concurrency_support`]: crate::Config::concurrency_support
1247    ///
1248    /// # Store-blocking behavior
1249    ///
1250    /// At this time there are certain situations in which the `Future` returned
1251    /// by the `AsyncFnOnce` passed to this function will not be polled for an
1252    /// extended period of time, despite one or more `Waker::wake` events having
1253    /// occurred for the task to which it belongs.  This can manifest as the
1254    /// `Future` seeming to be "blocked" or "locked up", but is actually due to
1255    /// the `Store` being held by e.g. a blocking host function, preventing the
1256    /// `Future` from being polled. A canonical example of this is when the
1257    /// `fun` provided to this function attempts to set a timeout for an
1258    /// invocation of a wasm function. In this situation the async closure is
1259    /// waiting both on (a) the wasm computation to finish, and (b) the timeout
1260    /// to elapse. At this time this setup will not always work and the timeout
1261    /// may not reliably fire.
1262    ///
1263    /// This function will not block the current thread and as such is always
1264    /// suitable to run in an `async` context, but the current implementation of
1265    /// Wasmtime can lead to situations where a certain wasm computation is
1266    /// required to make progress the closure to make progress. This is an
1267    /// artifact of Wasmtime's historical implementation of `async` functions
1268    /// and is the topic of [#11869] and [#11870]. In the timeout example from
1269    /// above it means that Wasmtime can get "wedged" for a bit where (a) must
1270    /// progress for a readiness notification of (b) to get delivered.
1271    ///
1272    /// This effectively means that it's not possible to reliably perform a
1273    /// "select" operation within the `fun` closure, which timeouts for example
1274    /// are based on. Fixing this requires some relatively major refactoring
1275    /// work within Wasmtime itself. This is a known pitfall otherwise and one
1276    /// that is intended to be fixed one day. In the meantime it's recommended
1277    /// to apply timeouts or such to the entire `run_concurrent` call itself
1278    /// rather than internally.
1279    ///
1280    /// [#11869]: https://github.com/bytecodealliance/wasmtime/issues/11869
1281    /// [#11870]: https://github.com/bytecodealliance/wasmtime/issues/11870
1282    ///
1283    /// # Example
1284    ///
1285    /// ```
1286    /// # use {
1287    /// #   wasmtime::{
1288    /// #     error::{Result},
1289    /// #     component::{ Component, Linker, Resource, ResourceTable},
1290    /// #     Config, Engine, Store
1291    /// #   },
1292    /// # };
1293    /// #
1294    /// # struct MyResource(u32);
1295    /// # struct Ctx { table: ResourceTable }
1296    /// #
1297    /// # async fn foo() -> Result<()> {
1298    /// # let mut config = Config::new();
1299    /// # let engine = Engine::new(&config)?;
1300    /// # let mut store = Store::new(&engine, Ctx { table: ResourceTable::new() });
1301    /// # let mut linker = Linker::new(&engine);
1302    /// # let component = Component::new(&engine, "")?;
1303    /// # let instance = linker.instantiate_async(&mut store, &component).await?;
1304    /// # let foo = instance.get_typed_func::<(Resource<MyResource>,), (Resource<MyResource>,)>(&mut store, "foo")?;
1305    /// # let bar = instance.get_typed_func::<(u32,), ()>(&mut store, "bar")?;
1306    /// store.run_concurrent(async |accessor| -> wasmtime::Result<_> {
1307    ///    let resource = accessor.with(|mut access| access.get().table.push(MyResource(42)))?;
1308    ///    let (another_resource,) = foo.call_concurrent(accessor, (resource,)).await?;
1309    ///    let value = accessor.with(|mut access| access.get().table.delete(another_resource))?;
1310    ///    bar.call_concurrent(accessor, (value.0,)).await?;
1311    ///    Ok(())
1312    /// }).await??;
1313    /// # Ok(())
1314    /// # }
1315    /// ```
1316    pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1317    where
1318        T: Send + 'static,
1319    {
1320        ensure!(
1321            self.0.concurrency_support(),
1322            "cannot use `run_concurrent` when Config::concurrency_support disabled",
1323        );
1324        self.do_run_concurrent(fun, false).await
1325    }
1326
1327    pub(super) async fn run_concurrent_trap_on_idle<R>(
1328        self,
1329        fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1330    ) -> Result<R> {
1331        self.do_run_concurrent(fun, true).await
1332    }
1333
1334    async fn do_run_concurrent<R>(
1335        mut self,
1336        fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1337        trap_on_idle: bool,
1338    ) -> Result<R> {
1339        debug_assert!(self.0.concurrency_support());
1340        self.0.check_event_loop_not_running()?;
1341        let token = StoreToken::new(self.as_context_mut());
1342
1343        struct Dropper<'a, T: 'static, V> {
1344            store: StoreContextMut<'a, T>,
1345            value: ManuallyDrop<V>,
1346        }
1347
1348        impl<'a, T, V> Drop for Dropper<'a, T, V> {
1349            fn drop(&mut self) {
1350                self.store
1351                    .0
1352                    .concurrent_state_mut_already_forced_current_thread()
1353                    .event_loop_running = false;
1354
1355                tls::set(self.store.0, || {
1356                    // SAFETY: Here we drop the value without moving it for the
1357                    // first and only time -- per the contract for `Drop::drop`,
1358                    // this code won't run again, and the `value` field will no
1359                    // longer be accessible.
1360                    unsafe { ManuallyDrop::drop(&mut self.value) }
1361                });
1362            }
1363        }
1364
1365        let accessor = &Accessor::new(token);
1366        self.0
1367            .concurrent_state_mut_already_forced_current_thread()
1368            .event_loop_running = true;
1369        let dropper = &mut Dropper {
1370            store: self,
1371            value: ManuallyDrop::new(fun(accessor)),
1372        };
1373        // SAFETY: We never move `dropper` nor its `value` field.
1374        let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1375
1376        let result = dropper
1377            .store
1378            .as_context_mut()
1379            .poll_until(future, trap_on_idle)
1380            .await;
1381
1382        if result.is_err() {
1383            dropper.store.0.set_trapped();
1384        }
1385
1386        result
1387    }
1388
1389    /// Run this store's event loop.
1390    ///
1391    /// The returned future will resolve when the specified future completes or,
1392    /// if `trap_on_idle` is true, when the event loop can't make further
1393    /// progress.
1394    async fn poll_until<R>(
1395        mut self,
1396        mut future: Pin<&mut impl Future<Output = R>>,
1397        trap_on_idle: bool,
1398    ) -> Result<R> {
1399        struct Reset<'a, T: 'static> {
1400            store: StoreContextMut<'a, T>,
1401            futures: Option<FuturesUnordered<HostTaskFuture>>,
1402        }
1403
1404        impl<'a, T> Drop for Reset<'a, T> {
1405            fn drop(&mut self) {
1406                if let Some(futures) = self.futures.take() {
1407                    *self
1408                        .store
1409                        .0
1410                        .concurrent_state_mut_already_forced_current_thread()
1411                        .futures
1412                        .get_mut() = Some(futures);
1413                }
1414            }
1415        }
1416
1417        // Keep a continuously busy event loop from holding one executor poll
1418        // indefinitely.  In particular, Tokio's cooperative budget is only
1419        // replenished when the future returns `Pending` to the executor.
1420        const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1421        let mut turns_without_yield = 0;
1422
1423        loop {
1424            // Take `ConcurrentState::futures` out of the store so we can poll
1425            // it while also safely giving any of the futures inside access to
1426            // `self`.
1427            let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1428            let mut reset = Reset {
1429                store: self.as_context_mut(),
1430                futures,
1431            };
1432            let mut next = match reset.futures.as_mut() {
1433                Some(f) => pin!(f.next()),
1434                None => bail_bug!("concurrent state missing futures field"),
1435            };
1436
1437            enum PollResult<R> {
1438                Complete(R),
1439                ProcessWork {
1440                    ready: Option<WorkItem>,
1441                    low_priority: bool,
1442                },
1443            }
1444
1445            let result = future::poll_fn(|cx| {
1446                // First, poll the future we were passed as an argument and
1447                // return immediately if it's ready.
1448                if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1449                    return Poll::Ready(Ok(PollResult::Complete(value)));
1450                }
1451
1452                // If the store has been poisoned by a trap, either before this
1453                // event loop started or while it was running, then refuse to
1454                // make any further progress. Any work items, suspended fibers,
1455                // or pending futures left over from the trap might otherwise
1456                // resume guest code or observe scheduler state which was never
1457                // unwound. Note that this is checked after polling `future`
1458                // above so that it can observe the original error, e.g. a
1459                // trap delivered over a oneshot channel by a work item.
1460                if reset.store.0.component_trapped() {
1461                    return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1462                }
1463
1464                // Next, poll `ConcurrentState::futures` (which includes any
1465                // pending host tasks and/or background tasks), returning
1466                // immediately if one of them fails.
1467                let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1468                    Poll::Ready(Some(output)) => {
1469                        match output {
1470                            Err(e) => return Poll::Ready(Err(e)),
1471                            Ok(()) => {}
1472                        }
1473                        Poll::Ready(true)
1474                    }
1475                    Poll::Ready(None) => Poll::Ready(false),
1476                    Poll::Pending => Poll::Pending,
1477                };
1478
1479                // Next, identify the next work item to process, if any, using
1480                // the following priority order:
1481                //
1482                // - `switch_item`: Represents the guest thread we _must_ switch
1483                // to before any other thread runs per the determinism
1484                // requirements in the Component Model spec.
1485                //
1486                // - `high_priority`: "Urgent" work items, e.g. async calls have
1487                // become freshly unblocked due to backpressure clearing or
1488                // similar, stream or future state updates, etc.
1489                //
1490                // - `low_priority`: Work items such as resuming a fiber after
1491                // it yields, in which case the point is to let other items run
1492                // first.
1493                let state = reset.store.0.concurrent_state_mut()?;
1494                let mut ready = state.switch_item.take();
1495                let mut low_priority = false;
1496                if ready.is_none() {
1497                    ready = state.high_priority.pop_back();
1498                    if ready.is_none() {
1499                        ready = state.low_priority.pop_back();
1500                        low_priority = true;
1501                    }
1502                }
1503                if ready.is_some() {
1504                    return Poll::Ready(Ok(PollResult::ProcessWork {
1505                        ready,
1506                        low_priority,
1507                    }));
1508                }
1509
1510                if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1511                    waker.wake();
1512                }
1513
1514                // Finally, if we have nothing else to do right now, determine what to do
1515                // based on whether there are any pending futures in
1516                // `ConcurrentState::futures`.
1517                return match next {
1518                    Poll::Ready(true) => {
1519                        // In this case, one of the futures in
1520                        // `ConcurrentState::futures` completed
1521                        // successfully, so we return now and continue
1522                        // the outer loop in case there is another one
1523                        // ready to complete.
1524                        Poll::Ready(Ok(PollResult::ProcessWork {
1525                            ready: None,
1526                            low_priority: false,
1527                        }))
1528                    }
1529                    Poll::Ready(false) => {
1530                        // Poll the future we were passed one last time
1531                        // in case one of `ConcurrentState::futures` had
1532                        // the side effect of unblocking it.
1533                        if let Poll::Ready(value) =
1534                            tls::set(reset.store.0, || future.as_mut().poll(cx))
1535                        {
1536                            Poll::Ready(Ok(PollResult::Complete(value)))
1537                        } else {
1538                            // In this case, there are no more pending
1539                            // futures in `ConcurrentState::futures`,
1540                            // there are no remaining work items, _and_
1541                            // the future we were passed as an argument
1542                            // still hasn't completed.
1543                            if trap_on_idle {
1544                                // `trap_on_idle` is true, so we exit
1545                                // immediately.
1546
1547                                // If there are any tasks belonging to
1548                                // an instance which may not suspend, trap with
1549                                // `CannotBlockSyncTask`:
1550                                Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1551                                    Trap::CannotBlockSyncTask.into()
1552                                } else {
1553                                    // Otherwise, trap with `AsyncDeadlock`:
1554                                    Trap::AsyncDeadlock.into()
1555                                }))
1556                            } else {
1557                                // `trap_on_idle` is false, so we assume
1558                                // that future will wake up and give us
1559                                // more work to do when it's ready to.
1560                                Poll::Pending
1561                            }
1562                        }
1563                    }
1564                    // There is at least one pending future in
1565                    // `ConcurrentState::futures` and we have nothing
1566                    // else to do but wait for now, so we return
1567                    // `Pending`.
1568                    Poll::Pending => Poll::Pending,
1569                };
1570            })
1571            .await;
1572
1573            // Put the `ConcurrentState::futures` back into the store before we
1574            // return or handle any work items since one or more of those items
1575            // might append more futures.
1576            drop(reset);
1577
1578            match result? {
1579                // The future we were passed as an argument completed, so we
1580                // return the result.
1581                PollResult::Complete(value) => break Ok(value),
1582                // The future we were passed has not yet completed, so handle
1583                // any work items and then loop again.
1584                PollResult::ProcessWork {
1585                    mut ready,
1586                    low_priority,
1587                } => {
1588                    // If we're about to run a low-priority task, first yield to
1589                    // the executor.  This ensures that it won't be starved of
1590                    // the ability to e.g. update the readiness of sockets,
1591                    // etc. which the guest may be using `thread.yield` along
1592                    // with `waitable-set.poll` to monitor in a CPU-heavy loop.
1593                    //
1594                    // This works for e.g. `thread.yield` and callbacks which
1595                    // return `CALLBACK_CODE_YIELD` because we queue a low
1596                    // priority item to resume the task (i.e. resume the thread
1597                    // or call the callback, respectively) just prior to
1598                    // suspending it.  Indeed, as of this writing those are the
1599                    // _only_ situations we queue low-priority tasks.
1600                    // Therefore, we interpret the guest's request to yield as
1601                    // meaning "yield to other guest tasks _and_/_or_ host
1602                    // operations such as updating socket readiness", the latter
1603                    // being the async runtime's responsibility.
1604                    //
1605                    // In the future, if this ends up causing measurable
1606                    // performance issues, this could be optimized such that we
1607                    // only yield periodically (e.g. for batches of low priority
1608                    // items) and not for each and every individual item.
1609                    //
1610                    // While we're yielded put the item back on the queue so it
1611                    // stays live within the store, and then if/when we come
1612                    // back take it back out. Note that the `await` point here
1613                    // owns the entire `StoreContextMut` so what we push should
1614                    // be what we pop.
1615                    if low_priority {
1616                        if let Some(item) = ready.take() {
1617                            self.0.concurrent_state_mut()?.low_priority.push_back(item);
1618                        }
1619                        self.0.yield_now().await;
1620                        turns_without_yield = 0;
1621                        ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1622                    }
1623
1624                    if let Some(item) = ready {
1625                        self.as_context_mut().handle_work_item(item).await?;
1626                    }
1627
1628                    turns_without_yield += 1;
1629                    if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1630                        turns_without_yield = 0;
1631                        self.0.yield_now().await;
1632                    }
1633                }
1634            }
1635        }
1636    }
1637
1638    /// Handle the specified work item, possibly resuming a fiber if applicable.
1639    fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1640        // Construct the guarded future before returning, even when the caller
1641        // never polls this work item. Other variants keep their fibers in the
1642        // store until their first poll.
1643        let item = match item {
1644            WorkItem::ResumeFiber {
1645                instance,
1646                thread,
1647                fiber,
1648            } => {
1649                let future = self.0.resume_fiber(fiber);
1650                log::trace!(
1651                    "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1652                );
1653                return futures::future::Either::Left(future);
1654            }
1655            item => item,
1656        };
1657        futures::future::Either::Right(async move {
1658            log::trace!("handle work item {item:?}");
1659            match item {
1660                WorkItem::PushFuture(future) => {
1661                    self.0
1662                        .concurrent_state_mut()?
1663                        .futures_mut()?
1664                        .push(future.into_inner());
1665                }
1666                WorkItem::ResumeFiber { .. } => unreachable!(),
1667                WorkItem::ResumeThread { thread, .. } => {
1668                    let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1669                    if !matches!(slot, GuestThreadState::Ready { .. }) {
1670                        bail_bug!("cannot resume non-pending thread {thread:?}");
1671                    }
1672                    let GuestThreadState::Ready { fiber } =
1673                        mem::replace(slot, GuestThreadState::Running)
1674                    else {
1675                        unreachable!()
1676                    };
1677                    self.0.resume_fiber(fiber).await?;
1678                }
1679                WorkItem::GuestCall { call, .. } => {
1680                    if call.is_ready(self.0)? {
1681                        self.0
1682                            .concurrent_state_mut()?
1683                            .get_mut(call.thread.thread)?
1684                            .wake_on_cancel = WakeOnCancel::None;
1685                        self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1686                    } else {
1687                        let state = self.0.concurrent_state_mut()?;
1688                        let task = state.get_mut(call.thread.task)?;
1689                        if !task.starting_sent {
1690                            task.starting_sent = true;
1691                            if let GuestCallKind::StartImplicit(_) = &call.kind {
1692                                Waitable::Guest(call.thread.task).set_event(
1693                                    state,
1694                                    Some(Event::Subtask {
1695                                        status: Status::Starting,
1696                                    }),
1697                                )?;
1698                            }
1699                        }
1700
1701                        let instance = state.get_mut(call.thread.task)?.instance;
1702                        // If this call is for a callback-lifted task waiting on a
1703                        // waitable set, it can't run until `instance` is enterable
1704                        // again. Meanwhile, the set may have an event which another
1705                        // thread waiting on it could take, and that thread may
1706                        // even be the reason `instance` isn't enterable, so wake
1707                        // the next waiter up as well if there's still an event
1708                        // for it. Whichever thread runs first takes the event, and
1709                        // the other goes back to waiting if none is left.
1710                        if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1711                            let state = self.0.concurrent_state_mut()?;
1712                            if !state.get_mut(*set)?.ready.is_empty() {
1713                                state.wake_waiter(*set)?;
1714                            }
1715                        }
1716                        self.0
1717                            .instance_state(instance)
1718                            .concurrent_state()
1719                            .pending
1720                            .insert(call.thread, call.kind);
1721
1722                        // Switch back to the caller (or canceller) immediately if
1723                        // applicable since we aren't yet able to run the subtask it
1724                        // yielded to.
1725                        self.0.concurrent_state_mut()?.take_next_switch_item()?;
1726                    }
1727                }
1728                WorkItem::WorkerFunction(fun) => {
1729                    self.run_on_worker(WorkerItem::Function(fun)).await?;
1730                }
1731            }
1732
1733            Ok(())
1734        })
1735    }
1736
1737    /// Execute the specified guest call on a worker fiber.
1738    async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1739        let state = self.0.concurrent_state_mut()?;
1740        if state.worker_item.is_some() {
1741            bail_bug!("worker item already set when taking worker fiber");
1742        }
1743        let worker = if let Some(fiber) = state.worker.take() {
1744            fiber
1745        } else {
1746            // SAFETY: the `make_fiber_unchecked` function is unsafe because the
1747            // returned fiber is unconditionally `Send` as opposed to being
1748            // conditionally send depending on the argument (in this case
1749            // `self.0`). This `async` function, however, is conditionally
1750            // `Send` depending on `self`, in this case `StoreContextMut<T>`,
1751            // which is already going to be conditionally `Send` depending on
1752            // `T`.
1753            //
1754            // The returned fiber is possibly stored within the `Store<T>` as
1755            // well. If `T: Send` then that's fine and everything's dandy. If
1756            // `T: !Send`, however, then the store is already not-`Send` meaning
1757            // that putting more actually-not-`Send` things inside of it isn't
1758            // an issue.
1759            //
1760            // The main issue here is that the returned fiber effectively can't
1761            // get transferred outside the context of the store. That's an
1762            // implementation detail we'll have to rely on, but is currently
1763            // true.
1764            unsafe {
1765                fiber::make_fiber_unchecked(self.0, move |store| {
1766                    loop {
1767                        let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1768                            bail_bug!("worker_item not present when resuming fiber")
1769                        };
1770                        match item {
1771                            WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1772                            WorkerItem::Function(fun) => fun.into_inner()(store)?,
1773                        }
1774
1775                        store.suspend(SuspendReason::NeedWork)?;
1776                    }
1777                })?
1778            }
1779        };
1780
1781        // No fallible operation between taking the fiber and guarding it in
1782        // resume_fiber; the current thread was forced before the take.
1783        self.0
1784            .concurrent_state_mut_already_forced_current_thread()
1785            .worker_item = Some(item);
1786
1787        self.0.resume_fiber(worker).await
1788    }
1789
1790    /// Wrap the specified host function in a future which will call it, passing
1791    /// it an `&Accessor<T>`.
1792    ///
1793    /// See the `Accessor` documentation for details.
1794    pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1795    where
1796        T: 'static,
1797        F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1798            + Send
1799            + Sync
1800            + 'static,
1801        R: Send + Sync + 'static,
1802    {
1803        let token = StoreToken::new(self);
1804        async move {
1805            let mut accessor = Accessor::new(token);
1806            closure(&mut accessor).await
1807        }
1808    }
1809
1810    pub(crate) async fn start_instance(
1811        &mut self,
1812        instance: ModuleInstance,
1813        callee: Option<RuntimeInstance>,
1814    ) -> Result<ModuleInstance> {
1815        // See `Func::call_async` for why this is checked up front.
1816        self.0.check_event_loop_not_running()?;
1817        let (tx, rx) = oneshot::channel();
1818        let token = StoreToken::new(self.as_context_mut());
1819        self.0.queue_task(move |store| {
1820            _ = tx.send(
1821                super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1822                    .map(|()| instance),
1823            );
1824            Ok(())
1825        })?;
1826        self.as_context_mut()
1827            .run_concurrent_trap_on_idle(async |_| {
1828                rx.await
1829                    .map_err(|_| format_err!("oneshot channel canceled"))
1830            })
1831            .await??
1832    }
1833}
1834
1835/// Return value of [`StoreOpaque::host_task_create`].
1836///
1837/// This is an `Option` to handle the dynamic `store.concurrency_support()`
1838/// property. When present this records the guest thread to restore when the
1839/// host call exits. The corresponding [`HostTask`] will need to be lazily
1840/// created if needed via [`StoreOpaque::materialize_host_task_id`].
1841pub type EnteredHostTask = Option<QualifiedThreadId>;
1842
1843impl StoreOpaque {
1844    pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1845        if self
1846            .concurrent_state_mut_already_forced_current_thread()
1847            .event_loop_running
1848        {
1849            bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1850        }
1851        Ok(())
1852    }
1853
1854    /// Returns the currently-running thread, promoting any deferred lazy guest
1855    /// thread into a fully-materialized `CurrentThread`. Deferred [`HostTask`]s
1856    /// are not materialized.
1857    #[inline]
1858    pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1859        // Without concurrency support there is nothing to force.
1860        if !self.concurrency_support() {
1861            return Ok(CurrentThread::None);
1862        }
1863
1864        // If the JIT-visible current thread isn't a deferred thread then
1865        // `ConcurrentState` is already up to date.
1866        if !self
1867            .vm_store_context_mut()
1868            .current_thread_mut()
1869            .is_deferred()
1870        {
1871            return Ok(self
1872                .concurrent_state_mut_already_forced_current_thread()
1873                .unforced_current_thread);
1874        }
1875
1876        self.force_deferred_current_thread()
1877    }
1878
1879    /// Slow path of [`Self::current_thread`]: promote the deferred lazy
1880    /// thread into a fully-materialized `CurrentThread`.
1881    #[cold]
1882    fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1883        // The component instance whose adapters pushed the deferred frames; all
1884        // frames in a guest-to-guest, sync-to-sync call chain of fused adapters
1885        // live within a single `wasmtime::component::Instance` (because
1886        // cross-`wasmtime::component::Instance` calls don't go through fused
1887        // adapters), and guest code only ever runs as a guest thread, so the
1888        // chain's base thread is already materialized in `ConcurrentState` and
1889        // we can get the `ComponentInstanceId` shared by the whole chain from
1890        // here.
1891        let state = self.concurrent_state_mut_without_forcing_current_thread();
1892        let id = match state.unforced_current_thread.guest_task() {
1893            Some(task) => state.get_mut(task)?.instance.instance,
1894            None => bail_bug!("deferred component-model thread with non-guest base"),
1895        };
1896
1897        // Collect the deferred frames pushed inline by fused adapters, walking
1898        // the `parent` chain from innermost to the base.
1899        let mut frames = SmallVec::<[_; 8]>::new();
1900        let mut cur = *self.vm_store_context_mut().current_thread_mut();
1901        while let Some(ptr) = cur.as_deferred() {
1902            // SAFETY: `ptr` points at a `VMDeferredThread` living in a fused
1903            // adapter's stack frame that is suspended below us on the stack
1904            // (mid-call, waiting for this nested call to return), so the
1905            // referent is still valid and exclusively ours to read.
1906            let deferred = unsafe { ptr.as_non_null().as_ref() };
1907            frames.push((
1908                deferred.callee_async != 0,
1909                deferred.callee_instance,
1910                deferred.saved_context,
1911                deferred.lazy_lenders_start,
1912            ));
1913            cur = deferred.parent;
1914        }
1915
1916        // Mark the current thread forced *before* replaying so that any
1917        // reentrant `force_current_thread` call short-circuits via the
1918        // non-deferred path above.
1919        *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1920
1921        // Save the current context, as we need to overwrite it while replaying
1922        // below.
1923        let current_context = *self.vm_store_context_mut().component_context_mut();
1924
1925        // Replay the deferred `enter_guest_sync_call`s outermost-first so that
1926        // the resulting `ConcurrentState` matches what the non-deferred path
1927        // would have otherwise produced.
1928        //
1929        // A frame's own lazy lends run from its start to that of the frame
1930        // pushed after it, or to the end of the buffer for the innermost.
1931        let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
1932        frames.reverse();
1933        let lazy_lenders_base = match frames.first() {
1934            Some((_, _, _, start)) => *start,
1935            None => bail_bug!("forcing a deferred thread with no deferred frames"),
1936        };
1937        let lazy_lender_ends = frames
1938            .iter()
1939            .skip(1)
1940            .map(|(_, _, _, start)| *start)
1941            .chain([lazy_lenders_end])
1942            .collect::<SmallVec<[_; 8]>>();
1943        for ((callee_async, callee_instance, saved_context, start), lenders_end) in
1944            frames.into_iter().zip(lazy_lender_ends)
1945        {
1946            // Restore the caller's context slots so that we save the correct
1947            // values into the caller's thread, exactly as the non-deferred path
1948            // would have on entry.
1949            *self.vm_store_context_mut().component_context_mut() = saved_context;
1950            let callee = RuntimeInstance {
1951                instance: id,
1952                index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
1953            };
1954            self.enter_guest_sync_call(callee_async, callee)?;
1955
1956            // Now that this frame has a real `GuestTask`, promote any lazy
1957            // lends it recorded while deferred.
1958            self.promote_current_lazy_lends(id, start, lenders_end)?;
1959        }
1960
1961        // Every lazy-lend record was promoted, so the length resets back to
1962        // where the outermost frame found it.
1963        debug_assert_eq!(
1964            *self.vm_store_context_mut().lazy_lenders_len_mut(),
1965            lazy_lenders_end
1966        );
1967        *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
1968
1969        // Replaying done; restore the current context.
1970        *self.vm_store_context_mut().component_context_mut() = current_context;
1971
1972        Ok(self
1973            .concurrent_state_mut_without_forcing_current_thread()
1974            .unforced_current_thread)
1975    }
1976
1977    /// Promotes the `lazy_lenders` entries in `start..end` into real lends
1978    /// owned by the current thread's `CallContext`.
1979    ///
1980    /// The task's `validate_scope_exit` then undoes them as if they had been
1981    /// lent there all along.
1982    fn promote_current_lazy_lends(
1983        &mut self,
1984        id: ComponentInstanceId,
1985        start: usize,
1986        end: usize,
1987    ) -> Result<()> {
1988        if start == end {
1989            return Ok(());
1990        }
1991
1992        let scope = match self
1993            .concurrent_state_mut_without_forcing_current_thread()
1994            .unforced_current_thread
1995            .guest_task()
1996        {
1997            Some(task) => VMScope::id(task.rep()),
1998            None => bail_bug!("replayed deferred frame is not a guest thread"),
1999        };
2000        let instance = Instance::from_wasmtime(self, id);
2001        self.promote_lazy_lends(instance, scope, start, end)
2002    }
2003
2004    fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2005        match self.current_thread()?.guest() {
2006            Some(id) => Ok(*id),
2007            None => bail_bug!("current thread is not a guest thread"),
2008        }
2009    }
2010
2011    // A result of `None` may indicate that this is either the top-level event
2012    // loop, a deferred host task, or concurrency support is disabled. In all
2013    // cases we don't have an ID for the task.
2014    pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2015        match self.current_thread()? {
2016            CurrentThread::Host(id) => Ok(Some(id)),
2017            CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2018            _ => bail_bug!("current thread is not a host thread"),
2019        }
2020    }
2021
2022    /// Returns the current host task ID, materializing a deferred host task if
2023    /// one is active. `None` represents a call from the top-level host.
2024    fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2025        Ok(self
2026            .concurrent_state_mut()?
2027            .materialize_current_host_task_id()?)
2028    }
2029
2030    fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2031        log::trace!("enter sync-typed call {callee:?}");
2032        let state = self.instance_state(callee).concurrent_state();
2033        let old_do_not_suspend = state.do_not_suspend;
2034        state.do_not_suspend = true;
2035
2036        let thread = self.current_guest_thread()?;
2037        let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2038        if thread.old_do_not_suspend.is_some() {
2039            bail_bug!("current thread already has `old_do_not_suspend` value");
2040        }
2041
2042        thread.old_do_not_suspend = Some(old_do_not_suspend);
2043
2044        Ok(())
2045    }
2046
2047    fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2048        log::trace!("exit sync-typed call {callee:?}");
2049        let thread = self.current_guest_thread()?;
2050        let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2051        let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2052            bail_bug!("current thread missing `old_do_not_suspend` value");
2053        };
2054        let state = self.instance_state(callee).concurrent_state();
2055        state.do_not_suspend = old_do_not_suspend;
2056        Ok(())
2057    }
2058
2059    /// Push a `GuestTask` onto the task stack for either a sync-to-sync,
2060    /// guest-to-guest call or a sync host-to-guest call.
2061    ///
2062    /// This task will only be used for the purpose of handling calls to
2063    /// intrinsic functions; both parameter lowering and result lifting are
2064    /// assumed to be taken care of elsewhere.
2065    ///
2066    /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
2067    /// fused adapters, see `StoreOpaque::current_thread`, `VMDeferredThread`,
2068    /// and `lower_fact_enter_sync_call`. Make sure all this stuff stays in
2069    /// sync!
2070    pub(crate) fn enter_guest_sync_call(
2071        &mut self,
2072        callee_async_typed: bool,
2073        callee: RuntimeInstance,
2074    ) -> Result<()> {
2075        log::trace!("enter sync-lifted call {callee:?}");
2076        if !self.concurrency_support() {
2077            return self.enter_call_not_concurrent();
2078        }
2079
2080        // Stash the next switch item (if any) for the duration of this call
2081        // since it's for the caller's caller, not for us.  We'll restore this
2082        // in `exit_guest_sync_call`.
2083        let state = self.concurrent_state_mut()?;
2084        let item = state.next_switch_item.take();
2085        state.saved_next_switch_items.push(item);
2086
2087        let thread = self.current_thread()?;
2088        let caller = if let Some(thread) = thread.guest() {
2089            Caller::Guest { thread: *thread }
2090        } else {
2091            Caller::Host {
2092                tx: None,
2093                caller: self.materialize_host_task_id()?,
2094            }
2095        };
2096
2097        let state = self.concurrent_state_mut()?;
2098        let guest_thread = GuestTask::new(
2099            state,
2100            Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2101            LiftResult {
2102                lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2103                ty: TypeTupleIndex::reserved_value(),
2104                memory: None,
2105                string_encoding: StringEncoding::Utf8,
2106            },
2107            caller,
2108            None,
2109            callee,
2110            callee_async_typed,
2111            false,
2112        )?;
2113
2114        Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2115            guest_thread.thread,
2116            self,
2117            callee.index,
2118        )?;
2119        self.set_thread(guest_thread)?;
2120
2121        if !callee_async_typed {
2122            self.enter_sync_call(callee)?;
2123        }
2124
2125        Ok(())
2126    }
2127
2128    /// Pop a `GuestTask` previously pushed using `enter_guest_sync_call`.
2129    ///
2130    /// NB: for sync-to-sync, guest-to-guest calls we delay task construction in
2131    /// fused adapters and then when the call returns we check to see if the
2132    /// task's contruction was forced and if not avoid calling out of the JIT
2133    /// code to this function. See `lower_fact_exit_sync_call`. Make sure all
2134    /// this stuff stays in sync!
2135    pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2136        if !self.concurrency_support() {
2137            return Ok(self.exit_call_not_concurrent());
2138        }
2139
2140        let thread = match self.current_thread()?.guest() {
2141            Some(t) => *t,
2142            None => bail_bug!("expected task when exiting"),
2143        };
2144        let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2145        let instance = task.instance;
2146
2147        let caller = match &task.caller {
2148            &Caller::Guest { thread } => thread.into(),
2149            &Caller::Host { caller, .. } => caller
2150                .map(CurrentThread::Host)
2151                .unwrap_or(CurrentThread::None),
2152        };
2153        task.lift_result = None;
2154        task.exited = true;
2155        let async_typed = task.async_typed;
2156
2157        if !async_typed {
2158            self.exit_sync_call(instance)?;
2159        }
2160
2161        self.set_thread(caller)?;
2162
2163        log::trace!("exit sync-lifted call {instance:?}");
2164
2165        if async_typed {
2166            // If we're async-typed, returning control to our caller won't help
2167            // resolve any outstanding sync-typed call which might be in
2168            // progress, so we may need to switch or trap before exiting this
2169            // thread:
2170            self.switch_or_trap_if_may_not_suspend(instance)?;
2171        }
2172
2173        self.cleanup_thread(thread, instance)?;
2174
2175        self.concurrent_state_mut()?
2176            .decrement_task_ref_count(thread.task)?;
2177
2178        let state = self.concurrent_state_mut()?;
2179        let Some(item) = state.saved_next_switch_items.pop() else {
2180            bail_bug!("unable to pop from `saved_next_switch_items`");
2181        };
2182        if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2183            // Stash it back in the store to ensure it's cleaned up on store
2184            // drop:
2185            state.push_high_priority(item);
2186            bail_bug!("`next_switch_item` unexpectedly already set");
2187        }
2188
2189        Ok(())
2190    }
2191
2192    /// Similar to `enter_guest_sync_call` except for when the guest makes a
2193    /// transition to the host.
2194    ///
2195    /// This initially records a deferred host call. A full [`HostTask`] should
2196    /// be allocated later if needed via
2197    /// [`StoreOpaque::materialize_host_task_id`].
2198    pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2199        if !self.concurrency_support() {
2200            self.enter_call_not_concurrent()?;
2201            return Ok(None);
2202        }
2203        let caller = self.current_guest_thread()?;
2204        log::trace!("new deferred host task with caller {caller:?}");
2205
2206        self.set_thread(CurrentThread::DeferredHost(caller))?;
2207        let state = self.concurrent_state_mut()?;
2208        debug_assert!(state.deferred_host_call_context.is_none());
2209        state.deferred_host_call_context = Some(CallContext::default());
2210        state.debug_assert_deferred_host_invariant();
2211        Ok(Some(caller))
2212    }
2213
2214    /// Dual of `host_task_create` and signifies that the host has finished and
2215    /// will be cleaned up.
2216    ///
2217    /// Note that this isn't invoked when the host is invoked asynchronously and
2218    /// the host isn't complete yet. In that situation the host task persists
2219    /// and will be cleaned up separately in `subtask_drop`
2220    pub(crate) fn host_task_delete(
2221        &mut self,
2222        original_task: EnteredHostTask,
2223        materialized_task: Option<TableId<HostTask>>,
2224    ) -> Result<()> {
2225        match original_task {
2226            Some(caller) => {
2227                self.set_thread(caller)?;
2228                if materialized_task.is_none() {
2229                    let state = self.concurrent_state_mut()?;
2230                    let context = state
2231                        .deferred_host_call_context
2232                        .take()
2233                        .expect("deferred host call context should be present");
2234                    debug_assert!(context.is_empty());
2235                    state.debug_assert_deferred_host_invariant();
2236                }
2237                log::trace!(
2238                    "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2239                );
2240                if let Some(task) = materialized_task {
2241                    self.concurrent_state_mut()?.delete_host_task(task)?;
2242                }
2243            }
2244            None => {
2245                debug_assert!(materialized_task.is_none());
2246                self.exit_call_not_concurrent();
2247            }
2248        }
2249        Ok(())
2250    }
2251
2252    /// Helper function to retrieve the `InstanceState` for the
2253    /// specified instance.
2254    fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2255        self.component_instance_mut(instance.instance)
2256            .instance_state(instance.index)
2257    }
2258
2259    /// Same as `instance_state`, but also returns the store's `HandleCounter`
2260    /// for inserting into the instance's handle tables.
2261    fn instance_state_and_handle_counter(
2262        &mut self,
2263        instance: RuntimeInstance,
2264    ) -> (&mut InstanceState, &mut HandleCounter) {
2265        let (component, counter) = self
2266            .store_data_mut()
2267            .component_instance_and_handle_counter_mut(instance.instance);
2268        (component.instance_state(instance.index), counter)
2269    }
2270
2271    /// Begins inserting into the handle table of the specified instance.
2272    fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2273        let (component, counter) = self
2274            .store_data_mut()
2275            .component_instance_and_handle_counter_mut(instance.instance);
2276        component.handle_table_insertion(instance.index, counter)
2277    }
2278
2279    /// Configure the currently running `thread`.
2280    ///
2281    /// This will save off any state necessary for the previous thread, if
2282    /// applicable, and then it'll additionally update state for `thread` if
2283    /// needed too.
2284    pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2285        let thread = thread.into();
2286        let state = self.concurrent_state_mut()?;
2287        state.debug_assert_deferred_host_invariant();
2288        let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2289
2290        state.handle_thread_switch(old_thread, thread)?;
2291
2292        // First thing to do after swapping threads is updating the context
2293        // slots for this thread within the store. This restores the behavior of
2294        // `context.{get,set}`. This involves taking the old state out of the
2295        // store, saving it in the thread that's being swapped from, and doing
2296        // the inverse for the new thread. When debug assertions are enabled
2297        // this also leaves behind sentinel values to try to uncover bugs where
2298        // this may be forgotten.
2299        if let Some(old_thread) = old_thread.guest() {
2300            let old_context = *self.vm_store_context_mut().component_context_mut();
2301            self.concurrent_state_mut()?
2302                .get_mut(old_thread.thread)?
2303                .context = old_context;
2304        }
2305        if cfg!(debug_assertions) {
2306            *self.vm_store_context_mut().component_context_mut() =
2307                [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2308        }
2309        if let Some(thread) = thread.guest() {
2310            let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2311            let context = thread.context;
2312            if cfg!(debug_assertions) {
2313                thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2314            }
2315            *self.vm_store_context_mut().component_context_mut() = context;
2316        }
2317
2318        // Keep the JIT-visible current-thread pointer in sync.
2319        *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2320            VMLazyThread::none()
2321        } else {
2322            VMLazyThread::forced()
2323        };
2324
2325        Ok(old_thread)
2326    }
2327
2328    /// Call `switch_if_may_not_suspend` and trap if it returns `false`.
2329    fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2330        if self.switch_if_may_not_suspend(instance)? {
2331            Ok(())
2332        } else {
2333            Err(Trap::CannotBlockSyncTask.into())
2334        }
2335    }
2336
2337    /// Check if the specified instance has a sync-typed call in progress; if so
2338    /// attempt to switch to another ready thread for that instance, and if no
2339    /// such thread exists, return false.
2340    fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2341        // Call this for the side effect of forcing any deferred task creation,
2342        // which may influence the value of `ConcurrentState::do_not_suspend`
2343        // below:
2344        self.concurrent_state_mut()?;
2345
2346        Ok(!self.concurrency_support()
2347            || !self
2348                .instance_state(instance)
2349                .concurrent_state()
2350                .do_not_suspend
2351            || self
2352                .concurrent_state_mut()?
2353                .promote_instance_local_thread_work_item(instance)?)
2354    }
2355
2356    /// Record that we're about to enter a (sub-)component instance which does
2357    /// not support more than one concurrent, stackful activation, meaning it
2358    /// cannot be entered again until the next call returns.
2359    fn enter_instance(&mut self, instance: RuntimeInstance) {
2360        log::trace!("enter {instance:?}");
2361        self.instance_state(instance)
2362            .concurrent_state()
2363            .do_not_enter = true;
2364    }
2365
2366    /// Record that we've exited a (sub-)component instance previously entered
2367    /// with `Self::enter_instance` and then calls `Self::partition_pending`.
2368    /// See the documentation for the latter for details.
2369    fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2370        log::trace!("exit {instance:?}");
2371        self.instance_state(instance)
2372            .concurrent_state()
2373            .do_not_enter = false;
2374        self.partition_pending(instance)
2375    }
2376
2377    /// Iterate over `InstanceState::pending`, moving any ready items into the
2378    /// "high priority" work item queue.
2379    ///
2380    /// Also, notify `ConcurrentState::ready_for_concurrent_call_waker` if
2381    /// present.
2382    ///
2383    /// See `GuestCall::is_ready` for details.
2384    fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2385        for (thread, kind) in
2386            mem::take(&mut self.instance_state(instance).concurrent_state().pending).into_iter()
2387        {
2388            let call = GuestCall { thread, kind };
2389            if call.is_ready(self)? {
2390                self.concurrent_state_mut()?
2391                    .push_high_priority(WorkItem::GuestCall { instance, call });
2392            } else {
2393                self.instance_state(instance)
2394                    .concurrent_state()
2395                    .pending
2396                    .insert(call.thread, call.kind);
2397            }
2398        }
2399
2400        if let Some(waker) = self
2401            .concurrent_state_mut()?
2402            .ready_for_concurrent_call_waker
2403            .take()
2404        {
2405            waker.wake();
2406        }
2407
2408        Ok(())
2409    }
2410
2411    /// Implements the `backpressure.{inc,dec}` intrinsics.
2412    pub(crate) fn backpressure_modify(
2413        &mut self,
2414        caller_instance: RuntimeInstance,
2415        modify: impl FnOnce(u16) -> Option<u16>,
2416    ) -> Result<()> {
2417        let state = self.instance_state(caller_instance).concurrent_state();
2418        let old = state.backpressure;
2419        let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2420        state.backpressure = new;
2421
2422        if old > 0 && new == 0 {
2423            // Backpressure was previously enabled and is now disabled; move any
2424            // newly-eligible guest calls to the "high priority" queue.
2425            self.partition_pending(caller_instance)?;
2426        }
2427
2428        Ok(())
2429    }
2430
2431    /// Resume the specified fiber, giving it exclusive access to the specified
2432    /// store.
2433    fn resume_fiber(
2434        &mut self,
2435        fiber: StoreFiber<'static>,
2436    ) -> impl Future<Output = Result<()>> + '_ {
2437        let mut resuming = ResumingFiber {
2438            store: self,
2439            fiber: Some(fiber),
2440        };
2441        async move {
2442            let old_thread = resuming.store.current_thread()?;
2443            log::trace!("resume_fiber: save current thread {old_thread:?}");
2444
2445            let fiber = resuming.fiber.take().unwrap();
2446            resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2447
2448            resuming.store.set_thread(old_thread)?;
2449
2450            let state = resuming.store.concurrent_state_mut()?;
2451
2452            if let Some(ot) = old_thread.guest() {
2453                state.set_thread_running(ot.thread)?;
2454            }
2455            log::trace!("resume_fiber: restore current thread {old_thread:?}");
2456
2457            if resuming.fiber.is_some() {
2458                log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2459                // See the `SuspendReason` documentation for what each case means.
2460                let reason = match state.suspend_reason.take() {
2461                    Some(r) => r,
2462                    None => bail_bug!("suspend reason missing when resuming fiber"),
2463                };
2464                match reason {
2465                    SuspendReason::NeedWork => {
2466                        if state.worker.is_none() {
2467                            state.worker = resuming.fiber.take();
2468                        }
2469                    }
2470                    SuspendReason::Yielding { thread } => {
2471                        let slot = &mut state.get_mut(thread.thread)?.state;
2472                        slot.check_no_fiber()?;
2473                        *slot = GuestThreadState::Ready {
2474                            fiber: resuming.fiber.take().unwrap(),
2475                        };
2476                        let instance = state.get_mut(thread.task)?.instance;
2477                        state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2478                    }
2479                    SuspendReason::ExplicitlySuspending { thread } => {
2480                        let slot = &mut state.get_mut(thread.thread)?.state;
2481                        slot.check_no_fiber()?;
2482                        *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2483                    }
2484                    SuspendReason::Waiting { set, thread } => {
2485                        match state.get_mut(set)?.waiting.entry(thread) {
2486                            Entry::Vacant(entry) => {
2487                                entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2488                            }
2489                            Entry::Occupied(_) => {
2490                                bail_bug!("entry unexpectedly already exists for {thread:?}");
2491                            }
2492                        }
2493                    }
2494                    SuspendReason::YieldingToSubtask { thread } => {
2495                        // In this case, the thread has either invoked or sent a
2496                        // cancel request to a subtask, and is now yielding to that
2497                        // subtask.  According to the CM spec, that subtask may only
2498                        // yield back to the original thread the first time it
2499                        // suspends or exits (or a thread that it has resumed
2500                        // suspends or exits, etc.), which we ensure by setting
2501                        // `ConcurrentState::next_switch_item` here.
2502
2503                        let instance = state.get_mut(thread.task)?.instance;
2504                        if state.next_switch_item.is_some() {
2505                            // This should be unreachable per the save/restore code
2506                            // in `Self::suspend`.
2507                            bail_bug!(
2508                                "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2509                             a thread wanted to wait on a subtask"
2510                            );
2511                        }
2512                        let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2513                            instance,
2514                            thread,
2515                            fiber: resuming.fiber.take().unwrap(),
2516                        });
2517                        log::trace!("set next switch item to {item:?}");
2518                    }
2519                };
2520            } else {
2521                log::trace!("resume_fiber: fiber has exited");
2522            }
2523
2524            Ok(())
2525        }
2526    }
2527
2528    /// Suspend the current fiber, storing the reason in
2529    /// `ConcurrentState::suspend_reason` to indicate the conditions under which
2530    /// it should be resumed.
2531    ///
2532    /// See the `SuspendReason` documentation for details.
2533    fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2534        log::trace!("suspend fiber: {reason:?}");
2535
2536        // We'll panic if we call `Self::with_blocking` when the fiber is being
2537        // disposed, so check for that and exit ASAP if appropriate.
2538        if !self.fiber_async_state_mut().can_block() {
2539            return Err(format_err!("future dropped"));
2540        }
2541
2542        let state = self.concurrent_state_mut()?;
2543
2544        // If we're yielding or waiting on behalf of a guest thread, we'll be
2545        // overwriting the current thread, so save it now and restore it once
2546        // we've resumed.
2547        //
2548        // Also, if we're yielding to a subtask, we're about to overwrite
2549        // `ConcurrentState::next_switch_item`, so also save and restore that.
2550        let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2551            SuspendReason::Yielding { .. }
2552            | SuspendReason::Waiting { .. }
2553            | SuspendReason::ExplicitlySuspending { .. } => {
2554                // If there's a thread waiting for this subtask to suspend, this
2555                // is a good time to switch back to it.
2556                if state.switch_item.is_none() {
2557                    state.take_next_switch_item()?;
2558                }
2559
2560                (true, false)
2561            }
2562            SuspendReason::YieldingToSubtask { .. } => (true, true),
2563            SuspendReason::NeedWork => (false, false),
2564        };
2565
2566        let old_next_switch_item = if save_and_restore_next_switch_item {
2567            Some(state.save_next_switch_item()?)
2568        } else {
2569            None
2570        };
2571
2572        let old_guest_thread = if save_and_restore_thread {
2573            self.current_thread()?
2574        } else {
2575            CurrentThread::None
2576        };
2577
2578        let waiting_set = match &reason {
2579            SuspendReason::Waiting { set, .. } => Some(*set),
2580            _ => None,
2581        };
2582
2583        let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2584        if suspend_reason.is_some() {
2585            bail_bug!("suspend reason unexpectedly already set for store");
2586        }
2587        *suspend_reason = Some(reason);
2588
2589        // Count this thread as waiting on `waiting_set`, if any, for as long as
2590        // it's suspended.
2591        if let Some(set) = waiting_set {
2592            self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2593        }
2594
2595        self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2596
2597        if let Some(set) = waiting_set {
2598            self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2599        }
2600
2601        if save_and_restore_thread {
2602            self.set_thread(old_guest_thread)?;
2603        }
2604
2605        if let Some(item) = old_next_switch_item {
2606            let state = self.concurrent_state_mut()?;
2607            state.restore_next_switch_item(item)?;
2608        }
2609
2610        Ok(())
2611    }
2612
2613    fn wait_for_event(
2614        &mut self,
2615        caller_instance: RuntimeInstance,
2616        waitable: Waitable,
2617    ) -> Result<()> {
2618        let caller = self.current_guest_thread()?;
2619        let state = self.concurrent_state_mut()?;
2620
2621        waitable.trap_if_in_waitable_set(state)?;
2622
2623        let set = state.get_mut(caller.thread)?.sync_call_set;
2624        waitable.join(state, Some(set))?;
2625
2626        self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2627
2628        self.suspend(SuspendReason::Waiting {
2629            set,
2630            thread: caller,
2631        })?;
2632        let state = self.concurrent_state_mut()?;
2633
2634        waitable.join(state, None)
2635    }
2636
2637    /// Cleans up the data structures backing the `guest_thread` specified,
2638    /// removing it from the internal tables of `runtime_instance` as well.
2639    ///
2640    /// This function is used whenever a guest thread has fully exited and
2641    /// completed. This'll clean up the associated `GuestThread` structure and
2642    /// related resources it contains.
2643    ///
2644    /// Other functionality that this implements is:
2645    ///
2646    /// * This will perform conditional cleanup of the `GuestTask` that owns
2647    ///   this thread if `cleanup_task` is `CleanupTask::Yes`.
2648    /// * If there are no more threads in the `GuestTask` that this thread is
2649    ///   associated with, and if the task hasn't produced a result (e.g. it's not
2650    ///   returned or cancelled), then a trap will be raised that a result
2651    ///   wasn't ever produced.
2652    /// * If this task is finished, meaning the top-level thread exited and
2653    ///   additionally it's been returned or cancelled, then this will handle
2654    ///   management of the store's "active interesting tasks" counter.
2655    ///
2656    /// Effectively this is intended to be a "narrow waist" through which many
2657    /// destruction operations are funneled through.
2658    fn cleanup_thread(
2659        &mut self,
2660        guest_thread: QualifiedThreadId,
2661        runtime_instance: RuntimeInstance,
2662    ) -> Result<()> {
2663        let state = self.concurrent_state_mut()?;
2664        // If we never suspended, we never had a chance to deliver a subtask
2665        // status update, if any, to our caller, so we do that here:
2666        state.take_next_switch_item()?;
2667        let thread_data = state.get_mut(guest_thread.thread)?;
2668        let sync_call_set = thread_data.sync_call_set;
2669        if let Some(guest_id) = thread_data.instance_rep {
2670            self.instance_state(runtime_instance)
2671                .thread_handle_table()
2672                .guest_thread_remove(guest_id)?;
2673        }
2674        let state = self.concurrent_state_mut()?;
2675
2676        // Clean up any pending subtasks in the sync_call_set
2677        for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2678            if let Some(Event::Subtask {
2679                status: Status::Returned | Status::ReturnCancelled,
2680            }) = waitable.common(state)?.event
2681            {
2682                waitable.delete_from(state)?;
2683            }
2684        }
2685
2686        state.delete(guest_thread.thread)?;
2687        state.delete(sync_call_set)?;
2688        let task = state.get_mut(guest_thread.task)?;
2689        task.threads.remove(&guest_thread.thread);
2690
2691        if task.threads.is_empty() && !task.returned_or_cancelled() {
2692            bail!(Trap::NoAsyncResult);
2693        }
2694
2695        if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2696            task.decremented_interesting_task_count = true;
2697
2698            debug_assert!(state.interesting_tasks > 0);
2699            state.interesting_tasks -= 1;
2700            if state.interesting_tasks == 0
2701                && let Some(waker) = state.interesting_tasks_empty_waker.take()
2702            {
2703                waker.wake();
2704            }
2705        }
2706
2707        state.decrement_task_ref_count(guest_thread.task)?;
2708
2709        Ok(())
2710    }
2711
2712    /// Performs cancellation of the `guest_task` specified with the
2713    /// precondition that the task hasn't lowered its parameters.
2714    ///
2715    /// In this situation the task hasn't ever been started meaning it hasn't
2716    /// actually run any wasm code yet. This requires cleaning up metadata such
2717    /// as thread information attached to the task.
2718    ///
2719    /// This is used for task cancellation via `subtask.cancel`, the intrinsic.
2720    fn cancel_guest_subtask_without_lowered_parameters(
2721        &mut self,
2722        caller_instance: RuntimeInstance,
2723        guest_task: TableId<GuestTask>,
2724    ) -> Result<()> {
2725        let concurrent_state = self.concurrent_state_mut()?;
2726        let task = concurrent_state.get_mut(guest_task)?;
2727        assert!(!task.already_lowered_parameters());
2728        // The task is in a `starting` state, meaning it hasn't run at
2729        // all yet.  Here we update its fields to indicate that it is
2730        // ready to delete immediately once `subtask.drop` is called.
2731        task.lower_params = None;
2732        task.lift_result = None;
2733        task.exited = true;
2734        let instance = task.instance;
2735
2736        // Clean up the thread within this task as it's now never going
2737        // to run.
2738        //
2739        // Note that `next_switch_item`, if any, belongs to the canceller and
2740        // not this thread since this thread never ran, so it's saved and
2741        // restored here to prevent `cleanup_thread` from taking it.
2742        assert_eq!(1, task.threads.len());
2743        let thread = *task.threads.iter().next().unwrap();
2744        let next_switch_item = concurrent_state.save_next_switch_item()?;
2745        self.cleanup_thread(
2746            QualifiedThreadId {
2747                task: guest_task,
2748                thread,
2749            },
2750            caller_instance,
2751        )?;
2752        self.concurrent_state_mut()?
2753            .restore_next_switch_item(next_switch_item)?;
2754
2755        // Not yet started; cancel and remove from pending
2756        let pending = &mut self.instance_state(instance).concurrent_state().pending;
2757        let pending_count = pending.len();
2758        pending.retain(|thread, _| thread.task != guest_task);
2759
2760        if pending.len() == pending_count {
2761            // If it wasn't found in the pending set, it might be waiting in one
2762            // of the task queues or switch items:
2763            let matches = |item: &WorkItem| match item {
2764                WorkItem::ResumeThread { thread, .. }
2765                | WorkItem::GuestCall {
2766                    call: GuestCall { thread, .. },
2767                    ..
2768                } => thread.task == guest_task,
2769                _ => false,
2770            };
2771
2772            let remove = |queue: &mut VecDeque<WorkItem>| {
2773                let count = queue.len();
2774                queue.retain(|item| !matches(item));
2775                count != queue.len()
2776            };
2777
2778            let state = self.concurrent_state_mut()?;
2779            let mut found = remove(&mut state.high_priority);
2780            found |= remove(&mut state.low_priority);
2781
2782            for item in [&mut state.switch_item, &mut state.next_switch_item] {
2783                if matches!(item, Some(item) if matches(item)) {
2784                    found = true;
2785                    *item = None;
2786                }
2787            }
2788
2789            if !found {
2790                bail!(Trap::SubtaskCancelAfterTerminal);
2791            }
2792        }
2793        Ok(())
2794    }
2795
2796    /// Used by `ResourceTables` to record the scope of a borrow to get undone
2797    /// in the future.
2798    pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2799        if !self.concurrency_support() {
2800            return Ok(self
2801                .current_scope_id_not_concurrent()?
2802                .map(|id| CurrentScope::Id(VMScope::id(id))));
2803        }
2804
2805        Ok(match self.current_thread()? {
2806            CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2807            CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2808            CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2809            CurrentThread::None => return Ok(None),
2810        })
2811    }
2812
2813    pub(crate) fn queue_task(
2814        &mut self,
2815        task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2816    ) -> Result<()> {
2817        self.concurrent_state_mut()?
2818            .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2819        Ok(())
2820    }
2821
2822    /// Used in `poll_until` just prior to trapping due to a "deadlock"
2823    /// condition.
2824    ///
2825    /// This helps us distinguish between a simple deadlock condition (where no
2826    /// work is available for the event loop to do, nor is there any way for new
2827    /// work to be added) and a "cannot block sync task" condition where at
2828    /// least once instance has an outstanding sync-typed task running, in which
2829    /// case we'll trap with a different error message.
2830    fn any_may_not_suspend(&mut self) -> Result<bool> {
2831        // Note that this currently requires a linear search across the whole
2832        // `ConcurrentState::table`.  We _could_ optimize that, but since (1)
2833        // this function is only used when trapping, (2) the only thing you can
2834        // really do with a store that's been poisoned by a trap is drop it, and
2835        // (3) we must do a linear search through the table when dropping a
2836        // store anyway to dispose of fibers, it's reasonable for us to also do
2837        // a linear search here.
2838        Ok(self
2839            .concurrent_state_mut()?
2840            .table
2841            .get_mut()
2842            .iter_mut()
2843            .filter_map(|(_, entry)| {
2844                if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2845                    Some(task.instance)
2846                } else {
2847                    None
2848                }
2849            })
2850            .collect::<Vec<_>>()
2851            .into_iter()
2852            .any(|instance| {
2853                self.instance_state(instance)
2854                    .concurrent_state()
2855                    .do_not_suspend
2856            }))
2857    }
2858}
2859
2860impl Instance {
2861    /// Get the next pending event for the specified task and (optional)
2862    /// waitable set, along with the waitable handle if applicable.
2863    fn get_event(
2864        self,
2865        store: &mut StoreOpaque,
2866        guest_task: TableId<GuestTask>,
2867        set: Option<TableId<WaitableSet>>,
2868        cancellable: bool,
2869    ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
2870        let state = store.concurrent_state_mut()?;
2871
2872        let task = state.get_mut(guest_task)?;
2873        let event = &mut task.event;
2874        if let Some(ev) = event
2875            && (cancellable || !matches!(ev, Event::Cancelled))
2876        {
2877            log::trace!("deliver event {ev:?} to {guest_task:?}");
2878
2879            if matches!(ev, Event::Cancelled) {
2880                task.cancel_request_delivered = true;
2881            }
2882
2883            let ev = *ev;
2884            *event = None;
2885            return Ok(Some((ev, None)));
2886        }
2887
2888        let set = match set {
2889            Some(set) => set,
2890            None => return Ok(None),
2891        };
2892        let waitable = match state.get_mut(set)?.ready.pop_first() {
2893            Some(v) => v,
2894            None => return Ok(None),
2895        };
2896
2897        let common = waitable.common(state)?;
2898        let handle = match common.handle {
2899            Some(h) => h,
2900            None => bail_bug!("handle not set when delivering event"),
2901        };
2902        let event = match common.event.take() {
2903            Some(e) => e,
2904            None => bail_bug!("event not set when delivering event"),
2905        };
2906
2907        log::trace!(
2908            "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
2909        );
2910
2911        waitable.on_delivery(store, self, event)?;
2912
2913        Ok(Some((event, Some((waitable, handle)))))
2914    }
2915
2916    /// Run `call` for the callback-lifted `guest_thread` and then handle the
2917    /// `CallbackCode` it returns.
2918    ///
2919    /// `call` runs with `guest_thread` as the current thread and with
2920    /// `runtime_instance` entered; both are restored before the final code is
2921    /// handled.
2922    fn handle_callback_code(
2923        self,
2924        store: &mut dyn VMStore,
2925        guest_thread: QualifiedThreadId,
2926        runtime_instance: RuntimeInstance,
2927        call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
2928    ) -> Result<()> {
2929        let old_thread = store.set_thread(guest_thread)?;
2930        log::trace!(
2931            "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
2932        );
2933
2934        store.enter_instance(runtime_instance);
2935
2936        let mut code = call(store)?;
2937
2938        // Implement the spec behavior here of effectively ignoring the return
2939        // code if there's a pending cancellation for this task available.
2940        //
2941        // TODO: the spec doesn't validate anything here, but seems like the
2942        // waitable set may want to be validated?
2943        if matches!(
2944            unpack_callback_code(code).0,
2945            callback_code::YIELD | callback_code::WAIT
2946        ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
2947        {
2948            let (unpacked, set) = unpack_callback_code(code);
2949            log::trace!(
2950                "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
2951                 cancellation; delivering it via callback"
2952            );
2953            code = invoke_callback(store, guest_thread.task, event, 0)?;
2954        }
2955
2956        store.exit_instance(runtime_instance)?;
2957
2958        store.set_thread(old_thread)?;
2959        log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
2960
2961        let store = store.store_opaque_mut();
2962        let (code, set) = unpack_callback_code(code);
2963
2964        log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
2965
2966        let state = store.concurrent_state_mut()?;
2967
2968        state.take_next_switch_item()?;
2969
2970        let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
2971            let set = store
2972                .instance_state(runtime_instance)
2973                .handle_table()
2974                .waitable_set_rep(handle)?;
2975
2976            Ok(TableId::<WaitableSet>::new(set))
2977        };
2978
2979        match code {
2980            callback_code::EXIT => {
2981                log::trace!("implicit thread {guest_thread:?} completed");
2982                let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
2983                task.exited = true;
2984                task.callback = None;
2985
2986                // Since we're async-typed, returning control to our caller
2987                // won't help resolve any outstanding sync-typed call which
2988                // might be in progress, so we may need to switch or trap before
2989                // exiting this thread:
2990                store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
2991
2992                store.cleanup_thread(guest_thread, runtime_instance)?;
2993            }
2994            callback_code::YIELD => {
2995                // Set `GuestTask::wake_on_cancel` to allow `subtask.cancel` to
2996                // promote this thread if appropriate.
2997                let old = state
2998                    .get_mut(guest_thread.thread)?
2999                    .wake_on_cancel
3000                    .replace(WakeOnCancel::Yielding);
3001                if !old.is_none() {
3002                    bail_bug!("thread unexpectedly had wake_on_cancel set");
3003                }
3004
3005                // If an `Event::Cancelled` is pending in `GuestTask::event`
3006                // when this call is delivered, we'll deliver that; otherwise,
3007                // we'll deliver `Event::None`.  Note that the `Event::None` is
3008                // deliberately not stored in `GuestTask::event` since that's
3009                // shared by all threads of the task and the `Event::None` is
3010                // only for this thread.
3011                let call = GuestCall {
3012                    thread: guest_thread,
3013                    kind: GuestCallKind::DeliverEvent {
3014                        instance: self,
3015                        set: None,
3016                    },
3017                };
3018                // Push this thread onto the "low priority" queue so it runs
3019                // after any other threads have had a chance to run.
3020                state.push_low_priority(WorkItem::GuestCall {
3021                    instance: runtime_instance,
3022                    call,
3023                });
3024            }
3025            callback_code::WAIT => {
3026                let set = get_set(store, set)?;
3027                self.wait_with_callback(store, guest_thread, set)?;
3028            }
3029            _ => bail!(Trap::UnsupportedCallbackCode),
3030        }
3031
3032        Ok(())
3033    }
3034
3035    /// Start waiting on `set` with `guest_thread`, which uses a callback.
3036    ///
3037    /// The thread will receive an event from `set` via a
3038    /// `GuestCallKind::DeliverEvent` work item, either immediately if one is
3039    /// already available, or once one is published.
3040    fn wait_with_callback(
3041        self,
3042        store: &mut StoreOpaque,
3043        guest_thread: QualifiedThreadId,
3044        set: TableId<WaitableSet>,
3045    ) -> Result<()> {
3046        let state = store.concurrent_state_mut()?;
3047        // This thread's wait ends when that work item is handled in
3048        // `handle_guest_call`.
3049        state.get_mut(set)?.num_waiting += 1;
3050
3051        if state.get_mut(guest_thread.task)?.event.is_some()
3052            || !state.get_mut(set)?.ready.is_empty()
3053        {
3054            // An event is immediately available; deliver it ASAP.
3055            let instance = state.get_mut(guest_thread.task)?.instance;
3056            state.push_high_priority(WorkItem::GuestCall {
3057                instance,
3058                call: GuestCall {
3059                    thread: guest_thread,
3060                    kind: GuestCallKind::DeliverEvent {
3061                        instance: self,
3062                        set: Some(set),
3063                    },
3064                },
3065            });
3066            return Ok(());
3067        }
3068
3069        // No event is immediately available, so this thread is about to block.
3070        // Like any other suspension point (e.g. the callback `EXIT` path), if
3071        // a sync-typed call is in progress for this instance we must first
3072        // switch to another ready thread of that instance, or trap if there is
3073        // none.
3074        let instance = store
3075            .concurrent_state_mut()?
3076            .get_mut(guest_thread.task)?
3077            .instance;
3078        store.switch_or_trap_if_may_not_suspend(instance)?;
3079
3080        // Register to be woken up when an event is published for this waitable
3081        // set.
3082        //
3083        // Here we also set `GuestTask::wake_on_cancel` which allows
3084        // `subtask.cancel` to interrupt the wait.
3085        let state = store.concurrent_state_mut()?;
3086        let old = state
3087            .get_mut(guest_thread.thread)?
3088            .wake_on_cancel
3089            .replace(WakeOnCancel::Waiting(set));
3090        if !old.is_none() {
3091            bail_bug!("thread unexpectedly had wake_on_cancel set");
3092        }
3093        let old = state
3094            .get_mut(set)?
3095            .waiting
3096            .insert(guest_thread, WaitMode::Callback(self));
3097        if !old.is_none() {
3098            bail_bug!("set's waiting set already had this thread registered");
3099        }
3100        Ok(())
3101    }
3102
3103    /// Stage the specified guest call as the work item to run next, to be
3104    /// started as soon as backpressure and/or reentrance rules allow.
3105    ///
3106    /// SAFETY: The raw pointer arguments must be valid references to guest
3107    /// functions (with the appropriate signatures) when the closures staged by
3108    /// this function are called.
3109    unsafe fn stage_call<T: 'static>(
3110        self,
3111        mut store: StoreContextMut<T>,
3112        guest_thread: QualifiedThreadId,
3113        callee: SendSyncPtr<VMFuncRef>,
3114        param_count: usize,
3115        result_count: usize,
3116        async_: bool,
3117        callback: Option<SendSyncPtr<VMFuncRef>>,
3118        post_return: Option<SendSyncPtr<VMFuncRef>>,
3119        host_caller: bool,
3120    ) -> Result<()> {
3121        /// Return a closure which will call the specified function in the scope
3122        /// of the specified task.
3123        ///
3124        /// This will use `GuestTask::lower_params` to lower the parameters, but
3125        /// will not lift the result; instead, it returns a
3126        /// `[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]` from which the result, if
3127        /// any, may be lifted.  Note that an async-lifted export will have
3128        /// returned its result using the `task.return` intrinsic (or not
3129        /// returned a result at all, in the case of `task.cancel`), in which
3130        /// case the "result" of this call will either be a callback code or
3131        /// nothing.
3132        ///
3133        /// SAFETY: `callee` must be a valid `*mut VMFuncRef` at the time when
3134        /// the returned closure is called.
3135        unsafe fn make_call<T: 'static>(
3136            store: StoreContextMut<T>,
3137            guest_thread: QualifiedThreadId,
3138            callee: SendSyncPtr<VMFuncRef>,
3139            param_count: usize,
3140            result_count: usize,
3141        ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3142        + Send
3143        + Sync
3144        + 'static
3145        + use<T> {
3146            let token = StoreToken::new(store);
3147            move |store: &mut dyn VMStore| {
3148                let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3149
3150                store
3151                    .concurrent_state_mut()?
3152                    .set_thread_running(guest_thread.thread)?;
3153                let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3154                let lower = match task.lower_params.take() {
3155                    Some(l) => l,
3156                    None => bail_bug!("lower_params missing"),
3157                };
3158
3159                lower(store, &mut storage[..param_count])?;
3160
3161                let mut store = token.as_context_mut(store);
3162
3163                // SAFETY: Per the contract documented in `make_call's`
3164                // documentation, `callee` must be a valid pointer.
3165                unsafe {
3166                    crate::Func::call_unchecked_raw(
3167                        &mut store,
3168                        callee.as_non_null(),
3169                        NonNull::new(
3170                            &mut storage[..param_count.max(result_count)]
3171                                as *mut [MaybeUninit<ValRaw>] as _,
3172                        )
3173                        .unwrap(),
3174                        UncaughtException::Trap,
3175                    )?;
3176                }
3177
3178                Ok(storage)
3179            }
3180        }
3181
3182        // SAFETY: Per the contract described in this function documentation,
3183        // the `callee` pointer which `call` closes over must be valid when
3184        // called by the closure we queue below.
3185        let call = unsafe {
3186            make_call(
3187                store.as_context_mut(),
3188                guest_thread,
3189                callee,
3190                param_count,
3191                result_count,
3192            )
3193        };
3194
3195        let callee_instance = store
3196            .0
3197            .concurrent_state_mut()?
3198            .get_mut(guest_thread.task)?
3199            .instance;
3200
3201        let fun = if callback.is_some() {
3202            assert!(async_);
3203
3204            Box::new(move |store: &mut dyn VMStore| {
3205                self.add_guest_thread_to_instance_table(
3206                    guest_thread.thread,
3207                    store,
3208                    callee_instance.index,
3209                )?;
3210                self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3211                    let storage = call(store)?;
3212
3213                    // SAFETY: `wasmparser` will have validated that the
3214                    // callback function returns a `i32` result.
3215                    Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3216                })
3217            }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3218        } else {
3219            let token = StoreToken::new(store.as_context_mut());
3220            Box::new(move |store: &mut dyn VMStore| {
3221                self.add_guest_thread_to_instance_table(
3222                    guest_thread.thread,
3223                    store,
3224                    callee_instance.index,
3225                )?;
3226                let old_thread = store.set_thread(guest_thread)?;
3227                log::trace!(
3228                    "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3229                );
3230                let flags = self.id().get(store).instance_flags(callee_instance.index);
3231
3232                let callee_async_typed = store
3233                    .concurrent_state_mut()?
3234                    .get_mut(guest_thread.task)?
3235                    .async_typed;
3236
3237                // Unless this is a callback-less (i.e. stackful) async-lifted
3238                // or sync-typed export, we need to record that the instance
3239                // cannot be entered until the call returns.
3240                if !async_ && callee_async_typed {
3241                    store.enter_instance(callee_instance);
3242                }
3243
3244                if !callee_async_typed {
3245                    store.enter_sync_call(callee_instance)?;
3246                }
3247
3248                // SAFETY: See the documentation for `make_call` to review the
3249                // contract we must uphold for `call` here.
3250                //
3251                // Per the contract described in the `stage_call`
3252                // documentation, the `callee` pointer which `call` closes
3253                // over must be valid.
3254                let storage = call(store)?;
3255
3256                if !callee_async_typed {
3257                    store.exit_sync_call(callee_instance)?;
3258                }
3259
3260                if !async_ {
3261                    // This is a sync-lifted export, so now is when we lift the
3262                    // result, optionally call the post-return function, if any,
3263                    // and finally notify any current or future waiters that the
3264                    // subtask has returned.
3265
3266                    if callee_async_typed {
3267                        store.exit_instance(callee_instance)?;
3268                    }
3269
3270                    let lift = {
3271                        let state = store.concurrent_state_mut()?;
3272                        if !state.get_mut(guest_thread.task)?.result.is_none() {
3273                            bail_bug!("task has already produced a result");
3274                        }
3275
3276                        match state.get_mut(guest_thread.task)?.lift_result.take() {
3277                            Some(lift) => lift,
3278                            None => bail_bug!("lift_result field is missing"),
3279                        }
3280                    };
3281
3282                    // SAFETY: `result_count` represents the number of core Wasm
3283                    // results returned, per `wasmparser`.
3284                    let result = (lift.lift)(store, unsafe {
3285                        mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3286                            &storage[..result_count],
3287                        )
3288                    })?;
3289
3290                    let post_return_arg = match result_count {
3291                        0 => ValRaw::i32(0),
3292                        // SAFETY: `result_count` represents the number of
3293                        // core Wasm results returned, per `wasmparser`.
3294                        1 => unsafe { storage[0].assume_init() },
3295                        _ => unreachable!(),
3296                    };
3297
3298                    unsafe {
3299                        call_post_return(
3300                            token.as_context_mut(store),
3301                            post_return.map(|v| v.as_non_null()),
3302                            post_return_arg,
3303                            flags,
3304                        )?;
3305                    }
3306
3307                    self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3308                }
3309
3310                store.set_thread(old_thread)?;
3311
3312                store
3313                    .concurrent_state_mut()?
3314                    .get_mut(guest_thread.task)?
3315                    .exited = true;
3316
3317                log::trace!(
3318                    "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3319                );
3320
3321                if callee_async_typed {
3322                    // If we're async-typed, returning control to our caller
3323                    // won't help resolve any outstanding sync-typed call which
3324                    // might be in progress, so we may need to switch or trap
3325                    // before exiting this thread:
3326                    store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3327                }
3328
3329                // This is a callback-less call, so the implicit thread has now completed
3330                store.cleanup_thread(guest_thread, callee_instance)?;
3331                Ok(())
3332            })
3333        };
3334
3335        store.0.concurrent_state_mut()?.push_work_item(
3336            WorkItem::GuestCall {
3337                instance: callee_instance,
3338                call: GuestCall {
3339                    thread: guest_thread,
3340                    kind: GuestCallKind::StartImplicit(fun),
3341                },
3342            },
3343            if host_caller {
3344                Priority::High
3345            } else {
3346                Priority::Switch
3347            },
3348        )?;
3349
3350        Ok(())
3351    }
3352
3353    /// Prepare (but do not start) a guest->guest call.
3354    ///
3355    /// This is called from fused adapter code generated in
3356    /// `wasmtime_environ::fact::trampoline::Compiler`.  `start` and `return_`
3357    /// are synthesized Wasm functions which move the parameters from the caller
3358    /// to the callee and the result from the callee to the caller,
3359    /// respectively.  The adapter will call `Self::start_call` immediately
3360    /// after calling this function.
3361    ///
3362    /// SAFETY: All the pointer arguments must be valid pointers to guest
3363    /// entities (and with the expected signatures for the function references
3364    /// -- see `wasmtime_environ::fact::trampoline::Compiler` for details).
3365    unsafe fn prepare_call<T: 'static>(
3366        self,
3367        mut store: StoreContextMut<T>,
3368        start: NonNull<VMFuncRef>,
3369        return_: NonNull<VMFuncRef>,
3370        caller_instance: RuntimeComponentInstanceIndex,
3371        callee_instance: RuntimeComponentInstanceIndex,
3372        task_return_type: TypeTupleIndex,
3373        callee_async_typed: bool,
3374        memory: *mut VMMemoryDefinition,
3375        string_encoding: StringEncoding,
3376        caller_info: CallerInfo,
3377    ) -> Result<()> {
3378        enum ResultInfo {
3379            Heap { results: u32 },
3380            Stack { result_count: u32 },
3381        }
3382
3383        let result_info = match &caller_info {
3384            CallerInfo::Async {
3385                has_result: true,
3386                params,
3387            } => ResultInfo::Heap {
3388                // FIXME(#4311): needs to read an i64 for memory64
3389                results: match params.last() {
3390                    Some(r) => r.get_u32(),
3391                    None => bail_bug!("retptr missing"),
3392                },
3393            },
3394            CallerInfo::Async {
3395                has_result: false, ..
3396            } => ResultInfo::Stack { result_count: 0 },
3397            CallerInfo::Sync {
3398                result_count,
3399                params,
3400            } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3401                // FIXME(#4311): needs to read an i64 for memory64
3402                results: match params.last() {
3403                    Some(r) => r.get_u32(),
3404                    None => bail_bug!("arg ptr missing"),
3405                },
3406            },
3407            CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3408                result_count: *result_count,
3409            },
3410        };
3411
3412        let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3413
3414        // Create a new guest task for the call, closing over the `start` and
3415        // `return_` functions to lift the parameters and lower the result,
3416        // respectively.
3417        let start = SendSyncPtr::new(start);
3418        let return_ = SendSyncPtr::new(return_);
3419        let token = StoreToken::new(store.as_context_mut());
3420        let old_thread = store.0.current_guest_thread()?;
3421
3422        let state = store.0.concurrent_state_mut()?;
3423
3424        debug_assert_eq!(
3425            state.get_mut(old_thread.task)?.instance,
3426            self.runtime_instance(caller_instance)
3427        );
3428
3429        let guest_thread = GuestTask::new(
3430            state,
3431            Box::new(move |store, dst| {
3432                let mut store = token.as_context_mut(store);
3433                assert!(dst.len() <= MAX_FLAT_PARAMS);
3434                // The `+ 1` here accounts for the return pointer, if any:
3435                let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3436                let count = match caller_info {
3437                    // Async callers, if they have a result, use the last
3438                    // parameter as a return pointer so chop that off if
3439                    // relevant here.
3440                    CallerInfo::Async { params, has_result } => {
3441                        let params = &params[..params.len() - usize::from(has_result)];
3442                        for (param, src) in params.iter().zip(&mut src) {
3443                            src.write(*param);
3444                        }
3445                        params.len()
3446                    }
3447
3448                    // Sync callers forward everything directly.
3449                    CallerInfo::Sync { params, .. } => {
3450                        for (param, src) in params.iter().zip(&mut src) {
3451                            src.write(*param);
3452                        }
3453                        params.len()
3454                    }
3455                };
3456                // SAFETY: `start` is a valid `*mut VMFuncRef` from
3457                // `wasmtime-cranelift`-generated fused adapter code.  Based on
3458                // how it was constructed (see
3459                // `wasmtime_environ::fact::trampoline::Compiler::compile_async_start_adapter`
3460                // for details) we know it takes count parameters and returns
3461                // `dst.len()` results.
3462                unsafe {
3463                    crate::Func::call_unchecked_raw(
3464                        &mut store,
3465                        start.as_non_null(),
3466                        NonNull::new(
3467                            &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3468                        )
3469                        .unwrap(),
3470                        UncaughtException::Trap,
3471                    )?;
3472                }
3473                dst.copy_from_slice(&src[..dst.len()]);
3474                let task = store.0.current_guest_thread()?.task;
3475                let state = store.0.concurrent_state_mut()?;
3476                Waitable::Guest(task).set_event(
3477                    state,
3478                    Some(Event::Subtask {
3479                        status: Status::Started,
3480                    }),
3481                )?;
3482                Ok(())
3483            }),
3484            LiftResult {
3485                lift: Box::new(move |store, src| {
3486                    // SAFETY: See comment in closure passed as `lower_params`
3487                    // parameter above.
3488                    let mut store = token.as_context_mut(store);
3489                    let mut my_src = src.to_owned(); // TODO: use stack to avoid allocation?
3490                    if let ResultInfo::Heap { results } = &result_info {
3491                        my_src.push(ValRaw::u32(*results));
3492                    }
3493
3494                    // SAFETY: `return_` is a valid `*mut VMFuncRef` from
3495                    // `wasmtime-cranelift`-generated fused adapter code.  Based
3496                    // on how it was constructed (see
3497                    // `wasmtime_environ::fact::trampoline::Compiler::compile_async_return_adapter`
3498                    // for details) we know it takes `src.len()` parameters and
3499                    // returns up to 1 result.
3500                    unsafe {
3501                        crate::Func::call_unchecked_raw(
3502                            &mut store,
3503                            return_.as_non_null(),
3504                            my_src.as_mut_slice().into(),
3505                            UncaughtException::Trap,
3506                        )?;
3507                    }
3508
3509                    let thread = store.0.current_guest_thread()?;
3510                    let state = store.0.concurrent_state_mut()?;
3511                    if sync_caller {
3512                        state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3513                            if let ResultInfo::Stack { result_count } = &result_info {
3514                                match result_count {
3515                                    0 => None,
3516                                    1 => Some(my_src[0]),
3517                                    _ => unreachable!(),
3518                                }
3519                            } else {
3520                                None
3521                            },
3522                        );
3523                    }
3524                    Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3525                }),
3526                ty: task_return_type,
3527                memory: NonNull::new(memory).map(SendSyncPtr::new),
3528                string_encoding,
3529            },
3530            Caller::Guest { thread: old_thread },
3531            None,
3532            self.runtime_instance(callee_instance),
3533            callee_async_typed,
3534            // We don't know whether the callee export was lifted sync or async
3535            // yet, but we'll update this in `start_call`:
3536            false,
3537        )?;
3538
3539        // Make the new thread the current one so that `Self::start_call` knows
3540        // which one to start.
3541        store.0.set_thread(guest_thread)?;
3542        log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3543
3544        Ok(())
3545    }
3546
3547    /// Call the specified callback function for an async-lifted export.
3548    ///
3549    /// SAFETY: `function` must be a valid reference to a guest function of the
3550    /// correct signature for a callback.
3551    unsafe fn call_callback<T>(
3552        self,
3553        mut store: StoreContextMut<T>,
3554        function: SendSyncPtr<VMFuncRef>,
3555        event: Event,
3556        handle: u32,
3557    ) -> Result<u32> {
3558        let (ordinal, result) = event.parts();
3559        let params = &mut [
3560            ValRaw::u32(ordinal),
3561            ValRaw::u32(handle),
3562            ValRaw::u32(result),
3563        ];
3564        // SAFETY: `func` is a valid `*mut VMFuncRef` from either
3565        // `wasmtime-cranelift`-generated fused adapter code or
3566        // `component::Options`.  Per `wasmparser` callback signature
3567        // validation, we know it takes three parameters and returns one.
3568        unsafe {
3569            crate::Func::call_unchecked_raw(
3570                &mut store,
3571                function.as_non_null(),
3572                params.as_mut_slice().into(),
3573                UncaughtException::Trap,
3574            )?;
3575        }
3576        Ok(params[0].get_u32())
3577    }
3578
3579    /// Start a guest->guest call previously prepared using
3580    /// `Self::prepare_call`.
3581    ///
3582    /// This is called from fused adapter code generated in
3583    /// `wasmtime_environ::fact::trampoline::Compiler`.  The adapter will call
3584    /// this function immediately after calling `Self::prepare_call`.
3585    ///
3586    /// If `flags` contains `START_FLAG_ASYNC_CALLER` then the caller used an
3587    /// async-lowered import and the call's packed status is written to
3588    /// `storage[0]`. Otherwise this blocks until the callee has produced its
3589    /// result, if any, which is written to `storage[0]`.
3590    ///
3591    /// SAFETY: The `*mut VMFuncRef` arguments must be valid pointers to guest
3592    /// functions with the appropriate signatures for the current guest task.
3593    /// If this is a call to an async-lowered import, the actual call may be
3594    /// deferred and run after this function returns, in which case the pointer
3595    /// arguments must also be valid when the call happens.
3596    unsafe fn start_call<T: 'static>(
3597        self,
3598        mut store: StoreContextMut<T>,
3599        callback: *mut VMFuncRef,
3600        post_return: *mut VMFuncRef,
3601        callee: NonNull<VMFuncRef>,
3602        param_count: u32,
3603        result_count: u32,
3604        flags: u32,
3605        storage: &mut [MaybeUninit<ValRaw>],
3606    ) -> Result<()> {
3607        let token = StoreToken::new(store.as_context_mut());
3608        let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3609        let guest_thread = store.0.current_guest_thread()?;
3610        let state = store.0.concurrent_state_mut()?;
3611
3612        // FIXME(#14950): this in theory should be a `bail_bug!`, but this can
3613        // be hit through `Func::call` for example. This isn't great because it
3614        // means that `Func::call` is sort of fundamentally incompatible with
3615        // component-model-async...
3616        if !state.event_loop_running {
3617            bail!(
3618                "async function started during an original synchronous \
3619                 entrypoint; please use an async-style entrypoint instead"
3620            );
3621        }
3622
3623        let callee = SendSyncPtr::new(callee);
3624        let param_count = usize::try_from(param_count)?;
3625        assert!(param_count <= MAX_FLAT_PARAMS);
3626        let result_count = usize::try_from(result_count)?;
3627        assert!(result_count <= MAX_FLAT_RESULTS);
3628
3629        let task = state.get_mut(guest_thread.task)?;
3630        let callee_async_typed = task.async_typed;
3631        let callee_instance = task.instance;
3632
3633        task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3634
3635        if let Some(callback) = NonNull::new(callback) {
3636            // We're calling an async-lifted export with a callback, so store
3637            // the callback and related context as part of the task so we can
3638            // call it later when needed.
3639            let callback = SendSyncPtr::new(callback);
3640            task.callback = Some(Box::new(move |store, event, handle| {
3641                let store = token.as_context_mut(store);
3642                unsafe { self.call_callback::<T>(store, callback, event, handle) }
3643            }));
3644        }
3645
3646        let Caller::Guest { thread: caller } = &task.caller else {
3647            // As of this writing, `start_call` is only used for guest->guest
3648            // calls.
3649            bail_bug!("start_call unexpectedly invoked for host->guest call");
3650        };
3651        let caller = *caller;
3652        let caller_instance = state.get_mut(caller.task)?.instance;
3653
3654        // Stage the call as the work item to run next, modulo backpressure, etc.
3655        unsafe {
3656            self.stage_call(
3657                store.as_context_mut(),
3658                guest_thread,
3659                callee,
3660                param_count,
3661                result_count,
3662                (flags & START_FLAG_ASYNC_CALLEE) != 0,
3663                NonNull::new(callback).map(SendSyncPtr::new),
3664                NonNull::new(post_return).map(SendSyncPtr::new),
3665                false,
3666            )?;
3667        }
3668
3669        let old_do_not_suspend = if callee_async_typed {
3670            // If we're starting an async-typed call, it is permitted to suspend
3671            // since that will just return control back to the caller, which may
3672            // be able to avoid blocking if needed.
3673            //
3674            // We'll restore the old value after the call either returns or
3675            // suspends.
3676            let state = store.0.instance_state(callee_instance).concurrent_state();
3677            let old_do_not_suspend = state.do_not_suspend;
3678            state.do_not_suspend = false;
3679            Some(old_do_not_suspend)
3680        } else {
3681            None
3682        };
3683
3684        let state = store.0.concurrent_state_mut()?;
3685
3686        // Use the caller's `GuestThread::sync_call_set` to register interest in
3687        // the subtask...
3688        let guest_waitable = Waitable::Guest(guest_thread.task);
3689        let old_set = guest_waitable.common(state)?.set;
3690        let set = state.get_mut(caller.thread)?.sync_call_set;
3691        guest_waitable.join(state, Some(set))?;
3692
3693        store.0.set_thread(CurrentThread::None)?;
3694
3695        // ... and suspend this fiber temporarily while we wait for it to start.
3696        //
3697        // Note that we _could_ call the callee directly using the current fiber
3698        // rather than suspend this one, but that would make reasoning about the
3699        // event loop more complicated and is probably only worth doing if
3700        // there's a measurable performance benefit.  In addition, it would mean
3701        // blocking the caller if the callee calls a blocking sync-lowered
3702        // import, and as of this writing the spec says we must not do that.
3703        //
3704        // Alternatively, the fused adapter code could be modified to call the
3705        // callee directly without calling a host-provided intrinsic at all (in
3706        // which case it would need to do its own, inline backpressure checks,
3707        // etc.).  Again, we'd want to see a measurable performance benefit
3708        // before committing to such an optimization.  And again, we'd need to
3709        // update the spec to allow that.
3710        let mut yielded = false;
3711        let (status, waitable) = loop {
3712            store.0.suspend(if yielded {
3713                SuspendReason::Waiting {
3714                    set,
3715                    thread: caller,
3716                }
3717            } else {
3718                yielded = true;
3719                SuspendReason::YieldingToSubtask { thread: caller }
3720            })?;
3721
3722            if let Some(old_do_not_suspend) = old_do_not_suspend {
3723                store
3724                    .0
3725                    .instance_state(callee_instance)
3726                    .concurrent_state()
3727                    .do_not_suspend = old_do_not_suspend;
3728            }
3729
3730            let state = store.0.concurrent_state_mut()?;
3731
3732            log::trace!("taking event for {:?}", guest_thread.task);
3733            let event = guest_waitable.take_event(state)?;
3734            let Some(Event::Subtask { status }) = event else {
3735                bail_bug!("subtasks should only get subtask events, got {event:?}")
3736            };
3737
3738            log::trace!("status {status:?} for {:?}", guest_thread.task);
3739
3740            if status == Status::Returned {
3741                // It returned, so we can stop waiting.
3742                break (status, None);
3743            } else if async_caller {
3744                // It hasn't returned yet, but the caller is calling via an
3745                // async-lowered import, so we generate a handle for the task
3746                // waitable and return the status.
3747                let handle = store
3748                    .0
3749                    .handle_table_insertion(caller_instance)
3750                    .subtask_guest(guest_thread.task.rep())?;
3751                store
3752                    .0
3753                    .concurrent_state_mut()?
3754                    .get_mut(guest_thread.task)?
3755                    .common
3756                    .handle = Some(handle);
3757                break (status, Some(handle));
3758            } else {
3759                // The callee hasn't returned yet, and the caller is calling via
3760                // a sync-lowered import, so we loop and keep waiting until the
3761                // callee returns.
3762                store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3763            }
3764        };
3765
3766        guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3767
3768        // Reset the current thread to point to the caller as it resumes control.
3769        store.0.set_thread(caller)?;
3770
3771        store
3772            .0
3773            .concurrent_state_mut()?
3774            .set_thread_running(caller.thread)?;
3775
3776        log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3777
3778        if async_caller {
3779            let Some(slot) = storage.first_mut() else {
3780                bail_bug!("no storage for async call status");
3781            };
3782            *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3783        } else {
3784            // The caller used a sync-lowered import, in which case the result,
3785            // if any, has been stashed in `GuestTask::sync_result`.
3786            let state = store.0.concurrent_state_mut()?;
3787            let task = state.get_mut(guest_thread.task)?;
3788            if let Some(Some(result)) = task.sync_result.take()? {
3789                storage[0] = MaybeUninit::new(result);
3790            }
3791        }
3792
3793        if status == Status::Returned {
3794            store
3795                .0
3796                .concurrent_state_mut()?
3797                .decrement_task_ref_count(guest_thread.task)?;
3798        }
3799
3800        Ok(())
3801    }
3802
3803    /// Poll the specified future once on behalf of a guest->host call using an
3804    /// async-lowered import.
3805    ///
3806    /// If it returns `Ready`, return `Ok(None)`.  Otherwise, if it returns
3807    /// `Pending`, add it to the set of futures to be polled as part of this
3808    /// instance's event loop until it completes, and then return
3809    /// `Ok(Some(handle))` where `handle` is the waitable handle to return.
3810    ///
3811    /// Whether the future returns `Ready` immediately or later, the `lower`
3812    /// function will be used to lower the result, if any, into the guest caller's
3813    /// stack and linear memory. The `lower` function is invoked with the
3814    /// `Option<R>` param `None` if the future is cancelled. The
3815    /// `Option<TableId<HostTask>>` is passed as `Some` if the host task was
3816    /// materialized during execution and allows `lower` to delete the task if
3817    /// needed.
3818    pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3819        self,
3820        mut store: StoreContextMut<'_, T>,
3821        host_task: EnteredHostTask,
3822        result_may_require_realloc: bool,
3823        future: impl Future<Output = Result<R>> + Send + 'static,
3824        lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3825        + Send
3826        + 'static,
3827    ) -> Result<u32> {
3828        let token = StoreToken::new(store.as_context_mut());
3829
3830        // Create an abortable future which hooks calls to poll and manages call
3831        // context state for the future.
3832        let (join_handle, future) = JoinHandle::run(future);
3833        let mut future = Box::pin(future);
3834
3835        // Finally, poll the future.  We can use a dummy `Waker` here because
3836        // we'll add the future to `ConcurrentState::futures` and poll it
3837        // automatically from the event loop if it doesn't complete immediately
3838        // here.
3839        let poll = tls::set(store.0, || {
3840            future
3841                .as_mut()
3842                .poll(&mut Context::from_waker(&Waker::noop()))
3843        });
3844
3845        match poll {
3846            // It finished immediately; lower the result and delete the task.
3847            Poll::Ready(result) => {
3848                let result = result.transpose()?;
3849                // Check if the host task was materialized so that it can be
3850                // deleted in `lower`.
3851                let task = store.0.current_materialized_host_task()?;
3852                lower(store.as_context_mut(), result, true, task)?;
3853                return Ok(Status::Returned.pack(None));
3854            }
3855
3856            // Future isn't ready yet, so fall through.
3857            Poll::Pending => {}
3858        }
3859
3860        // The future will outlive this call frame, so materialize the deferred
3861        // host task and attach its cancellation handle before publishing it to
3862        // the event loop.
3863        let Some(task) = store.0.materialize_host_task_id()? else {
3864            bail_bug!("current thread is not a host thread")
3865        };
3866        {
3867            let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
3868            assert!(matches!(state, HostTaskState::CalleeStarted));
3869            *state = HostTaskState::CalleeRunning(join_handle);
3870        }
3871
3872        // It hasn't finished yet; add the future to
3873        // `ConcurrentState::futures` so it will be polled by the event
3874        // loop and allocate a waitable handle to return to the guest.
3875
3876        // Wrap the future in a closure responsible for lowering the result into
3877        // the guest's stack and memory, as well as notifying any waiters that
3878        // the task returned.
3879        let future = Box::pin(async move {
3880            let result = match run_with_host_task_set(task, future).await? {
3881                Some(result) => Some(result?),
3882                None => None,
3883            };
3884            let on_complete = move |store: &mut dyn VMStore| {
3885                // Restore the `current_thread` to be the host so `lower` knows
3886                // how to manipulate borrows and knows which scope of borrows
3887                // to check.
3888                let mut store = token.as_context_mut(store);
3889                let old = store.0.set_thread(task)?;
3890
3891                let status = if result.is_some() {
3892                    Status::Returned
3893                } else {
3894                    Status::ReturnCancelled
3895                };
3896
3897                lower(store.as_context_mut(), result, false, Some(task))?;
3898                let state = store.0.concurrent_state_mut()?;
3899                match &mut state.get_mut(task)?.state {
3900                    // Cancellation is only complete once lowering has finished
3901                    // and the terminal event can be published.
3902                    pending @ HostTaskState::CalleeCancelling => {
3903                        *pending = HostTaskState::CalleeDone { cancelled: true };
3904                    }
3905
3906                    // Otherwise transition this task to the done state.
3907                    other => *other = HostTaskState::CalleeDone { cancelled: false },
3908                }
3909                Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
3910
3911                store.0.set_thread(old)?;
3912                Ok(())
3913            };
3914
3915            tls::get(move |store| {
3916                if result_may_require_realloc {
3917                    // Lowering may call the guest's realloc function, so run
3918                    // it on a worker fiber. This is required because the guest
3919                    // might try to suspend the fiber which is unsound when it's
3920                    // not actually on a fiber.
3921                    store
3922                        .concurrent_state_mut()?
3923                        .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
3924                            on_complete,
3925                        ))));
3926                    Ok(())
3927                } else {
3928                    // No guest code can be called while lowering this result,
3929                    // so avoid scheduling a worker fiber.
3930                    on_complete(store)
3931                }
3932            })
3933        });
3934
3935        // Make this task visible to the guest and then record what it
3936        // was made visible as.
3937        let caller = match host_task {
3938            Some(caller) => caller,
3939            None => bail_bug!("host task wasn't created but should have been"),
3940        };
3941        let state = store.0.concurrent_state_mut()?;
3942        state.push_future(future);
3943        let instance = state.get_mut(caller.task)?.instance;
3944        let handle = store
3945            .0
3946            .handle_table_insertion(instance)
3947            .subtask_host(task.rep())?;
3948        store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
3949        log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
3950
3951        // Restore the currently running thread to this host task's
3952        // caller. Note that the host task isn't deallocated as it's
3953        // within the store and will get deallocated later.
3954        store.0.set_thread(caller)?;
3955        Ok(Status::Started.pack(Some(handle)))
3956    }
3957
3958    /// Implements the `task.return` intrinsic, lifting the result for the
3959    /// current guest task.
3960    pub(crate) fn task_return(
3961        self,
3962        store: &mut dyn VMStore,
3963        ty: TypeTupleIndex,
3964        options: OptionsIndex,
3965        storage: &[ValRaw],
3966    ) -> Result<()> {
3967        let guest_thread = store.current_guest_thread()?;
3968        let state = store.concurrent_state_mut()?;
3969        if !state.get_mut(guest_thread.task)?.async_lifted {
3970            bail!(Trap::TaskReturnOrCancelSyncLifted);
3971        }
3972        let lift = state
3973            .get_mut(guest_thread.task)?
3974            .lift_result
3975            .take()
3976            .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
3977        if !state.get_mut(guest_thread.task)?.result.is_none() {
3978            bail_bug!("task result unexpectedly already set");
3979        }
3980
3981        let CanonicalOptions {
3982            string_encoding,
3983            data_model,
3984            ..
3985        } = &self.id().get(store).component().env_component().options[options];
3986
3987        let invalid = ty != lift.ty
3988            || string_encoding != &lift.string_encoding
3989            || match data_model {
3990                CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
3991                    Some(memory) => {
3992                        let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
3993                        let actual = self.id().get(store).runtime_memory(memory);
3994                        expected != actual.as_ptr()
3995                    }
3996                    // Memory not specified, meaning it didn't need to be
3997                    // specified per validation, so not invalid.
3998                    None => false,
3999                },
4000                // Always invalid as this isn't supported.
4001                CanonicalOptionsDataModel::Gc { .. } => true,
4002            };
4003
4004        if invalid {
4005            bail!(Trap::TaskReturnInvalid);
4006        }
4007
4008        log::trace!("task.return for {guest_thread:?}");
4009
4010        let result = (lift.lift)(store, storage)?;
4011        self.task_complete(store, guest_thread.task, result, Status::Returned)
4012    }
4013
4014    /// Implements the `task.cancel` intrinsic.
4015    pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4016        let guest_thread = store.current_guest_thread()?;
4017        let state = store.concurrent_state_mut()?;
4018        let task = state.get_mut(guest_thread.task)?;
4019        if !task.async_lifted {
4020            bail!(Trap::TaskReturnOrCancelSyncLifted);
4021        }
4022        if !task.cancel_request_delivered {
4023            bail!(Trap::TaskCancelNotCancelled);
4024        }
4025        _ = task
4026            .lift_result
4027            .take()
4028            .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4029
4030        if !task.result.is_none() {
4031            bail_bug!("task result should not bet set yet");
4032        }
4033
4034        log::trace!("task.cancel for {guest_thread:?}");
4035
4036        self.task_complete(
4037            store,
4038            guest_thread.task,
4039            Box::new(DummyResult),
4040            Status::ReturnCancelled,
4041        )
4042    }
4043
4044    /// Complete the specified guest task (i.e. indicate that it has either
4045    /// returned a (possibly empty) result or cancelled itself).
4046    ///
4047    /// This will return any resource borrows and notify any current or future
4048    /// waiters that the task has completed.
4049    fn task_complete(
4050        self,
4051        store: &mut StoreOpaque,
4052        guest_task: TableId<GuestTask>,
4053        result: Box<dyn Any + Send + Sync>,
4054        status: Status,
4055    ) -> Result<()> {
4056        store
4057            .component_resource_tables(Some(self))?
4058            .validate_scope_exit()?;
4059
4060        let state = store.concurrent_state_mut()?;
4061        let task = state.get_mut(guest_task)?;
4062
4063        // A cancellation request is only delivered to a task which hasn't yet
4064        // resolved, so drop any pending, undelivered `Event::Cancelled` now
4065        // that this task has resolved.
4066        task.event = None;
4067
4068        if let Caller::Host { tx, .. } = &mut task.caller {
4069            if let Some(tx) = tx.take() {
4070                _ = tx.send(result);
4071            }
4072        } else {
4073            task.result = Some(result);
4074            Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4075        }
4076
4077        Ok(())
4078    }
4079
4080    /// Implements the `waitable-set.new` intrinsic.
4081    pub(crate) fn waitable_set_new(
4082        self,
4083        store: &mut StoreOpaque,
4084        caller_instance: RuntimeComponentInstanceIndex,
4085    ) -> Result<u32> {
4086        let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4087        let handle = store
4088            .handle_table_insertion(self.runtime_instance(caller_instance))
4089            .waitable_set(set.rep())?;
4090        log::trace!("new waitable set {set:?} (handle {handle})");
4091        Ok(handle)
4092    }
4093
4094    /// Implements the `waitable-set.drop` intrinsic.
4095    pub(crate) fn waitable_set_drop(
4096        self,
4097        store: &mut StoreOpaque,
4098        caller_instance: RuntimeComponentInstanceIndex,
4099        set: u32,
4100    ) -> Result<()> {
4101        let rep = store
4102            .instance_state(self.runtime_instance(caller_instance))
4103            .handle_table()
4104            .waitable_set_remove(set)?;
4105
4106        log::trace!("drop waitable set {rep} (handle {set})");
4107
4108        // Note that we're careful to check for waiters _before_ deleting the
4109        // set to avoid dropping any waiters in `WaitMode::Fiber(_)`, which
4110        // would panic.  See `drop-waitable-set-with-waiters.wast` for details.
4111        let set = store
4112            .concurrent_state_mut()?
4113            .get_mut(TableId::<WaitableSet>::new(rep))?;
4114        if set.num_waiting > 0 {
4115            bail!(Trap::WaitableSetDropHasWaiters);
4116        }
4117
4118        store
4119            .concurrent_state_mut()?
4120            .delete(TableId::<WaitableSet>::new(rep))?;
4121
4122        Ok(())
4123    }
4124
4125    /// Implements the `waitable.join` intrinsic.
4126    pub(crate) fn waitable_join(
4127        self,
4128        store: &mut StoreOpaque,
4129        caller_instance: RuntimeComponentInstanceIndex,
4130        waitable_handle: u32,
4131        set_handle: u32,
4132    ) -> Result<()> {
4133        let mut instance = self.id().get_mut(store);
4134        let waitable =
4135            Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4136
4137        let set = if set_handle == 0 {
4138            None
4139        } else {
4140            let set = instance.instance_states().0[caller_instance]
4141                .handle_table()
4142                .waitable_set_rep(set_handle)?;
4143
4144            Some(TableId::<WaitableSet>::new(set))
4145        };
4146
4147        waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4148
4149        log::trace!(
4150            "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4151        );
4152
4153        waitable.join(store.concurrent_state_mut()?, set)
4154    }
4155
4156    /// Implements the `subtask.drop` intrinsic.
4157    pub(crate) fn subtask_drop(
4158        self,
4159        store: &mut StoreOpaque,
4160        caller_instance: RuntimeComponentInstanceIndex,
4161        task_id: u32,
4162    ) -> Result<()> {
4163        self.waitable_join(store, caller_instance, task_id, 0)?;
4164
4165        let (rep, is_host) = store
4166            .instance_state(self.runtime_instance(caller_instance))
4167            .handle_table()
4168            .subtask_remove(task_id)?;
4169
4170        let concurrent_state = store.concurrent_state_mut()?;
4171        let waitable = if is_host {
4172            let id = TableId::<HostTask>::new(rep);
4173            let task = concurrent_state.get_mut(id)?;
4174            match &task.state {
4175                HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4176                    bail!(Trap::SubtaskDropNotResolved)
4177                }
4178                HostTaskState::CalleeDone { .. } => {}
4179                HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4180                    bail_bug!("invalid state for callee in `subtask.drop`")
4181                }
4182            }
4183
4184            Waitable::Host(id)
4185        } else {
4186            let id = TableId::<GuestTask>::new(rep);
4187            let task = concurrent_state.get_mut(id)?;
4188            if task.lift_result.is_some() {
4189                bail!(Trap::SubtaskDropNotResolved);
4190            }
4191            Waitable::Guest(id)
4192        };
4193
4194        waitable.common(concurrent_state)?.handle = None;
4195
4196        // If this subtask has an event that means that the terminal status of
4197        // this subtask wasn't yet received so it can't be dropped yet.
4198        if waitable.take_event(concurrent_state)?.is_some() {
4199            bail!(Trap::SubtaskDropNotResolved);
4200        }
4201
4202        waitable.delete_from(concurrent_state)?;
4203
4204        log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4205        Ok(())
4206    }
4207
4208    /// Implements the `waitable-set.wait` intrinsic.
4209    pub(crate) fn waitable_set_wait(
4210        self,
4211        store: &mut StoreOpaque,
4212        options: OptionsIndex,
4213        set: u32,
4214        payload: u32,
4215    ) -> Result<u32> {
4216        let &CanonicalOptions {
4217            instance: caller_instance,
4218            ..
4219        } = &self.id().get(store).component().env_component().options[options];
4220        let caller = self.runtime_instance(caller_instance);
4221        let rep = store
4222            .instance_state(self.runtime_instance(caller_instance))
4223            .handle_table()
4224            .waitable_set_rep(set)?;
4225
4226        self.waitable_check(
4227            store,
4228            caller,
4229            WaitableCheck::Wait,
4230            WaitableCheckParams {
4231                set: TableId::new(rep),
4232                options,
4233                payload,
4234            },
4235        )
4236    }
4237
4238    /// Implements the `waitable-set.poll` intrinsic.
4239    pub(crate) fn waitable_set_poll(
4240        self,
4241        store: &mut StoreOpaque,
4242        options: OptionsIndex,
4243        set: u32,
4244        payload: u32,
4245    ) -> Result<u32> {
4246        let &CanonicalOptions {
4247            instance: caller_instance,
4248            ..
4249        } = &self.id().get(store).component().env_component().options[options];
4250        let caller = self.runtime_instance(caller_instance);
4251        let rep = store
4252            .instance_state(caller)
4253            .handle_table()
4254            .waitable_set_rep(set)?;
4255
4256        self.waitable_check(
4257            store,
4258            caller,
4259            WaitableCheck::Poll,
4260            WaitableCheckParams {
4261                set: TableId::new(rep),
4262                options,
4263                payload,
4264            },
4265        )
4266    }
4267
4268    /// Implements the `thread.index` intrinsic.
4269    pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4270        let thread_id = store.current_guest_thread()?.thread;
4271        match store
4272            .concurrent_state_mut()?
4273            .get_mut(thread_id)?
4274            .instance_rep
4275        {
4276            Some(r) => Ok(r),
4277            None => bail_bug!("thread should have instance_rep by now"),
4278        }
4279    }
4280
4281    /// Implements the `thread.new-indirect` intrinsic.
4282    pub(crate) fn thread_new_indirect<T: 'static>(
4283        self,
4284        mut store: StoreContextMut<T>,
4285        runtime_instance: RuntimeComponentInstanceIndex,
4286        start_func_ty: ModuleInternedTypeIndex,
4287        start_func_table_idx: RuntimeTableIndex,
4288        start_func_idx: u32,
4289        context: i32,
4290    ) -> Result<u32> {
4291        log::trace!("creating new thread");
4292
4293        let (instance, registry) = self.id().get_mut_and_registry(store.0);
4294        let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4295        else {
4296            bail_bug!("thread.new-indirect start function type should be registered");
4297        };
4298        let callee = instance
4299            .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4300            .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4301        if callee.type_index(store.0) != start_func_ty {
4302            bail!(Trap::ThreadNewIndirectInvalidType);
4303        }
4304
4305        let token = StoreToken::new(store.as_context_mut());
4306        let start_func = Box::new(
4307            move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4308                let old_thread = store.set_thread(guest_thread)?;
4309                log::trace!(
4310                    "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4311                );
4312
4313                let mut store = token.as_context_mut(store);
4314                let mut params = [ValRaw::i32(context)];
4315                // Use call_unchecked rather than call or call_async, as we don't want to run the function
4316                // on a separate fiber if we're running in an async store.
4317                unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4318
4319                store.0.set_thread(old_thread)?;
4320
4321                let runtime_instance = self.runtime_instance(runtime_instance);
4322
4323                // We're not returning to any caller, so we may need to switch
4324                // or trap if the instance has an outstanding sync-typed call:
4325                store
4326                    .0
4327                    .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4328
4329                store.0.cleanup_thread(guest_thread, runtime_instance)?;
4330
4331                log::trace!("explicit thread {guest_thread:?} completed");
4332                let state = store.0.concurrent_state_mut()?;
4333                if let Some(t) = old_thread.guest() {
4334                    state.set_thread_running(t.thread)?;
4335                }
4336                log::trace!("thread start: restored {old_thread:?} as current thread");
4337
4338                Ok(())
4339            },
4340        );
4341
4342        let current_thread = store.0.current_guest_thread()?;
4343        let state = store.0.concurrent_state_mut()?;
4344        let parent_task = current_thread.task;
4345
4346        let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4347        let thread_id = state.push(new_thread)?;
4348        state.get_mut(parent_task)?.threads.insert(thread_id);
4349
4350        log::trace!("new thread with id {thread_id:?} created");
4351
4352        self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4353    }
4354
4355    pub(crate) fn resume_thread(
4356        self,
4357        store: &mut StoreOpaque,
4358        runtime_instance: RuntimeComponentInstanceIndex,
4359        thread_idx: u32,
4360        how: ResumeThread,
4361    ) -> Result<bool> {
4362        let thread_id =
4363            GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4364        let state = store.concurrent_state_mut()?;
4365        let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4366
4367        if store.current_guest_thread()? == guest_thread {
4368            bail!(Trap::CannotResumeThread);
4369        }
4370
4371        let priority = match how {
4372            ResumeThread::Resume => Priority::Switch,
4373            ResumeThread::ResumeLater => Priority::Low,
4374
4375            // Promotion is a noop unless the thread is ready, so only search
4376            // through pending work items to see if something matches.
4377            ResumeThread::Promote => {
4378                let instance = self.runtime_instance(runtime_instance);
4379                let do_not_enter = store
4380                    .instance_state(instance)
4381                    .concurrent_state()
4382                    .do_not_enter;
4383                return store.concurrent_state_mut()?.promote_work_item_matching(
4384                    |item: &WorkItem| match item {
4385                        WorkItem::ResumeThread { thread, .. }
4386                        | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4387
4388                        WorkItem::GuestCall {
4389                            call: GuestCall { thread, kind },
4390                            ..
4391                        } => {
4392                            *thread == guest_thread
4393                                && match kind {
4394                                    // Events can only be delivered if this
4395                                    // instance's exclusive lock isn't held, so
4396                                    // gate on that.
4397                                    GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4398
4399                                    // A not-yet-started thread made ready via
4400                                    // `thread.resume-later`.
4401                                    GuestCallKind::StartExplicit(_) => true,
4402
4403                                    // Technically shouldn't be possible since
4404                                    // if this hasn't started nothing could get
4405                                    // its id...
4406                                    GuestCallKind::StartImplicit(_) => false,
4407                                }
4408                        }
4409
4410                        WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4411                    },
4412                );
4413            }
4414        };
4415
4416        let state = store.concurrent_state_mut()?;
4417        let thread = state.get_mut(guest_thread.thread)?;
4418
4419        // When resuming a thread it must be in a suspended state otherwise
4420        // this operation is a trap.
4421        match &thread.state {
4422            GuestThreadState::NotStartedExplicit(_) => {}
4423            GuestThreadState::Suspended(_) => {
4424                log::trace!("resuming thread {thread_id:?} that was suspended");
4425                state.schedule_suspended_thread(
4426                    guest_thread,
4427                    self.runtime_instance(runtime_instance),
4428                    priority,
4429                )?;
4430                return Ok(true);
4431            }
4432            _ => bail!(Trap::CannotResumeThread),
4433        }
4434        match mem::replace(&mut thread.state, GuestThreadState::Running) {
4435            GuestThreadState::NotStartedExplicit(start_func) => {
4436                log::trace!("starting thread {guest_thread:?}");
4437                let guest_call = WorkItem::GuestCall {
4438                    instance: self.runtime_instance(runtime_instance),
4439                    call: GuestCall {
4440                        thread: guest_thread,
4441                        kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4442                            start_func(store, guest_thread)
4443                        })),
4444                    },
4445                };
4446                state.push_work_item(guest_call, priority)?;
4447            }
4448            other => {
4449                thread.state = other;
4450                bail_bug!("thread state checked to be resumable above");
4451            }
4452        }
4453        Ok(true)
4454    }
4455
4456    fn add_guest_thread_to_instance_table(
4457        self,
4458        thread_id: TableId<GuestThread>,
4459        store: &mut StoreOpaque,
4460        runtime_instance: RuntimeComponentInstanceIndex,
4461    ) -> Result<u32> {
4462        let (state, counter) =
4463            store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4464        let guest_id = state
4465            .thread_handle_table()
4466            .guest_thread_insert(counter, thread_id.rep())?;
4467        store
4468            .concurrent_state_mut()?
4469            .get_mut(thread_id)?
4470            .instance_rep = Some(guest_id);
4471        Ok(guest_id)
4472    }
4473
4474    /// Helper function for the `thread.yield`, thread.suspend`,
4475    /// `thread.suspend-then-resume`, `thread.suspend-then-promote`,
4476    /// `thread.yield-then-resume`, and `thread.yield-then-promote` intrinsics.
4477    pub(crate) fn suspension_intrinsic(
4478        self,
4479        store: &mut StoreOpaque,
4480        caller: RuntimeComponentInstanceIndex,
4481        yielding: bool,
4482        to_thread: SuspensionTarget,
4483    ) -> Result<WaitResult> {
4484        let check_suspend = match to_thread {
4485            SuspensionTarget::Promote(thread) => {
4486                !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4487            }
4488            SuspensionTarget::Resume(thread) => {
4489                if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4490                    bail_bug!(
4491                        "`resume_thread` should only ever return false \
4492                         when `ResumeThread::Promote` is passed to it"
4493                    );
4494                }
4495                false
4496            }
4497            SuspensionTarget::None => true,
4498        };
4499
4500        if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4501            return if yielding {
4502                Ok(WaitResult::Completed)
4503            } else {
4504                Err(Trap::CannotBlockSyncTask.into())
4505            };
4506        }
4507
4508        let guest_thread = store.current_guest_thread()?;
4509
4510        let reason = if yielding {
4511            SuspendReason::Yielding {
4512                thread: guest_thread,
4513            }
4514        } else {
4515            SuspendReason::ExplicitlySuspending {
4516                thread: guest_thread,
4517            }
4518        };
4519
4520        store.suspend(reason)?;
4521
4522        Ok(WaitResult::Completed)
4523    }
4524
4525    /// Helper function for the `waitable-set.wait` and `waitable-set.poll` intrinsics.
4526    fn waitable_check(
4527        self,
4528        store: &mut StoreOpaque,
4529        caller: RuntimeInstance,
4530        check: WaitableCheck,
4531        params: WaitableCheckParams,
4532    ) -> Result<u32> {
4533        let guest_thread = store.current_guest_thread()?;
4534
4535        log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4536
4537        // If we're waiting, and there are no events immediately available,
4538        // suspend the fiber until that changes.
4539        match &check {
4540            WaitableCheck::Wait => {
4541                let set = params.set;
4542
4543                // Note that this is a loop since, by the time this fiber is
4544                // resumed after being woken up, another thread may have already
4545                // taken the event (e.g. via `waitable-set.poll`), in which case
4546                // we keep waiting.
4547                loop {
4548                    let state = store.concurrent_state_mut()?;
4549                    let task = state.get_mut(guest_thread.task)?;
4550                    if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4551                        || !state.get_mut(set)?.ready.is_empty()
4552                    {
4553                        break;
4554                    }
4555
4556                    store.switch_or_trap_if_may_not_suspend(caller)?;
4557
4558                    store.suspend(SuspendReason::Waiting {
4559                        set,
4560                        thread: guest_thread,
4561                    })?;
4562                }
4563            }
4564            WaitableCheck::Poll => {}
4565        }
4566
4567        log::trace!(
4568            "waitable check for {guest_thread:?}; set {:?}, part two",
4569            params.set
4570        );
4571
4572        // Deliver any pending events to the guest and return.
4573        let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4574
4575        let (ordinal, handle, result) = match &check {
4576            WaitableCheck::Wait => {
4577                let (event, waitable) = match event {
4578                    Some(p) => p,
4579                    None => bail_bug!("event expected to be present"),
4580                };
4581                let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4582                let (ordinal, result) = event.parts();
4583                (ordinal, handle, result)
4584            }
4585            WaitableCheck::Poll => {
4586                if let Some((event, waitable)) = event {
4587                    let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4588                    let (ordinal, result) = event.parts();
4589                    (ordinal, handle, result)
4590                } else {
4591                    log::trace!(
4592                        "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4593                        guest_thread.task,
4594                        params.set
4595                    );
4596                    let (ordinal, result) = Event::None.parts();
4597                    (ordinal, 0, result)
4598                }
4599            }
4600        };
4601        let ptr = crate::component::func::validate_inbounds_dynamic(
4602            &CanonicalAbiInfo::POINTER_PAIR,
4603            self.options_memory(store, params.options),
4604            &ValRaw::u32(params.payload),
4605        )?;
4606        let memory = self
4607            .options_memory_range_mut(store, params.options, ptr, 8)
4608            .unwrap();
4609        memory[0..4].copy_from_slice(&handle.to_le_bytes());
4610        memory[4..8].copy_from_slice(&result.to_le_bytes());
4611        Ok(ordinal)
4612    }
4613
4614    /// Implements the `subtask.cancel` intrinsic.
4615    pub(crate) fn subtask_cancel(
4616        self,
4617        store: &mut StoreOpaque,
4618        caller_instance: RuntimeComponentInstanceIndex,
4619        async_: bool,
4620        task_id: u32,
4621    ) -> Result<u32> {
4622        let (rep, is_host) = store
4623            .instance_state(self.runtime_instance(caller_instance))
4624            .handle_table()
4625            .subtask_rep(task_id)?;
4626        let waitable = if is_host {
4627            Waitable::Host(TableId::<HostTask>::new(rep))
4628        } else {
4629            Waitable::Guest(TableId::<GuestTask>::new(rep))
4630        };
4631        let concurrent_state = store.concurrent_state_mut()?;
4632
4633        log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4634
4635        waitable.trap_if_in_waitable_set(concurrent_state)?;
4636
4637        let needs_block;
4638        if let Waitable::Host(host_task) = waitable {
4639            let state = &mut concurrent_state.get_mut(host_task)?.state;
4640            match state {
4641                // If the callee is still running, signal an abort is requested.
4642                //
4643                // After cancelling this falls through to block waiting for the
4644                // host task to actually finish assuming that `async_` is false.
4645                // This blocking behavior resolves the race of `handle.abort()`
4646                // with the task actually getting cancelled or finishing.
4647                HostTaskState::CalleeRunning(handle) => {
4648                    handle.abort();
4649                    *state = HostTaskState::CalleeCancelling;
4650                    needs_block = true;
4651                }
4652
4653                // Cancellation was already requested, so fail as the task can't
4654                // be cancelled twice.
4655                HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4656                    bail!(Trap::SubtaskCancelAfterTerminal);
4657                }
4658                HostTaskState::CalleeDone { cancelled: false } => {
4659                    // The callee is already done so there's no need to
4660                    // block further for an event.
4661                    *state = HostTaskState::CalleeDone { cancelled: true };
4662                    needs_block = false;
4663                }
4664
4665                // These states should not be possible for a subtask that's
4666                // visible from the guest, so trap here.
4667                HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4668                    bail_bug!("invalid states for host callee")
4669                }
4670            }
4671        } else {
4672            let guest_task = TableId::<GuestTask>::new(rep);
4673            let task = concurrent_state.get_mut(guest_task)?;
4674
4675            // Cancellation was already requested, so fail as the task can't be
4676            // cancelled twice.
4677            if task.cancel_requested {
4678                bail!(Trap::SubtaskCancelAfterTerminal);
4679            }
4680
4681            if !task.already_lowered_parameters() {
4682                store.cancel_guest_subtask_without_lowered_parameters(
4683                    self.runtime_instance(caller_instance),
4684                    guest_task,
4685                )?;
4686                return Ok(Status::StartCancelled as u32);
4687            } else if !task.returned_or_cancelled() {
4688                // Started, but not yet returned or cancelled; send the
4689                // `CANCELLED` event
4690                //
4691                // Note that this might overwrite an event that was set earlier
4692                // (e.g. `Event::None` if the task is yielding), but that's
4693                // okay -- this should supersede the previous state.
4694                task.cancel_requested = true;
4695                task.event = Some(Event::Cancelled);
4696                let runtime_instance = task.instance;
4697                for thread in task.threads.clone() {
4698                    let thread = QualifiedThreadId {
4699                        task: guest_task,
4700                        thread,
4701                    };
4702                    let concurrent_state = store.concurrent_state_mut()?;
4703                    let thread_mut = concurrent_state.get_mut(thread.thread)?;
4704
4705                    let yield_ = |store: &mut StoreOpaque| {
4706                        // While we're yielding, temporarily set
4707                        // `do_not_suspend` to false on the subtask's instance
4708                        // since we'll be getting control back if it does
4709                        // suspend.
4710                        let state = store.instance_state(runtime_instance).concurrent_state();
4711                        let old_do_not_suspend = state.do_not_suspend;
4712                        state.do_not_suspend = false;
4713
4714                        let caller = store.current_guest_thread()?;
4715
4716                        // Temporarily add the waitable to the caller's
4717                        // `sync_call_set` to ensure that (1) it isn't already
4718                        // part of a different set and (2) it can't be added to
4719                        // a different set while we yield to the subtask.
4720                        let state = store.concurrent_state_mut()?;
4721                        let set = state.get_mut(caller.thread)?.sync_call_set;
4722                        waitable.join(state, Some(set))?;
4723
4724                        store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4725
4726                        let state = store.concurrent_state_mut()?;
4727                        waitable.join(state, None)?;
4728
4729                        store
4730                            .instance_state(runtime_instance)
4731                            .concurrent_state()
4732                            .do_not_suspend = old_do_not_suspend;
4733
4734                        Ok::<(), crate::Error>(())
4735                    };
4736
4737                    match thread_mut.wake_on_cancel.take() {
4738                        WakeOnCancel::Waiting(set) => {
4739                            // The thread is in a cancellable wait, so wake it up:
4740                            let instance = concurrent_state.take_callback_waiter(set, thread)?;
4741                            let item = WorkItem::GuestCall {
4742                                instance: runtime_instance,
4743                                call: GuestCall {
4744                                    thread,
4745                                    kind: GuestCallKind::DeliverEvent {
4746                                        instance,
4747                                        set: Some(set),
4748                                    },
4749                                },
4750                            };
4751                            concurrent_state.set_switch_item(item)?;
4752
4753                            yield_(store)?;
4754
4755                            break;
4756                        }
4757                        WakeOnCancel::Yielding => {
4758                            if concurrent_state.promote_thread_work_item(thread)? {
4759                                yield_(store)?;
4760                                break;
4761                            } else if store
4762                                .instance_state(runtime_instance)
4763                                .concurrent_state()
4764                                .pending
4765                                .contains_key(&thread)
4766                            {
4767                                // The thread's work item was parked in
4768                                // `pending` because its instance can't be
4769                                // entered right now, so the cancellation
4770                                // can't be delivered yet. It'll be delivered
4771                                // via `GuestTask::event` once the instance
4772                                // becomes enterable and the work item runs.
4773                                // Until then the thread is still yielding, so
4774                                // restore `wake_on_cancel` and move on to the
4775                                // next thread.
4776                                store
4777                                    .concurrent_state_mut()?
4778                                    .get_mut(thread.thread)?
4779                                    .wake_on_cancel = WakeOnCancel::Yielding;
4780                            } else {
4781                                bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4782                            }
4783                        }
4784                        WakeOnCancel::None => {}
4785                    }
4786                }
4787
4788                // Guest tasks need to block if they have not yet returned or
4789                // cancelled, even as a result of the event delivery above.
4790                needs_block = !store
4791                    .concurrent_state_mut()?
4792                    .get_mut(guest_task)?
4793                    .returned_or_cancelled()
4794            } else {
4795                needs_block = false;
4796            }
4797        };
4798
4799        // If we need to block waiting on the terminal status of this subtask
4800        // then return immediately in `async` mode, or otherwise wait for the
4801        // event to get signaled through the store.
4802        if needs_block {
4803            if async_ {
4804                return Ok(BLOCKED);
4805            }
4806
4807            // For cancelling host tasks save and later restore
4808            // `next_switch_item` so that our caller (or canceller) isn't
4809            // resumed in the meantime. This is our prerogative as a runtime
4810            // where we get to decide whether cancellation of host things blocks
4811            // or doesn't, and here we just need to get the final status from
4812            // the host side so it shouldn't block.
4813            //
4814            // A guest subtask, however, may genuinely block, so skip this in
4815            // that case.
4816            let old_next_switch_item = if is_host {
4817                Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4818            } else {
4819                None
4820            };
4821
4822            // Wait for this waitable to get signaled with its terminal
4823            // status. Once that's done fall through to the shared code.
4824            store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4825
4826            if let Some(item) = old_next_switch_item {
4827                let state = store.concurrent_state_mut()?;
4828                state.restore_next_switch_item(item)?;
4829            }
4830
4831            // .. fall through to determine what event's in store for us.
4832        }
4833
4834        let event = waitable.take_event(store.concurrent_state_mut()?)?;
4835        if let Some(Event::Subtask {
4836            status: status @ (Status::Returned | Status::ReturnCancelled),
4837        }) = event
4838        {
4839            Ok(status as u32)
4840        } else {
4841            bail!(Trap::SubtaskCancelAfterTerminal);
4842        }
4843    }
4844}
4845
4846/// Trait representing component model ABI async intrinsics and fused adapter
4847/// helper functions.
4848///
4849/// SAFETY (callers): Most of the methods in this trait accept raw pointers,
4850/// which must be valid for at least the duration of the call (and possibly for
4851/// as long as the relevant guest task exists, in the case of `*mut VMFuncRef`
4852/// pointers used for async calls).
4853pub trait VMComponentAsyncStore {
4854    /// A helper function for fused adapter modules involving calls where the
4855    /// one of the caller or callee is async.
4856    ///
4857    /// This helper is not used when the caller and callee both use the sync
4858    /// ABI, only when at least one is async is this used.
4859    unsafe fn prepare_call(
4860        &mut self,
4861        instance: Instance,
4862        memory: *mut VMMemoryDefinition,
4863        start: NonNull<VMFuncRef>,
4864        return_: NonNull<VMFuncRef>,
4865        caller_instance: RuntimeComponentInstanceIndex,
4866        callee_instance: RuntimeComponentInstanceIndex,
4867        task_return_type: TypeTupleIndex,
4868        callee_async: bool,
4869        string_encoding: StringEncoding,
4870        result_count: u32,
4871        storage: *mut ValRaw,
4872        storage_len: usize,
4873    ) -> Result<()>;
4874
4875    /// A helper function for fused adapter modules to start a call previously
4876    /// set up with `prepare_call`.
4877    unsafe fn start_call(
4878        &mut self,
4879        instance: Instance,
4880        callback: *mut VMFuncRef,
4881        post_return: *mut VMFuncRef,
4882        callee: NonNull<VMFuncRef>,
4883        param_count: u32,
4884        result_count: u32,
4885        flags: u32,
4886        storage: *mut MaybeUninit<ValRaw>,
4887        storage_len: usize,
4888    ) -> Result<()>;
4889
4890    /// The `future.write` intrinsic.
4891    fn future_write(
4892        &mut self,
4893        instance: Instance,
4894        caller: RuntimeComponentInstanceIndex,
4895        ty: TypeFutureTableIndex,
4896        options: OptionsIndex,
4897        future: u32,
4898        address: u32,
4899    ) -> Result<u32>;
4900
4901    /// The `future.read` intrinsic.
4902    fn future_read(
4903        &mut self,
4904        instance: Instance,
4905        caller: RuntimeComponentInstanceIndex,
4906        ty: TypeFutureTableIndex,
4907        options: OptionsIndex,
4908        future: u32,
4909        address: u32,
4910    ) -> Result<u32>;
4911
4912    /// The `future.drop-writable` intrinsic.
4913    fn future_drop_writable(
4914        &mut self,
4915        instance: Instance,
4916        ty: TypeFutureTableIndex,
4917        writer: u32,
4918    ) -> Result<()>;
4919
4920    /// The `stream.write` intrinsic.
4921    fn stream_write(
4922        &mut self,
4923        instance: Instance,
4924        caller: RuntimeComponentInstanceIndex,
4925        ty: TypeStreamTableIndex,
4926        options: OptionsIndex,
4927        stream: u32,
4928        address: u32,
4929        count: u32,
4930    ) -> Result<u32>;
4931
4932    /// The `stream.read` intrinsic.
4933    fn stream_read(
4934        &mut self,
4935        instance: Instance,
4936        caller: RuntimeComponentInstanceIndex,
4937        ty: TypeStreamTableIndex,
4938        options: OptionsIndex,
4939        stream: u32,
4940        address: u32,
4941        count: u32,
4942    ) -> Result<u32>;
4943
4944    /// The "fast-path" implementation of the `stream.write` intrinsic for
4945    /// "flat" (i.e. memcpy-able) payloads.
4946    fn flat_stream_write(
4947        &mut self,
4948        instance: Instance,
4949        caller: RuntimeComponentInstanceIndex,
4950        ty: TypeStreamTableIndex,
4951        options: OptionsIndex,
4952        payload_size: u32,
4953        payload_align: u32,
4954        stream: u32,
4955        address: u32,
4956        count: u32,
4957    ) -> Result<u32>;
4958
4959    /// The "fast-path" implementation of the `stream.read` intrinsic for "flat"
4960    /// (i.e. memcpy-able) payloads.
4961    fn flat_stream_read(
4962        &mut self,
4963        instance: Instance,
4964        caller: RuntimeComponentInstanceIndex,
4965        ty: TypeStreamTableIndex,
4966        options: OptionsIndex,
4967        payload_size: u32,
4968        payload_align: u32,
4969        stream: u32,
4970        address: u32,
4971        count: u32,
4972    ) -> Result<u32>;
4973
4974    /// The `stream.drop-writable` intrinsic.
4975    fn stream_drop_writable(
4976        &mut self,
4977        instance: Instance,
4978        ty: TypeStreamTableIndex,
4979        writer: u32,
4980    ) -> Result<()>;
4981
4982    /// The `error-context.debug-message` intrinsic.
4983    fn error_context_debug_message(
4984        &mut self,
4985        instance: Instance,
4986        ty: TypeComponentLocalErrorContextTableIndex,
4987        options: OptionsIndex,
4988        err_ctx_handle: u32,
4989        debug_msg_address: u32,
4990    ) -> Result<()>;
4991
4992    /// The `thread.new-indirect` intrinsic
4993    fn thread_new_indirect(
4994        &mut self,
4995        instance: Instance,
4996        caller: RuntimeComponentInstanceIndex,
4997        func_ty_idx: ModuleInternedTypeIndex,
4998        start_func_table_idx: RuntimeTableIndex,
4999        start_func_idx: u32,
5000        context: i32,
5001    ) -> Result<u32>;
5002}
5003
5004/// SAFETY: See trait docs.
5005impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5006    unsafe fn prepare_call(
5007        &mut self,
5008        instance: Instance,
5009        memory: *mut VMMemoryDefinition,
5010        start: NonNull<VMFuncRef>,
5011        return_: NonNull<VMFuncRef>,
5012        caller_instance: RuntimeComponentInstanceIndex,
5013        callee_instance: RuntimeComponentInstanceIndex,
5014        task_return_type: TypeTupleIndex,
5015        callee_async: bool,
5016        string_encoding: StringEncoding,
5017        result_count_or_max_if_async: u32,
5018        storage: *mut ValRaw,
5019        storage_len: usize,
5020    ) -> Result<()> {
5021        // SAFETY: The `wasmtime_cranelift`-generated code that calls
5022        // this method will have ensured that `storage` is a valid
5023        // pointer containing at least `storage_len` items.
5024        let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5025
5026        unsafe {
5027            instance.prepare_call(
5028                StoreContextMut(self),
5029                start,
5030                return_,
5031                caller_instance,
5032                callee_instance,
5033                task_return_type,
5034                callee_async,
5035                memory,
5036                string_encoding,
5037                match result_count_or_max_if_async {
5038                    PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5039                        params,
5040                        has_result: false,
5041                    },
5042                    PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5043                        params,
5044                        has_result: true,
5045                    },
5046                    result_count => CallerInfo::Sync {
5047                        params,
5048                        result_count,
5049                    },
5050                },
5051            )
5052        }
5053    }
5054
5055    unsafe fn start_call(
5056        &mut self,
5057        instance: Instance,
5058        callback: *mut VMFuncRef,
5059        post_return: *mut VMFuncRef,
5060        callee: NonNull<VMFuncRef>,
5061        param_count: u32,
5062        result_count: u32,
5063        flags: u32,
5064        storage: *mut MaybeUninit<ValRaw>,
5065        storage_len: usize,
5066    ) -> Result<()> {
5067        unsafe {
5068            instance.start_call(
5069                StoreContextMut(self),
5070                callback,
5071                post_return,
5072                callee,
5073                param_count,
5074                result_count,
5075                flags,
5076                // SAFETY: The `wasmtime_cranelift`-generated code that calls
5077                // this method will have ensured that `storage` is a valid
5078                // pointer containing at least `storage_len` items.
5079                core::slice::from_raw_parts_mut(storage, storage_len),
5080            )
5081        }
5082    }
5083
5084    fn future_write(
5085        &mut self,
5086        instance: Instance,
5087        caller: RuntimeComponentInstanceIndex,
5088        ty: TypeFutureTableIndex,
5089        options: OptionsIndex,
5090        future: u32,
5091        address: u32,
5092    ) -> Result<u32> {
5093        instance
5094            .guest_write(
5095                StoreContextMut(self),
5096                caller,
5097                TransmitIndex::Future(ty),
5098                options,
5099                None,
5100                future,
5101                address,
5102                1,
5103            )
5104            .map(|result| result.encode())
5105    }
5106
5107    fn future_read(
5108        &mut self,
5109        instance: Instance,
5110        caller: RuntimeComponentInstanceIndex,
5111        ty: TypeFutureTableIndex,
5112        options: OptionsIndex,
5113        future: u32,
5114        address: u32,
5115    ) -> Result<u32> {
5116        instance
5117            .guest_read(
5118                StoreContextMut(self),
5119                caller,
5120                TransmitIndex::Future(ty),
5121                options,
5122                None,
5123                future,
5124                address,
5125                1,
5126            )
5127            .map(|result| result.encode())
5128    }
5129
5130    fn stream_write(
5131        &mut self,
5132        instance: Instance,
5133        caller: RuntimeComponentInstanceIndex,
5134        ty: TypeStreamTableIndex,
5135        options: OptionsIndex,
5136        stream: u32,
5137        address: u32,
5138        count: u32,
5139    ) -> Result<u32> {
5140        instance
5141            .guest_write(
5142                StoreContextMut(self),
5143                caller,
5144                TransmitIndex::Stream(ty),
5145                options,
5146                None,
5147                stream,
5148                address,
5149                count,
5150            )
5151            .map(|result| result.encode())
5152    }
5153
5154    fn stream_read(
5155        &mut self,
5156        instance: Instance,
5157        caller: RuntimeComponentInstanceIndex,
5158        ty: TypeStreamTableIndex,
5159        options: OptionsIndex,
5160        stream: u32,
5161        address: u32,
5162        count: u32,
5163    ) -> Result<u32> {
5164        instance
5165            .guest_read(
5166                StoreContextMut(self),
5167                caller,
5168                TransmitIndex::Stream(ty),
5169                options,
5170                None,
5171                stream,
5172                address,
5173                count,
5174            )
5175            .map(|result| result.encode())
5176    }
5177
5178    fn future_drop_writable(
5179        &mut self,
5180        instance: Instance,
5181        ty: TypeFutureTableIndex,
5182        writer: u32,
5183    ) -> Result<()> {
5184        instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5185    }
5186
5187    fn flat_stream_write(
5188        &mut self,
5189        instance: Instance,
5190        caller: RuntimeComponentInstanceIndex,
5191        ty: TypeStreamTableIndex,
5192        options: OptionsIndex,
5193        payload_size: u32,
5194        payload_align: u32,
5195        stream: u32,
5196        address: u32,
5197        count: u32,
5198    ) -> Result<u32> {
5199        instance
5200            .guest_write(
5201                StoreContextMut(self),
5202                caller,
5203                TransmitIndex::Stream(ty),
5204                options,
5205                Some(FlatAbi {
5206                    size: payload_size,
5207                    align: payload_align,
5208                }),
5209                stream,
5210                address,
5211                count,
5212            )
5213            .map(|result| result.encode())
5214    }
5215
5216    fn flat_stream_read(
5217        &mut self,
5218        instance: Instance,
5219        caller: RuntimeComponentInstanceIndex,
5220        ty: TypeStreamTableIndex,
5221        options: OptionsIndex,
5222        payload_size: u32,
5223        payload_align: u32,
5224        stream: u32,
5225        address: u32,
5226        count: u32,
5227    ) -> Result<u32> {
5228        instance
5229            .guest_read(
5230                StoreContextMut(self),
5231                caller,
5232                TransmitIndex::Stream(ty),
5233                options,
5234                Some(FlatAbi {
5235                    size: payload_size,
5236                    align: payload_align,
5237                }),
5238                stream,
5239                address,
5240                count,
5241            )
5242            .map(|result| result.encode())
5243    }
5244
5245    fn stream_drop_writable(
5246        &mut self,
5247        instance: Instance,
5248        ty: TypeStreamTableIndex,
5249        writer: u32,
5250    ) -> Result<()> {
5251        instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5252    }
5253
5254    fn error_context_debug_message(
5255        &mut self,
5256        instance: Instance,
5257        ty: TypeComponentLocalErrorContextTableIndex,
5258        options: OptionsIndex,
5259        err_ctx_handle: u32,
5260        debug_msg_address: u32,
5261    ) -> Result<()> {
5262        instance.error_context_debug_message(
5263            StoreContextMut(self),
5264            ty,
5265            options,
5266            err_ctx_handle,
5267            debug_msg_address,
5268        )
5269    }
5270
5271    fn thread_new_indirect(
5272        &mut self,
5273        instance: Instance,
5274        caller: RuntimeComponentInstanceIndex,
5275        func_ty_idx: ModuleInternedTypeIndex,
5276        start_func_table_idx: RuntimeTableIndex,
5277        start_func_idx: u32,
5278        context: i32,
5279    ) -> Result<u32> {
5280        instance.thread_new_indirect(
5281            StoreContextMut(self),
5282            caller,
5283            func_ty_idx,
5284            start_func_table_idx,
5285            start_func_idx,
5286            context,
5287        )
5288    }
5289}
5290
5291type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5292
5293/// Runs the given future with the current thread set to `task` each time it is
5294/// polled.
5295async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5296where
5297    F: Future,
5298{
5299    let mut future = pin!(future);
5300    future::poll_fn(|cx| {
5301        let old_thread = match tls::get(|store| store.set_thread(task)) {
5302            Ok(thread) => thread,
5303            Err(error) => return Poll::Ready(Err(error)),
5304        };
5305        let result = future.as_mut().poll(cx);
5306        match tls::get(|store| store.set_thread(old_thread)) {
5307            Ok(_) => result.map(Ok),
5308            Err(error) => Poll::Ready(Err(error)),
5309        }
5310    })
5311    .await
5312}
5313
5314/// Represents the state of a pending host task.
5315///
5316/// This is used to represent tasks when the guest calls into the host.
5317pub(crate) struct HostTask {
5318    common: WaitableCommon,
5319
5320    /// State of borrows/etc the host needs to track. Used when the guest passes
5321    /// borrows to the host, for example.
5322    call_context: CallContext,
5323
5324    state: HostTaskState,
5325
5326    group: TaskGroupId,
5327}
5328
5329enum HostTaskState {
5330    /// A host task has been created and it's considered "started".
5331    ///
5332    /// The host task has yet to enter `first_poll` or `poll_and_block` which
5333    /// is where this will get updated further.
5334    CalleeStarted,
5335
5336    /// State used for tasks in `first_poll` meaning that the guest did an async
5337    /// lower of a host async function which is blocked. The specified handle is
5338    /// linked to the future in the main `FuturesUnordered` of a store which is
5339    /// used to cancel it if the guest requests cancellation.
5340    CalleeRunning(JoinHandle),
5341
5342    /// Cancellation was requested, but the completion worker has not finished
5343    /// lowering the result and publishing the terminal event. The task must
5344    /// remain in the table until that worker completes.
5345    CalleeCancelling,
5346
5347    /// Terminal state used for tasks in `poll_and_block` to store the result of
5348    /// their computation. Note that this state is not used for tasks in
5349    /// `first_poll`.
5350    CalleeFinished(LiftedResult),
5351
5352    /// Terminal state for host tasks meaning that the task was cancelled or the
5353    /// result was taken.
5354    CalleeDone { cancelled: bool },
5355}
5356
5357impl HostTask {
5358    fn new(
5359        concurrent_state: &mut ConcurrentState,
5360        state: HostTaskState,
5361        caller: QualifiedThreadId,
5362    ) -> Result<Self> {
5363        let group = concurrent_state.get_mut(caller.task)?.group;
5364        concurrent_state.increment_group_ref_count(group)?;
5365
5366        Ok(Self {
5367            common: WaitableCommon::default(),
5368            call_context: CallContext::default(),
5369            state,
5370            group,
5371        })
5372    }
5373}
5374
5375impl TableDebug for HostTask {
5376    fn type_name() -> &'static str {
5377        "HostTask"
5378    }
5379}
5380
5381type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5382
5383/// Represents the caller of a given guest task.
5384enum Caller {
5385    /// The host called the guest task.
5386    Host {
5387        /// If present, may be used to deliver the result.
5388        tx: Option<oneshot::Sender<LiftedResult>>,
5389        /// The host task which called into the guest, or `None` for a call from
5390        /// the top-level host. The task may belong to an entirely unrelated
5391        /// top-level component instance than the one the host called into.
5392        caller: Option<TableId<HostTask>>,
5393    },
5394    /// Another guest thread called the guest task
5395    Guest {
5396        /// The id of the caller
5397        thread: QualifiedThreadId,
5398    },
5399}
5400
5401/// Represents a closure and related canonical ABI parameters required to
5402/// validate a `task.return` call at runtime and lift the result.
5403struct LiftResult {
5404    lift: RawLift,
5405    ty: TypeTupleIndex,
5406    memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5407    string_encoding: StringEncoding,
5408}
5409
5410/// The table ID for a guest thread, qualified by the task to which it belongs.
5411///
5412/// This exists to minimize table lookups and the necessity to pass stores around mutably
5413/// for the common case of identifying the task to which a thread belongs.
5414#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5415pub(crate) struct QualifiedThreadId {
5416    task: TableId<GuestTask>,
5417    thread: TableId<GuestThread>,
5418}
5419
5420impl QualifiedThreadId {
5421    fn qualify(
5422        state: &mut ConcurrentState,
5423        thread: TableId<GuestThread>,
5424    ) -> Result<QualifiedThreadId> {
5425        Ok(QualifiedThreadId {
5426            task: state.get_mut(thread)?.parent_task,
5427            thread,
5428        })
5429    }
5430}
5431
5432impl fmt::Debug for QualifiedThreadId {
5433    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5434        f.debug_tuple("QualifiedThreadId")
5435            .field(&self.task.rep())
5436            .field(&self.thread.rep())
5437            .finish()
5438    }
5439}
5440
5441enum GuestThreadState {
5442    NotStartedImplicit,
5443    NotStartedExplicit(
5444        Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5445    ),
5446    Running,
5447    Suspended(StoreFiber<'static>),
5448    Ready {
5449        fiber: StoreFiber<'static>,
5450    },
5451    Completed,
5452}
5453
5454impl GuestThreadState {
5455    fn check_no_fiber(&self) -> Result<()> {
5456        match self {
5457            Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5458            Self::NotStartedImplicit
5459            | Self::NotStartedExplicit(_)
5460            | Self::Running
5461            | Self::Completed => Ok(()),
5462        }
5463    }
5464}
5465
5466impl fmt::Debug for GuestThreadState {
5467    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5468        match self {
5469            Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5470            Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5471            Self::Running => f.debug_tuple("Running").finish(),
5472            Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5473            Self::Ready { .. } => f.debug_struct("Ready").finish(),
5474            Self::Completed => f.debug_tuple("Completed").finish(),
5475        }
5476    }
5477}
5478
5479#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5480enum WakeOnCancel {
5481    None,
5482    Waiting(TableId<WaitableSet>),
5483    Yielding,
5484}
5485
5486impl WakeOnCancel {
5487    fn is_none(self) -> bool {
5488        matches!(self, WakeOnCancel::None)
5489    }
5490
5491    fn replace(&mut self, other: WakeOnCancel) -> Self {
5492        let old = *self;
5493        *self = other;
5494        old
5495    }
5496
5497    fn take(&mut self) -> Self {
5498        self.replace(WakeOnCancel::None)
5499    }
5500}
5501
5502pub struct GuestThread {
5503    /// Context-local state used to implement the `context.{get,set}`
5504    /// intrinsics.
5505    context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5506    /// The owning guest task.
5507    parent_task: TableId<GuestTask>,
5508    /// If non-`None`, indicates that the thread is currently either waiting a
5509    /// waitable set or yielding but may be cancelled and woken immediately.
5510    wake_on_cancel: WakeOnCancel,
5511    /// The execution state of this guest thread
5512    state: GuestThreadState,
5513    /// The index of this thread in the component instance's handle table.
5514    /// This must always be `Some` after initialization.
5515    instance_rep: Option<u32>,
5516    /// Scratch waitable set used to watch subtasks during synchronous calls.
5517    sync_call_set: TableId<WaitableSet>,
5518    /// The old value of `do_not_suspend` prior to the sync-typed task for which
5519    /// this thread was created, if relevant.
5520    old_do_not_suspend: Option<bool>,
5521}
5522
5523impl GuestThread {
5524    /// Retrieve the `GuestThread` corresponding to the specified guest-visible
5525    /// handle.
5526    fn from_instance(
5527        state: Pin<&mut ComponentInstance>,
5528        caller_instance: RuntimeComponentInstanceIndex,
5529        guest_thread: u32,
5530    ) -> Result<TableId<Self>> {
5531        let rep = state.instance_states().0[caller_instance]
5532            .thread_handle_table()
5533            .guest_thread_rep(guest_thread)?;
5534        Ok(TableId::new(rep))
5535    }
5536
5537    fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5538        let sync_call_set = state.push(WaitableSet {
5539            is_sync_call_set: true,
5540            ..WaitableSet::default()
5541        })?;
5542        Ok(Self {
5543            context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5544            parent_task,
5545            wake_on_cancel: WakeOnCancel::None,
5546            state: GuestThreadState::NotStartedImplicit,
5547            instance_rep: None,
5548            sync_call_set,
5549            old_do_not_suspend: None,
5550        })
5551    }
5552
5553    fn new_explicit(
5554        state: &mut ConcurrentState,
5555        parent_task: TableId<GuestTask>,
5556        start_func: Box<
5557            dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5558        >,
5559    ) -> Result<Self> {
5560        state.increment_task_ref_count(parent_task)?;
5561        let sync_call_set = state.push(WaitableSet {
5562            is_sync_call_set: true,
5563            ..WaitableSet::default()
5564        })?;
5565        Ok(Self {
5566            context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5567            parent_task,
5568            wake_on_cancel: WakeOnCancel::None,
5569            state: GuestThreadState::NotStartedExplicit(start_func),
5570            instance_rep: None,
5571            sync_call_set,
5572            old_do_not_suspend: None,
5573        })
5574    }
5575}
5576
5577impl TableDebug for GuestThread {
5578    fn type_name() -> &'static str {
5579        "GuestThread"
5580    }
5581}
5582
5583enum SyncResult {
5584    NotProduced,
5585    Produced(Option<ValRaw>),
5586    Taken,
5587}
5588
5589impl SyncResult {
5590    fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5591        Ok(match mem::replace(self, SyncResult::Taken) {
5592            SyncResult::NotProduced => None,
5593            SyncResult::Produced(val) => Some(val),
5594            SyncResult::Taken => {
5595                bail_bug!("attempted to take a synchronous result that was already taken")
5596            }
5597        })
5598    }
5599}
5600
5601/// Represents a pending guest task.
5602pub(crate) struct GuestTask {
5603    /// See `WaitableCommon`
5604    common: WaitableCommon,
5605    /// Closure to lower the parameters passed to this task.
5606    lower_params: Option<RawLower>,
5607    /// See `LiftResult`
5608    lift_result: Option<LiftResult>,
5609    /// A place to stash the type-erased lifted result if it can't be delivered
5610    /// immediately.
5611    result: Option<LiftedResult>,
5612    /// Closure to call the callback function for an async-lifted export, if
5613    /// provided.
5614    callback: Option<CallbackFn>,
5615    /// See `Caller`
5616    caller: Caller,
5617    /// Borrow state for this task.
5618    ///
5619    /// Keeps track of `borrow<T>` received to this task to ensure that
5620    /// everything is dropped by the time it exits.
5621    call_context: CallContext,
5622    /// A place to stash the lowered result for a sync-to-async call until it
5623    /// can be returned to the caller.
5624    sync_result: SyncResult,
5625    /// Whether or not the task has been cancelled (i.e. whether the
5626    /// cancellation request has been delivered to the task, and thus whether
5627    /// the task is permitted to call `task.cancel`).
5628    cancel_request_delivered: bool,
5629    /// Whether or not the caller has requested cancellation of this task via
5630    /// `subtask.cancel` while it was still unresolved. A subsequent
5631    /// `subtask.cancel` of the same task traps. This matches the spec's
5632    /// `Subtask.cancellation_requested`.
5633    cancel_requested: bool,
5634    /// Whether or not we've sent a `Status::Starting` event to any current or
5635    /// future waiters for this waitable.
5636    starting_sent: bool,
5637    /// The runtime instance to which the exported function for this guest task
5638    /// belongs.
5639    ///
5640    /// Note that the task may do a sync->sync call via a fused adapter which
5641    /// results in that task executing code in a different instance, and it may
5642    /// call host functions and intrinsics from that other instance.
5643    instance: RuntimeInstance,
5644    /// If present, a pending `Event::Cancelled` to be delivered to this task.
5645    event: Option<Event>,
5646    /// Whether or not the task has exited.
5647    exited: bool,
5648    /// Threads belonging to this task
5649    threads: HashSet<TableId<GuestThread>>,
5650    /// Indicates whether this task was created for a call to an async-typed
5651    /// export.
5652    async_typed: bool,
5653    /// Indicates whether this task was created for a call to an async-lifted
5654    /// export.
5655    async_lifted: bool,
5656    /// Whether `ConcurrentState::interesting_tasks` has been decremented for
5657    /// this task yet.
5658    decremented_interesting_task_count: bool,
5659    /// The `TaskGroup` to which this task belongs.
5660    group: TaskGroupId,
5661    /// The number of outstanding references to this task.
5662    ///
5663    /// When this goes to zero, the task will be removed from
5664    /// `ConcurrentState::table`.
5665    ref_count: usize,
5666}
5667
5668impl GuestTask {
5669    fn already_lowered_parameters(&self) -> bool {
5670        // We reset `lower_params` after we lower the parameters
5671        self.lower_params.is_none()
5672    }
5673
5674    fn returned_or_cancelled(&self) -> bool {
5675        // We reset `lift_result` after we return or exit
5676        self.lift_result.is_none()
5677    }
5678
5679    fn new(
5680        state: &mut ConcurrentState,
5681        lower_params: RawLower,
5682        lift_result: LiftResult,
5683        caller: Caller,
5684        callback: Option<CallbackFn>,
5685        instance: RuntimeInstance,
5686        async_typed: bool,
5687        async_lifted: bool,
5688    ) -> Result<QualifiedThreadId> {
5689        let group = match caller {
5690            Caller::Guest { thread } => {
5691                let group = state.get_mut(thread.task)?.group;
5692                state.increment_group_ref_count(group)?;
5693                group
5694            }
5695            Caller::Host { .. } => state.make_task_group()?,
5696        };
5697
5698        let task = state.push(Self {
5699            common: WaitableCommon::default(),
5700            lower_params: Some(lower_params),
5701            lift_result: Some(lift_result),
5702            result: None,
5703            callback,
5704            caller,
5705            call_context: CallContext::default(),
5706            sync_result: SyncResult::NotProduced,
5707            cancel_request_delivered: false,
5708            cancel_requested: false,
5709            starting_sent: false,
5710            instance,
5711            event: None,
5712            exited: false,
5713            threads: HashSet::new(),
5714            async_typed,
5715            async_lifted,
5716            decremented_interesting_task_count: false,
5717            group,
5718            // One for the caller and one for the implicit guest thread we're
5719            // about to create:
5720            ref_count: 2,
5721        })?;
5722        let new_thread = GuestThread::new_implicit(state, task)?;
5723        let thread = state.push(new_thread)?;
5724        state.get_mut(task)?.threads.insert(thread);
5725        state.interesting_tasks += 1;
5726        let thread = QualifiedThreadId { task, thread };
5727        log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5728        Ok(thread)
5729    }
5730}
5731
5732impl TableDebug for GuestTask {
5733    fn type_name() -> &'static str {
5734        "GuestTask"
5735    }
5736}
5737
5738/// Represents state common to all kinds of waitables.
5739#[derive(Default)]
5740struct WaitableCommon {
5741    /// The currently pending event for this waitable, if any.
5742    event: Option<Event>,
5743    /// The set to which this waitable belongs, if any.
5744    set: Option<TableId<WaitableSet>>,
5745    /// The handle with which the guest refers to this waitable, if any.
5746    handle: Option<u32>,
5747}
5748
5749/// Represents a Component Model Async `waitable`.
5750#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5751enum Waitable {
5752    /// A host task
5753    Host(TableId<HostTask>),
5754    /// A guest task
5755    Guest(TableId<GuestTask>),
5756    /// The read or write end of a stream or future
5757    Transmit(TableId<TransmitHandle>),
5758}
5759
5760impl Waitable {
5761    /// Retrieve the `Waitable` corresponding to the specified guest-visible
5762    /// handle.
5763    fn from_instance(
5764        state: Pin<&mut ComponentInstance>,
5765        caller_instance: RuntimeComponentInstanceIndex,
5766        waitable: u32,
5767    ) -> Result<Self> {
5768        use crate::runtime::vm::component::Waitable;
5769
5770        let (waitable, kind) = state.instance_states().0[caller_instance]
5771            .handle_table()
5772            .waitable_rep(waitable)?;
5773
5774        Ok(match kind {
5775            Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5776            Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5777            Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5778        })
5779    }
5780
5781    /// Retrieve the host-visible identifier for this `Waitable`.
5782    fn rep(&self) -> u32 {
5783        match self {
5784            Self::Host(id) => id.rep(),
5785            Self::Guest(id) => id.rep(),
5786            Self::Transmit(id) => id.rep(),
5787        }
5788    }
5789
5790    /// Move this `Waitable` to the specified set (when `set` is `Some(_)`) or
5791    /// remove it from any set it may currently belong to (when `set` is
5792    /// `None`).
5793    fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5794        log::trace!("waitable {self:?} join set {set:?}");
5795
5796        let old = mem::replace(&mut self.common(state)?.set, set);
5797
5798        if let Some(old) = old {
5799            match *self {
5800                Waitable::Host(id) => state.remove_child(id, old),
5801                Waitable::Guest(id) => state.remove_child(id, old),
5802                Waitable::Transmit(id) => state.remove_child(id, old),
5803            }?;
5804
5805            state.get_mut(old)?.ready.remove(self);
5806        }
5807
5808        if let Some(set) = set {
5809            match *self {
5810                Waitable::Host(id) => state.add_child(id, set),
5811                Waitable::Guest(id) => state.add_child(id, set),
5812                Waitable::Transmit(id) => state.add_child(id, set),
5813            }?;
5814
5815            if self.common(state)?.event.is_some() {
5816                self.mark_ready(state)?;
5817            }
5818        }
5819
5820        Ok(())
5821    }
5822
5823    /// Retrieve mutable access to the `WaitableCommon` for this `Waitable`.
5824    fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5825        Ok(match self {
5826            Self::Host(id) => &mut state.get_mut(*id)?.common,
5827            Self::Guest(id) => &mut state.get_mut(*id)?.common,
5828            Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5829        })
5830    }
5831
5832    /// Trap if this waitable is currently a member of a waitable set.
5833    ///
5834    /// A synchronous stream/future/subtask operation may end up blocking on
5835    /// this waitable, so it is not allowed to run while the waitable is also
5836    /// being watched by a waitable set.
5837    fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5838        if self.common(state)?.set.is_some() {
5839            bail!(Trap::WaitableSyncAndAsync);
5840        }
5841        Ok(())
5842    }
5843
5844    /// Trap if a thread is currently synchronously waiting on this waitable.
5845    ///
5846    /// This corresponds to the spec's `trap_if(w.has_sync_waiter)` and is
5847    /// used to prevent other threads from stealing or otherwise interfering
5848    /// with the event the sync waiter is expecting.
5849    fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5850        if let Some(set) = self.common(state)?.set
5851            && state.get_mut(set)?.is_sync_call_set
5852        {
5853            bail!(Trap::WaitableSyncAndAsync);
5854        }
5855        Ok(())
5856    }
5857
5858    /// Set or clear the pending event for this waitable and either deliver it
5859    /// to the first waiter, if any, or mark it as ready to be delivered to the
5860    /// next waiter that arrives.
5861    fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
5862        log::trace!("set event for {self:?}: {event:?}");
5863        self.common(state)?.event = event;
5864        self.mark_ready(state)
5865    }
5866
5867    /// Take the pending event from this waitable, leaving `None` in its place.
5868    fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
5869        let common = self.common(state)?;
5870        let event = common.event.take();
5871        if let Some(set) = self.common(state)?.set {
5872            state.get_mut(set)?.ready.remove(self);
5873        }
5874
5875        Ok(event)
5876    }
5877
5878    /// Deliver the current event for this waitable to the first waiter, if any,
5879    /// or else mark it as ready to be delivered to the next waiter that
5880    /// arrives.
5881    fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
5882        if let Some(set) = self.common(state)?.set {
5883            state.get_mut(set)?.ready.insert(*self);
5884            state.wake_waiter(set)?;
5885        }
5886        Ok(())
5887    }
5888
5889    /// Remove this waitable from the store's rep table (or decrement the ref
5890    /// count if it's a guest task).
5891    fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
5892        match self {
5893            Self::Host(task) => state.delete_host_task(*task),
5894            Self::Guest(task) => state.decrement_task_ref_count(*task),
5895            Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
5896        }
5897    }
5898}
5899
5900impl fmt::Debug for Waitable {
5901    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
5902        match self {
5903            Self::Host(id) => write!(f, "{id:?}"),
5904            Self::Guest(id) => write!(f, "{id:?}"),
5905            Self::Transmit(id) => write!(f, "{id:?}"),
5906        }
5907    }
5908}
5909
5910/// Represents a Component Model Async `waitable-set`.
5911#[derive(Default)]
5912struct WaitableSet {
5913    /// Which waitables in this set have pending events, if any.
5914    ready: BTreeSet<Waitable>,
5915    /// Which guest threads are currently waiting on this set, if any.
5916    waiting: BTreeMap<QualifiedThreadId, WaitMode>,
5917    /// How many guest threads are waiting on this set, mirroring the spec's
5918    /// `WaitableSet.num_waiting`.
5919    num_waiting: usize,
5920    /// Whether this set is a synthetic, internal one meant for handling
5921    /// synchronous calls.
5922    is_sync_call_set: bool,
5923}
5924
5925impl WaitableSet {
5926    /// Records that a thread counted in `num_waiting` is no longer waiting.
5927    fn stop_waiting(&mut self) -> Result<()> {
5928        self.num_waiting = match self.num_waiting.checked_sub(1) {
5929            Some(n) => n,
5930            None => bail_bug!("waiter not accounted for in waitable set"),
5931        };
5932        Ok(())
5933    }
5934}
5935
5936impl TableDebug for WaitableSet {
5937    fn type_name() -> &'static str {
5938        "WaitableSet"
5939    }
5940}
5941
5942/// Type-erased closure to lower the parameters for a guest task.
5943type RawLower =
5944    Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
5945
5946/// Type-erased closure to lift the result for a guest task.
5947type RawLift = Box<
5948    dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
5949>;
5950
5951/// Type erased result of a guest task which may be downcast to the expected
5952/// type by a host caller (or simply ignored in the case of a guest caller; see
5953/// `DummyResult`).
5954type LiftedResult = Box<dyn Any + Send + Sync>;
5955
5956/// Used to return a result from a `LiftFn` when the actual result has already
5957/// been lowered to a guest task's stack and linear memory.
5958struct DummyResult;
5959
5960/// Represents the Component Model Async state of a (sub-)component instance.
5961#[derive(Default)]
5962pub struct ConcurrentInstanceState {
5963    /// Whether backpressure is set for this instance (enabled if >0)
5964    backpressure: u16,
5965    /// Whether this instance can be entered
5966    do_not_enter: bool,
5967    /// Whether this instance may suspend (i.e. whether this instance is
5968    /// currently running a sync-typed function).
5969    do_not_suspend: bool,
5970    /// Pending calls for this instance which require `Self::backpressure` to be
5971    /// zero and/or `Self::do_not_enter` to be false before they can proceed.
5972    pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
5973}
5974
5975impl ConcurrentInstanceState {
5976    pub fn pending_is_empty(&self) -> bool {
5977        self.pending.is_empty()
5978    }
5979}
5980
5981#[derive(Debug, Copy, Clone)]
5982pub(crate) enum CurrentThread {
5983    /// The currently running thread is a guest, identified here with its
5984    /// task/thread id combo.
5985    Guest(QualifiedThreadId),
5986    /// The currently running thread is a host task.
5987    Host(TableId<HostTask>),
5988    /// The currently running thread is a host call whose task has not yet been
5989    /// materialized. The contained ID identifies its guest caller.
5990    DeferredHost(QualifiedThreadId),
5991    /// There is no currently running thread because we are in the main event
5992    /// loop or concurrency is disabled.
5993    None,
5994}
5995
5996impl CurrentThread {
5997    fn guest(&self) -> Option<&QualifiedThreadId> {
5998        match self {
5999            Self::Guest(id) => Some(id),
6000            _ => None,
6001        }
6002    }
6003
6004    fn guest_task(&self) -> Option<TableId<GuestTask>> {
6005        match self {
6006            Self::Guest(id) => Some(id.task),
6007            _ => None,
6008        }
6009    }
6010
6011    fn is_none(&self) -> bool {
6012        matches!(self, Self::None)
6013    }
6014}
6015
6016impl From<QualifiedThreadId> for CurrentThread {
6017    fn from(id: QualifiedThreadId) -> Self {
6018        Self::Guest(id)
6019    }
6020}
6021
6022impl From<TableId<HostTask>> for CurrentThread {
6023    fn from(id: TableId<HostTask>) -> Self {
6024        Self::Host(id)
6025    }
6026}
6027
6028enum Priority {
6029    Switch,
6030    High,
6031    Low,
6032}
6033
6034/// Represents the Component Model Async state of a store.
6035pub struct ConcurrentState {
6036    /// The currently running thread, if any.
6037    ///
6038    /// Note that we lazily materialize threads on-demand and this field is not
6039    /// necessarily up-to-date. The `StoreOpaque::current_thread` method should
6040    /// be preferred over directly accessing this field.
6041    unforced_current_thread: CurrentThread,
6042
6043    /// Borrow state for the deferred host call, if any.
6044    ///
6045    /// This is `Some` if and only if [`Self::unforced_current_thread`] is
6046    /// [`CurrentThread::DeferredHost`]. Materializing the host task moves this
6047    /// context into that task.
6048    deferred_host_call_context: Option<CallContext>,
6049
6050    /// The set of pending host and background tasks, if any.
6051    ///
6052    /// See `ComponentInstance::poll_until` for where we temporarily take this
6053    /// out, poll it, then put it back to avoid any mutable aliasing hazards.
6054    futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6055    /// The table of waitables, waitable sets, etc.
6056    table: AlwaysMut<ResourceTable>,
6057    /// The next item to switch to if any.
6058    ///
6059    /// This takes precedence over items in the `high_priority` queue below and
6060    /// should be used in cases such as subtask calls and thread resume/promote
6061    /// operations where we must switch to a specific thread at the next turn of
6062    /// the event loop regardless of what happens to be present in the
6063    /// `high_priority` queue.
6064    switch_item: Option<WorkItem>,
6065    /// The item to set `switch_item` to when the current thread suspends.
6066    ///
6067    /// This is used when ever an async-lowered import or `subtask.cancel` is
6068    /// called in order to track the thread to switch back to once the current
6069    /// subtask suspends, if any.
6070    next_switch_item: Option<WorkItem>,
6071    /// The "high priority" work queue for this store's event loop.
6072    high_priority: VecDeque<WorkItem>,
6073    /// The "low priority" work queue for this store's event loop.
6074    low_priority: VecDeque<WorkItem>,
6075    /// A place to stash the reason a fiber is suspending so that the code which
6076    /// resumed it will know under what conditions the fiber should be resumed
6077    /// again.
6078    suspend_reason: Option<SuspendReason>,
6079    /// A cached fiber which is waiting for work to do.
6080    ///
6081    /// This helps us avoid creating a new fiber for each `GuestCall` work item.
6082    worker: Option<StoreFiber<'static>>,
6083    /// A place to stash the work item for which we're resuming a worker fiber.
6084    worker_item: Option<WorkerItem>,
6085
6086    /// Reference counts for all component error contexts
6087    ///
6088    /// NOTE: it is possible the global ref count to be *greater* than the sum of
6089    /// (sub)component ref counts as tracked by `error_context_tables`, for
6090    /// example when the host holds one or more references to error contexts.
6091    ///
6092    /// The key of this primary map is often referred to as the "rep" (i.e. host-side
6093    /// component-wide representation) of the index into concurrent state for a given
6094    /// stored `ErrorContext`.
6095    ///
6096    /// Stated another way, `TypeComponentGlobalErrorContextTableIndex` is essentially the same
6097    /// as a `TableId<ErrorContextState>`.
6098    global_error_context_ref_counts:
6099        BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6100
6101    /// The number of "interesting tasks" currently executing in the store.
6102    ///
6103    /// This tracks the concept of a component instance lifetime as defined in
6104    /// https://github.com/WebAssembly/component-model/pull/643. Specifically
6105    /// all tasks currently increment this counter which then gets decremented
6106    /// when they exit. In the future some tasks might not increment this
6107    /// counter, but for now all do.
6108    ///
6109    /// This is used to implement `Accessor::poll_no_interesting_tasks` to
6110    /// inform the embedder when all tasks have completed. This is then
6111    /// used in wasmtime-wasi-http, for example, to know when an instance is
6112    /// idle.
6113    interesting_tasks: usize,
6114
6115    /// Single waker to notify when `interesting_tasks` reaches 0.
6116    ///
6117    /// Used in the implementation of `Accessor::poll_no_interesting_tasks`.
6118    interesting_tasks_empty_waker: Option<Waker>,
6119
6120    /// Single waker to notify when a component instance goes from
6121    /// not-concurrently-callable to concurrently-callable.
6122    ///
6123    /// Used in the implementation of `Accessor::poll_ready_for_concurrent_call`.
6124    ready_for_concurrent_call_waker: Option<Waker>,
6125
6126    /// Whether the `StoreContextMut::poll_until` event loop is running.
6127    event_loop_running: bool,
6128
6129    /// Stack of switch items to push to and pop from when entering and exiting
6130    /// sync-to-sync calls.
6131    saved_next_switch_items: Vec<Option<WorkItem>>,
6132
6133    /// See [TaskGroupHook].
6134    #[cfg(feature = "task-group-hook")]
6135    task_group_hook: Option<Box<dyn TaskGroupHook>>,
6136}
6137
6138impl Default for ConcurrentState {
6139    fn default() -> Self {
6140        Self {
6141            unforced_current_thread: CurrentThread::None,
6142            deferred_host_call_context: None,
6143            table: AlwaysMut::new(ResourceTable::new()),
6144            futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6145            switch_item: None,
6146            next_switch_item: None,
6147            high_priority: VecDeque::new(),
6148            low_priority: VecDeque::new(),
6149            suspend_reason: None,
6150            worker: None,
6151            worker_item: None,
6152            global_error_context_ref_counts: BTreeMap::new(),
6153            interesting_tasks: 0,
6154            interesting_tasks_empty_waker: None,
6155            ready_for_concurrent_call_waker: None,
6156            event_loop_running: false,
6157            saved_next_switch_items: Vec::new(),
6158            #[cfg(feature = "task-group-hook")]
6159            task_group_hook: None,
6160        }
6161    }
6162}
6163
6164impl ConcurrentState {
6165    /// Take ownership of any fibers and futures owned by this object.
6166    ///
6167    /// This should be used when disposing of the `Store` containing this object
6168    /// in order to gracefully resolve any and all fibers using
6169    /// `StoreFiber::dispose`.  This is necessary to avoid possible
6170    /// use-after-free bugs due to fibers which may still have access to the
6171    /// `Store`.
6172    ///
6173    /// Additionally, the futures collected with this function should be dropped
6174    /// within a `tls::set` call, which will ensure than any futures closing
6175    /// over an `&Accessor` will have access to the store when dropped, allowing
6176    /// e.g. `WithAccessor[AndValue]` instances to be disposed of without
6177    /// panicking.
6178    ///
6179    /// Note that this will leave the object in an inconsistent and unusable
6180    /// state, so it should only be used just prior to dropping it.
6181    pub(crate) fn take_fibers_and_futures(
6182        &mut self,
6183        fibers: &mut Vec<StoreFiber<'static>>,
6184        futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6185    ) {
6186        let ConcurrentState {
6187            table,
6188            worker,
6189            switch_item,
6190            next_switch_item,
6191            high_priority,
6192            low_priority,
6193            saved_next_switch_items,
6194            futures: pending_futures,
6195            worker_item: _,
6196            unforced_current_thread: _,
6197            deferred_host_call_context: _,
6198            suspend_reason: _,
6199            global_error_context_ref_counts: _,
6200            interesting_tasks: _,
6201            interesting_tasks_empty_waker: _,
6202            ready_for_concurrent_call_waker: _,
6203            event_loop_running: _,
6204            #[cfg(feature = "task-group-hook")]
6205                task_group_hook: _,
6206        } = self;
6207
6208        let mut items = Vec::new();
6209        for (_, entry) in table.get_mut().iter_mut() {
6210            let entry = entry.into_entry_mut();
6211            if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6212                for mode in mem::take(&mut set.waiting).into_values() {
6213                    match mode {
6214                        WaitMode::Fiber(fiber) => {
6215                            fibers.push(fiber);
6216                        }
6217                        WaitMode::Callback(_) => {}
6218                    }
6219                }
6220            } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6221                match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6222                    GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6223                        fibers.push(fiber);
6224                    }
6225                    GuestThreadState::NotStartedImplicit
6226                    | GuestThreadState::NotStartedExplicit(_)
6227                    | GuestThreadState::Running
6228                    | GuestThreadState::Completed => {}
6229                }
6230            } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6231                if let Some(item) = item.take() {
6232                    items.push(item);
6233                }
6234            }
6235        }
6236
6237        if let Some(fiber) = worker.take() {
6238            fibers.push(fiber);
6239        }
6240
6241        let mut handle_item = |item| match item {
6242            WorkItem::ResumeFiber { fiber, .. } => {
6243                fibers.push(fiber);
6244            }
6245            WorkItem::PushFuture(future) => {
6246                pending_futures
6247                    .get_mut()
6248                    .as_mut()
6249                    .unwrap()
6250                    .push(future.into_inner());
6251            }
6252            WorkItem::ResumeThread { .. }
6253            | WorkItem::GuestCall { .. }
6254            | WorkItem::WorkerFunction(_) => {}
6255        };
6256
6257        for item in items {
6258            handle_item(item);
6259        }
6260        if let Some(item) = switch_item.take() {
6261            handle_item(item);
6262        }
6263        if let Some(item) = next_switch_item.take() {
6264            handle_item(item);
6265        }
6266        for item in mem::take(high_priority) {
6267            handle_item(item);
6268        }
6269        for item in mem::take(low_priority) {
6270            handle_item(item);
6271        }
6272        for item in mem::take(saved_next_switch_items)
6273            .into_iter()
6274            .filter_map(|v| v)
6275        {
6276            handle_item(item);
6277        }
6278
6279        if let Some(them) = pending_futures.get_mut().take() {
6280            futures.push(them);
6281        }
6282    }
6283
6284    #[cfg(feature = "gc")]
6285    pub(crate) fn trace_fiber_roots(
6286        &mut self,
6287        modules: &ModuleRegistry,
6288        unwind: &dyn Unwind,
6289        gc_roots_list: &mut GcRootsList,
6290    ) {
6291        let ConcurrentState {
6292            table,
6293            worker,
6294            switch_item,
6295            next_switch_item,
6296            high_priority,
6297            low_priority,
6298            saved_next_switch_items,
6299
6300            // TODO(cm-gc): This field contains `ValRaw`s, but they are never GC
6301            // references because the component model doesn't support GC yet. We
6302            // will need to trace these somehow when it does.
6303            futures: _,
6304
6305            // These fields do not contain GC references.
6306            worker_item: _,
6307            unforced_current_thread: _,
6308            deferred_host_call_context: _,
6309            suspend_reason: _,
6310            global_error_context_ref_counts: _,
6311            interesting_tasks: _,
6312            interesting_tasks_empty_waker: _,
6313            ready_for_concurrent_call_waker: _,
6314            event_loop_running: _,
6315            #[cfg(feature = "task-group-hook")]
6316                task_group_hook: _,
6317        } = self;
6318
6319        for (_, entry) in table.get_mut().iter_mut() {
6320            let entry = entry.into_entry_mut();
6321            if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6322                for mode in set.waiting.values_mut() {
6323                    match mode {
6324                        WaitMode::Fiber(fiber) => {
6325                            fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6326                        }
6327                        WaitMode::Callback(_) => {}
6328                    }
6329                }
6330            } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6331                match &mut thread.state {
6332                    GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6333                        fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6334                    }
6335                    GuestThreadState::NotStartedImplicit
6336                    | GuestThreadState::NotStartedExplicit(_)
6337                    | GuestThreadState::Running
6338                    | GuestThreadState::Completed => {}
6339                }
6340            } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6341                entry.downcast_mut::<Option<WorkItem>>()
6342            {
6343                fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6344            }
6345        }
6346
6347        if let Some(fiber) = worker {
6348            fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6349        }
6350
6351        let mut handle_item = |item: &mut WorkItem| match item {
6352            WorkItem::ResumeFiber { fiber, .. } => {
6353                fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6354            }
6355            WorkItem::PushFuture(_future) => {
6356                // TODO(cm-gc): once futures can contain GC roots, we will need
6357                // to trace them.
6358            }
6359            WorkItem::ResumeThread { .. }
6360            | WorkItem::GuestCall { .. }
6361            | WorkItem::WorkerFunction(_) => {}
6362        };
6363
6364        if let Some(item) = switch_item {
6365            handle_item(item);
6366        }
6367        if let Some(item) = next_switch_item {
6368            handle_item(item);
6369        }
6370        for item in high_priority {
6371            handle_item(item);
6372        }
6373        for item in low_priority {
6374            handle_item(item);
6375        }
6376        for item in saved_next_switch_items
6377            .iter_mut()
6378            .filter_map(|v| v.as_mut())
6379        {
6380            handle_item(item);
6381        }
6382    }
6383
6384    fn schedule_suspended_thread(
6385        &mut self,
6386        thread: QualifiedThreadId,
6387        instance: RuntimeInstance,
6388        priority: Priority,
6389    ) -> Result<()> {
6390        // Validate the destination while the source still owns the fiber.
6391        if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6392            bail_bug!("switch item already set");
6393        }
6394        let slot = &mut self.get_mut(thread.thread)?.state;
6395        if !matches!(slot, GuestThreadState::Suspended(_)) {
6396            bail!(Trap::CannotResumeThread);
6397        }
6398        let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6399        else {
6400            unreachable!()
6401        };
6402        let item = WorkItem::ResumeFiber {
6403            instance,
6404            thread,
6405            fiber,
6406        };
6407        match priority {
6408            Priority::Switch => self.switch_item = Some(item),
6409            Priority::High => self.push_high_priority(item),
6410            Priority::Low => self.push_low_priority(item),
6411        }
6412        Ok(())
6413    }
6414
6415    fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6416        let slot = &mut self.get_mut(thread)?.state;
6417        slot.check_no_fiber()?;
6418        *slot = GuestThreadState::Running;
6419        Ok(())
6420    }
6421
6422    fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6423        // push can consume and drop its argument when the table is full.
6424        // Reserve with an empty item before taking the fiber-owning source.
6425        let saved = self.push(None::<WorkItem>)?;
6426        let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6427        *slot = self.next_switch_item.take();
6428        Ok(saved)
6429    }
6430
6431    fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6432        if self.next_switch_item.is_some() {
6433            bail_bug!("next switch item already set when restoring");
6434        }
6435        self.next_switch_item = self.delete(saved)?;
6436        Ok(())
6437    }
6438
6439    fn take_callback_waiter(
6440        &mut self,
6441        set: TableId<WaitableSet>,
6442        thread: QualifiedThreadId,
6443    ) -> Result<Instance> {
6444        let waiting = &mut self.get_mut(set)?.waiting;
6445        let instance = match waiting.get(&thread) {
6446            Some(WaitMode::Callback(instance)) => *instance,
6447            other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6448        };
6449        waiting.remove(&thread);
6450        Ok(instance)
6451    }
6452
6453    fn push<V: Send + Sync + 'static>(
6454        &mut self,
6455        value: V,
6456    ) -> Result<TableId<V>, ResourceTableError> {
6457        self.table.get_mut().push(value).map(TableId::from)
6458    }
6459
6460    fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6461        self.table.get_mut().get_mut(&Resource::from(id))
6462    }
6463
6464    pub fn add_child<T: 'static, U: 'static>(
6465        &mut self,
6466        child: TableId<T>,
6467        parent: TableId<U>,
6468    ) -> Result<(), ResourceTableError> {
6469        self.table
6470            .get_mut()
6471            .add_child(Resource::from(child), Resource::from(parent))
6472    }
6473
6474    pub fn remove_child<T: 'static, U: 'static>(
6475        &mut self,
6476        child: TableId<T>,
6477        parent: TableId<U>,
6478    ) -> Result<(), ResourceTableError> {
6479        self.table
6480            .get_mut()
6481            .remove_child(Resource::from(child), Resource::from(parent))
6482    }
6483
6484    fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6485        self.table.get_mut().delete(Resource::from(id))
6486    }
6487
6488    fn push_future(&mut self, future: HostTaskFuture) {
6489        // Note that we can't directly push to `ConcurrentState::futures` here
6490        // since this may be called from a future that's being polled inside
6491        // `Self::poll_until`, which temporarily removes the `FuturesUnordered`
6492        // so it has exclusive access while polling it.  Therefore, we push a
6493        // work item to the "high priority" queue, which will actually push to
6494        // `ConcurrentState::futures` later.
6495        self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6496    }
6497
6498    fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6499        if self.switch_item.is_some() {
6500            // Preserve ownership on this internal-error path. The caller
6501            // must stop scheduling after the error; teardown disposes the item.
6502            self.push_high_priority(item);
6503            bail_bug!("switch item already set");
6504        }
6505
6506        self.switch_item = Some(item);
6507        log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6508
6509        Ok(())
6510    }
6511
6512    fn take_next_switch_item(&mut self) -> Result<()> {
6513        if let Some(item) = self.next_switch_item.take() {
6514            self.set_switch_item(item)?;
6515        }
6516        Ok(())
6517    }
6518
6519    /// Wake the first thread waiting on `set`, if any, so that it may receive
6520    /// one of the set's pending events.
6521    fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6522        // Finish every fallible lookup and check before taking the waiter out
6523        // of the set, so that an error leaves its fiber owned by the set.
6524        let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6525            return Ok(());
6526        };
6527        let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6528        if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6529            bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6530        }
6531        let instance = self.get_mut(thread.task)?.instance;
6532        self.get_mut(thread.thread)?.wake_on_cancel.take();
6533        let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6534
6535        let item = match mode {
6536            WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6537                instance,
6538                thread,
6539                fiber,
6540            },
6541            WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6542                instance,
6543                call: GuestCall {
6544                    thread,
6545                    kind: GuestCallKind::DeliverEvent {
6546                        instance: callback_instance,
6547                        set: Some(set),
6548                    },
6549                },
6550            },
6551        };
6552        self.push_high_priority(item);
6553        Ok(())
6554    }
6555
6556    fn push_high_priority(&mut self, item: WorkItem) {
6557        self.high_priority.push_front(item);
6558        log::trace!(
6559            "push high priority: {:?}",
6560            self.high_priority.front().unwrap()
6561        );
6562    }
6563
6564    fn push_low_priority(&mut self, item: WorkItem) {
6565        self.low_priority.push_front(item);
6566        log::trace!(
6567            "push low priority: {:?}",
6568            self.low_priority.front().unwrap()
6569        );
6570    }
6571
6572    fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6573        match priority {
6574            Priority::Switch => self.set_switch_item(item)?,
6575            Priority::High => self.push_high_priority(item),
6576            Priority::Low => self.push_low_priority(item),
6577        }
6578
6579        Ok(())
6580    }
6581
6582    fn promote_instance_local_thread_work_item(
6583        &mut self,
6584        current_instance: RuntimeInstance,
6585    ) -> Result<bool> {
6586        log::trace!("promote thread work items for {current_instance:?}");
6587
6588        self.promote_work_item_matching(|item: &WorkItem| {
6589            let result = match item {
6590                WorkItem::ResumeThread { instance, .. }
6591                | WorkItem::ResumeFiber { instance, .. }
6592                | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6593                _ => false,
6594            };
6595
6596            log::trace!("candidate {item:?}: {result}");
6597            result
6598        })
6599    }
6600
6601    fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6602        self.promote_work_item_matching(|item: &WorkItem| match item {
6603            WorkItem::ResumeThread {
6604                thread: item_thread,
6605                ..
6606            }
6607            | WorkItem::GuestCall {
6608                call:
6609                    GuestCall {
6610                        thread: item_thread,
6611                        ..
6612                    },
6613                ..
6614            } => *item_thread == thread,
6615            _ => false,
6616        })
6617    }
6618
6619    fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6620    where
6621        F: FnMut(&WorkItem) -> bool,
6622    {
6623        // Scan from the back to preserve the ordering of items
6624        // are popped from the back of the `VecDeque`s by `poll_until` and
6625        // pushed to the front by `push_{high,low}_priority`.
6626
6627        // Inspect items while they are still owned by the store. A predicate
6628        // (including tracing) may panic; it must not drop an extracted queue.
6629        if self.switch_item.is_none() {
6630            if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6631                self.switch_item = self.high_priority.remove(index);
6632            }
6633        }
6634        if self.switch_item.is_none() {
6635            if let Some(index) = self.low_priority.iter().rposition(predicate) {
6636                self.switch_item = self.low_priority.remove(index);
6637            }
6638        }
6639        Ok(self.switch_item.is_some())
6640    }
6641
6642    /// Used by `ResourceTables` to acquire the current `CallContext` for the
6643    /// specified task.
6644    pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6645        match task.kind() {
6646            ScopeKind::HostId(task) => {
6647                let task: TableId<HostTask> = TableId::new(task);
6648                Ok(&mut self.get_mut(task)?.call_context)
6649            }
6650            ScopeKind::Id(task) => {
6651                let task: TableId<GuestTask> = TableId::new(task);
6652                Ok(&mut self.get_mut(task)?.call_context)
6653            }
6654        }
6655    }
6656
6657    pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6658        self.deferred_host_call_context.as_mut()
6659    }
6660
6661    fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6662        match self.futures.get_mut().as_mut() {
6663            Some(f) => Ok(f),
6664            None => bail_bug!("futures field of concurrent state is currently taken"),
6665        }
6666    }
6667
6668    pub(crate) fn table(&mut self) -> &mut ResourceTable {
6669        self.table.get_mut()
6670    }
6671
6672    fn debug_assert_deferred_host_invariant(&self) {
6673        debug_assert_eq!(
6674            self.deferred_host_call_context.is_some(),
6675            matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6676            "a deferred host thread and call context must exist together",
6677        );
6678    }
6679
6680    fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6681        self.debug_assert_deferred_host_invariant();
6682        let caller = match self.unforced_current_thread {
6683            CurrentThread::DeferredHost(caller) => caller,
6684            thread => return Ok(thread),
6685        };
6686
6687        // Push first so allocation failure leaves the deferred state intact.
6688        let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6689        let task = self.push(task)?;
6690        let call_context = self
6691            .deferred_host_call_context
6692            .take()
6693            .expect("deferred host call context should be present");
6694        self.get_mut(task)
6695            .expect("newly inserted host task should be present")
6696            .call_context = call_context;
6697        self.unforced_current_thread = CurrentThread::Host(task);
6698        self.debug_assert_deferred_host_invariant();
6699        log::trace!("new host task materialized {task:?}");
6700        Ok(CurrentThread::Host(task))
6701    }
6702
6703    fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6704        match self.materialize_host_task()? {
6705            CurrentThread::Host(id) => Ok(Some(id)),
6706            CurrentThread::None => Ok(None),
6707            CurrentThread::Guest(_) => {
6708                bail_bug!("tried to materialize a host task id from a guest thread")
6709            }
6710            CurrentThread::DeferredHost(_) => {
6711                bail_bug!(
6712                    "current thread is a deferred host thread which should have been materialized"
6713                )
6714            }
6715        }
6716    }
6717
6718    pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6719        match self.materialize_host_task()? {
6720            CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6721            _ => bail_bug!("current scope is not a deferred host scope"),
6722        }
6723    }
6724
6725    fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6726        let count = &mut self.get_mut(task)?.ref_count;
6727        *count += 1;
6728        log::trace!("increment {task:?} to {count}");
6729        Ok(())
6730    }
6731
6732    fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6733        let count = &mut self.get_mut(task)?.ref_count;
6734        if *count == 0 {
6735            bail_bug!("can't decrement task ref count that's already zero");
6736        }
6737        *count -= 1;
6738        log::trace!("decrement {task:?} to {count}");
6739        if *count == 0 {
6740            let task = self.delete(task)?;
6741
6742            self.decrement_group_ref_count(task.group)?;
6743
6744            // When a guest task is created it increments the
6745            // `ConcurrentState::interesting_tasks` counter, and that needs
6746            // to be paired with a decrement. There are a few situations in
6747            // which the decrement needs to happen which don't all funnel
6748            // through here, so in lieu of that at least try to catch issues
6749            // where we forgot to do a decrement.
6750            debug_assert!(task.decremented_interesting_task_count);
6751        }
6752        Ok(())
6753    }
6754
6755    fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6756        log::trace!("delete host task {task:?}");
6757        let task = self.delete(task)?;
6758
6759        self.decrement_group_ref_count(task.group)
6760    }
6761}
6762
6763/// Provide a type hint to compiler about the shape of a parameter lower
6764/// closure.
6765fn for_any_lower<
6766    F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6767>(
6768    fun: F,
6769) -> F {
6770    fun
6771}
6772
6773/// Provide a type hint to compiler about the shape of a result lift closure.
6774fn for_any_lift<
6775    F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6776>(
6777    fun: F,
6778) -> F {
6779    fun
6780}
6781
6782fn check_ambient_store(id: StoreId) {
6783    let message = "\
6784        `Future`s which depend on asynchronous component tasks, streams, or \
6785        futures to complete may only be polled from the event loop of the \
6786        store to which they belong.  Please use \
6787        `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6788    ";
6789    tls::try_get(|store| {
6790        let matched = match store {
6791            tls::TryGet::Some(store) => store.id() == id,
6792            tls::TryGet::Taken | tls::TryGet::None => false,
6793        };
6794
6795        if !matched {
6796            panic!("{message}")
6797        }
6798    });
6799}
6800
6801fn unpack_callback_code(code: u32) -> (u32, u32) {
6802    (code & 0xF, code >> 4)
6803}
6804
6805/// Helper struct for packaging parameters to be passed to
6806/// `ComponentInstance::waitable_check` for calls to `waitable-set.wait` or
6807/// `waitable-set.poll`.
6808struct WaitableCheckParams {
6809    set: TableId<WaitableSet>,
6810    options: OptionsIndex,
6811    payload: u32,
6812}
6813
6814/// Indicates whether `ComponentInstance::waitable_check` is being called for
6815/// `waitable-set.wait` or `waitable-set.poll`.
6816enum WaitableCheck {
6817    Wait,
6818    Poll,
6819}
6820
6821/// Represents a guest task called from the host, prepared using `prepare_call`.
6822pub(crate) struct PreparedCall<R> {
6823    /// The guest export to be called
6824    handle: Func,
6825    /// The guest thread created by `prepare_call`
6826    thread: QualifiedThreadId,
6827    /// The number of lowered core Wasm parameters to pass to the call.
6828    param_count: usize,
6829    /// The `oneshot::Receiver` to which the result of the call will be
6830    /// delivered when it is available.
6831    rx: oneshot::Receiver<LiftedResult>,
6832    _phantom: PhantomData<R>,
6833}
6834
6835impl<R> PreparedCall<R> {
6836    /// Acquire a reference to the `TaskId` for this `PreparedCall`.
6837    ///
6838    /// Note that this increments the reference count for the task, and
6839    /// `TaskId::host_future_dropped` decrements it.  Hence,
6840    /// `TaskId::host_future_dropped` _must_ be called before the `TaskId` is
6841    /// dropped to avoid leaking the task.
6842    pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6843        store
6844            .concurrent_state_mut()?
6845            .increment_task_ref_count(self.thread.task)?;
6846
6847        Ok(TaskId {
6848            task: self.thread.task,
6849        })
6850    }
6851}
6852
6853/// Represents a task created by `prepare_call`.
6854pub(crate) struct TaskId {
6855    task: TableId<GuestTask>,
6856}
6857
6858impl TaskId {
6859    /// The host future for an async task was dropped.
6860    ///
6861    /// If the parameters have not been lowered yet, it is no longer valid to
6862    /// do so, as the lowering closure would see a dangling pointer. In this
6863    /// case the closure is replaced with one that fails, so the call fails if
6864    /// it's ever started. This can't fail, since the task can't be deleted
6865    /// while its host future is live.
6866    ///
6867    /// Otherwise, there may be running threads, or ones that are suspended and
6868    /// can be resumed by other tasks for this component, so we mark the future
6869    /// as dropped and delete the task when all threads are done.
6870    pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
6871        let task = store.concurrent_state_mut()?.get_mut(self.task)?;
6872        if !task.already_lowered_parameters() {
6873            // TODO: avoid leaking the task and trapping in this case, e.g. by
6874            // allowing the replacement callback to return a value that tells
6875            // the event loop "nevermind, the host gave up on this call; just
6876            // skip it, clean up the task, and carry on with other work".
6877            task.lower_params = Some(Box::new(|store, _| {
6878                store.set_trapped();
6879                bail!("the host future for this call was dropped before it started")
6880            }));
6881        } else {
6882            // TODO: we're temporarily ignoring any error here to avoid a
6883            // panic-on-unwrap in `call_async_concurrent`.  Instead, we should
6884            // determine a safe way to defer the decrement (and any error it
6885            // might produce) outside of a `drop` call.
6886            _ = store
6887                .concurrent_state_mut()?
6888                .decrement_task_ref_count(self.task);
6889        }
6890
6891        Ok(())
6892    }
6893}
6894
6895/// Prepare a call to the specified exported Wasm function, providing functions
6896/// for lowering the parameters and lifting the result.
6897///
6898/// To enqueue the returned `PreparedCall` in the `ComponentInstance`'s event
6899/// loop, use `stage_call`.
6900pub(crate) fn prepare_call<T, R>(
6901    mut store: StoreContextMut<T>,
6902    handle: Func,
6903    param_count: usize,
6904    lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
6905    + Send
6906    + Sync
6907    + 'static,
6908    lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
6909    + Send
6910    + Sync
6911    + 'static,
6912) -> Result<PreparedCall<R>> {
6913    // This check should be redundant, but leave it here so that there is a
6914    // clear error when trying to execute wasm in a poisoned store instead of
6915    // possibly hitting some other `bail_bug!` because of inconsistent state.
6916    if !store.0.may_enter() {
6917        bail!(Trap::CannotEnterComponent);
6918    }
6919
6920    let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
6921
6922    let instance = handle.instance().id().get(store.0);
6923    let options = &instance.component().env_component().options[options];
6924    let ty = &instance.component().types()[ty];
6925    let async_typed = ty.async_;
6926    let async_lifted = raw_options.async_;
6927    let task_return_type = ty.results;
6928    let component_instance = raw_options.instance;
6929    let callback = options.callback.map(|i| instance.runtime_callback(i));
6930    let memory = options
6931        .memory()
6932        .map(|i| instance.runtime_memory(i))
6933        .map(SendSyncPtr::new);
6934    let string_encoding = options.string_encoding;
6935    let token = StoreToken::new(store.as_context_mut());
6936    let caller = store.0.materialize_host_task_id()?;
6937    let state = store.0.concurrent_state_mut()?;
6938
6939    let (tx, rx) = oneshot::channel();
6940
6941    let instance = handle.instance().runtime_instance(component_instance);
6942    let thread = GuestTask::new(
6943        state,
6944        Box::new(for_any_lower(move |store, params| {
6945            lower_params(token.as_context_mut(store), params)
6946        })),
6947        LiftResult {
6948            lift: Box::new(for_any_lift(move |store, result| {
6949                lift_result(store, result)
6950            })),
6951            ty: task_return_type,
6952            memory,
6953            string_encoding,
6954        },
6955        Caller::Host {
6956            tx: Some(tx),
6957            caller,
6958        },
6959        callback.map(|callback| {
6960            let callback = SendSyncPtr::new(callback);
6961            let instance = handle.instance();
6962            Box::new(move |store: &mut dyn VMStore, event, handle| {
6963                let store = token.as_context_mut(store);
6964                // SAFETY: Per the contract of `prepare_call`, the callback
6965                // will remain valid at least as long is this task exists.
6966                unsafe { instance.call_callback(store, callback, event, handle) }
6967            }) as CallbackFn
6968        }),
6969        instance,
6970        async_typed,
6971        async_lifted,
6972    )?;
6973
6974    Ok(PreparedCall {
6975        handle,
6976        thread,
6977        param_count,
6978        rx,
6979        _phantom: PhantomData,
6980    })
6981}
6982
6983pub(crate) struct StagedCall<R> {
6984    store: StoreId,
6985    rx: oneshot::Receiver<LiftedResult>,
6986    _marker: PhantomData<fn() -> R>,
6987    /// The task group to which the underlying task belongs.
6988    ///
6989    /// Note that this must be treated as opaque and not used to look up the
6990    /// corresponding `TaskGroup` since we decrement the task refcount in
6991    /// `StagedCall::new` which may cause the group to be disposed.  This field
6992    /// is only present to enable `[Typed]FuncCallConcurrent::group`, allowing
6993    /// the embedder to correlate the task group with `TaskGroupHook` events.
6994    group: TaskGroupId,
6995}
6996
6997impl<R> StagedCall<R> {
6998    /// Queue a call previously prepared using `prepare_call` to be run as part of
6999    /// the associated `ComponentInstance`'s event loop.
7000    ///
7001    /// Note that this has the side-effect of disposing the specified
7002    /// `PrepareCall` and decrementing the task reference count since
7003    /// `StagedCall` has no further need to access the task directly.
7004    ///
7005    /// The returned future will resolve to the result once it is available, but
7006    /// must only be polled via the instance's event loop. See
7007    /// `StoreContextMut::run_concurrent` for details.
7008    pub(crate) fn new<T: 'static>(
7009        mut store: StoreContextMut<T>,
7010        prepared: PreparedCall<R>,
7011    ) -> Result<StagedCall<R>> {
7012        let PreparedCall {
7013            handle,
7014            thread,
7015            param_count,
7016            rx,
7017            ..
7018        } = prepared;
7019
7020        stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7021
7022        let state = store.0.concurrent_state_mut()?;
7023        let group = state.get_mut(thread.task)?.group;
7024        state.decrement_task_ref_count(thread.task)?;
7025
7026        Ok(StagedCall {
7027            store: store.0.id(),
7028            rx,
7029            _marker: PhantomData,
7030            group,
7031        })
7032    }
7033}
7034
7035impl<R> Future for StagedCall<R>
7036where
7037    R: 'static,
7038{
7039    type Output = Result<R>;
7040
7041    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7042        check_ambient_store(self.store);
7043        Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7044            Ok(r) => match r.downcast() {
7045                Ok(r) => Ok(*r),
7046                Err(_) => bail_bug!("wrong type of value produced"),
7047            },
7048            Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7049        })
7050    }
7051}
7052
7053/// Queue a call previously prepared using `prepare_call` to be run as part of
7054/// the associated `ComponentInstance`'s event loop.
7055fn stage_call0<T: 'static>(
7056    store: StoreContextMut<T>,
7057    handle: Func,
7058    guest_thread: QualifiedThreadId,
7059    param_count: usize,
7060) -> Result<()> {
7061    let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7062    let is_concurrent = raw_options.async_;
7063    let callback = raw_options.callback;
7064    let instance = handle.instance();
7065    let callee = handle.lifted_core_func(store.0);
7066    let post_return = raw_options
7067        .post_return
7068        .map(|i| instance.id().get(store.0).runtime_post_return(i));
7069    let callback = callback.map(|i| {
7070        let instance = instance.id().get(store.0);
7071        SendSyncPtr::new(instance.runtime_callback(i))
7072    });
7073
7074    log::trace!("queueing call {guest_thread:?}");
7075
7076    // SAFETY: `callee`, `callback`, and `post_return` are valid pointers
7077    // (with signatures appropriate for this call) and will remain valid as
7078    // long as this instance is valid.
7079    unsafe {
7080        instance.stage_call(
7081            store,
7082            guest_thread,
7083            SendSyncPtr::new(callee),
7084            param_count,
7085            1,
7086            is_concurrent,
7087            callback,
7088            post_return.map(SendSyncPtr::new),
7089            true,
7090        )
7091    }
7092}
7093
7094#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7095mod tests {
7096    use super::*;
7097    use crate::component::{Component, Linker};
7098    use crate::store::AsStoreOpaque;
7099    use crate::{Config, Engine};
7100
7101    fn host_subtask(
7102        state: HostTaskState,
7103        event: Option<Event>,
7104    ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7105        let mut config = Config::new();
7106        config.wasm_component_model_async(true);
7107        let engine = Engine::new(&config)?;
7108        let component = Component::new(&engine, "(component)")?;
7109        let mut store = Store::new(&engine, ());
7110        let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7111        let store_opaque = store.as_store_opaque();
7112        let concurrent_state = store_opaque.concurrent_state_mut()?;
7113        // These tests create an isolated host task without a guest caller, so
7114        // give it its own task group instead of inheriting the caller's group.
7115        let group = concurrent_state.make_task_group()?;
7116        let task = concurrent_state.push(HostTask {
7117            common: WaitableCommon::default(),
7118            call_context: CallContext::default(),
7119            state,
7120            group,
7121        })?;
7122        let handle = store_opaque
7123            .handle_table_insertion(
7124                instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7125            )
7126            .subtask_host(task.rep())?;
7127        let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7128        common.handle = Some(handle);
7129        common.event = event;
7130        Ok((store, instance, task, handle))
7131    }
7132
7133    #[test]
7134    fn host_subtask_drop_during_cancellation() -> Result<()> {
7135        for abort_completed in [false, true] {
7136            let (handle, future) = JoinHandle::run(future::pending::<()>());
7137            let mut future = pin!(future);
7138            let (mut store, instance, task, handle) =
7139                host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7140            let store = store.as_store_opaque();
7141            let caller = RuntimeComponentInstanceIndex::from_u32(0);
7142            assert_eq!(
7143                instance.subtask_cancel(store, caller, true, handle)?,
7144                BLOCKED
7145            );
7146            if abort_completed {
7147                // Even after the abort resolves, the completion worker still
7148                // needs the task entry to lower the result and publish an event.
7149                assert!(matches!(
7150                    future
7151                        .as_mut()
7152                        .poll(&mut Context::from_waker(Waker::noop())),
7153                    Poll::Ready(None),
7154                ));
7155            }
7156            for async_ in [false, true] {
7157                let err = instance
7158                    .subtask_cancel(store, caller, async_, handle)
7159                    .unwrap_err();
7160                assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7161            }
7162            let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7163            assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7164            assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7165        }
7166        Ok(())
7167    }
7168
7169    #[test]
7170    fn host_subtask_cancel_after_completion() -> Result<()> {
7171        for async_ in [false, true] {
7172            let (mut store, instance, task, handle) = host_subtask(
7173                HostTaskState::CalleeDone { cancelled: false },
7174                Some(Event::Subtask {
7175                    status: Status::Returned,
7176                }),
7177            )?;
7178            let store = store.as_store_opaque();
7179            let caller = RuntimeComponentInstanceIndex::from_u32(0);
7180            assert_eq!(
7181                instance.subtask_cancel(store, caller, async_, handle)?,
7182                Status::Returned as u32,
7183            );
7184            let err = instance
7185                .subtask_cancel(store, caller, async_, handle)
7186                .unwrap_err();
7187            assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7188            instance.subtask_drop(store, caller, handle)?;
7189            assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7190        }
7191        Ok(())
7192    }
7193
7194    #[test]
7195    fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7196        for (cancelled, status) in [
7197            (false, Status::Returned),
7198            (true, Status::Returned),
7199            (true, Status::ReturnCancelled),
7200        ] {
7201            for delivered in [false, true] {
7202                let event = if delivered {
7203                    None
7204                } else {
7205                    Some(Event::Subtask { status })
7206                };
7207                let (mut store, instance, task, handle) =
7208                    host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7209                let store = store.as_store_opaque();
7210                let result = instance.subtask_drop(
7211                    store,
7212                    RuntimeComponentInstanceIndex::from_u32(0),
7213                    handle,
7214                );
7215                if delivered {
7216                    result?;
7217                    assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7218                } else {
7219                    let err = result.unwrap_err();
7220                    assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7221                    assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7222                }
7223            }
7224        }
7225        Ok(())
7226    }
7227}