1use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57 HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70 CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71 InstanceState,
72};
73use crate::vm::{
74 AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98 CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99 MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100 RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101 TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102 TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113 Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114 FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115 StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136const BLOCKED: u32 = 0xffff_ffff;
139
140#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143 Starting = 0,
144 Started = 1,
145 Returned = 2,
146 StartCancelled = 3,
147 ReturnCancelled = 4,
148}
149
150impl Status {
151 pub fn pack(self, waitable: Option<u32>) -> u32 {
157 assert!(matches!(self, Status::Returned) == waitable.is_none());
158 let waitable = waitable.unwrap_or(0);
159 assert!(waitable < (1 << 28));
160 (waitable << 4) | (self as u32)
161 }
162}
163
164#[derive(Clone, Copy, Debug)]
167enum Event {
168 None,
169 Subtask {
170 status: Status,
171 },
172 StreamRead {
173 code: ReturnCode,
174 pending: Option<(TypeStreamTableIndex, u32)>,
175 },
176 StreamWrite {
177 code: ReturnCode,
178 pending: Option<(TypeStreamTableIndex, u32)>,
179 },
180 FutureRead {
181 code: ReturnCode,
182 pending: Option<(TypeFutureTableIndex, u32)>,
183 },
184 FutureWrite {
185 code: ReturnCode,
186 pending: Option<(TypeFutureTableIndex, u32)>,
187 },
188 Cancelled,
189}
190
191impl Event {
192 fn parts(self) -> (u32, u32) {
197 const EVENT_NONE: u32 = 0;
198 const EVENT_SUBTASK: u32 = 1;
199 const EVENT_STREAM_READ: u32 = 2;
200 const EVENT_STREAM_WRITE: u32 = 3;
201 const EVENT_FUTURE_READ: u32 = 4;
202 const EVENT_FUTURE_WRITE: u32 = 5;
203 const EVENT_CANCELLED: u32 = 6;
204 match self {
205 Event::None => (EVENT_NONE, 0),
206 Event::Cancelled => (EVENT_CANCELLED, 0),
207 Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208 Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209 Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210 Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211 Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212 }
213 }
214}
215
216mod callback_code {
218 pub const EXIT: u32 = 0;
219 pub const YIELD: u32 = 1;
220 pub const WAIT: u32 = 2;
221}
222
223const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233 store: StoreContextMut<'a, T>,
234 get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239 D: HasData + ?Sized,
240 T: 'static,
241{
242 pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244 Self { store, get_data }
245 }
246
247 pub fn data_mut(&mut self) -> &mut T {
249 self.store.data_mut()
250 }
251
252 pub fn get(&mut self) -> D::Data<'_> {
254 (self.get_data)(self.data_mut())
255 }
256
257 pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261 where
262 T: 'static,
263 {
264 let accessor = Accessor {
265 get_data: self.get_data,
266 token: StoreToken::new(self.store.as_context_mut()),
267 };
268 self.store
269 .as_context_mut()
270 .spawn_with_accessor(accessor, task)
271 }
272
273 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276 self.get_data
277 }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282 D: HasData + ?Sized,
283 T: 'static,
284{
285 type Data = T;
286
287 fn as_context(&self) -> StoreContext<'_, T> {
288 self.store.as_context()
289 }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294 D: HasData + ?Sized,
295 T: 'static,
296{
297 fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298 self.store.as_context_mut()
299 }
300}
301
302pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363 D: HasData + ?Sized,
364{
365 token: StoreToken<T>,
366 get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369pub trait AsAccessor {
386 type Data: 'static;
388
389 type AccessorData: HasData + ?Sized;
392
393 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398 type Data = T::Data;
399 type AccessorData = T::AccessorData;
400
401 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402 T::as_accessor(self)
403 }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407 type Data = T;
408 type AccessorData = D;
409
410 fn as_accessor(&self) -> &Accessor<T, D> {
411 self
412 }
413}
414
415const _: () = {
438 const fn assert<T: Send + Sync>() {}
439 assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443 pub(crate) fn new(token: StoreToken<T>) -> Self {
452 Self {
453 token,
454 get_data: |x| x,
455 }
456 }
457}
458
459impl<T, D> Accessor<T, D>
460where
461 D: HasData + ?Sized,
462{
463 pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481 tls::get(|vmstore| {
482 fun(Access {
483 store: self.token.as_context_mut(vmstore),
484 get_data: self.get_data,
485 })
486 })
487 }
488
489 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492 self.get_data
493 }
494
495 pub fn with_getter<D2: HasData>(
512 &self,
513 get_data: fn(&mut T) -> D2::Data<'_>,
514 ) -> Accessor<T, D2> {
515 Accessor {
516 token: self.token,
517 get_data,
518 }
519 }
520
521 pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537 where
538 T: 'static,
539 {
540 let accessor = self.clone_for_spawn();
541 self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542 }
543
544 fn clone_for_spawn(&self) -> Self {
545 Self {
546 token: self.token,
547 get_data: self.get_data,
548 }
549 }
550
551 pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587 self.with(|mut access| {
588 let store = access.as_context_mut().0;
589 let state = store.concurrent_state_mut_without_forcing_current_thread();
590 if state.interesting_tasks == 0 {
591 Poll::Ready(())
592 } else {
593 state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594 Poll::Pending
595 }
596 })
597 }
598
599 pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623 self.with(|mut access| {
624 let store = access.as_context_mut().0;
625
626 {
627 let state = store.concurrent_state_mut_without_forcing_current_thread();
628 if state.switch_item.is_some()
629 || !state.high_priority.is_empty()
630 || !state.low_priority.is_empty()
631 {
632 state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633 return Poll::Pending;
634 }
635 }
636
637 let (_, _, _, raw_options) = func.abi_info(store);
638 let instance = func.instance().runtime_instance(raw_options.instance);
639 let state = store.instance_state(instance).concurrent_state();
640 if state.backpressure == 0 {
641 Poll::Ready(())
642 } else {
643 store
644 .concurrent_state_mut_without_forcing_current_thread()
645 .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646 Poll::Pending
647 }
648 })
649 }
650}
651
652pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674 AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676 D: HasData + ?Sized,
677{
678 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684 T: 'static,
685 F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686 F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687 Fut: Future<Output = Result<()>> + Send + 'fut,
688 D: HasData,
689{
690 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691 (self)(accessor)
692 }
693}
694
695enum CallerInfo {
698 Async {
700 params: Vec<ValRaw>,
701 has_result: bool,
702 },
703 Sync {
705 params: Vec<ValRaw>,
706 result_count: u32,
707 },
708}
709
710enum WaitMode {
712 Fiber(StoreFiber<'static>),
714 Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721 match self {
722 Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723 Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724 }
725 }
726}
727
728#[derive(Debug)]
730enum SuspendReason {
731 Waiting {
734 set: TableId<WaitableSet>,
735 thread: QualifiedThreadId,
736 },
737 YieldingToSubtask { thread: QualifiedThreadId },
740 NeedWork,
743 Yielding { thread: QualifiedThreadId },
746 ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751enum GuestCallKind {
753 DeliverEvent {
756 instance: Instance,
758 set: Option<TableId<WaitableSet>>,
763 },
764 StartImplicit {
767 closure: Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>,
769 was_pending: bool,
771 },
772 StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
773}
774
775impl fmt::Debug for GuestCallKind {
776 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
777 match self {
778 Self::DeliverEvent { instance, set } => f
779 .debug_struct("DeliverEvent")
780 .field("instance", instance)
781 .field("set", set)
782 .finish(),
783 Self::StartImplicit { was_pending, .. } => f
784 .debug_struct("StartImplicit")
785 .field("was_pending", was_pending)
786 .finish(),
787 Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
788 }
789 }
790}
791
792#[derive(Copy, Clone, Debug)]
794pub enum SuspensionTarget {
795 Resume(u32),
796 Promote(u32),
797 None,
798}
799
800#[derive(Copy, Clone, Debug)]
802pub enum ResumeThread {
803 Promote,
804 Resume,
805 ResumeLater,
806}
807
808#[derive(Debug)]
810struct GuestCall {
811 thread: QualifiedThreadId,
812 kind: GuestCallKind,
813}
814
815impl GuestCall {
816 fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
828 let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
829 let async_typed = task.async_typed;
830 let async_lifted = task.async_lifted;
831 let has_callback = task.callback.is_some();
832 let instance = task.instance;
833 let state = store.instance_state(instance).concurrent_state();
834
835 let ready = match &self.kind {
836 GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
837 GuestCallKind::StartImplicit { was_pending, .. } => {
838 if async_typed {
839 let needs_exclusive = !async_lifted || has_callback;
842
843 let may_enter =
848 !((needs_exclusive && state.do_not_enter) || state.backpressure > 0);
849
850 let skip_wait = *was_pending || state.pending_start_count == 0;
857
858 may_enter && skip_wait
861 } else {
862 true
863 }
864 }
865 GuestCallKind::StartExplicit(_) => true,
866 };
867 log::trace!(
868 "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {}; pending_start_count: {})",
869 state.do_not_enter,
870 state.backpressure,
871 state.pending_start_count,
872 );
873 Ok(ready)
874 }
875}
876
877enum WorkerItem {
879 GuestCall(GuestCall),
880 Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
881}
882
883enum WorkItem {
886 PushFuture(AlwaysMut<HostTaskFuture>),
888 ResumeFiber {
890 instance: RuntimeInstance,
891 thread: QualifiedThreadId,
892 fiber: StoreFiber<'static>,
893 },
894 ResumeThread {
896 instance: RuntimeInstance,
897 thread: QualifiedThreadId,
898 },
899 GuestCall {
901 instance: RuntimeInstance,
902 call: GuestCall,
903 },
904 WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
906}
907
908struct ResumingFiber<'a> {
914 store: &'a mut StoreOpaque,
915 fiber: Option<StoreFiber<'static>>,
916}
917
918impl Drop for ResumingFiber<'_> {
919 fn drop(&mut self) {
920 if let Some(mut fiber) = self.fiber.take() {
921 fiber.dispose(self.store);
922 }
923 }
924}
925
926impl fmt::Debug for WorkItem {
927 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
928 match self {
929 Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
930 Self::ResumeFiber {
931 instance, thread, ..
932 } => f
933 .debug_struct("ResumeFiber")
934 .field("instance", instance)
935 .field("thread", thread)
936 .finish(),
937 Self::ResumeThread { instance, thread } => f
938 .debug_struct("ResumeThread")
939 .field("instance", instance)
940 .field("thread", thread)
941 .finish(),
942 Self::GuestCall { instance, call } => f
943 .debug_struct("GuestCall")
944 .field("instance", instance)
945 .field("call", call)
946 .finish(),
947 Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
948 }
949 }
950}
951
952#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
954pub(crate) enum WaitResult {
955 Cancelled,
956 Completed,
957}
958
959pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
967 store: &mut dyn VMStore,
968 host_task: EnteredHostTask,
969 future: impl Future<Output = Result<R>> + Send + 'static,
970) -> Result<R> {
971 let mut future = Box::pin(future);
978 let poll = tls::set(store, || {
979 future
980 .as_mut()
981 .poll(&mut Context::from_waker(&Waker::noop()))
982 });
983
984 let caller = match host_task {
985 Some(caller) => caller,
986 None => bail_bug!("host task wasn't created but should have been"),
987 };
988
989 let task = match poll {
990 Poll::Ready(result) => return result,
992
993 Poll::Pending => {
998 let Some(task) = store.materialize_host_task_id()? else {
999 bail_bug!("current thread is not a host thread")
1000 };
1001
1002 let future = Box::pin(async move {
1005 let result = run_with_host_task_set(task, future).await??;
1006 tls::get(move |store| {
1007 let state = store.concurrent_state_mut()?;
1008 let host_state = &mut state.get_mut(task)?.state;
1009 assert!(matches!(host_state, HostTaskState::CalleeStarted));
1010 *host_state = HostTaskState::CalleeFinished(Box::new(result));
1011
1012 Waitable::Host(task).set_event(
1013 state,
1014 Some(Event::Subtask {
1015 status: Status::Returned,
1016 }),
1017 )?;
1018
1019 Ok(())
1020 })
1021 }) as HostTaskFuture;
1022
1023 let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
1024 store.switch_or_trap_if_may_not_suspend(caller_instance)?;
1025
1026 let state = store.concurrent_state_mut()?;
1027 state.push_future(future);
1028
1029 let set = state.get_mut(caller.thread)?.sync_call_set;
1030 Waitable::Host(task).join(state, Some(set))?;
1031
1032 store.suspend(SuspendReason::Waiting {
1033 set,
1034 thread: caller,
1035 })?;
1036
1037 Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1041 task
1042 }
1043 };
1044
1045 let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1047 match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1048 HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1049 Ok(result) => *result,
1050 Err(_) => bail_bug!("host task finished with wrong type of result"),
1051 }),
1052 _ => bail_bug!("unexpected host task state after completion"),
1053 }
1054}
1055
1056fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1058 match call.kind {
1059 GuestCallKind::DeliverEvent { instance, set } => {
1060 if let Some(set) = set {
1063 store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1064 }
1065 let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1066 Some(pair) => pair,
1067 None => match set {
1068 Some(set) => {
1073 log::trace!(
1074 "event for {:?} on {set:?} no longer present; waiting again",
1075 call.thread
1076 );
1077 return instance.wait_with_callback(&mut **store, call.thread, set);
1078 }
1079 None => (Event::None, None),
1084 },
1085 };
1086 let state = store.concurrent_state_mut()?;
1087 if let Some(set) = set
1092 && waitable.is_none()
1093 && !state.get_mut(set)?.ready.is_empty()
1094 {
1095 state.wake_waiter(set)?;
1096 }
1097 let task = state.get_mut(call.thread.task)?;
1098 let runtime_instance = task.instance;
1099 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1100
1101 log::trace!(
1102 "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1103 call.thread,
1104 );
1105
1106 instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1107 invoke_callback(store, call.thread.task, event, handle)
1108 })?;
1109 }
1110 GuestCallKind::StartImplicit { closure, .. } => {
1111 closure(store)?;
1112 }
1113 GuestCallKind::StartExplicit(fun) => {
1114 fun(store)?;
1115 }
1116 }
1117
1118 Ok(())
1119}
1120
1121fn invoke_callback(
1124 store: &mut dyn VMStore,
1125 task: TableId<GuestTask>,
1126 event: Event,
1127 handle: u32,
1128) -> Result<u32> {
1129 let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1130 bail_bug!("guest task callback field not present")
1131 };
1132
1133 let code = callback(store, event, handle)?;
1134
1135 store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1136
1137 Ok(code)
1138}
1139
1140impl<T> Store<T> {
1141 pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1143 where
1144 T: Send + 'static,
1145 {
1146 ensure!(
1147 self.as_context().0.concurrency_support(),
1148 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1149 );
1150 self.as_context_mut().run_concurrent(fun).await
1151 }
1152
1153 #[doc(hidden)]
1154 pub fn assert_concurrent_state_empty(&mut self) {
1155 self.as_context_mut().assert_concurrent_state_empty();
1156 }
1157
1158 #[doc(hidden)]
1159 pub fn concurrent_state_table_size(&mut self) -> usize {
1160 self.as_context_mut().concurrent_state_table_size()
1161 }
1162
1163 #[doc(hidden)]
1164 pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1165 self.as_context_mut()
1166 .set_max_concurrent_state_table_capacity(max)
1167 }
1168
1169 pub fn spawn(
1171 &mut self,
1172 task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1173 ) -> Result<JoinHandle>
1174 where
1175 T: 'static,
1176 {
1177 self.as_context_mut().spawn(task)
1178 }
1179}
1180
1181impl<T> StoreContextMut<'_, T> {
1182 #[doc(hidden)]
1193 pub fn assert_concurrent_state_empty(self) {
1194 let store = self.0;
1195 store
1196 .store_data_mut()
1197 .components
1198 .assert_instance_states_empty();
1199 let state = store.concurrent_state_mut().unwrap();
1200 assert!(
1201 state.table.get_mut().is_empty(),
1202 "non-empty table: {:?}",
1203 state.table.get_mut()
1204 );
1205 assert!(state.switch_item.is_none());
1206 assert!(state.next_switch_item.is_none());
1207 assert!(state.high_priority.is_empty());
1208 assert!(state.low_priority.is_empty());
1209 assert!(state.saved_next_switch_items.is_empty());
1210 assert!(state.unforced_current_thread.is_none());
1211 assert!(state.deferred_host_call_context.is_none());
1212 assert!(state.futures_mut().unwrap().is_empty());
1213 assert!(state.global_error_context_ref_counts.is_empty());
1214 }
1215
1216 #[doc(hidden)]
1221 pub fn concurrent_state_table_size(&mut self) -> usize {
1222 self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1223 }
1224
1225 #[doc(hidden)]
1231 pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1232 self.0
1233 .concurrent_state_mut_without_forcing_current_thread()
1234 .table
1235 .get_mut()
1236 .set_max_capacity(max);
1237 }
1238
1239 pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1249 where
1250 T: 'static,
1251 {
1252 let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1253 self.spawn_with_accessor(accessor, task)
1254 }
1255
1256 fn spawn_with_accessor<D>(
1259 self,
1260 accessor: Accessor<T, D>,
1261 task: impl for<'fut> AccessorTask<'fut, T, D>,
1262 ) -> Result<JoinHandle>
1263 where
1264 T: 'static,
1265 D: HasData + ?Sized,
1266 {
1267 let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1271 self.0
1272 .concurrent_state_mut()?
1273 .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1274 Ok(handle)
1275 }
1276
1277 pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1361 where
1362 T: Send + 'static,
1363 {
1364 ensure!(
1365 self.0.concurrency_support(),
1366 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1367 );
1368 self.do_run_concurrent(fun, false).await
1369 }
1370
1371 pub(super) async fn run_concurrent_trap_on_idle<R>(
1372 self,
1373 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1374 ) -> Result<R> {
1375 self.do_run_concurrent(fun, true).await
1376 }
1377
1378 async fn do_run_concurrent<R>(
1379 mut self,
1380 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1381 trap_on_idle: bool,
1382 ) -> Result<R> {
1383 debug_assert!(self.0.concurrency_support());
1384 self.0.check_event_loop_not_running()?;
1385
1386 self.0.concurrent_state_mut()?.materialize_host_task()?;
1391
1392 let token = StoreToken::new(self.as_context_mut());
1393
1394 struct Dropper<'a, T: 'static, V> {
1395 store: StoreContextMut<'a, T>,
1396 value: ManuallyDrop<V>,
1397 }
1398
1399 impl<'a, T, V> Drop for Dropper<'a, T, V> {
1400 fn drop(&mut self) {
1401 self.store
1402 .0
1403 .concurrent_state_mut_already_forced_current_thread()
1404 .event_loop_running = false;
1405
1406 tls::set(self.store.0, || {
1407 unsafe { ManuallyDrop::drop(&mut self.value) }
1412 });
1413 }
1414 }
1415
1416 let accessor = &Accessor::new(token);
1417 self.0
1418 .concurrent_state_mut_already_forced_current_thread()
1419 .event_loop_running = true;
1420 let dropper = &mut Dropper {
1421 store: self,
1422 value: ManuallyDrop::new(fun(accessor)),
1423 };
1424 let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1426
1427 let result = dropper
1428 .store
1429 .as_context_mut()
1430 .poll_until(future, trap_on_idle)
1431 .await;
1432
1433 if result.is_err() {
1434 dropper.store.0.set_trapped();
1435 }
1436
1437 result
1438 }
1439
1440 async fn poll_until<R>(
1446 mut self,
1447 mut future: Pin<&mut impl Future<Output = R>>,
1448 trap_on_idle: bool,
1449 ) -> Result<R> {
1450 struct Reset<'a, T: 'static> {
1451 store: StoreContextMut<'a, T>,
1452 futures: Option<FuturesUnordered<HostTaskFuture>>,
1453 }
1454
1455 impl<'a, T> Drop for Reset<'a, T> {
1456 fn drop(&mut self) {
1457 if let Some(futures) = self.futures.take() {
1458 *self
1459 .store
1460 .0
1461 .concurrent_state_mut_already_forced_current_thread()
1462 .futures
1463 .get_mut() = Some(futures);
1464 }
1465 }
1466 }
1467
1468 const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1472 let mut turns_without_yield = 0;
1473
1474 loop {
1475 let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1479 let mut reset = Reset {
1480 store: self.as_context_mut(),
1481 futures,
1482 };
1483 let mut next = match reset.futures.as_mut() {
1484 Some(f) => pin!(f.next()),
1485 None => bail_bug!("concurrent state missing futures field"),
1486 };
1487
1488 enum PollResult<R> {
1489 Complete(R),
1490 ProcessWork {
1491 ready: Option<WorkItem>,
1492 low_priority: bool,
1493 },
1494 }
1495
1496 let result = future::poll_fn(|cx| {
1497 if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1500 return Poll::Ready(Ok(PollResult::Complete(value)));
1501 }
1502
1503 if reset.store.0.component_trapped() {
1512 return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1513 }
1514
1515 let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1519 Poll::Ready(Some(output)) => {
1520 match output {
1521 Err(e) => return Poll::Ready(Err(e)),
1522 Ok(()) => {}
1523 }
1524 Poll::Ready(true)
1525 }
1526 Poll::Ready(None) => Poll::Ready(false),
1527 Poll::Pending => Poll::Pending,
1528 };
1529
1530 let state = reset.store.0.concurrent_state_mut()?;
1545 let mut ready = state.switch_item.take();
1546 let mut low_priority = false;
1547 if ready.is_none() {
1548 ready = state.high_priority.pop_back();
1549 if ready.is_none() {
1550 ready = state.low_priority.pop_back();
1551 low_priority = true;
1552 }
1553 }
1554 if ready.is_some() {
1555 return Poll::Ready(Ok(PollResult::ProcessWork {
1556 ready,
1557 low_priority,
1558 }));
1559 }
1560
1561 if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1562 waker.wake();
1563 }
1564
1565 return match next {
1569 Poll::Ready(true) => {
1570 Poll::Ready(Ok(PollResult::ProcessWork {
1576 ready: None,
1577 low_priority: false,
1578 }))
1579 }
1580 Poll::Ready(false) => {
1581 if let Poll::Ready(value) =
1585 tls::set(reset.store.0, || future.as_mut().poll(cx))
1586 {
1587 Poll::Ready(Ok(PollResult::Complete(value)))
1588 } else {
1589 if trap_on_idle {
1595 Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1602 Trap::CannotBlockSyncTask.into()
1603 } else {
1604 Trap::AsyncDeadlock.into()
1606 }))
1607 } else {
1608 Poll::Pending
1612 }
1613 }
1614 }
1615 Poll::Pending => Poll::Pending,
1620 };
1621 })
1622 .await;
1623
1624 drop(reset);
1628
1629 match result? {
1630 PollResult::Complete(value) => break Ok(value),
1633 PollResult::ProcessWork {
1636 mut ready,
1637 low_priority,
1638 } => {
1639 if low_priority {
1667 if let Some(item) = ready.take() {
1668 self.0.concurrent_state_mut()?.low_priority.push_back(item);
1669 }
1670 self.0.yield_now().await;
1671 turns_without_yield = 0;
1672 ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1673 }
1674
1675 if let Some(item) = ready {
1676 self.as_context_mut().handle_work_item(item).await?;
1677 }
1678
1679 turns_without_yield += 1;
1680 if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1681 turns_without_yield = 0;
1682 self.0.yield_now().await;
1683 }
1684 }
1685 }
1686 }
1687 }
1688
1689 fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1691 let item = match item {
1695 WorkItem::ResumeFiber {
1696 instance,
1697 thread,
1698 fiber,
1699 } => {
1700 let future = self.0.resume_fiber(fiber);
1701 log::trace!(
1702 "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1703 );
1704 return futures::future::Either::Left(future);
1705 }
1706 item => item,
1707 };
1708 futures::future::Either::Right(async move {
1709 log::trace!("handle work item {item:?}");
1710 match item {
1711 WorkItem::PushFuture(future) => {
1712 self.0
1713 .concurrent_state_mut()?
1714 .futures_mut()?
1715 .push(future.into_inner());
1716 }
1717 WorkItem::ResumeFiber { .. } => unreachable!(),
1718 WorkItem::ResumeThread { thread, .. } => {
1719 let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1720 if !matches!(slot, GuestThreadState::Ready { .. }) {
1721 bail_bug!("cannot resume non-pending thread {thread:?}");
1722 }
1723 let GuestThreadState::Ready { fiber } =
1724 mem::replace(slot, GuestThreadState::Running)
1725 else {
1726 unreachable!()
1727 };
1728 self.0.resume_fiber(fiber).await?;
1729 }
1730 WorkItem::GuestCall { mut call, .. } => {
1731 if call.is_ready(self.0)? {
1732 if let GuestCallKind::StartImplicit {
1733 was_pending: true, ..
1734 } = &call.kind
1735 {
1736 let instance = self
1737 .0
1738 .concurrent_state_mut()?
1739 .get_mut(call.thread.task)?
1740 .instance;
1741
1742 self.0.decrement_pending_start_count(instance, 1)?;
1743 }
1744
1745 self.0
1746 .concurrent_state_mut()?
1747 .get_mut(call.thread.thread)?
1748 .wake_on_cancel = WakeOnCancel::None;
1749
1750 self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1751 } else {
1752 let state = self.0.concurrent_state_mut()?;
1753 let task = state.get_mut(call.thread.task)?;
1754 if !task.starting_sent {
1755 task.starting_sent = true;
1756 if let GuestCallKind::StartImplicit { .. } = &call.kind {
1757 Waitable::Guest(call.thread.task).set_event(
1758 state,
1759 Some(Event::Subtask {
1760 status: Status::Starting,
1761 }),
1762 )?;
1763 }
1764 }
1765
1766 let increment = if let GuestCallKind::StartImplicit {
1767 was_pending, ..
1768 } = &mut call.kind
1769 {
1770 let increment = !*was_pending;
1771 *was_pending = true;
1772 increment
1773 } else {
1774 false
1775 };
1776
1777 let instance = state.get_mut(call.thread.task)?.instance;
1778 if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1787 if !state.get_mut(*set)?.ready.is_empty() {
1788 state.wake_waiter(*set)?;
1789 }
1790 }
1791
1792 state.take_next_switch_item()?;
1796
1797 let state = self.0.instance_state(instance).concurrent_state();
1798
1799 state.pending.insert(call.thread, call.kind);
1800
1801 if increment {
1802 state.pending_start_count += 1;
1803 }
1804 }
1805 }
1806 WorkItem::WorkerFunction(fun) => {
1807 self.run_on_worker(WorkerItem::Function(fun)).await?;
1808 }
1809 }
1810
1811 Ok(())
1812 })
1813 }
1814
1815 async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1817 let state = self.0.concurrent_state_mut()?;
1818 if state.worker_item.is_some() {
1819 bail_bug!("worker item already set when taking worker fiber");
1820 }
1821 let worker = if let Some(fiber) = state.worker.take() {
1822 fiber
1823 } else {
1824 unsafe {
1843 fiber::make_fiber_unchecked(self.0, move |store| {
1844 loop {
1845 let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1846 bail_bug!("worker_item not present when resuming fiber")
1847 };
1848 match item {
1849 WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1850 WorkerItem::Function(fun) => fun.into_inner()(store)?,
1851 }
1852
1853 store.suspend(SuspendReason::NeedWork)?;
1854 }
1855 })?
1856 }
1857 };
1858
1859 self.0
1862 .concurrent_state_mut_already_forced_current_thread()
1863 .worker_item = Some(item);
1864
1865 self.0.resume_fiber(worker).await
1866 }
1867
1868 pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1873 where
1874 T: 'static,
1875 F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1876 + Send
1877 + Sync
1878 + 'static,
1879 R: Send + Sync + 'static,
1880 {
1881 let token = StoreToken::new(self);
1882 async move {
1883 let mut accessor = Accessor::new(token);
1884 closure(&mut accessor).await
1885 }
1886 }
1887
1888 pub(crate) async fn start_instance(
1889 &mut self,
1890 instance: ModuleInstance,
1891 callee: Option<RuntimeInstance>,
1892 ) -> Result<ModuleInstance> {
1893 self.0.check_event_loop_not_running()?;
1895 let (tx, rx) = oneshot::channel();
1896 let token = StoreToken::new(self.as_context_mut());
1897 self.0.queue_task(move |store| {
1898 _ = tx.send(
1899 super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1900 .map(|()| instance),
1901 );
1902 Ok(())
1903 })?;
1904 self.as_context_mut()
1905 .run_concurrent_trap_on_idle(async |_| {
1906 rx.await
1907 .map_err(|_| format_err!("oneshot channel canceled"))
1908 })
1909 .await??
1910 }
1911}
1912
1913pub type EnteredHostTask = Option<QualifiedThreadId>;
1920
1921impl StoreOpaque {
1922 pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1923 if self
1924 .concurrent_state_mut_already_forced_current_thread()
1925 .event_loop_running
1926 {
1927 bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1928 }
1929 Ok(())
1930 }
1931
1932 #[inline]
1936 pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1937 if !self.concurrency_support() {
1939 return Ok(CurrentThread::None);
1940 }
1941
1942 if !self
1945 .vm_store_context_mut()
1946 .current_thread_mut()
1947 .is_deferred()
1948 {
1949 return Ok(self
1950 .concurrent_state_mut_already_forced_current_thread()
1951 .unforced_current_thread);
1952 }
1953
1954 self.force_deferred_current_thread()
1955 }
1956
1957 #[cold]
1960 fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1961 let state = self.concurrent_state_mut_without_forcing_current_thread();
1970 let id = match state.unforced_current_thread.guest_task() {
1971 Some(task) => state.get_mut(task)?.instance.instance,
1972 None => bail_bug!("deferred component-model thread with non-guest base"),
1973 };
1974
1975 let mut frames = SmallVec::<[_; 8]>::new();
1978 let mut cur = *self.vm_store_context_mut().current_thread_mut();
1979 while let Some(ptr) = cur.as_deferred() {
1980 let deferred = unsafe { ptr.as_non_null().as_ref() };
1985 frames.push((
1986 deferred.callee_async != 0,
1987 deferred.callee_instance,
1988 deferred.saved_context,
1989 deferred.lazy_lenders_start,
1990 ));
1991 cur = deferred.parent;
1992 }
1993
1994 *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1998
1999 let current_context = *self.vm_store_context_mut().component_context_mut();
2002
2003 let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
2010 frames.reverse();
2011 let lazy_lenders_base = match frames.first() {
2012 Some((_, _, _, start)) => *start,
2013 None => bail_bug!("forcing a deferred thread with no deferred frames"),
2014 };
2015 let lazy_lender_ends = frames
2016 .iter()
2017 .skip(1)
2018 .map(|(_, _, _, start)| *start)
2019 .chain([lazy_lenders_end])
2020 .collect::<SmallVec<[_; 8]>>();
2021 for ((callee_async, callee_instance, saved_context, start), lenders_end) in
2022 frames.into_iter().zip(lazy_lender_ends)
2023 {
2024 *self.vm_store_context_mut().component_context_mut() = saved_context;
2028 let callee = RuntimeInstance {
2029 instance: id,
2030 index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
2031 };
2032 self.enter_guest_sync_call(callee_async, callee)?;
2033
2034 self.promote_current_lazy_lends(id, start, lenders_end)?;
2037 }
2038
2039 debug_assert_eq!(
2042 *self.vm_store_context_mut().lazy_lenders_len_mut(),
2043 lazy_lenders_end
2044 );
2045 *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
2046
2047 *self.vm_store_context_mut().component_context_mut() = current_context;
2049
2050 Ok(self
2051 .concurrent_state_mut_without_forcing_current_thread()
2052 .unforced_current_thread)
2053 }
2054
2055 fn promote_current_lazy_lends(
2061 &mut self,
2062 id: ComponentInstanceId,
2063 start: usize,
2064 end: usize,
2065 ) -> Result<()> {
2066 if start == end {
2067 return Ok(());
2068 }
2069
2070 let scope = match self
2071 .concurrent_state_mut_without_forcing_current_thread()
2072 .unforced_current_thread
2073 .guest_task()
2074 {
2075 Some(task) => VMScope::id(task.rep()),
2076 None => bail_bug!("replayed deferred frame is not a guest thread"),
2077 };
2078 let instance = Instance::from_wasmtime(self, id);
2079 self.promote_lazy_lends(instance, scope, start, end)
2080 }
2081
2082 fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2083 match self.current_thread()?.guest() {
2084 Some(id) => Ok(*id),
2085 None => bail_bug!("current thread is not a guest thread"),
2086 }
2087 }
2088
2089 pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2093 match self.current_thread()? {
2094 CurrentThread::Host(id) => Ok(Some(id)),
2095 CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2096 _ => bail_bug!("current thread is not a host thread"),
2097 }
2098 }
2099
2100 fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2103 Ok(self
2104 .concurrent_state_mut()?
2105 .materialize_current_host_task_id()?)
2106 }
2107
2108 fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2109 log::trace!("enter sync-typed call {callee:?}");
2110 let state = self.instance_state(callee).concurrent_state();
2111 let old_do_not_suspend = state.do_not_suspend;
2112 state.do_not_suspend = true;
2113
2114 let thread = self.current_guest_thread()?;
2115 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2116 if thread.old_do_not_suspend.is_some() {
2117 bail_bug!("current thread already has `old_do_not_suspend` value");
2118 }
2119
2120 thread.old_do_not_suspend = Some(old_do_not_suspend);
2121
2122 Ok(())
2123 }
2124
2125 fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2126 log::trace!("exit sync-typed call {callee:?}");
2127 let thread = self.current_guest_thread()?;
2128 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2129 let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2130 bail_bug!("current thread missing `old_do_not_suspend` value");
2131 };
2132 let state = self.instance_state(callee).concurrent_state();
2133 state.do_not_suspend = old_do_not_suspend;
2134 Ok(())
2135 }
2136
2137 pub(crate) fn enter_guest_sync_call(
2149 &mut self,
2150 callee_async_typed: bool,
2151 callee: RuntimeInstance,
2152 ) -> Result<()> {
2153 log::trace!("enter sync-lifted call {callee:?}");
2154 if !self.concurrency_support() {
2155 return self.enter_call_not_concurrent();
2156 }
2157
2158 let state = self.concurrent_state_mut()?;
2162 let item = state.next_switch_item.take();
2163 state.saved_next_switch_items.push(item);
2164
2165 let thread = self.current_thread()?;
2166 let caller = if let Some(thread) = thread.guest() {
2167 Caller::Guest { thread: *thread }
2168 } else {
2169 Caller::Host {
2170 tx: None,
2171 caller: self.materialize_host_task_id()?,
2172 }
2173 };
2174
2175 let state = self.concurrent_state_mut()?;
2176 let guest_thread = GuestTask::new(
2177 state,
2178 Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2179 LiftResult {
2180 lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2181 ty: TypeTupleIndex::reserved_value(),
2182 memory: None,
2183 string_encoding: StringEncoding::Utf8,
2184 },
2185 caller,
2186 None,
2187 callee,
2188 callee_async_typed,
2189 false,
2190 )?;
2191
2192 Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2193 guest_thread.thread,
2194 self,
2195 callee.index,
2196 )?;
2197 self.set_thread(guest_thread)?;
2198
2199 if !callee_async_typed {
2200 self.enter_sync_call(callee)?;
2201 }
2202
2203 Ok(())
2204 }
2205
2206 pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2214 if !self.concurrency_support() {
2215 return Ok(self.exit_call_not_concurrent());
2216 }
2217
2218 let thread = match self.current_thread()?.guest() {
2219 Some(t) => *t,
2220 None => bail_bug!("expected task when exiting"),
2221 };
2222 let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2223 let instance = task.instance;
2224
2225 let caller = match &task.caller {
2226 &Caller::Guest { thread } => thread.into(),
2227 &Caller::Host { caller, .. } => caller
2228 .map(CurrentThread::Host)
2229 .unwrap_or(CurrentThread::None),
2230 };
2231 task.lift_result = None;
2232 task.exited = true;
2233 let async_typed = task.async_typed;
2234
2235 if !async_typed {
2236 self.exit_sync_call(instance)?;
2237 }
2238
2239 self.set_thread(caller)?;
2240
2241 log::trace!("exit sync-lifted call {instance:?}");
2242
2243 if async_typed {
2244 self.switch_or_trap_if_may_not_suspend(instance)?;
2249 }
2250
2251 self.cleanup_thread(thread, instance)?;
2252
2253 self.concurrent_state_mut()?
2254 .decrement_task_ref_count(thread.task)?;
2255
2256 let state = self.concurrent_state_mut()?;
2257 let Some(item) = state.saved_next_switch_items.pop() else {
2258 bail_bug!("unable to pop from `saved_next_switch_items`");
2259 };
2260 if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2261 state.push_high_priority(item);
2264 bail_bug!("`next_switch_item` unexpectedly already set");
2265 }
2266
2267 Ok(())
2268 }
2269
2270 pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2277 if !self.concurrency_support() {
2278 self.enter_call_not_concurrent()?;
2279 return Ok(None);
2280 }
2281 let caller = self.current_guest_thread()?;
2282 log::trace!("new deferred host task with caller {caller:?}");
2283
2284 self.set_thread(CurrentThread::DeferredHost(caller))?;
2285 let state = self.concurrent_state_mut()?;
2286 debug_assert!(state.deferred_host_call_context.is_none());
2287 state.deferred_host_call_context = Some(CallContext::default());
2288 state.debug_assert_deferred_host_invariant();
2289 Ok(Some(caller))
2290 }
2291
2292 pub(crate) fn host_task_delete(
2299 &mut self,
2300 original_task: EnteredHostTask,
2301 materialized_task: Option<TableId<HostTask>>,
2302 ) -> Result<()> {
2303 match original_task {
2304 Some(caller) => {
2305 self.set_thread(caller)?;
2306 if materialized_task.is_none() {
2307 let state = self.concurrent_state_mut()?;
2308 let context = state
2309 .deferred_host_call_context
2310 .take()
2311 .expect("deferred host call context should be present");
2312 debug_assert!(context.is_empty());
2313 state.debug_assert_deferred_host_invariant();
2314 }
2315 log::trace!(
2316 "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2317 );
2318 if let Some(task) = materialized_task {
2319 self.concurrent_state_mut()?.delete_host_task(task)?;
2320 }
2321 }
2322 None => {
2323 debug_assert!(materialized_task.is_none());
2324 self.exit_call_not_concurrent();
2325 }
2326 }
2327 Ok(())
2328 }
2329
2330 fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2333 self.component_instance_mut(instance.instance)
2334 .instance_state(instance.index)
2335 }
2336
2337 fn instance_state_and_handle_counter(
2340 &mut self,
2341 instance: RuntimeInstance,
2342 ) -> (&mut InstanceState, &mut HandleCounter) {
2343 let (component, counter) = self
2344 .store_data_mut()
2345 .component_instance_and_handle_counter_mut(instance.instance);
2346 (component.instance_state(instance.index), counter)
2347 }
2348
2349 fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2351 let (component, counter) = self
2352 .store_data_mut()
2353 .component_instance_and_handle_counter_mut(instance.instance);
2354 component.handle_table_insertion(instance.index, counter)
2355 }
2356
2357 pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2363 let thread = thread.into();
2364 let state = self.concurrent_state_mut()?;
2365 state.debug_assert_deferred_host_invariant();
2366 let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2367
2368 state.handle_thread_switch(old_thread, thread)?;
2369
2370 if let Some(old_thread) = old_thread.guest() {
2378 let old_context = *self.vm_store_context_mut().component_context_mut();
2379 self.concurrent_state_mut()?
2380 .get_mut(old_thread.thread)?
2381 .context = old_context;
2382 }
2383 if cfg!(debug_assertions) {
2384 *self.vm_store_context_mut().component_context_mut() =
2385 [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2386 }
2387 if let Some(thread) = thread.guest() {
2388 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2389 let context = thread.context;
2390 if cfg!(debug_assertions) {
2391 thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2392 }
2393 *self.vm_store_context_mut().component_context_mut() = context;
2394 }
2395
2396 *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2398 VMLazyThread::none()
2399 } else {
2400 VMLazyThread::forced()
2401 };
2402
2403 Ok(old_thread)
2404 }
2405
2406 fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2408 if self.switch_if_may_not_suspend(instance)? {
2409 Ok(())
2410 } else {
2411 Err(Trap::CannotBlockSyncTask.into())
2412 }
2413 }
2414
2415 fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2419 self.concurrent_state_mut()?;
2423
2424 Ok(!self.concurrency_support()
2425 || !self
2426 .instance_state(instance)
2427 .concurrent_state()
2428 .do_not_suspend
2429 || self
2430 .concurrent_state_mut()?
2431 .promote_instance_local_thread_work_item(instance)?)
2432 }
2433
2434 fn enter_instance(&mut self, instance: RuntimeInstance) {
2438 log::trace!("enter {instance:?}");
2439 self.instance_state(instance)
2440 .concurrent_state()
2441 .do_not_enter = true;
2442 }
2443
2444 fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2448 log::trace!("exit {instance:?}");
2449 self.instance_state(instance)
2450 .concurrent_state()
2451 .do_not_enter = false;
2452 self.partition_pending(instance)
2453 }
2454
2455 fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2463 let mut any_ready = false;
2464 for (thread, kind) in
2465 mem::take(&mut self.instance_state(instance).concurrent_state().pending)
2466 {
2467 let call = GuestCall { thread, kind };
2468 if call.is_ready(self)? {
2469 any_ready = true;
2470 self.concurrent_state_mut()?
2471 .push_high_priority(WorkItem::GuestCall { instance, call });
2472 } else {
2473 self.instance_state(instance)
2474 .concurrent_state()
2475 .pending
2476 .insert(call.thread, call.kind);
2477 }
2478 }
2479
2480 if any_ready {
2481 if let Some(waker) = self
2482 .concurrent_state_mut()?
2483 .ready_for_concurrent_call_waker
2484 .take()
2485 {
2486 waker.wake();
2487 }
2488 }
2489
2490 Ok(())
2491 }
2492
2493 pub(crate) fn backpressure_modify(
2495 &mut self,
2496 caller_instance: RuntimeInstance,
2497 modify: impl FnOnce(u16) -> Option<u16>,
2498 ) -> Result<()> {
2499 let state = self.instance_state(caller_instance).concurrent_state();
2500 let old = state.backpressure;
2501 let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2502 state.backpressure = new;
2503
2504 log::trace!("backpressure update for {caller_instance:?} from {old} to {new}");
2505
2506 if old > 0 && new == 0 {
2507 self.partition_pending(caller_instance)?;
2510 }
2511
2512 Ok(())
2513 }
2514
2515 fn resume_fiber(
2518 &mut self,
2519 fiber: StoreFiber<'static>,
2520 ) -> impl Future<Output = Result<()>> + '_ {
2521 let mut resuming = ResumingFiber {
2522 store: self,
2523 fiber: Some(fiber),
2524 };
2525 async move {
2526 let old_thread = resuming.store.current_thread()?;
2527 log::trace!("resume_fiber: save current thread {old_thread:?}");
2528
2529 let fiber = resuming.fiber.take().unwrap();
2530 resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2531
2532 resuming.store.set_thread(old_thread)?;
2533
2534 let state = resuming.store.concurrent_state_mut()?;
2535
2536 if let Some(ot) = old_thread.guest() {
2537 state.set_thread_running(ot.thread)?;
2538 }
2539 log::trace!("resume_fiber: restore current thread {old_thread:?}");
2540
2541 if resuming.fiber.is_some() {
2542 log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2543 let reason = match state.suspend_reason.take() {
2545 Some(r) => r,
2546 None => bail_bug!("suspend reason missing when resuming fiber"),
2547 };
2548 match reason {
2549 SuspendReason::NeedWork => {
2550 if state.worker.is_none() {
2551 state.worker = resuming.fiber.take();
2552 }
2553 }
2554 SuspendReason::Yielding { thread } => {
2555 let slot = &mut state.get_mut(thread.thread)?.state;
2556 slot.check_no_fiber()?;
2557 *slot = GuestThreadState::Ready {
2558 fiber: resuming.fiber.take().unwrap(),
2559 };
2560 let instance = state.get_mut(thread.task)?.instance;
2561 state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2562 }
2563 SuspendReason::ExplicitlySuspending { thread } => {
2564 let slot = &mut state.get_mut(thread.thread)?.state;
2565 slot.check_no_fiber()?;
2566 *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2567 }
2568 SuspendReason::Waiting { set, thread } => {
2569 match state.get_mut(set)?.waiting.entry(thread) {
2570 Entry::Vacant(entry) => {
2571 entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2572 }
2573 Entry::Occupied(_) => {
2574 bail_bug!("entry unexpectedly already exists for {thread:?}");
2575 }
2576 }
2577 }
2578 SuspendReason::YieldingToSubtask { thread } => {
2579 let instance = state.get_mut(thread.task)?.instance;
2588 if state.next_switch_item.is_some() {
2589 bail_bug!(
2592 "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2593 a thread wanted to wait on a subtask"
2594 );
2595 }
2596 let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2597 instance,
2598 thread,
2599 fiber: resuming.fiber.take().unwrap(),
2600 });
2601 log::trace!("set next switch item to {item:?}");
2602 }
2603 };
2604 } else {
2605 log::trace!("resume_fiber: fiber has exited");
2606 }
2607
2608 Ok(())
2609 }
2610 }
2611
2612 fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2618 log::trace!("suspend fiber: {reason:?}");
2619
2620 if !self.fiber_async_state_mut().can_block() {
2623 return Err(format_err!("future dropped"));
2624 }
2625
2626 let state = self.concurrent_state_mut()?;
2627
2628 let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2635 SuspendReason::Yielding { .. }
2636 | SuspendReason::Waiting { .. }
2637 | SuspendReason::ExplicitlySuspending { .. } => {
2638 if state.switch_item.is_none() {
2641 state.take_next_switch_item()?;
2642 }
2643
2644 (true, false)
2645 }
2646 SuspendReason::YieldingToSubtask { .. } => (true, true),
2647 SuspendReason::NeedWork => (false, false),
2648 };
2649
2650 let old_next_switch_item = if save_and_restore_next_switch_item {
2651 Some(state.save_next_switch_item()?)
2652 } else {
2653 None
2654 };
2655
2656 let old_guest_thread = if save_and_restore_thread {
2657 self.current_thread()?
2658 } else {
2659 CurrentThread::None
2660 };
2661
2662 let waiting_set = match &reason {
2663 SuspendReason::Waiting { set, .. } => Some(*set),
2664 _ => None,
2665 };
2666
2667 let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2668 if suspend_reason.is_some() {
2669 bail_bug!("suspend reason unexpectedly already set for store");
2670 }
2671 *suspend_reason = Some(reason);
2672
2673 if let Some(set) = waiting_set {
2676 self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2677 }
2678
2679 self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2680
2681 if let Some(set) = waiting_set {
2682 self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2683 }
2684
2685 if save_and_restore_thread {
2686 self.set_thread(old_guest_thread)?;
2687 }
2688
2689 if let Some(item) = old_next_switch_item {
2690 let state = self.concurrent_state_mut()?;
2691 state.restore_next_switch_item(item)?;
2692 }
2693
2694 Ok(())
2695 }
2696
2697 fn wait_for_event(
2698 &mut self,
2699 caller_instance: RuntimeInstance,
2700 waitable: Waitable,
2701 ) -> Result<()> {
2702 let caller = self.current_guest_thread()?;
2703 let state = self.concurrent_state_mut()?;
2704
2705 waitable.trap_if_in_waitable_set(state)?;
2706
2707 let set = state.get_mut(caller.thread)?.sync_call_set;
2708 waitable.join(state, Some(set))?;
2709
2710 self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2711
2712 self.suspend(SuspendReason::Waiting {
2713 set,
2714 thread: caller,
2715 })?;
2716 let state = self.concurrent_state_mut()?;
2717
2718 waitable.join(state, None)
2719 }
2720
2721 fn cleanup_thread(
2743 &mut self,
2744 guest_thread: QualifiedThreadId,
2745 runtime_instance: RuntimeInstance,
2746 ) -> Result<()> {
2747 let state = self.concurrent_state_mut()?;
2748 state.take_next_switch_item()?;
2751 let thread_data = state.get_mut(guest_thread.thread)?;
2752 let sync_call_set = thread_data.sync_call_set;
2753 if let Some(guest_id) = thread_data.instance_rep {
2754 self.instance_state(runtime_instance)
2755 .thread_handle_table()
2756 .guest_thread_remove(guest_id)?;
2757 }
2758 let state = self.concurrent_state_mut()?;
2759
2760 for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2762 if let Some(Event::Subtask {
2763 status: Status::Returned | Status::ReturnCancelled,
2764 }) = waitable.common(state)?.event
2765 {
2766 waitable.delete_from(state)?;
2767 }
2768 }
2769
2770 state.delete(guest_thread.thread)?;
2771 state.delete(sync_call_set)?;
2772 let task = state.get_mut(guest_thread.task)?;
2773 task.threads.remove(&guest_thread.thread);
2774
2775 if task.threads.is_empty() && !task.returned_or_cancelled() {
2776 bail!(Trap::NoAsyncResult);
2777 }
2778
2779 if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2780 task.decremented_interesting_task_count = true;
2781
2782 debug_assert!(state.interesting_tasks > 0);
2783 state.interesting_tasks -= 1;
2784 if state.interesting_tasks == 0
2785 && let Some(waker) = state.interesting_tasks_empty_waker.take()
2786 {
2787 waker.wake();
2788 }
2789 }
2790
2791 state.decrement_task_ref_count(guest_thread.task)?;
2792
2793 Ok(())
2794 }
2795
2796 fn cancel_guest_subtask_without_lowered_parameters(
2805 &mut self,
2806 caller_instance: RuntimeInstance,
2807 guest_task: TableId<GuestTask>,
2808 ) -> Result<()> {
2809 let concurrent_state = self.concurrent_state_mut()?;
2810 let task = concurrent_state.get_mut(guest_task)?;
2811 assert!(!task.already_lowered_parameters());
2812 task.lower_params = None;
2816 task.lift_result = None;
2817 task.exited = true;
2818 let instance = task.instance;
2819
2820 assert_eq!(1, task.threads.len());
2827 let thread = *task.threads.iter().next().unwrap();
2828 let next_switch_item = concurrent_state.save_next_switch_item()?;
2829 self.cleanup_thread(
2830 QualifiedThreadId {
2831 task: guest_task,
2832 thread,
2833 },
2834 caller_instance,
2835 )?;
2836 self.concurrent_state_mut()?
2837 .restore_next_switch_item(next_switch_item)?;
2838
2839 let pending = &mut self.instance_state(instance).concurrent_state().pending;
2841 let pending_count = pending.len();
2842 let mut decrement = 0;
2843 let mut consume = |kind: &GuestCallKind| {
2844 if let GuestCallKind::StartImplicit {
2845 was_pending: true, ..
2846 } = kind
2847 {
2848 decrement += 1;
2849 }
2850 };
2851 pending.retain(|thread, kind| {
2852 if thread.task == guest_task {
2853 consume(kind);
2854 false
2855 } else {
2856 true
2857 }
2858 });
2859
2860 if pending.len() == pending_count {
2861 let mut matches = |item: &WorkItem| match item {
2864 WorkItem::ResumeThread { thread, .. } => thread.task == guest_task,
2865 WorkItem::GuestCall {
2866 call: GuestCall { thread, kind },
2867 ..
2868 } => {
2869 if thread.task == guest_task {
2870 consume(kind);
2871 true
2872 } else {
2873 false
2874 }
2875 }
2876 _ => false,
2877 };
2878
2879 let mut remove = |queue: &mut VecDeque<WorkItem>| {
2880 let count = queue.len();
2881 queue.retain(|item| !matches(item));
2882 count != queue.len()
2883 };
2884
2885 let state = self.concurrent_state_mut()?;
2886 let mut found = remove(&mut state.high_priority);
2887 found |= remove(&mut state.low_priority);
2888
2889 for item in [&mut state.switch_item, &mut state.next_switch_item] {
2890 if matches!(item, Some(item) if matches(item)) {
2891 found = true;
2892 *item = None;
2893 }
2894 }
2895
2896 if !found {
2897 bail!(Trap::SubtaskCancelAfterTerminal);
2898 }
2899 }
2900
2901 self.decrement_pending_start_count(instance, decrement)?;
2902
2903 Ok(())
2904 }
2905
2906 fn decrement_pending_start_count(
2907 &mut self,
2908 instance: RuntimeInstance,
2909 decrement: usize,
2910 ) -> Result<()> {
2911 if decrement > 0 {
2912 let count = &mut self
2913 .instance_state(instance)
2914 .concurrent_state()
2915 .pending_start_count;
2916
2917 if *count < decrement {
2918 bail_bug!("`ConcurrentInstantState::pending_start_count` underflow");
2919 }
2920
2921 *count -= decrement;
2922
2923 self.partition_pending(instance)?;
2925 }
2926
2927 Ok(())
2928 }
2929
2930 pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2933 if !self.concurrency_support() {
2934 return Ok(self
2935 .current_scope_id_not_concurrent()?
2936 .map(|id| CurrentScope::Id(VMScope::id(id))));
2937 }
2938
2939 Ok(match self.current_thread()? {
2940 CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2941 CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2942 CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2943 CurrentThread::None => return Ok(None),
2944 })
2945 }
2946
2947 pub(crate) fn queue_task(
2948 &mut self,
2949 task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2950 ) -> Result<()> {
2951 self.concurrent_state_mut()?
2952 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2953 Ok(())
2954 }
2955
2956 fn any_may_not_suspend(&mut self) -> Result<bool> {
2965 Ok(self
2973 .concurrent_state_mut()?
2974 .table
2975 .get_mut()
2976 .iter_mut()
2977 .filter_map(|(_, entry)| {
2978 if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2979 Some(task.instance)
2980 } else {
2981 None
2982 }
2983 })
2984 .collect::<Vec<_>>()
2985 .into_iter()
2986 .any(|instance| {
2987 self.instance_state(instance)
2988 .concurrent_state()
2989 .do_not_suspend
2990 }))
2991 }
2992}
2993
2994impl Instance {
2995 fn get_event(
2998 self,
2999 store: &mut StoreOpaque,
3000 guest_task: TableId<GuestTask>,
3001 set: Option<TableId<WaitableSet>>,
3002 cancellable: bool,
3003 ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
3004 let state = store.concurrent_state_mut()?;
3005
3006 let task = state.get_mut(guest_task)?;
3007 let event = &mut task.event;
3008 if let Some(ev) = event
3009 && (cancellable || !matches!(ev, Event::Cancelled))
3010 {
3011 log::trace!("deliver event {ev:?} to {guest_task:?}");
3012
3013 if matches!(ev, Event::Cancelled) {
3014 task.cancel_request_delivered = true;
3015 }
3016
3017 let ev = *ev;
3018 *event = None;
3019 return Ok(Some((ev, None)));
3020 }
3021
3022 let set = match set {
3023 Some(set) => set,
3024 None => return Ok(None),
3025 };
3026 let waitable = match state.get_mut(set)?.ready.pop_first() {
3027 Some(v) => v,
3028 None => return Ok(None),
3029 };
3030
3031 let common = waitable.common(state)?;
3032 let handle = match common.handle {
3033 Some(h) => h,
3034 None => bail_bug!("handle not set when delivering event"),
3035 };
3036 let event = match common.event.take() {
3037 Some(e) => e,
3038 None => bail_bug!("event not set when delivering event"),
3039 };
3040
3041 log::trace!(
3042 "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
3043 );
3044
3045 waitable.on_delivery(store, self, event)?;
3046
3047 Ok(Some((event, Some((waitable, handle)))))
3048 }
3049
3050 fn handle_callback_code(
3057 self,
3058 store: &mut dyn VMStore,
3059 guest_thread: QualifiedThreadId,
3060 runtime_instance: RuntimeInstance,
3061 call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
3062 ) -> Result<()> {
3063 let old_thread = store.set_thread(guest_thread)?;
3064 log::trace!(
3065 "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
3066 );
3067
3068 store.enter_instance(runtime_instance);
3069
3070 let mut code = call(store)?;
3071
3072 if matches!(
3078 unpack_callback_code(code).0,
3079 callback_code::YIELD | callback_code::WAIT
3080 ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
3081 {
3082 let (unpacked, set) = unpack_callback_code(code);
3083 log::trace!(
3084 "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
3085 cancellation; delivering it via callback"
3086 );
3087 code = invoke_callback(store, guest_thread.task, event, 0)?;
3088 }
3089
3090 store.exit_instance(runtime_instance)?;
3091
3092 store.set_thread(old_thread)?;
3093 log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
3094
3095 let store = &mut **store;
3096 let (code, set) = unpack_callback_code(code);
3097
3098 log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
3099
3100 let state = store.concurrent_state_mut()?;
3101
3102 state.take_next_switch_item()?;
3103
3104 let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
3105 let set = store
3106 .instance_state(runtime_instance)
3107 .handle_table()
3108 .waitable_set_rep(handle)?;
3109
3110 Ok(TableId::<WaitableSet>::new(set))
3111 };
3112
3113 match code {
3114 callback_code::EXIT => {
3115 log::trace!("implicit thread {guest_thread:?} completed");
3116 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3117 task.exited = true;
3118 task.callback = None;
3119
3120 store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
3125
3126 store.cleanup_thread(guest_thread, runtime_instance)?;
3127 }
3128 callback_code::YIELD => {
3129 let old = state
3132 .get_mut(guest_thread.thread)?
3133 .wake_on_cancel
3134 .replace(WakeOnCancel::Yielding);
3135 if !old.is_none() {
3136 bail_bug!("thread unexpectedly had wake_on_cancel set");
3137 }
3138
3139 let call = GuestCall {
3146 thread: guest_thread,
3147 kind: GuestCallKind::DeliverEvent {
3148 instance: self,
3149 set: None,
3150 },
3151 };
3152 state.push_low_priority(WorkItem::GuestCall {
3155 instance: runtime_instance,
3156 call,
3157 });
3158 }
3159 callback_code::WAIT => {
3160 let set = get_set(store, set)?;
3161 self.wait_with_callback(store, guest_thread, set)?;
3162 }
3163 _ => bail!(Trap::UnsupportedCallbackCode),
3164 }
3165
3166 Ok(())
3167 }
3168
3169 fn wait_with_callback(
3175 self,
3176 store: &mut StoreOpaque,
3177 guest_thread: QualifiedThreadId,
3178 set: TableId<WaitableSet>,
3179 ) -> Result<()> {
3180 let state = store.concurrent_state_mut()?;
3181 state.get_mut(set)?.num_waiting += 1;
3184
3185 if state.get_mut(guest_thread.task)?.event.is_some()
3186 || !state.get_mut(set)?.ready.is_empty()
3187 {
3188 let instance = state.get_mut(guest_thread.task)?.instance;
3190 state.push_high_priority(WorkItem::GuestCall {
3191 instance,
3192 call: GuestCall {
3193 thread: guest_thread,
3194 kind: GuestCallKind::DeliverEvent {
3195 instance: self,
3196 set: Some(set),
3197 },
3198 },
3199 });
3200 return Ok(());
3201 }
3202
3203 let instance = store
3209 .concurrent_state_mut()?
3210 .get_mut(guest_thread.task)?
3211 .instance;
3212 store.switch_or_trap_if_may_not_suspend(instance)?;
3213
3214 let state = store.concurrent_state_mut()?;
3220 let old = state
3221 .get_mut(guest_thread.thread)?
3222 .wake_on_cancel
3223 .replace(WakeOnCancel::Waiting(set));
3224 if !old.is_none() {
3225 bail_bug!("thread unexpectedly had wake_on_cancel set");
3226 }
3227 let old = state
3228 .get_mut(set)?
3229 .waiting
3230 .insert(guest_thread, WaitMode::Callback(self));
3231 if !old.is_none() {
3232 bail_bug!("set's waiting set already had this thread registered");
3233 }
3234 Ok(())
3235 }
3236
3237 unsafe fn stage_call<T: 'static>(
3244 self,
3245 mut store: StoreContextMut<T>,
3246 guest_thread: QualifiedThreadId,
3247 callee: SendSyncPtr<VMFuncRef>,
3248 param_count: usize,
3249 result_count: usize,
3250 async_: bool,
3251 callback: Option<SendSyncPtr<VMFuncRef>>,
3252 post_return: Option<SendSyncPtr<VMFuncRef>>,
3253 host_caller: bool,
3254 ) -> Result<()> {
3255 unsafe fn make_call<T: 'static>(
3270 store: StoreContextMut<T>,
3271 guest_thread: QualifiedThreadId,
3272 callee: SendSyncPtr<VMFuncRef>,
3273 param_count: usize,
3274 result_count: usize,
3275 ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3276 + Send
3277 + Sync
3278 + 'static
3279 + use<T> {
3280 let token = StoreToken::new(store);
3281 move |store: &mut dyn VMStore| {
3282 let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3283
3284 store
3285 .concurrent_state_mut()?
3286 .set_thread_running(guest_thread.thread)?;
3287 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3288 let lower = match task.lower_params.take() {
3289 Some(l) => l,
3290 None => bail_bug!("lower_params missing"),
3291 };
3292
3293 lower(store, &mut storage[..param_count])?;
3294
3295 let mut store = token.as_context_mut(store);
3296
3297 unsafe {
3300 crate::Func::call_unchecked_raw(
3301 &mut store,
3302 callee.as_non_null(),
3303 NonNull::new(
3304 &mut storage[..param_count.max(result_count)]
3305 as *mut [MaybeUninit<ValRaw>] as _,
3306 )
3307 .unwrap(),
3308 UncaughtException::Trap,
3309 )?;
3310 }
3311
3312 Ok(storage)
3313 }
3314 }
3315
3316 let call = unsafe {
3320 make_call(
3321 store.as_context_mut(),
3322 guest_thread,
3323 callee,
3324 param_count,
3325 result_count,
3326 )
3327 };
3328
3329 let callee_instance = store
3330 .0
3331 .concurrent_state_mut()?
3332 .get_mut(guest_thread.task)?
3333 .instance;
3334
3335 let fun = if callback.is_some() {
3336 assert!(async_);
3337
3338 Box::new(move |store: &mut dyn VMStore| {
3339 self.add_guest_thread_to_instance_table(
3340 guest_thread.thread,
3341 store,
3342 callee_instance.index,
3343 )?;
3344 self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3345 let storage = call(store)?;
3346
3347 Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3350 })
3351 }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3352 } else {
3353 let token = StoreToken::new(store.as_context_mut());
3354 Box::new(move |store: &mut dyn VMStore| {
3355 self.add_guest_thread_to_instance_table(
3356 guest_thread.thread,
3357 store,
3358 callee_instance.index,
3359 )?;
3360 let old_thread = store.set_thread(guest_thread)?;
3361 log::trace!(
3362 "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3363 );
3364 let flags = self.id().get(store).instance_flags(callee_instance.index);
3365
3366 let callee_async_typed = store
3367 .concurrent_state_mut()?
3368 .get_mut(guest_thread.task)?
3369 .async_typed;
3370
3371 if !async_ && callee_async_typed {
3375 store.enter_instance(callee_instance);
3376 }
3377
3378 if !callee_async_typed {
3379 store.enter_sync_call(callee_instance)?;
3380 }
3381
3382 let storage = call(store)?;
3389
3390 if !callee_async_typed {
3391 store.exit_sync_call(callee_instance)?;
3392 }
3393
3394 if !async_ {
3395 if callee_async_typed {
3401 store.exit_instance(callee_instance)?;
3402 }
3403
3404 let lift = {
3405 let state = store.concurrent_state_mut()?;
3406 if !state.get_mut(guest_thread.task)?.result.is_none() {
3407 bail_bug!("task has already produced a result");
3408 }
3409
3410 match state.get_mut(guest_thread.task)?.lift_result.take() {
3411 Some(lift) => lift,
3412 None => bail_bug!("lift_result field is missing"),
3413 }
3414 };
3415
3416 let result = (lift.lift)(store, unsafe {
3419 mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3420 &storage[..result_count],
3421 )
3422 })?;
3423
3424 let post_return_arg = match result_count {
3425 0 => ValRaw::i32(0),
3426 1 => unsafe { storage[0].assume_init() },
3429 _ => unreachable!(),
3430 };
3431
3432 unsafe {
3433 call_post_return(
3434 token.as_context_mut(store),
3435 post_return.map(|v| v.as_non_null()),
3436 post_return_arg,
3437 flags,
3438 )?;
3439 }
3440
3441 self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3442 }
3443
3444 store.set_thread(old_thread)?;
3445
3446 store
3447 .concurrent_state_mut()?
3448 .get_mut(guest_thread.task)?
3449 .exited = true;
3450
3451 log::trace!(
3452 "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3453 );
3454
3455 if callee_async_typed {
3456 store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3461 }
3462
3463 store.cleanup_thread(guest_thread, callee_instance)?;
3465 Ok(())
3466 })
3467 };
3468
3469 store.0.concurrent_state_mut()?.push_work_item(
3470 WorkItem::GuestCall {
3471 instance: callee_instance,
3472 call: GuestCall {
3473 thread: guest_thread,
3474 kind: GuestCallKind::StartImplicit {
3475 closure: fun,
3476 was_pending: false,
3477 },
3478 },
3479 },
3480 if host_caller {
3481 Priority::Low
3487 } else {
3488 Priority::Switch
3489 },
3490 )?;
3491
3492 Ok(())
3493 }
3494
3495 unsafe fn prepare_call<T: 'static>(
3508 self,
3509 mut store: StoreContextMut<T>,
3510 start: NonNull<VMFuncRef>,
3511 return_: NonNull<VMFuncRef>,
3512 caller_instance: RuntimeComponentInstanceIndex,
3513 callee_instance: RuntimeComponentInstanceIndex,
3514 task_return_type: TypeTupleIndex,
3515 callee_async_typed: bool,
3516 memory: *mut VMMemoryDefinition,
3517 string_encoding: StringEncoding,
3518 caller_info: CallerInfo,
3519 ) -> Result<()> {
3520 enum ResultInfo {
3521 Heap { results: u32 },
3522 Stack { result_count: u32 },
3523 }
3524
3525 let result_info = match &caller_info {
3526 CallerInfo::Async {
3527 has_result: true,
3528 params,
3529 } => ResultInfo::Heap {
3530 results: match params.last() {
3532 Some(r) => r.get_u32(),
3533 None => bail_bug!("retptr missing"),
3534 },
3535 },
3536 CallerInfo::Async {
3537 has_result: false, ..
3538 } => ResultInfo::Stack { result_count: 0 },
3539 CallerInfo::Sync {
3540 result_count,
3541 params,
3542 } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3543 results: match params.last() {
3545 Some(r) => r.get_u32(),
3546 None => bail_bug!("arg ptr missing"),
3547 },
3548 },
3549 CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3550 result_count: *result_count,
3551 },
3552 };
3553
3554 let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3555
3556 let start = SendSyncPtr::new(start);
3560 let return_ = SendSyncPtr::new(return_);
3561 let token = StoreToken::new(store.as_context_mut());
3562 let old_thread = store.0.current_guest_thread()?;
3563
3564 let state = store.0.concurrent_state_mut()?;
3565
3566 debug_assert_eq!(
3567 state.get_mut(old_thread.task)?.instance,
3568 self.runtime_instance(caller_instance)
3569 );
3570
3571 let guest_thread = GuestTask::new(
3572 state,
3573 Box::new(move |store, dst| {
3574 let mut store = token.as_context_mut(store);
3575 assert!(dst.len() <= MAX_FLAT_PARAMS);
3576 let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3578 let count = match caller_info {
3579 CallerInfo::Async { params, has_result } => {
3583 let params = ¶ms[..params.len() - usize::from(has_result)];
3584 for (param, src) in params.iter().zip(&mut src) {
3585 src.write(*param);
3586 }
3587 params.len()
3588 }
3589
3590 CallerInfo::Sync { params, .. } => {
3592 for (param, src) in params.iter().zip(&mut src) {
3593 src.write(*param);
3594 }
3595 params.len()
3596 }
3597 };
3598 unsafe {
3605 crate::Func::call_unchecked_raw(
3606 &mut store,
3607 start.as_non_null(),
3608 NonNull::new(
3609 &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3610 )
3611 .unwrap(),
3612 UncaughtException::Trap,
3613 )?;
3614 }
3615 dst.copy_from_slice(&src[..dst.len()]);
3616 let task = store.0.current_guest_thread()?.task;
3617 let state = store.0.concurrent_state_mut()?;
3618 Waitable::Guest(task).set_event(
3619 state,
3620 Some(Event::Subtask {
3621 status: Status::Started,
3622 }),
3623 )?;
3624 Ok(())
3625 }),
3626 LiftResult {
3627 lift: Box::new(move |store, src| {
3628 let mut store = token.as_context_mut(store);
3631 let mut my_src = src.to_owned(); if let ResultInfo::Heap { results } = &result_info {
3633 my_src.push(ValRaw::u32(*results));
3634 }
3635
3636 unsafe {
3643 crate::Func::call_unchecked_raw(
3644 &mut store,
3645 return_.as_non_null(),
3646 my_src.as_mut_slice().into(),
3647 UncaughtException::Trap,
3648 )?;
3649 }
3650
3651 let thread = store.0.current_guest_thread()?;
3652 let state = store.0.concurrent_state_mut()?;
3653 if sync_caller {
3654 state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3655 if let ResultInfo::Stack { result_count } = &result_info {
3656 match result_count {
3657 0 => None,
3658 1 => Some(my_src[0]),
3659 _ => unreachable!(),
3660 }
3661 } else {
3662 None
3663 },
3664 );
3665 }
3666 Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3667 }),
3668 ty: task_return_type,
3669 memory: NonNull::new(memory).map(SendSyncPtr::new),
3670 string_encoding,
3671 },
3672 Caller::Guest { thread: old_thread },
3673 None,
3674 self.runtime_instance(callee_instance),
3675 callee_async_typed,
3676 false,
3679 )?;
3680
3681 store.0.set_thread(guest_thread)?;
3684 log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3685
3686 Ok(())
3687 }
3688
3689 unsafe fn call_callback<T>(
3694 self,
3695 mut store: StoreContextMut<T>,
3696 function: SendSyncPtr<VMFuncRef>,
3697 event: Event,
3698 handle: u32,
3699 ) -> Result<u32> {
3700 let (ordinal, result) = event.parts();
3701 let params = &mut [
3702 ValRaw::u32(ordinal),
3703 ValRaw::u32(handle),
3704 ValRaw::u32(result),
3705 ];
3706 unsafe {
3711 crate::Func::call_unchecked_raw(
3712 &mut store,
3713 function.as_non_null(),
3714 params.as_mut_slice().into(),
3715 UncaughtException::Trap,
3716 )?;
3717 }
3718 Ok(params[0].get_u32())
3719 }
3720
3721 unsafe fn start_call<T: 'static>(
3739 self,
3740 mut store: StoreContextMut<T>,
3741 callback: *mut VMFuncRef,
3742 post_return: *mut VMFuncRef,
3743 callee: NonNull<VMFuncRef>,
3744 param_count: u32,
3745 result_count: u32,
3746 flags: u32,
3747 storage: &mut [MaybeUninit<ValRaw>],
3748 ) -> Result<()> {
3749 let token = StoreToken::new(store.as_context_mut());
3750 let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3751 let guest_thread = store.0.current_guest_thread()?;
3752 let state = store.0.concurrent_state_mut()?;
3753
3754 if !state.event_loop_running {
3759 bail!(
3760 "async function started during an original synchronous \
3761 entrypoint; please use an async-style entrypoint instead"
3762 );
3763 }
3764
3765 let callee = SendSyncPtr::new(callee);
3766 let param_count = usize::try_from(param_count)?;
3767 assert!(param_count <= MAX_FLAT_PARAMS);
3768 let result_count = usize::try_from(result_count)?;
3769 assert!(result_count <= MAX_FLAT_RESULTS);
3770
3771 let task = state.get_mut(guest_thread.task)?;
3772 let callee_async_typed = task.async_typed;
3773 let callee_instance = task.instance;
3774
3775 task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3776
3777 if let Some(callback) = NonNull::new(callback) {
3778 let callback = SendSyncPtr::new(callback);
3782 task.callback = Some(Box::new(move |store, event, handle| {
3783 let store = token.as_context_mut(store);
3784 unsafe { self.call_callback::<T>(store, callback, event, handle) }
3785 }));
3786 }
3787
3788 let Caller::Guest { thread: caller } = &task.caller else {
3789 bail_bug!("start_call unexpectedly invoked for host->guest call");
3792 };
3793 let caller = *caller;
3794 let caller_instance = state.get_mut(caller.task)?.instance;
3795
3796 unsafe {
3798 self.stage_call(
3799 store.as_context_mut(),
3800 guest_thread,
3801 callee,
3802 param_count,
3803 result_count,
3804 (flags & START_FLAG_ASYNC_CALLEE) != 0,
3805 NonNull::new(callback).map(SendSyncPtr::new),
3806 NonNull::new(post_return).map(SendSyncPtr::new),
3807 false,
3808 )?;
3809 }
3810
3811 let old_do_not_suspend = if callee_async_typed {
3812 let state = store.0.instance_state(callee_instance).concurrent_state();
3819 let old_do_not_suspend = state.do_not_suspend;
3820 state.do_not_suspend = false;
3821 Some(old_do_not_suspend)
3822 } else {
3823 None
3824 };
3825
3826 let state = store.0.concurrent_state_mut()?;
3827
3828 let guest_waitable = Waitable::Guest(guest_thread.task);
3831 let old_set = guest_waitable.common(state)?.set;
3832 let set = state.get_mut(caller.thread)?.sync_call_set;
3833 guest_waitable.join(state, Some(set))?;
3834
3835 store.0.set_thread(CurrentThread::None)?;
3836
3837 let mut yielded = false;
3853 let (status, waitable) = loop {
3854 store.0.suspend(if yielded {
3855 SuspendReason::Waiting {
3856 set,
3857 thread: caller,
3858 }
3859 } else {
3860 yielded = true;
3861 SuspendReason::YieldingToSubtask { thread: caller }
3862 })?;
3863
3864 if let Some(old_do_not_suspend) = old_do_not_suspend {
3865 store
3866 .0
3867 .instance_state(callee_instance)
3868 .concurrent_state()
3869 .do_not_suspend = old_do_not_suspend;
3870 }
3871
3872 let state = store.0.concurrent_state_mut()?;
3873
3874 log::trace!("taking event for {:?}", guest_thread.task);
3875 let event = guest_waitable.take_event(state)?;
3876 let Some(Event::Subtask { status }) = event else {
3877 bail_bug!("subtasks should only get subtask events, got {event:?}")
3878 };
3879
3880 log::trace!("status {status:?} for {:?}", guest_thread.task);
3881
3882 if status == Status::Returned {
3883 break (status, None);
3885 } else if async_caller {
3886 let handle = store
3890 .0
3891 .handle_table_insertion(caller_instance)
3892 .subtask_guest(guest_thread.task.rep())?;
3893 store
3894 .0
3895 .concurrent_state_mut()?
3896 .get_mut(guest_thread.task)?
3897 .common
3898 .handle = Some(handle);
3899 break (status, Some(handle));
3900 } else {
3901 store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3905 }
3906 };
3907
3908 guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3909
3910 store.0.set_thread(caller)?;
3912
3913 store
3914 .0
3915 .concurrent_state_mut()?
3916 .set_thread_running(caller.thread)?;
3917
3918 log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3919
3920 if async_caller {
3921 let Some(slot) = storage.first_mut() else {
3922 bail_bug!("no storage for async call status");
3923 };
3924 *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3925 } else {
3926 let state = store.0.concurrent_state_mut()?;
3929 let task = state.get_mut(guest_thread.task)?;
3930 if let Some(Some(result)) = task.sync_result.take()? {
3931 storage[0] = MaybeUninit::new(result);
3932 }
3933 }
3934
3935 if status == Status::Returned {
3936 store
3937 .0
3938 .concurrent_state_mut()?
3939 .decrement_task_ref_count(guest_thread.task)?;
3940 }
3941
3942 Ok(())
3943 }
3944
3945 pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3961 self,
3962 mut store: StoreContextMut<'_, T>,
3963 host_task: EnteredHostTask,
3964 result_may_require_realloc: bool,
3965 future: impl Future<Output = Result<R>> + Send + 'static,
3966 lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3967 + Send
3968 + 'static,
3969 ) -> Result<u32> {
3970 let token = StoreToken::new(store.as_context_mut());
3971
3972 let (join_handle, future) = JoinHandle::run(future);
3975 let mut future = Box::pin(future);
3976
3977 let poll = tls::set(store.0, || {
3982 future
3983 .as_mut()
3984 .poll(&mut Context::from_waker(&Waker::noop()))
3985 });
3986
3987 match poll {
3988 Poll::Ready(result) => {
3990 let result = result.transpose()?;
3991 let task = store.0.current_materialized_host_task()?;
3994 lower(store.as_context_mut(), result, true, task)?;
3995 return Ok(Status::Returned.pack(None));
3996 }
3997
3998 Poll::Pending => {}
4000 }
4001
4002 let Some(task) = store.0.materialize_host_task_id()? else {
4006 bail_bug!("current thread is not a host thread")
4007 };
4008 {
4009 let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
4010 assert!(matches!(state, HostTaskState::CalleeStarted));
4011 *state = HostTaskState::CalleeRunning(join_handle);
4012 }
4013
4014 let future = Box::pin(async move {
4022 let result = match run_with_host_task_set(task, future).await? {
4023 Some(result) => Some(result?),
4024 None => None,
4025 };
4026 let on_complete = move |store: &mut dyn VMStore| {
4027 let mut store = token.as_context_mut(store);
4031 let old = store.0.set_thread(task)?;
4032
4033 let status = if result.is_some() {
4034 Status::Returned
4035 } else {
4036 Status::ReturnCancelled
4037 };
4038
4039 lower(store.as_context_mut(), result, false, Some(task))?;
4040 let state = store.0.concurrent_state_mut()?;
4041 match &mut state.get_mut(task)?.state {
4042 pending @ HostTaskState::CalleeCancelling => {
4045 *pending = HostTaskState::CalleeDone { cancelled: true };
4046 }
4047
4048 other => *other = HostTaskState::CalleeDone { cancelled: false },
4050 }
4051 Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
4052
4053 store.0.set_thread(old)?;
4054 Ok(())
4055 };
4056
4057 tls::get(move |store| {
4058 if result_may_require_realloc {
4059 store
4064 .concurrent_state_mut()?
4065 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
4066 on_complete,
4067 ))));
4068 Ok(())
4069 } else {
4070 on_complete(store)
4073 }
4074 })
4075 });
4076
4077 let caller = match host_task {
4080 Some(caller) => caller,
4081 None => bail_bug!("host task wasn't created but should have been"),
4082 };
4083 let state = store.0.concurrent_state_mut()?;
4084 state.push_future(future);
4085 let instance = state.get_mut(caller.task)?.instance;
4086 let handle = store
4087 .0
4088 .handle_table_insertion(instance)
4089 .subtask_host(task.rep())?;
4090 store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
4091 log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
4092
4093 store.0.set_thread(caller)?;
4097 Ok(Status::Started.pack(Some(handle)))
4098 }
4099
4100 pub(crate) fn task_return(
4103 self,
4104 store: &mut dyn VMStore,
4105 ty: TypeTupleIndex,
4106 options: OptionsIndex,
4107 storage: &[ValRaw],
4108 ) -> Result<()> {
4109 let guest_thread = store.current_guest_thread()?;
4110 let state = store.concurrent_state_mut()?;
4111 if !state.get_mut(guest_thread.task)?.async_lifted {
4112 bail!(Trap::TaskReturnOrCancelSyncLifted);
4113 }
4114 let lift = state
4115 .get_mut(guest_thread.task)?
4116 .lift_result
4117 .take()
4118 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4119 if !state.get_mut(guest_thread.task)?.result.is_none() {
4120 bail_bug!("task result unexpectedly already set");
4121 }
4122
4123 let CanonicalOptions {
4124 string_encoding,
4125 data_model,
4126 ..
4127 } = &self.id().get(store).component().env_component().options[options];
4128
4129 let invalid = ty != lift.ty
4130 || string_encoding != &lift.string_encoding
4131 || match data_model {
4132 CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
4133 Some(memory) => {
4134 let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
4135 let actual = self.id().get(store).runtime_memory(memory);
4136 expected != actual.as_ptr()
4137 }
4138 None => false,
4141 },
4142 CanonicalOptionsDataModel::Gc { .. } => true,
4144 };
4145
4146 if invalid {
4147 bail!(Trap::TaskReturnInvalid);
4148 }
4149
4150 log::trace!("task.return for {guest_thread:?}");
4151
4152 let result = (lift.lift)(store, storage)?;
4153 self.task_complete(store, guest_thread.task, result, Status::Returned)
4154 }
4155
4156 pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4158 let guest_thread = store.current_guest_thread()?;
4159 let state = store.concurrent_state_mut()?;
4160 let task = state.get_mut(guest_thread.task)?;
4161 if !task.async_lifted {
4162 bail!(Trap::TaskReturnOrCancelSyncLifted);
4163 }
4164 if !task.cancel_request_delivered {
4165 bail!(Trap::TaskCancelNotCancelled);
4166 }
4167 _ = task
4168 .lift_result
4169 .take()
4170 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4171
4172 if !task.result.is_none() {
4173 bail_bug!("task result should not bet set yet");
4174 }
4175
4176 log::trace!("task.cancel for {guest_thread:?}");
4177
4178 self.task_complete(
4179 store,
4180 guest_thread.task,
4181 Box::new(DummyResult),
4182 Status::ReturnCancelled,
4183 )
4184 }
4185
4186 fn task_complete(
4192 self,
4193 store: &mut StoreOpaque,
4194 guest_task: TableId<GuestTask>,
4195 result: Box<dyn Any + Send + Sync>,
4196 status: Status,
4197 ) -> Result<()> {
4198 store
4199 .component_resource_tables(Some(self))?
4200 .validate_scope_exit()?;
4201
4202 let state = store.concurrent_state_mut()?;
4203 let task = state.get_mut(guest_task)?;
4204
4205 task.event = None;
4209
4210 if let Caller::Host { tx, .. } = &mut task.caller {
4211 if let Some(tx) = tx.take() {
4212 _ = tx.send(result);
4213 }
4214 } else {
4215 task.result = Some(result);
4216 Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4217 }
4218
4219 Ok(())
4220 }
4221
4222 pub(crate) fn waitable_set_new(
4224 self,
4225 store: &mut StoreOpaque,
4226 caller_instance: RuntimeComponentInstanceIndex,
4227 ) -> Result<u32> {
4228 let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4229 let handle = store
4230 .handle_table_insertion(self.runtime_instance(caller_instance))
4231 .waitable_set(set.rep())?;
4232 log::trace!("new waitable set {set:?} (handle {handle})");
4233 Ok(handle)
4234 }
4235
4236 pub(crate) fn waitable_set_drop(
4238 self,
4239 store: &mut StoreOpaque,
4240 caller_instance: RuntimeComponentInstanceIndex,
4241 set: u32,
4242 ) -> Result<()> {
4243 let rep = store
4244 .instance_state(self.runtime_instance(caller_instance))
4245 .handle_table()
4246 .waitable_set_remove(set)?;
4247
4248 log::trace!("drop waitable set {rep} (handle {set})");
4249
4250 let set = store
4254 .concurrent_state_mut()?
4255 .get_mut(TableId::<WaitableSet>::new(rep))?;
4256 if set.num_waiting > 0 {
4257 bail!(Trap::WaitableSetDropHasWaiters);
4258 }
4259
4260 store
4261 .concurrent_state_mut()?
4262 .delete(TableId::<WaitableSet>::new(rep))?;
4263
4264 Ok(())
4265 }
4266
4267 pub(crate) fn waitable_join(
4269 self,
4270 store: &mut StoreOpaque,
4271 caller_instance: RuntimeComponentInstanceIndex,
4272 waitable_handle: u32,
4273 set_handle: u32,
4274 ) -> Result<()> {
4275 let mut instance = self.id().get_mut(store);
4276 let waitable =
4277 Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4278
4279 let set = if set_handle == 0 {
4280 None
4281 } else {
4282 let set = instance.instance_states().0[caller_instance]
4283 .handle_table()
4284 .waitable_set_rep(set_handle)?;
4285
4286 Some(TableId::<WaitableSet>::new(set))
4287 };
4288
4289 waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4290
4291 log::trace!(
4292 "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4293 );
4294
4295 waitable.join(store.concurrent_state_mut()?, set)
4296 }
4297
4298 pub(crate) fn subtask_drop(
4300 self,
4301 store: &mut StoreOpaque,
4302 caller_instance: RuntimeComponentInstanceIndex,
4303 task_id: u32,
4304 ) -> Result<()> {
4305 self.waitable_join(store, caller_instance, task_id, 0)?;
4306
4307 let (rep, is_host) = store
4308 .instance_state(self.runtime_instance(caller_instance))
4309 .handle_table()
4310 .subtask_remove(task_id)?;
4311
4312 let concurrent_state = store.concurrent_state_mut()?;
4313 let waitable = if is_host {
4314 let id = TableId::<HostTask>::new(rep);
4315 let task = concurrent_state.get_mut(id)?;
4316 match &task.state {
4317 HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4318 bail!(Trap::SubtaskDropNotResolved)
4319 }
4320 HostTaskState::CalleeDone { .. } => {}
4321 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4322 bail_bug!("invalid state for callee in `subtask.drop`")
4323 }
4324 }
4325
4326 Waitable::Host(id)
4327 } else {
4328 let id = TableId::<GuestTask>::new(rep);
4329 let task = concurrent_state.get_mut(id)?;
4330 if task.lift_result.is_some() {
4331 bail!(Trap::SubtaskDropNotResolved);
4332 }
4333 Waitable::Guest(id)
4334 };
4335
4336 waitable.common(concurrent_state)?.handle = None;
4337
4338 if waitable.take_event(concurrent_state)?.is_some() {
4341 bail!(Trap::SubtaskDropNotResolved);
4342 }
4343
4344 waitable.delete_from(concurrent_state)?;
4345
4346 log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4347 Ok(())
4348 }
4349
4350 pub(crate) fn waitable_set_wait(
4352 self,
4353 store: &mut StoreOpaque,
4354 options: OptionsIndex,
4355 set: u32,
4356 payload: u32,
4357 ) -> Result<u32> {
4358 let &CanonicalOptions {
4359 instance: caller_instance,
4360 ..
4361 } = &self.id().get(store).component().env_component().options[options];
4362 let caller = self.runtime_instance(caller_instance);
4363 let rep = store
4364 .instance_state(self.runtime_instance(caller_instance))
4365 .handle_table()
4366 .waitable_set_rep(set)?;
4367
4368 self.waitable_check(
4369 store,
4370 caller,
4371 WaitableCheck::Wait,
4372 WaitableCheckParams {
4373 set: TableId::new(rep),
4374 options,
4375 payload,
4376 },
4377 )
4378 }
4379
4380 pub(crate) fn waitable_set_poll(
4382 self,
4383 store: &mut StoreOpaque,
4384 options: OptionsIndex,
4385 set: u32,
4386 payload: u32,
4387 ) -> Result<u32> {
4388 let &CanonicalOptions {
4389 instance: caller_instance,
4390 ..
4391 } = &self.id().get(store).component().env_component().options[options];
4392 let caller = self.runtime_instance(caller_instance);
4393 let rep = store
4394 .instance_state(caller)
4395 .handle_table()
4396 .waitable_set_rep(set)?;
4397
4398 self.waitable_check(
4399 store,
4400 caller,
4401 WaitableCheck::Poll,
4402 WaitableCheckParams {
4403 set: TableId::new(rep),
4404 options,
4405 payload,
4406 },
4407 )
4408 }
4409
4410 pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4412 let thread_id = store.current_guest_thread()?.thread;
4413 match store
4414 .concurrent_state_mut()?
4415 .get_mut(thread_id)?
4416 .instance_rep
4417 {
4418 Some(r) => Ok(r),
4419 None => bail_bug!("thread should have instance_rep by now"),
4420 }
4421 }
4422
4423 pub(crate) fn thread_new_indirect<T: 'static>(
4425 self,
4426 mut store: StoreContextMut<T>,
4427 runtime_instance: RuntimeComponentInstanceIndex,
4428 start_func_ty: ModuleInternedTypeIndex,
4429 start_func_table_idx: RuntimeTableIndex,
4430 start_func_idx: u32,
4431 context: i32,
4432 ) -> Result<u32> {
4433 log::trace!("creating new thread");
4434
4435 let (instance, registry) = self.id().get_mut_and_registry(store.0);
4436 let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4437 else {
4438 bail_bug!("thread.new-indirect start function type should be registered");
4439 };
4440 let callee = instance
4441 .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4442 .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4443 if callee.type_index(store.0) != start_func_ty {
4444 bail!(Trap::ThreadNewIndirectInvalidType);
4445 }
4446
4447 let token = StoreToken::new(store.as_context_mut());
4448 let start_func = Box::new(
4449 move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4450 let old_thread = store.set_thread(guest_thread)?;
4451 log::trace!(
4452 "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4453 );
4454
4455 let mut store = token.as_context_mut(store);
4456 let mut params = [ValRaw::i32(context)];
4457 unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4460
4461 store.0.set_thread(old_thread)?;
4462
4463 let runtime_instance = self.runtime_instance(runtime_instance);
4464
4465 store
4468 .0
4469 .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4470
4471 store.0.cleanup_thread(guest_thread, runtime_instance)?;
4472
4473 log::trace!("explicit thread {guest_thread:?} completed");
4474 let state = store.0.concurrent_state_mut()?;
4475 if let Some(t) = old_thread.guest() {
4476 state.set_thread_running(t.thread)?;
4477 }
4478 log::trace!("thread start: restored {old_thread:?} as current thread");
4479
4480 Ok(())
4481 },
4482 );
4483
4484 let current_thread = store.0.current_guest_thread()?;
4485 let state = store.0.concurrent_state_mut()?;
4486 let parent_task = current_thread.task;
4487
4488 let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4489 let thread_id = state.push(new_thread)?;
4490 state.get_mut(parent_task)?.threads.insert(thread_id);
4491
4492 log::trace!("new thread with id {thread_id:?} created");
4493
4494 self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4495 }
4496
4497 pub(crate) fn resume_thread(
4498 self,
4499 store: &mut StoreOpaque,
4500 runtime_instance: RuntimeComponentInstanceIndex,
4501 thread_idx: u32,
4502 how: ResumeThread,
4503 ) -> Result<bool> {
4504 let thread_id =
4505 GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4506 let state = store.concurrent_state_mut()?;
4507 let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4508
4509 if store.current_guest_thread()? == guest_thread {
4510 bail!(Trap::CannotResumeThread);
4511 }
4512
4513 let priority = match how {
4514 ResumeThread::Resume => Priority::Switch,
4515 ResumeThread::ResumeLater => Priority::Low,
4516
4517 ResumeThread::Promote => {
4520 let instance = self.runtime_instance(runtime_instance);
4521 let do_not_enter = store
4522 .instance_state(instance)
4523 .concurrent_state()
4524 .do_not_enter;
4525 return store.concurrent_state_mut()?.promote_work_item_matching(
4526 |item: &WorkItem| match item {
4527 WorkItem::ResumeThread { thread, .. }
4528 | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4529
4530 WorkItem::GuestCall {
4531 call: GuestCall { thread, kind },
4532 ..
4533 } => {
4534 *thread == guest_thread
4535 && match kind {
4536 GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4540
4541 GuestCallKind::StartExplicit(_) => true,
4544
4545 GuestCallKind::StartImplicit { .. } => false,
4549 }
4550 }
4551
4552 WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4553 },
4554 );
4555 }
4556 };
4557
4558 let state = store.concurrent_state_mut()?;
4559 let thread = state.get_mut(guest_thread.thread)?;
4560
4561 match &thread.state {
4564 GuestThreadState::NotStartedExplicit(_) => {}
4565 GuestThreadState::Suspended(_) => {
4566 log::trace!("resuming thread {thread_id:?} that was suspended");
4567 state.schedule_suspended_thread(
4568 guest_thread,
4569 self.runtime_instance(runtime_instance),
4570 priority,
4571 )?;
4572 return Ok(true);
4573 }
4574 _ => bail!(Trap::CannotResumeThread),
4575 }
4576 match mem::replace(&mut thread.state, GuestThreadState::Running) {
4577 GuestThreadState::NotStartedExplicit(start_func) => {
4578 log::trace!("starting thread {guest_thread:?}");
4579 let guest_call = WorkItem::GuestCall {
4580 instance: self.runtime_instance(runtime_instance),
4581 call: GuestCall {
4582 thread: guest_thread,
4583 kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4584 start_func(store, guest_thread)
4585 })),
4586 },
4587 };
4588 state.push_work_item(guest_call, priority)?;
4589 }
4590 other => {
4591 thread.state = other;
4592 bail_bug!("thread state checked to be resumable above");
4593 }
4594 }
4595 Ok(true)
4596 }
4597
4598 fn add_guest_thread_to_instance_table(
4599 self,
4600 thread_id: TableId<GuestThread>,
4601 store: &mut StoreOpaque,
4602 runtime_instance: RuntimeComponentInstanceIndex,
4603 ) -> Result<u32> {
4604 let (state, counter) =
4605 store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4606 let guest_id = state
4607 .thread_handle_table()
4608 .guest_thread_insert(counter, thread_id.rep())?;
4609 store
4610 .concurrent_state_mut()?
4611 .get_mut(thread_id)?
4612 .instance_rep = Some(guest_id);
4613 Ok(guest_id)
4614 }
4615
4616 pub(crate) fn suspension_intrinsic(
4620 self,
4621 store: &mut StoreOpaque,
4622 caller: RuntimeComponentInstanceIndex,
4623 yielding: bool,
4624 to_thread: SuspensionTarget,
4625 ) -> Result<WaitResult> {
4626 let check_suspend = match to_thread {
4627 SuspensionTarget::Promote(thread) => {
4628 !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4629 }
4630 SuspensionTarget::Resume(thread) => {
4631 if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4632 bail_bug!(
4633 "`resume_thread` should only ever return false \
4634 when `ResumeThread::Promote` is passed to it"
4635 );
4636 }
4637 false
4638 }
4639 SuspensionTarget::None => true,
4640 };
4641
4642 if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4643 return if yielding {
4644 Ok(WaitResult::Completed)
4645 } else {
4646 Err(Trap::CannotBlockSyncTask.into())
4647 };
4648 }
4649
4650 let guest_thread = store.current_guest_thread()?;
4651
4652 let reason = if yielding {
4653 SuspendReason::Yielding {
4654 thread: guest_thread,
4655 }
4656 } else {
4657 SuspendReason::ExplicitlySuspending {
4658 thread: guest_thread,
4659 }
4660 };
4661
4662 store.suspend(reason)?;
4663
4664 Ok(WaitResult::Completed)
4665 }
4666
4667 fn waitable_check(
4669 self,
4670 store: &mut StoreOpaque,
4671 caller: RuntimeInstance,
4672 check: WaitableCheck,
4673 params: WaitableCheckParams,
4674 ) -> Result<u32> {
4675 let guest_thread = store.current_guest_thread()?;
4676
4677 log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4678
4679 match &check {
4682 WaitableCheck::Wait => {
4683 let set = params.set;
4684
4685 loop {
4690 let state = store.concurrent_state_mut()?;
4691 let task = state.get_mut(guest_thread.task)?;
4692 if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4693 || !state.get_mut(set)?.ready.is_empty()
4694 {
4695 break;
4696 }
4697
4698 store.switch_or_trap_if_may_not_suspend(caller)?;
4699
4700 store.suspend(SuspendReason::Waiting {
4701 set,
4702 thread: guest_thread,
4703 })?;
4704 }
4705 }
4706 WaitableCheck::Poll => {}
4707 }
4708
4709 log::trace!(
4710 "waitable check for {guest_thread:?}; set {:?}, part two",
4711 params.set
4712 );
4713
4714 let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4716
4717 let (ordinal, handle, result) = match &check {
4718 WaitableCheck::Wait => {
4719 let (event, waitable) = match event {
4720 Some(p) => p,
4721 None => bail_bug!("event expected to be present"),
4722 };
4723 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4724 let (ordinal, result) = event.parts();
4725 (ordinal, handle, result)
4726 }
4727 WaitableCheck::Poll => {
4728 if let Some((event, waitable)) = event {
4729 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4730 let (ordinal, result) = event.parts();
4731 (ordinal, handle, result)
4732 } else {
4733 log::trace!(
4734 "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4735 guest_thread.task,
4736 params.set
4737 );
4738 let (ordinal, result) = Event::None.parts();
4739 (ordinal, 0, result)
4740 }
4741 }
4742 };
4743 let ptr = crate::component::func::validate_inbounds_dynamic(
4744 &CanonicalAbiInfo::POINTER_PAIR,
4745 self.options_memory(store, params.options),
4746 &ValRaw::u32(params.payload),
4747 )?;
4748 let memory = self
4749 .options_memory_range_mut(store, params.options, ptr, 8)
4750 .unwrap();
4751 memory[0..4].copy_from_slice(&handle.to_le_bytes());
4752 memory[4..8].copy_from_slice(&result.to_le_bytes());
4753 Ok(ordinal)
4754 }
4755
4756 pub(crate) fn subtask_cancel(
4758 self,
4759 store: &mut StoreOpaque,
4760 caller_instance: RuntimeComponentInstanceIndex,
4761 async_: bool,
4762 task_id: u32,
4763 ) -> Result<u32> {
4764 let (rep, is_host) = store
4765 .instance_state(self.runtime_instance(caller_instance))
4766 .handle_table()
4767 .subtask_rep(task_id)?;
4768 let waitable = if is_host {
4769 Waitable::Host(TableId::<HostTask>::new(rep))
4770 } else {
4771 Waitable::Guest(TableId::<GuestTask>::new(rep))
4772 };
4773 let concurrent_state = store.concurrent_state_mut()?;
4774
4775 log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4776
4777 waitable.trap_if_in_waitable_set(concurrent_state)?;
4778
4779 let needs_block;
4780 if let Waitable::Host(host_task) = waitable {
4781 let state = &mut concurrent_state.get_mut(host_task)?.state;
4782 match state {
4783 HostTaskState::CalleeRunning(handle) => {
4790 handle.abort();
4791 *state = HostTaskState::CalleeCancelling;
4792 needs_block = true;
4793 }
4794
4795 HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4798 bail!(Trap::SubtaskCancelAfterTerminal);
4799 }
4800 HostTaskState::CalleeDone { cancelled: false } => {
4801 *state = HostTaskState::CalleeDone { cancelled: true };
4804 needs_block = false;
4805 }
4806
4807 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4810 bail_bug!("invalid states for host callee")
4811 }
4812 }
4813 } else {
4814 let guest_task = TableId::<GuestTask>::new(rep);
4815 let task = concurrent_state.get_mut(guest_task)?;
4816
4817 if task.cancel_requested {
4820 bail!(Trap::SubtaskCancelAfterTerminal);
4821 }
4822
4823 if !task.already_lowered_parameters() {
4824 store.cancel_guest_subtask_without_lowered_parameters(
4825 self.runtime_instance(caller_instance),
4826 guest_task,
4827 )?;
4828 return Ok(Status::StartCancelled as u32);
4829 } else if !task.returned_or_cancelled() {
4830 task.cancel_requested = true;
4837 task.event = Some(Event::Cancelled);
4838 let runtime_instance = task.instance;
4839 for thread in task.threads.clone() {
4840 let thread = QualifiedThreadId {
4841 task: guest_task,
4842 thread,
4843 };
4844 let concurrent_state = store.concurrent_state_mut()?;
4845 let thread_mut = concurrent_state.get_mut(thread.thread)?;
4846
4847 let yield_ = |store: &mut StoreOpaque| {
4848 let state = store.instance_state(runtime_instance).concurrent_state();
4853 let old_do_not_suspend = state.do_not_suspend;
4854 state.do_not_suspend = false;
4855
4856 let caller = store.current_guest_thread()?;
4857
4858 let state = store.concurrent_state_mut()?;
4863 let set = state.get_mut(caller.thread)?.sync_call_set;
4864 waitable.join(state, Some(set))?;
4865
4866 store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4867
4868 let state = store.concurrent_state_mut()?;
4869 waitable.join(state, None)?;
4870
4871 store
4872 .instance_state(runtime_instance)
4873 .concurrent_state()
4874 .do_not_suspend = old_do_not_suspend;
4875
4876 Ok::<(), crate::Error>(())
4877 };
4878
4879 match thread_mut.wake_on_cancel.take() {
4880 WakeOnCancel::Waiting(set) => {
4881 let instance = concurrent_state.take_callback_waiter(set, thread)?;
4883 let item = WorkItem::GuestCall {
4884 instance: runtime_instance,
4885 call: GuestCall {
4886 thread,
4887 kind: GuestCallKind::DeliverEvent {
4888 instance,
4889 set: Some(set),
4890 },
4891 },
4892 };
4893 concurrent_state.set_switch_item(item)?;
4894
4895 yield_(store)?;
4896
4897 break;
4898 }
4899 WakeOnCancel::Yielding => {
4900 if concurrent_state.promote_thread_work_item(thread)? {
4901 yield_(store)?;
4902 break;
4903 } else if store
4904 .instance_state(runtime_instance)
4905 .concurrent_state()
4906 .pending
4907 .contains_key(&thread)
4908 {
4909 store
4919 .concurrent_state_mut()?
4920 .get_mut(thread.thread)?
4921 .wake_on_cancel = WakeOnCancel::Yielding;
4922 } else {
4923 bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4924 }
4925 }
4926 WakeOnCancel::None => {}
4927 }
4928 }
4929
4930 needs_block = !store
4933 .concurrent_state_mut()?
4934 .get_mut(guest_task)?
4935 .returned_or_cancelled()
4936 } else {
4937 needs_block = false;
4938 }
4939 };
4940
4941 if needs_block {
4945 if async_ {
4946 return Ok(BLOCKED);
4947 }
4948
4949 let old_next_switch_item = if is_host {
4959 Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4960 } else {
4961 None
4962 };
4963
4964 store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4967
4968 if let Some(item) = old_next_switch_item {
4969 let state = store.concurrent_state_mut()?;
4970 state.restore_next_switch_item(item)?;
4971 }
4972
4973 }
4975
4976 let event = waitable.take_event(store.concurrent_state_mut()?)?;
4977 if let Some(Event::Subtask {
4978 status: status @ (Status::Returned | Status::ReturnCancelled),
4979 }) = event
4980 {
4981 Ok(status as u32)
4982 } else {
4983 bail!(Trap::SubtaskCancelAfterTerminal);
4984 }
4985 }
4986}
4987
4988pub trait VMComponentAsyncStore {
4996 unsafe fn prepare_call(
5002 &mut self,
5003 instance: Instance,
5004 memory: *mut VMMemoryDefinition,
5005 start: NonNull<VMFuncRef>,
5006 return_: NonNull<VMFuncRef>,
5007 caller_instance: RuntimeComponentInstanceIndex,
5008 callee_instance: RuntimeComponentInstanceIndex,
5009 task_return_type: TypeTupleIndex,
5010 callee_async: bool,
5011 string_encoding: StringEncoding,
5012 result_count: u32,
5013 storage: *mut ValRaw,
5014 storage_len: usize,
5015 ) -> Result<()>;
5016
5017 unsafe fn start_call(
5020 &mut self,
5021 instance: Instance,
5022 callback: *mut VMFuncRef,
5023 post_return: *mut VMFuncRef,
5024 callee: NonNull<VMFuncRef>,
5025 param_count: u32,
5026 result_count: u32,
5027 flags: u32,
5028 storage: *mut MaybeUninit<ValRaw>,
5029 storage_len: usize,
5030 ) -> Result<()>;
5031
5032 fn future_write(
5034 &mut self,
5035 instance: Instance,
5036 caller: RuntimeComponentInstanceIndex,
5037 ty: TypeFutureTableIndex,
5038 options: OptionsIndex,
5039 future: u32,
5040 address: u32,
5041 ) -> Result<u32>;
5042
5043 fn future_read(
5045 &mut self,
5046 instance: Instance,
5047 caller: RuntimeComponentInstanceIndex,
5048 ty: TypeFutureTableIndex,
5049 options: OptionsIndex,
5050 future: u32,
5051 address: u32,
5052 ) -> Result<u32>;
5053
5054 fn future_drop_writable(
5056 &mut self,
5057 instance: Instance,
5058 ty: TypeFutureTableIndex,
5059 writer: u32,
5060 ) -> Result<()>;
5061
5062 fn stream_write(
5064 &mut self,
5065 instance: Instance,
5066 caller: RuntimeComponentInstanceIndex,
5067 ty: TypeStreamTableIndex,
5068 options: OptionsIndex,
5069 stream: u32,
5070 address: u32,
5071 count: u32,
5072 ) -> Result<u32>;
5073
5074 fn stream_read(
5076 &mut self,
5077 instance: Instance,
5078 caller: RuntimeComponentInstanceIndex,
5079 ty: TypeStreamTableIndex,
5080 options: OptionsIndex,
5081 stream: u32,
5082 address: u32,
5083 count: u32,
5084 ) -> Result<u32>;
5085
5086 fn flat_stream_write(
5089 &mut self,
5090 instance: Instance,
5091 caller: RuntimeComponentInstanceIndex,
5092 ty: TypeStreamTableIndex,
5093 options: OptionsIndex,
5094 payload_size: u32,
5095 payload_align: u32,
5096 stream: u32,
5097 address: u32,
5098 count: u32,
5099 ) -> Result<u32>;
5100
5101 fn flat_stream_read(
5104 &mut self,
5105 instance: Instance,
5106 caller: RuntimeComponentInstanceIndex,
5107 ty: TypeStreamTableIndex,
5108 options: OptionsIndex,
5109 payload_size: u32,
5110 payload_align: u32,
5111 stream: u32,
5112 address: u32,
5113 count: u32,
5114 ) -> Result<u32>;
5115
5116 fn stream_drop_writable(
5118 &mut self,
5119 instance: Instance,
5120 ty: TypeStreamTableIndex,
5121 writer: u32,
5122 ) -> Result<()>;
5123
5124 fn error_context_debug_message(
5126 &mut self,
5127 instance: Instance,
5128 ty: TypeComponentLocalErrorContextTableIndex,
5129 options: OptionsIndex,
5130 err_ctx_handle: u32,
5131 debug_msg_address: u32,
5132 ) -> Result<()>;
5133
5134 fn thread_new_indirect(
5136 &mut self,
5137 instance: Instance,
5138 caller: RuntimeComponentInstanceIndex,
5139 func_ty_idx: ModuleInternedTypeIndex,
5140 start_func_table_idx: RuntimeTableIndex,
5141 start_func_idx: u32,
5142 context: i32,
5143 ) -> Result<u32>;
5144}
5145
5146impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5148 unsafe fn prepare_call(
5149 &mut self,
5150 instance: Instance,
5151 memory: *mut VMMemoryDefinition,
5152 start: NonNull<VMFuncRef>,
5153 return_: NonNull<VMFuncRef>,
5154 caller_instance: RuntimeComponentInstanceIndex,
5155 callee_instance: RuntimeComponentInstanceIndex,
5156 task_return_type: TypeTupleIndex,
5157 callee_async: bool,
5158 string_encoding: StringEncoding,
5159 result_count_or_max_if_async: u32,
5160 storage: *mut ValRaw,
5161 storage_len: usize,
5162 ) -> Result<()> {
5163 let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5167
5168 unsafe {
5169 instance.prepare_call(
5170 StoreContextMut(self),
5171 start,
5172 return_,
5173 caller_instance,
5174 callee_instance,
5175 task_return_type,
5176 callee_async,
5177 memory,
5178 string_encoding,
5179 match result_count_or_max_if_async {
5180 PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5181 params,
5182 has_result: false,
5183 },
5184 PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5185 params,
5186 has_result: true,
5187 },
5188 result_count => CallerInfo::Sync {
5189 params,
5190 result_count,
5191 },
5192 },
5193 )
5194 }
5195 }
5196
5197 unsafe fn start_call(
5198 &mut self,
5199 instance: Instance,
5200 callback: *mut VMFuncRef,
5201 post_return: *mut VMFuncRef,
5202 callee: NonNull<VMFuncRef>,
5203 param_count: u32,
5204 result_count: u32,
5205 flags: u32,
5206 storage: *mut MaybeUninit<ValRaw>,
5207 storage_len: usize,
5208 ) -> Result<()> {
5209 unsafe {
5210 instance.start_call(
5211 StoreContextMut(self),
5212 callback,
5213 post_return,
5214 callee,
5215 param_count,
5216 result_count,
5217 flags,
5218 core::slice::from_raw_parts_mut(storage, storage_len),
5222 )
5223 }
5224 }
5225
5226 fn future_write(
5227 &mut self,
5228 instance: Instance,
5229 caller: RuntimeComponentInstanceIndex,
5230 ty: TypeFutureTableIndex,
5231 options: OptionsIndex,
5232 future: u32,
5233 address: u32,
5234 ) -> Result<u32> {
5235 instance
5236 .guest_write(
5237 StoreContextMut(self),
5238 caller,
5239 TransmitIndex::Future(ty),
5240 options,
5241 None,
5242 future,
5243 address,
5244 1,
5245 )
5246 .map(|result| result.encode())
5247 }
5248
5249 fn future_read(
5250 &mut self,
5251 instance: Instance,
5252 caller: RuntimeComponentInstanceIndex,
5253 ty: TypeFutureTableIndex,
5254 options: OptionsIndex,
5255 future: u32,
5256 address: u32,
5257 ) -> Result<u32> {
5258 instance
5259 .guest_read(
5260 StoreContextMut(self),
5261 caller,
5262 TransmitIndex::Future(ty),
5263 options,
5264 None,
5265 future,
5266 address,
5267 1,
5268 )
5269 .map(|result| result.encode())
5270 }
5271
5272 fn stream_write(
5273 &mut self,
5274 instance: Instance,
5275 caller: RuntimeComponentInstanceIndex,
5276 ty: TypeStreamTableIndex,
5277 options: OptionsIndex,
5278 stream: u32,
5279 address: u32,
5280 count: u32,
5281 ) -> Result<u32> {
5282 instance
5283 .guest_write(
5284 StoreContextMut(self),
5285 caller,
5286 TransmitIndex::Stream(ty),
5287 options,
5288 None,
5289 stream,
5290 address,
5291 count,
5292 )
5293 .map(|result| result.encode())
5294 }
5295
5296 fn stream_read(
5297 &mut self,
5298 instance: Instance,
5299 caller: RuntimeComponentInstanceIndex,
5300 ty: TypeStreamTableIndex,
5301 options: OptionsIndex,
5302 stream: u32,
5303 address: u32,
5304 count: u32,
5305 ) -> Result<u32> {
5306 instance
5307 .guest_read(
5308 StoreContextMut(self),
5309 caller,
5310 TransmitIndex::Stream(ty),
5311 options,
5312 None,
5313 stream,
5314 address,
5315 count,
5316 )
5317 .map(|result| result.encode())
5318 }
5319
5320 fn future_drop_writable(
5321 &mut self,
5322 instance: Instance,
5323 ty: TypeFutureTableIndex,
5324 writer: u32,
5325 ) -> Result<()> {
5326 instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5327 }
5328
5329 fn flat_stream_write(
5330 &mut self,
5331 instance: Instance,
5332 caller: RuntimeComponentInstanceIndex,
5333 ty: TypeStreamTableIndex,
5334 options: OptionsIndex,
5335 payload_size: u32,
5336 payload_align: u32,
5337 stream: u32,
5338 address: u32,
5339 count: u32,
5340 ) -> Result<u32> {
5341 instance
5342 .guest_write(
5343 StoreContextMut(self),
5344 caller,
5345 TransmitIndex::Stream(ty),
5346 options,
5347 Some(FlatAbi {
5348 size: payload_size,
5349 align: payload_align,
5350 }),
5351 stream,
5352 address,
5353 count,
5354 )
5355 .map(|result| result.encode())
5356 }
5357
5358 fn flat_stream_read(
5359 &mut self,
5360 instance: Instance,
5361 caller: RuntimeComponentInstanceIndex,
5362 ty: TypeStreamTableIndex,
5363 options: OptionsIndex,
5364 payload_size: u32,
5365 payload_align: u32,
5366 stream: u32,
5367 address: u32,
5368 count: u32,
5369 ) -> Result<u32> {
5370 instance
5371 .guest_read(
5372 StoreContextMut(self),
5373 caller,
5374 TransmitIndex::Stream(ty),
5375 options,
5376 Some(FlatAbi {
5377 size: payload_size,
5378 align: payload_align,
5379 }),
5380 stream,
5381 address,
5382 count,
5383 )
5384 .map(|result| result.encode())
5385 }
5386
5387 fn stream_drop_writable(
5388 &mut self,
5389 instance: Instance,
5390 ty: TypeStreamTableIndex,
5391 writer: u32,
5392 ) -> Result<()> {
5393 instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5394 }
5395
5396 fn error_context_debug_message(
5397 &mut self,
5398 instance: Instance,
5399 ty: TypeComponentLocalErrorContextTableIndex,
5400 options: OptionsIndex,
5401 err_ctx_handle: u32,
5402 debug_msg_address: u32,
5403 ) -> Result<()> {
5404 instance.error_context_debug_message(
5405 StoreContextMut(self),
5406 ty,
5407 options,
5408 err_ctx_handle,
5409 debug_msg_address,
5410 )
5411 }
5412
5413 fn thread_new_indirect(
5414 &mut self,
5415 instance: Instance,
5416 caller: RuntimeComponentInstanceIndex,
5417 func_ty_idx: ModuleInternedTypeIndex,
5418 start_func_table_idx: RuntimeTableIndex,
5419 start_func_idx: u32,
5420 context: i32,
5421 ) -> Result<u32> {
5422 instance.thread_new_indirect(
5423 StoreContextMut(self),
5424 caller,
5425 func_ty_idx,
5426 start_func_table_idx,
5427 start_func_idx,
5428 context,
5429 )
5430 }
5431}
5432
5433type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5434
5435async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5438where
5439 F: Future,
5440{
5441 let mut future = pin!(future);
5442 future::poll_fn(|cx| {
5443 let old_thread = match tls::get(|store| store.set_thread(task)) {
5444 Ok(thread) => thread,
5445 Err(error) => return Poll::Ready(Err(error)),
5446 };
5447 let result = future.as_mut().poll(cx);
5448 match tls::get(|store| store.set_thread(old_thread)) {
5449 Ok(_) => result.map(Ok),
5450 Err(error) => Poll::Ready(Err(error)),
5451 }
5452 })
5453 .await
5454}
5455
5456pub(crate) struct HostTask {
5460 common: WaitableCommon,
5461
5462 call_context: CallContext,
5465
5466 state: HostTaskState,
5467
5468 group: TaskGroupId,
5469}
5470
5471enum HostTaskState {
5472 CalleeStarted,
5477
5478 CalleeRunning(JoinHandle),
5483
5484 CalleeCancelling,
5488
5489 CalleeFinished(LiftedResult),
5493
5494 CalleeDone { cancelled: bool },
5497}
5498
5499impl HostTask {
5500 fn new(
5501 concurrent_state: &mut ConcurrentState,
5502 state: HostTaskState,
5503 caller: QualifiedThreadId,
5504 ) -> Result<Self> {
5505 let group = concurrent_state.get_mut(caller.task)?.group;
5506 concurrent_state.increment_group_ref_count(group)?;
5507
5508 Ok(Self {
5509 common: WaitableCommon::default(),
5510 call_context: CallContext::default(),
5511 state,
5512 group,
5513 })
5514 }
5515}
5516
5517impl TableDebug for HostTask {
5518 fn type_name() -> &'static str {
5519 "HostTask"
5520 }
5521}
5522
5523type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5524
5525enum Caller {
5527 Host {
5529 tx: Option<oneshot::Sender<LiftedResult>>,
5531 caller: Option<TableId<HostTask>>,
5535 },
5536 Guest {
5538 thread: QualifiedThreadId,
5540 },
5541}
5542
5543struct LiftResult {
5546 lift: RawLift,
5547 ty: TypeTupleIndex,
5548 memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5549 string_encoding: StringEncoding,
5550}
5551
5552#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5557pub(crate) struct QualifiedThreadId {
5558 task: TableId<GuestTask>,
5559 thread: TableId<GuestThread>,
5560}
5561
5562impl QualifiedThreadId {
5563 fn qualify(
5564 state: &mut ConcurrentState,
5565 thread: TableId<GuestThread>,
5566 ) -> Result<QualifiedThreadId> {
5567 Ok(QualifiedThreadId {
5568 task: state.get_mut(thread)?.parent_task,
5569 thread,
5570 })
5571 }
5572}
5573
5574impl fmt::Debug for QualifiedThreadId {
5575 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5576 f.debug_tuple("QualifiedThreadId")
5577 .field(&self.task.rep())
5578 .field(&self.thread.rep())
5579 .finish()
5580 }
5581}
5582
5583enum GuestThreadState {
5584 NotStartedImplicit,
5585 NotStartedExplicit(
5586 Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5587 ),
5588 Running,
5589 Suspended(StoreFiber<'static>),
5590 Ready {
5591 fiber: StoreFiber<'static>,
5592 },
5593 Completed,
5594}
5595
5596impl GuestThreadState {
5597 fn check_no_fiber(&self) -> Result<()> {
5598 match self {
5599 Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5600 Self::NotStartedImplicit
5601 | Self::NotStartedExplicit(_)
5602 | Self::Running
5603 | Self::Completed => Ok(()),
5604 }
5605 }
5606}
5607
5608impl fmt::Debug for GuestThreadState {
5609 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5610 match self {
5611 Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5612 Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5613 Self::Running => f.debug_tuple("Running").finish(),
5614 Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5615 Self::Ready { .. } => f.debug_struct("Ready").finish(),
5616 Self::Completed => f.debug_tuple("Completed").finish(),
5617 }
5618 }
5619}
5620
5621#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5622enum WakeOnCancel {
5623 None,
5624 Waiting(TableId<WaitableSet>),
5625 Yielding,
5626}
5627
5628impl WakeOnCancel {
5629 fn is_none(self) -> bool {
5630 matches!(self, WakeOnCancel::None)
5631 }
5632
5633 fn replace(&mut self, other: WakeOnCancel) -> Self {
5634 let old = *self;
5635 *self = other;
5636 old
5637 }
5638
5639 fn take(&mut self) -> Self {
5640 self.replace(WakeOnCancel::None)
5641 }
5642}
5643
5644pub struct GuestThread {
5645 context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5648 parent_task: TableId<GuestTask>,
5650 wake_on_cancel: WakeOnCancel,
5653 state: GuestThreadState,
5655 instance_rep: Option<u32>,
5658 sync_call_set: TableId<WaitableSet>,
5660 old_do_not_suspend: Option<bool>,
5663}
5664
5665impl GuestThread {
5666 fn from_instance(
5669 state: Pin<&mut ComponentInstance>,
5670 caller_instance: RuntimeComponentInstanceIndex,
5671 guest_thread: u32,
5672 ) -> Result<TableId<Self>> {
5673 let rep = state.instance_states().0[caller_instance]
5674 .thread_handle_table()
5675 .guest_thread_rep(guest_thread)?;
5676 Ok(TableId::new(rep))
5677 }
5678
5679 fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5680 let sync_call_set = state.push(WaitableSet {
5681 is_sync_call_set: true,
5682 ..WaitableSet::default()
5683 })?;
5684 Ok(Self {
5685 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5686 parent_task,
5687 wake_on_cancel: WakeOnCancel::None,
5688 state: GuestThreadState::NotStartedImplicit,
5689 instance_rep: None,
5690 sync_call_set,
5691 old_do_not_suspend: None,
5692 })
5693 }
5694
5695 fn new_explicit(
5696 state: &mut ConcurrentState,
5697 parent_task: TableId<GuestTask>,
5698 start_func: Box<
5699 dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5700 >,
5701 ) -> Result<Self> {
5702 state.increment_task_ref_count(parent_task)?;
5703 let sync_call_set = state.push(WaitableSet {
5704 is_sync_call_set: true,
5705 ..WaitableSet::default()
5706 })?;
5707 Ok(Self {
5708 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5709 parent_task,
5710 wake_on_cancel: WakeOnCancel::None,
5711 state: GuestThreadState::NotStartedExplicit(start_func),
5712 instance_rep: None,
5713 sync_call_set,
5714 old_do_not_suspend: None,
5715 })
5716 }
5717}
5718
5719impl TableDebug for GuestThread {
5720 fn type_name() -> &'static str {
5721 "GuestThread"
5722 }
5723}
5724
5725enum SyncResult {
5726 NotProduced,
5727 Produced(Option<ValRaw>),
5728 Taken,
5729}
5730
5731impl SyncResult {
5732 fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5733 Ok(match mem::replace(self, SyncResult::Taken) {
5734 SyncResult::NotProduced => None,
5735 SyncResult::Produced(val) => Some(val),
5736 SyncResult::Taken => {
5737 bail_bug!("attempted to take a synchronous result that was already taken")
5738 }
5739 })
5740 }
5741}
5742
5743pub(crate) struct GuestTask {
5745 common: WaitableCommon,
5747 lower_params: Option<RawLower>,
5749 lift_result: Option<LiftResult>,
5751 result: Option<LiftedResult>,
5754 callback: Option<CallbackFn>,
5757 caller: Caller,
5759 call_context: CallContext,
5764 sync_result: SyncResult,
5767 cancel_request_delivered: bool,
5771 cancel_requested: bool,
5776 starting_sent: bool,
5779 instance: RuntimeInstance,
5786 event: Option<Event>,
5788 exited: bool,
5790 threads: HashSet<TableId<GuestThread>>,
5792 async_typed: bool,
5795 async_lifted: bool,
5798 decremented_interesting_task_count: bool,
5801 group: TaskGroupId,
5803 ref_count: usize,
5808}
5809
5810impl GuestTask {
5811 fn already_lowered_parameters(&self) -> bool {
5812 self.lower_params.is_none()
5814 }
5815
5816 fn returned_or_cancelled(&self) -> bool {
5817 self.lift_result.is_none()
5819 }
5820
5821 fn new(
5822 state: &mut ConcurrentState,
5823 lower_params: RawLower,
5824 lift_result: LiftResult,
5825 caller: Caller,
5826 callback: Option<CallbackFn>,
5827 instance: RuntimeInstance,
5828 async_typed: bool,
5829 async_lifted: bool,
5830 ) -> Result<QualifiedThreadId> {
5831 let group = match caller {
5832 Caller::Guest { thread } => {
5833 let group = state.get_mut(thread.task)?.group;
5834 state.increment_group_ref_count(group)?;
5835 group
5836 }
5837 Caller::Host { .. } => state.make_task_group()?,
5838 };
5839
5840 let task = state.push(Self {
5841 common: WaitableCommon::default(),
5842 lower_params: Some(lower_params),
5843 lift_result: Some(lift_result),
5844 result: None,
5845 callback,
5846 caller,
5847 call_context: CallContext::default(),
5848 sync_result: SyncResult::NotProduced,
5849 cancel_request_delivered: false,
5850 cancel_requested: false,
5851 starting_sent: false,
5852 instance,
5853 event: None,
5854 exited: false,
5855 threads: HashSet::new(),
5856 async_typed,
5857 async_lifted,
5858 decremented_interesting_task_count: false,
5859 group,
5860 ref_count: 2,
5863 })?;
5864 let new_thread = GuestThread::new_implicit(state, task)?;
5865 let thread = state.push(new_thread)?;
5866 state.get_mut(task)?.threads.insert(thread);
5867 state.interesting_tasks += 1;
5868 let thread = QualifiedThreadId { task, thread };
5869 log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5870 Ok(thread)
5871 }
5872}
5873
5874impl TableDebug for GuestTask {
5875 fn type_name() -> &'static str {
5876 "GuestTask"
5877 }
5878}
5879
5880#[derive(Default)]
5882struct WaitableCommon {
5883 event: Option<Event>,
5885 set: Option<TableId<WaitableSet>>,
5887 handle: Option<u32>,
5889}
5890
5891#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5893enum Waitable {
5894 Host(TableId<HostTask>),
5896 Guest(TableId<GuestTask>),
5898 Transmit(TableId<TransmitHandle>),
5900}
5901
5902impl Waitable {
5903 fn from_instance(
5906 state: Pin<&mut ComponentInstance>,
5907 caller_instance: RuntimeComponentInstanceIndex,
5908 waitable: u32,
5909 ) -> Result<Self> {
5910 use crate::runtime::vm::component::Waitable;
5911
5912 let (waitable, kind) = state.instance_states().0[caller_instance]
5913 .handle_table()
5914 .waitable_rep(waitable)?;
5915
5916 Ok(match kind {
5917 Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5918 Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5919 Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5920 })
5921 }
5922
5923 fn rep(&self) -> u32 {
5925 match self {
5926 Self::Host(id) => id.rep(),
5927 Self::Guest(id) => id.rep(),
5928 Self::Transmit(id) => id.rep(),
5929 }
5930 }
5931
5932 fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5936 log::trace!("waitable {self:?} join set {set:?}");
5937
5938 let old = mem::replace(&mut self.common(state)?.set, set);
5939
5940 if let Some(old) = old {
5941 match *self {
5942 Waitable::Host(id) => state.remove_child(id, old),
5943 Waitable::Guest(id) => state.remove_child(id, old),
5944 Waitable::Transmit(id) => state.remove_child(id, old),
5945 }?;
5946
5947 state.get_mut(old)?.ready.remove(self);
5948 }
5949
5950 if let Some(set) = set {
5951 match *self {
5952 Waitable::Host(id) => state.add_child(id, set),
5953 Waitable::Guest(id) => state.add_child(id, set),
5954 Waitable::Transmit(id) => state.add_child(id, set),
5955 }?;
5956
5957 if self.common(state)?.event.is_some() {
5958 self.mark_ready(state)?;
5959 }
5960 }
5961
5962 Ok(())
5963 }
5964
5965 fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5967 Ok(match self {
5968 Self::Host(id) => &mut state.get_mut(*id)?.common,
5969 Self::Guest(id) => &mut state.get_mut(*id)?.common,
5970 Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5971 })
5972 }
5973
5974 fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5980 if self.common(state)?.set.is_some() {
5981 bail!(Trap::WaitableSyncAndAsync);
5982 }
5983 Ok(())
5984 }
5985
5986 fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5992 if let Some(set) = self.common(state)?.set
5993 && state.get_mut(set)?.is_sync_call_set
5994 {
5995 bail!(Trap::WaitableSyncAndAsync);
5996 }
5997 Ok(())
5998 }
5999
6000 fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
6004 log::trace!("set event for {self:?}: {event:?}");
6005 self.common(state)?.event = event;
6006 self.mark_ready(state)
6007 }
6008
6009 fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
6011 let common = self.common(state)?;
6012 let event = common.event.take();
6013 if let Some(set) = self.common(state)?.set {
6014 state.get_mut(set)?.ready.remove(self);
6015 }
6016
6017 Ok(event)
6018 }
6019
6020 fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
6024 if let Some(set) = self.common(state)?.set {
6025 state.get_mut(set)?.ready.insert(*self);
6026 state.wake_waiter(set)?;
6027 }
6028 Ok(())
6029 }
6030
6031 fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
6034 match self {
6035 Self::Host(task) => state.delete_host_task(*task),
6036 Self::Guest(task) => state.decrement_task_ref_count(*task),
6037 Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
6038 }
6039 }
6040}
6041
6042impl fmt::Debug for Waitable {
6043 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6044 match self {
6045 Self::Host(id) => write!(f, "{id:?}"),
6046 Self::Guest(id) => write!(f, "{id:?}"),
6047 Self::Transmit(id) => write!(f, "{id:?}"),
6048 }
6049 }
6050}
6051
6052#[derive(Default)]
6054struct WaitableSet {
6055 ready: BTreeSet<Waitable>,
6057 waiting: BTreeMap<QualifiedThreadId, WaitMode>,
6059 num_waiting: usize,
6062 is_sync_call_set: bool,
6065}
6066
6067impl WaitableSet {
6068 fn stop_waiting(&mut self) -> Result<()> {
6070 self.num_waiting = match self.num_waiting.checked_sub(1) {
6071 Some(n) => n,
6072 None => bail_bug!("waiter not accounted for in waitable set"),
6073 };
6074 Ok(())
6075 }
6076}
6077
6078impl TableDebug for WaitableSet {
6079 fn type_name() -> &'static str {
6080 "WaitableSet"
6081 }
6082}
6083
6084type RawLower =
6086 Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
6087
6088type RawLift = Box<
6090 dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6091>;
6092
6093type LiftedResult = Box<dyn Any + Send + Sync>;
6097
6098struct DummyResult;
6101
6102#[derive(Default)]
6104pub struct ConcurrentInstanceState {
6105 backpressure: u16,
6107 do_not_enter: bool,
6109 do_not_suspend: bool,
6112 pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
6115 pending_start_count: usize,
6118}
6119
6120impl ConcurrentInstanceState {
6121 pub fn pending_is_empty(&self) -> bool {
6122 self.pending.is_empty()
6123 }
6124}
6125
6126#[derive(Debug, Copy, Clone)]
6127pub(crate) enum CurrentThread {
6128 Guest(QualifiedThreadId),
6131 Host(TableId<HostTask>),
6133 DeferredHost(QualifiedThreadId),
6136 None,
6139}
6140
6141impl CurrentThread {
6142 fn guest(&self) -> Option<&QualifiedThreadId> {
6143 match self {
6144 Self::Guest(id) => Some(id),
6145 _ => None,
6146 }
6147 }
6148
6149 fn guest_task(&self) -> Option<TableId<GuestTask>> {
6150 match self {
6151 Self::Guest(id) => Some(id.task),
6152 _ => None,
6153 }
6154 }
6155
6156 fn is_none(&self) -> bool {
6157 matches!(self, Self::None)
6158 }
6159}
6160
6161impl From<QualifiedThreadId> for CurrentThread {
6162 fn from(id: QualifiedThreadId) -> Self {
6163 Self::Guest(id)
6164 }
6165}
6166
6167impl From<TableId<HostTask>> for CurrentThread {
6168 fn from(id: TableId<HostTask>) -> Self {
6169 Self::Host(id)
6170 }
6171}
6172
6173enum Priority {
6174 Switch,
6175 #[expect(
6176 dead_code,
6177 reason = "Here for completeness and in case we need it later"
6178 )]
6179 High,
6180 Low,
6181}
6182
6183pub struct ConcurrentState {
6185 unforced_current_thread: CurrentThread,
6191
6192 deferred_host_call_context: Option<CallContext>,
6198
6199 futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6204 table: AlwaysMut<ResourceTable>,
6206 switch_item: Option<WorkItem>,
6214 next_switch_item: Option<WorkItem>,
6220 high_priority: VecDeque<WorkItem>,
6222 low_priority: VecDeque<WorkItem>,
6224 suspend_reason: Option<SuspendReason>,
6228 worker: Option<StoreFiber<'static>>,
6232 worker_item: Option<WorkerItem>,
6234
6235 global_error_context_ref_counts:
6248 BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6249
6250 interesting_tasks: usize,
6263
6264 interesting_tasks_empty_waker: Option<Waker>,
6268
6269 ready_for_concurrent_call_waker: Option<Waker>,
6274
6275 event_loop_running: bool,
6277
6278 saved_next_switch_items: Vec<Option<WorkItem>>,
6281
6282 #[cfg(feature = "task-group-hook")]
6284 task_group_hook: Option<Box<dyn TaskGroupHook>>,
6285}
6286
6287impl Default for ConcurrentState {
6288 fn default() -> Self {
6289 Self {
6290 unforced_current_thread: CurrentThread::None,
6291 deferred_host_call_context: None,
6292 table: AlwaysMut::new(ResourceTable::new()),
6293 futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6294 switch_item: None,
6295 next_switch_item: None,
6296 high_priority: VecDeque::new(),
6297 low_priority: VecDeque::new(),
6298 suspend_reason: None,
6299 worker: None,
6300 worker_item: None,
6301 global_error_context_ref_counts: BTreeMap::new(),
6302 interesting_tasks: 0,
6303 interesting_tasks_empty_waker: None,
6304 ready_for_concurrent_call_waker: None,
6305 event_loop_running: false,
6306 saved_next_switch_items: Vec::new(),
6307 #[cfg(feature = "task-group-hook")]
6308 task_group_hook: None,
6309 }
6310 }
6311}
6312
6313impl ConcurrentState {
6314 pub(crate) fn take_fibers_and_futures(
6331 &mut self,
6332 fibers: &mut Vec<StoreFiber<'static>>,
6333 futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6334 ) {
6335 let ConcurrentState {
6336 table,
6337 worker,
6338 switch_item,
6339 next_switch_item,
6340 high_priority,
6341 low_priority,
6342 saved_next_switch_items,
6343 futures: pending_futures,
6344 worker_item: _,
6345 unforced_current_thread: _,
6346 deferred_host_call_context: _,
6347 suspend_reason: _,
6348 global_error_context_ref_counts: _,
6349 interesting_tasks: _,
6350 interesting_tasks_empty_waker: _,
6351 ready_for_concurrent_call_waker: _,
6352 event_loop_running: _,
6353 #[cfg(feature = "task-group-hook")]
6354 task_group_hook: _,
6355 } = self;
6356
6357 let mut items = Vec::new();
6358 for (_, entry) in table.get_mut().iter_mut() {
6359 let entry = entry.into_entry_mut();
6360 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6361 for mode in mem::take(&mut set.waiting).into_values() {
6362 match mode {
6363 WaitMode::Fiber(fiber) => {
6364 fibers.push(fiber);
6365 }
6366 WaitMode::Callback(_) => {}
6367 }
6368 }
6369 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6370 match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6371 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6372 fibers.push(fiber);
6373 }
6374 GuestThreadState::NotStartedImplicit
6375 | GuestThreadState::NotStartedExplicit(_)
6376 | GuestThreadState::Running
6377 | GuestThreadState::Completed => {}
6378 }
6379 } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6380 if let Some(item) = item.take() {
6381 items.push(item);
6382 }
6383 }
6384 }
6385
6386 if let Some(fiber) = worker.take() {
6387 fibers.push(fiber);
6388 }
6389
6390 let mut handle_item = |item| match item {
6391 WorkItem::ResumeFiber { fiber, .. } => {
6392 fibers.push(fiber);
6393 }
6394 WorkItem::PushFuture(future) => {
6395 pending_futures
6396 .get_mut()
6397 .as_mut()
6398 .unwrap()
6399 .push(future.into_inner());
6400 }
6401 WorkItem::ResumeThread { .. }
6402 | WorkItem::GuestCall { .. }
6403 | WorkItem::WorkerFunction(_) => {}
6404 };
6405
6406 for item in items {
6407 handle_item(item);
6408 }
6409 if let Some(item) = switch_item.take() {
6410 handle_item(item);
6411 }
6412 if let Some(item) = next_switch_item.take() {
6413 handle_item(item);
6414 }
6415 for item in mem::take(high_priority) {
6416 handle_item(item);
6417 }
6418 for item in mem::take(low_priority) {
6419 handle_item(item);
6420 }
6421 for item in mem::take(saved_next_switch_items)
6422 .into_iter()
6423 .filter_map(|v| v)
6424 {
6425 handle_item(item);
6426 }
6427
6428 if let Some(them) = pending_futures.get_mut().take() {
6429 futures.push(them);
6430 }
6431 }
6432
6433 #[cfg(feature = "gc")]
6434 pub(crate) fn trace_fiber_roots(
6435 &mut self,
6436 modules: &ModuleRegistry,
6437 unwind: &dyn Unwind,
6438 gc_roots_list: &mut GcRootsList,
6439 ) {
6440 let ConcurrentState {
6441 table,
6442 worker,
6443 switch_item,
6444 next_switch_item,
6445 high_priority,
6446 low_priority,
6447 saved_next_switch_items,
6448
6449 futures: _,
6453
6454 worker_item: _,
6456 unforced_current_thread: _,
6457 deferred_host_call_context: _,
6458 suspend_reason: _,
6459 global_error_context_ref_counts: _,
6460 interesting_tasks: _,
6461 interesting_tasks_empty_waker: _,
6462 ready_for_concurrent_call_waker: _,
6463 event_loop_running: _,
6464 #[cfg(feature = "task-group-hook")]
6465 task_group_hook: _,
6466 } = self;
6467
6468 for (_, entry) in table.get_mut().iter_mut() {
6469 let entry = entry.into_entry_mut();
6470 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6471 for mode in set.waiting.values_mut() {
6472 match mode {
6473 WaitMode::Fiber(fiber) => {
6474 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6475 }
6476 WaitMode::Callback(_) => {}
6477 }
6478 }
6479 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6480 match &mut thread.state {
6481 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6482 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6483 }
6484 GuestThreadState::NotStartedImplicit
6485 | GuestThreadState::NotStartedExplicit(_)
6486 | GuestThreadState::Running
6487 | GuestThreadState::Completed => {}
6488 }
6489 } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6490 entry.downcast_mut::<Option<WorkItem>>()
6491 {
6492 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6493 }
6494 }
6495
6496 if let Some(fiber) = worker {
6497 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6498 }
6499
6500 let mut handle_item = |item: &mut WorkItem| match item {
6501 WorkItem::ResumeFiber { fiber, .. } => {
6502 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6503 }
6504 WorkItem::PushFuture(_future) => {
6505 }
6508 WorkItem::ResumeThread { .. }
6509 | WorkItem::GuestCall { .. }
6510 | WorkItem::WorkerFunction(_) => {}
6511 };
6512
6513 if let Some(item) = switch_item {
6514 handle_item(item);
6515 }
6516 if let Some(item) = next_switch_item {
6517 handle_item(item);
6518 }
6519 for item in high_priority {
6520 handle_item(item);
6521 }
6522 for item in low_priority {
6523 handle_item(item);
6524 }
6525 for item in saved_next_switch_items
6526 .iter_mut()
6527 .filter_map(|v| v.as_mut())
6528 {
6529 handle_item(item);
6530 }
6531 }
6532
6533 fn schedule_suspended_thread(
6534 &mut self,
6535 thread: QualifiedThreadId,
6536 instance: RuntimeInstance,
6537 priority: Priority,
6538 ) -> Result<()> {
6539 if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6541 bail_bug!("switch item already set");
6542 }
6543 let slot = &mut self.get_mut(thread.thread)?.state;
6544 if !matches!(slot, GuestThreadState::Suspended(_)) {
6545 bail!(Trap::CannotResumeThread);
6546 }
6547 let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6548 else {
6549 unreachable!()
6550 };
6551 let item = WorkItem::ResumeFiber {
6552 instance,
6553 thread,
6554 fiber,
6555 };
6556 match priority {
6557 Priority::Switch => self.switch_item = Some(item),
6558 Priority::High => self.push_high_priority(item),
6559 Priority::Low => self.push_low_priority(item),
6560 }
6561 Ok(())
6562 }
6563
6564 fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6565 let slot = &mut self.get_mut(thread)?.state;
6566 slot.check_no_fiber()?;
6567 *slot = GuestThreadState::Running;
6568 Ok(())
6569 }
6570
6571 fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6572 let saved = self.push(None::<WorkItem>)?;
6575 let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6576 *slot = self.next_switch_item.take();
6577 Ok(saved)
6578 }
6579
6580 fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6581 if self.next_switch_item.is_some() {
6582 bail_bug!("next switch item already set when restoring");
6583 }
6584 self.next_switch_item = self.delete(saved)?;
6585 Ok(())
6586 }
6587
6588 fn take_callback_waiter(
6589 &mut self,
6590 set: TableId<WaitableSet>,
6591 thread: QualifiedThreadId,
6592 ) -> Result<Instance> {
6593 let waiting = &mut self.get_mut(set)?.waiting;
6594 let instance = match waiting.get(&thread) {
6595 Some(WaitMode::Callback(instance)) => *instance,
6596 other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6597 };
6598 waiting.remove(&thread);
6599 Ok(instance)
6600 }
6601
6602 fn push<V: Send + Sync + 'static>(
6603 &mut self,
6604 value: V,
6605 ) -> Result<TableId<V>, ResourceTableError> {
6606 self.table.get_mut().push(value).map(TableId::from)
6607 }
6608
6609 fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6610 self.table.get_mut().get_mut(&Resource::from(id))
6611 }
6612
6613 pub fn add_child<T: 'static, U: 'static>(
6614 &mut self,
6615 child: TableId<T>,
6616 parent: TableId<U>,
6617 ) -> Result<(), ResourceTableError> {
6618 self.table
6619 .get_mut()
6620 .add_child(Resource::from(child), Resource::from(parent))
6621 }
6622
6623 pub fn remove_child<T: 'static, U: 'static>(
6624 &mut self,
6625 child: TableId<T>,
6626 parent: TableId<U>,
6627 ) -> Result<(), ResourceTableError> {
6628 self.table
6629 .get_mut()
6630 .remove_child(Resource::from(child), Resource::from(parent))
6631 }
6632
6633 fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6634 self.table.get_mut().delete(Resource::from(id))
6635 }
6636
6637 fn push_future(&mut self, future: HostTaskFuture) {
6638 self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6645 }
6646
6647 fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6648 if self.switch_item.is_some() {
6649 self.push_high_priority(item);
6652 bail_bug!("switch item already set");
6653 }
6654
6655 self.switch_item = Some(item);
6656 log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6657
6658 Ok(())
6659 }
6660
6661 fn take_next_switch_item(&mut self) -> Result<()> {
6662 if let Some(item) = self.next_switch_item.take() {
6663 self.set_switch_item(item)?;
6664 }
6665 Ok(())
6666 }
6667
6668 fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6671 let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6674 return Ok(());
6675 };
6676 let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6677 if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6678 bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6679 }
6680 let instance = self.get_mut(thread.task)?.instance;
6681 self.get_mut(thread.thread)?.wake_on_cancel.take();
6682 let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6683
6684 let item = match mode {
6685 WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6686 instance,
6687 thread,
6688 fiber,
6689 },
6690 WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6691 instance,
6692 call: GuestCall {
6693 thread,
6694 kind: GuestCallKind::DeliverEvent {
6695 instance: callback_instance,
6696 set: Some(set),
6697 },
6698 },
6699 },
6700 };
6701 self.push_high_priority(item);
6702 Ok(())
6703 }
6704
6705 fn push_high_priority(&mut self, item: WorkItem) {
6706 self.high_priority.push_front(item);
6707 log::trace!(
6708 "push high priority: {:?}",
6709 self.high_priority.front().unwrap()
6710 );
6711 }
6712
6713 fn push_low_priority(&mut self, item: WorkItem) {
6714 self.low_priority.push_front(item);
6715 log::trace!(
6716 "push low priority: {:?}",
6717 self.low_priority.front().unwrap()
6718 );
6719 }
6720
6721 fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6722 match priority {
6723 Priority::Switch => self.set_switch_item(item)?,
6724 Priority::High => self.push_high_priority(item),
6725 Priority::Low => self.push_low_priority(item),
6726 }
6727
6728 Ok(())
6729 }
6730
6731 fn promote_instance_local_thread_work_item(
6732 &mut self,
6733 current_instance: RuntimeInstance,
6734 ) -> Result<bool> {
6735 log::trace!("promote thread work items for {current_instance:?}");
6736
6737 self.promote_work_item_matching(|item: &WorkItem| {
6738 let result = match item {
6739 WorkItem::ResumeThread { instance, .. }
6740 | WorkItem::ResumeFiber { instance, .. }
6741 | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6742 _ => false,
6743 };
6744
6745 log::trace!("candidate {item:?}: {result}");
6746 result
6747 })
6748 }
6749
6750 fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6751 self.promote_work_item_matching(|item: &WorkItem| match item {
6752 WorkItem::ResumeThread {
6753 thread: item_thread,
6754 ..
6755 }
6756 | WorkItem::GuestCall {
6757 call:
6758 GuestCall {
6759 thread: item_thread,
6760 ..
6761 },
6762 ..
6763 } => *item_thread == thread,
6764 _ => false,
6765 })
6766 }
6767
6768 fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6769 where
6770 F: FnMut(&WorkItem) -> bool,
6771 {
6772 if self.switch_item.is_none() {
6779 if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6780 self.switch_item = self.high_priority.remove(index);
6781 }
6782 }
6783 if self.switch_item.is_none() {
6784 if let Some(index) = self.low_priority.iter().rposition(predicate) {
6785 self.switch_item = self.low_priority.remove(index);
6786 }
6787 }
6788 Ok(self.switch_item.is_some())
6789 }
6790
6791 pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6794 match task.kind() {
6795 ScopeKind::HostId(task) => {
6796 let task: TableId<HostTask> = TableId::new(task);
6797 Ok(&mut self.get_mut(task)?.call_context)
6798 }
6799 ScopeKind::Id(task) => {
6800 let task: TableId<GuestTask> = TableId::new(task);
6801 Ok(&mut self.get_mut(task)?.call_context)
6802 }
6803 }
6804 }
6805
6806 pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6807 self.deferred_host_call_context.as_mut()
6808 }
6809
6810 fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6811 match self.futures.get_mut().as_mut() {
6812 Some(f) => Ok(f),
6813 None => bail_bug!("futures field of concurrent state is currently taken"),
6814 }
6815 }
6816
6817 pub(crate) fn table(&mut self) -> &mut ResourceTable {
6818 self.table.get_mut()
6819 }
6820
6821 fn debug_assert_deferred_host_invariant(&self) {
6822 debug_assert_eq!(
6823 self.deferred_host_call_context.is_some(),
6824 matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6825 "a deferred host thread and call context must exist together",
6826 );
6827 }
6828
6829 fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6830 self.debug_assert_deferred_host_invariant();
6831 let caller = match self.unforced_current_thread {
6832 CurrentThread::DeferredHost(caller) => caller,
6833 thread => return Ok(thread),
6834 };
6835
6836 let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6838 let task = self.push(task)?;
6839 let call_context = self
6840 .deferred_host_call_context
6841 .take()
6842 .expect("deferred host call context should be present");
6843 self.get_mut(task)
6844 .expect("newly inserted host task should be present")
6845 .call_context = call_context;
6846 self.unforced_current_thread = CurrentThread::Host(task);
6847 self.debug_assert_deferred_host_invariant();
6848 log::trace!("new host task materialized {task:?}");
6849 Ok(CurrentThread::Host(task))
6850 }
6851
6852 fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6853 match self.materialize_host_task()? {
6854 CurrentThread::Host(id) => Ok(Some(id)),
6855 CurrentThread::None => Ok(None),
6856 CurrentThread::Guest(_) => {
6857 bail_bug!("tried to materialize a host task id from a guest thread")
6858 }
6859 CurrentThread::DeferredHost(_) => {
6860 bail_bug!(
6861 "current thread is a deferred host thread which should have been materialized"
6862 )
6863 }
6864 }
6865 }
6866
6867 pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6868 match self.materialize_host_task()? {
6869 CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6870 _ => bail_bug!("current scope is not a deferred host scope"),
6871 }
6872 }
6873
6874 fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6875 let count = &mut self.get_mut(task)?.ref_count;
6876 *count += 1;
6877 log::trace!("increment {task:?} to {count}");
6878 Ok(())
6879 }
6880
6881 fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6882 let count = &mut self.get_mut(task)?.ref_count;
6883 if *count == 0 {
6884 bail_bug!("can't decrement task ref count that's already zero");
6885 }
6886 *count -= 1;
6887 log::trace!("decrement {task:?} to {count}");
6888 if *count == 0 {
6889 let task = self.delete(task)?;
6890
6891 self.decrement_group_ref_count(task.group)?;
6892
6893 debug_assert!(task.decremented_interesting_task_count);
6900 }
6901 Ok(())
6902 }
6903
6904 fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6905 log::trace!("delete host task {task:?}");
6906 let task = self.delete(task)?;
6907
6908 self.decrement_group_ref_count(task.group)
6909 }
6910}
6911
6912fn for_any_lower<
6915 F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6916>(
6917 fun: F,
6918) -> F {
6919 fun
6920}
6921
6922fn for_any_lift<
6924 F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6925>(
6926 fun: F,
6927) -> F {
6928 fun
6929}
6930
6931fn check_ambient_store(id: StoreId) {
6932 let message = "\
6933 `Future`s which depend on asynchronous component tasks, streams, or \
6934 futures to complete may only be polled from the event loop of the \
6935 store to which they belong. Please use \
6936 `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6937 ";
6938 tls::try_get(|store| {
6939 let matched = match store {
6940 tls::TryGet::Some(store) => store.id() == id,
6941 tls::TryGet::Taken | tls::TryGet::None => false,
6942 };
6943
6944 if !matched {
6945 panic!("{message}")
6946 }
6947 });
6948}
6949
6950fn unpack_callback_code(code: u32) -> (u32, u32) {
6951 (code & 0xF, code >> 4)
6952}
6953
6954struct WaitableCheckParams {
6958 set: TableId<WaitableSet>,
6959 options: OptionsIndex,
6960 payload: u32,
6961}
6962
6963enum WaitableCheck {
6966 Wait,
6967 Poll,
6968}
6969
6970pub(crate) struct PreparedCall<R> {
6972 handle: Func,
6974 thread: QualifiedThreadId,
6976 param_count: usize,
6978 rx: oneshot::Receiver<LiftedResult>,
6981 _phantom: PhantomData<R>,
6982}
6983
6984impl<R> PreparedCall<R> {
6985 pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6992 store
6993 .concurrent_state_mut()?
6994 .increment_task_ref_count(self.thread.task)?;
6995
6996 Ok(TaskId {
6997 task: self.thread.task,
6998 })
6999 }
7000}
7001
7002pub(crate) struct TaskId {
7004 task: TableId<GuestTask>,
7005}
7006
7007impl TaskId {
7008 pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
7020 let task = store.concurrent_state_mut()?.get_mut(self.task)?;
7021 if !task.already_lowered_parameters() {
7022 task.lower_params = Some(Box::new(|store, _| {
7027 store.set_trapped();
7028 bail!("the host future for this call was dropped before it started")
7029 }));
7030 } else {
7031 _ = store
7036 .concurrent_state_mut()?
7037 .decrement_task_ref_count(self.task);
7038 }
7039
7040 Ok(())
7041 }
7042}
7043
7044pub(crate) fn prepare_call<T, R>(
7050 mut store: StoreContextMut<T>,
7051 handle: Func,
7052 param_count: usize,
7053 lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
7054 + Send
7055 + Sync
7056 + 'static,
7057 lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
7058 + Send
7059 + Sync
7060 + 'static,
7061) -> Result<PreparedCall<R>> {
7062 if !store.0.may_enter() {
7066 bail!(Trap::CannotEnterComponent);
7067 }
7068
7069 let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
7070
7071 let instance = handle.instance().id().get(store.0);
7072 let options = &instance.component().env_component().options[options];
7073 let ty = &instance.component().types()[ty];
7074 let async_typed = ty.async_;
7075 let async_lifted = raw_options.async_;
7076 let task_return_type = ty.results;
7077 let component_instance = raw_options.instance;
7078 let callback = options.callback.map(|i| instance.runtime_callback(i));
7079 let memory = options
7080 .memory()
7081 .map(|i| instance.runtime_memory(i))
7082 .map(SendSyncPtr::new);
7083 let string_encoding = options.string_encoding;
7084 let token = StoreToken::new(store.as_context_mut());
7085 let caller = store.0.materialize_host_task_id()?;
7086 let state = store.0.concurrent_state_mut()?;
7087
7088 let (tx, rx) = oneshot::channel();
7089
7090 let instance = handle.instance().runtime_instance(component_instance);
7091 let thread = GuestTask::new(
7092 state,
7093 Box::new(for_any_lower(move |store, params| {
7094 lower_params(token.as_context_mut(store), params)
7095 })),
7096 LiftResult {
7097 lift: Box::new(for_any_lift(move |store, result| {
7098 lift_result(store, result)
7099 })),
7100 ty: task_return_type,
7101 memory,
7102 string_encoding,
7103 },
7104 Caller::Host {
7105 tx: Some(tx),
7106 caller,
7107 },
7108 callback.map(|callback| {
7109 let callback = SendSyncPtr::new(callback);
7110 let instance = handle.instance();
7111 Box::new(move |store: &mut dyn VMStore, event, handle| {
7112 let store = token.as_context_mut(store);
7113 unsafe { instance.call_callback(store, callback, event, handle) }
7116 }) as CallbackFn
7117 }),
7118 instance,
7119 async_typed,
7120 async_lifted,
7121 )?;
7122
7123 Ok(PreparedCall {
7124 handle,
7125 thread,
7126 param_count,
7127 rx,
7128 _phantom: PhantomData,
7129 })
7130}
7131
7132pub(crate) struct StagedCall<R> {
7133 store: StoreId,
7134 rx: oneshot::Receiver<LiftedResult>,
7135 _marker: PhantomData<fn() -> R>,
7136 group: TaskGroupId,
7144}
7145
7146impl<R> StagedCall<R> {
7147 pub(crate) fn new<T: 'static>(
7158 mut store: StoreContextMut<T>,
7159 prepared: PreparedCall<R>,
7160 ) -> Result<StagedCall<R>> {
7161 let PreparedCall {
7162 handle,
7163 thread,
7164 param_count,
7165 rx,
7166 ..
7167 } = prepared;
7168
7169 stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7170
7171 let state = store.0.concurrent_state_mut()?;
7172 let group = state.get_mut(thread.task)?.group;
7173 state.decrement_task_ref_count(thread.task)?;
7174
7175 Ok(StagedCall {
7176 store: store.0.id(),
7177 rx,
7178 _marker: PhantomData,
7179 group,
7180 })
7181 }
7182}
7183
7184impl<R> Future for StagedCall<R>
7185where
7186 R: 'static,
7187{
7188 type Output = Result<R>;
7189
7190 fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7191 check_ambient_store(self.store);
7192 Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7193 Ok(r) => match r.downcast() {
7194 Ok(r) => Ok(*r),
7195 Err(_) => bail_bug!("wrong type of value produced"),
7196 },
7197 Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7198 })
7199 }
7200}
7201
7202fn stage_call0<T: 'static>(
7205 store: StoreContextMut<T>,
7206 handle: Func,
7207 guest_thread: QualifiedThreadId,
7208 param_count: usize,
7209) -> Result<()> {
7210 let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7211 let is_concurrent = raw_options.async_;
7212 let callback = raw_options.callback;
7213 let instance = handle.instance();
7214 let callee = handle.lifted_core_func(store.0);
7215 let post_return = raw_options
7216 .post_return
7217 .map(|i| instance.id().get(store.0).runtime_post_return(i));
7218 let callback = callback.map(|i| {
7219 let instance = instance.id().get(store.0);
7220 SendSyncPtr::new(instance.runtime_callback(i))
7221 });
7222
7223 log::trace!("queueing call {guest_thread:?}");
7224
7225 unsafe {
7229 instance.stage_call(
7230 store,
7231 guest_thread,
7232 SendSyncPtr::new(callee),
7233 param_count,
7234 1,
7235 is_concurrent,
7236 callback,
7237 post_return.map(SendSyncPtr::new),
7238 true,
7239 )
7240 }
7241}
7242
7243#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7244mod tests {
7245 use super::*;
7246 use crate::component::{Component, Linker};
7247 use crate::store::AsStoreOpaque;
7248 use crate::{Config, Engine};
7249
7250 fn host_subtask(
7251 state: HostTaskState,
7252 event: Option<Event>,
7253 ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7254 let mut config = Config::new();
7255 config.wasm_component_model_async(true);
7256 let engine = Engine::new(&config)?;
7257 let component = Component::new(&engine, "(component)")?;
7258 let mut store = Store::new(&engine, ());
7259 let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7260 let store_opaque = store.as_store_opaque();
7261 let concurrent_state = store_opaque.concurrent_state_mut()?;
7262 let group = concurrent_state.make_task_group()?;
7265 let task = concurrent_state.push(HostTask {
7266 common: WaitableCommon::default(),
7267 call_context: CallContext::default(),
7268 state,
7269 group,
7270 })?;
7271 let handle = store_opaque
7272 .handle_table_insertion(
7273 instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7274 )
7275 .subtask_host(task.rep())?;
7276 let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7277 common.handle = Some(handle);
7278 common.event = event;
7279 Ok((store, instance, task, handle))
7280 }
7281
7282 #[test]
7283 fn host_subtask_drop_during_cancellation() -> Result<()> {
7284 for abort_completed in [false, true] {
7285 let (handle, future) = JoinHandle::run(future::pending::<()>());
7286 let mut future = pin!(future);
7287 let (mut store, instance, task, handle) =
7288 host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7289 let store = store.as_store_opaque();
7290 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7291 assert_eq!(
7292 instance.subtask_cancel(store, caller, true, handle)?,
7293 BLOCKED
7294 );
7295 if abort_completed {
7296 assert!(matches!(
7299 future
7300 .as_mut()
7301 .poll(&mut Context::from_waker(Waker::noop())),
7302 Poll::Ready(None),
7303 ));
7304 }
7305 for async_ in [false, true] {
7306 let err = instance
7307 .subtask_cancel(store, caller, async_, handle)
7308 .unwrap_err();
7309 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7310 }
7311 let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7312 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7313 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7314 }
7315 Ok(())
7316 }
7317
7318 #[test]
7319 fn host_subtask_cancel_after_completion() -> Result<()> {
7320 for async_ in [false, true] {
7321 let (mut store, instance, task, handle) = host_subtask(
7322 HostTaskState::CalleeDone { cancelled: false },
7323 Some(Event::Subtask {
7324 status: Status::Returned,
7325 }),
7326 )?;
7327 let store = store.as_store_opaque();
7328 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7329 assert_eq!(
7330 instance.subtask_cancel(store, caller, async_, handle)?,
7331 Status::Returned as u32,
7332 );
7333 let err = instance
7334 .subtask_cancel(store, caller, async_, handle)
7335 .unwrap_err();
7336 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7337 instance.subtask_drop(store, caller, handle)?;
7338 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7339 }
7340 Ok(())
7341 }
7342
7343 #[test]
7344 fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7345 for (cancelled, status) in [
7346 (false, Status::Returned),
7347 (true, Status::Returned),
7348 (true, Status::ReturnCancelled),
7349 ] {
7350 for delivered in [false, true] {
7351 let event = if delivered {
7352 None
7353 } else {
7354 Some(Event::Subtask { status })
7355 };
7356 let (mut store, instance, task, handle) =
7357 host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7358 let store = store.as_store_opaque();
7359 let result = instance.subtask_drop(
7360 store,
7361 RuntimeComponentInstanceIndex::from_u32(0),
7362 handle,
7363 );
7364 if delivered {
7365 result?;
7366 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7367 } else {
7368 let err = result.unwrap_err();
7369 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7370 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7371 }
7372 }
7373 }
7374 Ok(())
7375 }
7376}