1use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57 HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70 CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71 InstanceState,
72};
73use crate::vm::{
74 AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98 CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99 MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100 RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101 TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102 TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113 Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114 FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115 StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136const BLOCKED: u32 = 0xffff_ffff;
139
140#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143 Starting = 0,
144 Started = 1,
145 Returned = 2,
146 StartCancelled = 3,
147 ReturnCancelled = 4,
148}
149
150impl Status {
151 pub fn pack(self, waitable: Option<u32>) -> u32 {
157 assert!(matches!(self, Status::Returned) == waitable.is_none());
158 let waitable = waitable.unwrap_or(0);
159 assert!(waitable < (1 << 28));
160 (waitable << 4) | (self as u32)
161 }
162}
163
164#[derive(Clone, Copy, Debug)]
167enum Event {
168 None,
169 Subtask {
170 status: Status,
171 },
172 StreamRead {
173 code: ReturnCode,
174 pending: Option<(TypeStreamTableIndex, u32)>,
175 },
176 StreamWrite {
177 code: ReturnCode,
178 pending: Option<(TypeStreamTableIndex, u32)>,
179 },
180 FutureRead {
181 code: ReturnCode,
182 pending: Option<(TypeFutureTableIndex, u32)>,
183 },
184 FutureWrite {
185 code: ReturnCode,
186 pending: Option<(TypeFutureTableIndex, u32)>,
187 },
188 Cancelled,
189}
190
191impl Event {
192 fn parts(self) -> (u32, u32) {
197 const EVENT_NONE: u32 = 0;
198 const EVENT_SUBTASK: u32 = 1;
199 const EVENT_STREAM_READ: u32 = 2;
200 const EVENT_STREAM_WRITE: u32 = 3;
201 const EVENT_FUTURE_READ: u32 = 4;
202 const EVENT_FUTURE_WRITE: u32 = 5;
203 const EVENT_CANCELLED: u32 = 6;
204 match self {
205 Event::None => (EVENT_NONE, 0),
206 Event::Cancelled => (EVENT_CANCELLED, 0),
207 Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208 Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209 Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210 Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211 Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212 }
213 }
214}
215
216mod callback_code {
218 pub const EXIT: u32 = 0;
219 pub const YIELD: u32 = 1;
220 pub const WAIT: u32 = 2;
221}
222
223const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233 store: StoreContextMut<'a, T>,
234 get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239 D: HasData + ?Sized,
240 T: 'static,
241{
242 pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244 Self { store, get_data }
245 }
246
247 pub fn data_mut(&mut self) -> &mut T {
249 self.store.data_mut()
250 }
251
252 pub fn get(&mut self) -> D::Data<'_> {
254 (self.get_data)(self.data_mut())
255 }
256
257 pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261 where
262 T: 'static,
263 {
264 let accessor = Accessor {
265 get_data: self.get_data,
266 token: StoreToken::new(self.store.as_context_mut()),
267 };
268 self.store
269 .as_context_mut()
270 .spawn_with_accessor(accessor, task)
271 }
272
273 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276 self.get_data
277 }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282 D: HasData + ?Sized,
283 T: 'static,
284{
285 type Data = T;
286
287 fn as_context(&self) -> StoreContext<'_, T> {
288 self.store.as_context()
289 }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294 D: HasData + ?Sized,
295 T: 'static,
296{
297 fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298 self.store.as_context_mut()
299 }
300}
301
302pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363 D: HasData + ?Sized,
364{
365 token: StoreToken<T>,
366 get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369pub trait AsAccessor {
386 type Data: 'static;
388
389 type AccessorData: HasData + ?Sized;
392
393 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398 type Data = T::Data;
399 type AccessorData = T::AccessorData;
400
401 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402 T::as_accessor(self)
403 }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407 type Data = T;
408 type AccessorData = D;
409
410 fn as_accessor(&self) -> &Accessor<T, D> {
411 self
412 }
413}
414
415const _: () = {
438 const fn assert<T: Send + Sync>() {}
439 assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443 pub(crate) fn new(token: StoreToken<T>) -> Self {
452 Self {
453 token,
454 get_data: |x| x,
455 }
456 }
457}
458
459impl<T, D> Accessor<T, D>
460where
461 D: HasData + ?Sized,
462{
463 pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481 tls::get(|vmstore| {
482 fun(Access {
483 store: self.token.as_context_mut(vmstore),
484 get_data: self.get_data,
485 })
486 })
487 }
488
489 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492 self.get_data
493 }
494
495 pub fn with_getter<D2: HasData>(
512 &self,
513 get_data: fn(&mut T) -> D2::Data<'_>,
514 ) -> Accessor<T, D2> {
515 Accessor {
516 token: self.token,
517 get_data,
518 }
519 }
520
521 pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537 where
538 T: 'static,
539 {
540 let accessor = self.clone_for_spawn();
541 self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542 }
543
544 fn clone_for_spawn(&self) -> Self {
545 Self {
546 token: self.token,
547 get_data: self.get_data,
548 }
549 }
550
551 pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587 self.with(|mut access| {
588 let store = access.as_context_mut().0;
589 let state = store.concurrent_state_mut_without_forcing_current_thread();
590 if state.interesting_tasks == 0 {
591 Poll::Ready(())
592 } else {
593 state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594 Poll::Pending
595 }
596 })
597 }
598
599 pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623 self.with(|mut access| {
624 let store = access.as_context_mut().0;
625
626 {
627 let state = store.concurrent_state_mut_without_forcing_current_thread();
628 if state.switch_item.is_some()
629 || !state.high_priority.is_empty()
630 || !state.low_priority.is_empty()
631 {
632 state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633 return Poll::Pending;
634 }
635 }
636
637 let (_, _, _, raw_options) = func.abi_info(store);
638 let instance = func.instance().runtime_instance(raw_options.instance);
639 let state = store.instance_state(instance).concurrent_state();
640 if state.backpressure == 0 {
641 Poll::Ready(())
642 } else {
643 store
644 .concurrent_state_mut_without_forcing_current_thread()
645 .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646 Poll::Pending
647 }
648 })
649 }
650}
651
652pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674 AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676 D: HasData + ?Sized,
677{
678 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684 T: 'static,
685 F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686 F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687 Fut: Future<Output = Result<()>> + Send + 'fut,
688 D: HasData,
689{
690 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691 (self)(accessor)
692 }
693}
694
695enum CallerInfo {
698 Async {
700 params: Vec<ValRaw>,
701 has_result: bool,
702 },
703 Sync {
705 params: Vec<ValRaw>,
706 result_count: u32,
707 },
708}
709
710enum WaitMode {
712 Fiber(StoreFiber<'static>),
714 Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721 match self {
722 Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723 Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724 }
725 }
726}
727
728#[derive(Debug)]
730enum SuspendReason {
731 Waiting {
734 set: TableId<WaitableSet>,
735 thread: QualifiedThreadId,
736 },
737 YieldingToSubtask { thread: QualifiedThreadId },
740 NeedWork,
743 Yielding { thread: QualifiedThreadId },
746 ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751enum GuestCallKind {
753 DeliverEvent {
756 instance: Instance,
758 set: Option<TableId<WaitableSet>>,
763 },
764 StartImplicit {
767 closure: Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>,
769 was_pending: bool,
771 },
772 StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
773}
774
775impl fmt::Debug for GuestCallKind {
776 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
777 match self {
778 Self::DeliverEvent { instance, set } => f
779 .debug_struct("DeliverEvent")
780 .field("instance", instance)
781 .field("set", set)
782 .finish(),
783 Self::StartImplicit { was_pending, .. } => f
784 .debug_struct("StartImplicit")
785 .field("was_pending", was_pending)
786 .finish(),
787 Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
788 }
789 }
790}
791
792#[derive(Copy, Clone, Debug)]
794pub enum SuspensionTarget {
795 Resume(u32),
796 Promote(u32),
797 None,
798}
799
800#[derive(Copy, Clone, Debug)]
802pub enum ResumeThread {
803 Promote,
804 Resume,
805 ResumeLater,
806}
807
808#[derive(Debug)]
810struct GuestCall {
811 thread: QualifiedThreadId,
812 kind: GuestCallKind,
813}
814
815impl GuestCall {
816 fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
828 let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
829 let async_typed = task.async_typed;
830 let async_lifted = task.async_lifted;
831 let has_callback = task.callback.is_some();
832 let instance = task.instance;
833 let state = store.instance_state(instance).concurrent_state();
834
835 let ready = match &self.kind {
836 GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
837 GuestCallKind::StartImplicit { was_pending, .. } => {
838 if async_typed {
839 let needs_exclusive = !async_lifted || has_callback;
842
843 let may_enter =
848 !((needs_exclusive && state.do_not_enter) || state.backpressure > 0);
849
850 let skip_wait = *was_pending || state.pending_start_count == 0;
857
858 may_enter && skip_wait
861 } else {
862 true
863 }
864 }
865 GuestCallKind::StartExplicit(_) => true,
866 };
867 log::trace!(
868 "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {}; pending_start_count: {})",
869 state.do_not_enter,
870 state.backpressure,
871 state.pending_start_count,
872 );
873 Ok(ready)
874 }
875}
876
877enum WorkerItem {
879 GuestCall(GuestCall),
880 Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
881}
882
883enum WorkItem {
886 PushFuture(AlwaysMut<HostTaskFuture>),
888 ResumeFiber {
890 instance: RuntimeInstance,
891 thread: QualifiedThreadId,
892 fiber: StoreFiber<'static>,
893 },
894 ResumeThread {
896 instance: RuntimeInstance,
897 thread: QualifiedThreadId,
898 },
899 GuestCall {
901 instance: RuntimeInstance,
902 call: GuestCall,
903 },
904 WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
906}
907
908struct ResumingFiber<'a> {
914 store: &'a mut StoreOpaque,
915 fiber: Option<StoreFiber<'static>>,
916}
917
918impl Drop for ResumingFiber<'_> {
919 fn drop(&mut self) {
920 if let Some(mut fiber) = self.fiber.take() {
921 fiber.dispose(self.store);
922 }
923 }
924}
925
926impl fmt::Debug for WorkItem {
927 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
928 match self {
929 Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
930 Self::ResumeFiber {
931 instance, thread, ..
932 } => f
933 .debug_struct("ResumeFiber")
934 .field("instance", instance)
935 .field("thread", thread)
936 .finish(),
937 Self::ResumeThread { instance, thread } => f
938 .debug_struct("ResumeThread")
939 .field("instance", instance)
940 .field("thread", thread)
941 .finish(),
942 Self::GuestCall { instance, call } => f
943 .debug_struct("GuestCall")
944 .field("instance", instance)
945 .field("call", call)
946 .finish(),
947 Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
948 }
949 }
950}
951
952#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
954pub(crate) enum WaitResult {
955 Cancelled,
956 Completed,
957}
958
959pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
967 store: &mut dyn VMStore,
968 host_task: EnteredHostTask,
969 future: impl Future<Output = Result<R>> + Send + 'static,
970) -> Result<R> {
971 let mut future = Box::pin(future);
978 let poll = tls::set(store, || {
979 future
980 .as_mut()
981 .poll(&mut Context::from_waker(&Waker::noop()))
982 });
983
984 let caller = match host_task {
985 Some(caller) => caller,
986 None => bail_bug!("host task wasn't created but should have been"),
987 };
988
989 let task = match poll {
990 Poll::Ready(result) => return result,
992
993 Poll::Pending => {
998 let Some(task) = store.materialize_host_task_id()? else {
999 bail_bug!("current thread is not a host thread")
1000 };
1001
1002 let future = Box::pin(async move {
1005 let result = run_with_host_task_set(task, future).await??;
1006 tls::get(move |store| {
1007 let state = store.concurrent_state_mut()?;
1008 let host_state = &mut state.get_mut(task)?.state;
1009 assert!(matches!(host_state, HostTaskState::CalleeStarted));
1010 *host_state = HostTaskState::CalleeFinished(Box::new(result));
1011
1012 Waitable::Host(task).set_event(
1013 state,
1014 Some(Event::Subtask {
1015 status: Status::Returned,
1016 }),
1017 )?;
1018
1019 Ok(())
1020 })
1021 }) as HostTaskFuture;
1022
1023 let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
1024 store.switch_or_trap_if_may_not_suspend(caller_instance)?;
1025
1026 let state = store.concurrent_state_mut()?;
1027 state.push_future(future);
1028
1029 let set = state.get_mut(caller.thread)?.sync_call_set;
1030 Waitable::Host(task).join(state, Some(set))?;
1031
1032 store.suspend(SuspendReason::Waiting {
1033 set,
1034 thread: caller,
1035 })?;
1036
1037 Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1041 task
1042 }
1043 };
1044
1045 let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1047 match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1048 HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1049 Ok(result) => *result,
1050 Err(_) => bail_bug!("host task finished with wrong type of result"),
1051 }),
1052 _ => bail_bug!("unexpected host task state after completion"),
1053 }
1054}
1055
1056fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1058 match call.kind {
1059 GuestCallKind::DeliverEvent { instance, set } => {
1060 if let Some(set) = set {
1063 store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1064 }
1065 let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1066 Some(pair) => pair,
1067 None => match set {
1068 Some(set) => {
1073 log::trace!(
1074 "event for {:?} on {set:?} no longer present; waiting again",
1075 call.thread
1076 );
1077 return instance.wait_with_callback(&mut **store, call.thread, set);
1078 }
1079 None => (Event::None, None),
1084 },
1085 };
1086 let state = store.concurrent_state_mut()?;
1087 if let Some(set) = set
1092 && waitable.is_none()
1093 && !state.get_mut(set)?.ready.is_empty()
1094 {
1095 state.wake_waiter(set)?;
1096 }
1097 let task = state.get_mut(call.thread.task)?;
1098 let runtime_instance = task.instance;
1099 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1100
1101 log::trace!(
1102 "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1103 call.thread,
1104 );
1105
1106 instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1107 invoke_callback(store, call.thread.task, event, handle)
1108 })?;
1109 }
1110 GuestCallKind::StartImplicit { closure, .. } => {
1111 closure(store)?;
1112 }
1113 GuestCallKind::StartExplicit(fun) => {
1114 fun(store)?;
1115 }
1116 }
1117
1118 Ok(())
1119}
1120
1121fn invoke_callback(
1124 store: &mut dyn VMStore,
1125 task: TableId<GuestTask>,
1126 event: Event,
1127 handle: u32,
1128) -> Result<u32> {
1129 let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1130 bail_bug!("guest task callback field not present")
1131 };
1132
1133 let code = callback(store, event, handle)?;
1134
1135 store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1136
1137 Ok(code)
1138}
1139
1140impl<T> Store<T> {
1141 pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1143 where
1144 T: Send + 'static,
1145 {
1146 ensure!(
1147 self.as_context().0.concurrency_support(),
1148 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1149 );
1150 self.as_context_mut().run_concurrent(fun).await
1151 }
1152
1153 #[doc(hidden)]
1154 pub fn assert_concurrent_state_empty(&mut self) {
1155 self.as_context_mut().assert_concurrent_state_empty();
1156 }
1157
1158 #[doc(hidden)]
1159 pub fn concurrent_state_table_size(&mut self) -> usize {
1160 self.as_context_mut().concurrent_state_table_size()
1161 }
1162
1163 #[doc(hidden)]
1164 pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1165 self.as_context_mut()
1166 .set_max_concurrent_state_table_capacity(max)
1167 }
1168
1169 pub fn spawn(
1171 &mut self,
1172 task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1173 ) -> Result<JoinHandle>
1174 where
1175 T: 'static,
1176 {
1177 self.as_context_mut().spawn(task)
1178 }
1179}
1180
1181impl<T> StoreContextMut<'_, T> {
1182 #[doc(hidden)]
1193 pub fn assert_concurrent_state_empty(self) {
1194 let store = self.0;
1195 store
1196 .store_data_mut()
1197 .components
1198 .assert_instance_states_empty();
1199 let state = store.concurrent_state_mut().unwrap();
1200 assert!(
1201 state.table.get_mut().is_empty(),
1202 "non-empty table: {:?}",
1203 state.table.get_mut()
1204 );
1205 assert!(state.switch_item.is_none());
1206 assert!(state.next_switch_item.is_none());
1207 assert!(state.high_priority.is_empty());
1208 assert!(state.low_priority.is_empty());
1209 assert!(state.saved_next_switch_items.is_empty());
1210 assert!(state.unforced_current_thread.is_none());
1211 assert!(state.deferred_host_call_context.is_none());
1212 assert!(state.futures_mut().unwrap().is_empty());
1213 assert!(state.global_error_context_ref_counts.is_empty());
1214 }
1215
1216 #[doc(hidden)]
1221 pub fn concurrent_state_table_size(&mut self) -> usize {
1222 self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1223 }
1224
1225 #[doc(hidden)]
1231 pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1232 self.0
1233 .concurrent_state_mut_without_forcing_current_thread()
1234 .table
1235 .get_mut()
1236 .set_max_capacity(max);
1237 }
1238
1239 pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1249 where
1250 T: 'static,
1251 {
1252 let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1253 self.spawn_with_accessor(accessor, task)
1254 }
1255
1256 fn spawn_with_accessor<D>(
1259 self,
1260 accessor: Accessor<T, D>,
1261 task: impl for<'fut> AccessorTask<'fut, T, D>,
1262 ) -> Result<JoinHandle>
1263 where
1264 T: 'static,
1265 D: HasData + ?Sized,
1266 {
1267 let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1271 self.0
1272 .concurrent_state_mut()?
1273 .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1274 Ok(handle)
1275 }
1276
1277 pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1361 where
1362 T: Send + 'static,
1363 {
1364 ensure!(
1365 self.0.concurrency_support(),
1366 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1367 );
1368 self.do_run_concurrent(fun, false).await
1369 }
1370
1371 pub(super) async fn run_concurrent_trap_on_idle<R>(
1372 self,
1373 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1374 ) -> Result<R> {
1375 self.do_run_concurrent(fun, true).await
1376 }
1377
1378 async fn do_run_concurrent<R>(
1379 mut self,
1380 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1381 trap_on_idle: bool,
1382 ) -> Result<R> {
1383 debug_assert!(self.0.concurrency_support());
1384 self.0.check_event_loop_not_running()?;
1385 let token = StoreToken::new(self.as_context_mut());
1386
1387 struct Dropper<'a, T: 'static, V> {
1388 store: StoreContextMut<'a, T>,
1389 value: ManuallyDrop<V>,
1390 }
1391
1392 impl<'a, T, V> Drop for Dropper<'a, T, V> {
1393 fn drop(&mut self) {
1394 self.store
1395 .0
1396 .concurrent_state_mut_already_forced_current_thread()
1397 .event_loop_running = false;
1398
1399 tls::set(self.store.0, || {
1400 unsafe { ManuallyDrop::drop(&mut self.value) }
1405 });
1406 }
1407 }
1408
1409 let accessor = &Accessor::new(token);
1410 self.0
1411 .concurrent_state_mut_already_forced_current_thread()
1412 .event_loop_running = true;
1413 let dropper = &mut Dropper {
1414 store: self,
1415 value: ManuallyDrop::new(fun(accessor)),
1416 };
1417 let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1419
1420 let result = dropper
1421 .store
1422 .as_context_mut()
1423 .poll_until(future, trap_on_idle)
1424 .await;
1425
1426 if result.is_err() {
1427 dropper.store.0.set_trapped();
1428 }
1429
1430 result
1431 }
1432
1433 async fn poll_until<R>(
1439 mut self,
1440 mut future: Pin<&mut impl Future<Output = R>>,
1441 trap_on_idle: bool,
1442 ) -> Result<R> {
1443 struct Reset<'a, T: 'static> {
1444 store: StoreContextMut<'a, T>,
1445 futures: Option<FuturesUnordered<HostTaskFuture>>,
1446 }
1447
1448 impl<'a, T> Drop for Reset<'a, T> {
1449 fn drop(&mut self) {
1450 if let Some(futures) = self.futures.take() {
1451 *self
1452 .store
1453 .0
1454 .concurrent_state_mut_already_forced_current_thread()
1455 .futures
1456 .get_mut() = Some(futures);
1457 }
1458 }
1459 }
1460
1461 const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1465 let mut turns_without_yield = 0;
1466
1467 loop {
1468 let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1472 let mut reset = Reset {
1473 store: self.as_context_mut(),
1474 futures,
1475 };
1476 let mut next = match reset.futures.as_mut() {
1477 Some(f) => pin!(f.next()),
1478 None => bail_bug!("concurrent state missing futures field"),
1479 };
1480
1481 enum PollResult<R> {
1482 Complete(R),
1483 ProcessWork {
1484 ready: Option<WorkItem>,
1485 low_priority: bool,
1486 },
1487 }
1488
1489 let result = future::poll_fn(|cx| {
1490 if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1493 return Poll::Ready(Ok(PollResult::Complete(value)));
1494 }
1495
1496 if reset.store.0.component_trapped() {
1505 return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1506 }
1507
1508 let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1512 Poll::Ready(Some(output)) => {
1513 match output {
1514 Err(e) => return Poll::Ready(Err(e)),
1515 Ok(()) => {}
1516 }
1517 Poll::Ready(true)
1518 }
1519 Poll::Ready(None) => Poll::Ready(false),
1520 Poll::Pending => Poll::Pending,
1521 };
1522
1523 let state = reset.store.0.concurrent_state_mut()?;
1538 let mut ready = state.switch_item.take();
1539 let mut low_priority = false;
1540 if ready.is_none() {
1541 ready = state.high_priority.pop_back();
1542 if ready.is_none() {
1543 ready = state.low_priority.pop_back();
1544 low_priority = true;
1545 }
1546 }
1547 if ready.is_some() {
1548 return Poll::Ready(Ok(PollResult::ProcessWork {
1549 ready,
1550 low_priority,
1551 }));
1552 }
1553
1554 if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1555 waker.wake();
1556 }
1557
1558 return match next {
1562 Poll::Ready(true) => {
1563 Poll::Ready(Ok(PollResult::ProcessWork {
1569 ready: None,
1570 low_priority: false,
1571 }))
1572 }
1573 Poll::Ready(false) => {
1574 if let Poll::Ready(value) =
1578 tls::set(reset.store.0, || future.as_mut().poll(cx))
1579 {
1580 Poll::Ready(Ok(PollResult::Complete(value)))
1581 } else {
1582 if trap_on_idle {
1588 Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1595 Trap::CannotBlockSyncTask.into()
1596 } else {
1597 Trap::AsyncDeadlock.into()
1599 }))
1600 } else {
1601 Poll::Pending
1605 }
1606 }
1607 }
1608 Poll::Pending => Poll::Pending,
1613 };
1614 })
1615 .await;
1616
1617 drop(reset);
1621
1622 match result? {
1623 PollResult::Complete(value) => break Ok(value),
1626 PollResult::ProcessWork {
1629 mut ready,
1630 low_priority,
1631 } => {
1632 if low_priority {
1660 if let Some(item) = ready.take() {
1661 self.0.concurrent_state_mut()?.low_priority.push_back(item);
1662 }
1663 self.0.yield_now().await;
1664 turns_without_yield = 0;
1665 ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1666 }
1667
1668 if let Some(item) = ready {
1669 self.as_context_mut().handle_work_item(item).await?;
1670 }
1671
1672 turns_without_yield += 1;
1673 if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1674 turns_without_yield = 0;
1675 self.0.yield_now().await;
1676 }
1677 }
1678 }
1679 }
1680 }
1681
1682 fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1684 let item = match item {
1688 WorkItem::ResumeFiber {
1689 instance,
1690 thread,
1691 fiber,
1692 } => {
1693 let future = self.0.resume_fiber(fiber);
1694 log::trace!(
1695 "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1696 );
1697 return futures::future::Either::Left(future);
1698 }
1699 item => item,
1700 };
1701 futures::future::Either::Right(async move {
1702 log::trace!("handle work item {item:?}");
1703 match item {
1704 WorkItem::PushFuture(future) => {
1705 self.0
1706 .concurrent_state_mut()?
1707 .futures_mut()?
1708 .push(future.into_inner());
1709 }
1710 WorkItem::ResumeFiber { .. } => unreachable!(),
1711 WorkItem::ResumeThread { thread, .. } => {
1712 let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1713 if !matches!(slot, GuestThreadState::Ready { .. }) {
1714 bail_bug!("cannot resume non-pending thread {thread:?}");
1715 }
1716 let GuestThreadState::Ready { fiber } =
1717 mem::replace(slot, GuestThreadState::Running)
1718 else {
1719 unreachable!()
1720 };
1721 self.0.resume_fiber(fiber).await?;
1722 }
1723 WorkItem::GuestCall { mut call, .. } => {
1724 if call.is_ready(self.0)? {
1725 if let GuestCallKind::StartImplicit {
1726 was_pending: true, ..
1727 } = &call.kind
1728 {
1729 let instance = self
1730 .0
1731 .concurrent_state_mut()?
1732 .get_mut(call.thread.task)?
1733 .instance;
1734
1735 self.0.decrement_pending_start_count(instance, 1)?;
1736 }
1737
1738 self.0
1739 .concurrent_state_mut()?
1740 .get_mut(call.thread.thread)?
1741 .wake_on_cancel = WakeOnCancel::None;
1742
1743 self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1744 } else {
1745 let state = self.0.concurrent_state_mut()?;
1746 let task = state.get_mut(call.thread.task)?;
1747 if !task.starting_sent {
1748 task.starting_sent = true;
1749 if let GuestCallKind::StartImplicit { .. } = &call.kind {
1750 Waitable::Guest(call.thread.task).set_event(
1751 state,
1752 Some(Event::Subtask {
1753 status: Status::Starting,
1754 }),
1755 )?;
1756 }
1757 }
1758
1759 let increment = if let GuestCallKind::StartImplicit {
1760 was_pending, ..
1761 } = &mut call.kind
1762 {
1763 let increment = !*was_pending;
1764 *was_pending = true;
1765 increment
1766 } else {
1767 false
1768 };
1769
1770 let instance = state.get_mut(call.thread.task)?.instance;
1771 if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1780 if !state.get_mut(*set)?.ready.is_empty() {
1781 state.wake_waiter(*set)?;
1782 }
1783 }
1784
1785 state.take_next_switch_item()?;
1789
1790 let state = self.0.instance_state(instance).concurrent_state();
1791
1792 state.pending.insert(call.thread, call.kind);
1793
1794 if increment {
1795 state.pending_start_count += 1;
1796 }
1797 }
1798 }
1799 WorkItem::WorkerFunction(fun) => {
1800 self.run_on_worker(WorkerItem::Function(fun)).await?;
1801 }
1802 }
1803
1804 Ok(())
1805 })
1806 }
1807
1808 async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1810 let state = self.0.concurrent_state_mut()?;
1811 if state.worker_item.is_some() {
1812 bail_bug!("worker item already set when taking worker fiber");
1813 }
1814 let worker = if let Some(fiber) = state.worker.take() {
1815 fiber
1816 } else {
1817 unsafe {
1836 fiber::make_fiber_unchecked(self.0, move |store| {
1837 loop {
1838 let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1839 bail_bug!("worker_item not present when resuming fiber")
1840 };
1841 match item {
1842 WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1843 WorkerItem::Function(fun) => fun.into_inner()(store)?,
1844 }
1845
1846 store.suspend(SuspendReason::NeedWork)?;
1847 }
1848 })?
1849 }
1850 };
1851
1852 self.0
1855 .concurrent_state_mut_already_forced_current_thread()
1856 .worker_item = Some(item);
1857
1858 self.0.resume_fiber(worker).await
1859 }
1860
1861 pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1866 where
1867 T: 'static,
1868 F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1869 + Send
1870 + Sync
1871 + 'static,
1872 R: Send + Sync + 'static,
1873 {
1874 let token = StoreToken::new(self);
1875 async move {
1876 let mut accessor = Accessor::new(token);
1877 closure(&mut accessor).await
1878 }
1879 }
1880
1881 pub(crate) async fn start_instance(
1882 &mut self,
1883 instance: ModuleInstance,
1884 callee: Option<RuntimeInstance>,
1885 ) -> Result<ModuleInstance> {
1886 self.0.check_event_loop_not_running()?;
1888 let (tx, rx) = oneshot::channel();
1889 let token = StoreToken::new(self.as_context_mut());
1890 self.0.queue_task(move |store| {
1891 _ = tx.send(
1892 super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1893 .map(|()| instance),
1894 );
1895 Ok(())
1896 })?;
1897 self.as_context_mut()
1898 .run_concurrent_trap_on_idle(async |_| {
1899 rx.await
1900 .map_err(|_| format_err!("oneshot channel canceled"))
1901 })
1902 .await??
1903 }
1904}
1905
1906pub type EnteredHostTask = Option<QualifiedThreadId>;
1913
1914impl StoreOpaque {
1915 pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1916 if self
1917 .concurrent_state_mut_already_forced_current_thread()
1918 .event_loop_running
1919 {
1920 bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1921 }
1922 Ok(())
1923 }
1924
1925 #[inline]
1929 pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1930 if !self.concurrency_support() {
1932 return Ok(CurrentThread::None);
1933 }
1934
1935 if !self
1938 .vm_store_context_mut()
1939 .current_thread_mut()
1940 .is_deferred()
1941 {
1942 return Ok(self
1943 .concurrent_state_mut_already_forced_current_thread()
1944 .unforced_current_thread);
1945 }
1946
1947 self.force_deferred_current_thread()
1948 }
1949
1950 #[cold]
1953 fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1954 let state = self.concurrent_state_mut_without_forcing_current_thread();
1963 let id = match state.unforced_current_thread.guest_task() {
1964 Some(task) => state.get_mut(task)?.instance.instance,
1965 None => bail_bug!("deferred component-model thread with non-guest base"),
1966 };
1967
1968 let mut frames = SmallVec::<[_; 8]>::new();
1971 let mut cur = *self.vm_store_context_mut().current_thread_mut();
1972 while let Some(ptr) = cur.as_deferred() {
1973 let deferred = unsafe { ptr.as_non_null().as_ref() };
1978 frames.push((
1979 deferred.callee_async != 0,
1980 deferred.callee_instance,
1981 deferred.saved_context,
1982 deferred.lazy_lenders_start,
1983 ));
1984 cur = deferred.parent;
1985 }
1986
1987 *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1991
1992 let current_context = *self.vm_store_context_mut().component_context_mut();
1995
1996 let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
2003 frames.reverse();
2004 let lazy_lenders_base = match frames.first() {
2005 Some((_, _, _, start)) => *start,
2006 None => bail_bug!("forcing a deferred thread with no deferred frames"),
2007 };
2008 let lazy_lender_ends = frames
2009 .iter()
2010 .skip(1)
2011 .map(|(_, _, _, start)| *start)
2012 .chain([lazy_lenders_end])
2013 .collect::<SmallVec<[_; 8]>>();
2014 for ((callee_async, callee_instance, saved_context, start), lenders_end) in
2015 frames.into_iter().zip(lazy_lender_ends)
2016 {
2017 *self.vm_store_context_mut().component_context_mut() = saved_context;
2021 let callee = RuntimeInstance {
2022 instance: id,
2023 index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
2024 };
2025 self.enter_guest_sync_call(callee_async, callee)?;
2026
2027 self.promote_current_lazy_lends(id, start, lenders_end)?;
2030 }
2031
2032 debug_assert_eq!(
2035 *self.vm_store_context_mut().lazy_lenders_len_mut(),
2036 lazy_lenders_end
2037 );
2038 *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
2039
2040 *self.vm_store_context_mut().component_context_mut() = current_context;
2042
2043 Ok(self
2044 .concurrent_state_mut_without_forcing_current_thread()
2045 .unforced_current_thread)
2046 }
2047
2048 fn promote_current_lazy_lends(
2054 &mut self,
2055 id: ComponentInstanceId,
2056 start: usize,
2057 end: usize,
2058 ) -> Result<()> {
2059 if start == end {
2060 return Ok(());
2061 }
2062
2063 let scope = match self
2064 .concurrent_state_mut_without_forcing_current_thread()
2065 .unforced_current_thread
2066 .guest_task()
2067 {
2068 Some(task) => VMScope::id(task.rep()),
2069 None => bail_bug!("replayed deferred frame is not a guest thread"),
2070 };
2071 let instance = Instance::from_wasmtime(self, id);
2072 self.promote_lazy_lends(instance, scope, start, end)
2073 }
2074
2075 fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2076 match self.current_thread()?.guest() {
2077 Some(id) => Ok(*id),
2078 None => bail_bug!("current thread is not a guest thread"),
2079 }
2080 }
2081
2082 pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2086 match self.current_thread()? {
2087 CurrentThread::Host(id) => Ok(Some(id)),
2088 CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2089 _ => bail_bug!("current thread is not a host thread"),
2090 }
2091 }
2092
2093 fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2096 Ok(self
2097 .concurrent_state_mut()?
2098 .materialize_current_host_task_id()?)
2099 }
2100
2101 fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2102 log::trace!("enter sync-typed call {callee:?}");
2103 let state = self.instance_state(callee).concurrent_state();
2104 let old_do_not_suspend = state.do_not_suspend;
2105 state.do_not_suspend = true;
2106
2107 let thread = self.current_guest_thread()?;
2108 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2109 if thread.old_do_not_suspend.is_some() {
2110 bail_bug!("current thread already has `old_do_not_suspend` value");
2111 }
2112
2113 thread.old_do_not_suspend = Some(old_do_not_suspend);
2114
2115 Ok(())
2116 }
2117
2118 fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2119 log::trace!("exit sync-typed call {callee:?}");
2120 let thread = self.current_guest_thread()?;
2121 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2122 let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2123 bail_bug!("current thread missing `old_do_not_suspend` value");
2124 };
2125 let state = self.instance_state(callee).concurrent_state();
2126 state.do_not_suspend = old_do_not_suspend;
2127 Ok(())
2128 }
2129
2130 pub(crate) fn enter_guest_sync_call(
2142 &mut self,
2143 callee_async_typed: bool,
2144 callee: RuntimeInstance,
2145 ) -> Result<()> {
2146 log::trace!("enter sync-lifted call {callee:?}");
2147 if !self.concurrency_support() {
2148 return self.enter_call_not_concurrent();
2149 }
2150
2151 let state = self.concurrent_state_mut()?;
2155 let item = state.next_switch_item.take();
2156 state.saved_next_switch_items.push(item);
2157
2158 let thread = self.current_thread()?;
2159 let caller = if let Some(thread) = thread.guest() {
2160 Caller::Guest { thread: *thread }
2161 } else {
2162 Caller::Host {
2163 tx: None,
2164 caller: self.materialize_host_task_id()?,
2165 }
2166 };
2167
2168 let state = self.concurrent_state_mut()?;
2169 let guest_thread = GuestTask::new(
2170 state,
2171 Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2172 LiftResult {
2173 lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2174 ty: TypeTupleIndex::reserved_value(),
2175 memory: None,
2176 string_encoding: StringEncoding::Utf8,
2177 },
2178 caller,
2179 None,
2180 callee,
2181 callee_async_typed,
2182 false,
2183 )?;
2184
2185 Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2186 guest_thread.thread,
2187 self,
2188 callee.index,
2189 )?;
2190 self.set_thread(guest_thread)?;
2191
2192 if !callee_async_typed {
2193 self.enter_sync_call(callee)?;
2194 }
2195
2196 Ok(())
2197 }
2198
2199 pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2207 if !self.concurrency_support() {
2208 return Ok(self.exit_call_not_concurrent());
2209 }
2210
2211 let thread = match self.current_thread()?.guest() {
2212 Some(t) => *t,
2213 None => bail_bug!("expected task when exiting"),
2214 };
2215 let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2216 let instance = task.instance;
2217
2218 let caller = match &task.caller {
2219 &Caller::Guest { thread } => thread.into(),
2220 &Caller::Host { caller, .. } => caller
2221 .map(CurrentThread::Host)
2222 .unwrap_or(CurrentThread::None),
2223 };
2224 task.lift_result = None;
2225 task.exited = true;
2226 let async_typed = task.async_typed;
2227
2228 if !async_typed {
2229 self.exit_sync_call(instance)?;
2230 }
2231
2232 self.set_thread(caller)?;
2233
2234 log::trace!("exit sync-lifted call {instance:?}");
2235
2236 if async_typed {
2237 self.switch_or_trap_if_may_not_suspend(instance)?;
2242 }
2243
2244 self.cleanup_thread(thread, instance)?;
2245
2246 self.concurrent_state_mut()?
2247 .decrement_task_ref_count(thread.task)?;
2248
2249 let state = self.concurrent_state_mut()?;
2250 let Some(item) = state.saved_next_switch_items.pop() else {
2251 bail_bug!("unable to pop from `saved_next_switch_items`");
2252 };
2253 if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2254 state.push_high_priority(item);
2257 bail_bug!("`next_switch_item` unexpectedly already set");
2258 }
2259
2260 Ok(())
2261 }
2262
2263 pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2270 if !self.concurrency_support() {
2271 self.enter_call_not_concurrent()?;
2272 return Ok(None);
2273 }
2274 let caller = self.current_guest_thread()?;
2275 log::trace!("new deferred host task with caller {caller:?}");
2276
2277 self.set_thread(CurrentThread::DeferredHost(caller))?;
2278 let state = self.concurrent_state_mut()?;
2279 debug_assert!(state.deferred_host_call_context.is_none());
2280 state.deferred_host_call_context = Some(CallContext::default());
2281 state.debug_assert_deferred_host_invariant();
2282 Ok(Some(caller))
2283 }
2284
2285 pub(crate) fn host_task_delete(
2292 &mut self,
2293 original_task: EnteredHostTask,
2294 materialized_task: Option<TableId<HostTask>>,
2295 ) -> Result<()> {
2296 match original_task {
2297 Some(caller) => {
2298 self.set_thread(caller)?;
2299 if materialized_task.is_none() {
2300 let state = self.concurrent_state_mut()?;
2301 let context = state
2302 .deferred_host_call_context
2303 .take()
2304 .expect("deferred host call context should be present");
2305 debug_assert!(context.is_empty());
2306 state.debug_assert_deferred_host_invariant();
2307 }
2308 log::trace!(
2309 "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2310 );
2311 if let Some(task) = materialized_task {
2312 self.concurrent_state_mut()?.delete_host_task(task)?;
2313 }
2314 }
2315 None => {
2316 debug_assert!(materialized_task.is_none());
2317 self.exit_call_not_concurrent();
2318 }
2319 }
2320 Ok(())
2321 }
2322
2323 fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2326 self.component_instance_mut(instance.instance)
2327 .instance_state(instance.index)
2328 }
2329
2330 fn instance_state_and_handle_counter(
2333 &mut self,
2334 instance: RuntimeInstance,
2335 ) -> (&mut InstanceState, &mut HandleCounter) {
2336 let (component, counter) = self
2337 .store_data_mut()
2338 .component_instance_and_handle_counter_mut(instance.instance);
2339 (component.instance_state(instance.index), counter)
2340 }
2341
2342 fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2344 let (component, counter) = self
2345 .store_data_mut()
2346 .component_instance_and_handle_counter_mut(instance.instance);
2347 component.handle_table_insertion(instance.index, counter)
2348 }
2349
2350 pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2356 let thread = thread.into();
2357 let state = self.concurrent_state_mut()?;
2358 state.debug_assert_deferred_host_invariant();
2359 let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2360
2361 state.handle_thread_switch(old_thread, thread)?;
2362
2363 if let Some(old_thread) = old_thread.guest() {
2371 let old_context = *self.vm_store_context_mut().component_context_mut();
2372 self.concurrent_state_mut()?
2373 .get_mut(old_thread.thread)?
2374 .context = old_context;
2375 }
2376 if cfg!(debug_assertions) {
2377 *self.vm_store_context_mut().component_context_mut() =
2378 [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2379 }
2380 if let Some(thread) = thread.guest() {
2381 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2382 let context = thread.context;
2383 if cfg!(debug_assertions) {
2384 thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2385 }
2386 *self.vm_store_context_mut().component_context_mut() = context;
2387 }
2388
2389 *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2391 VMLazyThread::none()
2392 } else {
2393 VMLazyThread::forced()
2394 };
2395
2396 Ok(old_thread)
2397 }
2398
2399 fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2401 if self.switch_if_may_not_suspend(instance)? {
2402 Ok(())
2403 } else {
2404 Err(Trap::CannotBlockSyncTask.into())
2405 }
2406 }
2407
2408 fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2412 self.concurrent_state_mut()?;
2416
2417 Ok(!self.concurrency_support()
2418 || !self
2419 .instance_state(instance)
2420 .concurrent_state()
2421 .do_not_suspend
2422 || self
2423 .concurrent_state_mut()?
2424 .promote_instance_local_thread_work_item(instance)?)
2425 }
2426
2427 fn enter_instance(&mut self, instance: RuntimeInstance) {
2431 log::trace!("enter {instance:?}");
2432 self.instance_state(instance)
2433 .concurrent_state()
2434 .do_not_enter = true;
2435 }
2436
2437 fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2441 log::trace!("exit {instance:?}");
2442 self.instance_state(instance)
2443 .concurrent_state()
2444 .do_not_enter = false;
2445 self.partition_pending(instance)
2446 }
2447
2448 fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2456 let mut any_ready = false;
2457 for (thread, kind) in
2458 mem::take(&mut self.instance_state(instance).concurrent_state().pending)
2459 {
2460 let call = GuestCall { thread, kind };
2461 if call.is_ready(self)? {
2462 any_ready = true;
2463 self.concurrent_state_mut()?
2464 .push_high_priority(WorkItem::GuestCall { instance, call });
2465 } else {
2466 self.instance_state(instance)
2467 .concurrent_state()
2468 .pending
2469 .insert(call.thread, call.kind);
2470 }
2471 }
2472
2473 if any_ready {
2474 if let Some(waker) = self
2475 .concurrent_state_mut()?
2476 .ready_for_concurrent_call_waker
2477 .take()
2478 {
2479 waker.wake();
2480 }
2481 }
2482
2483 Ok(())
2484 }
2485
2486 pub(crate) fn backpressure_modify(
2488 &mut self,
2489 caller_instance: RuntimeInstance,
2490 modify: impl FnOnce(u16) -> Option<u16>,
2491 ) -> Result<()> {
2492 let state = self.instance_state(caller_instance).concurrent_state();
2493 let old = state.backpressure;
2494 let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2495 state.backpressure = new;
2496
2497 log::trace!("backpressure update for {caller_instance:?} from {old} to {new}");
2498
2499 if old > 0 && new == 0 {
2500 self.partition_pending(caller_instance)?;
2503 }
2504
2505 Ok(())
2506 }
2507
2508 fn resume_fiber(
2511 &mut self,
2512 fiber: StoreFiber<'static>,
2513 ) -> impl Future<Output = Result<()>> + '_ {
2514 let mut resuming = ResumingFiber {
2515 store: self,
2516 fiber: Some(fiber),
2517 };
2518 async move {
2519 let old_thread = resuming.store.current_thread()?;
2520 log::trace!("resume_fiber: save current thread {old_thread:?}");
2521
2522 let fiber = resuming.fiber.take().unwrap();
2523 resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2524
2525 resuming.store.set_thread(old_thread)?;
2526
2527 let state = resuming.store.concurrent_state_mut()?;
2528
2529 if let Some(ot) = old_thread.guest() {
2530 state.set_thread_running(ot.thread)?;
2531 }
2532 log::trace!("resume_fiber: restore current thread {old_thread:?}");
2533
2534 if resuming.fiber.is_some() {
2535 log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2536 let reason = match state.suspend_reason.take() {
2538 Some(r) => r,
2539 None => bail_bug!("suspend reason missing when resuming fiber"),
2540 };
2541 match reason {
2542 SuspendReason::NeedWork => {
2543 if state.worker.is_none() {
2544 state.worker = resuming.fiber.take();
2545 }
2546 }
2547 SuspendReason::Yielding { thread } => {
2548 let slot = &mut state.get_mut(thread.thread)?.state;
2549 slot.check_no_fiber()?;
2550 *slot = GuestThreadState::Ready {
2551 fiber: resuming.fiber.take().unwrap(),
2552 };
2553 let instance = state.get_mut(thread.task)?.instance;
2554 state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2555 }
2556 SuspendReason::ExplicitlySuspending { thread } => {
2557 let slot = &mut state.get_mut(thread.thread)?.state;
2558 slot.check_no_fiber()?;
2559 *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2560 }
2561 SuspendReason::Waiting { set, thread } => {
2562 match state.get_mut(set)?.waiting.entry(thread) {
2563 Entry::Vacant(entry) => {
2564 entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2565 }
2566 Entry::Occupied(_) => {
2567 bail_bug!("entry unexpectedly already exists for {thread:?}");
2568 }
2569 }
2570 }
2571 SuspendReason::YieldingToSubtask { thread } => {
2572 let instance = state.get_mut(thread.task)?.instance;
2581 if state.next_switch_item.is_some() {
2582 bail_bug!(
2585 "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2586 a thread wanted to wait on a subtask"
2587 );
2588 }
2589 let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2590 instance,
2591 thread,
2592 fiber: resuming.fiber.take().unwrap(),
2593 });
2594 log::trace!("set next switch item to {item:?}");
2595 }
2596 };
2597 } else {
2598 log::trace!("resume_fiber: fiber has exited");
2599 }
2600
2601 Ok(())
2602 }
2603 }
2604
2605 fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2611 log::trace!("suspend fiber: {reason:?}");
2612
2613 if !self.fiber_async_state_mut().can_block() {
2616 return Err(format_err!("future dropped"));
2617 }
2618
2619 let state = self.concurrent_state_mut()?;
2620
2621 let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2628 SuspendReason::Yielding { .. }
2629 | SuspendReason::Waiting { .. }
2630 | SuspendReason::ExplicitlySuspending { .. } => {
2631 if state.switch_item.is_none() {
2634 state.take_next_switch_item()?;
2635 }
2636
2637 (true, false)
2638 }
2639 SuspendReason::YieldingToSubtask { .. } => (true, true),
2640 SuspendReason::NeedWork => (false, false),
2641 };
2642
2643 let old_next_switch_item = if save_and_restore_next_switch_item {
2644 Some(state.save_next_switch_item()?)
2645 } else {
2646 None
2647 };
2648
2649 let old_guest_thread = if save_and_restore_thread {
2650 self.current_thread()?
2651 } else {
2652 CurrentThread::None
2653 };
2654
2655 let waiting_set = match &reason {
2656 SuspendReason::Waiting { set, .. } => Some(*set),
2657 _ => None,
2658 };
2659
2660 let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2661 if suspend_reason.is_some() {
2662 bail_bug!("suspend reason unexpectedly already set for store");
2663 }
2664 *suspend_reason = Some(reason);
2665
2666 if let Some(set) = waiting_set {
2669 self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2670 }
2671
2672 self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2673
2674 if let Some(set) = waiting_set {
2675 self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2676 }
2677
2678 if save_and_restore_thread {
2679 self.set_thread(old_guest_thread)?;
2680 }
2681
2682 if let Some(item) = old_next_switch_item {
2683 let state = self.concurrent_state_mut()?;
2684 state.restore_next_switch_item(item)?;
2685 }
2686
2687 Ok(())
2688 }
2689
2690 fn wait_for_event(
2691 &mut self,
2692 caller_instance: RuntimeInstance,
2693 waitable: Waitable,
2694 ) -> Result<()> {
2695 let caller = self.current_guest_thread()?;
2696 let state = self.concurrent_state_mut()?;
2697
2698 waitable.trap_if_in_waitable_set(state)?;
2699
2700 let set = state.get_mut(caller.thread)?.sync_call_set;
2701 waitable.join(state, Some(set))?;
2702
2703 self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2704
2705 self.suspend(SuspendReason::Waiting {
2706 set,
2707 thread: caller,
2708 })?;
2709 let state = self.concurrent_state_mut()?;
2710
2711 waitable.join(state, None)
2712 }
2713
2714 fn cleanup_thread(
2736 &mut self,
2737 guest_thread: QualifiedThreadId,
2738 runtime_instance: RuntimeInstance,
2739 ) -> Result<()> {
2740 let state = self.concurrent_state_mut()?;
2741 state.take_next_switch_item()?;
2744 let thread_data = state.get_mut(guest_thread.thread)?;
2745 let sync_call_set = thread_data.sync_call_set;
2746 if let Some(guest_id) = thread_data.instance_rep {
2747 self.instance_state(runtime_instance)
2748 .thread_handle_table()
2749 .guest_thread_remove(guest_id)?;
2750 }
2751 let state = self.concurrent_state_mut()?;
2752
2753 for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2755 if let Some(Event::Subtask {
2756 status: Status::Returned | Status::ReturnCancelled,
2757 }) = waitable.common(state)?.event
2758 {
2759 waitable.delete_from(state)?;
2760 }
2761 }
2762
2763 state.delete(guest_thread.thread)?;
2764 state.delete(sync_call_set)?;
2765 let task = state.get_mut(guest_thread.task)?;
2766 task.threads.remove(&guest_thread.thread);
2767
2768 if task.threads.is_empty() && !task.returned_or_cancelled() {
2769 bail!(Trap::NoAsyncResult);
2770 }
2771
2772 if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2773 task.decremented_interesting_task_count = true;
2774
2775 debug_assert!(state.interesting_tasks > 0);
2776 state.interesting_tasks -= 1;
2777 if state.interesting_tasks == 0
2778 && let Some(waker) = state.interesting_tasks_empty_waker.take()
2779 {
2780 waker.wake();
2781 }
2782 }
2783
2784 state.decrement_task_ref_count(guest_thread.task)?;
2785
2786 Ok(())
2787 }
2788
2789 fn cancel_guest_subtask_without_lowered_parameters(
2798 &mut self,
2799 caller_instance: RuntimeInstance,
2800 guest_task: TableId<GuestTask>,
2801 ) -> Result<()> {
2802 let concurrent_state = self.concurrent_state_mut()?;
2803 let task = concurrent_state.get_mut(guest_task)?;
2804 assert!(!task.already_lowered_parameters());
2805 task.lower_params = None;
2809 task.lift_result = None;
2810 task.exited = true;
2811 let instance = task.instance;
2812
2813 assert_eq!(1, task.threads.len());
2820 let thread = *task.threads.iter().next().unwrap();
2821 let next_switch_item = concurrent_state.save_next_switch_item()?;
2822 self.cleanup_thread(
2823 QualifiedThreadId {
2824 task: guest_task,
2825 thread,
2826 },
2827 caller_instance,
2828 )?;
2829 self.concurrent_state_mut()?
2830 .restore_next_switch_item(next_switch_item)?;
2831
2832 let pending = &mut self.instance_state(instance).concurrent_state().pending;
2834 let pending_count = pending.len();
2835 let mut decrement = 0;
2836 let mut consume = |kind: &GuestCallKind| {
2837 if let GuestCallKind::StartImplicit {
2838 was_pending: true, ..
2839 } = kind
2840 {
2841 decrement += 1;
2842 }
2843 };
2844 pending.retain(|thread, kind| {
2845 if thread.task == guest_task {
2846 consume(kind);
2847 false
2848 } else {
2849 true
2850 }
2851 });
2852
2853 if pending.len() == pending_count {
2854 let mut matches = |item: &WorkItem| match item {
2857 WorkItem::ResumeThread { thread, .. } => thread.task == guest_task,
2858 WorkItem::GuestCall {
2859 call: GuestCall { thread, kind },
2860 ..
2861 } => {
2862 if thread.task == guest_task {
2863 consume(kind);
2864 true
2865 } else {
2866 false
2867 }
2868 }
2869 _ => false,
2870 };
2871
2872 let mut remove = |queue: &mut VecDeque<WorkItem>| {
2873 let count = queue.len();
2874 queue.retain(|item| !matches(item));
2875 count != queue.len()
2876 };
2877
2878 let state = self.concurrent_state_mut()?;
2879 let mut found = remove(&mut state.high_priority);
2880 found |= remove(&mut state.low_priority);
2881
2882 for item in [&mut state.switch_item, &mut state.next_switch_item] {
2883 if matches!(item, Some(item) if matches(item)) {
2884 found = true;
2885 *item = None;
2886 }
2887 }
2888
2889 if !found {
2890 bail!(Trap::SubtaskCancelAfterTerminal);
2891 }
2892 }
2893
2894 self.decrement_pending_start_count(instance, decrement)?;
2895
2896 Ok(())
2897 }
2898
2899 fn decrement_pending_start_count(
2900 &mut self,
2901 instance: RuntimeInstance,
2902 decrement: usize,
2903 ) -> Result<()> {
2904 if decrement > 0 {
2905 let count = &mut self
2906 .instance_state(instance)
2907 .concurrent_state()
2908 .pending_start_count;
2909
2910 if *count < decrement {
2911 bail_bug!("`ConcurrentInstantState::pending_start_count` underflow");
2912 }
2913
2914 *count -= decrement;
2915
2916 self.partition_pending(instance)?;
2918 }
2919
2920 Ok(())
2921 }
2922
2923 pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2926 if !self.concurrency_support() {
2927 return Ok(self
2928 .current_scope_id_not_concurrent()?
2929 .map(|id| CurrentScope::Id(VMScope::id(id))));
2930 }
2931
2932 Ok(match self.current_thread()? {
2933 CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2934 CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2935 CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2936 CurrentThread::None => return Ok(None),
2937 })
2938 }
2939
2940 pub(crate) fn queue_task(
2941 &mut self,
2942 task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2943 ) -> Result<()> {
2944 self.concurrent_state_mut()?
2945 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2946 Ok(())
2947 }
2948
2949 fn any_may_not_suspend(&mut self) -> Result<bool> {
2958 Ok(self
2966 .concurrent_state_mut()?
2967 .table
2968 .get_mut()
2969 .iter_mut()
2970 .filter_map(|(_, entry)| {
2971 if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2972 Some(task.instance)
2973 } else {
2974 None
2975 }
2976 })
2977 .collect::<Vec<_>>()
2978 .into_iter()
2979 .any(|instance| {
2980 self.instance_state(instance)
2981 .concurrent_state()
2982 .do_not_suspend
2983 }))
2984 }
2985}
2986
2987impl Instance {
2988 fn get_event(
2991 self,
2992 store: &mut StoreOpaque,
2993 guest_task: TableId<GuestTask>,
2994 set: Option<TableId<WaitableSet>>,
2995 cancellable: bool,
2996 ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
2997 let state = store.concurrent_state_mut()?;
2998
2999 let task = state.get_mut(guest_task)?;
3000 let event = &mut task.event;
3001 if let Some(ev) = event
3002 && (cancellable || !matches!(ev, Event::Cancelled))
3003 {
3004 log::trace!("deliver event {ev:?} to {guest_task:?}");
3005
3006 if matches!(ev, Event::Cancelled) {
3007 task.cancel_request_delivered = true;
3008 }
3009
3010 let ev = *ev;
3011 *event = None;
3012 return Ok(Some((ev, None)));
3013 }
3014
3015 let set = match set {
3016 Some(set) => set,
3017 None => return Ok(None),
3018 };
3019 let waitable = match state.get_mut(set)?.ready.pop_first() {
3020 Some(v) => v,
3021 None => return Ok(None),
3022 };
3023
3024 let common = waitable.common(state)?;
3025 let handle = match common.handle {
3026 Some(h) => h,
3027 None => bail_bug!("handle not set when delivering event"),
3028 };
3029 let event = match common.event.take() {
3030 Some(e) => e,
3031 None => bail_bug!("event not set when delivering event"),
3032 };
3033
3034 log::trace!(
3035 "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
3036 );
3037
3038 waitable.on_delivery(store, self, event)?;
3039
3040 Ok(Some((event, Some((waitable, handle)))))
3041 }
3042
3043 fn handle_callback_code(
3050 self,
3051 store: &mut dyn VMStore,
3052 guest_thread: QualifiedThreadId,
3053 runtime_instance: RuntimeInstance,
3054 call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
3055 ) -> Result<()> {
3056 let old_thread = store.set_thread(guest_thread)?;
3057 log::trace!(
3058 "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
3059 );
3060
3061 store.enter_instance(runtime_instance);
3062
3063 let mut code = call(store)?;
3064
3065 if matches!(
3071 unpack_callback_code(code).0,
3072 callback_code::YIELD | callback_code::WAIT
3073 ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
3074 {
3075 let (unpacked, set) = unpack_callback_code(code);
3076 log::trace!(
3077 "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
3078 cancellation; delivering it via callback"
3079 );
3080 code = invoke_callback(store, guest_thread.task, event, 0)?;
3081 }
3082
3083 store.exit_instance(runtime_instance)?;
3084
3085 store.set_thread(old_thread)?;
3086 log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
3087
3088 let store = &mut **store;
3089 let (code, set) = unpack_callback_code(code);
3090
3091 log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
3092
3093 let state = store.concurrent_state_mut()?;
3094
3095 state.take_next_switch_item()?;
3096
3097 let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
3098 let set = store
3099 .instance_state(runtime_instance)
3100 .handle_table()
3101 .waitable_set_rep(handle)?;
3102
3103 Ok(TableId::<WaitableSet>::new(set))
3104 };
3105
3106 match code {
3107 callback_code::EXIT => {
3108 log::trace!("implicit thread {guest_thread:?} completed");
3109 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3110 task.exited = true;
3111 task.callback = None;
3112
3113 store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
3118
3119 store.cleanup_thread(guest_thread, runtime_instance)?;
3120 }
3121 callback_code::YIELD => {
3122 let old = state
3125 .get_mut(guest_thread.thread)?
3126 .wake_on_cancel
3127 .replace(WakeOnCancel::Yielding);
3128 if !old.is_none() {
3129 bail_bug!("thread unexpectedly had wake_on_cancel set");
3130 }
3131
3132 let call = GuestCall {
3139 thread: guest_thread,
3140 kind: GuestCallKind::DeliverEvent {
3141 instance: self,
3142 set: None,
3143 },
3144 };
3145 state.push_low_priority(WorkItem::GuestCall {
3148 instance: runtime_instance,
3149 call,
3150 });
3151 }
3152 callback_code::WAIT => {
3153 let set = get_set(store, set)?;
3154 self.wait_with_callback(store, guest_thread, set)?;
3155 }
3156 _ => bail!(Trap::UnsupportedCallbackCode),
3157 }
3158
3159 Ok(())
3160 }
3161
3162 fn wait_with_callback(
3168 self,
3169 store: &mut StoreOpaque,
3170 guest_thread: QualifiedThreadId,
3171 set: TableId<WaitableSet>,
3172 ) -> Result<()> {
3173 let state = store.concurrent_state_mut()?;
3174 state.get_mut(set)?.num_waiting += 1;
3177
3178 if state.get_mut(guest_thread.task)?.event.is_some()
3179 || !state.get_mut(set)?.ready.is_empty()
3180 {
3181 let instance = state.get_mut(guest_thread.task)?.instance;
3183 state.push_high_priority(WorkItem::GuestCall {
3184 instance,
3185 call: GuestCall {
3186 thread: guest_thread,
3187 kind: GuestCallKind::DeliverEvent {
3188 instance: self,
3189 set: Some(set),
3190 },
3191 },
3192 });
3193 return Ok(());
3194 }
3195
3196 let instance = store
3202 .concurrent_state_mut()?
3203 .get_mut(guest_thread.task)?
3204 .instance;
3205 store.switch_or_trap_if_may_not_suspend(instance)?;
3206
3207 let state = store.concurrent_state_mut()?;
3213 let old = state
3214 .get_mut(guest_thread.thread)?
3215 .wake_on_cancel
3216 .replace(WakeOnCancel::Waiting(set));
3217 if !old.is_none() {
3218 bail_bug!("thread unexpectedly had wake_on_cancel set");
3219 }
3220 let old = state
3221 .get_mut(set)?
3222 .waiting
3223 .insert(guest_thread, WaitMode::Callback(self));
3224 if !old.is_none() {
3225 bail_bug!("set's waiting set already had this thread registered");
3226 }
3227 Ok(())
3228 }
3229
3230 unsafe fn stage_call<T: 'static>(
3237 self,
3238 mut store: StoreContextMut<T>,
3239 guest_thread: QualifiedThreadId,
3240 callee: SendSyncPtr<VMFuncRef>,
3241 param_count: usize,
3242 result_count: usize,
3243 async_: bool,
3244 callback: Option<SendSyncPtr<VMFuncRef>>,
3245 post_return: Option<SendSyncPtr<VMFuncRef>>,
3246 host_caller: bool,
3247 ) -> Result<()> {
3248 unsafe fn make_call<T: 'static>(
3263 store: StoreContextMut<T>,
3264 guest_thread: QualifiedThreadId,
3265 callee: SendSyncPtr<VMFuncRef>,
3266 param_count: usize,
3267 result_count: usize,
3268 ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3269 + Send
3270 + Sync
3271 + 'static
3272 + use<T> {
3273 let token = StoreToken::new(store);
3274 move |store: &mut dyn VMStore| {
3275 let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3276
3277 store
3278 .concurrent_state_mut()?
3279 .set_thread_running(guest_thread.thread)?;
3280 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3281 let lower = match task.lower_params.take() {
3282 Some(l) => l,
3283 None => bail_bug!("lower_params missing"),
3284 };
3285
3286 lower(store, &mut storage[..param_count])?;
3287
3288 let mut store = token.as_context_mut(store);
3289
3290 unsafe {
3293 crate::Func::call_unchecked_raw(
3294 &mut store,
3295 callee.as_non_null(),
3296 NonNull::new(
3297 &mut storage[..param_count.max(result_count)]
3298 as *mut [MaybeUninit<ValRaw>] as _,
3299 )
3300 .unwrap(),
3301 UncaughtException::Trap,
3302 )?;
3303 }
3304
3305 Ok(storage)
3306 }
3307 }
3308
3309 let call = unsafe {
3313 make_call(
3314 store.as_context_mut(),
3315 guest_thread,
3316 callee,
3317 param_count,
3318 result_count,
3319 )
3320 };
3321
3322 let callee_instance = store
3323 .0
3324 .concurrent_state_mut()?
3325 .get_mut(guest_thread.task)?
3326 .instance;
3327
3328 let fun = if callback.is_some() {
3329 assert!(async_);
3330
3331 Box::new(move |store: &mut dyn VMStore| {
3332 self.add_guest_thread_to_instance_table(
3333 guest_thread.thread,
3334 store,
3335 callee_instance.index,
3336 )?;
3337 self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3338 let storage = call(store)?;
3339
3340 Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3343 })
3344 }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3345 } else {
3346 let token = StoreToken::new(store.as_context_mut());
3347 Box::new(move |store: &mut dyn VMStore| {
3348 self.add_guest_thread_to_instance_table(
3349 guest_thread.thread,
3350 store,
3351 callee_instance.index,
3352 )?;
3353 let old_thread = store.set_thread(guest_thread)?;
3354 log::trace!(
3355 "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3356 );
3357 let flags = self.id().get(store).instance_flags(callee_instance.index);
3358
3359 let callee_async_typed = store
3360 .concurrent_state_mut()?
3361 .get_mut(guest_thread.task)?
3362 .async_typed;
3363
3364 if !async_ && callee_async_typed {
3368 store.enter_instance(callee_instance);
3369 }
3370
3371 if !callee_async_typed {
3372 store.enter_sync_call(callee_instance)?;
3373 }
3374
3375 let storage = call(store)?;
3382
3383 if !callee_async_typed {
3384 store.exit_sync_call(callee_instance)?;
3385 }
3386
3387 if !async_ {
3388 if callee_async_typed {
3394 store.exit_instance(callee_instance)?;
3395 }
3396
3397 let lift = {
3398 let state = store.concurrent_state_mut()?;
3399 if !state.get_mut(guest_thread.task)?.result.is_none() {
3400 bail_bug!("task has already produced a result");
3401 }
3402
3403 match state.get_mut(guest_thread.task)?.lift_result.take() {
3404 Some(lift) => lift,
3405 None => bail_bug!("lift_result field is missing"),
3406 }
3407 };
3408
3409 let result = (lift.lift)(store, unsafe {
3412 mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3413 &storage[..result_count],
3414 )
3415 })?;
3416
3417 let post_return_arg = match result_count {
3418 0 => ValRaw::i32(0),
3419 1 => unsafe { storage[0].assume_init() },
3422 _ => unreachable!(),
3423 };
3424
3425 unsafe {
3426 call_post_return(
3427 token.as_context_mut(store),
3428 post_return.map(|v| v.as_non_null()),
3429 post_return_arg,
3430 flags,
3431 )?;
3432 }
3433
3434 self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3435 }
3436
3437 store.set_thread(old_thread)?;
3438
3439 store
3440 .concurrent_state_mut()?
3441 .get_mut(guest_thread.task)?
3442 .exited = true;
3443
3444 log::trace!(
3445 "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3446 );
3447
3448 if callee_async_typed {
3449 store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3454 }
3455
3456 store.cleanup_thread(guest_thread, callee_instance)?;
3458 Ok(())
3459 })
3460 };
3461
3462 store.0.concurrent_state_mut()?.push_work_item(
3463 WorkItem::GuestCall {
3464 instance: callee_instance,
3465 call: GuestCall {
3466 thread: guest_thread,
3467 kind: GuestCallKind::StartImplicit {
3468 closure: fun,
3469 was_pending: false,
3470 },
3471 },
3472 },
3473 if host_caller {
3474 Priority::Low
3480 } else {
3481 Priority::Switch
3482 },
3483 )?;
3484
3485 Ok(())
3486 }
3487
3488 unsafe fn prepare_call<T: 'static>(
3501 self,
3502 mut store: StoreContextMut<T>,
3503 start: NonNull<VMFuncRef>,
3504 return_: NonNull<VMFuncRef>,
3505 caller_instance: RuntimeComponentInstanceIndex,
3506 callee_instance: RuntimeComponentInstanceIndex,
3507 task_return_type: TypeTupleIndex,
3508 callee_async_typed: bool,
3509 memory: *mut VMMemoryDefinition,
3510 string_encoding: StringEncoding,
3511 caller_info: CallerInfo,
3512 ) -> Result<()> {
3513 enum ResultInfo {
3514 Heap { results: u32 },
3515 Stack { result_count: u32 },
3516 }
3517
3518 let result_info = match &caller_info {
3519 CallerInfo::Async {
3520 has_result: true,
3521 params,
3522 } => ResultInfo::Heap {
3523 results: match params.last() {
3525 Some(r) => r.get_u32(),
3526 None => bail_bug!("retptr missing"),
3527 },
3528 },
3529 CallerInfo::Async {
3530 has_result: false, ..
3531 } => ResultInfo::Stack { result_count: 0 },
3532 CallerInfo::Sync {
3533 result_count,
3534 params,
3535 } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3536 results: match params.last() {
3538 Some(r) => r.get_u32(),
3539 None => bail_bug!("arg ptr missing"),
3540 },
3541 },
3542 CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3543 result_count: *result_count,
3544 },
3545 };
3546
3547 let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3548
3549 let start = SendSyncPtr::new(start);
3553 let return_ = SendSyncPtr::new(return_);
3554 let token = StoreToken::new(store.as_context_mut());
3555 let old_thread = store.0.current_guest_thread()?;
3556
3557 let state = store.0.concurrent_state_mut()?;
3558
3559 debug_assert_eq!(
3560 state.get_mut(old_thread.task)?.instance,
3561 self.runtime_instance(caller_instance)
3562 );
3563
3564 let guest_thread = GuestTask::new(
3565 state,
3566 Box::new(move |store, dst| {
3567 let mut store = token.as_context_mut(store);
3568 assert!(dst.len() <= MAX_FLAT_PARAMS);
3569 let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3571 let count = match caller_info {
3572 CallerInfo::Async { params, has_result } => {
3576 let params = ¶ms[..params.len() - usize::from(has_result)];
3577 for (param, src) in params.iter().zip(&mut src) {
3578 src.write(*param);
3579 }
3580 params.len()
3581 }
3582
3583 CallerInfo::Sync { params, .. } => {
3585 for (param, src) in params.iter().zip(&mut src) {
3586 src.write(*param);
3587 }
3588 params.len()
3589 }
3590 };
3591 unsafe {
3598 crate::Func::call_unchecked_raw(
3599 &mut store,
3600 start.as_non_null(),
3601 NonNull::new(
3602 &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3603 )
3604 .unwrap(),
3605 UncaughtException::Trap,
3606 )?;
3607 }
3608 dst.copy_from_slice(&src[..dst.len()]);
3609 let task = store.0.current_guest_thread()?.task;
3610 let state = store.0.concurrent_state_mut()?;
3611 Waitable::Guest(task).set_event(
3612 state,
3613 Some(Event::Subtask {
3614 status: Status::Started,
3615 }),
3616 )?;
3617 Ok(())
3618 }),
3619 LiftResult {
3620 lift: Box::new(move |store, src| {
3621 let mut store = token.as_context_mut(store);
3624 let mut my_src = src.to_owned(); if let ResultInfo::Heap { results } = &result_info {
3626 my_src.push(ValRaw::u32(*results));
3627 }
3628
3629 unsafe {
3636 crate::Func::call_unchecked_raw(
3637 &mut store,
3638 return_.as_non_null(),
3639 my_src.as_mut_slice().into(),
3640 UncaughtException::Trap,
3641 )?;
3642 }
3643
3644 let thread = store.0.current_guest_thread()?;
3645 let state = store.0.concurrent_state_mut()?;
3646 if sync_caller {
3647 state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3648 if let ResultInfo::Stack { result_count } = &result_info {
3649 match result_count {
3650 0 => None,
3651 1 => Some(my_src[0]),
3652 _ => unreachable!(),
3653 }
3654 } else {
3655 None
3656 },
3657 );
3658 }
3659 Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3660 }),
3661 ty: task_return_type,
3662 memory: NonNull::new(memory).map(SendSyncPtr::new),
3663 string_encoding,
3664 },
3665 Caller::Guest { thread: old_thread },
3666 None,
3667 self.runtime_instance(callee_instance),
3668 callee_async_typed,
3669 false,
3672 )?;
3673
3674 store.0.set_thread(guest_thread)?;
3677 log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3678
3679 Ok(())
3680 }
3681
3682 unsafe fn call_callback<T>(
3687 self,
3688 mut store: StoreContextMut<T>,
3689 function: SendSyncPtr<VMFuncRef>,
3690 event: Event,
3691 handle: u32,
3692 ) -> Result<u32> {
3693 let (ordinal, result) = event.parts();
3694 let params = &mut [
3695 ValRaw::u32(ordinal),
3696 ValRaw::u32(handle),
3697 ValRaw::u32(result),
3698 ];
3699 unsafe {
3704 crate::Func::call_unchecked_raw(
3705 &mut store,
3706 function.as_non_null(),
3707 params.as_mut_slice().into(),
3708 UncaughtException::Trap,
3709 )?;
3710 }
3711 Ok(params[0].get_u32())
3712 }
3713
3714 unsafe fn start_call<T: 'static>(
3732 self,
3733 mut store: StoreContextMut<T>,
3734 callback: *mut VMFuncRef,
3735 post_return: *mut VMFuncRef,
3736 callee: NonNull<VMFuncRef>,
3737 param_count: u32,
3738 result_count: u32,
3739 flags: u32,
3740 storage: &mut [MaybeUninit<ValRaw>],
3741 ) -> Result<()> {
3742 let token = StoreToken::new(store.as_context_mut());
3743 let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3744 let guest_thread = store.0.current_guest_thread()?;
3745 let state = store.0.concurrent_state_mut()?;
3746
3747 if !state.event_loop_running {
3752 bail!(
3753 "async function started during an original synchronous \
3754 entrypoint; please use an async-style entrypoint instead"
3755 );
3756 }
3757
3758 let callee = SendSyncPtr::new(callee);
3759 let param_count = usize::try_from(param_count)?;
3760 assert!(param_count <= MAX_FLAT_PARAMS);
3761 let result_count = usize::try_from(result_count)?;
3762 assert!(result_count <= MAX_FLAT_RESULTS);
3763
3764 let task = state.get_mut(guest_thread.task)?;
3765 let callee_async_typed = task.async_typed;
3766 let callee_instance = task.instance;
3767
3768 task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3769
3770 if let Some(callback) = NonNull::new(callback) {
3771 let callback = SendSyncPtr::new(callback);
3775 task.callback = Some(Box::new(move |store, event, handle| {
3776 let store = token.as_context_mut(store);
3777 unsafe { self.call_callback::<T>(store, callback, event, handle) }
3778 }));
3779 }
3780
3781 let Caller::Guest { thread: caller } = &task.caller else {
3782 bail_bug!("start_call unexpectedly invoked for host->guest call");
3785 };
3786 let caller = *caller;
3787 let caller_instance = state.get_mut(caller.task)?.instance;
3788
3789 unsafe {
3791 self.stage_call(
3792 store.as_context_mut(),
3793 guest_thread,
3794 callee,
3795 param_count,
3796 result_count,
3797 (flags & START_FLAG_ASYNC_CALLEE) != 0,
3798 NonNull::new(callback).map(SendSyncPtr::new),
3799 NonNull::new(post_return).map(SendSyncPtr::new),
3800 false,
3801 )?;
3802 }
3803
3804 let old_do_not_suspend = if callee_async_typed {
3805 let state = store.0.instance_state(callee_instance).concurrent_state();
3812 let old_do_not_suspend = state.do_not_suspend;
3813 state.do_not_suspend = false;
3814 Some(old_do_not_suspend)
3815 } else {
3816 None
3817 };
3818
3819 let state = store.0.concurrent_state_mut()?;
3820
3821 let guest_waitable = Waitable::Guest(guest_thread.task);
3824 let old_set = guest_waitable.common(state)?.set;
3825 let set = state.get_mut(caller.thread)?.sync_call_set;
3826 guest_waitable.join(state, Some(set))?;
3827
3828 store.0.set_thread(CurrentThread::None)?;
3829
3830 let mut yielded = false;
3846 let (status, waitable) = loop {
3847 store.0.suspend(if yielded {
3848 SuspendReason::Waiting {
3849 set,
3850 thread: caller,
3851 }
3852 } else {
3853 yielded = true;
3854 SuspendReason::YieldingToSubtask { thread: caller }
3855 })?;
3856
3857 if let Some(old_do_not_suspend) = old_do_not_suspend {
3858 store
3859 .0
3860 .instance_state(callee_instance)
3861 .concurrent_state()
3862 .do_not_suspend = old_do_not_suspend;
3863 }
3864
3865 let state = store.0.concurrent_state_mut()?;
3866
3867 log::trace!("taking event for {:?}", guest_thread.task);
3868 let event = guest_waitable.take_event(state)?;
3869 let Some(Event::Subtask { status }) = event else {
3870 bail_bug!("subtasks should only get subtask events, got {event:?}")
3871 };
3872
3873 log::trace!("status {status:?} for {:?}", guest_thread.task);
3874
3875 if status == Status::Returned {
3876 break (status, None);
3878 } else if async_caller {
3879 let handle = store
3883 .0
3884 .handle_table_insertion(caller_instance)
3885 .subtask_guest(guest_thread.task.rep())?;
3886 store
3887 .0
3888 .concurrent_state_mut()?
3889 .get_mut(guest_thread.task)?
3890 .common
3891 .handle = Some(handle);
3892 break (status, Some(handle));
3893 } else {
3894 store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3898 }
3899 };
3900
3901 guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3902
3903 store.0.set_thread(caller)?;
3905
3906 store
3907 .0
3908 .concurrent_state_mut()?
3909 .set_thread_running(caller.thread)?;
3910
3911 log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3912
3913 if async_caller {
3914 let Some(slot) = storage.first_mut() else {
3915 bail_bug!("no storage for async call status");
3916 };
3917 *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3918 } else {
3919 let state = store.0.concurrent_state_mut()?;
3922 let task = state.get_mut(guest_thread.task)?;
3923 if let Some(Some(result)) = task.sync_result.take()? {
3924 storage[0] = MaybeUninit::new(result);
3925 }
3926 }
3927
3928 if status == Status::Returned {
3929 store
3930 .0
3931 .concurrent_state_mut()?
3932 .decrement_task_ref_count(guest_thread.task)?;
3933 }
3934
3935 Ok(())
3936 }
3937
3938 pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3954 self,
3955 mut store: StoreContextMut<'_, T>,
3956 host_task: EnteredHostTask,
3957 result_may_require_realloc: bool,
3958 future: impl Future<Output = Result<R>> + Send + 'static,
3959 lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3960 + Send
3961 + 'static,
3962 ) -> Result<u32> {
3963 let token = StoreToken::new(store.as_context_mut());
3964
3965 let (join_handle, future) = JoinHandle::run(future);
3968 let mut future = Box::pin(future);
3969
3970 let poll = tls::set(store.0, || {
3975 future
3976 .as_mut()
3977 .poll(&mut Context::from_waker(&Waker::noop()))
3978 });
3979
3980 match poll {
3981 Poll::Ready(result) => {
3983 let result = result.transpose()?;
3984 let task = store.0.current_materialized_host_task()?;
3987 lower(store.as_context_mut(), result, true, task)?;
3988 return Ok(Status::Returned.pack(None));
3989 }
3990
3991 Poll::Pending => {}
3993 }
3994
3995 let Some(task) = store.0.materialize_host_task_id()? else {
3999 bail_bug!("current thread is not a host thread")
4000 };
4001 {
4002 let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
4003 assert!(matches!(state, HostTaskState::CalleeStarted));
4004 *state = HostTaskState::CalleeRunning(join_handle);
4005 }
4006
4007 let future = Box::pin(async move {
4015 let result = match run_with_host_task_set(task, future).await? {
4016 Some(result) => Some(result?),
4017 None => None,
4018 };
4019 let on_complete = move |store: &mut dyn VMStore| {
4020 let mut store = token.as_context_mut(store);
4024 let old = store.0.set_thread(task)?;
4025
4026 let status = if result.is_some() {
4027 Status::Returned
4028 } else {
4029 Status::ReturnCancelled
4030 };
4031
4032 lower(store.as_context_mut(), result, false, Some(task))?;
4033 let state = store.0.concurrent_state_mut()?;
4034 match &mut state.get_mut(task)?.state {
4035 pending @ HostTaskState::CalleeCancelling => {
4038 *pending = HostTaskState::CalleeDone { cancelled: true };
4039 }
4040
4041 other => *other = HostTaskState::CalleeDone { cancelled: false },
4043 }
4044 Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
4045
4046 store.0.set_thread(old)?;
4047 Ok(())
4048 };
4049
4050 tls::get(move |store| {
4051 if result_may_require_realloc {
4052 store
4057 .concurrent_state_mut()?
4058 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
4059 on_complete,
4060 ))));
4061 Ok(())
4062 } else {
4063 on_complete(store)
4066 }
4067 })
4068 });
4069
4070 let caller = match host_task {
4073 Some(caller) => caller,
4074 None => bail_bug!("host task wasn't created but should have been"),
4075 };
4076 let state = store.0.concurrent_state_mut()?;
4077 state.push_future(future);
4078 let instance = state.get_mut(caller.task)?.instance;
4079 let handle = store
4080 .0
4081 .handle_table_insertion(instance)
4082 .subtask_host(task.rep())?;
4083 store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
4084 log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
4085
4086 store.0.set_thread(caller)?;
4090 Ok(Status::Started.pack(Some(handle)))
4091 }
4092
4093 pub(crate) fn task_return(
4096 self,
4097 store: &mut dyn VMStore,
4098 ty: TypeTupleIndex,
4099 options: OptionsIndex,
4100 storage: &[ValRaw],
4101 ) -> Result<()> {
4102 let guest_thread = store.current_guest_thread()?;
4103 let state = store.concurrent_state_mut()?;
4104 if !state.get_mut(guest_thread.task)?.async_lifted {
4105 bail!(Trap::TaskReturnOrCancelSyncLifted);
4106 }
4107 let lift = state
4108 .get_mut(guest_thread.task)?
4109 .lift_result
4110 .take()
4111 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4112 if !state.get_mut(guest_thread.task)?.result.is_none() {
4113 bail_bug!("task result unexpectedly already set");
4114 }
4115
4116 let CanonicalOptions {
4117 string_encoding,
4118 data_model,
4119 ..
4120 } = &self.id().get(store).component().env_component().options[options];
4121
4122 let invalid = ty != lift.ty
4123 || string_encoding != &lift.string_encoding
4124 || match data_model {
4125 CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
4126 Some(memory) => {
4127 let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
4128 let actual = self.id().get(store).runtime_memory(memory);
4129 expected != actual.as_ptr()
4130 }
4131 None => false,
4134 },
4135 CanonicalOptionsDataModel::Gc { .. } => true,
4137 };
4138
4139 if invalid {
4140 bail!(Trap::TaskReturnInvalid);
4141 }
4142
4143 log::trace!("task.return for {guest_thread:?}");
4144
4145 let result = (lift.lift)(store, storage)?;
4146 self.task_complete(store, guest_thread.task, result, Status::Returned)
4147 }
4148
4149 pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4151 let guest_thread = store.current_guest_thread()?;
4152 let state = store.concurrent_state_mut()?;
4153 let task = state.get_mut(guest_thread.task)?;
4154 if !task.async_lifted {
4155 bail!(Trap::TaskReturnOrCancelSyncLifted);
4156 }
4157 if !task.cancel_request_delivered {
4158 bail!(Trap::TaskCancelNotCancelled);
4159 }
4160 _ = task
4161 .lift_result
4162 .take()
4163 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4164
4165 if !task.result.is_none() {
4166 bail_bug!("task result should not bet set yet");
4167 }
4168
4169 log::trace!("task.cancel for {guest_thread:?}");
4170
4171 self.task_complete(
4172 store,
4173 guest_thread.task,
4174 Box::new(DummyResult),
4175 Status::ReturnCancelled,
4176 )
4177 }
4178
4179 fn task_complete(
4185 self,
4186 store: &mut StoreOpaque,
4187 guest_task: TableId<GuestTask>,
4188 result: Box<dyn Any + Send + Sync>,
4189 status: Status,
4190 ) -> Result<()> {
4191 store
4192 .component_resource_tables(Some(self))?
4193 .validate_scope_exit()?;
4194
4195 let state = store.concurrent_state_mut()?;
4196 let task = state.get_mut(guest_task)?;
4197
4198 task.event = None;
4202
4203 if let Caller::Host { tx, .. } = &mut task.caller {
4204 if let Some(tx) = tx.take() {
4205 _ = tx.send(result);
4206 }
4207 } else {
4208 task.result = Some(result);
4209 Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4210 }
4211
4212 Ok(())
4213 }
4214
4215 pub(crate) fn waitable_set_new(
4217 self,
4218 store: &mut StoreOpaque,
4219 caller_instance: RuntimeComponentInstanceIndex,
4220 ) -> Result<u32> {
4221 let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4222 let handle = store
4223 .handle_table_insertion(self.runtime_instance(caller_instance))
4224 .waitable_set(set.rep())?;
4225 log::trace!("new waitable set {set:?} (handle {handle})");
4226 Ok(handle)
4227 }
4228
4229 pub(crate) fn waitable_set_drop(
4231 self,
4232 store: &mut StoreOpaque,
4233 caller_instance: RuntimeComponentInstanceIndex,
4234 set: u32,
4235 ) -> Result<()> {
4236 let rep = store
4237 .instance_state(self.runtime_instance(caller_instance))
4238 .handle_table()
4239 .waitable_set_remove(set)?;
4240
4241 log::trace!("drop waitable set {rep} (handle {set})");
4242
4243 let set = store
4247 .concurrent_state_mut()?
4248 .get_mut(TableId::<WaitableSet>::new(rep))?;
4249 if set.num_waiting > 0 {
4250 bail!(Trap::WaitableSetDropHasWaiters);
4251 }
4252
4253 store
4254 .concurrent_state_mut()?
4255 .delete(TableId::<WaitableSet>::new(rep))?;
4256
4257 Ok(())
4258 }
4259
4260 pub(crate) fn waitable_join(
4262 self,
4263 store: &mut StoreOpaque,
4264 caller_instance: RuntimeComponentInstanceIndex,
4265 waitable_handle: u32,
4266 set_handle: u32,
4267 ) -> Result<()> {
4268 let mut instance = self.id().get_mut(store);
4269 let waitable =
4270 Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4271
4272 let set = if set_handle == 0 {
4273 None
4274 } else {
4275 let set = instance.instance_states().0[caller_instance]
4276 .handle_table()
4277 .waitable_set_rep(set_handle)?;
4278
4279 Some(TableId::<WaitableSet>::new(set))
4280 };
4281
4282 waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4283
4284 log::trace!(
4285 "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4286 );
4287
4288 waitable.join(store.concurrent_state_mut()?, set)
4289 }
4290
4291 pub(crate) fn subtask_drop(
4293 self,
4294 store: &mut StoreOpaque,
4295 caller_instance: RuntimeComponentInstanceIndex,
4296 task_id: u32,
4297 ) -> Result<()> {
4298 self.waitable_join(store, caller_instance, task_id, 0)?;
4299
4300 let (rep, is_host) = store
4301 .instance_state(self.runtime_instance(caller_instance))
4302 .handle_table()
4303 .subtask_remove(task_id)?;
4304
4305 let concurrent_state = store.concurrent_state_mut()?;
4306 let waitable = if is_host {
4307 let id = TableId::<HostTask>::new(rep);
4308 let task = concurrent_state.get_mut(id)?;
4309 match &task.state {
4310 HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4311 bail!(Trap::SubtaskDropNotResolved)
4312 }
4313 HostTaskState::CalleeDone { .. } => {}
4314 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4315 bail_bug!("invalid state for callee in `subtask.drop`")
4316 }
4317 }
4318
4319 Waitable::Host(id)
4320 } else {
4321 let id = TableId::<GuestTask>::new(rep);
4322 let task = concurrent_state.get_mut(id)?;
4323 if task.lift_result.is_some() {
4324 bail!(Trap::SubtaskDropNotResolved);
4325 }
4326 Waitable::Guest(id)
4327 };
4328
4329 waitable.common(concurrent_state)?.handle = None;
4330
4331 if waitable.take_event(concurrent_state)?.is_some() {
4334 bail!(Trap::SubtaskDropNotResolved);
4335 }
4336
4337 waitable.delete_from(concurrent_state)?;
4338
4339 log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4340 Ok(())
4341 }
4342
4343 pub(crate) fn waitable_set_wait(
4345 self,
4346 store: &mut StoreOpaque,
4347 options: OptionsIndex,
4348 set: u32,
4349 payload: u32,
4350 ) -> Result<u32> {
4351 let &CanonicalOptions {
4352 instance: caller_instance,
4353 ..
4354 } = &self.id().get(store).component().env_component().options[options];
4355 let caller = self.runtime_instance(caller_instance);
4356 let rep = store
4357 .instance_state(self.runtime_instance(caller_instance))
4358 .handle_table()
4359 .waitable_set_rep(set)?;
4360
4361 self.waitable_check(
4362 store,
4363 caller,
4364 WaitableCheck::Wait,
4365 WaitableCheckParams {
4366 set: TableId::new(rep),
4367 options,
4368 payload,
4369 },
4370 )
4371 }
4372
4373 pub(crate) fn waitable_set_poll(
4375 self,
4376 store: &mut StoreOpaque,
4377 options: OptionsIndex,
4378 set: u32,
4379 payload: u32,
4380 ) -> Result<u32> {
4381 let &CanonicalOptions {
4382 instance: caller_instance,
4383 ..
4384 } = &self.id().get(store).component().env_component().options[options];
4385 let caller = self.runtime_instance(caller_instance);
4386 let rep = store
4387 .instance_state(caller)
4388 .handle_table()
4389 .waitable_set_rep(set)?;
4390
4391 self.waitable_check(
4392 store,
4393 caller,
4394 WaitableCheck::Poll,
4395 WaitableCheckParams {
4396 set: TableId::new(rep),
4397 options,
4398 payload,
4399 },
4400 )
4401 }
4402
4403 pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4405 let thread_id = store.current_guest_thread()?.thread;
4406 match store
4407 .concurrent_state_mut()?
4408 .get_mut(thread_id)?
4409 .instance_rep
4410 {
4411 Some(r) => Ok(r),
4412 None => bail_bug!("thread should have instance_rep by now"),
4413 }
4414 }
4415
4416 pub(crate) fn thread_new_indirect<T: 'static>(
4418 self,
4419 mut store: StoreContextMut<T>,
4420 runtime_instance: RuntimeComponentInstanceIndex,
4421 start_func_ty: ModuleInternedTypeIndex,
4422 start_func_table_idx: RuntimeTableIndex,
4423 start_func_idx: u32,
4424 context: i32,
4425 ) -> Result<u32> {
4426 log::trace!("creating new thread");
4427
4428 let (instance, registry) = self.id().get_mut_and_registry(store.0);
4429 let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4430 else {
4431 bail_bug!("thread.new-indirect start function type should be registered");
4432 };
4433 let callee = instance
4434 .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4435 .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4436 if callee.type_index(store.0) != start_func_ty {
4437 bail!(Trap::ThreadNewIndirectInvalidType);
4438 }
4439
4440 let token = StoreToken::new(store.as_context_mut());
4441 let start_func = Box::new(
4442 move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4443 let old_thread = store.set_thread(guest_thread)?;
4444 log::trace!(
4445 "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4446 );
4447
4448 let mut store = token.as_context_mut(store);
4449 let mut params = [ValRaw::i32(context)];
4450 unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4453
4454 store.0.set_thread(old_thread)?;
4455
4456 let runtime_instance = self.runtime_instance(runtime_instance);
4457
4458 store
4461 .0
4462 .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4463
4464 store.0.cleanup_thread(guest_thread, runtime_instance)?;
4465
4466 log::trace!("explicit thread {guest_thread:?} completed");
4467 let state = store.0.concurrent_state_mut()?;
4468 if let Some(t) = old_thread.guest() {
4469 state.set_thread_running(t.thread)?;
4470 }
4471 log::trace!("thread start: restored {old_thread:?} as current thread");
4472
4473 Ok(())
4474 },
4475 );
4476
4477 let current_thread = store.0.current_guest_thread()?;
4478 let state = store.0.concurrent_state_mut()?;
4479 let parent_task = current_thread.task;
4480
4481 let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4482 let thread_id = state.push(new_thread)?;
4483 state.get_mut(parent_task)?.threads.insert(thread_id);
4484
4485 log::trace!("new thread with id {thread_id:?} created");
4486
4487 self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4488 }
4489
4490 pub(crate) fn resume_thread(
4491 self,
4492 store: &mut StoreOpaque,
4493 runtime_instance: RuntimeComponentInstanceIndex,
4494 thread_idx: u32,
4495 how: ResumeThread,
4496 ) -> Result<bool> {
4497 let thread_id =
4498 GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4499 let state = store.concurrent_state_mut()?;
4500 let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4501
4502 if store.current_guest_thread()? == guest_thread {
4503 bail!(Trap::CannotResumeThread);
4504 }
4505
4506 let priority = match how {
4507 ResumeThread::Resume => Priority::Switch,
4508 ResumeThread::ResumeLater => Priority::Low,
4509
4510 ResumeThread::Promote => {
4513 let instance = self.runtime_instance(runtime_instance);
4514 let do_not_enter = store
4515 .instance_state(instance)
4516 .concurrent_state()
4517 .do_not_enter;
4518 return store.concurrent_state_mut()?.promote_work_item_matching(
4519 |item: &WorkItem| match item {
4520 WorkItem::ResumeThread { thread, .. }
4521 | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4522
4523 WorkItem::GuestCall {
4524 call: GuestCall { thread, kind },
4525 ..
4526 } => {
4527 *thread == guest_thread
4528 && match kind {
4529 GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4533
4534 GuestCallKind::StartExplicit(_) => true,
4537
4538 GuestCallKind::StartImplicit { .. } => false,
4542 }
4543 }
4544
4545 WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4546 },
4547 );
4548 }
4549 };
4550
4551 let state = store.concurrent_state_mut()?;
4552 let thread = state.get_mut(guest_thread.thread)?;
4553
4554 match &thread.state {
4557 GuestThreadState::NotStartedExplicit(_) => {}
4558 GuestThreadState::Suspended(_) => {
4559 log::trace!("resuming thread {thread_id:?} that was suspended");
4560 state.schedule_suspended_thread(
4561 guest_thread,
4562 self.runtime_instance(runtime_instance),
4563 priority,
4564 )?;
4565 return Ok(true);
4566 }
4567 _ => bail!(Trap::CannotResumeThread),
4568 }
4569 match mem::replace(&mut thread.state, GuestThreadState::Running) {
4570 GuestThreadState::NotStartedExplicit(start_func) => {
4571 log::trace!("starting thread {guest_thread:?}");
4572 let guest_call = WorkItem::GuestCall {
4573 instance: self.runtime_instance(runtime_instance),
4574 call: GuestCall {
4575 thread: guest_thread,
4576 kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4577 start_func(store, guest_thread)
4578 })),
4579 },
4580 };
4581 state.push_work_item(guest_call, priority)?;
4582 }
4583 other => {
4584 thread.state = other;
4585 bail_bug!("thread state checked to be resumable above");
4586 }
4587 }
4588 Ok(true)
4589 }
4590
4591 fn add_guest_thread_to_instance_table(
4592 self,
4593 thread_id: TableId<GuestThread>,
4594 store: &mut StoreOpaque,
4595 runtime_instance: RuntimeComponentInstanceIndex,
4596 ) -> Result<u32> {
4597 let (state, counter) =
4598 store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4599 let guest_id = state
4600 .thread_handle_table()
4601 .guest_thread_insert(counter, thread_id.rep())?;
4602 store
4603 .concurrent_state_mut()?
4604 .get_mut(thread_id)?
4605 .instance_rep = Some(guest_id);
4606 Ok(guest_id)
4607 }
4608
4609 pub(crate) fn suspension_intrinsic(
4613 self,
4614 store: &mut StoreOpaque,
4615 caller: RuntimeComponentInstanceIndex,
4616 yielding: bool,
4617 to_thread: SuspensionTarget,
4618 ) -> Result<WaitResult> {
4619 let check_suspend = match to_thread {
4620 SuspensionTarget::Promote(thread) => {
4621 !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4622 }
4623 SuspensionTarget::Resume(thread) => {
4624 if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4625 bail_bug!(
4626 "`resume_thread` should only ever return false \
4627 when `ResumeThread::Promote` is passed to it"
4628 );
4629 }
4630 false
4631 }
4632 SuspensionTarget::None => true,
4633 };
4634
4635 if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4636 return if yielding {
4637 Ok(WaitResult::Completed)
4638 } else {
4639 Err(Trap::CannotBlockSyncTask.into())
4640 };
4641 }
4642
4643 let guest_thread = store.current_guest_thread()?;
4644
4645 let reason = if yielding {
4646 SuspendReason::Yielding {
4647 thread: guest_thread,
4648 }
4649 } else {
4650 SuspendReason::ExplicitlySuspending {
4651 thread: guest_thread,
4652 }
4653 };
4654
4655 store.suspend(reason)?;
4656
4657 Ok(WaitResult::Completed)
4658 }
4659
4660 fn waitable_check(
4662 self,
4663 store: &mut StoreOpaque,
4664 caller: RuntimeInstance,
4665 check: WaitableCheck,
4666 params: WaitableCheckParams,
4667 ) -> Result<u32> {
4668 let guest_thread = store.current_guest_thread()?;
4669
4670 log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4671
4672 match &check {
4675 WaitableCheck::Wait => {
4676 let set = params.set;
4677
4678 loop {
4683 let state = store.concurrent_state_mut()?;
4684 let task = state.get_mut(guest_thread.task)?;
4685 if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4686 || !state.get_mut(set)?.ready.is_empty()
4687 {
4688 break;
4689 }
4690
4691 store.switch_or_trap_if_may_not_suspend(caller)?;
4692
4693 store.suspend(SuspendReason::Waiting {
4694 set,
4695 thread: guest_thread,
4696 })?;
4697 }
4698 }
4699 WaitableCheck::Poll => {}
4700 }
4701
4702 log::trace!(
4703 "waitable check for {guest_thread:?}; set {:?}, part two",
4704 params.set
4705 );
4706
4707 let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4709
4710 let (ordinal, handle, result) = match &check {
4711 WaitableCheck::Wait => {
4712 let (event, waitable) = match event {
4713 Some(p) => p,
4714 None => bail_bug!("event expected to be present"),
4715 };
4716 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4717 let (ordinal, result) = event.parts();
4718 (ordinal, handle, result)
4719 }
4720 WaitableCheck::Poll => {
4721 if let Some((event, waitable)) = event {
4722 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4723 let (ordinal, result) = event.parts();
4724 (ordinal, handle, result)
4725 } else {
4726 log::trace!(
4727 "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4728 guest_thread.task,
4729 params.set
4730 );
4731 let (ordinal, result) = Event::None.parts();
4732 (ordinal, 0, result)
4733 }
4734 }
4735 };
4736 let ptr = crate::component::func::validate_inbounds_dynamic(
4737 &CanonicalAbiInfo::POINTER_PAIR,
4738 self.options_memory(store, params.options),
4739 &ValRaw::u32(params.payload),
4740 )?;
4741 let memory = self
4742 .options_memory_range_mut(store, params.options, ptr, 8)
4743 .unwrap();
4744 memory[0..4].copy_from_slice(&handle.to_le_bytes());
4745 memory[4..8].copy_from_slice(&result.to_le_bytes());
4746 Ok(ordinal)
4747 }
4748
4749 pub(crate) fn subtask_cancel(
4751 self,
4752 store: &mut StoreOpaque,
4753 caller_instance: RuntimeComponentInstanceIndex,
4754 async_: bool,
4755 task_id: u32,
4756 ) -> Result<u32> {
4757 let (rep, is_host) = store
4758 .instance_state(self.runtime_instance(caller_instance))
4759 .handle_table()
4760 .subtask_rep(task_id)?;
4761 let waitable = if is_host {
4762 Waitable::Host(TableId::<HostTask>::new(rep))
4763 } else {
4764 Waitable::Guest(TableId::<GuestTask>::new(rep))
4765 };
4766 let concurrent_state = store.concurrent_state_mut()?;
4767
4768 log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4769
4770 waitable.trap_if_in_waitable_set(concurrent_state)?;
4771
4772 let needs_block;
4773 if let Waitable::Host(host_task) = waitable {
4774 let state = &mut concurrent_state.get_mut(host_task)?.state;
4775 match state {
4776 HostTaskState::CalleeRunning(handle) => {
4783 handle.abort();
4784 *state = HostTaskState::CalleeCancelling;
4785 needs_block = true;
4786 }
4787
4788 HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4791 bail!(Trap::SubtaskCancelAfterTerminal);
4792 }
4793 HostTaskState::CalleeDone { cancelled: false } => {
4794 *state = HostTaskState::CalleeDone { cancelled: true };
4797 needs_block = false;
4798 }
4799
4800 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4803 bail_bug!("invalid states for host callee")
4804 }
4805 }
4806 } else {
4807 let guest_task = TableId::<GuestTask>::new(rep);
4808 let task = concurrent_state.get_mut(guest_task)?;
4809
4810 if task.cancel_requested {
4813 bail!(Trap::SubtaskCancelAfterTerminal);
4814 }
4815
4816 if !task.already_lowered_parameters() {
4817 store.cancel_guest_subtask_without_lowered_parameters(
4818 self.runtime_instance(caller_instance),
4819 guest_task,
4820 )?;
4821 return Ok(Status::StartCancelled as u32);
4822 } else if !task.returned_or_cancelled() {
4823 task.cancel_requested = true;
4830 task.event = Some(Event::Cancelled);
4831 let runtime_instance = task.instance;
4832 for thread in task.threads.clone() {
4833 let thread = QualifiedThreadId {
4834 task: guest_task,
4835 thread,
4836 };
4837 let concurrent_state = store.concurrent_state_mut()?;
4838 let thread_mut = concurrent_state.get_mut(thread.thread)?;
4839
4840 let yield_ = |store: &mut StoreOpaque| {
4841 let state = store.instance_state(runtime_instance).concurrent_state();
4846 let old_do_not_suspend = state.do_not_suspend;
4847 state.do_not_suspend = false;
4848
4849 let caller = store.current_guest_thread()?;
4850
4851 let state = store.concurrent_state_mut()?;
4856 let set = state.get_mut(caller.thread)?.sync_call_set;
4857 waitable.join(state, Some(set))?;
4858
4859 store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4860
4861 let state = store.concurrent_state_mut()?;
4862 waitable.join(state, None)?;
4863
4864 store
4865 .instance_state(runtime_instance)
4866 .concurrent_state()
4867 .do_not_suspend = old_do_not_suspend;
4868
4869 Ok::<(), crate::Error>(())
4870 };
4871
4872 match thread_mut.wake_on_cancel.take() {
4873 WakeOnCancel::Waiting(set) => {
4874 let instance = concurrent_state.take_callback_waiter(set, thread)?;
4876 let item = WorkItem::GuestCall {
4877 instance: runtime_instance,
4878 call: GuestCall {
4879 thread,
4880 kind: GuestCallKind::DeliverEvent {
4881 instance,
4882 set: Some(set),
4883 },
4884 },
4885 };
4886 concurrent_state.set_switch_item(item)?;
4887
4888 yield_(store)?;
4889
4890 break;
4891 }
4892 WakeOnCancel::Yielding => {
4893 if concurrent_state.promote_thread_work_item(thread)? {
4894 yield_(store)?;
4895 break;
4896 } else if store
4897 .instance_state(runtime_instance)
4898 .concurrent_state()
4899 .pending
4900 .contains_key(&thread)
4901 {
4902 store
4912 .concurrent_state_mut()?
4913 .get_mut(thread.thread)?
4914 .wake_on_cancel = WakeOnCancel::Yielding;
4915 } else {
4916 bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4917 }
4918 }
4919 WakeOnCancel::None => {}
4920 }
4921 }
4922
4923 needs_block = !store
4926 .concurrent_state_mut()?
4927 .get_mut(guest_task)?
4928 .returned_or_cancelled()
4929 } else {
4930 needs_block = false;
4931 }
4932 };
4933
4934 if needs_block {
4938 if async_ {
4939 return Ok(BLOCKED);
4940 }
4941
4942 let old_next_switch_item = if is_host {
4952 Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4953 } else {
4954 None
4955 };
4956
4957 store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4960
4961 if let Some(item) = old_next_switch_item {
4962 let state = store.concurrent_state_mut()?;
4963 state.restore_next_switch_item(item)?;
4964 }
4965
4966 }
4968
4969 let event = waitable.take_event(store.concurrent_state_mut()?)?;
4970 if let Some(Event::Subtask {
4971 status: status @ (Status::Returned | Status::ReturnCancelled),
4972 }) = event
4973 {
4974 Ok(status as u32)
4975 } else {
4976 bail!(Trap::SubtaskCancelAfterTerminal);
4977 }
4978 }
4979}
4980
4981pub trait VMComponentAsyncStore {
4989 unsafe fn prepare_call(
4995 &mut self,
4996 instance: Instance,
4997 memory: *mut VMMemoryDefinition,
4998 start: NonNull<VMFuncRef>,
4999 return_: NonNull<VMFuncRef>,
5000 caller_instance: RuntimeComponentInstanceIndex,
5001 callee_instance: RuntimeComponentInstanceIndex,
5002 task_return_type: TypeTupleIndex,
5003 callee_async: bool,
5004 string_encoding: StringEncoding,
5005 result_count: u32,
5006 storage: *mut ValRaw,
5007 storage_len: usize,
5008 ) -> Result<()>;
5009
5010 unsafe fn start_call(
5013 &mut self,
5014 instance: Instance,
5015 callback: *mut VMFuncRef,
5016 post_return: *mut VMFuncRef,
5017 callee: NonNull<VMFuncRef>,
5018 param_count: u32,
5019 result_count: u32,
5020 flags: u32,
5021 storage: *mut MaybeUninit<ValRaw>,
5022 storage_len: usize,
5023 ) -> Result<()>;
5024
5025 fn future_write(
5027 &mut self,
5028 instance: Instance,
5029 caller: RuntimeComponentInstanceIndex,
5030 ty: TypeFutureTableIndex,
5031 options: OptionsIndex,
5032 future: u32,
5033 address: u32,
5034 ) -> Result<u32>;
5035
5036 fn future_read(
5038 &mut self,
5039 instance: Instance,
5040 caller: RuntimeComponentInstanceIndex,
5041 ty: TypeFutureTableIndex,
5042 options: OptionsIndex,
5043 future: u32,
5044 address: u32,
5045 ) -> Result<u32>;
5046
5047 fn future_drop_writable(
5049 &mut self,
5050 instance: Instance,
5051 ty: TypeFutureTableIndex,
5052 writer: u32,
5053 ) -> Result<()>;
5054
5055 fn stream_write(
5057 &mut self,
5058 instance: Instance,
5059 caller: RuntimeComponentInstanceIndex,
5060 ty: TypeStreamTableIndex,
5061 options: OptionsIndex,
5062 stream: u32,
5063 address: u32,
5064 count: u32,
5065 ) -> Result<u32>;
5066
5067 fn stream_read(
5069 &mut self,
5070 instance: Instance,
5071 caller: RuntimeComponentInstanceIndex,
5072 ty: TypeStreamTableIndex,
5073 options: OptionsIndex,
5074 stream: u32,
5075 address: u32,
5076 count: u32,
5077 ) -> Result<u32>;
5078
5079 fn flat_stream_write(
5082 &mut self,
5083 instance: Instance,
5084 caller: RuntimeComponentInstanceIndex,
5085 ty: TypeStreamTableIndex,
5086 options: OptionsIndex,
5087 payload_size: u32,
5088 payload_align: u32,
5089 stream: u32,
5090 address: u32,
5091 count: u32,
5092 ) -> Result<u32>;
5093
5094 fn flat_stream_read(
5097 &mut self,
5098 instance: Instance,
5099 caller: RuntimeComponentInstanceIndex,
5100 ty: TypeStreamTableIndex,
5101 options: OptionsIndex,
5102 payload_size: u32,
5103 payload_align: u32,
5104 stream: u32,
5105 address: u32,
5106 count: u32,
5107 ) -> Result<u32>;
5108
5109 fn stream_drop_writable(
5111 &mut self,
5112 instance: Instance,
5113 ty: TypeStreamTableIndex,
5114 writer: u32,
5115 ) -> Result<()>;
5116
5117 fn error_context_debug_message(
5119 &mut self,
5120 instance: Instance,
5121 ty: TypeComponentLocalErrorContextTableIndex,
5122 options: OptionsIndex,
5123 err_ctx_handle: u32,
5124 debug_msg_address: u32,
5125 ) -> Result<()>;
5126
5127 fn thread_new_indirect(
5129 &mut self,
5130 instance: Instance,
5131 caller: RuntimeComponentInstanceIndex,
5132 func_ty_idx: ModuleInternedTypeIndex,
5133 start_func_table_idx: RuntimeTableIndex,
5134 start_func_idx: u32,
5135 context: i32,
5136 ) -> Result<u32>;
5137}
5138
5139impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5141 unsafe fn prepare_call(
5142 &mut self,
5143 instance: Instance,
5144 memory: *mut VMMemoryDefinition,
5145 start: NonNull<VMFuncRef>,
5146 return_: NonNull<VMFuncRef>,
5147 caller_instance: RuntimeComponentInstanceIndex,
5148 callee_instance: RuntimeComponentInstanceIndex,
5149 task_return_type: TypeTupleIndex,
5150 callee_async: bool,
5151 string_encoding: StringEncoding,
5152 result_count_or_max_if_async: u32,
5153 storage: *mut ValRaw,
5154 storage_len: usize,
5155 ) -> Result<()> {
5156 let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5160
5161 unsafe {
5162 instance.prepare_call(
5163 StoreContextMut(self),
5164 start,
5165 return_,
5166 caller_instance,
5167 callee_instance,
5168 task_return_type,
5169 callee_async,
5170 memory,
5171 string_encoding,
5172 match result_count_or_max_if_async {
5173 PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5174 params,
5175 has_result: false,
5176 },
5177 PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5178 params,
5179 has_result: true,
5180 },
5181 result_count => CallerInfo::Sync {
5182 params,
5183 result_count,
5184 },
5185 },
5186 )
5187 }
5188 }
5189
5190 unsafe fn start_call(
5191 &mut self,
5192 instance: Instance,
5193 callback: *mut VMFuncRef,
5194 post_return: *mut VMFuncRef,
5195 callee: NonNull<VMFuncRef>,
5196 param_count: u32,
5197 result_count: u32,
5198 flags: u32,
5199 storage: *mut MaybeUninit<ValRaw>,
5200 storage_len: usize,
5201 ) -> Result<()> {
5202 unsafe {
5203 instance.start_call(
5204 StoreContextMut(self),
5205 callback,
5206 post_return,
5207 callee,
5208 param_count,
5209 result_count,
5210 flags,
5211 core::slice::from_raw_parts_mut(storage, storage_len),
5215 )
5216 }
5217 }
5218
5219 fn future_write(
5220 &mut self,
5221 instance: Instance,
5222 caller: RuntimeComponentInstanceIndex,
5223 ty: TypeFutureTableIndex,
5224 options: OptionsIndex,
5225 future: u32,
5226 address: u32,
5227 ) -> Result<u32> {
5228 instance
5229 .guest_write(
5230 StoreContextMut(self),
5231 caller,
5232 TransmitIndex::Future(ty),
5233 options,
5234 None,
5235 future,
5236 address,
5237 1,
5238 )
5239 .map(|result| result.encode())
5240 }
5241
5242 fn future_read(
5243 &mut self,
5244 instance: Instance,
5245 caller: RuntimeComponentInstanceIndex,
5246 ty: TypeFutureTableIndex,
5247 options: OptionsIndex,
5248 future: u32,
5249 address: u32,
5250 ) -> Result<u32> {
5251 instance
5252 .guest_read(
5253 StoreContextMut(self),
5254 caller,
5255 TransmitIndex::Future(ty),
5256 options,
5257 None,
5258 future,
5259 address,
5260 1,
5261 )
5262 .map(|result| result.encode())
5263 }
5264
5265 fn stream_write(
5266 &mut self,
5267 instance: Instance,
5268 caller: RuntimeComponentInstanceIndex,
5269 ty: TypeStreamTableIndex,
5270 options: OptionsIndex,
5271 stream: u32,
5272 address: u32,
5273 count: u32,
5274 ) -> Result<u32> {
5275 instance
5276 .guest_write(
5277 StoreContextMut(self),
5278 caller,
5279 TransmitIndex::Stream(ty),
5280 options,
5281 None,
5282 stream,
5283 address,
5284 count,
5285 )
5286 .map(|result| result.encode())
5287 }
5288
5289 fn stream_read(
5290 &mut self,
5291 instance: Instance,
5292 caller: RuntimeComponentInstanceIndex,
5293 ty: TypeStreamTableIndex,
5294 options: OptionsIndex,
5295 stream: u32,
5296 address: u32,
5297 count: u32,
5298 ) -> Result<u32> {
5299 instance
5300 .guest_read(
5301 StoreContextMut(self),
5302 caller,
5303 TransmitIndex::Stream(ty),
5304 options,
5305 None,
5306 stream,
5307 address,
5308 count,
5309 )
5310 .map(|result| result.encode())
5311 }
5312
5313 fn future_drop_writable(
5314 &mut self,
5315 instance: Instance,
5316 ty: TypeFutureTableIndex,
5317 writer: u32,
5318 ) -> Result<()> {
5319 instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5320 }
5321
5322 fn flat_stream_write(
5323 &mut self,
5324 instance: Instance,
5325 caller: RuntimeComponentInstanceIndex,
5326 ty: TypeStreamTableIndex,
5327 options: OptionsIndex,
5328 payload_size: u32,
5329 payload_align: u32,
5330 stream: u32,
5331 address: u32,
5332 count: u32,
5333 ) -> Result<u32> {
5334 instance
5335 .guest_write(
5336 StoreContextMut(self),
5337 caller,
5338 TransmitIndex::Stream(ty),
5339 options,
5340 Some(FlatAbi {
5341 size: payload_size,
5342 align: payload_align,
5343 }),
5344 stream,
5345 address,
5346 count,
5347 )
5348 .map(|result| result.encode())
5349 }
5350
5351 fn flat_stream_read(
5352 &mut self,
5353 instance: Instance,
5354 caller: RuntimeComponentInstanceIndex,
5355 ty: TypeStreamTableIndex,
5356 options: OptionsIndex,
5357 payload_size: u32,
5358 payload_align: u32,
5359 stream: u32,
5360 address: u32,
5361 count: u32,
5362 ) -> Result<u32> {
5363 instance
5364 .guest_read(
5365 StoreContextMut(self),
5366 caller,
5367 TransmitIndex::Stream(ty),
5368 options,
5369 Some(FlatAbi {
5370 size: payload_size,
5371 align: payload_align,
5372 }),
5373 stream,
5374 address,
5375 count,
5376 )
5377 .map(|result| result.encode())
5378 }
5379
5380 fn stream_drop_writable(
5381 &mut self,
5382 instance: Instance,
5383 ty: TypeStreamTableIndex,
5384 writer: u32,
5385 ) -> Result<()> {
5386 instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5387 }
5388
5389 fn error_context_debug_message(
5390 &mut self,
5391 instance: Instance,
5392 ty: TypeComponentLocalErrorContextTableIndex,
5393 options: OptionsIndex,
5394 err_ctx_handle: u32,
5395 debug_msg_address: u32,
5396 ) -> Result<()> {
5397 instance.error_context_debug_message(
5398 StoreContextMut(self),
5399 ty,
5400 options,
5401 err_ctx_handle,
5402 debug_msg_address,
5403 )
5404 }
5405
5406 fn thread_new_indirect(
5407 &mut self,
5408 instance: Instance,
5409 caller: RuntimeComponentInstanceIndex,
5410 func_ty_idx: ModuleInternedTypeIndex,
5411 start_func_table_idx: RuntimeTableIndex,
5412 start_func_idx: u32,
5413 context: i32,
5414 ) -> Result<u32> {
5415 instance.thread_new_indirect(
5416 StoreContextMut(self),
5417 caller,
5418 func_ty_idx,
5419 start_func_table_idx,
5420 start_func_idx,
5421 context,
5422 )
5423 }
5424}
5425
5426type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5427
5428async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5431where
5432 F: Future,
5433{
5434 let mut future = pin!(future);
5435 future::poll_fn(|cx| {
5436 let old_thread = match tls::get(|store| store.set_thread(task)) {
5437 Ok(thread) => thread,
5438 Err(error) => return Poll::Ready(Err(error)),
5439 };
5440 let result = future.as_mut().poll(cx);
5441 match tls::get(|store| store.set_thread(old_thread)) {
5442 Ok(_) => result.map(Ok),
5443 Err(error) => Poll::Ready(Err(error)),
5444 }
5445 })
5446 .await
5447}
5448
5449pub(crate) struct HostTask {
5453 common: WaitableCommon,
5454
5455 call_context: CallContext,
5458
5459 state: HostTaskState,
5460
5461 group: TaskGroupId,
5462}
5463
5464enum HostTaskState {
5465 CalleeStarted,
5470
5471 CalleeRunning(JoinHandle),
5476
5477 CalleeCancelling,
5481
5482 CalleeFinished(LiftedResult),
5486
5487 CalleeDone { cancelled: bool },
5490}
5491
5492impl HostTask {
5493 fn new(
5494 concurrent_state: &mut ConcurrentState,
5495 state: HostTaskState,
5496 caller: QualifiedThreadId,
5497 ) -> Result<Self> {
5498 let group = concurrent_state.get_mut(caller.task)?.group;
5499 concurrent_state.increment_group_ref_count(group)?;
5500
5501 Ok(Self {
5502 common: WaitableCommon::default(),
5503 call_context: CallContext::default(),
5504 state,
5505 group,
5506 })
5507 }
5508}
5509
5510impl TableDebug for HostTask {
5511 fn type_name() -> &'static str {
5512 "HostTask"
5513 }
5514}
5515
5516type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5517
5518enum Caller {
5520 Host {
5522 tx: Option<oneshot::Sender<LiftedResult>>,
5524 caller: Option<TableId<HostTask>>,
5528 },
5529 Guest {
5531 thread: QualifiedThreadId,
5533 },
5534}
5535
5536struct LiftResult {
5539 lift: RawLift,
5540 ty: TypeTupleIndex,
5541 memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5542 string_encoding: StringEncoding,
5543}
5544
5545#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5550pub(crate) struct QualifiedThreadId {
5551 task: TableId<GuestTask>,
5552 thread: TableId<GuestThread>,
5553}
5554
5555impl QualifiedThreadId {
5556 fn qualify(
5557 state: &mut ConcurrentState,
5558 thread: TableId<GuestThread>,
5559 ) -> Result<QualifiedThreadId> {
5560 Ok(QualifiedThreadId {
5561 task: state.get_mut(thread)?.parent_task,
5562 thread,
5563 })
5564 }
5565}
5566
5567impl fmt::Debug for QualifiedThreadId {
5568 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5569 f.debug_tuple("QualifiedThreadId")
5570 .field(&self.task.rep())
5571 .field(&self.thread.rep())
5572 .finish()
5573 }
5574}
5575
5576enum GuestThreadState {
5577 NotStartedImplicit,
5578 NotStartedExplicit(
5579 Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5580 ),
5581 Running,
5582 Suspended(StoreFiber<'static>),
5583 Ready {
5584 fiber: StoreFiber<'static>,
5585 },
5586 Completed,
5587}
5588
5589impl GuestThreadState {
5590 fn check_no_fiber(&self) -> Result<()> {
5591 match self {
5592 Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5593 Self::NotStartedImplicit
5594 | Self::NotStartedExplicit(_)
5595 | Self::Running
5596 | Self::Completed => Ok(()),
5597 }
5598 }
5599}
5600
5601impl fmt::Debug for GuestThreadState {
5602 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5603 match self {
5604 Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5605 Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5606 Self::Running => f.debug_tuple("Running").finish(),
5607 Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5608 Self::Ready { .. } => f.debug_struct("Ready").finish(),
5609 Self::Completed => f.debug_tuple("Completed").finish(),
5610 }
5611 }
5612}
5613
5614#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5615enum WakeOnCancel {
5616 None,
5617 Waiting(TableId<WaitableSet>),
5618 Yielding,
5619}
5620
5621impl WakeOnCancel {
5622 fn is_none(self) -> bool {
5623 matches!(self, WakeOnCancel::None)
5624 }
5625
5626 fn replace(&mut self, other: WakeOnCancel) -> Self {
5627 let old = *self;
5628 *self = other;
5629 old
5630 }
5631
5632 fn take(&mut self) -> Self {
5633 self.replace(WakeOnCancel::None)
5634 }
5635}
5636
5637pub struct GuestThread {
5638 context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5641 parent_task: TableId<GuestTask>,
5643 wake_on_cancel: WakeOnCancel,
5646 state: GuestThreadState,
5648 instance_rep: Option<u32>,
5651 sync_call_set: TableId<WaitableSet>,
5653 old_do_not_suspend: Option<bool>,
5656}
5657
5658impl GuestThread {
5659 fn from_instance(
5662 state: Pin<&mut ComponentInstance>,
5663 caller_instance: RuntimeComponentInstanceIndex,
5664 guest_thread: u32,
5665 ) -> Result<TableId<Self>> {
5666 let rep = state.instance_states().0[caller_instance]
5667 .thread_handle_table()
5668 .guest_thread_rep(guest_thread)?;
5669 Ok(TableId::new(rep))
5670 }
5671
5672 fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5673 let sync_call_set = state.push(WaitableSet {
5674 is_sync_call_set: true,
5675 ..WaitableSet::default()
5676 })?;
5677 Ok(Self {
5678 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5679 parent_task,
5680 wake_on_cancel: WakeOnCancel::None,
5681 state: GuestThreadState::NotStartedImplicit,
5682 instance_rep: None,
5683 sync_call_set,
5684 old_do_not_suspend: None,
5685 })
5686 }
5687
5688 fn new_explicit(
5689 state: &mut ConcurrentState,
5690 parent_task: TableId<GuestTask>,
5691 start_func: Box<
5692 dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5693 >,
5694 ) -> Result<Self> {
5695 state.increment_task_ref_count(parent_task)?;
5696 let sync_call_set = state.push(WaitableSet {
5697 is_sync_call_set: true,
5698 ..WaitableSet::default()
5699 })?;
5700 Ok(Self {
5701 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5702 parent_task,
5703 wake_on_cancel: WakeOnCancel::None,
5704 state: GuestThreadState::NotStartedExplicit(start_func),
5705 instance_rep: None,
5706 sync_call_set,
5707 old_do_not_suspend: None,
5708 })
5709 }
5710}
5711
5712impl TableDebug for GuestThread {
5713 fn type_name() -> &'static str {
5714 "GuestThread"
5715 }
5716}
5717
5718enum SyncResult {
5719 NotProduced,
5720 Produced(Option<ValRaw>),
5721 Taken,
5722}
5723
5724impl SyncResult {
5725 fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5726 Ok(match mem::replace(self, SyncResult::Taken) {
5727 SyncResult::NotProduced => None,
5728 SyncResult::Produced(val) => Some(val),
5729 SyncResult::Taken => {
5730 bail_bug!("attempted to take a synchronous result that was already taken")
5731 }
5732 })
5733 }
5734}
5735
5736pub(crate) struct GuestTask {
5738 common: WaitableCommon,
5740 lower_params: Option<RawLower>,
5742 lift_result: Option<LiftResult>,
5744 result: Option<LiftedResult>,
5747 callback: Option<CallbackFn>,
5750 caller: Caller,
5752 call_context: CallContext,
5757 sync_result: SyncResult,
5760 cancel_request_delivered: bool,
5764 cancel_requested: bool,
5769 starting_sent: bool,
5772 instance: RuntimeInstance,
5779 event: Option<Event>,
5781 exited: bool,
5783 threads: HashSet<TableId<GuestThread>>,
5785 async_typed: bool,
5788 async_lifted: bool,
5791 decremented_interesting_task_count: bool,
5794 group: TaskGroupId,
5796 ref_count: usize,
5801}
5802
5803impl GuestTask {
5804 fn already_lowered_parameters(&self) -> bool {
5805 self.lower_params.is_none()
5807 }
5808
5809 fn returned_or_cancelled(&self) -> bool {
5810 self.lift_result.is_none()
5812 }
5813
5814 fn new(
5815 state: &mut ConcurrentState,
5816 lower_params: RawLower,
5817 lift_result: LiftResult,
5818 caller: Caller,
5819 callback: Option<CallbackFn>,
5820 instance: RuntimeInstance,
5821 async_typed: bool,
5822 async_lifted: bool,
5823 ) -> Result<QualifiedThreadId> {
5824 let group = match caller {
5825 Caller::Guest { thread } => {
5826 let group = state.get_mut(thread.task)?.group;
5827 state.increment_group_ref_count(group)?;
5828 group
5829 }
5830 Caller::Host { .. } => state.make_task_group()?,
5831 };
5832
5833 let task = state.push(Self {
5834 common: WaitableCommon::default(),
5835 lower_params: Some(lower_params),
5836 lift_result: Some(lift_result),
5837 result: None,
5838 callback,
5839 caller,
5840 call_context: CallContext::default(),
5841 sync_result: SyncResult::NotProduced,
5842 cancel_request_delivered: false,
5843 cancel_requested: false,
5844 starting_sent: false,
5845 instance,
5846 event: None,
5847 exited: false,
5848 threads: HashSet::new(),
5849 async_typed,
5850 async_lifted,
5851 decremented_interesting_task_count: false,
5852 group,
5853 ref_count: 2,
5856 })?;
5857 let new_thread = GuestThread::new_implicit(state, task)?;
5858 let thread = state.push(new_thread)?;
5859 state.get_mut(task)?.threads.insert(thread);
5860 state.interesting_tasks += 1;
5861 let thread = QualifiedThreadId { task, thread };
5862 log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5863 Ok(thread)
5864 }
5865}
5866
5867impl TableDebug for GuestTask {
5868 fn type_name() -> &'static str {
5869 "GuestTask"
5870 }
5871}
5872
5873#[derive(Default)]
5875struct WaitableCommon {
5876 event: Option<Event>,
5878 set: Option<TableId<WaitableSet>>,
5880 handle: Option<u32>,
5882}
5883
5884#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5886enum Waitable {
5887 Host(TableId<HostTask>),
5889 Guest(TableId<GuestTask>),
5891 Transmit(TableId<TransmitHandle>),
5893}
5894
5895impl Waitable {
5896 fn from_instance(
5899 state: Pin<&mut ComponentInstance>,
5900 caller_instance: RuntimeComponentInstanceIndex,
5901 waitable: u32,
5902 ) -> Result<Self> {
5903 use crate::runtime::vm::component::Waitable;
5904
5905 let (waitable, kind) = state.instance_states().0[caller_instance]
5906 .handle_table()
5907 .waitable_rep(waitable)?;
5908
5909 Ok(match kind {
5910 Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5911 Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5912 Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5913 })
5914 }
5915
5916 fn rep(&self) -> u32 {
5918 match self {
5919 Self::Host(id) => id.rep(),
5920 Self::Guest(id) => id.rep(),
5921 Self::Transmit(id) => id.rep(),
5922 }
5923 }
5924
5925 fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5929 log::trace!("waitable {self:?} join set {set:?}");
5930
5931 let old = mem::replace(&mut self.common(state)?.set, set);
5932
5933 if let Some(old) = old {
5934 match *self {
5935 Waitable::Host(id) => state.remove_child(id, old),
5936 Waitable::Guest(id) => state.remove_child(id, old),
5937 Waitable::Transmit(id) => state.remove_child(id, old),
5938 }?;
5939
5940 state.get_mut(old)?.ready.remove(self);
5941 }
5942
5943 if let Some(set) = set {
5944 match *self {
5945 Waitable::Host(id) => state.add_child(id, set),
5946 Waitable::Guest(id) => state.add_child(id, set),
5947 Waitable::Transmit(id) => state.add_child(id, set),
5948 }?;
5949
5950 if self.common(state)?.event.is_some() {
5951 self.mark_ready(state)?;
5952 }
5953 }
5954
5955 Ok(())
5956 }
5957
5958 fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5960 Ok(match self {
5961 Self::Host(id) => &mut state.get_mut(*id)?.common,
5962 Self::Guest(id) => &mut state.get_mut(*id)?.common,
5963 Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5964 })
5965 }
5966
5967 fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5973 if self.common(state)?.set.is_some() {
5974 bail!(Trap::WaitableSyncAndAsync);
5975 }
5976 Ok(())
5977 }
5978
5979 fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5985 if let Some(set) = self.common(state)?.set
5986 && state.get_mut(set)?.is_sync_call_set
5987 {
5988 bail!(Trap::WaitableSyncAndAsync);
5989 }
5990 Ok(())
5991 }
5992
5993 fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
5997 log::trace!("set event for {self:?}: {event:?}");
5998 self.common(state)?.event = event;
5999 self.mark_ready(state)
6000 }
6001
6002 fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
6004 let common = self.common(state)?;
6005 let event = common.event.take();
6006 if let Some(set) = self.common(state)?.set {
6007 state.get_mut(set)?.ready.remove(self);
6008 }
6009
6010 Ok(event)
6011 }
6012
6013 fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
6017 if let Some(set) = self.common(state)?.set {
6018 state.get_mut(set)?.ready.insert(*self);
6019 state.wake_waiter(set)?;
6020 }
6021 Ok(())
6022 }
6023
6024 fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
6027 match self {
6028 Self::Host(task) => state.delete_host_task(*task),
6029 Self::Guest(task) => state.decrement_task_ref_count(*task),
6030 Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
6031 }
6032 }
6033}
6034
6035impl fmt::Debug for Waitable {
6036 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6037 match self {
6038 Self::Host(id) => write!(f, "{id:?}"),
6039 Self::Guest(id) => write!(f, "{id:?}"),
6040 Self::Transmit(id) => write!(f, "{id:?}"),
6041 }
6042 }
6043}
6044
6045#[derive(Default)]
6047struct WaitableSet {
6048 ready: BTreeSet<Waitable>,
6050 waiting: BTreeMap<QualifiedThreadId, WaitMode>,
6052 num_waiting: usize,
6055 is_sync_call_set: bool,
6058}
6059
6060impl WaitableSet {
6061 fn stop_waiting(&mut self) -> Result<()> {
6063 self.num_waiting = match self.num_waiting.checked_sub(1) {
6064 Some(n) => n,
6065 None => bail_bug!("waiter not accounted for in waitable set"),
6066 };
6067 Ok(())
6068 }
6069}
6070
6071impl TableDebug for WaitableSet {
6072 fn type_name() -> &'static str {
6073 "WaitableSet"
6074 }
6075}
6076
6077type RawLower =
6079 Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
6080
6081type RawLift = Box<
6083 dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6084>;
6085
6086type LiftedResult = Box<dyn Any + Send + Sync>;
6090
6091struct DummyResult;
6094
6095#[derive(Default)]
6097pub struct ConcurrentInstanceState {
6098 backpressure: u16,
6100 do_not_enter: bool,
6102 do_not_suspend: bool,
6105 pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
6108 pending_start_count: usize,
6111}
6112
6113impl ConcurrentInstanceState {
6114 pub fn pending_is_empty(&self) -> bool {
6115 self.pending.is_empty()
6116 }
6117}
6118
6119#[derive(Debug, Copy, Clone)]
6120pub(crate) enum CurrentThread {
6121 Guest(QualifiedThreadId),
6124 Host(TableId<HostTask>),
6126 DeferredHost(QualifiedThreadId),
6129 None,
6132}
6133
6134impl CurrentThread {
6135 fn guest(&self) -> Option<&QualifiedThreadId> {
6136 match self {
6137 Self::Guest(id) => Some(id),
6138 _ => None,
6139 }
6140 }
6141
6142 fn guest_task(&self) -> Option<TableId<GuestTask>> {
6143 match self {
6144 Self::Guest(id) => Some(id.task),
6145 _ => None,
6146 }
6147 }
6148
6149 fn is_none(&self) -> bool {
6150 matches!(self, Self::None)
6151 }
6152}
6153
6154impl From<QualifiedThreadId> for CurrentThread {
6155 fn from(id: QualifiedThreadId) -> Self {
6156 Self::Guest(id)
6157 }
6158}
6159
6160impl From<TableId<HostTask>> for CurrentThread {
6161 fn from(id: TableId<HostTask>) -> Self {
6162 Self::Host(id)
6163 }
6164}
6165
6166enum Priority {
6167 Switch,
6168 #[expect(
6169 dead_code,
6170 reason = "Here for completeness and in case we need it later"
6171 )]
6172 High,
6173 Low,
6174}
6175
6176pub struct ConcurrentState {
6178 unforced_current_thread: CurrentThread,
6184
6185 deferred_host_call_context: Option<CallContext>,
6191
6192 futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6197 table: AlwaysMut<ResourceTable>,
6199 switch_item: Option<WorkItem>,
6207 next_switch_item: Option<WorkItem>,
6213 high_priority: VecDeque<WorkItem>,
6215 low_priority: VecDeque<WorkItem>,
6217 suspend_reason: Option<SuspendReason>,
6221 worker: Option<StoreFiber<'static>>,
6225 worker_item: Option<WorkerItem>,
6227
6228 global_error_context_ref_counts:
6241 BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6242
6243 interesting_tasks: usize,
6256
6257 interesting_tasks_empty_waker: Option<Waker>,
6261
6262 ready_for_concurrent_call_waker: Option<Waker>,
6267
6268 event_loop_running: bool,
6270
6271 saved_next_switch_items: Vec<Option<WorkItem>>,
6274
6275 #[cfg(feature = "task-group-hook")]
6277 task_group_hook: Option<Box<dyn TaskGroupHook>>,
6278}
6279
6280impl Default for ConcurrentState {
6281 fn default() -> Self {
6282 Self {
6283 unforced_current_thread: CurrentThread::None,
6284 deferred_host_call_context: None,
6285 table: AlwaysMut::new(ResourceTable::new()),
6286 futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6287 switch_item: None,
6288 next_switch_item: None,
6289 high_priority: VecDeque::new(),
6290 low_priority: VecDeque::new(),
6291 suspend_reason: None,
6292 worker: None,
6293 worker_item: None,
6294 global_error_context_ref_counts: BTreeMap::new(),
6295 interesting_tasks: 0,
6296 interesting_tasks_empty_waker: None,
6297 ready_for_concurrent_call_waker: None,
6298 event_loop_running: false,
6299 saved_next_switch_items: Vec::new(),
6300 #[cfg(feature = "task-group-hook")]
6301 task_group_hook: None,
6302 }
6303 }
6304}
6305
6306impl ConcurrentState {
6307 pub(crate) fn take_fibers_and_futures(
6324 &mut self,
6325 fibers: &mut Vec<StoreFiber<'static>>,
6326 futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6327 ) {
6328 let ConcurrentState {
6329 table,
6330 worker,
6331 switch_item,
6332 next_switch_item,
6333 high_priority,
6334 low_priority,
6335 saved_next_switch_items,
6336 futures: pending_futures,
6337 worker_item: _,
6338 unforced_current_thread: _,
6339 deferred_host_call_context: _,
6340 suspend_reason: _,
6341 global_error_context_ref_counts: _,
6342 interesting_tasks: _,
6343 interesting_tasks_empty_waker: _,
6344 ready_for_concurrent_call_waker: _,
6345 event_loop_running: _,
6346 #[cfg(feature = "task-group-hook")]
6347 task_group_hook: _,
6348 } = self;
6349
6350 let mut items = Vec::new();
6351 for (_, entry) in table.get_mut().iter_mut() {
6352 let entry = entry.into_entry_mut();
6353 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6354 for mode in mem::take(&mut set.waiting).into_values() {
6355 match mode {
6356 WaitMode::Fiber(fiber) => {
6357 fibers.push(fiber);
6358 }
6359 WaitMode::Callback(_) => {}
6360 }
6361 }
6362 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6363 match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6364 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6365 fibers.push(fiber);
6366 }
6367 GuestThreadState::NotStartedImplicit
6368 | GuestThreadState::NotStartedExplicit(_)
6369 | GuestThreadState::Running
6370 | GuestThreadState::Completed => {}
6371 }
6372 } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6373 if let Some(item) = item.take() {
6374 items.push(item);
6375 }
6376 }
6377 }
6378
6379 if let Some(fiber) = worker.take() {
6380 fibers.push(fiber);
6381 }
6382
6383 let mut handle_item = |item| match item {
6384 WorkItem::ResumeFiber { fiber, .. } => {
6385 fibers.push(fiber);
6386 }
6387 WorkItem::PushFuture(future) => {
6388 pending_futures
6389 .get_mut()
6390 .as_mut()
6391 .unwrap()
6392 .push(future.into_inner());
6393 }
6394 WorkItem::ResumeThread { .. }
6395 | WorkItem::GuestCall { .. }
6396 | WorkItem::WorkerFunction(_) => {}
6397 };
6398
6399 for item in items {
6400 handle_item(item);
6401 }
6402 if let Some(item) = switch_item.take() {
6403 handle_item(item);
6404 }
6405 if let Some(item) = next_switch_item.take() {
6406 handle_item(item);
6407 }
6408 for item in mem::take(high_priority) {
6409 handle_item(item);
6410 }
6411 for item in mem::take(low_priority) {
6412 handle_item(item);
6413 }
6414 for item in mem::take(saved_next_switch_items)
6415 .into_iter()
6416 .filter_map(|v| v)
6417 {
6418 handle_item(item);
6419 }
6420
6421 if let Some(them) = pending_futures.get_mut().take() {
6422 futures.push(them);
6423 }
6424 }
6425
6426 #[cfg(feature = "gc")]
6427 pub(crate) fn trace_fiber_roots(
6428 &mut self,
6429 modules: &ModuleRegistry,
6430 unwind: &dyn Unwind,
6431 gc_roots_list: &mut GcRootsList,
6432 ) {
6433 let ConcurrentState {
6434 table,
6435 worker,
6436 switch_item,
6437 next_switch_item,
6438 high_priority,
6439 low_priority,
6440 saved_next_switch_items,
6441
6442 futures: _,
6446
6447 worker_item: _,
6449 unforced_current_thread: _,
6450 deferred_host_call_context: _,
6451 suspend_reason: _,
6452 global_error_context_ref_counts: _,
6453 interesting_tasks: _,
6454 interesting_tasks_empty_waker: _,
6455 ready_for_concurrent_call_waker: _,
6456 event_loop_running: _,
6457 #[cfg(feature = "task-group-hook")]
6458 task_group_hook: _,
6459 } = self;
6460
6461 for (_, entry) in table.get_mut().iter_mut() {
6462 let entry = entry.into_entry_mut();
6463 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6464 for mode in set.waiting.values_mut() {
6465 match mode {
6466 WaitMode::Fiber(fiber) => {
6467 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6468 }
6469 WaitMode::Callback(_) => {}
6470 }
6471 }
6472 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6473 match &mut thread.state {
6474 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6475 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6476 }
6477 GuestThreadState::NotStartedImplicit
6478 | GuestThreadState::NotStartedExplicit(_)
6479 | GuestThreadState::Running
6480 | GuestThreadState::Completed => {}
6481 }
6482 } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6483 entry.downcast_mut::<Option<WorkItem>>()
6484 {
6485 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6486 }
6487 }
6488
6489 if let Some(fiber) = worker {
6490 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6491 }
6492
6493 let mut handle_item = |item: &mut WorkItem| match item {
6494 WorkItem::ResumeFiber { fiber, .. } => {
6495 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6496 }
6497 WorkItem::PushFuture(_future) => {
6498 }
6501 WorkItem::ResumeThread { .. }
6502 | WorkItem::GuestCall { .. }
6503 | WorkItem::WorkerFunction(_) => {}
6504 };
6505
6506 if let Some(item) = switch_item {
6507 handle_item(item);
6508 }
6509 if let Some(item) = next_switch_item {
6510 handle_item(item);
6511 }
6512 for item in high_priority {
6513 handle_item(item);
6514 }
6515 for item in low_priority {
6516 handle_item(item);
6517 }
6518 for item in saved_next_switch_items
6519 .iter_mut()
6520 .filter_map(|v| v.as_mut())
6521 {
6522 handle_item(item);
6523 }
6524 }
6525
6526 fn schedule_suspended_thread(
6527 &mut self,
6528 thread: QualifiedThreadId,
6529 instance: RuntimeInstance,
6530 priority: Priority,
6531 ) -> Result<()> {
6532 if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6534 bail_bug!("switch item already set");
6535 }
6536 let slot = &mut self.get_mut(thread.thread)?.state;
6537 if !matches!(slot, GuestThreadState::Suspended(_)) {
6538 bail!(Trap::CannotResumeThread);
6539 }
6540 let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6541 else {
6542 unreachable!()
6543 };
6544 let item = WorkItem::ResumeFiber {
6545 instance,
6546 thread,
6547 fiber,
6548 };
6549 match priority {
6550 Priority::Switch => self.switch_item = Some(item),
6551 Priority::High => self.push_high_priority(item),
6552 Priority::Low => self.push_low_priority(item),
6553 }
6554 Ok(())
6555 }
6556
6557 fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6558 let slot = &mut self.get_mut(thread)?.state;
6559 slot.check_no_fiber()?;
6560 *slot = GuestThreadState::Running;
6561 Ok(())
6562 }
6563
6564 fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6565 let saved = self.push(None::<WorkItem>)?;
6568 let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6569 *slot = self.next_switch_item.take();
6570 Ok(saved)
6571 }
6572
6573 fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6574 if self.next_switch_item.is_some() {
6575 bail_bug!("next switch item already set when restoring");
6576 }
6577 self.next_switch_item = self.delete(saved)?;
6578 Ok(())
6579 }
6580
6581 fn take_callback_waiter(
6582 &mut self,
6583 set: TableId<WaitableSet>,
6584 thread: QualifiedThreadId,
6585 ) -> Result<Instance> {
6586 let waiting = &mut self.get_mut(set)?.waiting;
6587 let instance = match waiting.get(&thread) {
6588 Some(WaitMode::Callback(instance)) => *instance,
6589 other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6590 };
6591 waiting.remove(&thread);
6592 Ok(instance)
6593 }
6594
6595 fn push<V: Send + Sync + 'static>(
6596 &mut self,
6597 value: V,
6598 ) -> Result<TableId<V>, ResourceTableError> {
6599 self.table.get_mut().push(value).map(TableId::from)
6600 }
6601
6602 fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6603 self.table.get_mut().get_mut(&Resource::from(id))
6604 }
6605
6606 pub fn add_child<T: 'static, U: 'static>(
6607 &mut self,
6608 child: TableId<T>,
6609 parent: TableId<U>,
6610 ) -> Result<(), ResourceTableError> {
6611 self.table
6612 .get_mut()
6613 .add_child(Resource::from(child), Resource::from(parent))
6614 }
6615
6616 pub fn remove_child<T: 'static, U: 'static>(
6617 &mut self,
6618 child: TableId<T>,
6619 parent: TableId<U>,
6620 ) -> Result<(), ResourceTableError> {
6621 self.table
6622 .get_mut()
6623 .remove_child(Resource::from(child), Resource::from(parent))
6624 }
6625
6626 fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6627 self.table.get_mut().delete(Resource::from(id))
6628 }
6629
6630 fn push_future(&mut self, future: HostTaskFuture) {
6631 self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6638 }
6639
6640 fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6641 if self.switch_item.is_some() {
6642 self.push_high_priority(item);
6645 bail_bug!("switch item already set");
6646 }
6647
6648 self.switch_item = Some(item);
6649 log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6650
6651 Ok(())
6652 }
6653
6654 fn take_next_switch_item(&mut self) -> Result<()> {
6655 if let Some(item) = self.next_switch_item.take() {
6656 self.set_switch_item(item)?;
6657 }
6658 Ok(())
6659 }
6660
6661 fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6664 let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6667 return Ok(());
6668 };
6669 let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6670 if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6671 bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6672 }
6673 let instance = self.get_mut(thread.task)?.instance;
6674 self.get_mut(thread.thread)?.wake_on_cancel.take();
6675 let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6676
6677 let item = match mode {
6678 WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6679 instance,
6680 thread,
6681 fiber,
6682 },
6683 WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6684 instance,
6685 call: GuestCall {
6686 thread,
6687 kind: GuestCallKind::DeliverEvent {
6688 instance: callback_instance,
6689 set: Some(set),
6690 },
6691 },
6692 },
6693 };
6694 self.push_high_priority(item);
6695 Ok(())
6696 }
6697
6698 fn push_high_priority(&mut self, item: WorkItem) {
6699 self.high_priority.push_front(item);
6700 log::trace!(
6701 "push high priority: {:?}",
6702 self.high_priority.front().unwrap()
6703 );
6704 }
6705
6706 fn push_low_priority(&mut self, item: WorkItem) {
6707 self.low_priority.push_front(item);
6708 log::trace!(
6709 "push low priority: {:?}",
6710 self.low_priority.front().unwrap()
6711 );
6712 }
6713
6714 fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6715 match priority {
6716 Priority::Switch => self.set_switch_item(item)?,
6717 Priority::High => self.push_high_priority(item),
6718 Priority::Low => self.push_low_priority(item),
6719 }
6720
6721 Ok(())
6722 }
6723
6724 fn promote_instance_local_thread_work_item(
6725 &mut self,
6726 current_instance: RuntimeInstance,
6727 ) -> Result<bool> {
6728 log::trace!("promote thread work items for {current_instance:?}");
6729
6730 self.promote_work_item_matching(|item: &WorkItem| {
6731 let result = match item {
6732 WorkItem::ResumeThread { instance, .. }
6733 | WorkItem::ResumeFiber { instance, .. }
6734 | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6735 _ => false,
6736 };
6737
6738 log::trace!("candidate {item:?}: {result}");
6739 result
6740 })
6741 }
6742
6743 fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6744 self.promote_work_item_matching(|item: &WorkItem| match item {
6745 WorkItem::ResumeThread {
6746 thread: item_thread,
6747 ..
6748 }
6749 | WorkItem::GuestCall {
6750 call:
6751 GuestCall {
6752 thread: item_thread,
6753 ..
6754 },
6755 ..
6756 } => *item_thread == thread,
6757 _ => false,
6758 })
6759 }
6760
6761 fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6762 where
6763 F: FnMut(&WorkItem) -> bool,
6764 {
6765 if self.switch_item.is_none() {
6772 if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6773 self.switch_item = self.high_priority.remove(index);
6774 }
6775 }
6776 if self.switch_item.is_none() {
6777 if let Some(index) = self.low_priority.iter().rposition(predicate) {
6778 self.switch_item = self.low_priority.remove(index);
6779 }
6780 }
6781 Ok(self.switch_item.is_some())
6782 }
6783
6784 pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6787 match task.kind() {
6788 ScopeKind::HostId(task) => {
6789 let task: TableId<HostTask> = TableId::new(task);
6790 Ok(&mut self.get_mut(task)?.call_context)
6791 }
6792 ScopeKind::Id(task) => {
6793 let task: TableId<GuestTask> = TableId::new(task);
6794 Ok(&mut self.get_mut(task)?.call_context)
6795 }
6796 }
6797 }
6798
6799 pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6800 self.deferred_host_call_context.as_mut()
6801 }
6802
6803 fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6804 match self.futures.get_mut().as_mut() {
6805 Some(f) => Ok(f),
6806 None => bail_bug!("futures field of concurrent state is currently taken"),
6807 }
6808 }
6809
6810 pub(crate) fn table(&mut self) -> &mut ResourceTable {
6811 self.table.get_mut()
6812 }
6813
6814 fn debug_assert_deferred_host_invariant(&self) {
6815 debug_assert_eq!(
6816 self.deferred_host_call_context.is_some(),
6817 matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6818 "a deferred host thread and call context must exist together",
6819 );
6820 }
6821
6822 fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6823 self.debug_assert_deferred_host_invariant();
6824 let caller = match self.unforced_current_thread {
6825 CurrentThread::DeferredHost(caller) => caller,
6826 thread => return Ok(thread),
6827 };
6828
6829 let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6831 let task = self.push(task)?;
6832 let call_context = self
6833 .deferred_host_call_context
6834 .take()
6835 .expect("deferred host call context should be present");
6836 self.get_mut(task)
6837 .expect("newly inserted host task should be present")
6838 .call_context = call_context;
6839 self.unforced_current_thread = CurrentThread::Host(task);
6840 self.debug_assert_deferred_host_invariant();
6841 log::trace!("new host task materialized {task:?}");
6842 Ok(CurrentThread::Host(task))
6843 }
6844
6845 fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6846 match self.materialize_host_task()? {
6847 CurrentThread::Host(id) => Ok(Some(id)),
6848 CurrentThread::None => Ok(None),
6849 CurrentThread::Guest(_) => {
6850 bail_bug!("tried to materialize a host task id from a guest thread")
6851 }
6852 CurrentThread::DeferredHost(_) => {
6853 bail_bug!(
6854 "current thread is a deferred host thread which should have been materialized"
6855 )
6856 }
6857 }
6858 }
6859
6860 pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6861 match self.materialize_host_task()? {
6862 CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6863 _ => bail_bug!("current scope is not a deferred host scope"),
6864 }
6865 }
6866
6867 fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6868 let count = &mut self.get_mut(task)?.ref_count;
6869 *count += 1;
6870 log::trace!("increment {task:?} to {count}");
6871 Ok(())
6872 }
6873
6874 fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6875 let count = &mut self.get_mut(task)?.ref_count;
6876 if *count == 0 {
6877 bail_bug!("can't decrement task ref count that's already zero");
6878 }
6879 *count -= 1;
6880 log::trace!("decrement {task:?} to {count}");
6881 if *count == 0 {
6882 let task = self.delete(task)?;
6883
6884 self.decrement_group_ref_count(task.group)?;
6885
6886 debug_assert!(task.decremented_interesting_task_count);
6893 }
6894 Ok(())
6895 }
6896
6897 fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6898 log::trace!("delete host task {task:?}");
6899 let task = self.delete(task)?;
6900
6901 self.decrement_group_ref_count(task.group)
6902 }
6903}
6904
6905fn for_any_lower<
6908 F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6909>(
6910 fun: F,
6911) -> F {
6912 fun
6913}
6914
6915fn for_any_lift<
6917 F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6918>(
6919 fun: F,
6920) -> F {
6921 fun
6922}
6923
6924fn check_ambient_store(id: StoreId) {
6925 let message = "\
6926 `Future`s which depend on asynchronous component tasks, streams, or \
6927 futures to complete may only be polled from the event loop of the \
6928 store to which they belong. Please use \
6929 `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6930 ";
6931 tls::try_get(|store| {
6932 let matched = match store {
6933 tls::TryGet::Some(store) => store.id() == id,
6934 tls::TryGet::Taken | tls::TryGet::None => false,
6935 };
6936
6937 if !matched {
6938 panic!("{message}")
6939 }
6940 });
6941}
6942
6943fn unpack_callback_code(code: u32) -> (u32, u32) {
6944 (code & 0xF, code >> 4)
6945}
6946
6947struct WaitableCheckParams {
6951 set: TableId<WaitableSet>,
6952 options: OptionsIndex,
6953 payload: u32,
6954}
6955
6956enum WaitableCheck {
6959 Wait,
6960 Poll,
6961}
6962
6963pub(crate) struct PreparedCall<R> {
6965 handle: Func,
6967 thread: QualifiedThreadId,
6969 param_count: usize,
6971 rx: oneshot::Receiver<LiftedResult>,
6974 _phantom: PhantomData<R>,
6975}
6976
6977impl<R> PreparedCall<R> {
6978 pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6985 store
6986 .concurrent_state_mut()?
6987 .increment_task_ref_count(self.thread.task)?;
6988
6989 Ok(TaskId {
6990 task: self.thread.task,
6991 })
6992 }
6993}
6994
6995pub(crate) struct TaskId {
6997 task: TableId<GuestTask>,
6998}
6999
7000impl TaskId {
7001 pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
7013 let task = store.concurrent_state_mut()?.get_mut(self.task)?;
7014 if !task.already_lowered_parameters() {
7015 task.lower_params = Some(Box::new(|store, _| {
7020 store.set_trapped();
7021 bail!("the host future for this call was dropped before it started")
7022 }));
7023 } else {
7024 _ = store
7029 .concurrent_state_mut()?
7030 .decrement_task_ref_count(self.task);
7031 }
7032
7033 Ok(())
7034 }
7035}
7036
7037pub(crate) fn prepare_call<T, R>(
7043 mut store: StoreContextMut<T>,
7044 handle: Func,
7045 param_count: usize,
7046 lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
7047 + Send
7048 + Sync
7049 + 'static,
7050 lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
7051 + Send
7052 + Sync
7053 + 'static,
7054) -> Result<PreparedCall<R>> {
7055 if !store.0.may_enter() {
7059 bail!(Trap::CannotEnterComponent);
7060 }
7061
7062 let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
7063
7064 let instance = handle.instance().id().get(store.0);
7065 let options = &instance.component().env_component().options[options];
7066 let ty = &instance.component().types()[ty];
7067 let async_typed = ty.async_;
7068 let async_lifted = raw_options.async_;
7069 let task_return_type = ty.results;
7070 let component_instance = raw_options.instance;
7071 let callback = options.callback.map(|i| instance.runtime_callback(i));
7072 let memory = options
7073 .memory()
7074 .map(|i| instance.runtime_memory(i))
7075 .map(SendSyncPtr::new);
7076 let string_encoding = options.string_encoding;
7077 let token = StoreToken::new(store.as_context_mut());
7078 let caller = store.0.materialize_host_task_id()?;
7079 let state = store.0.concurrent_state_mut()?;
7080
7081 let (tx, rx) = oneshot::channel();
7082
7083 let instance = handle.instance().runtime_instance(component_instance);
7084 let thread = GuestTask::new(
7085 state,
7086 Box::new(for_any_lower(move |store, params| {
7087 lower_params(token.as_context_mut(store), params)
7088 })),
7089 LiftResult {
7090 lift: Box::new(for_any_lift(move |store, result| {
7091 lift_result(store, result)
7092 })),
7093 ty: task_return_type,
7094 memory,
7095 string_encoding,
7096 },
7097 Caller::Host {
7098 tx: Some(tx),
7099 caller,
7100 },
7101 callback.map(|callback| {
7102 let callback = SendSyncPtr::new(callback);
7103 let instance = handle.instance();
7104 Box::new(move |store: &mut dyn VMStore, event, handle| {
7105 let store = token.as_context_mut(store);
7106 unsafe { instance.call_callback(store, callback, event, handle) }
7109 }) as CallbackFn
7110 }),
7111 instance,
7112 async_typed,
7113 async_lifted,
7114 )?;
7115
7116 Ok(PreparedCall {
7117 handle,
7118 thread,
7119 param_count,
7120 rx,
7121 _phantom: PhantomData,
7122 })
7123}
7124
7125pub(crate) struct StagedCall<R> {
7126 store: StoreId,
7127 rx: oneshot::Receiver<LiftedResult>,
7128 _marker: PhantomData<fn() -> R>,
7129 group: TaskGroupId,
7137}
7138
7139impl<R> StagedCall<R> {
7140 pub(crate) fn new<T: 'static>(
7151 mut store: StoreContextMut<T>,
7152 prepared: PreparedCall<R>,
7153 ) -> Result<StagedCall<R>> {
7154 let PreparedCall {
7155 handle,
7156 thread,
7157 param_count,
7158 rx,
7159 ..
7160 } = prepared;
7161
7162 stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7163
7164 let state = store.0.concurrent_state_mut()?;
7165 let group = state.get_mut(thread.task)?.group;
7166 state.decrement_task_ref_count(thread.task)?;
7167
7168 Ok(StagedCall {
7169 store: store.0.id(),
7170 rx,
7171 _marker: PhantomData,
7172 group,
7173 })
7174 }
7175}
7176
7177impl<R> Future for StagedCall<R>
7178where
7179 R: 'static,
7180{
7181 type Output = Result<R>;
7182
7183 fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7184 check_ambient_store(self.store);
7185 Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7186 Ok(r) => match r.downcast() {
7187 Ok(r) => Ok(*r),
7188 Err(_) => bail_bug!("wrong type of value produced"),
7189 },
7190 Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7191 })
7192 }
7193}
7194
7195fn stage_call0<T: 'static>(
7198 store: StoreContextMut<T>,
7199 handle: Func,
7200 guest_thread: QualifiedThreadId,
7201 param_count: usize,
7202) -> Result<()> {
7203 let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7204 let is_concurrent = raw_options.async_;
7205 let callback = raw_options.callback;
7206 let instance = handle.instance();
7207 let callee = handle.lifted_core_func(store.0);
7208 let post_return = raw_options
7209 .post_return
7210 .map(|i| instance.id().get(store.0).runtime_post_return(i));
7211 let callback = callback.map(|i| {
7212 let instance = instance.id().get(store.0);
7213 SendSyncPtr::new(instance.runtime_callback(i))
7214 });
7215
7216 log::trace!("queueing call {guest_thread:?}");
7217
7218 unsafe {
7222 instance.stage_call(
7223 store,
7224 guest_thread,
7225 SendSyncPtr::new(callee),
7226 param_count,
7227 1,
7228 is_concurrent,
7229 callback,
7230 post_return.map(SendSyncPtr::new),
7231 true,
7232 )
7233 }
7234}
7235
7236#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7237mod tests {
7238 use super::*;
7239 use crate::component::{Component, Linker};
7240 use crate::store::AsStoreOpaque;
7241 use crate::{Config, Engine};
7242
7243 fn host_subtask(
7244 state: HostTaskState,
7245 event: Option<Event>,
7246 ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7247 let mut config = Config::new();
7248 config.wasm_component_model_async(true);
7249 let engine = Engine::new(&config)?;
7250 let component = Component::new(&engine, "(component)")?;
7251 let mut store = Store::new(&engine, ());
7252 let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7253 let store_opaque = store.as_store_opaque();
7254 let concurrent_state = store_opaque.concurrent_state_mut()?;
7255 let group = concurrent_state.make_task_group()?;
7258 let task = concurrent_state.push(HostTask {
7259 common: WaitableCommon::default(),
7260 call_context: CallContext::default(),
7261 state,
7262 group,
7263 })?;
7264 let handle = store_opaque
7265 .handle_table_insertion(
7266 instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7267 )
7268 .subtask_host(task.rep())?;
7269 let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7270 common.handle = Some(handle);
7271 common.event = event;
7272 Ok((store, instance, task, handle))
7273 }
7274
7275 #[test]
7276 fn host_subtask_drop_during_cancellation() -> Result<()> {
7277 for abort_completed in [false, true] {
7278 let (handle, future) = JoinHandle::run(future::pending::<()>());
7279 let mut future = pin!(future);
7280 let (mut store, instance, task, handle) =
7281 host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7282 let store = store.as_store_opaque();
7283 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7284 assert_eq!(
7285 instance.subtask_cancel(store, caller, true, handle)?,
7286 BLOCKED
7287 );
7288 if abort_completed {
7289 assert!(matches!(
7292 future
7293 .as_mut()
7294 .poll(&mut Context::from_waker(Waker::noop())),
7295 Poll::Ready(None),
7296 ));
7297 }
7298 for async_ in [false, true] {
7299 let err = instance
7300 .subtask_cancel(store, caller, async_, handle)
7301 .unwrap_err();
7302 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7303 }
7304 let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7305 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7306 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7307 }
7308 Ok(())
7309 }
7310
7311 #[test]
7312 fn host_subtask_cancel_after_completion() -> Result<()> {
7313 for async_ in [false, true] {
7314 let (mut store, instance, task, handle) = host_subtask(
7315 HostTaskState::CalleeDone { cancelled: false },
7316 Some(Event::Subtask {
7317 status: Status::Returned,
7318 }),
7319 )?;
7320 let store = store.as_store_opaque();
7321 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7322 assert_eq!(
7323 instance.subtask_cancel(store, caller, async_, handle)?,
7324 Status::Returned as u32,
7325 );
7326 let err = instance
7327 .subtask_cancel(store, caller, async_, handle)
7328 .unwrap_err();
7329 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7330 instance.subtask_drop(store, caller, handle)?;
7331 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7332 }
7333 Ok(())
7334 }
7335
7336 #[test]
7337 fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7338 for (cancelled, status) in [
7339 (false, Status::Returned),
7340 (true, Status::Returned),
7341 (true, Status::ReturnCancelled),
7342 ] {
7343 for delivered in [false, true] {
7344 let event = if delivered {
7345 None
7346 } else {
7347 Some(Event::Subtask { status })
7348 };
7349 let (mut store, instance, task, handle) =
7350 host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7351 let store = store.as_store_opaque();
7352 let result = instance.subtask_drop(
7353 store,
7354 RuntimeComponentInstanceIndex::from_u32(0),
7355 handle,
7356 );
7357 if delivered {
7358 result?;
7359 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7360 } else {
7361 let err = result.unwrap_err();
7362 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7363 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7364 }
7365 }
7366 }
7367 Ok(())
7368 }
7369}