1use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57 HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70 CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71 InstanceState,
72};
73use crate::vm::{
74 AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98 CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99 MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100 RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101 TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102 TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113 Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114 FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115 StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136const BLOCKED: u32 = 0xffff_ffff;
139
140#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143 Starting = 0,
144 Started = 1,
145 Returned = 2,
146 StartCancelled = 3,
147 ReturnCancelled = 4,
148}
149
150impl Status {
151 pub fn pack(self, waitable: Option<u32>) -> u32 {
157 assert!(matches!(self, Status::Returned) == waitable.is_none());
158 let waitable = waitable.unwrap_or(0);
159 assert!(waitable < (1 << 28));
160 (waitable << 4) | (self as u32)
161 }
162}
163
164#[derive(Clone, Copy, Debug)]
167enum Event {
168 None,
169 Subtask {
170 status: Status,
171 },
172 StreamRead {
173 code: ReturnCode,
174 pending: Option<(TypeStreamTableIndex, u32)>,
175 },
176 StreamWrite {
177 code: ReturnCode,
178 pending: Option<(TypeStreamTableIndex, u32)>,
179 },
180 FutureRead {
181 code: ReturnCode,
182 pending: Option<(TypeFutureTableIndex, u32)>,
183 },
184 FutureWrite {
185 code: ReturnCode,
186 pending: Option<(TypeFutureTableIndex, u32)>,
187 },
188 Cancelled,
189}
190
191impl Event {
192 fn parts(self) -> (u32, u32) {
197 const EVENT_NONE: u32 = 0;
198 const EVENT_SUBTASK: u32 = 1;
199 const EVENT_STREAM_READ: u32 = 2;
200 const EVENT_STREAM_WRITE: u32 = 3;
201 const EVENT_FUTURE_READ: u32 = 4;
202 const EVENT_FUTURE_WRITE: u32 = 5;
203 const EVENT_CANCELLED: u32 = 6;
204 match self {
205 Event::None => (EVENT_NONE, 0),
206 Event::Cancelled => (EVENT_CANCELLED, 0),
207 Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208 Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209 Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210 Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211 Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212 }
213 }
214}
215
216mod callback_code {
218 pub const EXIT: u32 = 0;
219 pub const YIELD: u32 = 1;
220 pub const WAIT: u32 = 2;
221}
222
223const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233 store: StoreContextMut<'a, T>,
234 get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239 D: HasData + ?Sized,
240 T: 'static,
241{
242 pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244 Self { store, get_data }
245 }
246
247 pub fn data_mut(&mut self) -> &mut T {
249 self.store.data_mut()
250 }
251
252 pub fn get(&mut self) -> D::Data<'_> {
254 (self.get_data)(self.data_mut())
255 }
256
257 pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261 where
262 T: 'static,
263 {
264 let accessor = Accessor {
265 get_data: self.get_data,
266 token: StoreToken::new(self.store.as_context_mut()),
267 };
268 self.store
269 .as_context_mut()
270 .spawn_with_accessor(accessor, task)
271 }
272
273 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276 self.get_data
277 }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282 D: HasData + ?Sized,
283 T: 'static,
284{
285 type Data = T;
286
287 fn as_context(&self) -> StoreContext<'_, T> {
288 self.store.as_context()
289 }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294 D: HasData + ?Sized,
295 T: 'static,
296{
297 fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298 self.store.as_context_mut()
299 }
300}
301
302pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363 D: HasData + ?Sized,
364{
365 token: StoreToken<T>,
366 get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369pub trait AsAccessor {
386 type Data: 'static;
388
389 type AccessorData: HasData + ?Sized;
392
393 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398 type Data = T::Data;
399 type AccessorData = T::AccessorData;
400
401 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402 T::as_accessor(self)
403 }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407 type Data = T;
408 type AccessorData = D;
409
410 fn as_accessor(&self) -> &Accessor<T, D> {
411 self
412 }
413}
414
415const _: () = {
438 const fn assert<T: Send + Sync>() {}
439 assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443 pub(crate) fn new(token: StoreToken<T>) -> Self {
452 Self {
453 token,
454 get_data: |x| x,
455 }
456 }
457}
458
459impl<T, D> Accessor<T, D>
460where
461 D: HasData + ?Sized,
462{
463 pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481 tls::get(|vmstore| {
482 fun(Access {
483 store: self.token.as_context_mut(vmstore),
484 get_data: self.get_data,
485 })
486 })
487 }
488
489 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492 self.get_data
493 }
494
495 pub fn with_getter<D2: HasData>(
512 &self,
513 get_data: fn(&mut T) -> D2::Data<'_>,
514 ) -> Accessor<T, D2> {
515 Accessor {
516 token: self.token,
517 get_data,
518 }
519 }
520
521 pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537 where
538 T: 'static,
539 {
540 let accessor = self.clone_for_spawn();
541 self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542 }
543
544 fn clone_for_spawn(&self) -> Self {
545 Self {
546 token: self.token,
547 get_data: self.get_data,
548 }
549 }
550
551 pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587 self.with(|mut access| {
588 let store = access.as_context_mut().0;
589 let state = store.concurrent_state_mut_without_forcing_current_thread();
590 if state.interesting_tasks == 0 {
591 Poll::Ready(())
592 } else {
593 state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594 Poll::Pending
595 }
596 })
597 }
598
599 pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623 self.with(|mut access| {
624 let store = access.as_context_mut().0;
625
626 {
627 let state = store.concurrent_state_mut_without_forcing_current_thread();
628 if state.switch_item.is_some()
629 || !state.high_priority.is_empty()
630 || !state.low_priority.is_empty()
631 {
632 state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633 return Poll::Pending;
634 }
635 }
636
637 let (_, _, _, raw_options) = func.abi_info(store);
638 let instance = func.instance().runtime_instance(raw_options.instance);
639 let state = store.instance_state(instance).concurrent_state();
640 if state.backpressure == 0 {
641 Poll::Ready(())
642 } else {
643 store
644 .concurrent_state_mut_without_forcing_current_thread()
645 .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646 Poll::Pending
647 }
648 })
649 }
650}
651
652pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674 AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676 D: HasData + ?Sized,
677{
678 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684 T: 'static,
685 F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686 F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687 Fut: Future<Output = Result<()>> + Send + 'fut,
688 D: HasData,
689{
690 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691 (self)(accessor)
692 }
693}
694
695enum CallerInfo {
698 Async {
700 params: Vec<ValRaw>,
701 has_result: bool,
702 },
703 Sync {
705 params: Vec<ValRaw>,
706 result_count: u32,
707 },
708}
709
710enum WaitMode {
712 Fiber(StoreFiber<'static>),
714 Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721 match self {
722 Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723 Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724 }
725 }
726}
727
728#[derive(Debug)]
730enum SuspendReason {
731 Waiting {
734 set: TableId<WaitableSet>,
735 thread: QualifiedThreadId,
736 },
737 YieldingToSubtask { thread: QualifiedThreadId },
740 NeedWork,
743 Yielding { thread: QualifiedThreadId },
746 ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751enum GuestCallKind {
753 DeliverEvent {
756 instance: Instance,
758 set: Option<TableId<WaitableSet>>,
763 },
764 StartImplicit {
767 closure: Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>,
769 was_pending: bool,
771 },
772 StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
773}
774
775impl fmt::Debug for GuestCallKind {
776 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
777 match self {
778 Self::DeliverEvent { instance, set } => f
779 .debug_struct("DeliverEvent")
780 .field("instance", instance)
781 .field("set", set)
782 .finish(),
783 Self::StartImplicit { was_pending, .. } => f
784 .debug_struct("StartImplicit")
785 .field("was_pending", was_pending)
786 .finish(),
787 Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
788 }
789 }
790}
791
792#[derive(Copy, Clone, Debug)]
794pub enum SuspensionTarget {
795 Resume(u32),
796 Promote(u32),
797 None,
798}
799
800#[derive(Copy, Clone, Debug)]
802pub enum ResumeThread {
803 Promote,
804 Resume,
805 ResumeLater,
806}
807
808#[derive(Debug)]
810struct GuestCall {
811 thread: QualifiedThreadId,
812 kind: GuestCallKind,
813}
814
815impl GuestCall {
816 fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
828 let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
829 let async_typed = task.async_typed;
830 let async_lifted = task.async_lifted;
831 let has_callback = task.callback.is_some();
832 let instance = task.instance;
833 let state = store.instance_state(instance).concurrent_state();
834
835 let ready = match &self.kind {
836 GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
837 GuestCallKind::StartImplicit { was_pending, .. } => {
838 if async_typed {
839 let needs_exclusive = !async_lifted || has_callback;
842
843 let may_enter =
848 !((needs_exclusive && state.do_not_enter) || state.backpressure > 0);
849
850 let skip_wait = *was_pending || state.pending_start_count == 0;
857
858 may_enter && skip_wait
861 } else {
862 true
863 }
864 }
865 GuestCallKind::StartExplicit(_) => true,
866 };
867 log::trace!(
868 "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {}; pending_start_count: {})",
869 state.do_not_enter,
870 state.backpressure,
871 state.pending_start_count,
872 );
873 Ok(ready)
874 }
875}
876
877enum WorkerItem {
879 GuestCall(GuestCall),
880 Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
881}
882
883enum WorkItem {
886 PushFuture(AlwaysMut<HostTaskFuture>),
888 ResumeFiber {
890 instance: RuntimeInstance,
891 thread: QualifiedThreadId,
892 fiber: StoreFiber<'static>,
893 },
894 ResumeThread {
896 instance: RuntimeInstance,
897 thread: QualifiedThreadId,
898 },
899 GuestCall {
901 instance: RuntimeInstance,
902 call: GuestCall,
903 },
904 WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
906}
907
908struct ResumingFiber<'a> {
914 store: &'a mut StoreOpaque,
915 fiber: Option<StoreFiber<'static>>,
916}
917
918impl Drop for ResumingFiber<'_> {
919 fn drop(&mut self) {
920 if let Some(mut fiber) = self.fiber.take() {
921 fiber.dispose(self.store);
922 }
923 }
924}
925
926impl fmt::Debug for WorkItem {
927 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
928 match self {
929 Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
930 Self::ResumeFiber {
931 instance, thread, ..
932 } => f
933 .debug_struct("ResumeFiber")
934 .field("instance", instance)
935 .field("thread", thread)
936 .finish(),
937 Self::ResumeThread { instance, thread } => f
938 .debug_struct("ResumeThread")
939 .field("instance", instance)
940 .field("thread", thread)
941 .finish(),
942 Self::GuestCall { instance, call } => f
943 .debug_struct("GuestCall")
944 .field("instance", instance)
945 .field("call", call)
946 .finish(),
947 Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
948 }
949 }
950}
951
952#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
954pub(crate) enum WaitResult {
955 Cancelled,
956 Completed,
957}
958
959pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
967 store: &mut dyn VMStore,
968 host_task: EnteredHostTask,
969 future: impl Future<Output = Result<R>> + Send + 'static,
970) -> Result<R> {
971 let mut future = Box::pin(future);
978 let poll = tls::set(store, || {
979 future
980 .as_mut()
981 .poll(&mut Context::from_waker(&Waker::noop()))
982 });
983
984 let caller = match host_task {
985 Some(caller) => caller,
986 None => bail_bug!("host task wasn't created but should have been"),
987 };
988
989 let task = match poll {
990 Poll::Ready(result) => return result,
992
993 Poll::Pending => {
998 let Some(task) = store.materialize_host_task_id()? else {
999 bail_bug!("current thread is not a host thread")
1000 };
1001
1002 let future = Box::pin(async move {
1005 let result = run_with_host_task_set(task, future).await??;
1006 tls::get(move |store| {
1007 let state = store.concurrent_state_mut()?;
1008 let host_state = &mut state.get_mut(task)?.state;
1009 assert!(matches!(host_state, HostTaskState::CalleeStarted));
1010 *host_state = HostTaskState::CalleeFinished(Box::new(result));
1011
1012 Waitable::Host(task).set_event(
1013 state,
1014 Some(Event::Subtask {
1015 status: Status::Returned,
1016 }),
1017 )?;
1018
1019 Ok(())
1020 })
1021 }) as HostTaskFuture;
1022
1023 let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
1024 store.switch_or_trap_if_may_not_suspend(caller_instance)?;
1025
1026 let state = store.concurrent_state_mut()?;
1027 state.push_future(future);
1028
1029 let set = state.get_mut(caller.thread)?.sync_call_set;
1030 Waitable::Host(task).join(state, Some(set))?;
1031
1032 store.suspend(SuspendReason::Waiting {
1033 set,
1034 thread: caller,
1035 })?;
1036
1037 Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1041 task
1042 }
1043 };
1044
1045 let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1047 match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1048 HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1049 Ok(result) => *result,
1050 Err(_) => bail_bug!("host task finished with wrong type of result"),
1051 }),
1052 _ => bail_bug!("unexpected host task state after completion"),
1053 }
1054}
1055
1056fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1058 match call.kind {
1059 GuestCallKind::DeliverEvent { instance, set } => {
1060 if let Some(set) = set {
1063 store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1064 }
1065 let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1066 Some(pair) => pair,
1067 None => match set {
1068 Some(set) => {
1073 log::trace!(
1074 "event for {:?} on {set:?} no longer present; waiting again",
1075 call.thread
1076 );
1077 return instance.wait_with_callback(
1078 store.store_opaque_mut(),
1079 call.thread,
1080 set,
1081 );
1082 }
1083 None => (Event::None, None),
1088 },
1089 };
1090 let state = store.concurrent_state_mut()?;
1091 if let Some(set) = set
1096 && waitable.is_none()
1097 && !state.get_mut(set)?.ready.is_empty()
1098 {
1099 state.wake_waiter(set)?;
1100 }
1101 let task = state.get_mut(call.thread.task)?;
1102 let runtime_instance = task.instance;
1103 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1104
1105 log::trace!(
1106 "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1107 call.thread,
1108 );
1109
1110 instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1111 invoke_callback(store, call.thread.task, event, handle)
1112 })?;
1113 }
1114 GuestCallKind::StartImplicit { closure, .. } => {
1115 closure(store)?;
1116 }
1117 GuestCallKind::StartExplicit(fun) => {
1118 fun(store)?;
1119 }
1120 }
1121
1122 Ok(())
1123}
1124
1125fn invoke_callback(
1128 store: &mut dyn VMStore,
1129 task: TableId<GuestTask>,
1130 event: Event,
1131 handle: u32,
1132) -> Result<u32> {
1133 let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1134 bail_bug!("guest task callback field not present")
1135 };
1136
1137 let code = callback(store, event, handle)?;
1138
1139 store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1140
1141 Ok(code)
1142}
1143
1144impl<T> Store<T> {
1145 pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1147 where
1148 T: Send + 'static,
1149 {
1150 ensure!(
1151 self.as_context().0.concurrency_support(),
1152 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1153 );
1154 self.as_context_mut().run_concurrent(fun).await
1155 }
1156
1157 #[doc(hidden)]
1158 pub fn assert_concurrent_state_empty(&mut self) {
1159 self.as_context_mut().assert_concurrent_state_empty();
1160 }
1161
1162 #[doc(hidden)]
1163 pub fn concurrent_state_table_size(&mut self) -> usize {
1164 self.as_context_mut().concurrent_state_table_size()
1165 }
1166
1167 #[doc(hidden)]
1168 pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1169 self.as_context_mut()
1170 .set_max_concurrent_state_table_capacity(max)
1171 }
1172
1173 pub fn spawn(
1175 &mut self,
1176 task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1177 ) -> Result<JoinHandle>
1178 where
1179 T: 'static,
1180 {
1181 self.as_context_mut().spawn(task)
1182 }
1183}
1184
1185impl<T> StoreContextMut<'_, T> {
1186 #[doc(hidden)]
1197 pub fn assert_concurrent_state_empty(self) {
1198 let store = self.0;
1199 store
1200 .store_data_mut()
1201 .components
1202 .assert_instance_states_empty();
1203 let state = store.concurrent_state_mut().unwrap();
1204 assert!(
1205 state.table.get_mut().is_empty(),
1206 "non-empty table: {:?}",
1207 state.table.get_mut()
1208 );
1209 assert!(state.switch_item.is_none());
1210 assert!(state.next_switch_item.is_none());
1211 assert!(state.high_priority.is_empty());
1212 assert!(state.low_priority.is_empty());
1213 assert!(state.saved_next_switch_items.is_empty());
1214 assert!(state.unforced_current_thread.is_none());
1215 assert!(state.deferred_host_call_context.is_none());
1216 assert!(state.futures_mut().unwrap().is_empty());
1217 assert!(state.global_error_context_ref_counts.is_empty());
1218 }
1219
1220 #[doc(hidden)]
1225 pub fn concurrent_state_table_size(&mut self) -> usize {
1226 self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1227 }
1228
1229 #[doc(hidden)]
1235 pub fn set_max_concurrent_state_table_capacity(&mut self, max: usize) {
1236 self.0
1237 .concurrent_state_mut_without_forcing_current_thread()
1238 .table
1239 .get_mut()
1240 .set_max_capacity(max);
1241 }
1242
1243 pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1253 where
1254 T: 'static,
1255 {
1256 let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1257 self.spawn_with_accessor(accessor, task)
1258 }
1259
1260 fn spawn_with_accessor<D>(
1263 self,
1264 accessor: Accessor<T, D>,
1265 task: impl for<'fut> AccessorTask<'fut, T, D>,
1266 ) -> Result<JoinHandle>
1267 where
1268 T: 'static,
1269 D: HasData + ?Sized,
1270 {
1271 let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1275 self.0
1276 .concurrent_state_mut()?
1277 .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1278 Ok(handle)
1279 }
1280
1281 pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1365 where
1366 T: Send + 'static,
1367 {
1368 ensure!(
1369 self.0.concurrency_support(),
1370 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1371 );
1372 self.do_run_concurrent(fun, false).await
1373 }
1374
1375 pub(super) async fn run_concurrent_trap_on_idle<R>(
1376 self,
1377 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1378 ) -> Result<R> {
1379 self.do_run_concurrent(fun, true).await
1380 }
1381
1382 async fn do_run_concurrent<R>(
1383 mut self,
1384 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1385 trap_on_idle: bool,
1386 ) -> Result<R> {
1387 debug_assert!(self.0.concurrency_support());
1388 self.0.check_event_loop_not_running()?;
1389 let token = StoreToken::new(self.as_context_mut());
1390
1391 struct Dropper<'a, T: 'static, V> {
1392 store: StoreContextMut<'a, T>,
1393 value: ManuallyDrop<V>,
1394 }
1395
1396 impl<'a, T, V> Drop for Dropper<'a, T, V> {
1397 fn drop(&mut self) {
1398 self.store
1399 .0
1400 .concurrent_state_mut_already_forced_current_thread()
1401 .event_loop_running = false;
1402
1403 tls::set(self.store.0, || {
1404 unsafe { ManuallyDrop::drop(&mut self.value) }
1409 });
1410 }
1411 }
1412
1413 let accessor = &Accessor::new(token);
1414 self.0
1415 .concurrent_state_mut_already_forced_current_thread()
1416 .event_loop_running = true;
1417 let dropper = &mut Dropper {
1418 store: self,
1419 value: ManuallyDrop::new(fun(accessor)),
1420 };
1421 let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1423
1424 let result = dropper
1425 .store
1426 .as_context_mut()
1427 .poll_until(future, trap_on_idle)
1428 .await;
1429
1430 if result.is_err() {
1431 dropper.store.0.set_trapped();
1432 }
1433
1434 result
1435 }
1436
1437 async fn poll_until<R>(
1443 mut self,
1444 mut future: Pin<&mut impl Future<Output = R>>,
1445 trap_on_idle: bool,
1446 ) -> Result<R> {
1447 struct Reset<'a, T: 'static> {
1448 store: StoreContextMut<'a, T>,
1449 futures: Option<FuturesUnordered<HostTaskFuture>>,
1450 }
1451
1452 impl<'a, T> Drop for Reset<'a, T> {
1453 fn drop(&mut self) {
1454 if let Some(futures) = self.futures.take() {
1455 *self
1456 .store
1457 .0
1458 .concurrent_state_mut_already_forced_current_thread()
1459 .futures
1460 .get_mut() = Some(futures);
1461 }
1462 }
1463 }
1464
1465 const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1469 let mut turns_without_yield = 0;
1470
1471 loop {
1472 let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1476 let mut reset = Reset {
1477 store: self.as_context_mut(),
1478 futures,
1479 };
1480 let mut next = match reset.futures.as_mut() {
1481 Some(f) => pin!(f.next()),
1482 None => bail_bug!("concurrent state missing futures field"),
1483 };
1484
1485 enum PollResult<R> {
1486 Complete(R),
1487 ProcessWork {
1488 ready: Option<WorkItem>,
1489 low_priority: bool,
1490 },
1491 }
1492
1493 let result = future::poll_fn(|cx| {
1494 if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1497 return Poll::Ready(Ok(PollResult::Complete(value)));
1498 }
1499
1500 if reset.store.0.component_trapped() {
1509 return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1510 }
1511
1512 let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1516 Poll::Ready(Some(output)) => {
1517 match output {
1518 Err(e) => return Poll::Ready(Err(e)),
1519 Ok(()) => {}
1520 }
1521 Poll::Ready(true)
1522 }
1523 Poll::Ready(None) => Poll::Ready(false),
1524 Poll::Pending => Poll::Pending,
1525 };
1526
1527 let state = reset.store.0.concurrent_state_mut()?;
1542 let mut ready = state.switch_item.take();
1543 let mut low_priority = false;
1544 if ready.is_none() {
1545 ready = state.high_priority.pop_back();
1546 if ready.is_none() {
1547 ready = state.low_priority.pop_back();
1548 low_priority = true;
1549 }
1550 }
1551 if ready.is_some() {
1552 return Poll::Ready(Ok(PollResult::ProcessWork {
1553 ready,
1554 low_priority,
1555 }));
1556 }
1557
1558 if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1559 waker.wake();
1560 }
1561
1562 return match next {
1566 Poll::Ready(true) => {
1567 Poll::Ready(Ok(PollResult::ProcessWork {
1573 ready: None,
1574 low_priority: false,
1575 }))
1576 }
1577 Poll::Ready(false) => {
1578 if let Poll::Ready(value) =
1582 tls::set(reset.store.0, || future.as_mut().poll(cx))
1583 {
1584 Poll::Ready(Ok(PollResult::Complete(value)))
1585 } else {
1586 if trap_on_idle {
1592 Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1599 Trap::CannotBlockSyncTask.into()
1600 } else {
1601 Trap::AsyncDeadlock.into()
1603 }))
1604 } else {
1605 Poll::Pending
1609 }
1610 }
1611 }
1612 Poll::Pending => Poll::Pending,
1617 };
1618 })
1619 .await;
1620
1621 drop(reset);
1625
1626 match result? {
1627 PollResult::Complete(value) => break Ok(value),
1630 PollResult::ProcessWork {
1633 mut ready,
1634 low_priority,
1635 } => {
1636 if low_priority {
1664 if let Some(item) = ready.take() {
1665 self.0.concurrent_state_mut()?.low_priority.push_back(item);
1666 }
1667 self.0.yield_now().await;
1668 turns_without_yield = 0;
1669 ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1670 }
1671
1672 if let Some(item) = ready {
1673 self.as_context_mut().handle_work_item(item).await?;
1674 }
1675
1676 turns_without_yield += 1;
1677 if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1678 turns_without_yield = 0;
1679 self.0.yield_now().await;
1680 }
1681 }
1682 }
1683 }
1684 }
1685
1686 fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1688 let item = match item {
1692 WorkItem::ResumeFiber {
1693 instance,
1694 thread,
1695 fiber,
1696 } => {
1697 let future = self.0.resume_fiber(fiber);
1698 log::trace!(
1699 "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1700 );
1701 return futures::future::Either::Left(future);
1702 }
1703 item => item,
1704 };
1705 futures::future::Either::Right(async move {
1706 log::trace!("handle work item {item:?}");
1707 match item {
1708 WorkItem::PushFuture(future) => {
1709 self.0
1710 .concurrent_state_mut()?
1711 .futures_mut()?
1712 .push(future.into_inner());
1713 }
1714 WorkItem::ResumeFiber { .. } => unreachable!(),
1715 WorkItem::ResumeThread { thread, .. } => {
1716 let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1717 if !matches!(slot, GuestThreadState::Ready { .. }) {
1718 bail_bug!("cannot resume non-pending thread {thread:?}");
1719 }
1720 let GuestThreadState::Ready { fiber } =
1721 mem::replace(slot, GuestThreadState::Running)
1722 else {
1723 unreachable!()
1724 };
1725 self.0.resume_fiber(fiber).await?;
1726 }
1727 WorkItem::GuestCall { mut call, .. } => {
1728 if call.is_ready(self.0)? {
1729 if let GuestCallKind::StartImplicit {
1730 was_pending: true, ..
1731 } = &call.kind
1732 {
1733 let instance = self
1734 .0
1735 .concurrent_state_mut()?
1736 .get_mut(call.thread.task)?
1737 .instance;
1738
1739 self.0.decrement_pending_start_count(instance, 1)?;
1740 }
1741
1742 self.0
1743 .concurrent_state_mut()?
1744 .get_mut(call.thread.thread)?
1745 .wake_on_cancel = WakeOnCancel::None;
1746
1747 self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1748 } else {
1749 let state = self.0.concurrent_state_mut()?;
1750 let task = state.get_mut(call.thread.task)?;
1751 if !task.starting_sent {
1752 task.starting_sent = true;
1753 if let GuestCallKind::StartImplicit { .. } = &call.kind {
1754 Waitable::Guest(call.thread.task).set_event(
1755 state,
1756 Some(Event::Subtask {
1757 status: Status::Starting,
1758 }),
1759 )?;
1760 }
1761 }
1762
1763 let increment = if let GuestCallKind::StartImplicit {
1764 was_pending, ..
1765 } = &mut call.kind
1766 {
1767 let increment = !*was_pending;
1768 *was_pending = true;
1769 increment
1770 } else {
1771 false
1772 };
1773
1774 let instance = state.get_mut(call.thread.task)?.instance;
1775 if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1784 if !state.get_mut(*set)?.ready.is_empty() {
1785 state.wake_waiter(*set)?;
1786 }
1787 }
1788
1789 state.take_next_switch_item()?;
1793
1794 let state = self.0.instance_state(instance).concurrent_state();
1795
1796 state.pending.insert(call.thread, call.kind);
1797
1798 if increment {
1799 state.pending_start_count += 1;
1800 }
1801 }
1802 }
1803 WorkItem::WorkerFunction(fun) => {
1804 self.run_on_worker(WorkerItem::Function(fun)).await?;
1805 }
1806 }
1807
1808 Ok(())
1809 })
1810 }
1811
1812 async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1814 let state = self.0.concurrent_state_mut()?;
1815 if state.worker_item.is_some() {
1816 bail_bug!("worker item already set when taking worker fiber");
1817 }
1818 let worker = if let Some(fiber) = state.worker.take() {
1819 fiber
1820 } else {
1821 unsafe {
1840 fiber::make_fiber_unchecked(self.0, move |store| {
1841 loop {
1842 let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1843 bail_bug!("worker_item not present when resuming fiber")
1844 };
1845 match item {
1846 WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1847 WorkerItem::Function(fun) => fun.into_inner()(store)?,
1848 }
1849
1850 store.suspend(SuspendReason::NeedWork)?;
1851 }
1852 })?
1853 }
1854 };
1855
1856 self.0
1859 .concurrent_state_mut_already_forced_current_thread()
1860 .worker_item = Some(item);
1861
1862 self.0.resume_fiber(worker).await
1863 }
1864
1865 pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1870 where
1871 T: 'static,
1872 F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1873 + Send
1874 + Sync
1875 + 'static,
1876 R: Send + Sync + 'static,
1877 {
1878 let token = StoreToken::new(self);
1879 async move {
1880 let mut accessor = Accessor::new(token);
1881 closure(&mut accessor).await
1882 }
1883 }
1884
1885 pub(crate) async fn start_instance(
1886 &mut self,
1887 instance: ModuleInstance,
1888 callee: Option<RuntimeInstance>,
1889 ) -> Result<ModuleInstance> {
1890 self.0.check_event_loop_not_running()?;
1892 let (tx, rx) = oneshot::channel();
1893 let token = StoreToken::new(self.as_context_mut());
1894 self.0.queue_task(move |store| {
1895 _ = tx.send(
1896 super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1897 .map(|()| instance),
1898 );
1899 Ok(())
1900 })?;
1901 self.as_context_mut()
1902 .run_concurrent_trap_on_idle(async |_| {
1903 rx.await
1904 .map_err(|_| format_err!("oneshot channel canceled"))
1905 })
1906 .await??
1907 }
1908}
1909
1910pub type EnteredHostTask = Option<QualifiedThreadId>;
1917
1918impl StoreOpaque {
1919 pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1920 if self
1921 .concurrent_state_mut_already_forced_current_thread()
1922 .event_loop_running
1923 {
1924 bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1925 }
1926 Ok(())
1927 }
1928
1929 #[inline]
1933 pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1934 if !self.concurrency_support() {
1936 return Ok(CurrentThread::None);
1937 }
1938
1939 if !self
1942 .vm_store_context_mut()
1943 .current_thread_mut()
1944 .is_deferred()
1945 {
1946 return Ok(self
1947 .concurrent_state_mut_already_forced_current_thread()
1948 .unforced_current_thread);
1949 }
1950
1951 self.force_deferred_current_thread()
1952 }
1953
1954 #[cold]
1957 fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1958 let state = self.concurrent_state_mut_without_forcing_current_thread();
1967 let id = match state.unforced_current_thread.guest_task() {
1968 Some(task) => state.get_mut(task)?.instance.instance,
1969 None => bail_bug!("deferred component-model thread with non-guest base"),
1970 };
1971
1972 let mut frames = SmallVec::<[_; 8]>::new();
1975 let mut cur = *self.vm_store_context_mut().current_thread_mut();
1976 while let Some(ptr) = cur.as_deferred() {
1977 let deferred = unsafe { ptr.as_non_null().as_ref() };
1982 frames.push((
1983 deferred.callee_async != 0,
1984 deferred.callee_instance,
1985 deferred.saved_context,
1986 deferred.lazy_lenders_start,
1987 ));
1988 cur = deferred.parent;
1989 }
1990
1991 *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1995
1996 let current_context = *self.vm_store_context_mut().component_context_mut();
1999
2000 let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
2007 frames.reverse();
2008 let lazy_lenders_base = match frames.first() {
2009 Some((_, _, _, start)) => *start,
2010 None => bail_bug!("forcing a deferred thread with no deferred frames"),
2011 };
2012 let lazy_lender_ends = frames
2013 .iter()
2014 .skip(1)
2015 .map(|(_, _, _, start)| *start)
2016 .chain([lazy_lenders_end])
2017 .collect::<SmallVec<[_; 8]>>();
2018 for ((callee_async, callee_instance, saved_context, start), lenders_end) in
2019 frames.into_iter().zip(lazy_lender_ends)
2020 {
2021 *self.vm_store_context_mut().component_context_mut() = saved_context;
2025 let callee = RuntimeInstance {
2026 instance: id,
2027 index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
2028 };
2029 self.enter_guest_sync_call(callee_async, callee)?;
2030
2031 self.promote_current_lazy_lends(id, start, lenders_end)?;
2034 }
2035
2036 debug_assert_eq!(
2039 *self.vm_store_context_mut().lazy_lenders_len_mut(),
2040 lazy_lenders_end
2041 );
2042 *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
2043
2044 *self.vm_store_context_mut().component_context_mut() = current_context;
2046
2047 Ok(self
2048 .concurrent_state_mut_without_forcing_current_thread()
2049 .unforced_current_thread)
2050 }
2051
2052 fn promote_current_lazy_lends(
2058 &mut self,
2059 id: ComponentInstanceId,
2060 start: usize,
2061 end: usize,
2062 ) -> Result<()> {
2063 if start == end {
2064 return Ok(());
2065 }
2066
2067 let scope = match self
2068 .concurrent_state_mut_without_forcing_current_thread()
2069 .unforced_current_thread
2070 .guest_task()
2071 {
2072 Some(task) => VMScope::id(task.rep()),
2073 None => bail_bug!("replayed deferred frame is not a guest thread"),
2074 };
2075 let instance = Instance::from_wasmtime(self, id);
2076 self.promote_lazy_lends(instance, scope, start, end)
2077 }
2078
2079 fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2080 match self.current_thread()?.guest() {
2081 Some(id) => Ok(*id),
2082 None => bail_bug!("current thread is not a guest thread"),
2083 }
2084 }
2085
2086 pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2090 match self.current_thread()? {
2091 CurrentThread::Host(id) => Ok(Some(id)),
2092 CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2093 _ => bail_bug!("current thread is not a host thread"),
2094 }
2095 }
2096
2097 fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2100 Ok(self
2101 .concurrent_state_mut()?
2102 .materialize_current_host_task_id()?)
2103 }
2104
2105 fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2106 log::trace!("enter sync-typed call {callee:?}");
2107 let state = self.instance_state(callee).concurrent_state();
2108 let old_do_not_suspend = state.do_not_suspend;
2109 state.do_not_suspend = true;
2110
2111 let thread = self.current_guest_thread()?;
2112 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2113 if thread.old_do_not_suspend.is_some() {
2114 bail_bug!("current thread already has `old_do_not_suspend` value");
2115 }
2116
2117 thread.old_do_not_suspend = Some(old_do_not_suspend);
2118
2119 Ok(())
2120 }
2121
2122 fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2123 log::trace!("exit sync-typed call {callee:?}");
2124 let thread = self.current_guest_thread()?;
2125 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2126 let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2127 bail_bug!("current thread missing `old_do_not_suspend` value");
2128 };
2129 let state = self.instance_state(callee).concurrent_state();
2130 state.do_not_suspend = old_do_not_suspend;
2131 Ok(())
2132 }
2133
2134 pub(crate) fn enter_guest_sync_call(
2146 &mut self,
2147 callee_async_typed: bool,
2148 callee: RuntimeInstance,
2149 ) -> Result<()> {
2150 log::trace!("enter sync-lifted call {callee:?}");
2151 if !self.concurrency_support() {
2152 return self.enter_call_not_concurrent();
2153 }
2154
2155 let state = self.concurrent_state_mut()?;
2159 let item = state.next_switch_item.take();
2160 state.saved_next_switch_items.push(item);
2161
2162 let thread = self.current_thread()?;
2163 let caller = if let Some(thread) = thread.guest() {
2164 Caller::Guest { thread: *thread }
2165 } else {
2166 Caller::Host {
2167 tx: None,
2168 caller: self.materialize_host_task_id()?,
2169 }
2170 };
2171
2172 let state = self.concurrent_state_mut()?;
2173 let guest_thread = GuestTask::new(
2174 state,
2175 Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2176 LiftResult {
2177 lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2178 ty: TypeTupleIndex::reserved_value(),
2179 memory: None,
2180 string_encoding: StringEncoding::Utf8,
2181 },
2182 caller,
2183 None,
2184 callee,
2185 callee_async_typed,
2186 false,
2187 )?;
2188
2189 Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2190 guest_thread.thread,
2191 self,
2192 callee.index,
2193 )?;
2194 self.set_thread(guest_thread)?;
2195
2196 if !callee_async_typed {
2197 self.enter_sync_call(callee)?;
2198 }
2199
2200 Ok(())
2201 }
2202
2203 pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2211 if !self.concurrency_support() {
2212 return Ok(self.exit_call_not_concurrent());
2213 }
2214
2215 let thread = match self.current_thread()?.guest() {
2216 Some(t) => *t,
2217 None => bail_bug!("expected task when exiting"),
2218 };
2219 let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2220 let instance = task.instance;
2221
2222 let caller = match &task.caller {
2223 &Caller::Guest { thread } => thread.into(),
2224 &Caller::Host { caller, .. } => caller
2225 .map(CurrentThread::Host)
2226 .unwrap_or(CurrentThread::None),
2227 };
2228 task.lift_result = None;
2229 task.exited = true;
2230 let async_typed = task.async_typed;
2231
2232 if !async_typed {
2233 self.exit_sync_call(instance)?;
2234 }
2235
2236 self.set_thread(caller)?;
2237
2238 log::trace!("exit sync-lifted call {instance:?}");
2239
2240 if async_typed {
2241 self.switch_or_trap_if_may_not_suspend(instance)?;
2246 }
2247
2248 self.cleanup_thread(thread, instance)?;
2249
2250 self.concurrent_state_mut()?
2251 .decrement_task_ref_count(thread.task)?;
2252
2253 let state = self.concurrent_state_mut()?;
2254 let Some(item) = state.saved_next_switch_items.pop() else {
2255 bail_bug!("unable to pop from `saved_next_switch_items`");
2256 };
2257 if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2258 state.push_high_priority(item);
2261 bail_bug!("`next_switch_item` unexpectedly already set");
2262 }
2263
2264 Ok(())
2265 }
2266
2267 pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2274 if !self.concurrency_support() {
2275 self.enter_call_not_concurrent()?;
2276 return Ok(None);
2277 }
2278 let caller = self.current_guest_thread()?;
2279 log::trace!("new deferred host task with caller {caller:?}");
2280
2281 self.set_thread(CurrentThread::DeferredHost(caller))?;
2282 let state = self.concurrent_state_mut()?;
2283 debug_assert!(state.deferred_host_call_context.is_none());
2284 state.deferred_host_call_context = Some(CallContext::default());
2285 state.debug_assert_deferred_host_invariant();
2286 Ok(Some(caller))
2287 }
2288
2289 pub(crate) fn host_task_delete(
2296 &mut self,
2297 original_task: EnteredHostTask,
2298 materialized_task: Option<TableId<HostTask>>,
2299 ) -> Result<()> {
2300 match original_task {
2301 Some(caller) => {
2302 self.set_thread(caller)?;
2303 if materialized_task.is_none() {
2304 let state = self.concurrent_state_mut()?;
2305 let context = state
2306 .deferred_host_call_context
2307 .take()
2308 .expect("deferred host call context should be present");
2309 debug_assert!(context.is_empty());
2310 state.debug_assert_deferred_host_invariant();
2311 }
2312 log::trace!(
2313 "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2314 );
2315 if let Some(task) = materialized_task {
2316 self.concurrent_state_mut()?.delete_host_task(task)?;
2317 }
2318 }
2319 None => {
2320 debug_assert!(materialized_task.is_none());
2321 self.exit_call_not_concurrent();
2322 }
2323 }
2324 Ok(())
2325 }
2326
2327 fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2330 self.component_instance_mut(instance.instance)
2331 .instance_state(instance.index)
2332 }
2333
2334 fn instance_state_and_handle_counter(
2337 &mut self,
2338 instance: RuntimeInstance,
2339 ) -> (&mut InstanceState, &mut HandleCounter) {
2340 let (component, counter) = self
2341 .store_data_mut()
2342 .component_instance_and_handle_counter_mut(instance.instance);
2343 (component.instance_state(instance.index), counter)
2344 }
2345
2346 fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2348 let (component, counter) = self
2349 .store_data_mut()
2350 .component_instance_and_handle_counter_mut(instance.instance);
2351 component.handle_table_insertion(instance.index, counter)
2352 }
2353
2354 pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2360 let thread = thread.into();
2361 let state = self.concurrent_state_mut()?;
2362 state.debug_assert_deferred_host_invariant();
2363 let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2364
2365 state.handle_thread_switch(old_thread, thread)?;
2366
2367 if let Some(old_thread) = old_thread.guest() {
2375 let old_context = *self.vm_store_context_mut().component_context_mut();
2376 self.concurrent_state_mut()?
2377 .get_mut(old_thread.thread)?
2378 .context = old_context;
2379 }
2380 if cfg!(debug_assertions) {
2381 *self.vm_store_context_mut().component_context_mut() =
2382 [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2383 }
2384 if let Some(thread) = thread.guest() {
2385 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2386 let context = thread.context;
2387 if cfg!(debug_assertions) {
2388 thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2389 }
2390 *self.vm_store_context_mut().component_context_mut() = context;
2391 }
2392
2393 *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2395 VMLazyThread::none()
2396 } else {
2397 VMLazyThread::forced()
2398 };
2399
2400 Ok(old_thread)
2401 }
2402
2403 fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2405 if self.switch_if_may_not_suspend(instance)? {
2406 Ok(())
2407 } else {
2408 Err(Trap::CannotBlockSyncTask.into())
2409 }
2410 }
2411
2412 fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2416 self.concurrent_state_mut()?;
2420
2421 Ok(!self.concurrency_support()
2422 || !self
2423 .instance_state(instance)
2424 .concurrent_state()
2425 .do_not_suspend
2426 || self
2427 .concurrent_state_mut()?
2428 .promote_instance_local_thread_work_item(instance)?)
2429 }
2430
2431 fn enter_instance(&mut self, instance: RuntimeInstance) {
2435 log::trace!("enter {instance:?}");
2436 self.instance_state(instance)
2437 .concurrent_state()
2438 .do_not_enter = true;
2439 }
2440
2441 fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2445 log::trace!("exit {instance:?}");
2446 self.instance_state(instance)
2447 .concurrent_state()
2448 .do_not_enter = false;
2449 self.partition_pending(instance)
2450 }
2451
2452 fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2460 let mut any_ready = false;
2461 for (thread, kind) in
2462 mem::take(&mut self.instance_state(instance).concurrent_state().pending)
2463 {
2464 let call = GuestCall { thread, kind };
2465 if call.is_ready(self)? {
2466 any_ready = true;
2467 self.concurrent_state_mut()?
2468 .push_high_priority(WorkItem::GuestCall { instance, call });
2469 } else {
2470 self.instance_state(instance)
2471 .concurrent_state()
2472 .pending
2473 .insert(call.thread, call.kind);
2474 }
2475 }
2476
2477 if any_ready {
2478 if let Some(waker) = self
2479 .concurrent_state_mut()?
2480 .ready_for_concurrent_call_waker
2481 .take()
2482 {
2483 waker.wake();
2484 }
2485 }
2486
2487 Ok(())
2488 }
2489
2490 pub(crate) fn backpressure_modify(
2492 &mut self,
2493 caller_instance: RuntimeInstance,
2494 modify: impl FnOnce(u16) -> Option<u16>,
2495 ) -> Result<()> {
2496 let state = self.instance_state(caller_instance).concurrent_state();
2497 let old = state.backpressure;
2498 let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2499 state.backpressure = new;
2500
2501 log::trace!("backpressure update for {caller_instance:?} from {old} to {new}");
2502
2503 if old > 0 && new == 0 {
2504 self.partition_pending(caller_instance)?;
2507 }
2508
2509 Ok(())
2510 }
2511
2512 fn resume_fiber(
2515 &mut self,
2516 fiber: StoreFiber<'static>,
2517 ) -> impl Future<Output = Result<()>> + '_ {
2518 let mut resuming = ResumingFiber {
2519 store: self,
2520 fiber: Some(fiber),
2521 };
2522 async move {
2523 let old_thread = resuming.store.current_thread()?;
2524 log::trace!("resume_fiber: save current thread {old_thread:?}");
2525
2526 let fiber = resuming.fiber.take().unwrap();
2527 resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2528
2529 resuming.store.set_thread(old_thread)?;
2530
2531 let state = resuming.store.concurrent_state_mut()?;
2532
2533 if let Some(ot) = old_thread.guest() {
2534 state.set_thread_running(ot.thread)?;
2535 }
2536 log::trace!("resume_fiber: restore current thread {old_thread:?}");
2537
2538 if resuming.fiber.is_some() {
2539 log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2540 let reason = match state.suspend_reason.take() {
2542 Some(r) => r,
2543 None => bail_bug!("suspend reason missing when resuming fiber"),
2544 };
2545 match reason {
2546 SuspendReason::NeedWork => {
2547 if state.worker.is_none() {
2548 state.worker = resuming.fiber.take();
2549 }
2550 }
2551 SuspendReason::Yielding { thread } => {
2552 let slot = &mut state.get_mut(thread.thread)?.state;
2553 slot.check_no_fiber()?;
2554 *slot = GuestThreadState::Ready {
2555 fiber: resuming.fiber.take().unwrap(),
2556 };
2557 let instance = state.get_mut(thread.task)?.instance;
2558 state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2559 }
2560 SuspendReason::ExplicitlySuspending { thread } => {
2561 let slot = &mut state.get_mut(thread.thread)?.state;
2562 slot.check_no_fiber()?;
2563 *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2564 }
2565 SuspendReason::Waiting { set, thread } => {
2566 match state.get_mut(set)?.waiting.entry(thread) {
2567 Entry::Vacant(entry) => {
2568 entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2569 }
2570 Entry::Occupied(_) => {
2571 bail_bug!("entry unexpectedly already exists for {thread:?}");
2572 }
2573 }
2574 }
2575 SuspendReason::YieldingToSubtask { thread } => {
2576 let instance = state.get_mut(thread.task)?.instance;
2585 if state.next_switch_item.is_some() {
2586 bail_bug!(
2589 "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2590 a thread wanted to wait on a subtask"
2591 );
2592 }
2593 let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2594 instance,
2595 thread,
2596 fiber: resuming.fiber.take().unwrap(),
2597 });
2598 log::trace!("set next switch item to {item:?}");
2599 }
2600 };
2601 } else {
2602 log::trace!("resume_fiber: fiber has exited");
2603 }
2604
2605 Ok(())
2606 }
2607 }
2608
2609 fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2615 log::trace!("suspend fiber: {reason:?}");
2616
2617 if !self.fiber_async_state_mut().can_block() {
2620 return Err(format_err!("future dropped"));
2621 }
2622
2623 let state = self.concurrent_state_mut()?;
2624
2625 let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2632 SuspendReason::Yielding { .. }
2633 | SuspendReason::Waiting { .. }
2634 | SuspendReason::ExplicitlySuspending { .. } => {
2635 if state.switch_item.is_none() {
2638 state.take_next_switch_item()?;
2639 }
2640
2641 (true, false)
2642 }
2643 SuspendReason::YieldingToSubtask { .. } => (true, true),
2644 SuspendReason::NeedWork => (false, false),
2645 };
2646
2647 let old_next_switch_item = if save_and_restore_next_switch_item {
2648 Some(state.save_next_switch_item()?)
2649 } else {
2650 None
2651 };
2652
2653 let old_guest_thread = if save_and_restore_thread {
2654 self.current_thread()?
2655 } else {
2656 CurrentThread::None
2657 };
2658
2659 let waiting_set = match &reason {
2660 SuspendReason::Waiting { set, .. } => Some(*set),
2661 _ => None,
2662 };
2663
2664 let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2665 if suspend_reason.is_some() {
2666 bail_bug!("suspend reason unexpectedly already set for store");
2667 }
2668 *suspend_reason = Some(reason);
2669
2670 if let Some(set) = waiting_set {
2673 self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2674 }
2675
2676 self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2677
2678 if let Some(set) = waiting_set {
2679 self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2680 }
2681
2682 if save_and_restore_thread {
2683 self.set_thread(old_guest_thread)?;
2684 }
2685
2686 if let Some(item) = old_next_switch_item {
2687 let state = self.concurrent_state_mut()?;
2688 state.restore_next_switch_item(item)?;
2689 }
2690
2691 Ok(())
2692 }
2693
2694 fn wait_for_event(
2695 &mut self,
2696 caller_instance: RuntimeInstance,
2697 waitable: Waitable,
2698 ) -> Result<()> {
2699 let caller = self.current_guest_thread()?;
2700 let state = self.concurrent_state_mut()?;
2701
2702 waitable.trap_if_in_waitable_set(state)?;
2703
2704 let set = state.get_mut(caller.thread)?.sync_call_set;
2705 waitable.join(state, Some(set))?;
2706
2707 self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2708
2709 self.suspend(SuspendReason::Waiting {
2710 set,
2711 thread: caller,
2712 })?;
2713 let state = self.concurrent_state_mut()?;
2714
2715 waitable.join(state, None)
2716 }
2717
2718 fn cleanup_thread(
2740 &mut self,
2741 guest_thread: QualifiedThreadId,
2742 runtime_instance: RuntimeInstance,
2743 ) -> Result<()> {
2744 let state = self.concurrent_state_mut()?;
2745 state.take_next_switch_item()?;
2748 let thread_data = state.get_mut(guest_thread.thread)?;
2749 let sync_call_set = thread_data.sync_call_set;
2750 if let Some(guest_id) = thread_data.instance_rep {
2751 self.instance_state(runtime_instance)
2752 .thread_handle_table()
2753 .guest_thread_remove(guest_id)?;
2754 }
2755 let state = self.concurrent_state_mut()?;
2756
2757 for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2759 if let Some(Event::Subtask {
2760 status: Status::Returned | Status::ReturnCancelled,
2761 }) = waitable.common(state)?.event
2762 {
2763 waitable.delete_from(state)?;
2764 }
2765 }
2766
2767 state.delete(guest_thread.thread)?;
2768 state.delete(sync_call_set)?;
2769 let task = state.get_mut(guest_thread.task)?;
2770 task.threads.remove(&guest_thread.thread);
2771
2772 if task.threads.is_empty() && !task.returned_or_cancelled() {
2773 bail!(Trap::NoAsyncResult);
2774 }
2775
2776 if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2777 task.decremented_interesting_task_count = true;
2778
2779 debug_assert!(state.interesting_tasks > 0);
2780 state.interesting_tasks -= 1;
2781 if state.interesting_tasks == 0
2782 && let Some(waker) = state.interesting_tasks_empty_waker.take()
2783 {
2784 waker.wake();
2785 }
2786 }
2787
2788 state.decrement_task_ref_count(guest_thread.task)?;
2789
2790 Ok(())
2791 }
2792
2793 fn cancel_guest_subtask_without_lowered_parameters(
2802 &mut self,
2803 caller_instance: RuntimeInstance,
2804 guest_task: TableId<GuestTask>,
2805 ) -> Result<()> {
2806 let concurrent_state = self.concurrent_state_mut()?;
2807 let task = concurrent_state.get_mut(guest_task)?;
2808 assert!(!task.already_lowered_parameters());
2809 task.lower_params = None;
2813 task.lift_result = None;
2814 task.exited = true;
2815 let instance = task.instance;
2816
2817 assert_eq!(1, task.threads.len());
2824 let thread = *task.threads.iter().next().unwrap();
2825 let next_switch_item = concurrent_state.save_next_switch_item()?;
2826 self.cleanup_thread(
2827 QualifiedThreadId {
2828 task: guest_task,
2829 thread,
2830 },
2831 caller_instance,
2832 )?;
2833 self.concurrent_state_mut()?
2834 .restore_next_switch_item(next_switch_item)?;
2835
2836 let pending = &mut self.instance_state(instance).concurrent_state().pending;
2838 let pending_count = pending.len();
2839 let mut decrement = 0;
2840 let mut consume = |kind: &GuestCallKind| {
2841 if let GuestCallKind::StartImplicit {
2842 was_pending: true, ..
2843 } = kind
2844 {
2845 decrement += 1;
2846 }
2847 };
2848 pending.retain(|thread, kind| {
2849 if thread.task == guest_task {
2850 consume(kind);
2851 false
2852 } else {
2853 true
2854 }
2855 });
2856
2857 if pending.len() == pending_count {
2858 let mut matches = |item: &WorkItem| match item {
2861 WorkItem::ResumeThread { thread, .. } => thread.task == guest_task,
2862 WorkItem::GuestCall {
2863 call: GuestCall { thread, kind },
2864 ..
2865 } => {
2866 if thread.task == guest_task {
2867 consume(kind);
2868 true
2869 } else {
2870 false
2871 }
2872 }
2873 _ => false,
2874 };
2875
2876 let mut remove = |queue: &mut VecDeque<WorkItem>| {
2877 let count = queue.len();
2878 queue.retain(|item| !matches(item));
2879 count != queue.len()
2880 };
2881
2882 let state = self.concurrent_state_mut()?;
2883 let mut found = remove(&mut state.high_priority);
2884 found |= remove(&mut state.low_priority);
2885
2886 for item in [&mut state.switch_item, &mut state.next_switch_item] {
2887 if matches!(item, Some(item) if matches(item)) {
2888 found = true;
2889 *item = None;
2890 }
2891 }
2892
2893 if !found {
2894 bail!(Trap::SubtaskCancelAfterTerminal);
2895 }
2896 }
2897
2898 self.decrement_pending_start_count(instance, decrement)?;
2899
2900 Ok(())
2901 }
2902
2903 fn decrement_pending_start_count(
2904 &mut self,
2905 instance: RuntimeInstance,
2906 decrement: usize,
2907 ) -> Result<()> {
2908 if decrement > 0 {
2909 let count = &mut self
2910 .instance_state(instance)
2911 .concurrent_state()
2912 .pending_start_count;
2913
2914 if *count < decrement {
2915 bail_bug!("`ConcurrentInstantState::pending_start_count` underflow");
2916 }
2917
2918 *count -= decrement;
2919
2920 self.partition_pending(instance)?;
2922 }
2923
2924 Ok(())
2925 }
2926
2927 pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2930 if !self.concurrency_support() {
2931 return Ok(self
2932 .current_scope_id_not_concurrent()?
2933 .map(|id| CurrentScope::Id(VMScope::id(id))));
2934 }
2935
2936 Ok(match self.current_thread()? {
2937 CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2938 CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2939 CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2940 CurrentThread::None => return Ok(None),
2941 })
2942 }
2943
2944 pub(crate) fn queue_task(
2945 &mut self,
2946 task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2947 ) -> Result<()> {
2948 self.concurrent_state_mut()?
2949 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2950 Ok(())
2951 }
2952
2953 fn any_may_not_suspend(&mut self) -> Result<bool> {
2962 Ok(self
2970 .concurrent_state_mut()?
2971 .table
2972 .get_mut()
2973 .iter_mut()
2974 .filter_map(|(_, entry)| {
2975 if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2976 Some(task.instance)
2977 } else {
2978 None
2979 }
2980 })
2981 .collect::<Vec<_>>()
2982 .into_iter()
2983 .any(|instance| {
2984 self.instance_state(instance)
2985 .concurrent_state()
2986 .do_not_suspend
2987 }))
2988 }
2989}
2990
2991impl Instance {
2992 fn get_event(
2995 self,
2996 store: &mut StoreOpaque,
2997 guest_task: TableId<GuestTask>,
2998 set: Option<TableId<WaitableSet>>,
2999 cancellable: bool,
3000 ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
3001 let state = store.concurrent_state_mut()?;
3002
3003 let task = state.get_mut(guest_task)?;
3004 let event = &mut task.event;
3005 if let Some(ev) = event
3006 && (cancellable || !matches!(ev, Event::Cancelled))
3007 {
3008 log::trace!("deliver event {ev:?} to {guest_task:?}");
3009
3010 if matches!(ev, Event::Cancelled) {
3011 task.cancel_request_delivered = true;
3012 }
3013
3014 let ev = *ev;
3015 *event = None;
3016 return Ok(Some((ev, None)));
3017 }
3018
3019 let set = match set {
3020 Some(set) => set,
3021 None => return Ok(None),
3022 };
3023 let waitable = match state.get_mut(set)?.ready.pop_first() {
3024 Some(v) => v,
3025 None => return Ok(None),
3026 };
3027
3028 let common = waitable.common(state)?;
3029 let handle = match common.handle {
3030 Some(h) => h,
3031 None => bail_bug!("handle not set when delivering event"),
3032 };
3033 let event = match common.event.take() {
3034 Some(e) => e,
3035 None => bail_bug!("event not set when delivering event"),
3036 };
3037
3038 log::trace!(
3039 "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
3040 );
3041
3042 waitable.on_delivery(store, self, event)?;
3043
3044 Ok(Some((event, Some((waitable, handle)))))
3045 }
3046
3047 fn handle_callback_code(
3054 self,
3055 store: &mut dyn VMStore,
3056 guest_thread: QualifiedThreadId,
3057 runtime_instance: RuntimeInstance,
3058 call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
3059 ) -> Result<()> {
3060 let old_thread = store.set_thread(guest_thread)?;
3061 log::trace!(
3062 "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
3063 );
3064
3065 store.enter_instance(runtime_instance);
3066
3067 let mut code = call(store)?;
3068
3069 if matches!(
3075 unpack_callback_code(code).0,
3076 callback_code::YIELD | callback_code::WAIT
3077 ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
3078 {
3079 let (unpacked, set) = unpack_callback_code(code);
3080 log::trace!(
3081 "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
3082 cancellation; delivering it via callback"
3083 );
3084 code = invoke_callback(store, guest_thread.task, event, 0)?;
3085 }
3086
3087 store.exit_instance(runtime_instance)?;
3088
3089 store.set_thread(old_thread)?;
3090 log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
3091
3092 let store = store.store_opaque_mut();
3093 let (code, set) = unpack_callback_code(code);
3094
3095 log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
3096
3097 let state = store.concurrent_state_mut()?;
3098
3099 state.take_next_switch_item()?;
3100
3101 let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
3102 let set = store
3103 .instance_state(runtime_instance)
3104 .handle_table()
3105 .waitable_set_rep(handle)?;
3106
3107 Ok(TableId::<WaitableSet>::new(set))
3108 };
3109
3110 match code {
3111 callback_code::EXIT => {
3112 log::trace!("implicit thread {guest_thread:?} completed");
3113 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3114 task.exited = true;
3115 task.callback = None;
3116
3117 store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
3122
3123 store.cleanup_thread(guest_thread, runtime_instance)?;
3124 }
3125 callback_code::YIELD => {
3126 let old = state
3129 .get_mut(guest_thread.thread)?
3130 .wake_on_cancel
3131 .replace(WakeOnCancel::Yielding);
3132 if !old.is_none() {
3133 bail_bug!("thread unexpectedly had wake_on_cancel set");
3134 }
3135
3136 let call = GuestCall {
3143 thread: guest_thread,
3144 kind: GuestCallKind::DeliverEvent {
3145 instance: self,
3146 set: None,
3147 },
3148 };
3149 state.push_low_priority(WorkItem::GuestCall {
3152 instance: runtime_instance,
3153 call,
3154 });
3155 }
3156 callback_code::WAIT => {
3157 let set = get_set(store, set)?;
3158 self.wait_with_callback(store, guest_thread, set)?;
3159 }
3160 _ => bail!(Trap::UnsupportedCallbackCode),
3161 }
3162
3163 Ok(())
3164 }
3165
3166 fn wait_with_callback(
3172 self,
3173 store: &mut StoreOpaque,
3174 guest_thread: QualifiedThreadId,
3175 set: TableId<WaitableSet>,
3176 ) -> Result<()> {
3177 let state = store.concurrent_state_mut()?;
3178 state.get_mut(set)?.num_waiting += 1;
3181
3182 if state.get_mut(guest_thread.task)?.event.is_some()
3183 || !state.get_mut(set)?.ready.is_empty()
3184 {
3185 let instance = state.get_mut(guest_thread.task)?.instance;
3187 state.push_high_priority(WorkItem::GuestCall {
3188 instance,
3189 call: GuestCall {
3190 thread: guest_thread,
3191 kind: GuestCallKind::DeliverEvent {
3192 instance: self,
3193 set: Some(set),
3194 },
3195 },
3196 });
3197 return Ok(());
3198 }
3199
3200 let instance = store
3206 .concurrent_state_mut()?
3207 .get_mut(guest_thread.task)?
3208 .instance;
3209 store.switch_or_trap_if_may_not_suspend(instance)?;
3210
3211 let state = store.concurrent_state_mut()?;
3217 let old = state
3218 .get_mut(guest_thread.thread)?
3219 .wake_on_cancel
3220 .replace(WakeOnCancel::Waiting(set));
3221 if !old.is_none() {
3222 bail_bug!("thread unexpectedly had wake_on_cancel set");
3223 }
3224 let old = state
3225 .get_mut(set)?
3226 .waiting
3227 .insert(guest_thread, WaitMode::Callback(self));
3228 if !old.is_none() {
3229 bail_bug!("set's waiting set already had this thread registered");
3230 }
3231 Ok(())
3232 }
3233
3234 unsafe fn stage_call<T: 'static>(
3241 self,
3242 mut store: StoreContextMut<T>,
3243 guest_thread: QualifiedThreadId,
3244 callee: SendSyncPtr<VMFuncRef>,
3245 param_count: usize,
3246 result_count: usize,
3247 async_: bool,
3248 callback: Option<SendSyncPtr<VMFuncRef>>,
3249 post_return: Option<SendSyncPtr<VMFuncRef>>,
3250 host_caller: bool,
3251 ) -> Result<()> {
3252 unsafe fn make_call<T: 'static>(
3267 store: StoreContextMut<T>,
3268 guest_thread: QualifiedThreadId,
3269 callee: SendSyncPtr<VMFuncRef>,
3270 param_count: usize,
3271 result_count: usize,
3272 ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3273 + Send
3274 + Sync
3275 + 'static
3276 + use<T> {
3277 let token = StoreToken::new(store);
3278 move |store: &mut dyn VMStore| {
3279 let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3280
3281 store
3282 .concurrent_state_mut()?
3283 .set_thread_running(guest_thread.thread)?;
3284 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3285 let lower = match task.lower_params.take() {
3286 Some(l) => l,
3287 None => bail_bug!("lower_params missing"),
3288 };
3289
3290 lower(store, &mut storage[..param_count])?;
3291
3292 let mut store = token.as_context_mut(store);
3293
3294 unsafe {
3297 crate::Func::call_unchecked_raw(
3298 &mut store,
3299 callee.as_non_null(),
3300 NonNull::new(
3301 &mut storage[..param_count.max(result_count)]
3302 as *mut [MaybeUninit<ValRaw>] as _,
3303 )
3304 .unwrap(),
3305 UncaughtException::Trap,
3306 )?;
3307 }
3308
3309 Ok(storage)
3310 }
3311 }
3312
3313 let call = unsafe {
3317 make_call(
3318 store.as_context_mut(),
3319 guest_thread,
3320 callee,
3321 param_count,
3322 result_count,
3323 )
3324 };
3325
3326 let callee_instance = store
3327 .0
3328 .concurrent_state_mut()?
3329 .get_mut(guest_thread.task)?
3330 .instance;
3331
3332 let fun = if callback.is_some() {
3333 assert!(async_);
3334
3335 Box::new(move |store: &mut dyn VMStore| {
3336 self.add_guest_thread_to_instance_table(
3337 guest_thread.thread,
3338 store,
3339 callee_instance.index,
3340 )?;
3341 self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3342 let storage = call(store)?;
3343
3344 Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3347 })
3348 }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3349 } else {
3350 let token = StoreToken::new(store.as_context_mut());
3351 Box::new(move |store: &mut dyn VMStore| {
3352 self.add_guest_thread_to_instance_table(
3353 guest_thread.thread,
3354 store,
3355 callee_instance.index,
3356 )?;
3357 let old_thread = store.set_thread(guest_thread)?;
3358 log::trace!(
3359 "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3360 );
3361 let flags = self.id().get(store).instance_flags(callee_instance.index);
3362
3363 let callee_async_typed = store
3364 .concurrent_state_mut()?
3365 .get_mut(guest_thread.task)?
3366 .async_typed;
3367
3368 if !async_ && callee_async_typed {
3372 store.enter_instance(callee_instance);
3373 }
3374
3375 if !callee_async_typed {
3376 store.enter_sync_call(callee_instance)?;
3377 }
3378
3379 let storage = call(store)?;
3386
3387 if !callee_async_typed {
3388 store.exit_sync_call(callee_instance)?;
3389 }
3390
3391 if !async_ {
3392 if callee_async_typed {
3398 store.exit_instance(callee_instance)?;
3399 }
3400
3401 let lift = {
3402 let state = store.concurrent_state_mut()?;
3403 if !state.get_mut(guest_thread.task)?.result.is_none() {
3404 bail_bug!("task has already produced a result");
3405 }
3406
3407 match state.get_mut(guest_thread.task)?.lift_result.take() {
3408 Some(lift) => lift,
3409 None => bail_bug!("lift_result field is missing"),
3410 }
3411 };
3412
3413 let result = (lift.lift)(store, unsafe {
3416 mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3417 &storage[..result_count],
3418 )
3419 })?;
3420
3421 let post_return_arg = match result_count {
3422 0 => ValRaw::i32(0),
3423 1 => unsafe { storage[0].assume_init() },
3426 _ => unreachable!(),
3427 };
3428
3429 unsafe {
3430 call_post_return(
3431 token.as_context_mut(store),
3432 post_return.map(|v| v.as_non_null()),
3433 post_return_arg,
3434 flags,
3435 )?;
3436 }
3437
3438 self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3439 }
3440
3441 store.set_thread(old_thread)?;
3442
3443 store
3444 .concurrent_state_mut()?
3445 .get_mut(guest_thread.task)?
3446 .exited = true;
3447
3448 log::trace!(
3449 "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3450 );
3451
3452 if callee_async_typed {
3453 store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3458 }
3459
3460 store.cleanup_thread(guest_thread, callee_instance)?;
3462 Ok(())
3463 })
3464 };
3465
3466 store.0.concurrent_state_mut()?.push_work_item(
3467 WorkItem::GuestCall {
3468 instance: callee_instance,
3469 call: GuestCall {
3470 thread: guest_thread,
3471 kind: GuestCallKind::StartImplicit {
3472 closure: fun,
3473 was_pending: false,
3474 },
3475 },
3476 },
3477 if host_caller {
3478 Priority::Low
3484 } else {
3485 Priority::Switch
3486 },
3487 )?;
3488
3489 Ok(())
3490 }
3491
3492 unsafe fn prepare_call<T: 'static>(
3505 self,
3506 mut store: StoreContextMut<T>,
3507 start: NonNull<VMFuncRef>,
3508 return_: NonNull<VMFuncRef>,
3509 caller_instance: RuntimeComponentInstanceIndex,
3510 callee_instance: RuntimeComponentInstanceIndex,
3511 task_return_type: TypeTupleIndex,
3512 callee_async_typed: bool,
3513 memory: *mut VMMemoryDefinition,
3514 string_encoding: StringEncoding,
3515 caller_info: CallerInfo,
3516 ) -> Result<()> {
3517 enum ResultInfo {
3518 Heap { results: u32 },
3519 Stack { result_count: u32 },
3520 }
3521
3522 let result_info = match &caller_info {
3523 CallerInfo::Async {
3524 has_result: true,
3525 params,
3526 } => ResultInfo::Heap {
3527 results: match params.last() {
3529 Some(r) => r.get_u32(),
3530 None => bail_bug!("retptr missing"),
3531 },
3532 },
3533 CallerInfo::Async {
3534 has_result: false, ..
3535 } => ResultInfo::Stack { result_count: 0 },
3536 CallerInfo::Sync {
3537 result_count,
3538 params,
3539 } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3540 results: match params.last() {
3542 Some(r) => r.get_u32(),
3543 None => bail_bug!("arg ptr missing"),
3544 },
3545 },
3546 CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3547 result_count: *result_count,
3548 },
3549 };
3550
3551 let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3552
3553 let start = SendSyncPtr::new(start);
3557 let return_ = SendSyncPtr::new(return_);
3558 let token = StoreToken::new(store.as_context_mut());
3559 let old_thread = store.0.current_guest_thread()?;
3560
3561 let state = store.0.concurrent_state_mut()?;
3562
3563 debug_assert_eq!(
3564 state.get_mut(old_thread.task)?.instance,
3565 self.runtime_instance(caller_instance)
3566 );
3567
3568 let guest_thread = GuestTask::new(
3569 state,
3570 Box::new(move |store, dst| {
3571 let mut store = token.as_context_mut(store);
3572 assert!(dst.len() <= MAX_FLAT_PARAMS);
3573 let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3575 let count = match caller_info {
3576 CallerInfo::Async { params, has_result } => {
3580 let params = ¶ms[..params.len() - usize::from(has_result)];
3581 for (param, src) in params.iter().zip(&mut src) {
3582 src.write(*param);
3583 }
3584 params.len()
3585 }
3586
3587 CallerInfo::Sync { params, .. } => {
3589 for (param, src) in params.iter().zip(&mut src) {
3590 src.write(*param);
3591 }
3592 params.len()
3593 }
3594 };
3595 unsafe {
3602 crate::Func::call_unchecked_raw(
3603 &mut store,
3604 start.as_non_null(),
3605 NonNull::new(
3606 &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3607 )
3608 .unwrap(),
3609 UncaughtException::Trap,
3610 )?;
3611 }
3612 dst.copy_from_slice(&src[..dst.len()]);
3613 let task = store.0.current_guest_thread()?.task;
3614 let state = store.0.concurrent_state_mut()?;
3615 Waitable::Guest(task).set_event(
3616 state,
3617 Some(Event::Subtask {
3618 status: Status::Started,
3619 }),
3620 )?;
3621 Ok(())
3622 }),
3623 LiftResult {
3624 lift: Box::new(move |store, src| {
3625 let mut store = token.as_context_mut(store);
3628 let mut my_src = src.to_owned(); if let ResultInfo::Heap { results } = &result_info {
3630 my_src.push(ValRaw::u32(*results));
3631 }
3632
3633 unsafe {
3640 crate::Func::call_unchecked_raw(
3641 &mut store,
3642 return_.as_non_null(),
3643 my_src.as_mut_slice().into(),
3644 UncaughtException::Trap,
3645 )?;
3646 }
3647
3648 let thread = store.0.current_guest_thread()?;
3649 let state = store.0.concurrent_state_mut()?;
3650 if sync_caller {
3651 state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3652 if let ResultInfo::Stack { result_count } = &result_info {
3653 match result_count {
3654 0 => None,
3655 1 => Some(my_src[0]),
3656 _ => unreachable!(),
3657 }
3658 } else {
3659 None
3660 },
3661 );
3662 }
3663 Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3664 }),
3665 ty: task_return_type,
3666 memory: NonNull::new(memory).map(SendSyncPtr::new),
3667 string_encoding,
3668 },
3669 Caller::Guest { thread: old_thread },
3670 None,
3671 self.runtime_instance(callee_instance),
3672 callee_async_typed,
3673 false,
3676 )?;
3677
3678 store.0.set_thread(guest_thread)?;
3681 log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3682
3683 Ok(())
3684 }
3685
3686 unsafe fn call_callback<T>(
3691 self,
3692 mut store: StoreContextMut<T>,
3693 function: SendSyncPtr<VMFuncRef>,
3694 event: Event,
3695 handle: u32,
3696 ) -> Result<u32> {
3697 let (ordinal, result) = event.parts();
3698 let params = &mut [
3699 ValRaw::u32(ordinal),
3700 ValRaw::u32(handle),
3701 ValRaw::u32(result),
3702 ];
3703 unsafe {
3708 crate::Func::call_unchecked_raw(
3709 &mut store,
3710 function.as_non_null(),
3711 params.as_mut_slice().into(),
3712 UncaughtException::Trap,
3713 )?;
3714 }
3715 Ok(params[0].get_u32())
3716 }
3717
3718 unsafe fn start_call<T: 'static>(
3736 self,
3737 mut store: StoreContextMut<T>,
3738 callback: *mut VMFuncRef,
3739 post_return: *mut VMFuncRef,
3740 callee: NonNull<VMFuncRef>,
3741 param_count: u32,
3742 result_count: u32,
3743 flags: u32,
3744 storage: &mut [MaybeUninit<ValRaw>],
3745 ) -> Result<()> {
3746 let token = StoreToken::new(store.as_context_mut());
3747 let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3748 let guest_thread = store.0.current_guest_thread()?;
3749 let state = store.0.concurrent_state_mut()?;
3750
3751 if !state.event_loop_running {
3756 bail!(
3757 "async function started during an original synchronous \
3758 entrypoint; please use an async-style entrypoint instead"
3759 );
3760 }
3761
3762 let callee = SendSyncPtr::new(callee);
3763 let param_count = usize::try_from(param_count)?;
3764 assert!(param_count <= MAX_FLAT_PARAMS);
3765 let result_count = usize::try_from(result_count)?;
3766 assert!(result_count <= MAX_FLAT_RESULTS);
3767
3768 let task = state.get_mut(guest_thread.task)?;
3769 let callee_async_typed = task.async_typed;
3770 let callee_instance = task.instance;
3771
3772 task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3773
3774 if let Some(callback) = NonNull::new(callback) {
3775 let callback = SendSyncPtr::new(callback);
3779 task.callback = Some(Box::new(move |store, event, handle| {
3780 let store = token.as_context_mut(store);
3781 unsafe { self.call_callback::<T>(store, callback, event, handle) }
3782 }));
3783 }
3784
3785 let Caller::Guest { thread: caller } = &task.caller else {
3786 bail_bug!("start_call unexpectedly invoked for host->guest call");
3789 };
3790 let caller = *caller;
3791 let caller_instance = state.get_mut(caller.task)?.instance;
3792
3793 unsafe {
3795 self.stage_call(
3796 store.as_context_mut(),
3797 guest_thread,
3798 callee,
3799 param_count,
3800 result_count,
3801 (flags & START_FLAG_ASYNC_CALLEE) != 0,
3802 NonNull::new(callback).map(SendSyncPtr::new),
3803 NonNull::new(post_return).map(SendSyncPtr::new),
3804 false,
3805 )?;
3806 }
3807
3808 let old_do_not_suspend = if callee_async_typed {
3809 let state = store.0.instance_state(callee_instance).concurrent_state();
3816 let old_do_not_suspend = state.do_not_suspend;
3817 state.do_not_suspend = false;
3818 Some(old_do_not_suspend)
3819 } else {
3820 None
3821 };
3822
3823 let state = store.0.concurrent_state_mut()?;
3824
3825 let guest_waitable = Waitable::Guest(guest_thread.task);
3828 let old_set = guest_waitable.common(state)?.set;
3829 let set = state.get_mut(caller.thread)?.sync_call_set;
3830 guest_waitable.join(state, Some(set))?;
3831
3832 store.0.set_thread(CurrentThread::None)?;
3833
3834 let mut yielded = false;
3850 let (status, waitable) = loop {
3851 store.0.suspend(if yielded {
3852 SuspendReason::Waiting {
3853 set,
3854 thread: caller,
3855 }
3856 } else {
3857 yielded = true;
3858 SuspendReason::YieldingToSubtask { thread: caller }
3859 })?;
3860
3861 if let Some(old_do_not_suspend) = old_do_not_suspend {
3862 store
3863 .0
3864 .instance_state(callee_instance)
3865 .concurrent_state()
3866 .do_not_suspend = old_do_not_suspend;
3867 }
3868
3869 let state = store.0.concurrent_state_mut()?;
3870
3871 log::trace!("taking event for {:?}", guest_thread.task);
3872 let event = guest_waitable.take_event(state)?;
3873 let Some(Event::Subtask { status }) = event else {
3874 bail_bug!("subtasks should only get subtask events, got {event:?}")
3875 };
3876
3877 log::trace!("status {status:?} for {:?}", guest_thread.task);
3878
3879 if status == Status::Returned {
3880 break (status, None);
3882 } else if async_caller {
3883 let handle = store
3887 .0
3888 .handle_table_insertion(caller_instance)
3889 .subtask_guest(guest_thread.task.rep())?;
3890 store
3891 .0
3892 .concurrent_state_mut()?
3893 .get_mut(guest_thread.task)?
3894 .common
3895 .handle = Some(handle);
3896 break (status, Some(handle));
3897 } else {
3898 store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3902 }
3903 };
3904
3905 guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3906
3907 store.0.set_thread(caller)?;
3909
3910 store
3911 .0
3912 .concurrent_state_mut()?
3913 .set_thread_running(caller.thread)?;
3914
3915 log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3916
3917 if async_caller {
3918 let Some(slot) = storage.first_mut() else {
3919 bail_bug!("no storage for async call status");
3920 };
3921 *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3922 } else {
3923 let state = store.0.concurrent_state_mut()?;
3926 let task = state.get_mut(guest_thread.task)?;
3927 if let Some(Some(result)) = task.sync_result.take()? {
3928 storage[0] = MaybeUninit::new(result);
3929 }
3930 }
3931
3932 if status == Status::Returned {
3933 store
3934 .0
3935 .concurrent_state_mut()?
3936 .decrement_task_ref_count(guest_thread.task)?;
3937 }
3938
3939 Ok(())
3940 }
3941
3942 pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3958 self,
3959 mut store: StoreContextMut<'_, T>,
3960 host_task: EnteredHostTask,
3961 result_may_require_realloc: bool,
3962 future: impl Future<Output = Result<R>> + Send + 'static,
3963 lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3964 + Send
3965 + 'static,
3966 ) -> Result<u32> {
3967 let token = StoreToken::new(store.as_context_mut());
3968
3969 let (join_handle, future) = JoinHandle::run(future);
3972 let mut future = Box::pin(future);
3973
3974 let poll = tls::set(store.0, || {
3979 future
3980 .as_mut()
3981 .poll(&mut Context::from_waker(&Waker::noop()))
3982 });
3983
3984 match poll {
3985 Poll::Ready(result) => {
3987 let result = result.transpose()?;
3988 let task = store.0.current_materialized_host_task()?;
3991 lower(store.as_context_mut(), result, true, task)?;
3992 return Ok(Status::Returned.pack(None));
3993 }
3994
3995 Poll::Pending => {}
3997 }
3998
3999 let Some(task) = store.0.materialize_host_task_id()? else {
4003 bail_bug!("current thread is not a host thread")
4004 };
4005 {
4006 let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
4007 assert!(matches!(state, HostTaskState::CalleeStarted));
4008 *state = HostTaskState::CalleeRunning(join_handle);
4009 }
4010
4011 let future = Box::pin(async move {
4019 let result = match run_with_host_task_set(task, future).await? {
4020 Some(result) => Some(result?),
4021 None => None,
4022 };
4023 let on_complete = move |store: &mut dyn VMStore| {
4024 let mut store = token.as_context_mut(store);
4028 let old = store.0.set_thread(task)?;
4029
4030 let status = if result.is_some() {
4031 Status::Returned
4032 } else {
4033 Status::ReturnCancelled
4034 };
4035
4036 lower(store.as_context_mut(), result, false, Some(task))?;
4037 let state = store.0.concurrent_state_mut()?;
4038 match &mut state.get_mut(task)?.state {
4039 pending @ HostTaskState::CalleeCancelling => {
4042 *pending = HostTaskState::CalleeDone { cancelled: true };
4043 }
4044
4045 other => *other = HostTaskState::CalleeDone { cancelled: false },
4047 }
4048 Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
4049
4050 store.0.set_thread(old)?;
4051 Ok(())
4052 };
4053
4054 tls::get(move |store| {
4055 if result_may_require_realloc {
4056 store
4061 .concurrent_state_mut()?
4062 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
4063 on_complete,
4064 ))));
4065 Ok(())
4066 } else {
4067 on_complete(store)
4070 }
4071 })
4072 });
4073
4074 let caller = match host_task {
4077 Some(caller) => caller,
4078 None => bail_bug!("host task wasn't created but should have been"),
4079 };
4080 let state = store.0.concurrent_state_mut()?;
4081 state.push_future(future);
4082 let instance = state.get_mut(caller.task)?.instance;
4083 let handle = store
4084 .0
4085 .handle_table_insertion(instance)
4086 .subtask_host(task.rep())?;
4087 store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
4088 log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
4089
4090 store.0.set_thread(caller)?;
4094 Ok(Status::Started.pack(Some(handle)))
4095 }
4096
4097 pub(crate) fn task_return(
4100 self,
4101 store: &mut dyn VMStore,
4102 ty: TypeTupleIndex,
4103 options: OptionsIndex,
4104 storage: &[ValRaw],
4105 ) -> Result<()> {
4106 let guest_thread = store.current_guest_thread()?;
4107 let state = store.concurrent_state_mut()?;
4108 if !state.get_mut(guest_thread.task)?.async_lifted {
4109 bail!(Trap::TaskReturnOrCancelSyncLifted);
4110 }
4111 let lift = state
4112 .get_mut(guest_thread.task)?
4113 .lift_result
4114 .take()
4115 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4116 if !state.get_mut(guest_thread.task)?.result.is_none() {
4117 bail_bug!("task result unexpectedly already set");
4118 }
4119
4120 let CanonicalOptions {
4121 string_encoding,
4122 data_model,
4123 ..
4124 } = &self.id().get(store).component().env_component().options[options];
4125
4126 let invalid = ty != lift.ty
4127 || string_encoding != &lift.string_encoding
4128 || match data_model {
4129 CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
4130 Some(memory) => {
4131 let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
4132 let actual = self.id().get(store).runtime_memory(memory);
4133 expected != actual.as_ptr()
4134 }
4135 None => false,
4138 },
4139 CanonicalOptionsDataModel::Gc { .. } => true,
4141 };
4142
4143 if invalid {
4144 bail!(Trap::TaskReturnInvalid);
4145 }
4146
4147 log::trace!("task.return for {guest_thread:?}");
4148
4149 let result = (lift.lift)(store, storage)?;
4150 self.task_complete(store, guest_thread.task, result, Status::Returned)
4151 }
4152
4153 pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4155 let guest_thread = store.current_guest_thread()?;
4156 let state = store.concurrent_state_mut()?;
4157 let task = state.get_mut(guest_thread.task)?;
4158 if !task.async_lifted {
4159 bail!(Trap::TaskReturnOrCancelSyncLifted);
4160 }
4161 if !task.cancel_request_delivered {
4162 bail!(Trap::TaskCancelNotCancelled);
4163 }
4164 _ = task
4165 .lift_result
4166 .take()
4167 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4168
4169 if !task.result.is_none() {
4170 bail_bug!("task result should not bet set yet");
4171 }
4172
4173 log::trace!("task.cancel for {guest_thread:?}");
4174
4175 self.task_complete(
4176 store,
4177 guest_thread.task,
4178 Box::new(DummyResult),
4179 Status::ReturnCancelled,
4180 )
4181 }
4182
4183 fn task_complete(
4189 self,
4190 store: &mut StoreOpaque,
4191 guest_task: TableId<GuestTask>,
4192 result: Box<dyn Any + Send + Sync>,
4193 status: Status,
4194 ) -> Result<()> {
4195 store
4196 .component_resource_tables(Some(self))?
4197 .validate_scope_exit()?;
4198
4199 let state = store.concurrent_state_mut()?;
4200 let task = state.get_mut(guest_task)?;
4201
4202 task.event = None;
4206
4207 if let Caller::Host { tx, .. } = &mut task.caller {
4208 if let Some(tx) = tx.take() {
4209 _ = tx.send(result);
4210 }
4211 } else {
4212 task.result = Some(result);
4213 Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4214 }
4215
4216 Ok(())
4217 }
4218
4219 pub(crate) fn waitable_set_new(
4221 self,
4222 store: &mut StoreOpaque,
4223 caller_instance: RuntimeComponentInstanceIndex,
4224 ) -> Result<u32> {
4225 let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4226 let handle = store
4227 .handle_table_insertion(self.runtime_instance(caller_instance))
4228 .waitable_set(set.rep())?;
4229 log::trace!("new waitable set {set:?} (handle {handle})");
4230 Ok(handle)
4231 }
4232
4233 pub(crate) fn waitable_set_drop(
4235 self,
4236 store: &mut StoreOpaque,
4237 caller_instance: RuntimeComponentInstanceIndex,
4238 set: u32,
4239 ) -> Result<()> {
4240 let rep = store
4241 .instance_state(self.runtime_instance(caller_instance))
4242 .handle_table()
4243 .waitable_set_remove(set)?;
4244
4245 log::trace!("drop waitable set {rep} (handle {set})");
4246
4247 let set = store
4251 .concurrent_state_mut()?
4252 .get_mut(TableId::<WaitableSet>::new(rep))?;
4253 if set.num_waiting > 0 {
4254 bail!(Trap::WaitableSetDropHasWaiters);
4255 }
4256
4257 store
4258 .concurrent_state_mut()?
4259 .delete(TableId::<WaitableSet>::new(rep))?;
4260
4261 Ok(())
4262 }
4263
4264 pub(crate) fn waitable_join(
4266 self,
4267 store: &mut StoreOpaque,
4268 caller_instance: RuntimeComponentInstanceIndex,
4269 waitable_handle: u32,
4270 set_handle: u32,
4271 ) -> Result<()> {
4272 let mut instance = self.id().get_mut(store);
4273 let waitable =
4274 Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4275
4276 let set = if set_handle == 0 {
4277 None
4278 } else {
4279 let set = instance.instance_states().0[caller_instance]
4280 .handle_table()
4281 .waitable_set_rep(set_handle)?;
4282
4283 Some(TableId::<WaitableSet>::new(set))
4284 };
4285
4286 waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4287
4288 log::trace!(
4289 "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4290 );
4291
4292 waitable.join(store.concurrent_state_mut()?, set)
4293 }
4294
4295 pub(crate) fn subtask_drop(
4297 self,
4298 store: &mut StoreOpaque,
4299 caller_instance: RuntimeComponentInstanceIndex,
4300 task_id: u32,
4301 ) -> Result<()> {
4302 self.waitable_join(store, caller_instance, task_id, 0)?;
4303
4304 let (rep, is_host) = store
4305 .instance_state(self.runtime_instance(caller_instance))
4306 .handle_table()
4307 .subtask_remove(task_id)?;
4308
4309 let concurrent_state = store.concurrent_state_mut()?;
4310 let waitable = if is_host {
4311 let id = TableId::<HostTask>::new(rep);
4312 let task = concurrent_state.get_mut(id)?;
4313 match &task.state {
4314 HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4315 bail!(Trap::SubtaskDropNotResolved)
4316 }
4317 HostTaskState::CalleeDone { .. } => {}
4318 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4319 bail_bug!("invalid state for callee in `subtask.drop`")
4320 }
4321 }
4322
4323 Waitable::Host(id)
4324 } else {
4325 let id = TableId::<GuestTask>::new(rep);
4326 let task = concurrent_state.get_mut(id)?;
4327 if task.lift_result.is_some() {
4328 bail!(Trap::SubtaskDropNotResolved);
4329 }
4330 Waitable::Guest(id)
4331 };
4332
4333 waitable.common(concurrent_state)?.handle = None;
4334
4335 if waitable.take_event(concurrent_state)?.is_some() {
4338 bail!(Trap::SubtaskDropNotResolved);
4339 }
4340
4341 waitable.delete_from(concurrent_state)?;
4342
4343 log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4344 Ok(())
4345 }
4346
4347 pub(crate) fn waitable_set_wait(
4349 self,
4350 store: &mut StoreOpaque,
4351 options: OptionsIndex,
4352 set: u32,
4353 payload: u32,
4354 ) -> Result<u32> {
4355 let &CanonicalOptions {
4356 instance: caller_instance,
4357 ..
4358 } = &self.id().get(store).component().env_component().options[options];
4359 let caller = self.runtime_instance(caller_instance);
4360 let rep = store
4361 .instance_state(self.runtime_instance(caller_instance))
4362 .handle_table()
4363 .waitable_set_rep(set)?;
4364
4365 self.waitable_check(
4366 store,
4367 caller,
4368 WaitableCheck::Wait,
4369 WaitableCheckParams {
4370 set: TableId::new(rep),
4371 options,
4372 payload,
4373 },
4374 )
4375 }
4376
4377 pub(crate) fn waitable_set_poll(
4379 self,
4380 store: &mut StoreOpaque,
4381 options: OptionsIndex,
4382 set: u32,
4383 payload: u32,
4384 ) -> Result<u32> {
4385 let &CanonicalOptions {
4386 instance: caller_instance,
4387 ..
4388 } = &self.id().get(store).component().env_component().options[options];
4389 let caller = self.runtime_instance(caller_instance);
4390 let rep = store
4391 .instance_state(caller)
4392 .handle_table()
4393 .waitable_set_rep(set)?;
4394
4395 self.waitable_check(
4396 store,
4397 caller,
4398 WaitableCheck::Poll,
4399 WaitableCheckParams {
4400 set: TableId::new(rep),
4401 options,
4402 payload,
4403 },
4404 )
4405 }
4406
4407 pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4409 let thread_id = store.current_guest_thread()?.thread;
4410 match store
4411 .concurrent_state_mut()?
4412 .get_mut(thread_id)?
4413 .instance_rep
4414 {
4415 Some(r) => Ok(r),
4416 None => bail_bug!("thread should have instance_rep by now"),
4417 }
4418 }
4419
4420 pub(crate) fn thread_new_indirect<T: 'static>(
4422 self,
4423 mut store: StoreContextMut<T>,
4424 runtime_instance: RuntimeComponentInstanceIndex,
4425 start_func_ty: ModuleInternedTypeIndex,
4426 start_func_table_idx: RuntimeTableIndex,
4427 start_func_idx: u32,
4428 context: i32,
4429 ) -> Result<u32> {
4430 log::trace!("creating new thread");
4431
4432 let (instance, registry) = self.id().get_mut_and_registry(store.0);
4433 let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4434 else {
4435 bail_bug!("thread.new-indirect start function type should be registered");
4436 };
4437 let callee = instance
4438 .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4439 .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4440 if callee.type_index(store.0) != start_func_ty {
4441 bail!(Trap::ThreadNewIndirectInvalidType);
4442 }
4443
4444 let token = StoreToken::new(store.as_context_mut());
4445 let start_func = Box::new(
4446 move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4447 let old_thread = store.set_thread(guest_thread)?;
4448 log::trace!(
4449 "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4450 );
4451
4452 let mut store = token.as_context_mut(store);
4453 let mut params = [ValRaw::i32(context)];
4454 unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4457
4458 store.0.set_thread(old_thread)?;
4459
4460 let runtime_instance = self.runtime_instance(runtime_instance);
4461
4462 store
4465 .0
4466 .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4467
4468 store.0.cleanup_thread(guest_thread, runtime_instance)?;
4469
4470 log::trace!("explicit thread {guest_thread:?} completed");
4471 let state = store.0.concurrent_state_mut()?;
4472 if let Some(t) = old_thread.guest() {
4473 state.set_thread_running(t.thread)?;
4474 }
4475 log::trace!("thread start: restored {old_thread:?} as current thread");
4476
4477 Ok(())
4478 },
4479 );
4480
4481 let current_thread = store.0.current_guest_thread()?;
4482 let state = store.0.concurrent_state_mut()?;
4483 let parent_task = current_thread.task;
4484
4485 let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4486 let thread_id = state.push(new_thread)?;
4487 state.get_mut(parent_task)?.threads.insert(thread_id);
4488
4489 log::trace!("new thread with id {thread_id:?} created");
4490
4491 self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4492 }
4493
4494 pub(crate) fn resume_thread(
4495 self,
4496 store: &mut StoreOpaque,
4497 runtime_instance: RuntimeComponentInstanceIndex,
4498 thread_idx: u32,
4499 how: ResumeThread,
4500 ) -> Result<bool> {
4501 let thread_id =
4502 GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4503 let state = store.concurrent_state_mut()?;
4504 let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4505
4506 if store.current_guest_thread()? == guest_thread {
4507 bail!(Trap::CannotResumeThread);
4508 }
4509
4510 let priority = match how {
4511 ResumeThread::Resume => Priority::Switch,
4512 ResumeThread::ResumeLater => Priority::Low,
4513
4514 ResumeThread::Promote => {
4517 let instance = self.runtime_instance(runtime_instance);
4518 let do_not_enter = store
4519 .instance_state(instance)
4520 .concurrent_state()
4521 .do_not_enter;
4522 return store.concurrent_state_mut()?.promote_work_item_matching(
4523 |item: &WorkItem| match item {
4524 WorkItem::ResumeThread { thread, .. }
4525 | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4526
4527 WorkItem::GuestCall {
4528 call: GuestCall { thread, kind },
4529 ..
4530 } => {
4531 *thread == guest_thread
4532 && match kind {
4533 GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4537
4538 GuestCallKind::StartExplicit(_) => true,
4541
4542 GuestCallKind::StartImplicit { .. } => false,
4546 }
4547 }
4548
4549 WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4550 },
4551 );
4552 }
4553 };
4554
4555 let state = store.concurrent_state_mut()?;
4556 let thread = state.get_mut(guest_thread.thread)?;
4557
4558 match &thread.state {
4561 GuestThreadState::NotStartedExplicit(_) => {}
4562 GuestThreadState::Suspended(_) => {
4563 log::trace!("resuming thread {thread_id:?} that was suspended");
4564 state.schedule_suspended_thread(
4565 guest_thread,
4566 self.runtime_instance(runtime_instance),
4567 priority,
4568 )?;
4569 return Ok(true);
4570 }
4571 _ => bail!(Trap::CannotResumeThread),
4572 }
4573 match mem::replace(&mut thread.state, GuestThreadState::Running) {
4574 GuestThreadState::NotStartedExplicit(start_func) => {
4575 log::trace!("starting thread {guest_thread:?}");
4576 let guest_call = WorkItem::GuestCall {
4577 instance: self.runtime_instance(runtime_instance),
4578 call: GuestCall {
4579 thread: guest_thread,
4580 kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4581 start_func(store, guest_thread)
4582 })),
4583 },
4584 };
4585 state.push_work_item(guest_call, priority)?;
4586 }
4587 other => {
4588 thread.state = other;
4589 bail_bug!("thread state checked to be resumable above");
4590 }
4591 }
4592 Ok(true)
4593 }
4594
4595 fn add_guest_thread_to_instance_table(
4596 self,
4597 thread_id: TableId<GuestThread>,
4598 store: &mut StoreOpaque,
4599 runtime_instance: RuntimeComponentInstanceIndex,
4600 ) -> Result<u32> {
4601 let (state, counter) =
4602 store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4603 let guest_id = state
4604 .thread_handle_table()
4605 .guest_thread_insert(counter, thread_id.rep())?;
4606 store
4607 .concurrent_state_mut()?
4608 .get_mut(thread_id)?
4609 .instance_rep = Some(guest_id);
4610 Ok(guest_id)
4611 }
4612
4613 pub(crate) fn suspension_intrinsic(
4617 self,
4618 store: &mut StoreOpaque,
4619 caller: RuntimeComponentInstanceIndex,
4620 yielding: bool,
4621 to_thread: SuspensionTarget,
4622 ) -> Result<WaitResult> {
4623 let check_suspend = match to_thread {
4624 SuspensionTarget::Promote(thread) => {
4625 !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4626 }
4627 SuspensionTarget::Resume(thread) => {
4628 if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4629 bail_bug!(
4630 "`resume_thread` should only ever return false \
4631 when `ResumeThread::Promote` is passed to it"
4632 );
4633 }
4634 false
4635 }
4636 SuspensionTarget::None => true,
4637 };
4638
4639 if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4640 return if yielding {
4641 Ok(WaitResult::Completed)
4642 } else {
4643 Err(Trap::CannotBlockSyncTask.into())
4644 };
4645 }
4646
4647 let guest_thread = store.current_guest_thread()?;
4648
4649 let reason = if yielding {
4650 SuspendReason::Yielding {
4651 thread: guest_thread,
4652 }
4653 } else {
4654 SuspendReason::ExplicitlySuspending {
4655 thread: guest_thread,
4656 }
4657 };
4658
4659 store.suspend(reason)?;
4660
4661 Ok(WaitResult::Completed)
4662 }
4663
4664 fn waitable_check(
4666 self,
4667 store: &mut StoreOpaque,
4668 caller: RuntimeInstance,
4669 check: WaitableCheck,
4670 params: WaitableCheckParams,
4671 ) -> Result<u32> {
4672 let guest_thread = store.current_guest_thread()?;
4673
4674 log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4675
4676 match &check {
4679 WaitableCheck::Wait => {
4680 let set = params.set;
4681
4682 loop {
4687 let state = store.concurrent_state_mut()?;
4688 let task = state.get_mut(guest_thread.task)?;
4689 if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4690 || !state.get_mut(set)?.ready.is_empty()
4691 {
4692 break;
4693 }
4694
4695 store.switch_or_trap_if_may_not_suspend(caller)?;
4696
4697 store.suspend(SuspendReason::Waiting {
4698 set,
4699 thread: guest_thread,
4700 })?;
4701 }
4702 }
4703 WaitableCheck::Poll => {}
4704 }
4705
4706 log::trace!(
4707 "waitable check for {guest_thread:?}; set {:?}, part two",
4708 params.set
4709 );
4710
4711 let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4713
4714 let (ordinal, handle, result) = match &check {
4715 WaitableCheck::Wait => {
4716 let (event, waitable) = match event {
4717 Some(p) => p,
4718 None => bail_bug!("event expected to be present"),
4719 };
4720 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4721 let (ordinal, result) = event.parts();
4722 (ordinal, handle, result)
4723 }
4724 WaitableCheck::Poll => {
4725 if let Some((event, waitable)) = event {
4726 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4727 let (ordinal, result) = event.parts();
4728 (ordinal, handle, result)
4729 } else {
4730 log::trace!(
4731 "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4732 guest_thread.task,
4733 params.set
4734 );
4735 let (ordinal, result) = Event::None.parts();
4736 (ordinal, 0, result)
4737 }
4738 }
4739 };
4740 let ptr = crate::component::func::validate_inbounds_dynamic(
4741 &CanonicalAbiInfo::POINTER_PAIR,
4742 self.options_memory(store, params.options),
4743 &ValRaw::u32(params.payload),
4744 )?;
4745 let memory = self
4746 .options_memory_range_mut(store, params.options, ptr, 8)
4747 .unwrap();
4748 memory[0..4].copy_from_slice(&handle.to_le_bytes());
4749 memory[4..8].copy_from_slice(&result.to_le_bytes());
4750 Ok(ordinal)
4751 }
4752
4753 pub(crate) fn subtask_cancel(
4755 self,
4756 store: &mut StoreOpaque,
4757 caller_instance: RuntimeComponentInstanceIndex,
4758 async_: bool,
4759 task_id: u32,
4760 ) -> Result<u32> {
4761 let (rep, is_host) = store
4762 .instance_state(self.runtime_instance(caller_instance))
4763 .handle_table()
4764 .subtask_rep(task_id)?;
4765 let waitable = if is_host {
4766 Waitable::Host(TableId::<HostTask>::new(rep))
4767 } else {
4768 Waitable::Guest(TableId::<GuestTask>::new(rep))
4769 };
4770 let concurrent_state = store.concurrent_state_mut()?;
4771
4772 log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4773
4774 waitable.trap_if_in_waitable_set(concurrent_state)?;
4775
4776 let needs_block;
4777 if let Waitable::Host(host_task) = waitable {
4778 let state = &mut concurrent_state.get_mut(host_task)?.state;
4779 match state {
4780 HostTaskState::CalleeRunning(handle) => {
4787 handle.abort();
4788 *state = HostTaskState::CalleeCancelling;
4789 needs_block = true;
4790 }
4791
4792 HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4795 bail!(Trap::SubtaskCancelAfterTerminal);
4796 }
4797 HostTaskState::CalleeDone { cancelled: false } => {
4798 *state = HostTaskState::CalleeDone { cancelled: true };
4801 needs_block = false;
4802 }
4803
4804 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4807 bail_bug!("invalid states for host callee")
4808 }
4809 }
4810 } else {
4811 let guest_task = TableId::<GuestTask>::new(rep);
4812 let task = concurrent_state.get_mut(guest_task)?;
4813
4814 if task.cancel_requested {
4817 bail!(Trap::SubtaskCancelAfterTerminal);
4818 }
4819
4820 if !task.already_lowered_parameters() {
4821 store.cancel_guest_subtask_without_lowered_parameters(
4822 self.runtime_instance(caller_instance),
4823 guest_task,
4824 )?;
4825 return Ok(Status::StartCancelled as u32);
4826 } else if !task.returned_or_cancelled() {
4827 task.cancel_requested = true;
4834 task.event = Some(Event::Cancelled);
4835 let runtime_instance = task.instance;
4836 for thread in task.threads.clone() {
4837 let thread = QualifiedThreadId {
4838 task: guest_task,
4839 thread,
4840 };
4841 let concurrent_state = store.concurrent_state_mut()?;
4842 let thread_mut = concurrent_state.get_mut(thread.thread)?;
4843
4844 let yield_ = |store: &mut StoreOpaque| {
4845 let state = store.instance_state(runtime_instance).concurrent_state();
4850 let old_do_not_suspend = state.do_not_suspend;
4851 state.do_not_suspend = false;
4852
4853 let caller = store.current_guest_thread()?;
4854
4855 let state = store.concurrent_state_mut()?;
4860 let set = state.get_mut(caller.thread)?.sync_call_set;
4861 waitable.join(state, Some(set))?;
4862
4863 store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4864
4865 let state = store.concurrent_state_mut()?;
4866 waitable.join(state, None)?;
4867
4868 store
4869 .instance_state(runtime_instance)
4870 .concurrent_state()
4871 .do_not_suspend = old_do_not_suspend;
4872
4873 Ok::<(), crate::Error>(())
4874 };
4875
4876 match thread_mut.wake_on_cancel.take() {
4877 WakeOnCancel::Waiting(set) => {
4878 let instance = concurrent_state.take_callback_waiter(set, thread)?;
4880 let item = WorkItem::GuestCall {
4881 instance: runtime_instance,
4882 call: GuestCall {
4883 thread,
4884 kind: GuestCallKind::DeliverEvent {
4885 instance,
4886 set: Some(set),
4887 },
4888 },
4889 };
4890 concurrent_state.set_switch_item(item)?;
4891
4892 yield_(store)?;
4893
4894 break;
4895 }
4896 WakeOnCancel::Yielding => {
4897 if concurrent_state.promote_thread_work_item(thread)? {
4898 yield_(store)?;
4899 break;
4900 } else if store
4901 .instance_state(runtime_instance)
4902 .concurrent_state()
4903 .pending
4904 .contains_key(&thread)
4905 {
4906 store
4916 .concurrent_state_mut()?
4917 .get_mut(thread.thread)?
4918 .wake_on_cancel = WakeOnCancel::Yielding;
4919 } else {
4920 bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4921 }
4922 }
4923 WakeOnCancel::None => {}
4924 }
4925 }
4926
4927 needs_block = !store
4930 .concurrent_state_mut()?
4931 .get_mut(guest_task)?
4932 .returned_or_cancelled()
4933 } else {
4934 needs_block = false;
4935 }
4936 };
4937
4938 if needs_block {
4942 if async_ {
4943 return Ok(BLOCKED);
4944 }
4945
4946 let old_next_switch_item = if is_host {
4956 Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4957 } else {
4958 None
4959 };
4960
4961 store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4964
4965 if let Some(item) = old_next_switch_item {
4966 let state = store.concurrent_state_mut()?;
4967 state.restore_next_switch_item(item)?;
4968 }
4969
4970 }
4972
4973 let event = waitable.take_event(store.concurrent_state_mut()?)?;
4974 if let Some(Event::Subtask {
4975 status: status @ (Status::Returned | Status::ReturnCancelled),
4976 }) = event
4977 {
4978 Ok(status as u32)
4979 } else {
4980 bail!(Trap::SubtaskCancelAfterTerminal);
4981 }
4982 }
4983}
4984
4985pub trait VMComponentAsyncStore {
4993 unsafe fn prepare_call(
4999 &mut self,
5000 instance: Instance,
5001 memory: *mut VMMemoryDefinition,
5002 start: NonNull<VMFuncRef>,
5003 return_: NonNull<VMFuncRef>,
5004 caller_instance: RuntimeComponentInstanceIndex,
5005 callee_instance: RuntimeComponentInstanceIndex,
5006 task_return_type: TypeTupleIndex,
5007 callee_async: bool,
5008 string_encoding: StringEncoding,
5009 result_count: u32,
5010 storage: *mut ValRaw,
5011 storage_len: usize,
5012 ) -> Result<()>;
5013
5014 unsafe fn start_call(
5017 &mut self,
5018 instance: Instance,
5019 callback: *mut VMFuncRef,
5020 post_return: *mut VMFuncRef,
5021 callee: NonNull<VMFuncRef>,
5022 param_count: u32,
5023 result_count: u32,
5024 flags: u32,
5025 storage: *mut MaybeUninit<ValRaw>,
5026 storage_len: usize,
5027 ) -> Result<()>;
5028
5029 fn future_write(
5031 &mut self,
5032 instance: Instance,
5033 caller: RuntimeComponentInstanceIndex,
5034 ty: TypeFutureTableIndex,
5035 options: OptionsIndex,
5036 future: u32,
5037 address: u32,
5038 ) -> Result<u32>;
5039
5040 fn future_read(
5042 &mut self,
5043 instance: Instance,
5044 caller: RuntimeComponentInstanceIndex,
5045 ty: TypeFutureTableIndex,
5046 options: OptionsIndex,
5047 future: u32,
5048 address: u32,
5049 ) -> Result<u32>;
5050
5051 fn future_drop_writable(
5053 &mut self,
5054 instance: Instance,
5055 ty: TypeFutureTableIndex,
5056 writer: u32,
5057 ) -> Result<()>;
5058
5059 fn stream_write(
5061 &mut self,
5062 instance: Instance,
5063 caller: RuntimeComponentInstanceIndex,
5064 ty: TypeStreamTableIndex,
5065 options: OptionsIndex,
5066 stream: u32,
5067 address: u32,
5068 count: u32,
5069 ) -> Result<u32>;
5070
5071 fn stream_read(
5073 &mut self,
5074 instance: Instance,
5075 caller: RuntimeComponentInstanceIndex,
5076 ty: TypeStreamTableIndex,
5077 options: OptionsIndex,
5078 stream: u32,
5079 address: u32,
5080 count: u32,
5081 ) -> Result<u32>;
5082
5083 fn flat_stream_write(
5086 &mut self,
5087 instance: Instance,
5088 caller: RuntimeComponentInstanceIndex,
5089 ty: TypeStreamTableIndex,
5090 options: OptionsIndex,
5091 payload_size: u32,
5092 payload_align: u32,
5093 stream: u32,
5094 address: u32,
5095 count: u32,
5096 ) -> Result<u32>;
5097
5098 fn flat_stream_read(
5101 &mut self,
5102 instance: Instance,
5103 caller: RuntimeComponentInstanceIndex,
5104 ty: TypeStreamTableIndex,
5105 options: OptionsIndex,
5106 payload_size: u32,
5107 payload_align: u32,
5108 stream: u32,
5109 address: u32,
5110 count: u32,
5111 ) -> Result<u32>;
5112
5113 fn stream_drop_writable(
5115 &mut self,
5116 instance: Instance,
5117 ty: TypeStreamTableIndex,
5118 writer: u32,
5119 ) -> Result<()>;
5120
5121 fn error_context_debug_message(
5123 &mut self,
5124 instance: Instance,
5125 ty: TypeComponentLocalErrorContextTableIndex,
5126 options: OptionsIndex,
5127 err_ctx_handle: u32,
5128 debug_msg_address: u32,
5129 ) -> Result<()>;
5130
5131 fn thread_new_indirect(
5133 &mut self,
5134 instance: Instance,
5135 caller: RuntimeComponentInstanceIndex,
5136 func_ty_idx: ModuleInternedTypeIndex,
5137 start_func_table_idx: RuntimeTableIndex,
5138 start_func_idx: u32,
5139 context: i32,
5140 ) -> Result<u32>;
5141}
5142
5143impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5145 unsafe fn prepare_call(
5146 &mut self,
5147 instance: Instance,
5148 memory: *mut VMMemoryDefinition,
5149 start: NonNull<VMFuncRef>,
5150 return_: NonNull<VMFuncRef>,
5151 caller_instance: RuntimeComponentInstanceIndex,
5152 callee_instance: RuntimeComponentInstanceIndex,
5153 task_return_type: TypeTupleIndex,
5154 callee_async: bool,
5155 string_encoding: StringEncoding,
5156 result_count_or_max_if_async: u32,
5157 storage: *mut ValRaw,
5158 storage_len: usize,
5159 ) -> Result<()> {
5160 let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5164
5165 unsafe {
5166 instance.prepare_call(
5167 StoreContextMut(self),
5168 start,
5169 return_,
5170 caller_instance,
5171 callee_instance,
5172 task_return_type,
5173 callee_async,
5174 memory,
5175 string_encoding,
5176 match result_count_or_max_if_async {
5177 PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5178 params,
5179 has_result: false,
5180 },
5181 PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5182 params,
5183 has_result: true,
5184 },
5185 result_count => CallerInfo::Sync {
5186 params,
5187 result_count,
5188 },
5189 },
5190 )
5191 }
5192 }
5193
5194 unsafe fn start_call(
5195 &mut self,
5196 instance: Instance,
5197 callback: *mut VMFuncRef,
5198 post_return: *mut VMFuncRef,
5199 callee: NonNull<VMFuncRef>,
5200 param_count: u32,
5201 result_count: u32,
5202 flags: u32,
5203 storage: *mut MaybeUninit<ValRaw>,
5204 storage_len: usize,
5205 ) -> Result<()> {
5206 unsafe {
5207 instance.start_call(
5208 StoreContextMut(self),
5209 callback,
5210 post_return,
5211 callee,
5212 param_count,
5213 result_count,
5214 flags,
5215 core::slice::from_raw_parts_mut(storage, storage_len),
5219 )
5220 }
5221 }
5222
5223 fn future_write(
5224 &mut self,
5225 instance: Instance,
5226 caller: RuntimeComponentInstanceIndex,
5227 ty: TypeFutureTableIndex,
5228 options: OptionsIndex,
5229 future: u32,
5230 address: u32,
5231 ) -> Result<u32> {
5232 instance
5233 .guest_write(
5234 StoreContextMut(self),
5235 caller,
5236 TransmitIndex::Future(ty),
5237 options,
5238 None,
5239 future,
5240 address,
5241 1,
5242 )
5243 .map(|result| result.encode())
5244 }
5245
5246 fn future_read(
5247 &mut self,
5248 instance: Instance,
5249 caller: RuntimeComponentInstanceIndex,
5250 ty: TypeFutureTableIndex,
5251 options: OptionsIndex,
5252 future: u32,
5253 address: u32,
5254 ) -> Result<u32> {
5255 instance
5256 .guest_read(
5257 StoreContextMut(self),
5258 caller,
5259 TransmitIndex::Future(ty),
5260 options,
5261 None,
5262 future,
5263 address,
5264 1,
5265 )
5266 .map(|result| result.encode())
5267 }
5268
5269 fn stream_write(
5270 &mut self,
5271 instance: Instance,
5272 caller: RuntimeComponentInstanceIndex,
5273 ty: TypeStreamTableIndex,
5274 options: OptionsIndex,
5275 stream: u32,
5276 address: u32,
5277 count: u32,
5278 ) -> Result<u32> {
5279 instance
5280 .guest_write(
5281 StoreContextMut(self),
5282 caller,
5283 TransmitIndex::Stream(ty),
5284 options,
5285 None,
5286 stream,
5287 address,
5288 count,
5289 )
5290 .map(|result| result.encode())
5291 }
5292
5293 fn stream_read(
5294 &mut self,
5295 instance: Instance,
5296 caller: RuntimeComponentInstanceIndex,
5297 ty: TypeStreamTableIndex,
5298 options: OptionsIndex,
5299 stream: u32,
5300 address: u32,
5301 count: u32,
5302 ) -> Result<u32> {
5303 instance
5304 .guest_read(
5305 StoreContextMut(self),
5306 caller,
5307 TransmitIndex::Stream(ty),
5308 options,
5309 None,
5310 stream,
5311 address,
5312 count,
5313 )
5314 .map(|result| result.encode())
5315 }
5316
5317 fn future_drop_writable(
5318 &mut self,
5319 instance: Instance,
5320 ty: TypeFutureTableIndex,
5321 writer: u32,
5322 ) -> Result<()> {
5323 instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5324 }
5325
5326 fn flat_stream_write(
5327 &mut self,
5328 instance: Instance,
5329 caller: RuntimeComponentInstanceIndex,
5330 ty: TypeStreamTableIndex,
5331 options: OptionsIndex,
5332 payload_size: u32,
5333 payload_align: u32,
5334 stream: u32,
5335 address: u32,
5336 count: u32,
5337 ) -> Result<u32> {
5338 instance
5339 .guest_write(
5340 StoreContextMut(self),
5341 caller,
5342 TransmitIndex::Stream(ty),
5343 options,
5344 Some(FlatAbi {
5345 size: payload_size,
5346 align: payload_align,
5347 }),
5348 stream,
5349 address,
5350 count,
5351 )
5352 .map(|result| result.encode())
5353 }
5354
5355 fn flat_stream_read(
5356 &mut self,
5357 instance: Instance,
5358 caller: RuntimeComponentInstanceIndex,
5359 ty: TypeStreamTableIndex,
5360 options: OptionsIndex,
5361 payload_size: u32,
5362 payload_align: u32,
5363 stream: u32,
5364 address: u32,
5365 count: u32,
5366 ) -> Result<u32> {
5367 instance
5368 .guest_read(
5369 StoreContextMut(self),
5370 caller,
5371 TransmitIndex::Stream(ty),
5372 options,
5373 Some(FlatAbi {
5374 size: payload_size,
5375 align: payload_align,
5376 }),
5377 stream,
5378 address,
5379 count,
5380 )
5381 .map(|result| result.encode())
5382 }
5383
5384 fn stream_drop_writable(
5385 &mut self,
5386 instance: Instance,
5387 ty: TypeStreamTableIndex,
5388 writer: u32,
5389 ) -> Result<()> {
5390 instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5391 }
5392
5393 fn error_context_debug_message(
5394 &mut self,
5395 instance: Instance,
5396 ty: TypeComponentLocalErrorContextTableIndex,
5397 options: OptionsIndex,
5398 err_ctx_handle: u32,
5399 debug_msg_address: u32,
5400 ) -> Result<()> {
5401 instance.error_context_debug_message(
5402 StoreContextMut(self),
5403 ty,
5404 options,
5405 err_ctx_handle,
5406 debug_msg_address,
5407 )
5408 }
5409
5410 fn thread_new_indirect(
5411 &mut self,
5412 instance: Instance,
5413 caller: RuntimeComponentInstanceIndex,
5414 func_ty_idx: ModuleInternedTypeIndex,
5415 start_func_table_idx: RuntimeTableIndex,
5416 start_func_idx: u32,
5417 context: i32,
5418 ) -> Result<u32> {
5419 instance.thread_new_indirect(
5420 StoreContextMut(self),
5421 caller,
5422 func_ty_idx,
5423 start_func_table_idx,
5424 start_func_idx,
5425 context,
5426 )
5427 }
5428}
5429
5430type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5431
5432async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5435where
5436 F: Future,
5437{
5438 let mut future = pin!(future);
5439 future::poll_fn(|cx| {
5440 let old_thread = match tls::get(|store| store.set_thread(task)) {
5441 Ok(thread) => thread,
5442 Err(error) => return Poll::Ready(Err(error)),
5443 };
5444 let result = future.as_mut().poll(cx);
5445 match tls::get(|store| store.set_thread(old_thread)) {
5446 Ok(_) => result.map(Ok),
5447 Err(error) => Poll::Ready(Err(error)),
5448 }
5449 })
5450 .await
5451}
5452
5453pub(crate) struct HostTask {
5457 common: WaitableCommon,
5458
5459 call_context: CallContext,
5462
5463 state: HostTaskState,
5464
5465 group: TaskGroupId,
5466}
5467
5468enum HostTaskState {
5469 CalleeStarted,
5474
5475 CalleeRunning(JoinHandle),
5480
5481 CalleeCancelling,
5485
5486 CalleeFinished(LiftedResult),
5490
5491 CalleeDone { cancelled: bool },
5494}
5495
5496impl HostTask {
5497 fn new(
5498 concurrent_state: &mut ConcurrentState,
5499 state: HostTaskState,
5500 caller: QualifiedThreadId,
5501 ) -> Result<Self> {
5502 let group = concurrent_state.get_mut(caller.task)?.group;
5503 concurrent_state.increment_group_ref_count(group)?;
5504
5505 Ok(Self {
5506 common: WaitableCommon::default(),
5507 call_context: CallContext::default(),
5508 state,
5509 group,
5510 })
5511 }
5512}
5513
5514impl TableDebug for HostTask {
5515 fn type_name() -> &'static str {
5516 "HostTask"
5517 }
5518}
5519
5520type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5521
5522enum Caller {
5524 Host {
5526 tx: Option<oneshot::Sender<LiftedResult>>,
5528 caller: Option<TableId<HostTask>>,
5532 },
5533 Guest {
5535 thread: QualifiedThreadId,
5537 },
5538}
5539
5540struct LiftResult {
5543 lift: RawLift,
5544 ty: TypeTupleIndex,
5545 memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5546 string_encoding: StringEncoding,
5547}
5548
5549#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5554pub(crate) struct QualifiedThreadId {
5555 task: TableId<GuestTask>,
5556 thread: TableId<GuestThread>,
5557}
5558
5559impl QualifiedThreadId {
5560 fn qualify(
5561 state: &mut ConcurrentState,
5562 thread: TableId<GuestThread>,
5563 ) -> Result<QualifiedThreadId> {
5564 Ok(QualifiedThreadId {
5565 task: state.get_mut(thread)?.parent_task,
5566 thread,
5567 })
5568 }
5569}
5570
5571impl fmt::Debug for QualifiedThreadId {
5572 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5573 f.debug_tuple("QualifiedThreadId")
5574 .field(&self.task.rep())
5575 .field(&self.thread.rep())
5576 .finish()
5577 }
5578}
5579
5580enum GuestThreadState {
5581 NotStartedImplicit,
5582 NotStartedExplicit(
5583 Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5584 ),
5585 Running,
5586 Suspended(StoreFiber<'static>),
5587 Ready {
5588 fiber: StoreFiber<'static>,
5589 },
5590 Completed,
5591}
5592
5593impl GuestThreadState {
5594 fn check_no_fiber(&self) -> Result<()> {
5595 match self {
5596 Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5597 Self::NotStartedImplicit
5598 | Self::NotStartedExplicit(_)
5599 | Self::Running
5600 | Self::Completed => Ok(()),
5601 }
5602 }
5603}
5604
5605impl fmt::Debug for GuestThreadState {
5606 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5607 match self {
5608 Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5609 Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5610 Self::Running => f.debug_tuple("Running").finish(),
5611 Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5612 Self::Ready { .. } => f.debug_struct("Ready").finish(),
5613 Self::Completed => f.debug_tuple("Completed").finish(),
5614 }
5615 }
5616}
5617
5618#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5619enum WakeOnCancel {
5620 None,
5621 Waiting(TableId<WaitableSet>),
5622 Yielding,
5623}
5624
5625impl WakeOnCancel {
5626 fn is_none(self) -> bool {
5627 matches!(self, WakeOnCancel::None)
5628 }
5629
5630 fn replace(&mut self, other: WakeOnCancel) -> Self {
5631 let old = *self;
5632 *self = other;
5633 old
5634 }
5635
5636 fn take(&mut self) -> Self {
5637 self.replace(WakeOnCancel::None)
5638 }
5639}
5640
5641pub struct GuestThread {
5642 context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5645 parent_task: TableId<GuestTask>,
5647 wake_on_cancel: WakeOnCancel,
5650 state: GuestThreadState,
5652 instance_rep: Option<u32>,
5655 sync_call_set: TableId<WaitableSet>,
5657 old_do_not_suspend: Option<bool>,
5660}
5661
5662impl GuestThread {
5663 fn from_instance(
5666 state: Pin<&mut ComponentInstance>,
5667 caller_instance: RuntimeComponentInstanceIndex,
5668 guest_thread: u32,
5669 ) -> Result<TableId<Self>> {
5670 let rep = state.instance_states().0[caller_instance]
5671 .thread_handle_table()
5672 .guest_thread_rep(guest_thread)?;
5673 Ok(TableId::new(rep))
5674 }
5675
5676 fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5677 let sync_call_set = state.push(WaitableSet {
5678 is_sync_call_set: true,
5679 ..WaitableSet::default()
5680 })?;
5681 Ok(Self {
5682 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5683 parent_task,
5684 wake_on_cancel: WakeOnCancel::None,
5685 state: GuestThreadState::NotStartedImplicit,
5686 instance_rep: None,
5687 sync_call_set,
5688 old_do_not_suspend: None,
5689 })
5690 }
5691
5692 fn new_explicit(
5693 state: &mut ConcurrentState,
5694 parent_task: TableId<GuestTask>,
5695 start_func: Box<
5696 dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5697 >,
5698 ) -> Result<Self> {
5699 state.increment_task_ref_count(parent_task)?;
5700 let sync_call_set = state.push(WaitableSet {
5701 is_sync_call_set: true,
5702 ..WaitableSet::default()
5703 })?;
5704 Ok(Self {
5705 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5706 parent_task,
5707 wake_on_cancel: WakeOnCancel::None,
5708 state: GuestThreadState::NotStartedExplicit(start_func),
5709 instance_rep: None,
5710 sync_call_set,
5711 old_do_not_suspend: None,
5712 })
5713 }
5714}
5715
5716impl TableDebug for GuestThread {
5717 fn type_name() -> &'static str {
5718 "GuestThread"
5719 }
5720}
5721
5722enum SyncResult {
5723 NotProduced,
5724 Produced(Option<ValRaw>),
5725 Taken,
5726}
5727
5728impl SyncResult {
5729 fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5730 Ok(match mem::replace(self, SyncResult::Taken) {
5731 SyncResult::NotProduced => None,
5732 SyncResult::Produced(val) => Some(val),
5733 SyncResult::Taken => {
5734 bail_bug!("attempted to take a synchronous result that was already taken")
5735 }
5736 })
5737 }
5738}
5739
5740pub(crate) struct GuestTask {
5742 common: WaitableCommon,
5744 lower_params: Option<RawLower>,
5746 lift_result: Option<LiftResult>,
5748 result: Option<LiftedResult>,
5751 callback: Option<CallbackFn>,
5754 caller: Caller,
5756 call_context: CallContext,
5761 sync_result: SyncResult,
5764 cancel_request_delivered: bool,
5768 cancel_requested: bool,
5773 starting_sent: bool,
5776 instance: RuntimeInstance,
5783 event: Option<Event>,
5785 exited: bool,
5787 threads: HashSet<TableId<GuestThread>>,
5789 async_typed: bool,
5792 async_lifted: bool,
5795 decremented_interesting_task_count: bool,
5798 group: TaskGroupId,
5800 ref_count: usize,
5805}
5806
5807impl GuestTask {
5808 fn already_lowered_parameters(&self) -> bool {
5809 self.lower_params.is_none()
5811 }
5812
5813 fn returned_or_cancelled(&self) -> bool {
5814 self.lift_result.is_none()
5816 }
5817
5818 fn new(
5819 state: &mut ConcurrentState,
5820 lower_params: RawLower,
5821 lift_result: LiftResult,
5822 caller: Caller,
5823 callback: Option<CallbackFn>,
5824 instance: RuntimeInstance,
5825 async_typed: bool,
5826 async_lifted: bool,
5827 ) -> Result<QualifiedThreadId> {
5828 let group = match caller {
5829 Caller::Guest { thread } => {
5830 let group = state.get_mut(thread.task)?.group;
5831 state.increment_group_ref_count(group)?;
5832 group
5833 }
5834 Caller::Host { .. } => state.make_task_group()?,
5835 };
5836
5837 let task = state.push(Self {
5838 common: WaitableCommon::default(),
5839 lower_params: Some(lower_params),
5840 lift_result: Some(lift_result),
5841 result: None,
5842 callback,
5843 caller,
5844 call_context: CallContext::default(),
5845 sync_result: SyncResult::NotProduced,
5846 cancel_request_delivered: false,
5847 cancel_requested: false,
5848 starting_sent: false,
5849 instance,
5850 event: None,
5851 exited: false,
5852 threads: HashSet::new(),
5853 async_typed,
5854 async_lifted,
5855 decremented_interesting_task_count: false,
5856 group,
5857 ref_count: 2,
5860 })?;
5861 let new_thread = GuestThread::new_implicit(state, task)?;
5862 let thread = state.push(new_thread)?;
5863 state.get_mut(task)?.threads.insert(thread);
5864 state.interesting_tasks += 1;
5865 let thread = QualifiedThreadId { task, thread };
5866 log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5867 Ok(thread)
5868 }
5869}
5870
5871impl TableDebug for GuestTask {
5872 fn type_name() -> &'static str {
5873 "GuestTask"
5874 }
5875}
5876
5877#[derive(Default)]
5879struct WaitableCommon {
5880 event: Option<Event>,
5882 set: Option<TableId<WaitableSet>>,
5884 handle: Option<u32>,
5886}
5887
5888#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5890enum Waitable {
5891 Host(TableId<HostTask>),
5893 Guest(TableId<GuestTask>),
5895 Transmit(TableId<TransmitHandle>),
5897}
5898
5899impl Waitable {
5900 fn from_instance(
5903 state: Pin<&mut ComponentInstance>,
5904 caller_instance: RuntimeComponentInstanceIndex,
5905 waitable: u32,
5906 ) -> Result<Self> {
5907 use crate::runtime::vm::component::Waitable;
5908
5909 let (waitable, kind) = state.instance_states().0[caller_instance]
5910 .handle_table()
5911 .waitable_rep(waitable)?;
5912
5913 Ok(match kind {
5914 Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5915 Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5916 Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5917 })
5918 }
5919
5920 fn rep(&self) -> u32 {
5922 match self {
5923 Self::Host(id) => id.rep(),
5924 Self::Guest(id) => id.rep(),
5925 Self::Transmit(id) => id.rep(),
5926 }
5927 }
5928
5929 fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5933 log::trace!("waitable {self:?} join set {set:?}");
5934
5935 let old = mem::replace(&mut self.common(state)?.set, set);
5936
5937 if let Some(old) = old {
5938 match *self {
5939 Waitable::Host(id) => state.remove_child(id, old),
5940 Waitable::Guest(id) => state.remove_child(id, old),
5941 Waitable::Transmit(id) => state.remove_child(id, old),
5942 }?;
5943
5944 state.get_mut(old)?.ready.remove(self);
5945 }
5946
5947 if let Some(set) = set {
5948 match *self {
5949 Waitable::Host(id) => state.add_child(id, set),
5950 Waitable::Guest(id) => state.add_child(id, set),
5951 Waitable::Transmit(id) => state.add_child(id, set),
5952 }?;
5953
5954 if self.common(state)?.event.is_some() {
5955 self.mark_ready(state)?;
5956 }
5957 }
5958
5959 Ok(())
5960 }
5961
5962 fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5964 Ok(match self {
5965 Self::Host(id) => &mut state.get_mut(*id)?.common,
5966 Self::Guest(id) => &mut state.get_mut(*id)?.common,
5967 Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5968 })
5969 }
5970
5971 fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5977 if self.common(state)?.set.is_some() {
5978 bail!(Trap::WaitableSyncAndAsync);
5979 }
5980 Ok(())
5981 }
5982
5983 fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5989 if let Some(set) = self.common(state)?.set
5990 && state.get_mut(set)?.is_sync_call_set
5991 {
5992 bail!(Trap::WaitableSyncAndAsync);
5993 }
5994 Ok(())
5995 }
5996
5997 fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
6001 log::trace!("set event for {self:?}: {event:?}");
6002 self.common(state)?.event = event;
6003 self.mark_ready(state)
6004 }
6005
6006 fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
6008 let common = self.common(state)?;
6009 let event = common.event.take();
6010 if let Some(set) = self.common(state)?.set {
6011 state.get_mut(set)?.ready.remove(self);
6012 }
6013
6014 Ok(event)
6015 }
6016
6017 fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
6021 if let Some(set) = self.common(state)?.set {
6022 state.get_mut(set)?.ready.insert(*self);
6023 state.wake_waiter(set)?;
6024 }
6025 Ok(())
6026 }
6027
6028 fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
6031 match self {
6032 Self::Host(task) => state.delete_host_task(*task),
6033 Self::Guest(task) => state.decrement_task_ref_count(*task),
6034 Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
6035 }
6036 }
6037}
6038
6039impl fmt::Debug for Waitable {
6040 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6041 match self {
6042 Self::Host(id) => write!(f, "{id:?}"),
6043 Self::Guest(id) => write!(f, "{id:?}"),
6044 Self::Transmit(id) => write!(f, "{id:?}"),
6045 }
6046 }
6047}
6048
6049#[derive(Default)]
6051struct WaitableSet {
6052 ready: BTreeSet<Waitable>,
6054 waiting: BTreeMap<QualifiedThreadId, WaitMode>,
6056 num_waiting: usize,
6059 is_sync_call_set: bool,
6062}
6063
6064impl WaitableSet {
6065 fn stop_waiting(&mut self) -> Result<()> {
6067 self.num_waiting = match self.num_waiting.checked_sub(1) {
6068 Some(n) => n,
6069 None => bail_bug!("waiter not accounted for in waitable set"),
6070 };
6071 Ok(())
6072 }
6073}
6074
6075impl TableDebug for WaitableSet {
6076 fn type_name() -> &'static str {
6077 "WaitableSet"
6078 }
6079}
6080
6081type RawLower =
6083 Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
6084
6085type RawLift = Box<
6087 dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6088>;
6089
6090type LiftedResult = Box<dyn Any + Send + Sync>;
6094
6095struct DummyResult;
6098
6099#[derive(Default)]
6101pub struct ConcurrentInstanceState {
6102 backpressure: u16,
6104 do_not_enter: bool,
6106 do_not_suspend: bool,
6109 pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
6112 pending_start_count: usize,
6115}
6116
6117impl ConcurrentInstanceState {
6118 pub fn pending_is_empty(&self) -> bool {
6119 self.pending.is_empty()
6120 }
6121}
6122
6123#[derive(Debug, Copy, Clone)]
6124pub(crate) enum CurrentThread {
6125 Guest(QualifiedThreadId),
6128 Host(TableId<HostTask>),
6130 DeferredHost(QualifiedThreadId),
6133 None,
6136}
6137
6138impl CurrentThread {
6139 fn guest(&self) -> Option<&QualifiedThreadId> {
6140 match self {
6141 Self::Guest(id) => Some(id),
6142 _ => None,
6143 }
6144 }
6145
6146 fn guest_task(&self) -> Option<TableId<GuestTask>> {
6147 match self {
6148 Self::Guest(id) => Some(id.task),
6149 _ => None,
6150 }
6151 }
6152
6153 fn is_none(&self) -> bool {
6154 matches!(self, Self::None)
6155 }
6156}
6157
6158impl From<QualifiedThreadId> for CurrentThread {
6159 fn from(id: QualifiedThreadId) -> Self {
6160 Self::Guest(id)
6161 }
6162}
6163
6164impl From<TableId<HostTask>> for CurrentThread {
6165 fn from(id: TableId<HostTask>) -> Self {
6166 Self::Host(id)
6167 }
6168}
6169
6170enum Priority {
6171 Switch,
6172 #[expect(
6173 dead_code,
6174 reason = "Here for completeness and in case we need it later"
6175 )]
6176 High,
6177 Low,
6178}
6179
6180pub struct ConcurrentState {
6182 unforced_current_thread: CurrentThread,
6188
6189 deferred_host_call_context: Option<CallContext>,
6195
6196 futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6201 table: AlwaysMut<ResourceTable>,
6203 switch_item: Option<WorkItem>,
6211 next_switch_item: Option<WorkItem>,
6217 high_priority: VecDeque<WorkItem>,
6219 low_priority: VecDeque<WorkItem>,
6221 suspend_reason: Option<SuspendReason>,
6225 worker: Option<StoreFiber<'static>>,
6229 worker_item: Option<WorkerItem>,
6231
6232 global_error_context_ref_counts:
6245 BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6246
6247 interesting_tasks: usize,
6260
6261 interesting_tasks_empty_waker: Option<Waker>,
6265
6266 ready_for_concurrent_call_waker: Option<Waker>,
6271
6272 event_loop_running: bool,
6274
6275 saved_next_switch_items: Vec<Option<WorkItem>>,
6278
6279 #[cfg(feature = "task-group-hook")]
6281 task_group_hook: Option<Box<dyn TaskGroupHook>>,
6282}
6283
6284impl Default for ConcurrentState {
6285 fn default() -> Self {
6286 Self {
6287 unforced_current_thread: CurrentThread::None,
6288 deferred_host_call_context: None,
6289 table: AlwaysMut::new(ResourceTable::new()),
6290 futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6291 switch_item: None,
6292 next_switch_item: None,
6293 high_priority: VecDeque::new(),
6294 low_priority: VecDeque::new(),
6295 suspend_reason: None,
6296 worker: None,
6297 worker_item: None,
6298 global_error_context_ref_counts: BTreeMap::new(),
6299 interesting_tasks: 0,
6300 interesting_tasks_empty_waker: None,
6301 ready_for_concurrent_call_waker: None,
6302 event_loop_running: false,
6303 saved_next_switch_items: Vec::new(),
6304 #[cfg(feature = "task-group-hook")]
6305 task_group_hook: None,
6306 }
6307 }
6308}
6309
6310impl ConcurrentState {
6311 pub(crate) fn take_fibers_and_futures(
6328 &mut self,
6329 fibers: &mut Vec<StoreFiber<'static>>,
6330 futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6331 ) {
6332 let ConcurrentState {
6333 table,
6334 worker,
6335 switch_item,
6336 next_switch_item,
6337 high_priority,
6338 low_priority,
6339 saved_next_switch_items,
6340 futures: pending_futures,
6341 worker_item: _,
6342 unforced_current_thread: _,
6343 deferred_host_call_context: _,
6344 suspend_reason: _,
6345 global_error_context_ref_counts: _,
6346 interesting_tasks: _,
6347 interesting_tasks_empty_waker: _,
6348 ready_for_concurrent_call_waker: _,
6349 event_loop_running: _,
6350 #[cfg(feature = "task-group-hook")]
6351 task_group_hook: _,
6352 } = self;
6353
6354 let mut items = Vec::new();
6355 for (_, entry) in table.get_mut().iter_mut() {
6356 let entry = entry.into_entry_mut();
6357 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6358 for mode in mem::take(&mut set.waiting).into_values() {
6359 match mode {
6360 WaitMode::Fiber(fiber) => {
6361 fibers.push(fiber);
6362 }
6363 WaitMode::Callback(_) => {}
6364 }
6365 }
6366 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6367 match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6368 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6369 fibers.push(fiber);
6370 }
6371 GuestThreadState::NotStartedImplicit
6372 | GuestThreadState::NotStartedExplicit(_)
6373 | GuestThreadState::Running
6374 | GuestThreadState::Completed => {}
6375 }
6376 } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6377 if let Some(item) = item.take() {
6378 items.push(item);
6379 }
6380 }
6381 }
6382
6383 if let Some(fiber) = worker.take() {
6384 fibers.push(fiber);
6385 }
6386
6387 let mut handle_item = |item| match item {
6388 WorkItem::ResumeFiber { fiber, .. } => {
6389 fibers.push(fiber);
6390 }
6391 WorkItem::PushFuture(future) => {
6392 pending_futures
6393 .get_mut()
6394 .as_mut()
6395 .unwrap()
6396 .push(future.into_inner());
6397 }
6398 WorkItem::ResumeThread { .. }
6399 | WorkItem::GuestCall { .. }
6400 | WorkItem::WorkerFunction(_) => {}
6401 };
6402
6403 for item in items {
6404 handle_item(item);
6405 }
6406 if let Some(item) = switch_item.take() {
6407 handle_item(item);
6408 }
6409 if let Some(item) = next_switch_item.take() {
6410 handle_item(item);
6411 }
6412 for item in mem::take(high_priority) {
6413 handle_item(item);
6414 }
6415 for item in mem::take(low_priority) {
6416 handle_item(item);
6417 }
6418 for item in mem::take(saved_next_switch_items)
6419 .into_iter()
6420 .filter_map(|v| v)
6421 {
6422 handle_item(item);
6423 }
6424
6425 if let Some(them) = pending_futures.get_mut().take() {
6426 futures.push(them);
6427 }
6428 }
6429
6430 #[cfg(feature = "gc")]
6431 pub(crate) fn trace_fiber_roots(
6432 &mut self,
6433 modules: &ModuleRegistry,
6434 unwind: &dyn Unwind,
6435 gc_roots_list: &mut GcRootsList,
6436 ) {
6437 let ConcurrentState {
6438 table,
6439 worker,
6440 switch_item,
6441 next_switch_item,
6442 high_priority,
6443 low_priority,
6444 saved_next_switch_items,
6445
6446 futures: _,
6450
6451 worker_item: _,
6453 unforced_current_thread: _,
6454 deferred_host_call_context: _,
6455 suspend_reason: _,
6456 global_error_context_ref_counts: _,
6457 interesting_tasks: _,
6458 interesting_tasks_empty_waker: _,
6459 ready_for_concurrent_call_waker: _,
6460 event_loop_running: _,
6461 #[cfg(feature = "task-group-hook")]
6462 task_group_hook: _,
6463 } = self;
6464
6465 for (_, entry) in table.get_mut().iter_mut() {
6466 let entry = entry.into_entry_mut();
6467 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6468 for mode in set.waiting.values_mut() {
6469 match mode {
6470 WaitMode::Fiber(fiber) => {
6471 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6472 }
6473 WaitMode::Callback(_) => {}
6474 }
6475 }
6476 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6477 match &mut thread.state {
6478 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6479 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6480 }
6481 GuestThreadState::NotStartedImplicit
6482 | GuestThreadState::NotStartedExplicit(_)
6483 | GuestThreadState::Running
6484 | GuestThreadState::Completed => {}
6485 }
6486 } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6487 entry.downcast_mut::<Option<WorkItem>>()
6488 {
6489 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6490 }
6491 }
6492
6493 if let Some(fiber) = worker {
6494 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6495 }
6496
6497 let mut handle_item = |item: &mut WorkItem| match item {
6498 WorkItem::ResumeFiber { fiber, .. } => {
6499 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6500 }
6501 WorkItem::PushFuture(_future) => {
6502 }
6505 WorkItem::ResumeThread { .. }
6506 | WorkItem::GuestCall { .. }
6507 | WorkItem::WorkerFunction(_) => {}
6508 };
6509
6510 if let Some(item) = switch_item {
6511 handle_item(item);
6512 }
6513 if let Some(item) = next_switch_item {
6514 handle_item(item);
6515 }
6516 for item in high_priority {
6517 handle_item(item);
6518 }
6519 for item in low_priority {
6520 handle_item(item);
6521 }
6522 for item in saved_next_switch_items
6523 .iter_mut()
6524 .filter_map(|v| v.as_mut())
6525 {
6526 handle_item(item);
6527 }
6528 }
6529
6530 fn schedule_suspended_thread(
6531 &mut self,
6532 thread: QualifiedThreadId,
6533 instance: RuntimeInstance,
6534 priority: Priority,
6535 ) -> Result<()> {
6536 if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6538 bail_bug!("switch item already set");
6539 }
6540 let slot = &mut self.get_mut(thread.thread)?.state;
6541 if !matches!(slot, GuestThreadState::Suspended(_)) {
6542 bail!(Trap::CannotResumeThread);
6543 }
6544 let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6545 else {
6546 unreachable!()
6547 };
6548 let item = WorkItem::ResumeFiber {
6549 instance,
6550 thread,
6551 fiber,
6552 };
6553 match priority {
6554 Priority::Switch => self.switch_item = Some(item),
6555 Priority::High => self.push_high_priority(item),
6556 Priority::Low => self.push_low_priority(item),
6557 }
6558 Ok(())
6559 }
6560
6561 fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6562 let slot = &mut self.get_mut(thread)?.state;
6563 slot.check_no_fiber()?;
6564 *slot = GuestThreadState::Running;
6565 Ok(())
6566 }
6567
6568 fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6569 let saved = self.push(None::<WorkItem>)?;
6572 let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6573 *slot = self.next_switch_item.take();
6574 Ok(saved)
6575 }
6576
6577 fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6578 if self.next_switch_item.is_some() {
6579 bail_bug!("next switch item already set when restoring");
6580 }
6581 self.next_switch_item = self.delete(saved)?;
6582 Ok(())
6583 }
6584
6585 fn take_callback_waiter(
6586 &mut self,
6587 set: TableId<WaitableSet>,
6588 thread: QualifiedThreadId,
6589 ) -> Result<Instance> {
6590 let waiting = &mut self.get_mut(set)?.waiting;
6591 let instance = match waiting.get(&thread) {
6592 Some(WaitMode::Callback(instance)) => *instance,
6593 other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6594 };
6595 waiting.remove(&thread);
6596 Ok(instance)
6597 }
6598
6599 fn push<V: Send + Sync + 'static>(
6600 &mut self,
6601 value: V,
6602 ) -> Result<TableId<V>, ResourceTableError> {
6603 self.table.get_mut().push(value).map(TableId::from)
6604 }
6605
6606 fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6607 self.table.get_mut().get_mut(&Resource::from(id))
6608 }
6609
6610 pub fn add_child<T: 'static, U: 'static>(
6611 &mut self,
6612 child: TableId<T>,
6613 parent: TableId<U>,
6614 ) -> Result<(), ResourceTableError> {
6615 self.table
6616 .get_mut()
6617 .add_child(Resource::from(child), Resource::from(parent))
6618 }
6619
6620 pub fn remove_child<T: 'static, U: 'static>(
6621 &mut self,
6622 child: TableId<T>,
6623 parent: TableId<U>,
6624 ) -> Result<(), ResourceTableError> {
6625 self.table
6626 .get_mut()
6627 .remove_child(Resource::from(child), Resource::from(parent))
6628 }
6629
6630 fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6631 self.table.get_mut().delete(Resource::from(id))
6632 }
6633
6634 fn push_future(&mut self, future: HostTaskFuture) {
6635 self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6642 }
6643
6644 fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6645 if self.switch_item.is_some() {
6646 self.push_high_priority(item);
6649 bail_bug!("switch item already set");
6650 }
6651
6652 self.switch_item = Some(item);
6653 log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6654
6655 Ok(())
6656 }
6657
6658 fn take_next_switch_item(&mut self) -> Result<()> {
6659 if let Some(item) = self.next_switch_item.take() {
6660 self.set_switch_item(item)?;
6661 }
6662 Ok(())
6663 }
6664
6665 fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6668 let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6671 return Ok(());
6672 };
6673 let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6674 if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6675 bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6676 }
6677 let instance = self.get_mut(thread.task)?.instance;
6678 self.get_mut(thread.thread)?.wake_on_cancel.take();
6679 let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6680
6681 let item = match mode {
6682 WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6683 instance,
6684 thread,
6685 fiber,
6686 },
6687 WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6688 instance,
6689 call: GuestCall {
6690 thread,
6691 kind: GuestCallKind::DeliverEvent {
6692 instance: callback_instance,
6693 set: Some(set),
6694 },
6695 },
6696 },
6697 };
6698 self.push_high_priority(item);
6699 Ok(())
6700 }
6701
6702 fn push_high_priority(&mut self, item: WorkItem) {
6703 self.high_priority.push_front(item);
6704 log::trace!(
6705 "push high priority: {:?}",
6706 self.high_priority.front().unwrap()
6707 );
6708 }
6709
6710 fn push_low_priority(&mut self, item: WorkItem) {
6711 self.low_priority.push_front(item);
6712 log::trace!(
6713 "push low priority: {:?}",
6714 self.low_priority.front().unwrap()
6715 );
6716 }
6717
6718 fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6719 match priority {
6720 Priority::Switch => self.set_switch_item(item)?,
6721 Priority::High => self.push_high_priority(item),
6722 Priority::Low => self.push_low_priority(item),
6723 }
6724
6725 Ok(())
6726 }
6727
6728 fn promote_instance_local_thread_work_item(
6729 &mut self,
6730 current_instance: RuntimeInstance,
6731 ) -> Result<bool> {
6732 log::trace!("promote thread work items for {current_instance:?}");
6733
6734 self.promote_work_item_matching(|item: &WorkItem| {
6735 let result = match item {
6736 WorkItem::ResumeThread { instance, .. }
6737 | WorkItem::ResumeFiber { instance, .. }
6738 | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6739 _ => false,
6740 };
6741
6742 log::trace!("candidate {item:?}: {result}");
6743 result
6744 })
6745 }
6746
6747 fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6748 self.promote_work_item_matching(|item: &WorkItem| match item {
6749 WorkItem::ResumeThread {
6750 thread: item_thread,
6751 ..
6752 }
6753 | WorkItem::GuestCall {
6754 call:
6755 GuestCall {
6756 thread: item_thread,
6757 ..
6758 },
6759 ..
6760 } => *item_thread == thread,
6761 _ => false,
6762 })
6763 }
6764
6765 fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6766 where
6767 F: FnMut(&WorkItem) -> bool,
6768 {
6769 if self.switch_item.is_none() {
6776 if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6777 self.switch_item = self.high_priority.remove(index);
6778 }
6779 }
6780 if self.switch_item.is_none() {
6781 if let Some(index) = self.low_priority.iter().rposition(predicate) {
6782 self.switch_item = self.low_priority.remove(index);
6783 }
6784 }
6785 Ok(self.switch_item.is_some())
6786 }
6787
6788 pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6791 match task.kind() {
6792 ScopeKind::HostId(task) => {
6793 let task: TableId<HostTask> = TableId::new(task);
6794 Ok(&mut self.get_mut(task)?.call_context)
6795 }
6796 ScopeKind::Id(task) => {
6797 let task: TableId<GuestTask> = TableId::new(task);
6798 Ok(&mut self.get_mut(task)?.call_context)
6799 }
6800 }
6801 }
6802
6803 pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6804 self.deferred_host_call_context.as_mut()
6805 }
6806
6807 fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6808 match self.futures.get_mut().as_mut() {
6809 Some(f) => Ok(f),
6810 None => bail_bug!("futures field of concurrent state is currently taken"),
6811 }
6812 }
6813
6814 pub(crate) fn table(&mut self) -> &mut ResourceTable {
6815 self.table.get_mut()
6816 }
6817
6818 fn debug_assert_deferred_host_invariant(&self) {
6819 debug_assert_eq!(
6820 self.deferred_host_call_context.is_some(),
6821 matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6822 "a deferred host thread and call context must exist together",
6823 );
6824 }
6825
6826 fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6827 self.debug_assert_deferred_host_invariant();
6828 let caller = match self.unforced_current_thread {
6829 CurrentThread::DeferredHost(caller) => caller,
6830 thread => return Ok(thread),
6831 };
6832
6833 let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6835 let task = self.push(task)?;
6836 let call_context = self
6837 .deferred_host_call_context
6838 .take()
6839 .expect("deferred host call context should be present");
6840 self.get_mut(task)
6841 .expect("newly inserted host task should be present")
6842 .call_context = call_context;
6843 self.unforced_current_thread = CurrentThread::Host(task);
6844 self.debug_assert_deferred_host_invariant();
6845 log::trace!("new host task materialized {task:?}");
6846 Ok(CurrentThread::Host(task))
6847 }
6848
6849 fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6850 match self.materialize_host_task()? {
6851 CurrentThread::Host(id) => Ok(Some(id)),
6852 CurrentThread::None => Ok(None),
6853 CurrentThread::Guest(_) => {
6854 bail_bug!("tried to materialize a host task id from a guest thread")
6855 }
6856 CurrentThread::DeferredHost(_) => {
6857 bail_bug!(
6858 "current thread is a deferred host thread which should have been materialized"
6859 )
6860 }
6861 }
6862 }
6863
6864 pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6865 match self.materialize_host_task()? {
6866 CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6867 _ => bail_bug!("current scope is not a deferred host scope"),
6868 }
6869 }
6870
6871 fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6872 let count = &mut self.get_mut(task)?.ref_count;
6873 *count += 1;
6874 log::trace!("increment {task:?} to {count}");
6875 Ok(())
6876 }
6877
6878 fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6879 let count = &mut self.get_mut(task)?.ref_count;
6880 if *count == 0 {
6881 bail_bug!("can't decrement task ref count that's already zero");
6882 }
6883 *count -= 1;
6884 log::trace!("decrement {task:?} to {count}");
6885 if *count == 0 {
6886 let task = self.delete(task)?;
6887
6888 self.decrement_group_ref_count(task.group)?;
6889
6890 debug_assert!(task.decremented_interesting_task_count);
6897 }
6898 Ok(())
6899 }
6900
6901 fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6902 log::trace!("delete host task {task:?}");
6903 let task = self.delete(task)?;
6904
6905 self.decrement_group_ref_count(task.group)
6906 }
6907}
6908
6909fn for_any_lower<
6912 F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6913>(
6914 fun: F,
6915) -> F {
6916 fun
6917}
6918
6919fn for_any_lift<
6921 F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6922>(
6923 fun: F,
6924) -> F {
6925 fun
6926}
6927
6928fn check_ambient_store(id: StoreId) {
6929 let message = "\
6930 `Future`s which depend on asynchronous component tasks, streams, or \
6931 futures to complete may only be polled from the event loop of the \
6932 store to which they belong. Please use \
6933 `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6934 ";
6935 tls::try_get(|store| {
6936 let matched = match store {
6937 tls::TryGet::Some(store) => store.id() == id,
6938 tls::TryGet::Taken | tls::TryGet::None => false,
6939 };
6940
6941 if !matched {
6942 panic!("{message}")
6943 }
6944 });
6945}
6946
6947fn unpack_callback_code(code: u32) -> (u32, u32) {
6948 (code & 0xF, code >> 4)
6949}
6950
6951struct WaitableCheckParams {
6955 set: TableId<WaitableSet>,
6956 options: OptionsIndex,
6957 payload: u32,
6958}
6959
6960enum WaitableCheck {
6963 Wait,
6964 Poll,
6965}
6966
6967pub(crate) struct PreparedCall<R> {
6969 handle: Func,
6971 thread: QualifiedThreadId,
6973 param_count: usize,
6975 rx: oneshot::Receiver<LiftedResult>,
6978 _phantom: PhantomData<R>,
6979}
6980
6981impl<R> PreparedCall<R> {
6982 pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6989 store
6990 .concurrent_state_mut()?
6991 .increment_task_ref_count(self.thread.task)?;
6992
6993 Ok(TaskId {
6994 task: self.thread.task,
6995 })
6996 }
6997}
6998
6999pub(crate) struct TaskId {
7001 task: TableId<GuestTask>,
7002}
7003
7004impl TaskId {
7005 pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
7017 let task = store.concurrent_state_mut()?.get_mut(self.task)?;
7018 if !task.already_lowered_parameters() {
7019 task.lower_params = Some(Box::new(|store, _| {
7024 store.set_trapped();
7025 bail!("the host future for this call was dropped before it started")
7026 }));
7027 } else {
7028 _ = store
7033 .concurrent_state_mut()?
7034 .decrement_task_ref_count(self.task);
7035 }
7036
7037 Ok(())
7038 }
7039}
7040
7041pub(crate) fn prepare_call<T, R>(
7047 mut store: StoreContextMut<T>,
7048 handle: Func,
7049 param_count: usize,
7050 lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
7051 + Send
7052 + Sync
7053 + 'static,
7054 lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
7055 + Send
7056 + Sync
7057 + 'static,
7058) -> Result<PreparedCall<R>> {
7059 if !store.0.may_enter() {
7063 bail!(Trap::CannotEnterComponent);
7064 }
7065
7066 let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
7067
7068 let instance = handle.instance().id().get(store.0);
7069 let options = &instance.component().env_component().options[options];
7070 let ty = &instance.component().types()[ty];
7071 let async_typed = ty.async_;
7072 let async_lifted = raw_options.async_;
7073 let task_return_type = ty.results;
7074 let component_instance = raw_options.instance;
7075 let callback = options.callback.map(|i| instance.runtime_callback(i));
7076 let memory = options
7077 .memory()
7078 .map(|i| instance.runtime_memory(i))
7079 .map(SendSyncPtr::new);
7080 let string_encoding = options.string_encoding;
7081 let token = StoreToken::new(store.as_context_mut());
7082 let caller = store.0.materialize_host_task_id()?;
7083 let state = store.0.concurrent_state_mut()?;
7084
7085 let (tx, rx) = oneshot::channel();
7086
7087 let instance = handle.instance().runtime_instance(component_instance);
7088 let thread = GuestTask::new(
7089 state,
7090 Box::new(for_any_lower(move |store, params| {
7091 lower_params(token.as_context_mut(store), params)
7092 })),
7093 LiftResult {
7094 lift: Box::new(for_any_lift(move |store, result| {
7095 lift_result(store, result)
7096 })),
7097 ty: task_return_type,
7098 memory,
7099 string_encoding,
7100 },
7101 Caller::Host {
7102 tx: Some(tx),
7103 caller,
7104 },
7105 callback.map(|callback| {
7106 let callback = SendSyncPtr::new(callback);
7107 let instance = handle.instance();
7108 Box::new(move |store: &mut dyn VMStore, event, handle| {
7109 let store = token.as_context_mut(store);
7110 unsafe { instance.call_callback(store, callback, event, handle) }
7113 }) as CallbackFn
7114 }),
7115 instance,
7116 async_typed,
7117 async_lifted,
7118 )?;
7119
7120 Ok(PreparedCall {
7121 handle,
7122 thread,
7123 param_count,
7124 rx,
7125 _phantom: PhantomData,
7126 })
7127}
7128
7129pub(crate) struct StagedCall<R> {
7130 store: StoreId,
7131 rx: oneshot::Receiver<LiftedResult>,
7132 _marker: PhantomData<fn() -> R>,
7133 group: TaskGroupId,
7141}
7142
7143impl<R> StagedCall<R> {
7144 pub(crate) fn new<T: 'static>(
7155 mut store: StoreContextMut<T>,
7156 prepared: PreparedCall<R>,
7157 ) -> Result<StagedCall<R>> {
7158 let PreparedCall {
7159 handle,
7160 thread,
7161 param_count,
7162 rx,
7163 ..
7164 } = prepared;
7165
7166 stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7167
7168 let state = store.0.concurrent_state_mut()?;
7169 let group = state.get_mut(thread.task)?.group;
7170 state.decrement_task_ref_count(thread.task)?;
7171
7172 Ok(StagedCall {
7173 store: store.0.id(),
7174 rx,
7175 _marker: PhantomData,
7176 group,
7177 })
7178 }
7179}
7180
7181impl<R> Future for StagedCall<R>
7182where
7183 R: 'static,
7184{
7185 type Output = Result<R>;
7186
7187 fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7188 check_ambient_store(self.store);
7189 Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7190 Ok(r) => match r.downcast() {
7191 Ok(r) => Ok(*r),
7192 Err(_) => bail_bug!("wrong type of value produced"),
7193 },
7194 Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7195 })
7196 }
7197}
7198
7199fn stage_call0<T: 'static>(
7202 store: StoreContextMut<T>,
7203 handle: Func,
7204 guest_thread: QualifiedThreadId,
7205 param_count: usize,
7206) -> Result<()> {
7207 let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7208 let is_concurrent = raw_options.async_;
7209 let callback = raw_options.callback;
7210 let instance = handle.instance();
7211 let callee = handle.lifted_core_func(store.0);
7212 let post_return = raw_options
7213 .post_return
7214 .map(|i| instance.id().get(store.0).runtime_post_return(i));
7215 let callback = callback.map(|i| {
7216 let instance = instance.id().get(store.0);
7217 SendSyncPtr::new(instance.runtime_callback(i))
7218 });
7219
7220 log::trace!("queueing call {guest_thread:?}");
7221
7222 unsafe {
7226 instance.stage_call(
7227 store,
7228 guest_thread,
7229 SendSyncPtr::new(callee),
7230 param_count,
7231 1,
7232 is_concurrent,
7233 callback,
7234 post_return.map(SendSyncPtr::new),
7235 true,
7236 )
7237 }
7238}
7239
7240#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7241mod tests {
7242 use super::*;
7243 use crate::component::{Component, Linker};
7244 use crate::store::AsStoreOpaque;
7245 use crate::{Config, Engine};
7246
7247 fn host_subtask(
7248 state: HostTaskState,
7249 event: Option<Event>,
7250 ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7251 let mut config = Config::new();
7252 config.wasm_component_model_async(true);
7253 let engine = Engine::new(&config)?;
7254 let component = Component::new(&engine, "(component)")?;
7255 let mut store = Store::new(&engine, ());
7256 let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7257 let store_opaque = store.as_store_opaque();
7258 let concurrent_state = store_opaque.concurrent_state_mut()?;
7259 let group = concurrent_state.make_task_group()?;
7262 let task = concurrent_state.push(HostTask {
7263 common: WaitableCommon::default(),
7264 call_context: CallContext::default(),
7265 state,
7266 group,
7267 })?;
7268 let handle = store_opaque
7269 .handle_table_insertion(
7270 instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7271 )
7272 .subtask_host(task.rep())?;
7273 let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7274 common.handle = Some(handle);
7275 common.event = event;
7276 Ok((store, instance, task, handle))
7277 }
7278
7279 #[test]
7280 fn host_subtask_drop_during_cancellation() -> Result<()> {
7281 for abort_completed in [false, true] {
7282 let (handle, future) = JoinHandle::run(future::pending::<()>());
7283 let mut future = pin!(future);
7284 let (mut store, instance, task, handle) =
7285 host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7286 let store = store.as_store_opaque();
7287 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7288 assert_eq!(
7289 instance.subtask_cancel(store, caller, true, handle)?,
7290 BLOCKED
7291 );
7292 if abort_completed {
7293 assert!(matches!(
7296 future
7297 .as_mut()
7298 .poll(&mut Context::from_waker(Waker::noop())),
7299 Poll::Ready(None),
7300 ));
7301 }
7302 for async_ in [false, true] {
7303 let err = instance
7304 .subtask_cancel(store, caller, async_, handle)
7305 .unwrap_err();
7306 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7307 }
7308 let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7309 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7310 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7311 }
7312 Ok(())
7313 }
7314
7315 #[test]
7316 fn host_subtask_cancel_after_completion() -> Result<()> {
7317 for async_ in [false, true] {
7318 let (mut store, instance, task, handle) = host_subtask(
7319 HostTaskState::CalleeDone { cancelled: false },
7320 Some(Event::Subtask {
7321 status: Status::Returned,
7322 }),
7323 )?;
7324 let store = store.as_store_opaque();
7325 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7326 assert_eq!(
7327 instance.subtask_cancel(store, caller, async_, handle)?,
7328 Status::Returned as u32,
7329 );
7330 let err = instance
7331 .subtask_cancel(store, caller, async_, handle)
7332 .unwrap_err();
7333 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7334 instance.subtask_drop(store, caller, handle)?;
7335 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7336 }
7337 Ok(())
7338 }
7339
7340 #[test]
7341 fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7342 for (cancelled, status) in [
7343 (false, Status::Returned),
7344 (true, Status::Returned),
7345 (true, Status::ReturnCancelled),
7346 ] {
7347 for delivered in [false, true] {
7348 let event = if delivered {
7349 None
7350 } else {
7351 Some(Event::Subtask { status })
7352 };
7353 let (mut store, instance, task, handle) =
7354 host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7355 let store = store.as_store_opaque();
7356 let result = instance.subtask_drop(
7357 store,
7358 RuntimeComponentInstanceIndex::from_u32(0),
7359 handle,
7360 );
7361 if delivered {
7362 result?;
7363 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7364 } else {
7365 let err = result.unwrap_err();
7366 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7367 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7368 }
7369 }
7370 }
7371 Ok(())
7372 }
7373}