1use self::error_contexts::GlobalErrorContextRefCount;
54use crate::component::func::{Func, call_post_return};
55use crate::component::store::ComponentInstanceId;
56use crate::component::{
57 HasData, HasSelf, Instance, Resource, ResourceTable, ResourceTableError, RuntimeInstance,
58};
59use crate::fiber::{self, StoreFiber, StoreFiberYield};
60use crate::hash_set::HashSet;
61#[cfg(feature = "gc")]
62use crate::module::ModuleRegistry;
63use crate::prelude::*;
64use crate::store::{Store, StoreId, StoreInner, StoreOpaque, StoreToken};
65#[cfg(feature = "gc")]
66use crate::vm::GcRootsList;
67use crate::vm::VMScope;
68use crate::vm::component::ScopeKind;
69use crate::vm::component::{
70 CallContext, ComponentInstance, CurrentScope, HandleCounter, HandleTableInsertion,
71 InstanceState,
72};
73use crate::vm::{
74 AlwaysMut, SendSyncPtr, UncaughtException, VMFuncRef, VMLazyThread, VMMemoryDefinition, VMStore,
75};
76use crate::{AsContext, AsContextMut, Result, StoreContext, StoreContextMut, ValRaw, bail};
77use crate::{Instance as ModuleInstance, bail_bug};
78use alloc::borrow::ToOwned;
79use alloc::collections::btree_map::Entry;
80use alloc::collections::{BTreeMap, BTreeSet, VecDeque};
81use core::any::Any;
82use core::cell::UnsafeCell;
83use core::fmt;
84use core::future;
85use core::future::Future;
86use core::marker::PhantomData;
87use core::mem::{self, ManuallyDrop, MaybeUninit};
88use core::ops::DerefMut;
89use core::pin::{Pin, pin};
90use core::ptr::{self, NonNull};
91use core::task::{Context, Poll, Waker};
92use futures::channel::oneshot;
93use futures::stream::{FuturesUnordered, StreamExt};
94use futures_and_streams::{FlatAbi, ReturnCode, TransmitHandle, TransmitIndex};
95use smallvec::SmallVec;
96use table::{TableDebug, TableId};
97use wasmtime_environ::component::{
98 CanonicalAbiInfo, CanonicalOptions, CanonicalOptionsDataModel, MAX_FLAT_PARAMS,
99 MAX_FLAT_RESULTS, OptionsIndex, PREPARE_ASYNC_NO_RESULT, PREPARE_ASYNC_WITH_RESULT,
100 RuntimeComponentInstanceIndex, RuntimeTableIndex, StringEncoding,
101 TypeComponentGlobalErrorContextTableIndex, TypeComponentLocalErrorContextTableIndex,
102 TypeFutureTableIndex, TypeStreamTableIndex, TypeTupleIndex,
103};
104use wasmtime_environ::packed_option::ReservedValue;
105use wasmtime_environ::{ModuleInternedTypeIndex, NUM_COMPONENT_CONTEXT_SLOTS, Trap};
106#[cfg(feature = "gc")]
107use wasmtime_unwinder::Unwind;
108
109pub use abort::JoinHandle;
110pub use func::{FuncCallConcurrent, TypedFuncCallConcurrent};
111pub use future_stream_any::{FutureAny, StreamAny};
112pub use futures_and_streams::{
113 Destination, DirectDestination, DirectSource, ErrorContext, FutureConsumer, FutureProducer,
114 FutureReader, GuardedFutureReader, GuardedStreamReader, ReadBuffer, Source, StreamConsumer,
115 StreamProducer, StreamReader, StreamResult, VecBuffer, WriteBuffer,
116};
117pub(crate) use futures_and_streams::{ResourcePair, lower_error_context_to_index};
118#[cfg(feature = "task-group-hook")]
119pub use task_group_hook::TaskGroupHook;
120pub use task_group_hook::TaskGroupId;
121
122mod abort;
123mod error_contexts;
124mod func;
125mod future_stream_any;
126mod futures_and_streams;
127pub(crate) mod table;
128#[cfg(feature = "task-group-hook")]
129mod task_group_hook;
130#[cfg(not(feature = "task-group-hook"))]
131mod task_group_hook_disabled;
132#[cfg(not(feature = "task-group-hook"))]
133use task_group_hook_disabled as task_group_hook;
134pub(crate) mod tls;
135
136const BLOCKED: u32 = 0xffff_ffff;
139
140#[derive(Clone, Copy, Eq, PartialEq, Debug)]
142pub enum Status {
143 Starting = 0,
144 Started = 1,
145 Returned = 2,
146 StartCancelled = 3,
147 ReturnCancelled = 4,
148}
149
150impl Status {
151 pub fn pack(self, waitable: Option<u32>) -> u32 {
157 assert!(matches!(self, Status::Returned) == waitable.is_none());
158 let waitable = waitable.unwrap_or(0);
159 assert!(waitable < (1 << 28));
160 (waitable << 4) | (self as u32)
161 }
162}
163
164#[derive(Clone, Copy, Debug)]
167enum Event {
168 None,
169 Subtask {
170 status: Status,
171 },
172 StreamRead {
173 code: ReturnCode,
174 pending: Option<(TypeStreamTableIndex, u32)>,
175 },
176 StreamWrite {
177 code: ReturnCode,
178 pending: Option<(TypeStreamTableIndex, u32)>,
179 },
180 FutureRead {
181 code: ReturnCode,
182 pending: Option<(TypeFutureTableIndex, u32)>,
183 },
184 FutureWrite {
185 code: ReturnCode,
186 pending: Option<(TypeFutureTableIndex, u32)>,
187 },
188 Cancelled,
189}
190
191impl Event {
192 fn parts(self) -> (u32, u32) {
197 const EVENT_NONE: u32 = 0;
198 const EVENT_SUBTASK: u32 = 1;
199 const EVENT_STREAM_READ: u32 = 2;
200 const EVENT_STREAM_WRITE: u32 = 3;
201 const EVENT_FUTURE_READ: u32 = 4;
202 const EVENT_FUTURE_WRITE: u32 = 5;
203 const EVENT_CANCELLED: u32 = 6;
204 match self {
205 Event::None => (EVENT_NONE, 0),
206 Event::Cancelled => (EVENT_CANCELLED, 0),
207 Event::Subtask { status } => (EVENT_SUBTASK, status as u32),
208 Event::StreamRead { code, .. } => (EVENT_STREAM_READ, code.encode()),
209 Event::StreamWrite { code, .. } => (EVENT_STREAM_WRITE, code.encode()),
210 Event::FutureRead { code, .. } => (EVENT_FUTURE_READ, code.encode()),
211 Event::FutureWrite { code, .. } => (EVENT_FUTURE_WRITE, code.encode()),
212 }
213 }
214}
215
216mod callback_code {
218 pub const EXIT: u32 = 0;
219 pub const YIELD: u32 = 1;
220 pub const WAIT: u32 = 2;
221}
222
223const START_FLAG_ASYNC_CALLEE: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLEE as u32;
225const START_FLAG_ASYNC_CALLER: u32 = wasmtime_environ::component::START_FLAG_ASYNC_CALLER as u32;
226
227pub struct Access<'a, T: 'static, D: HasData + ?Sized = HasSelf<T>> {
233 store: StoreContextMut<'a, T>,
234 get_data: fn(&mut T) -> D::Data<'_>,
235}
236
237impl<'a, T, D> Access<'a, T, D>
238where
239 D: HasData + ?Sized,
240 T: 'static,
241{
242 pub fn new(store: StoreContextMut<'a, T>, get_data: fn(&mut T) -> D::Data<'_>) -> Self {
244 Self { store, get_data }
245 }
246
247 pub fn data_mut(&mut self) -> &mut T {
249 self.store.data_mut()
250 }
251
252 pub fn get(&mut self) -> D::Data<'_> {
254 (self.get_data)(self.data_mut())
255 }
256
257 pub fn spawn(&mut self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
261 where
262 T: 'static,
263 {
264 let accessor = Accessor {
265 get_data: self.get_data,
266 token: StoreToken::new(self.store.as_context_mut()),
267 };
268 self.store
269 .as_context_mut()
270 .spawn_with_accessor(accessor, task)
271 }
272
273 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
276 self.get_data
277 }
278}
279
280impl<'a, T, D> AsContext for Access<'a, T, D>
281where
282 D: HasData + ?Sized,
283 T: 'static,
284{
285 type Data = T;
286
287 fn as_context(&self) -> StoreContext<'_, T> {
288 self.store.as_context()
289 }
290}
291
292impl<'a, T, D> AsContextMut for Access<'a, T, D>
293where
294 D: HasData + ?Sized,
295 T: 'static,
296{
297 fn as_context_mut(&mut self) -> StoreContextMut<'_, T> {
298 self.store.as_context_mut()
299 }
300}
301
302pub struct Accessor<T: 'static, D = HasSelf<T>>
362where
363 D: HasData + ?Sized,
364{
365 token: StoreToken<T>,
366 get_data: fn(&mut T) -> D::Data<'_>,
367}
368
369pub trait AsAccessor {
386 type Data: 'static;
388
389 type AccessorData: HasData + ?Sized;
392
393 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData>;
395}
396
397impl<T: AsAccessor + ?Sized> AsAccessor for &T {
398 type Data = T::Data;
399 type AccessorData = T::AccessorData;
400
401 fn as_accessor(&self) -> &Accessor<Self::Data, Self::AccessorData> {
402 T::as_accessor(self)
403 }
404}
405
406impl<T, D: HasData + ?Sized> AsAccessor for Accessor<T, D> {
407 type Data = T;
408 type AccessorData = D;
409
410 fn as_accessor(&self) -> &Accessor<T, D> {
411 self
412 }
413}
414
415const _: () = {
438 const fn assert<T: Send + Sync>() {}
439 assert::<Accessor<UnsafeCell<u32>>>();
440};
441
442impl<T> Accessor<T> {
443 pub(crate) fn new(token: StoreToken<T>) -> Self {
452 Self {
453 token,
454 get_data: |x| x,
455 }
456 }
457}
458
459impl<T, D> Accessor<T, D>
460where
461 D: HasData + ?Sized,
462{
463 pub fn with<R>(&self, fun: impl FnOnce(Access<'_, T, D>) -> R) -> R {
481 tls::get(|vmstore| {
482 fun(Access {
483 store: self.token.as_context_mut(vmstore),
484 get_data: self.get_data,
485 })
486 })
487 }
488
489 pub fn getter(&self) -> fn(&mut T) -> D::Data<'_> {
492 self.get_data
493 }
494
495 pub fn with_getter<D2: HasData>(
512 &self,
513 get_data: fn(&mut T) -> D2::Data<'_>,
514 ) -> Accessor<T, D2> {
515 Accessor {
516 token: self.token,
517 get_data,
518 }
519 }
520
521 pub fn spawn(&self, task: impl for<'fut> AccessorTask<'fut, T, D>) -> Result<JoinHandle>
537 where
538 T: 'static,
539 {
540 let accessor = self.clone_for_spawn();
541 self.with(|mut access| access.as_context_mut().spawn_with_accessor(accessor, task))
542 }
543
544 fn clone_for_spawn(&self) -> Self {
545 Self {
546 token: self.token,
547 get_data: self.get_data,
548 }
549 }
550
551 pub fn poll_no_interesting_tasks(&self, cx: &mut Context<'_>) -> Poll<()> {
587 self.with(|mut access| {
588 let store = access.as_context_mut().0;
589 let state = store.concurrent_state_mut_without_forcing_current_thread();
590 if state.interesting_tasks == 0 {
591 Poll::Ready(())
592 } else {
593 state.interesting_tasks_empty_waker = Some(cx.waker().clone());
594 Poll::Pending
595 }
596 })
597 }
598
599 pub fn poll_ready_for_concurrent_call(&self, func: Func, cx: &mut Context<'_>) -> Poll<()> {
623 self.with(|mut access| {
624 let store = access.as_context_mut().0;
625
626 {
627 let state = store.concurrent_state_mut_without_forcing_current_thread();
628 if state.switch_item.is_some()
629 || !state.high_priority.is_empty()
630 || !state.low_priority.is_empty()
631 {
632 state.ready_for_concurrent_call_waker = Some(cx.waker().clone());
633 return Poll::Pending;
634 }
635 }
636
637 let (_, _, _, raw_options) = func.abi_info(store);
638 let instance = func.instance().runtime_instance(raw_options.instance);
639 let state = store.instance_state(instance).concurrent_state();
640 if state.backpressure == 0 {
641 Poll::Ready(())
642 } else {
643 store
644 .concurrent_state_mut_without_forcing_current_thread()
645 .ready_for_concurrent_call_waker = Some(cx.waker().clone());
646 Poll::Pending
647 }
648 })
649 }
650}
651
652pub trait AccessorTask<'fut, T, D = HasSelf<T>>:
674 AsyncFnOnce(&Accessor<T, D>) -> Result<()> + Send + 'static
675where
676 D: HasData + ?Sized,
677{
678 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut;
680}
681
682impl<'fut, F, Fut, T, D> AccessorTask<'fut, T, D> for F
683where
684 T: 'static,
685 F: AsyncFnOnce(&Accessor<T, D>) -> Result<()>,
686 F: FnOnce(&'fut Accessor<T, D>) -> Fut + Send + 'static,
687 Fut: Future<Output = Result<()>> + Send + 'fut,
688 D: HasData,
689{
690 fn run(self, accessor: &'fut Accessor<T, D>) -> impl Future<Output = Result<()>> + Send + 'fut {
691 (self)(accessor)
692 }
693}
694
695enum CallerInfo {
698 Async {
700 params: Vec<ValRaw>,
701 has_result: bool,
702 },
703 Sync {
705 params: Vec<ValRaw>,
706 result_count: u32,
707 },
708}
709
710enum WaitMode {
712 Fiber(StoreFiber<'static>),
714 Callback(Instance),
717}
718
719impl fmt::Debug for WaitMode {
720 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
721 match self {
722 Self::Fiber(_) => f.debug_tuple("Fiber").finish(),
723 Self::Callback(instance) => f.debug_tuple("Callback").field(instance).finish(),
724 }
725 }
726}
727
728#[derive(Debug)]
730enum SuspendReason {
731 Waiting {
734 set: TableId<WaitableSet>,
735 thread: QualifiedThreadId,
736 },
737 YieldingToSubtask { thread: QualifiedThreadId },
740 NeedWork,
743 Yielding { thread: QualifiedThreadId },
746 ExplicitlySuspending { thread: QualifiedThreadId },
749}
750
751enum GuestCallKind {
753 DeliverEvent {
756 instance: Instance,
758 set: Option<TableId<WaitableSet>>,
763 },
764 StartImplicit {
767 closure: Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>,
769 was_pending: bool,
771 },
772 StartExplicit(Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>),
773}
774
775impl fmt::Debug for GuestCallKind {
776 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
777 match self {
778 Self::DeliverEvent { instance, set } => f
779 .debug_struct("DeliverEvent")
780 .field("instance", instance)
781 .field("set", set)
782 .finish(),
783 Self::StartImplicit { was_pending, .. } => f
784 .debug_struct("StartImplicit")
785 .field("was_pending", was_pending)
786 .finish(),
787 Self::StartExplicit(_) => f.debug_tuple("StartExplicit").finish(),
788 }
789 }
790}
791
792#[derive(Copy, Clone, Debug)]
794pub enum SuspensionTarget {
795 Resume(u32),
796 Promote(u32),
797 None,
798}
799
800#[derive(Copy, Clone, Debug)]
802pub enum ResumeThread {
803 Promote,
804 Resume,
805 ResumeLater,
806}
807
808#[derive(Debug)]
810struct GuestCall {
811 thread: QualifiedThreadId,
812 kind: GuestCallKind,
813}
814
815impl GuestCall {
816 fn is_ready(&self, store: &mut StoreOpaque) -> Result<bool> {
828 let task = store.concurrent_state_mut()?.get_mut(self.thread.task)?;
829 let async_typed = task.async_typed;
830 let async_lifted = task.async_lifted;
831 let has_callback = task.callback.is_some();
832 let instance = task.instance;
833 let state = store.instance_state(instance).concurrent_state();
834
835 let ready = match &self.kind {
836 GuestCallKind::DeliverEvent { .. } => !state.do_not_enter,
837 GuestCallKind::StartImplicit { was_pending, .. } => {
838 if async_typed {
839 let needs_exclusive = !async_lifted || has_callback;
842
843 let may_enter =
848 !((needs_exclusive && state.do_not_enter) || state.backpressure > 0);
849
850 let skip_wait = *was_pending || state.pending_start_count == 0;
857
858 may_enter && skip_wait
861 } else {
862 true
863 }
864 }
865 GuestCallKind::StartExplicit(_) => true,
866 };
867 log::trace!(
868 "call {self:?} ready? {ready} (do_not_enter: {}; backpressure: {}; pending_start_count: {})",
869 state.do_not_enter,
870 state.backpressure,
871 state.pending_start_count,
872 );
873 Ok(ready)
874 }
875}
876
877enum WorkerItem {
879 GuestCall(GuestCall),
880 Function(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
881}
882
883enum WorkItem {
886 PushFuture(AlwaysMut<HostTaskFuture>),
888 ResumeFiber {
890 instance: RuntimeInstance,
891 thread: QualifiedThreadId,
892 fiber: StoreFiber<'static>,
893 },
894 ResumeThread {
896 instance: RuntimeInstance,
897 thread: QualifiedThreadId,
898 },
899 GuestCall {
901 instance: RuntimeInstance,
902 call: GuestCall,
903 },
904 WorkerFunction(AlwaysMut<Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send>>),
906}
907
908struct ResumingFiber<'a> {
914 store: &'a mut StoreOpaque,
915 fiber: Option<StoreFiber<'static>>,
916}
917
918impl Drop for ResumingFiber<'_> {
919 fn drop(&mut self) {
920 if let Some(mut fiber) = self.fiber.take() {
921 fiber.dispose(self.store);
922 }
923 }
924}
925
926impl fmt::Debug for WorkItem {
927 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
928 match self {
929 Self::PushFuture(_) => f.debug_tuple("PushFuture").finish(),
930 Self::ResumeFiber {
931 instance, thread, ..
932 } => f
933 .debug_struct("ResumeFiber")
934 .field("instance", instance)
935 .field("thread", thread)
936 .finish(),
937 Self::ResumeThread { instance, thread } => f
938 .debug_struct("ResumeThread")
939 .field("instance", instance)
940 .field("thread", thread)
941 .finish(),
942 Self::GuestCall { instance, call } => f
943 .debug_struct("GuestCall")
944 .field("instance", instance)
945 .field("call", call)
946 .finish(),
947 Self::WorkerFunction(_) => f.debug_tuple("WorkerFunction").finish(),
948 }
949 }
950}
951
952#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
954pub(crate) enum WaitResult {
955 Cancelled,
956 Completed,
957}
958
959pub(crate) fn poll_and_block<R: Send + Sync + 'static>(
967 store: &mut dyn VMStore,
968 host_task: EnteredHostTask,
969 future: impl Future<Output = Result<R>> + Send + 'static,
970) -> Result<R> {
971 let mut future = Box::pin(future);
978 let poll = tls::set(store, || {
979 future
980 .as_mut()
981 .poll(&mut Context::from_waker(&Waker::noop()))
982 });
983
984 let caller = match host_task {
985 Some(caller) => caller,
986 None => bail_bug!("host task wasn't created but should have been"),
987 };
988
989 let task = match poll {
990 Poll::Ready(result) => return result,
992
993 Poll::Pending => {
998 let Some(task) = store.materialize_host_task_id()? else {
999 bail_bug!("current thread is not a host thread")
1000 };
1001
1002 let future = Box::pin(async move {
1005 let result = run_with_host_task_set(task, future).await??;
1006 tls::get(move |store| {
1007 let state = store.concurrent_state_mut()?;
1008 let host_state = &mut state.get_mut(task)?.state;
1009 assert!(matches!(host_state, HostTaskState::CalleeStarted));
1010 *host_state = HostTaskState::CalleeFinished(Box::new(result));
1011
1012 Waitable::Host(task).set_event(
1013 state,
1014 Some(Event::Subtask {
1015 status: Status::Returned,
1016 }),
1017 )?;
1018
1019 Ok(())
1020 })
1021 }) as HostTaskFuture;
1022
1023 let caller_instance = store.concurrent_state_mut()?.get_mut(caller.task)?.instance;
1024 store.switch_or_trap_if_may_not_suspend(caller_instance)?;
1025
1026 let state = store.concurrent_state_mut()?;
1027 state.push_future(future);
1028
1029 let set = state.get_mut(caller.thread)?.sync_call_set;
1030 Waitable::Host(task).join(state, Some(set))?;
1031
1032 store.suspend(SuspendReason::Waiting {
1033 set,
1034 thread: caller,
1035 })?;
1036
1037 Waitable::Host(task).join(store.concurrent_state_mut()?, None)?;
1041 task
1042 }
1043 };
1044
1045 let host_state = &mut store.concurrent_state_mut()?.get_mut(task)?.state;
1047 match mem::replace(host_state, HostTaskState::CalleeDone { cancelled: false }) {
1048 HostTaskState::CalleeFinished(result) => Ok(match result.downcast() {
1049 Ok(result) => *result,
1050 Err(_) => bail_bug!("host task finished with wrong type of result"),
1051 }),
1052 _ => bail_bug!("unexpected host task state after completion"),
1053 }
1054}
1055
1056fn handle_guest_call(store: &mut dyn VMStore, call: GuestCall) -> Result<()> {
1058 match call.kind {
1059 GuestCallKind::DeliverEvent { instance, set } => {
1060 if let Some(set) = set {
1063 store.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
1064 }
1065 let (event, waitable) = match instance.get_event(store, call.thread.task, set, true)? {
1066 Some(pair) => pair,
1067 None => match set {
1068 Some(set) => {
1073 log::trace!(
1074 "event for {:?} on {set:?} no longer present; waiting again",
1075 call.thread
1076 );
1077 return instance.wait_with_callback(
1078 store.store_opaque_mut(),
1079 call.thread,
1080 set,
1081 );
1082 }
1083 None => (Event::None, None),
1088 },
1089 };
1090 let state = store.concurrent_state_mut()?;
1091 if let Some(set) = set
1096 && waitable.is_none()
1097 && !state.get_mut(set)?.ready.is_empty()
1098 {
1099 state.wake_waiter(set)?;
1100 }
1101 let task = state.get_mut(call.thread.task)?;
1102 let runtime_instance = task.instance;
1103 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
1104
1105 log::trace!(
1106 "use callback to deliver event {event:?} to {:?} for {waitable:?}",
1107 call.thread,
1108 );
1109
1110 instance.handle_callback_code(store, call.thread, runtime_instance, |store| {
1111 invoke_callback(store, call.thread.task, event, handle)
1112 })?;
1113 }
1114 GuestCallKind::StartImplicit { closure, .. } => {
1115 closure(store)?;
1116 }
1117 GuestCallKind::StartExplicit(fun) => {
1118 fun(store)?;
1119 }
1120 }
1121
1122 Ok(())
1123}
1124
1125fn invoke_callback(
1128 store: &mut dyn VMStore,
1129 task: TableId<GuestTask>,
1130 event: Event,
1131 handle: u32,
1132) -> Result<u32> {
1133 let Some(callback) = store.concurrent_state_mut()?.get_mut(task)?.callback.take() else {
1134 bail_bug!("guest task callback field not present")
1135 };
1136
1137 let code = callback(store, event, handle)?;
1138
1139 store.concurrent_state_mut()?.get_mut(task)?.callback = Some(callback);
1140
1141 Ok(code)
1142}
1143
1144impl<T> Store<T> {
1145 pub async fn run_concurrent<R>(&mut self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1147 where
1148 T: Send + 'static,
1149 {
1150 ensure!(
1151 self.as_context().0.concurrency_support(),
1152 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1153 );
1154 self.as_context_mut().run_concurrent(fun).await
1155 }
1156
1157 #[doc(hidden)]
1158 pub fn assert_concurrent_state_empty(&mut self) {
1159 self.as_context_mut().assert_concurrent_state_empty();
1160 }
1161
1162 #[doc(hidden)]
1163 pub fn concurrent_state_table_size(&mut self) -> usize {
1164 self.as_context_mut().concurrent_state_table_size()
1165 }
1166
1167 pub fn spawn(
1169 &mut self,
1170 task: impl for<'fut> AccessorTask<'fut, T, HasSelf<T>>,
1171 ) -> Result<JoinHandle>
1172 where
1173 T: 'static,
1174 {
1175 self.as_context_mut().spawn(task)
1176 }
1177}
1178
1179impl<T> StoreContextMut<'_, T> {
1180 #[doc(hidden)]
1191 pub fn assert_concurrent_state_empty(self) {
1192 let store = self.0;
1193 store
1194 .store_data_mut()
1195 .components
1196 .assert_instance_states_empty();
1197 let state = store.concurrent_state_mut().unwrap();
1198 assert!(
1199 state.table.get_mut().is_empty(),
1200 "non-empty table: {:?}",
1201 state.table.get_mut()
1202 );
1203 assert!(state.switch_item.is_none());
1204 assert!(state.next_switch_item.is_none());
1205 assert!(state.high_priority.is_empty());
1206 assert!(state.low_priority.is_empty());
1207 assert!(state.saved_next_switch_items.is_empty());
1208 assert!(state.unforced_current_thread.is_none());
1209 assert!(state.deferred_host_call_context.is_none());
1210 assert!(state.futures_mut().unwrap().is_empty());
1211 assert!(state.global_error_context_ref_counts.is_empty());
1212 }
1213
1214 #[doc(hidden)]
1219 pub fn concurrent_state_table_size(&mut self) -> usize {
1220 self.0.concurrent_state_mut().unwrap().table.get_mut().len()
1221 }
1222
1223 pub fn spawn(mut self, task: impl for<'fut> AccessorTask<'fut, T>) -> Result<JoinHandle>
1233 where
1234 T: 'static,
1235 {
1236 let accessor = Accessor::new(StoreToken::new(self.as_context_mut()));
1237 self.spawn_with_accessor(accessor, task)
1238 }
1239
1240 fn spawn_with_accessor<D>(
1243 self,
1244 accessor: Accessor<T, D>,
1245 task: impl for<'fut> AccessorTask<'fut, T, D>,
1246 ) -> Result<JoinHandle>
1247 where
1248 T: 'static,
1249 D: HasData + ?Sized,
1250 {
1251 let (handle, future) = JoinHandle::run(async move { task.run(&accessor).await });
1255 self.0
1256 .concurrent_state_mut()?
1257 .push_future(Box::pin(async move { future.await.unwrap_or(Ok(())) }));
1258 Ok(handle)
1259 }
1260
1261 pub async fn run_concurrent<R>(self, fun: impl AsyncFnOnce(&Accessor<T>) -> R) -> Result<R>
1345 where
1346 T: Send + 'static,
1347 {
1348 ensure!(
1349 self.0.concurrency_support(),
1350 "cannot use `run_concurrent` when Config::concurrency_support disabled",
1351 );
1352 self.do_run_concurrent(fun, false).await
1353 }
1354
1355 pub(super) async fn run_concurrent_trap_on_idle<R>(
1356 self,
1357 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1358 ) -> Result<R> {
1359 self.do_run_concurrent(fun, true).await
1360 }
1361
1362 async fn do_run_concurrent<R>(
1363 mut self,
1364 fun: impl AsyncFnOnce(&Accessor<T>) -> R,
1365 trap_on_idle: bool,
1366 ) -> Result<R> {
1367 debug_assert!(self.0.concurrency_support());
1368 self.0.check_event_loop_not_running()?;
1369 let token = StoreToken::new(self.as_context_mut());
1370
1371 struct Dropper<'a, T: 'static, V> {
1372 store: StoreContextMut<'a, T>,
1373 value: ManuallyDrop<V>,
1374 }
1375
1376 impl<'a, T, V> Drop for Dropper<'a, T, V> {
1377 fn drop(&mut self) {
1378 self.store
1379 .0
1380 .concurrent_state_mut_already_forced_current_thread()
1381 .event_loop_running = false;
1382
1383 tls::set(self.store.0, || {
1384 unsafe { ManuallyDrop::drop(&mut self.value) }
1389 });
1390 }
1391 }
1392
1393 let accessor = &Accessor::new(token);
1394 self.0
1395 .concurrent_state_mut_already_forced_current_thread()
1396 .event_loop_running = true;
1397 let dropper = &mut Dropper {
1398 store: self,
1399 value: ManuallyDrop::new(fun(accessor)),
1400 };
1401 let future = unsafe { Pin::new_unchecked(dropper.value.deref_mut()) };
1403
1404 let result = dropper
1405 .store
1406 .as_context_mut()
1407 .poll_until(future, trap_on_idle)
1408 .await;
1409
1410 if result.is_err() {
1411 dropper.store.0.set_trapped();
1412 }
1413
1414 result
1415 }
1416
1417 async fn poll_until<R>(
1423 mut self,
1424 mut future: Pin<&mut impl Future<Output = R>>,
1425 trap_on_idle: bool,
1426 ) -> Result<R> {
1427 struct Reset<'a, T: 'static> {
1428 store: StoreContextMut<'a, T>,
1429 futures: Option<FuturesUnordered<HostTaskFuture>>,
1430 }
1431
1432 impl<'a, T> Drop for Reset<'a, T> {
1433 fn drop(&mut self) {
1434 if let Some(futures) = self.futures.take() {
1435 *self
1436 .store
1437 .0
1438 .concurrent_state_mut_already_forced_current_thread()
1439 .futures
1440 .get_mut() = Some(futures);
1441 }
1442 }
1443 }
1444
1445 const MAX_TURNS_WITHOUT_YIELD: usize = 128;
1449 let mut turns_without_yield = 0;
1450
1451 loop {
1452 let futures = self.0.concurrent_state_mut()?.futures.get_mut().take();
1456 let mut reset = Reset {
1457 store: self.as_context_mut(),
1458 futures,
1459 };
1460 let mut next = match reset.futures.as_mut() {
1461 Some(f) => pin!(f.next()),
1462 None => bail_bug!("concurrent state missing futures field"),
1463 };
1464
1465 enum PollResult<R> {
1466 Complete(R),
1467 ProcessWork {
1468 ready: Option<WorkItem>,
1469 low_priority: bool,
1470 },
1471 }
1472
1473 let result = future::poll_fn(|cx| {
1474 if let Poll::Ready(value) = tls::set(reset.store.0, || future.as_mut().poll(cx)) {
1477 return Poll::Ready(Ok(PollResult::Complete(value)));
1478 }
1479
1480 if reset.store.0.component_trapped() {
1489 return Poll::Ready(Err(Trap::CannotEnterComponent.into()));
1490 }
1491
1492 let next = match tls::set(reset.store.0, || next.as_mut().poll(cx)) {
1496 Poll::Ready(Some(output)) => {
1497 match output {
1498 Err(e) => return Poll::Ready(Err(e)),
1499 Ok(()) => {}
1500 }
1501 Poll::Ready(true)
1502 }
1503 Poll::Ready(None) => Poll::Ready(false),
1504 Poll::Pending => Poll::Pending,
1505 };
1506
1507 let state = reset.store.0.concurrent_state_mut()?;
1522 let mut ready = state.switch_item.take();
1523 let mut low_priority = false;
1524 if ready.is_none() {
1525 ready = state.high_priority.pop_back();
1526 if ready.is_none() {
1527 ready = state.low_priority.pop_back();
1528 low_priority = true;
1529 }
1530 }
1531 if ready.is_some() {
1532 return Poll::Ready(Ok(PollResult::ProcessWork {
1533 ready,
1534 low_priority,
1535 }));
1536 }
1537
1538 if let Some(waker) = state.ready_for_concurrent_call_waker.take() {
1539 waker.wake();
1540 }
1541
1542 return match next {
1546 Poll::Ready(true) => {
1547 Poll::Ready(Ok(PollResult::ProcessWork {
1553 ready: None,
1554 low_priority: false,
1555 }))
1556 }
1557 Poll::Ready(false) => {
1558 if let Poll::Ready(value) =
1562 tls::set(reset.store.0, || future.as_mut().poll(cx))
1563 {
1564 Poll::Ready(Ok(PollResult::Complete(value)))
1565 } else {
1566 if trap_on_idle {
1572 Poll::Ready(Err(if reset.store.0.any_may_not_suspend()? {
1579 Trap::CannotBlockSyncTask.into()
1580 } else {
1581 Trap::AsyncDeadlock.into()
1583 }))
1584 } else {
1585 Poll::Pending
1589 }
1590 }
1591 }
1592 Poll::Pending => Poll::Pending,
1597 };
1598 })
1599 .await;
1600
1601 drop(reset);
1605
1606 match result? {
1607 PollResult::Complete(value) => break Ok(value),
1610 PollResult::ProcessWork {
1613 mut ready,
1614 low_priority,
1615 } => {
1616 if low_priority {
1644 if let Some(item) = ready.take() {
1645 self.0.concurrent_state_mut()?.low_priority.push_back(item);
1646 }
1647 self.0.yield_now().await;
1648 turns_without_yield = 0;
1649 ready = self.0.concurrent_state_mut()?.low_priority.pop_back();
1650 }
1651
1652 if let Some(item) = ready {
1653 self.as_context_mut().handle_work_item(item).await?;
1654 }
1655
1656 turns_without_yield += 1;
1657 if turns_without_yield == MAX_TURNS_WITHOUT_YIELD {
1658 turns_without_yield = 0;
1659 self.0.yield_now().await;
1660 }
1661 }
1662 }
1663 }
1664 }
1665
1666 fn handle_work_item(self, item: WorkItem) -> impl Future<Output = Result<()>> {
1668 let item = match item {
1672 WorkItem::ResumeFiber {
1673 instance,
1674 thread,
1675 fiber,
1676 } => {
1677 let future = self.0.resume_fiber(fiber);
1678 log::trace!(
1679 "handle work item ResumeFiber {{ instance: {instance:?}, thread: {thread:?} }}"
1680 );
1681 return futures::future::Either::Left(future);
1682 }
1683 item => item,
1684 };
1685 futures::future::Either::Right(async move {
1686 log::trace!("handle work item {item:?}");
1687 match item {
1688 WorkItem::PushFuture(future) => {
1689 self.0
1690 .concurrent_state_mut()?
1691 .futures_mut()?
1692 .push(future.into_inner());
1693 }
1694 WorkItem::ResumeFiber { .. } => unreachable!(),
1695 WorkItem::ResumeThread { thread, .. } => {
1696 let slot = &mut self.0.concurrent_state_mut()?.get_mut(thread.thread)?.state;
1697 if !matches!(slot, GuestThreadState::Ready { .. }) {
1698 bail_bug!("cannot resume non-pending thread {thread:?}");
1699 }
1700 let GuestThreadState::Ready { fiber } =
1701 mem::replace(slot, GuestThreadState::Running)
1702 else {
1703 unreachable!()
1704 };
1705 self.0.resume_fiber(fiber).await?;
1706 }
1707 WorkItem::GuestCall { mut call, .. } => {
1708 if call.is_ready(self.0)? {
1709 if let GuestCallKind::StartImplicit {
1710 was_pending: true, ..
1711 } = &call.kind
1712 {
1713 let instance = self
1714 .0
1715 .concurrent_state_mut()?
1716 .get_mut(call.thread.task)?
1717 .instance;
1718
1719 self.0.decrement_pending_start_count(instance, 1)?;
1720 }
1721
1722 self.0
1723 .concurrent_state_mut()?
1724 .get_mut(call.thread.thread)?
1725 .wake_on_cancel = WakeOnCancel::None;
1726
1727 self.run_on_worker(WorkerItem::GuestCall(call)).await?;
1728 } else {
1729 let state = self.0.concurrent_state_mut()?;
1730 let task = state.get_mut(call.thread.task)?;
1731 if !task.starting_sent {
1732 task.starting_sent = true;
1733 if let GuestCallKind::StartImplicit { .. } = &call.kind {
1734 Waitable::Guest(call.thread.task).set_event(
1735 state,
1736 Some(Event::Subtask {
1737 status: Status::Starting,
1738 }),
1739 )?;
1740 }
1741 }
1742
1743 let increment = if let GuestCallKind::StartImplicit {
1744 was_pending, ..
1745 } = &mut call.kind
1746 {
1747 let increment = !*was_pending;
1748 *was_pending = true;
1749 increment
1750 } else {
1751 false
1752 };
1753
1754 let instance = state.get_mut(call.thread.task)?.instance;
1755 if let GuestCallKind::DeliverEvent { set: Some(set), .. } = &call.kind {
1764 if !state.get_mut(*set)?.ready.is_empty() {
1765 state.wake_waiter(*set)?;
1766 }
1767 }
1768
1769 state.take_next_switch_item()?;
1773
1774 let state = self.0.instance_state(instance).concurrent_state();
1775
1776 state.pending.insert(call.thread, call.kind);
1777
1778 if increment {
1779 state.pending_start_count += 1;
1780 }
1781 }
1782 }
1783 WorkItem::WorkerFunction(fun) => {
1784 self.run_on_worker(WorkerItem::Function(fun)).await?;
1785 }
1786 }
1787
1788 Ok(())
1789 })
1790 }
1791
1792 async fn run_on_worker(self, item: WorkerItem) -> Result<()> {
1794 let state = self.0.concurrent_state_mut()?;
1795 if state.worker_item.is_some() {
1796 bail_bug!("worker item already set when taking worker fiber");
1797 }
1798 let worker = if let Some(fiber) = state.worker.take() {
1799 fiber
1800 } else {
1801 unsafe {
1820 fiber::make_fiber_unchecked(self.0, move |store| {
1821 loop {
1822 let Some(item) = store.concurrent_state_mut()?.worker_item.take() else {
1823 bail_bug!("worker_item not present when resuming fiber")
1824 };
1825 match item {
1826 WorkerItem::GuestCall(call) => handle_guest_call(store, call)?,
1827 WorkerItem::Function(fun) => fun.into_inner()(store)?,
1828 }
1829
1830 store.suspend(SuspendReason::NeedWork)?;
1831 }
1832 })?
1833 }
1834 };
1835
1836 self.0
1839 .concurrent_state_mut_already_forced_current_thread()
1840 .worker_item = Some(item);
1841
1842 self.0.resume_fiber(worker).await
1843 }
1844
1845 pub(crate) fn wrap_call<F, R>(self, closure: F) -> impl Future<Output = Result<R>> + 'static
1850 where
1851 T: 'static,
1852 F: FnOnce(&Accessor<T>) -> Pin<Box<dyn Future<Output = Result<R>> + Send + '_>>
1853 + Send
1854 + Sync
1855 + 'static,
1856 R: Send + Sync + 'static,
1857 {
1858 let token = StoreToken::new(self);
1859 async move {
1860 let mut accessor = Accessor::new(token);
1861 closure(&mut accessor).await
1862 }
1863 }
1864
1865 pub(crate) async fn start_instance(
1866 &mut self,
1867 instance: ModuleInstance,
1868 callee: Option<RuntimeInstance>,
1869 ) -> Result<ModuleInstance> {
1870 self.0.check_event_loop_not_running()?;
1872 let (tx, rx) = oneshot::channel();
1873 let token = StoreToken::new(self.as_context_mut());
1874 self.0.queue_task(move |store| {
1875 _ = tx.send(
1876 super::instance::start_raw(&mut token.as_context_mut(store), instance, callee)
1877 .map(|()| instance),
1878 );
1879 Ok(())
1880 })?;
1881 self.as_context_mut()
1882 .run_concurrent_trap_on_idle(async |_| {
1883 rx.await
1884 .map_err(|_| format_err!("oneshot channel canceled"))
1885 })
1886 .await??
1887 }
1888}
1889
1890pub type EnteredHostTask = Option<QualifiedThreadId>;
1897
1898impl StoreOpaque {
1899 pub(crate) fn check_event_loop_not_running(&mut self) -> Result<()> {
1900 if self
1901 .concurrent_state_mut_already_forced_current_thread()
1902 .event_loop_running
1903 {
1904 bail!("Recursive `StoreContextMut::run_concurrent` calls not supported")
1905 }
1906 Ok(())
1907 }
1908
1909 #[inline]
1913 pub(crate) fn current_thread(&mut self) -> Result<CurrentThread> {
1914 if !self.concurrency_support() {
1916 return Ok(CurrentThread::None);
1917 }
1918
1919 if !self
1922 .vm_store_context_mut()
1923 .current_thread_mut()
1924 .is_deferred()
1925 {
1926 return Ok(self
1927 .concurrent_state_mut_already_forced_current_thread()
1928 .unforced_current_thread);
1929 }
1930
1931 self.force_deferred_current_thread()
1932 }
1933
1934 #[cold]
1937 fn force_deferred_current_thread(&mut self) -> Result<CurrentThread> {
1938 let state = self.concurrent_state_mut_without_forcing_current_thread();
1947 let id = match state.unforced_current_thread.guest_task() {
1948 Some(task) => state.get_mut(task)?.instance.instance,
1949 None => bail_bug!("deferred component-model thread with non-guest base"),
1950 };
1951
1952 let mut frames = SmallVec::<[_; 8]>::new();
1955 let mut cur = *self.vm_store_context_mut().current_thread_mut();
1956 while let Some(ptr) = cur.as_deferred() {
1957 let deferred = unsafe { ptr.as_non_null().as_ref() };
1962 frames.push((
1963 deferred.callee_async != 0,
1964 deferred.callee_instance,
1965 deferred.saved_context,
1966 deferred.lazy_lenders_start,
1967 ));
1968 cur = deferred.parent;
1969 }
1970
1971 *self.vm_store_context_mut().current_thread_mut() = VMLazyThread::forced();
1975
1976 let current_context = *self.vm_store_context_mut().component_context_mut();
1979
1980 let lazy_lenders_end = *self.vm_store_context_mut().lazy_lenders_len_mut();
1987 frames.reverse();
1988 let lazy_lenders_base = match frames.first() {
1989 Some((_, _, _, start)) => *start,
1990 None => bail_bug!("forcing a deferred thread with no deferred frames"),
1991 };
1992 let lazy_lender_ends = frames
1993 .iter()
1994 .skip(1)
1995 .map(|(_, _, _, start)| *start)
1996 .chain([lazy_lenders_end])
1997 .collect::<SmallVec<[_; 8]>>();
1998 for ((callee_async, callee_instance, saved_context, start), lenders_end) in
1999 frames.into_iter().zip(lazy_lender_ends)
2000 {
2001 *self.vm_store_context_mut().component_context_mut() = saved_context;
2005 let callee = RuntimeInstance {
2006 instance: id,
2007 index: RuntimeComponentInstanceIndex::from_u32(callee_instance),
2008 };
2009 self.enter_guest_sync_call(callee_async, callee)?;
2010
2011 self.promote_current_lazy_lends(id, start, lenders_end)?;
2014 }
2015
2016 debug_assert_eq!(
2019 *self.vm_store_context_mut().lazy_lenders_len_mut(),
2020 lazy_lenders_end
2021 );
2022 *self.vm_store_context_mut().lazy_lenders_len_mut() = lazy_lenders_base;
2023
2024 *self.vm_store_context_mut().component_context_mut() = current_context;
2026
2027 Ok(self
2028 .concurrent_state_mut_without_forcing_current_thread()
2029 .unforced_current_thread)
2030 }
2031
2032 fn promote_current_lazy_lends(
2038 &mut self,
2039 id: ComponentInstanceId,
2040 start: usize,
2041 end: usize,
2042 ) -> Result<()> {
2043 if start == end {
2044 return Ok(());
2045 }
2046
2047 let scope = match self
2048 .concurrent_state_mut_without_forcing_current_thread()
2049 .unforced_current_thread
2050 .guest_task()
2051 {
2052 Some(task) => VMScope::id(task.rep()),
2053 None => bail_bug!("replayed deferred frame is not a guest thread"),
2054 };
2055 let instance = Instance::from_wasmtime(self, id);
2056 self.promote_lazy_lends(instance, scope, start, end)
2057 }
2058
2059 fn current_guest_thread(&mut self) -> Result<QualifiedThreadId> {
2060 match self.current_thread()?.guest() {
2061 Some(id) => Ok(*id),
2062 None => bail_bug!("current thread is not a guest thread"),
2063 }
2064 }
2065
2066 pub(crate) fn current_materialized_host_task(&mut self) -> Result<Option<TableId<HostTask>>> {
2070 match self.current_thread()? {
2071 CurrentThread::Host(id) => Ok(Some(id)),
2072 CurrentThread::DeferredHost(_) | CurrentThread::None => Ok(None),
2073 _ => bail_bug!("current thread is not a host thread"),
2074 }
2075 }
2076
2077 fn materialize_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
2080 Ok(self
2081 .concurrent_state_mut()?
2082 .materialize_current_host_task_id()?)
2083 }
2084
2085 fn enter_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2086 log::trace!("enter sync-typed call {callee:?}");
2087 let state = self.instance_state(callee).concurrent_state();
2088 let old_do_not_suspend = state.do_not_suspend;
2089 state.do_not_suspend = true;
2090
2091 let thread = self.current_guest_thread()?;
2092 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2093 if thread.old_do_not_suspend.is_some() {
2094 bail_bug!("current thread already has `old_do_not_suspend` value");
2095 }
2096
2097 thread.old_do_not_suspend = Some(old_do_not_suspend);
2098
2099 Ok(())
2100 }
2101
2102 fn exit_sync_call(&mut self, callee: RuntimeInstance) -> Result<()> {
2103 log::trace!("exit sync-typed call {callee:?}");
2104 let thread = self.current_guest_thread()?;
2105 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2106 let Some(old_do_not_suspend) = thread.old_do_not_suspend.take() else {
2107 bail_bug!("current thread missing `old_do_not_suspend` value");
2108 };
2109 let state = self.instance_state(callee).concurrent_state();
2110 state.do_not_suspend = old_do_not_suspend;
2111 Ok(())
2112 }
2113
2114 pub(crate) fn enter_guest_sync_call(
2126 &mut self,
2127 callee_async_typed: bool,
2128 callee: RuntimeInstance,
2129 ) -> Result<()> {
2130 log::trace!("enter sync-lifted call {callee:?}");
2131 if !self.concurrency_support() {
2132 return self.enter_call_not_concurrent();
2133 }
2134
2135 let state = self.concurrent_state_mut()?;
2139 let item = state.next_switch_item.take();
2140 state.saved_next_switch_items.push(item);
2141
2142 let thread = self.current_thread()?;
2143 let caller = if let Some(thread) = thread.guest() {
2144 Caller::Guest { thread: *thread }
2145 } else {
2146 Caller::Host {
2147 tx: None,
2148 caller: self.materialize_host_task_id()?,
2149 }
2150 };
2151
2152 let state = self.concurrent_state_mut()?;
2153 let guest_thread = GuestTask::new(
2154 state,
2155 Box::new(move |_, _| bail_bug!("cannot lower params in sync call")),
2156 LiftResult {
2157 lift: Box::new(move |_, _| bail_bug!("cannot lift result in sync call")),
2158 ty: TypeTupleIndex::reserved_value(),
2159 memory: None,
2160 string_encoding: StringEncoding::Utf8,
2161 },
2162 caller,
2163 None,
2164 callee,
2165 callee_async_typed,
2166 false,
2167 )?;
2168
2169 Instance::from_wasmtime(self, callee.instance).add_guest_thread_to_instance_table(
2170 guest_thread.thread,
2171 self,
2172 callee.index,
2173 )?;
2174 self.set_thread(guest_thread)?;
2175
2176 if !callee_async_typed {
2177 self.enter_sync_call(callee)?;
2178 }
2179
2180 Ok(())
2181 }
2182
2183 pub(crate) fn exit_guest_sync_call(&mut self) -> Result<()> {
2191 if !self.concurrency_support() {
2192 return Ok(self.exit_call_not_concurrent());
2193 }
2194
2195 let thread = match self.current_thread()?.guest() {
2196 Some(t) => *t,
2197 None => bail_bug!("expected task when exiting"),
2198 };
2199 let task = self.concurrent_state_mut()?.get_mut(thread.task)?;
2200 let instance = task.instance;
2201
2202 let caller = match &task.caller {
2203 &Caller::Guest { thread } => thread.into(),
2204 &Caller::Host { caller, .. } => caller
2205 .map(CurrentThread::Host)
2206 .unwrap_or(CurrentThread::None),
2207 };
2208 task.lift_result = None;
2209 task.exited = true;
2210 let async_typed = task.async_typed;
2211
2212 if !async_typed {
2213 self.exit_sync_call(instance)?;
2214 }
2215
2216 self.set_thread(caller)?;
2217
2218 log::trace!("exit sync-lifted call {instance:?}");
2219
2220 if async_typed {
2221 self.switch_or_trap_if_may_not_suspend(instance)?;
2226 }
2227
2228 self.cleanup_thread(thread, instance)?;
2229
2230 self.concurrent_state_mut()?
2231 .decrement_task_ref_count(thread.task)?;
2232
2233 let state = self.concurrent_state_mut()?;
2234 let Some(item) = state.saved_next_switch_items.pop() else {
2235 bail_bug!("unable to pop from `saved_next_switch_items`");
2236 };
2237 if let Some(item) = mem::replace(&mut state.next_switch_item, item) {
2238 state.push_high_priority(item);
2241 bail_bug!("`next_switch_item` unexpectedly already set");
2242 }
2243
2244 Ok(())
2245 }
2246
2247 pub(crate) fn host_task_create(&mut self) -> Result<EnteredHostTask> {
2254 if !self.concurrency_support() {
2255 self.enter_call_not_concurrent()?;
2256 return Ok(None);
2257 }
2258 let caller = self.current_guest_thread()?;
2259 log::trace!("new deferred host task with caller {caller:?}");
2260
2261 self.set_thread(CurrentThread::DeferredHost(caller))?;
2262 let state = self.concurrent_state_mut()?;
2263 debug_assert!(state.deferred_host_call_context.is_none());
2264 state.deferred_host_call_context = Some(CallContext::default());
2265 state.debug_assert_deferred_host_invariant();
2266 Ok(Some(caller))
2267 }
2268
2269 pub(crate) fn host_task_delete(
2276 &mut self,
2277 original_task: EnteredHostTask,
2278 materialized_task: Option<TableId<HostTask>>,
2279 ) -> Result<()> {
2280 match original_task {
2281 Some(caller) => {
2282 self.set_thread(caller)?;
2283 if materialized_task.is_none() {
2284 let state = self.concurrent_state_mut()?;
2285 let context = state
2286 .deferred_host_call_context
2287 .take()
2288 .expect("deferred host call context should be present");
2289 debug_assert!(context.is_empty());
2290 state.debug_assert_deferred_host_invariant();
2291 }
2292 log::trace!(
2293 "delete host task with caller {original_task:?} and materialized as {materialized_task:?}"
2294 );
2295 if let Some(task) = materialized_task {
2296 self.concurrent_state_mut()?.delete_host_task(task)?;
2297 }
2298 }
2299 None => {
2300 debug_assert!(materialized_task.is_none());
2301 self.exit_call_not_concurrent();
2302 }
2303 }
2304 Ok(())
2305 }
2306
2307 fn instance_state(&mut self, instance: RuntimeInstance) -> &mut InstanceState {
2310 self.component_instance_mut(instance.instance)
2311 .instance_state(instance.index)
2312 }
2313
2314 fn instance_state_and_handle_counter(
2317 &mut self,
2318 instance: RuntimeInstance,
2319 ) -> (&mut InstanceState, &mut HandleCounter) {
2320 let (component, counter) = self
2321 .store_data_mut()
2322 .component_instance_and_handle_counter_mut(instance.instance);
2323 (component.instance_state(instance.index), counter)
2324 }
2325
2326 fn handle_table_insertion(&mut self, instance: RuntimeInstance) -> HandleTableInsertion<'_> {
2328 let (component, counter) = self
2329 .store_data_mut()
2330 .component_instance_and_handle_counter_mut(instance.instance);
2331 component.handle_table_insertion(instance.index, counter)
2332 }
2333
2334 pub(crate) fn set_thread(&mut self, thread: impl Into<CurrentThread>) -> Result<CurrentThread> {
2340 let thread = thread.into();
2341 let state = self.concurrent_state_mut()?;
2342 state.debug_assert_deferred_host_invariant();
2343 let old_thread = mem::replace(&mut state.unforced_current_thread, thread);
2344
2345 state.handle_thread_switch(old_thread, thread)?;
2346
2347 if let Some(old_thread) = old_thread.guest() {
2355 let old_context = *self.vm_store_context_mut().component_context_mut();
2356 self.concurrent_state_mut()?
2357 .get_mut(old_thread.thread)?
2358 .context = old_context;
2359 }
2360 if cfg!(debug_assertions) {
2361 *self.vm_store_context_mut().component_context_mut() =
2362 [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2363 }
2364 if let Some(thread) = thread.guest() {
2365 let thread = self.concurrent_state_mut()?.get_mut(thread.thread)?;
2366 let context = thread.context;
2367 if cfg!(debug_assertions) {
2368 thread.context = [u32::MAX; NUM_COMPONENT_CONTEXT_SLOTS];
2369 }
2370 *self.vm_store_context_mut().component_context_mut() = context;
2371 }
2372
2373 *self.vm_store_context_mut().current_thread_mut() = if thread.is_none() {
2375 VMLazyThread::none()
2376 } else {
2377 VMLazyThread::forced()
2378 };
2379
2380 Ok(old_thread)
2381 }
2382
2383 fn switch_or_trap_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<()> {
2385 if self.switch_if_may_not_suspend(instance)? {
2386 Ok(())
2387 } else {
2388 Err(Trap::CannotBlockSyncTask.into())
2389 }
2390 }
2391
2392 fn switch_if_may_not_suspend(&mut self, instance: RuntimeInstance) -> Result<bool> {
2396 self.concurrent_state_mut()?;
2400
2401 Ok(!self.concurrency_support()
2402 || !self
2403 .instance_state(instance)
2404 .concurrent_state()
2405 .do_not_suspend
2406 || self
2407 .concurrent_state_mut()?
2408 .promote_instance_local_thread_work_item(instance)?)
2409 }
2410
2411 fn enter_instance(&mut self, instance: RuntimeInstance) {
2415 log::trace!("enter {instance:?}");
2416 self.instance_state(instance)
2417 .concurrent_state()
2418 .do_not_enter = true;
2419 }
2420
2421 fn exit_instance(&mut self, instance: RuntimeInstance) -> Result<()> {
2425 log::trace!("exit {instance:?}");
2426 self.instance_state(instance)
2427 .concurrent_state()
2428 .do_not_enter = false;
2429 self.partition_pending(instance)
2430 }
2431
2432 fn partition_pending(&mut self, instance: RuntimeInstance) -> Result<()> {
2440 let mut any_ready = false;
2441 for (thread, kind) in
2442 mem::take(&mut self.instance_state(instance).concurrent_state().pending)
2443 {
2444 let call = GuestCall { thread, kind };
2445 if call.is_ready(self)? {
2446 any_ready = true;
2447 self.concurrent_state_mut()?
2448 .push_high_priority(WorkItem::GuestCall { instance, call });
2449 } else {
2450 self.instance_state(instance)
2451 .concurrent_state()
2452 .pending
2453 .insert(call.thread, call.kind);
2454 }
2455 }
2456
2457 if any_ready {
2458 if let Some(waker) = self
2459 .concurrent_state_mut()?
2460 .ready_for_concurrent_call_waker
2461 .take()
2462 {
2463 waker.wake();
2464 }
2465 }
2466
2467 Ok(())
2468 }
2469
2470 pub(crate) fn backpressure_modify(
2472 &mut self,
2473 caller_instance: RuntimeInstance,
2474 modify: impl FnOnce(u16) -> Option<u16>,
2475 ) -> Result<()> {
2476 let state = self.instance_state(caller_instance).concurrent_state();
2477 let old = state.backpressure;
2478 let new = modify(old).ok_or_else(|| Trap::BackpressureOverflow)?;
2479 state.backpressure = new;
2480
2481 log::trace!("backpressure update for {caller_instance:?} from {old} to {new}");
2482
2483 if old > 0 && new == 0 {
2484 self.partition_pending(caller_instance)?;
2487 }
2488
2489 Ok(())
2490 }
2491
2492 fn resume_fiber(
2495 &mut self,
2496 fiber: StoreFiber<'static>,
2497 ) -> impl Future<Output = Result<()>> + '_ {
2498 let mut resuming = ResumingFiber {
2499 store: self,
2500 fiber: Some(fiber),
2501 };
2502 async move {
2503 let old_thread = resuming.store.current_thread()?;
2504 log::trace!("resume_fiber: save current thread {old_thread:?}");
2505
2506 let fiber = resuming.fiber.take().unwrap();
2507 resuming.fiber = fiber::resolve_or_release(resuming.store, fiber).await?;
2508
2509 resuming.store.set_thread(old_thread)?;
2510
2511 let state = resuming.store.concurrent_state_mut()?;
2512
2513 if let Some(ot) = old_thread.guest() {
2514 state.set_thread_running(ot.thread)?;
2515 }
2516 log::trace!("resume_fiber: restore current thread {old_thread:?}");
2517
2518 if resuming.fiber.is_some() {
2519 log::trace!("resume_fiber: suspend reason {:?}", &state.suspend_reason);
2520 let reason = match state.suspend_reason.take() {
2522 Some(r) => r,
2523 None => bail_bug!("suspend reason missing when resuming fiber"),
2524 };
2525 match reason {
2526 SuspendReason::NeedWork => {
2527 if state.worker.is_none() {
2528 state.worker = resuming.fiber.take();
2529 }
2530 }
2531 SuspendReason::Yielding { thread } => {
2532 let slot = &mut state.get_mut(thread.thread)?.state;
2533 slot.check_no_fiber()?;
2534 *slot = GuestThreadState::Ready {
2535 fiber: resuming.fiber.take().unwrap(),
2536 };
2537 let instance = state.get_mut(thread.task)?.instance;
2538 state.push_low_priority(WorkItem::ResumeThread { instance, thread });
2539 }
2540 SuspendReason::ExplicitlySuspending { thread } => {
2541 let slot = &mut state.get_mut(thread.thread)?.state;
2542 slot.check_no_fiber()?;
2543 *slot = GuestThreadState::Suspended(resuming.fiber.take().unwrap());
2544 }
2545 SuspendReason::Waiting { set, thread } => {
2546 match state.get_mut(set)?.waiting.entry(thread) {
2547 Entry::Vacant(entry) => {
2548 entry.insert(WaitMode::Fiber(resuming.fiber.take().unwrap()));
2549 }
2550 Entry::Occupied(_) => {
2551 bail_bug!("entry unexpectedly already exists for {thread:?}");
2552 }
2553 }
2554 }
2555 SuspendReason::YieldingToSubtask { thread } => {
2556 let instance = state.get_mut(thread.task)?.instance;
2565 if state.next_switch_item.is_some() {
2566 bail_bug!(
2569 "`ConcurrentState::next_switch_item` was already `Some(_)` when \
2570 a thread wanted to wait on a subtask"
2571 );
2572 }
2573 let item = state.next_switch_item.insert(WorkItem::ResumeFiber {
2574 instance,
2575 thread,
2576 fiber: resuming.fiber.take().unwrap(),
2577 });
2578 log::trace!("set next switch item to {item:?}");
2579 }
2580 };
2581 } else {
2582 log::trace!("resume_fiber: fiber has exited");
2583 }
2584
2585 Ok(())
2586 }
2587 }
2588
2589 fn suspend(&mut self, reason: SuspendReason) -> Result<()> {
2595 log::trace!("suspend fiber: {reason:?}");
2596
2597 if !self.fiber_async_state_mut().can_block() {
2600 return Err(format_err!("future dropped"));
2601 }
2602
2603 let state = self.concurrent_state_mut()?;
2604
2605 let (save_and_restore_thread, save_and_restore_next_switch_item) = match &reason {
2612 SuspendReason::Yielding { .. }
2613 | SuspendReason::Waiting { .. }
2614 | SuspendReason::ExplicitlySuspending { .. } => {
2615 if state.switch_item.is_none() {
2618 state.take_next_switch_item()?;
2619 }
2620
2621 (true, false)
2622 }
2623 SuspendReason::YieldingToSubtask { .. } => (true, true),
2624 SuspendReason::NeedWork => (false, false),
2625 };
2626
2627 let old_next_switch_item = if save_and_restore_next_switch_item {
2628 Some(state.save_next_switch_item()?)
2629 } else {
2630 None
2631 };
2632
2633 let old_guest_thread = if save_and_restore_thread {
2634 self.current_thread()?
2635 } else {
2636 CurrentThread::None
2637 };
2638
2639 let waiting_set = match &reason {
2640 SuspendReason::Waiting { set, .. } => Some(*set),
2641 _ => None,
2642 };
2643
2644 let suspend_reason = &mut self.concurrent_state_mut()?.suspend_reason;
2645 if suspend_reason.is_some() {
2646 bail_bug!("suspend reason unexpectedly already set for store");
2647 }
2648 *suspend_reason = Some(reason);
2649
2650 if let Some(set) = waiting_set {
2653 self.concurrent_state_mut()?.get_mut(set)?.num_waiting += 1;
2654 }
2655
2656 self.with_blocking(|_, cx| cx.suspend(StoreFiberYield::ReleaseStore))?;
2657
2658 if let Some(set) = waiting_set {
2659 self.concurrent_state_mut()?.get_mut(set)?.stop_waiting()?;
2660 }
2661
2662 if save_and_restore_thread {
2663 self.set_thread(old_guest_thread)?;
2664 }
2665
2666 if let Some(item) = old_next_switch_item {
2667 let state = self.concurrent_state_mut()?;
2668 state.restore_next_switch_item(item)?;
2669 }
2670
2671 Ok(())
2672 }
2673
2674 fn wait_for_event(
2675 &mut self,
2676 caller_instance: RuntimeInstance,
2677 waitable: Waitable,
2678 ) -> Result<()> {
2679 let caller = self.current_guest_thread()?;
2680 let state = self.concurrent_state_mut()?;
2681
2682 waitable.trap_if_in_waitable_set(state)?;
2683
2684 let set = state.get_mut(caller.thread)?.sync_call_set;
2685 waitable.join(state, Some(set))?;
2686
2687 self.switch_or_trap_if_may_not_suspend(caller_instance)?;
2688
2689 self.suspend(SuspendReason::Waiting {
2690 set,
2691 thread: caller,
2692 })?;
2693 let state = self.concurrent_state_mut()?;
2694
2695 waitable.join(state, None)
2696 }
2697
2698 fn cleanup_thread(
2720 &mut self,
2721 guest_thread: QualifiedThreadId,
2722 runtime_instance: RuntimeInstance,
2723 ) -> Result<()> {
2724 let state = self.concurrent_state_mut()?;
2725 state.take_next_switch_item()?;
2728 let thread_data = state.get_mut(guest_thread.thread)?;
2729 let sync_call_set = thread_data.sync_call_set;
2730 if let Some(guest_id) = thread_data.instance_rep {
2731 self.instance_state(runtime_instance)
2732 .thread_handle_table()
2733 .guest_thread_remove(guest_id)?;
2734 }
2735 let state = self.concurrent_state_mut()?;
2736
2737 for waitable in mem::take(&mut state.get_mut(sync_call_set)?.ready) {
2739 if let Some(Event::Subtask {
2740 status: Status::Returned | Status::ReturnCancelled,
2741 }) = waitable.common(state)?.event
2742 {
2743 waitable.delete_from(state)?;
2744 }
2745 }
2746
2747 state.delete(guest_thread.thread)?;
2748 state.delete(sync_call_set)?;
2749 let task = state.get_mut(guest_thread.task)?;
2750 task.threads.remove(&guest_thread.thread);
2751
2752 if task.threads.is_empty() && !task.returned_or_cancelled() {
2753 bail!(Trap::NoAsyncResult);
2754 }
2755
2756 if !task.decremented_interesting_task_count && task.exited && task.returned_or_cancelled() {
2757 task.decremented_interesting_task_count = true;
2758
2759 debug_assert!(state.interesting_tasks > 0);
2760 state.interesting_tasks -= 1;
2761 if state.interesting_tasks == 0
2762 && let Some(waker) = state.interesting_tasks_empty_waker.take()
2763 {
2764 waker.wake();
2765 }
2766 }
2767
2768 state.decrement_task_ref_count(guest_thread.task)?;
2769
2770 Ok(())
2771 }
2772
2773 fn cancel_guest_subtask_without_lowered_parameters(
2782 &mut self,
2783 caller_instance: RuntimeInstance,
2784 guest_task: TableId<GuestTask>,
2785 ) -> Result<()> {
2786 let concurrent_state = self.concurrent_state_mut()?;
2787 let task = concurrent_state.get_mut(guest_task)?;
2788 assert!(!task.already_lowered_parameters());
2789 task.lower_params = None;
2793 task.lift_result = None;
2794 task.exited = true;
2795 let instance = task.instance;
2796
2797 assert_eq!(1, task.threads.len());
2804 let thread = *task.threads.iter().next().unwrap();
2805 let next_switch_item = concurrent_state.save_next_switch_item()?;
2806 self.cleanup_thread(
2807 QualifiedThreadId {
2808 task: guest_task,
2809 thread,
2810 },
2811 caller_instance,
2812 )?;
2813 self.concurrent_state_mut()?
2814 .restore_next_switch_item(next_switch_item)?;
2815
2816 let pending = &mut self.instance_state(instance).concurrent_state().pending;
2818 let pending_count = pending.len();
2819 let mut decrement = 0;
2820 let mut consume = |kind: &GuestCallKind| {
2821 if let GuestCallKind::StartImplicit {
2822 was_pending: true, ..
2823 } = kind
2824 {
2825 decrement += 1;
2826 }
2827 };
2828 pending.retain(|thread, kind| {
2829 if thread.task == guest_task {
2830 consume(kind);
2831 false
2832 } else {
2833 true
2834 }
2835 });
2836
2837 if pending.len() == pending_count {
2838 let mut matches = |item: &WorkItem| match item {
2841 WorkItem::ResumeThread { thread, .. } => thread.task == guest_task,
2842 WorkItem::GuestCall {
2843 call: GuestCall { thread, kind },
2844 ..
2845 } => {
2846 if thread.task == guest_task {
2847 consume(kind);
2848 true
2849 } else {
2850 false
2851 }
2852 }
2853 _ => false,
2854 };
2855
2856 let mut remove = |queue: &mut VecDeque<WorkItem>| {
2857 let count = queue.len();
2858 queue.retain(|item| !matches(item));
2859 count != queue.len()
2860 };
2861
2862 let state = self.concurrent_state_mut()?;
2863 let mut found = remove(&mut state.high_priority);
2864 found |= remove(&mut state.low_priority);
2865
2866 for item in [&mut state.switch_item, &mut state.next_switch_item] {
2867 if matches!(item, Some(item) if matches(item)) {
2868 found = true;
2869 *item = None;
2870 }
2871 }
2872
2873 if !found {
2874 bail!(Trap::SubtaskCancelAfterTerminal);
2875 }
2876 }
2877
2878 self.decrement_pending_start_count(instance, decrement)?;
2879
2880 Ok(())
2881 }
2882
2883 fn decrement_pending_start_count(
2884 &mut self,
2885 instance: RuntimeInstance,
2886 decrement: usize,
2887 ) -> Result<()> {
2888 if decrement > 0 {
2889 let count = &mut self
2890 .instance_state(instance)
2891 .concurrent_state()
2892 .pending_start_count;
2893
2894 if *count < decrement {
2895 bail_bug!("`ConcurrentInstantState::pending_start_count` underflow");
2896 }
2897
2898 *count -= decrement;
2899
2900 self.partition_pending(instance)?;
2902 }
2903
2904 Ok(())
2905 }
2906
2907 pub(crate) fn current_scope(&mut self) -> Result<Option<CurrentScope>> {
2910 if !self.concurrency_support() {
2911 return Ok(self
2912 .current_scope_id_not_concurrent()?
2913 .map(|id| CurrentScope::Id(VMScope::id(id))));
2914 }
2915
2916 Ok(match self.current_thread()? {
2917 CurrentThread::Guest(id) => Some(CurrentScope::Id(VMScope::id(id.task.rep()))),
2918 CurrentThread::Host(id) => Some(CurrentScope::Id(VMScope::host_id(id.rep()))),
2919 CurrentThread::DeferredHost(_) => Some(CurrentScope::DeferredHost),
2920 CurrentThread::None => return Ok(None),
2921 })
2922 }
2923
2924 pub(crate) fn queue_task(
2925 &mut self,
2926 task: impl FnOnce(&mut dyn VMStore) -> Result<()> + Send + 'static,
2927 ) -> Result<()> {
2928 self.concurrent_state_mut()?
2929 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(task))));
2930 Ok(())
2931 }
2932
2933 fn any_may_not_suspend(&mut self) -> Result<bool> {
2942 Ok(self
2950 .concurrent_state_mut()?
2951 .table
2952 .get_mut()
2953 .iter_mut()
2954 .filter_map(|(_, entry)| {
2955 if let Some(task) = entry.entry().downcast_ref::<GuestTask>() {
2956 Some(task.instance)
2957 } else {
2958 None
2959 }
2960 })
2961 .collect::<Vec<_>>()
2962 .into_iter()
2963 .any(|instance| {
2964 self.instance_state(instance)
2965 .concurrent_state()
2966 .do_not_suspend
2967 }))
2968 }
2969}
2970
2971impl Instance {
2972 fn get_event(
2975 self,
2976 store: &mut StoreOpaque,
2977 guest_task: TableId<GuestTask>,
2978 set: Option<TableId<WaitableSet>>,
2979 cancellable: bool,
2980 ) -> Result<Option<(Event, Option<(Waitable, u32)>)>> {
2981 let state = store.concurrent_state_mut()?;
2982
2983 let task = state.get_mut(guest_task)?;
2984 let event = &mut task.event;
2985 if let Some(ev) = event
2986 && (cancellable || !matches!(ev, Event::Cancelled))
2987 {
2988 log::trace!("deliver event {ev:?} to {guest_task:?}");
2989
2990 if matches!(ev, Event::Cancelled) {
2991 task.cancel_request_delivered = true;
2992 }
2993
2994 let ev = *ev;
2995 *event = None;
2996 return Ok(Some((ev, None)));
2997 }
2998
2999 let set = match set {
3000 Some(set) => set,
3001 None => return Ok(None),
3002 };
3003 let waitable = match state.get_mut(set)?.ready.pop_first() {
3004 Some(v) => v,
3005 None => return Ok(None),
3006 };
3007
3008 let common = waitable.common(state)?;
3009 let handle = match common.handle {
3010 Some(h) => h,
3011 None => bail_bug!("handle not set when delivering event"),
3012 };
3013 let event = match common.event.take() {
3014 Some(e) => e,
3015 None => bail_bug!("event not set when delivering event"),
3016 };
3017
3018 log::trace!(
3019 "deliver event {event:?} to {guest_task:?} for {waitable:?} (handle {handle}); set {set:?}"
3020 );
3021
3022 waitable.on_delivery(store, self, event)?;
3023
3024 Ok(Some((event, Some((waitable, handle)))))
3025 }
3026
3027 fn handle_callback_code(
3034 self,
3035 store: &mut dyn VMStore,
3036 guest_thread: QualifiedThreadId,
3037 runtime_instance: RuntimeInstance,
3038 call: impl FnOnce(&mut dyn VMStore) -> Result<u32>,
3039 ) -> Result<()> {
3040 let old_thread = store.set_thread(guest_thread)?;
3041 log::trace!(
3042 "callback-lifted call: replaced {old_thread:?} with {guest_thread:?} as current thread"
3043 );
3044
3045 store.enter_instance(runtime_instance);
3046
3047 let mut code = call(store)?;
3048
3049 if matches!(
3055 unpack_callback_code(code).0,
3056 callback_code::YIELD | callback_code::WAIT
3057 ) && let Some((event, _)) = self.get_event(store, guest_thread.task, None, true)?
3058 {
3059 let (unpacked, set) = unpack_callback_code(code);
3060 log::trace!(
3061 "{guest_thread:?} returned callback code {unpacked} (set: {set}) with a pending \
3062 cancellation; delivering it via callback"
3063 );
3064 code = invoke_callback(store, guest_thread.task, event, 0)?;
3065 }
3066
3067 store.exit_instance(runtime_instance)?;
3068
3069 store.set_thread(old_thread)?;
3070 log::trace!("callback-lifted call: restored {old_thread:?} as current thread");
3071
3072 let store = store.store_opaque_mut();
3073 let (code, set) = unpack_callback_code(code);
3074
3075 log::trace!("received callback code from {guest_thread:?}: {code} (set: {set})");
3076
3077 let state = store.concurrent_state_mut()?;
3078
3079 state.take_next_switch_item()?;
3080
3081 let get_set = |store: &mut StoreOpaque, handle| -> Result<_> {
3082 let set = store
3083 .instance_state(runtime_instance)
3084 .handle_table()
3085 .waitable_set_rep(handle)?;
3086
3087 Ok(TableId::<WaitableSet>::new(set))
3088 };
3089
3090 match code {
3091 callback_code::EXIT => {
3092 log::trace!("implicit thread {guest_thread:?} completed");
3093 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3094 task.exited = true;
3095 task.callback = None;
3096
3097 store.switch_or_trap_if_may_not_suspend(runtime_instance)?;
3102
3103 store.cleanup_thread(guest_thread, runtime_instance)?;
3104 }
3105 callback_code::YIELD => {
3106 let old = state
3109 .get_mut(guest_thread.thread)?
3110 .wake_on_cancel
3111 .replace(WakeOnCancel::Yielding);
3112 if !old.is_none() {
3113 bail_bug!("thread unexpectedly had wake_on_cancel set");
3114 }
3115
3116 let call = GuestCall {
3123 thread: guest_thread,
3124 kind: GuestCallKind::DeliverEvent {
3125 instance: self,
3126 set: None,
3127 },
3128 };
3129 state.push_low_priority(WorkItem::GuestCall {
3132 instance: runtime_instance,
3133 call,
3134 });
3135 }
3136 callback_code::WAIT => {
3137 let set = get_set(store, set)?;
3138 self.wait_with_callback(store, guest_thread, set)?;
3139 }
3140 _ => bail!(Trap::UnsupportedCallbackCode),
3141 }
3142
3143 Ok(())
3144 }
3145
3146 fn wait_with_callback(
3152 self,
3153 store: &mut StoreOpaque,
3154 guest_thread: QualifiedThreadId,
3155 set: TableId<WaitableSet>,
3156 ) -> Result<()> {
3157 let state = store.concurrent_state_mut()?;
3158 state.get_mut(set)?.num_waiting += 1;
3161
3162 if state.get_mut(guest_thread.task)?.event.is_some()
3163 || !state.get_mut(set)?.ready.is_empty()
3164 {
3165 let instance = state.get_mut(guest_thread.task)?.instance;
3167 state.push_high_priority(WorkItem::GuestCall {
3168 instance,
3169 call: GuestCall {
3170 thread: guest_thread,
3171 kind: GuestCallKind::DeliverEvent {
3172 instance: self,
3173 set: Some(set),
3174 },
3175 },
3176 });
3177 return Ok(());
3178 }
3179
3180 let instance = store
3186 .concurrent_state_mut()?
3187 .get_mut(guest_thread.task)?
3188 .instance;
3189 store.switch_or_trap_if_may_not_suspend(instance)?;
3190
3191 let state = store.concurrent_state_mut()?;
3197 let old = state
3198 .get_mut(guest_thread.thread)?
3199 .wake_on_cancel
3200 .replace(WakeOnCancel::Waiting(set));
3201 if !old.is_none() {
3202 bail_bug!("thread unexpectedly had wake_on_cancel set");
3203 }
3204 let old = state
3205 .get_mut(set)?
3206 .waiting
3207 .insert(guest_thread, WaitMode::Callback(self));
3208 if !old.is_none() {
3209 bail_bug!("set's waiting set already had this thread registered");
3210 }
3211 Ok(())
3212 }
3213
3214 unsafe fn stage_call<T: 'static>(
3221 self,
3222 mut store: StoreContextMut<T>,
3223 guest_thread: QualifiedThreadId,
3224 callee: SendSyncPtr<VMFuncRef>,
3225 param_count: usize,
3226 result_count: usize,
3227 async_: bool,
3228 callback: Option<SendSyncPtr<VMFuncRef>>,
3229 post_return: Option<SendSyncPtr<VMFuncRef>>,
3230 host_caller: bool,
3231 ) -> Result<()> {
3232 unsafe fn make_call<T: 'static>(
3247 store: StoreContextMut<T>,
3248 guest_thread: QualifiedThreadId,
3249 callee: SendSyncPtr<VMFuncRef>,
3250 param_count: usize,
3251 result_count: usize,
3252 ) -> impl FnOnce(&mut dyn VMStore) -> Result<[MaybeUninit<ValRaw>; MAX_FLAT_PARAMS]>
3253 + Send
3254 + Sync
3255 + 'static
3256 + use<T> {
3257 let token = StoreToken::new(store);
3258 move |store: &mut dyn VMStore| {
3259 let mut storage = [MaybeUninit::uninit(); MAX_FLAT_PARAMS];
3260
3261 store
3262 .concurrent_state_mut()?
3263 .set_thread_running(guest_thread.thread)?;
3264 let task = store.concurrent_state_mut()?.get_mut(guest_thread.task)?;
3265 let lower = match task.lower_params.take() {
3266 Some(l) => l,
3267 None => bail_bug!("lower_params missing"),
3268 };
3269
3270 lower(store, &mut storage[..param_count])?;
3271
3272 let mut store = token.as_context_mut(store);
3273
3274 unsafe {
3277 crate::Func::call_unchecked_raw(
3278 &mut store,
3279 callee.as_non_null(),
3280 NonNull::new(
3281 &mut storage[..param_count.max(result_count)]
3282 as *mut [MaybeUninit<ValRaw>] as _,
3283 )
3284 .unwrap(),
3285 UncaughtException::Trap,
3286 )?;
3287 }
3288
3289 Ok(storage)
3290 }
3291 }
3292
3293 let call = unsafe {
3297 make_call(
3298 store.as_context_mut(),
3299 guest_thread,
3300 callee,
3301 param_count,
3302 result_count,
3303 )
3304 };
3305
3306 let callee_instance = store
3307 .0
3308 .concurrent_state_mut()?
3309 .get_mut(guest_thread.task)?
3310 .instance;
3311
3312 let fun = if callback.is_some() {
3313 assert!(async_);
3314
3315 Box::new(move |store: &mut dyn VMStore| {
3316 self.add_guest_thread_to_instance_table(
3317 guest_thread.thread,
3318 store,
3319 callee_instance.index,
3320 )?;
3321 self.handle_callback_code(store, guest_thread, callee_instance, |store| {
3322 let storage = call(store)?;
3323
3324 Ok(unsafe { storage[0].assume_init() }.get_i32() as u32)
3327 })
3328 }) as Box<dyn FnOnce(&mut dyn VMStore) -> Result<()> + Send + Sync>
3329 } else {
3330 let token = StoreToken::new(store.as_context_mut());
3331 Box::new(move |store: &mut dyn VMStore| {
3332 self.add_guest_thread_to_instance_table(
3333 guest_thread.thread,
3334 store,
3335 callee_instance.index,
3336 )?;
3337 let old_thread = store.set_thread(guest_thread)?;
3338 log::trace!(
3339 "sync/async-stackful call: replaced {old_thread:?} with {guest_thread:?} as current thread",
3340 );
3341 let flags = self.id().get(store).instance_flags(callee_instance.index);
3342
3343 let callee_async_typed = store
3344 .concurrent_state_mut()?
3345 .get_mut(guest_thread.task)?
3346 .async_typed;
3347
3348 if !async_ && callee_async_typed {
3352 store.enter_instance(callee_instance);
3353 }
3354
3355 if !callee_async_typed {
3356 store.enter_sync_call(callee_instance)?;
3357 }
3358
3359 let storage = call(store)?;
3366
3367 if !callee_async_typed {
3368 store.exit_sync_call(callee_instance)?;
3369 }
3370
3371 if !async_ {
3372 if callee_async_typed {
3378 store.exit_instance(callee_instance)?;
3379 }
3380
3381 let lift = {
3382 let state = store.concurrent_state_mut()?;
3383 if !state.get_mut(guest_thread.task)?.result.is_none() {
3384 bail_bug!("task has already produced a result");
3385 }
3386
3387 match state.get_mut(guest_thread.task)?.lift_result.take() {
3388 Some(lift) => lift,
3389 None => bail_bug!("lift_result field is missing"),
3390 }
3391 };
3392
3393 let result = (lift.lift)(store, unsafe {
3396 mem::transmute::<&[MaybeUninit<ValRaw>], &[ValRaw]>(
3397 &storage[..result_count],
3398 )
3399 })?;
3400
3401 let post_return_arg = match result_count {
3402 0 => ValRaw::i32(0),
3403 1 => unsafe { storage[0].assume_init() },
3406 _ => unreachable!(),
3407 };
3408
3409 unsafe {
3410 call_post_return(
3411 token.as_context_mut(store),
3412 post_return.map(|v| v.as_non_null()),
3413 post_return_arg,
3414 flags,
3415 )?;
3416 }
3417
3418 self.task_complete(store, guest_thread.task, result, Status::Returned)?;
3419 }
3420
3421 store.set_thread(old_thread)?;
3422
3423 store
3424 .concurrent_state_mut()?
3425 .get_mut(guest_thread.task)?
3426 .exited = true;
3427
3428 log::trace!(
3429 "clean up thread; async lifted? {async_} async typed? {callee_async_typed}"
3430 );
3431
3432 if callee_async_typed {
3433 store.switch_or_trap_if_may_not_suspend(callee_instance)?;
3438 }
3439
3440 store.cleanup_thread(guest_thread, callee_instance)?;
3442 Ok(())
3443 })
3444 };
3445
3446 store.0.concurrent_state_mut()?.push_work_item(
3447 WorkItem::GuestCall {
3448 instance: callee_instance,
3449 call: GuestCall {
3450 thread: guest_thread,
3451 kind: GuestCallKind::StartImplicit {
3452 closure: fun,
3453 was_pending: false,
3454 },
3455 },
3456 },
3457 if host_caller {
3458 Priority::High
3459 } else {
3460 Priority::Switch
3461 },
3462 )?;
3463
3464 Ok(())
3465 }
3466
3467 unsafe fn prepare_call<T: 'static>(
3480 self,
3481 mut store: StoreContextMut<T>,
3482 start: NonNull<VMFuncRef>,
3483 return_: NonNull<VMFuncRef>,
3484 caller_instance: RuntimeComponentInstanceIndex,
3485 callee_instance: RuntimeComponentInstanceIndex,
3486 task_return_type: TypeTupleIndex,
3487 callee_async_typed: bool,
3488 memory: *mut VMMemoryDefinition,
3489 string_encoding: StringEncoding,
3490 caller_info: CallerInfo,
3491 ) -> Result<()> {
3492 enum ResultInfo {
3493 Heap { results: u32 },
3494 Stack { result_count: u32 },
3495 }
3496
3497 let result_info = match &caller_info {
3498 CallerInfo::Async {
3499 has_result: true,
3500 params,
3501 } => ResultInfo::Heap {
3502 results: match params.last() {
3504 Some(r) => r.get_u32(),
3505 None => bail_bug!("retptr missing"),
3506 },
3507 },
3508 CallerInfo::Async {
3509 has_result: false, ..
3510 } => ResultInfo::Stack { result_count: 0 },
3511 CallerInfo::Sync {
3512 result_count,
3513 params,
3514 } if *result_count > u32::try_from(MAX_FLAT_RESULTS)? => ResultInfo::Heap {
3515 results: match params.last() {
3517 Some(r) => r.get_u32(),
3518 None => bail_bug!("arg ptr missing"),
3519 },
3520 },
3521 CallerInfo::Sync { result_count, .. } => ResultInfo::Stack {
3522 result_count: *result_count,
3523 },
3524 };
3525
3526 let sync_caller = matches!(caller_info, CallerInfo::Sync { .. });
3527
3528 let start = SendSyncPtr::new(start);
3532 let return_ = SendSyncPtr::new(return_);
3533 let token = StoreToken::new(store.as_context_mut());
3534 let old_thread = store.0.current_guest_thread()?;
3535
3536 let state = store.0.concurrent_state_mut()?;
3537
3538 debug_assert_eq!(
3539 state.get_mut(old_thread.task)?.instance,
3540 self.runtime_instance(caller_instance)
3541 );
3542
3543 let guest_thread = GuestTask::new(
3544 state,
3545 Box::new(move |store, dst| {
3546 let mut store = token.as_context_mut(store);
3547 assert!(dst.len() <= MAX_FLAT_PARAMS);
3548 let mut src = [MaybeUninit::uninit(); MAX_FLAT_PARAMS + 1];
3550 let count = match caller_info {
3551 CallerInfo::Async { params, has_result } => {
3555 let params = ¶ms[..params.len() - usize::from(has_result)];
3556 for (param, src) in params.iter().zip(&mut src) {
3557 src.write(*param);
3558 }
3559 params.len()
3560 }
3561
3562 CallerInfo::Sync { params, .. } => {
3564 for (param, src) in params.iter().zip(&mut src) {
3565 src.write(*param);
3566 }
3567 params.len()
3568 }
3569 };
3570 unsafe {
3577 crate::Func::call_unchecked_raw(
3578 &mut store,
3579 start.as_non_null(),
3580 NonNull::new(
3581 &mut src[..count.max(dst.len())] as *mut [MaybeUninit<ValRaw>] as _,
3582 )
3583 .unwrap(),
3584 UncaughtException::Trap,
3585 )?;
3586 }
3587 dst.copy_from_slice(&src[..dst.len()]);
3588 let task = store.0.current_guest_thread()?.task;
3589 let state = store.0.concurrent_state_mut()?;
3590 Waitable::Guest(task).set_event(
3591 state,
3592 Some(Event::Subtask {
3593 status: Status::Started,
3594 }),
3595 )?;
3596 Ok(())
3597 }),
3598 LiftResult {
3599 lift: Box::new(move |store, src| {
3600 let mut store = token.as_context_mut(store);
3603 let mut my_src = src.to_owned(); if let ResultInfo::Heap { results } = &result_info {
3605 my_src.push(ValRaw::u32(*results));
3606 }
3607
3608 unsafe {
3615 crate::Func::call_unchecked_raw(
3616 &mut store,
3617 return_.as_non_null(),
3618 my_src.as_mut_slice().into(),
3619 UncaughtException::Trap,
3620 )?;
3621 }
3622
3623 let thread = store.0.current_guest_thread()?;
3624 let state = store.0.concurrent_state_mut()?;
3625 if sync_caller {
3626 state.get_mut(thread.task)?.sync_result = SyncResult::Produced(
3627 if let ResultInfo::Stack { result_count } = &result_info {
3628 match result_count {
3629 0 => None,
3630 1 => Some(my_src[0]),
3631 _ => unreachable!(),
3632 }
3633 } else {
3634 None
3635 },
3636 );
3637 }
3638 Ok(Box::new(DummyResult) as Box<dyn Any + Send + Sync>)
3639 }),
3640 ty: task_return_type,
3641 memory: NonNull::new(memory).map(SendSyncPtr::new),
3642 string_encoding,
3643 },
3644 Caller::Guest { thread: old_thread },
3645 None,
3646 self.runtime_instance(callee_instance),
3647 callee_async_typed,
3648 false,
3651 )?;
3652
3653 store.0.set_thread(guest_thread)?;
3656 log::trace!("pushed {guest_thread:?} as current thread; old thread was {old_thread:?}");
3657
3658 Ok(())
3659 }
3660
3661 unsafe fn call_callback<T>(
3666 self,
3667 mut store: StoreContextMut<T>,
3668 function: SendSyncPtr<VMFuncRef>,
3669 event: Event,
3670 handle: u32,
3671 ) -> Result<u32> {
3672 let (ordinal, result) = event.parts();
3673 let params = &mut [
3674 ValRaw::u32(ordinal),
3675 ValRaw::u32(handle),
3676 ValRaw::u32(result),
3677 ];
3678 unsafe {
3683 crate::Func::call_unchecked_raw(
3684 &mut store,
3685 function.as_non_null(),
3686 params.as_mut_slice().into(),
3687 UncaughtException::Trap,
3688 )?;
3689 }
3690 Ok(params[0].get_u32())
3691 }
3692
3693 unsafe fn start_call<T: 'static>(
3711 self,
3712 mut store: StoreContextMut<T>,
3713 callback: *mut VMFuncRef,
3714 post_return: *mut VMFuncRef,
3715 callee: NonNull<VMFuncRef>,
3716 param_count: u32,
3717 result_count: u32,
3718 flags: u32,
3719 storage: &mut [MaybeUninit<ValRaw>],
3720 ) -> Result<()> {
3721 let token = StoreToken::new(store.as_context_mut());
3722 let async_caller = (flags & START_FLAG_ASYNC_CALLER) != 0;
3723 let guest_thread = store.0.current_guest_thread()?;
3724 let state = store.0.concurrent_state_mut()?;
3725
3726 if !state.event_loop_running {
3731 bail!(
3732 "async function started during an original synchronous \
3733 entrypoint; please use an async-style entrypoint instead"
3734 );
3735 }
3736
3737 let callee = SendSyncPtr::new(callee);
3738 let param_count = usize::try_from(param_count)?;
3739 assert!(param_count <= MAX_FLAT_PARAMS);
3740 let result_count = usize::try_from(result_count)?;
3741 assert!(result_count <= MAX_FLAT_RESULTS);
3742
3743 let task = state.get_mut(guest_thread.task)?;
3744 let callee_async_typed = task.async_typed;
3745 let callee_instance = task.instance;
3746
3747 task.async_lifted = (flags & START_FLAG_ASYNC_CALLEE) != 0;
3748
3749 if let Some(callback) = NonNull::new(callback) {
3750 let callback = SendSyncPtr::new(callback);
3754 task.callback = Some(Box::new(move |store, event, handle| {
3755 let store = token.as_context_mut(store);
3756 unsafe { self.call_callback::<T>(store, callback, event, handle) }
3757 }));
3758 }
3759
3760 let Caller::Guest { thread: caller } = &task.caller else {
3761 bail_bug!("start_call unexpectedly invoked for host->guest call");
3764 };
3765 let caller = *caller;
3766 let caller_instance = state.get_mut(caller.task)?.instance;
3767
3768 unsafe {
3770 self.stage_call(
3771 store.as_context_mut(),
3772 guest_thread,
3773 callee,
3774 param_count,
3775 result_count,
3776 (flags & START_FLAG_ASYNC_CALLEE) != 0,
3777 NonNull::new(callback).map(SendSyncPtr::new),
3778 NonNull::new(post_return).map(SendSyncPtr::new),
3779 false,
3780 )?;
3781 }
3782
3783 let old_do_not_suspend = if callee_async_typed {
3784 let state = store.0.instance_state(callee_instance).concurrent_state();
3791 let old_do_not_suspend = state.do_not_suspend;
3792 state.do_not_suspend = false;
3793 Some(old_do_not_suspend)
3794 } else {
3795 None
3796 };
3797
3798 let state = store.0.concurrent_state_mut()?;
3799
3800 let guest_waitable = Waitable::Guest(guest_thread.task);
3803 let old_set = guest_waitable.common(state)?.set;
3804 let set = state.get_mut(caller.thread)?.sync_call_set;
3805 guest_waitable.join(state, Some(set))?;
3806
3807 store.0.set_thread(CurrentThread::None)?;
3808
3809 let mut yielded = false;
3825 let (status, waitable) = loop {
3826 store.0.suspend(if yielded {
3827 SuspendReason::Waiting {
3828 set,
3829 thread: caller,
3830 }
3831 } else {
3832 yielded = true;
3833 SuspendReason::YieldingToSubtask { thread: caller }
3834 })?;
3835
3836 if let Some(old_do_not_suspend) = old_do_not_suspend {
3837 store
3838 .0
3839 .instance_state(callee_instance)
3840 .concurrent_state()
3841 .do_not_suspend = old_do_not_suspend;
3842 }
3843
3844 let state = store.0.concurrent_state_mut()?;
3845
3846 log::trace!("taking event for {:?}", guest_thread.task);
3847 let event = guest_waitable.take_event(state)?;
3848 let Some(Event::Subtask { status }) = event else {
3849 bail_bug!("subtasks should only get subtask events, got {event:?}")
3850 };
3851
3852 log::trace!("status {status:?} for {:?}", guest_thread.task);
3853
3854 if status == Status::Returned {
3855 break (status, None);
3857 } else if async_caller {
3858 let handle = store
3862 .0
3863 .handle_table_insertion(caller_instance)
3864 .subtask_guest(guest_thread.task.rep())?;
3865 store
3866 .0
3867 .concurrent_state_mut()?
3868 .get_mut(guest_thread.task)?
3869 .common
3870 .handle = Some(handle);
3871 break (status, Some(handle));
3872 } else {
3873 store.0.switch_or_trap_if_may_not_suspend(caller_instance)?;
3877 }
3878 };
3879
3880 guest_waitable.join(store.0.concurrent_state_mut()?, old_set)?;
3881
3882 store.0.set_thread(caller)?;
3884
3885 store
3886 .0
3887 .concurrent_state_mut()?
3888 .set_thread_running(caller.thread)?;
3889
3890 log::trace!("popped current thread {guest_thread:?}; new thread is {caller:?}");
3891
3892 if async_caller {
3893 let Some(slot) = storage.first_mut() else {
3894 bail_bug!("no storage for async call status");
3895 };
3896 *slot = MaybeUninit::new(ValRaw::u32(status.pack(waitable)));
3897 } else {
3898 let state = store.0.concurrent_state_mut()?;
3901 let task = state.get_mut(guest_thread.task)?;
3902 if let Some(Some(result)) = task.sync_result.take()? {
3903 storage[0] = MaybeUninit::new(result);
3904 }
3905 }
3906
3907 if status == Status::Returned {
3908 store
3909 .0
3910 .concurrent_state_mut()?
3911 .decrement_task_ref_count(guest_thread.task)?;
3912 }
3913
3914 Ok(())
3915 }
3916
3917 pub(crate) fn first_poll<T: 'static, R: Send + 'static>(
3933 self,
3934 mut store: StoreContextMut<'_, T>,
3935 host_task: EnteredHostTask,
3936 result_may_require_realloc: bool,
3937 future: impl Future<Output = Result<R>> + Send + 'static,
3938 lower: impl FnOnce(StoreContextMut<T>, Option<R>, bool, Option<TableId<HostTask>>) -> Result<()>
3939 + Send
3940 + 'static,
3941 ) -> Result<u32> {
3942 let token = StoreToken::new(store.as_context_mut());
3943
3944 let (join_handle, future) = JoinHandle::run(future);
3947 let mut future = Box::pin(future);
3948
3949 let poll = tls::set(store.0, || {
3954 future
3955 .as_mut()
3956 .poll(&mut Context::from_waker(&Waker::noop()))
3957 });
3958
3959 match poll {
3960 Poll::Ready(result) => {
3962 let result = result.transpose()?;
3963 let task = store.0.current_materialized_host_task()?;
3966 lower(store.as_context_mut(), result, true, task)?;
3967 return Ok(Status::Returned.pack(None));
3968 }
3969
3970 Poll::Pending => {}
3972 }
3973
3974 let Some(task) = store.0.materialize_host_task_id()? else {
3978 bail_bug!("current thread is not a host thread")
3979 };
3980 {
3981 let state = &mut store.0.concurrent_state_mut()?.get_mut(task)?.state;
3982 assert!(matches!(state, HostTaskState::CalleeStarted));
3983 *state = HostTaskState::CalleeRunning(join_handle);
3984 }
3985
3986 let future = Box::pin(async move {
3994 let result = match run_with_host_task_set(task, future).await? {
3995 Some(result) => Some(result?),
3996 None => None,
3997 };
3998 let on_complete = move |store: &mut dyn VMStore| {
3999 let mut store = token.as_context_mut(store);
4003 let old = store.0.set_thread(task)?;
4004
4005 let status = if result.is_some() {
4006 Status::Returned
4007 } else {
4008 Status::ReturnCancelled
4009 };
4010
4011 lower(store.as_context_mut(), result, false, Some(task))?;
4012 let state = store.0.concurrent_state_mut()?;
4013 match &mut state.get_mut(task)?.state {
4014 pending @ HostTaskState::CalleeCancelling => {
4017 *pending = HostTaskState::CalleeDone { cancelled: true };
4018 }
4019
4020 other => *other = HostTaskState::CalleeDone { cancelled: false },
4022 }
4023 Waitable::Host(task).set_event(state, Some(Event::Subtask { status }))?;
4024
4025 store.0.set_thread(old)?;
4026 Ok(())
4027 };
4028
4029 tls::get(move |store| {
4030 if result_may_require_realloc {
4031 store
4036 .concurrent_state_mut()?
4037 .push_high_priority(WorkItem::WorkerFunction(AlwaysMut::new(Box::new(
4038 on_complete,
4039 ))));
4040 Ok(())
4041 } else {
4042 on_complete(store)
4045 }
4046 })
4047 });
4048
4049 let caller = match host_task {
4052 Some(caller) => caller,
4053 None => bail_bug!("host task wasn't created but should have been"),
4054 };
4055 let state = store.0.concurrent_state_mut()?;
4056 state.push_future(future);
4057 let instance = state.get_mut(caller.task)?.instance;
4058 let handle = store
4059 .0
4060 .handle_table_insertion(instance)
4061 .subtask_host(task.rep())?;
4062 store.0.concurrent_state_mut()?.get_mut(task)?.common.handle = Some(handle);
4063 log::trace!("assign {task:?} handle {handle} for {caller:?} instance {instance:?}");
4064
4065 store.0.set_thread(caller)?;
4069 Ok(Status::Started.pack(Some(handle)))
4070 }
4071
4072 pub(crate) fn task_return(
4075 self,
4076 store: &mut dyn VMStore,
4077 ty: TypeTupleIndex,
4078 options: OptionsIndex,
4079 storage: &[ValRaw],
4080 ) -> Result<()> {
4081 let guest_thread = store.current_guest_thread()?;
4082 let state = store.concurrent_state_mut()?;
4083 if !state.get_mut(guest_thread.task)?.async_lifted {
4084 bail!(Trap::TaskReturnOrCancelSyncLifted);
4085 }
4086 let lift = state
4087 .get_mut(guest_thread.task)?
4088 .lift_result
4089 .take()
4090 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4091 if !state.get_mut(guest_thread.task)?.result.is_none() {
4092 bail_bug!("task result unexpectedly already set");
4093 }
4094
4095 let CanonicalOptions {
4096 string_encoding,
4097 data_model,
4098 ..
4099 } = &self.id().get(store).component().env_component().options[options];
4100
4101 let invalid = ty != lift.ty
4102 || string_encoding != &lift.string_encoding
4103 || match data_model {
4104 CanonicalOptionsDataModel::LinearMemory(opts) => match opts.memory {
4105 Some(memory) => {
4106 let expected = lift.memory.map(|v| v.as_ptr()).unwrap_or(ptr::null_mut());
4107 let actual = self.id().get(store).runtime_memory(memory);
4108 expected != actual.as_ptr()
4109 }
4110 None => false,
4113 },
4114 CanonicalOptionsDataModel::Gc { .. } => true,
4116 };
4117
4118 if invalid {
4119 bail!(Trap::TaskReturnInvalid);
4120 }
4121
4122 log::trace!("task.return for {guest_thread:?}");
4123
4124 let result = (lift.lift)(store, storage)?;
4125 self.task_complete(store, guest_thread.task, result, Status::Returned)
4126 }
4127
4128 pub(crate) fn task_cancel(self, store: &mut StoreOpaque) -> Result<()> {
4130 let guest_thread = store.current_guest_thread()?;
4131 let state = store.concurrent_state_mut()?;
4132 let task = state.get_mut(guest_thread.task)?;
4133 if !task.async_lifted {
4134 bail!(Trap::TaskReturnOrCancelSyncLifted);
4135 }
4136 if !task.cancel_request_delivered {
4137 bail!(Trap::TaskCancelNotCancelled);
4138 }
4139 _ = task
4140 .lift_result
4141 .take()
4142 .ok_or_else(|| Trap::TaskCancelOrReturnTwice)?;
4143
4144 if !task.result.is_none() {
4145 bail_bug!("task result should not bet set yet");
4146 }
4147
4148 log::trace!("task.cancel for {guest_thread:?}");
4149
4150 self.task_complete(
4151 store,
4152 guest_thread.task,
4153 Box::new(DummyResult),
4154 Status::ReturnCancelled,
4155 )
4156 }
4157
4158 fn task_complete(
4164 self,
4165 store: &mut StoreOpaque,
4166 guest_task: TableId<GuestTask>,
4167 result: Box<dyn Any + Send + Sync>,
4168 status: Status,
4169 ) -> Result<()> {
4170 store
4171 .component_resource_tables(Some(self))?
4172 .validate_scope_exit()?;
4173
4174 let state = store.concurrent_state_mut()?;
4175 let task = state.get_mut(guest_task)?;
4176
4177 task.event = None;
4181
4182 if let Caller::Host { tx, .. } = &mut task.caller {
4183 if let Some(tx) = tx.take() {
4184 _ = tx.send(result);
4185 }
4186 } else {
4187 task.result = Some(result);
4188 Waitable::Guest(guest_task).set_event(state, Some(Event::Subtask { status }))?;
4189 }
4190
4191 Ok(())
4192 }
4193
4194 pub(crate) fn waitable_set_new(
4196 self,
4197 store: &mut StoreOpaque,
4198 caller_instance: RuntimeComponentInstanceIndex,
4199 ) -> Result<u32> {
4200 let set = store.concurrent_state_mut()?.push(WaitableSet::default())?;
4201 let handle = store
4202 .handle_table_insertion(self.runtime_instance(caller_instance))
4203 .waitable_set(set.rep())?;
4204 log::trace!("new waitable set {set:?} (handle {handle})");
4205 Ok(handle)
4206 }
4207
4208 pub(crate) fn waitable_set_drop(
4210 self,
4211 store: &mut StoreOpaque,
4212 caller_instance: RuntimeComponentInstanceIndex,
4213 set: u32,
4214 ) -> Result<()> {
4215 let rep = store
4216 .instance_state(self.runtime_instance(caller_instance))
4217 .handle_table()
4218 .waitable_set_remove(set)?;
4219
4220 log::trace!("drop waitable set {rep} (handle {set})");
4221
4222 let set = store
4226 .concurrent_state_mut()?
4227 .get_mut(TableId::<WaitableSet>::new(rep))?;
4228 if set.num_waiting > 0 {
4229 bail!(Trap::WaitableSetDropHasWaiters);
4230 }
4231
4232 store
4233 .concurrent_state_mut()?
4234 .delete(TableId::<WaitableSet>::new(rep))?;
4235
4236 Ok(())
4237 }
4238
4239 pub(crate) fn waitable_join(
4241 self,
4242 store: &mut StoreOpaque,
4243 caller_instance: RuntimeComponentInstanceIndex,
4244 waitable_handle: u32,
4245 set_handle: u32,
4246 ) -> Result<()> {
4247 let mut instance = self.id().get_mut(store);
4248 let waitable =
4249 Waitable::from_instance(instance.as_mut(), caller_instance, waitable_handle)?;
4250
4251 let set = if set_handle == 0 {
4252 None
4253 } else {
4254 let set = instance.instance_states().0[caller_instance]
4255 .handle_table()
4256 .waitable_set_rep(set_handle)?;
4257
4258 Some(TableId::<WaitableSet>::new(set))
4259 };
4260
4261 waitable.trap_if_has_sync_waiter(store.concurrent_state_mut()?)?;
4262
4263 log::trace!(
4264 "waitable {waitable:?} (handle {waitable_handle}) join set {set:?} (handle {set_handle})",
4265 );
4266
4267 waitable.join(store.concurrent_state_mut()?, set)
4268 }
4269
4270 pub(crate) fn subtask_drop(
4272 self,
4273 store: &mut StoreOpaque,
4274 caller_instance: RuntimeComponentInstanceIndex,
4275 task_id: u32,
4276 ) -> Result<()> {
4277 self.waitable_join(store, caller_instance, task_id, 0)?;
4278
4279 let (rep, is_host) = store
4280 .instance_state(self.runtime_instance(caller_instance))
4281 .handle_table()
4282 .subtask_remove(task_id)?;
4283
4284 let concurrent_state = store.concurrent_state_mut()?;
4285 let waitable = if is_host {
4286 let id = TableId::<HostTask>::new(rep);
4287 let task = concurrent_state.get_mut(id)?;
4288 match &task.state {
4289 HostTaskState::CalleeRunning(_) | HostTaskState::CalleeCancelling => {
4290 bail!(Trap::SubtaskDropNotResolved)
4291 }
4292 HostTaskState::CalleeDone { .. } => {}
4293 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4294 bail_bug!("invalid state for callee in `subtask.drop`")
4295 }
4296 }
4297
4298 Waitable::Host(id)
4299 } else {
4300 let id = TableId::<GuestTask>::new(rep);
4301 let task = concurrent_state.get_mut(id)?;
4302 if task.lift_result.is_some() {
4303 bail!(Trap::SubtaskDropNotResolved);
4304 }
4305 Waitable::Guest(id)
4306 };
4307
4308 waitable.common(concurrent_state)?.handle = None;
4309
4310 if waitable.take_event(concurrent_state)?.is_some() {
4313 bail!(Trap::SubtaskDropNotResolved);
4314 }
4315
4316 waitable.delete_from(concurrent_state)?;
4317
4318 log::trace!("subtask_drop {waitable:?} (handle {task_id})");
4319 Ok(())
4320 }
4321
4322 pub(crate) fn waitable_set_wait(
4324 self,
4325 store: &mut StoreOpaque,
4326 options: OptionsIndex,
4327 set: u32,
4328 payload: u32,
4329 ) -> Result<u32> {
4330 let &CanonicalOptions {
4331 instance: caller_instance,
4332 ..
4333 } = &self.id().get(store).component().env_component().options[options];
4334 let caller = self.runtime_instance(caller_instance);
4335 let rep = store
4336 .instance_state(self.runtime_instance(caller_instance))
4337 .handle_table()
4338 .waitable_set_rep(set)?;
4339
4340 self.waitable_check(
4341 store,
4342 caller,
4343 WaitableCheck::Wait,
4344 WaitableCheckParams {
4345 set: TableId::new(rep),
4346 options,
4347 payload,
4348 },
4349 )
4350 }
4351
4352 pub(crate) fn waitable_set_poll(
4354 self,
4355 store: &mut StoreOpaque,
4356 options: OptionsIndex,
4357 set: u32,
4358 payload: u32,
4359 ) -> Result<u32> {
4360 let &CanonicalOptions {
4361 instance: caller_instance,
4362 ..
4363 } = &self.id().get(store).component().env_component().options[options];
4364 let caller = self.runtime_instance(caller_instance);
4365 let rep = store
4366 .instance_state(caller)
4367 .handle_table()
4368 .waitable_set_rep(set)?;
4369
4370 self.waitable_check(
4371 store,
4372 caller,
4373 WaitableCheck::Poll,
4374 WaitableCheckParams {
4375 set: TableId::new(rep),
4376 options,
4377 payload,
4378 },
4379 )
4380 }
4381
4382 pub(crate) fn thread_index(&self, store: &mut dyn VMStore) -> Result<u32> {
4384 let thread_id = store.current_guest_thread()?.thread;
4385 match store
4386 .concurrent_state_mut()?
4387 .get_mut(thread_id)?
4388 .instance_rep
4389 {
4390 Some(r) => Ok(r),
4391 None => bail_bug!("thread should have instance_rep by now"),
4392 }
4393 }
4394
4395 pub(crate) fn thread_new_indirect<T: 'static>(
4397 self,
4398 mut store: StoreContextMut<T>,
4399 runtime_instance: RuntimeComponentInstanceIndex,
4400 start_func_ty: ModuleInternedTypeIndex,
4401 start_func_table_idx: RuntimeTableIndex,
4402 start_func_idx: u32,
4403 context: i32,
4404 ) -> Result<u32> {
4405 log::trace!("creating new thread");
4406
4407 let (instance, registry) = self.id().get_mut_and_registry(store.0);
4408 let Some(start_func_ty) = instance.component().signatures().shared_type(start_func_ty)
4409 else {
4410 bail_bug!("thread.new-indirect start function type should be registered");
4411 };
4412 let callee = instance
4413 .index_runtime_func_table(registry, start_func_table_idx, start_func_idx as u64)?
4414 .ok_or_else(|| Trap::ThreadNewIndirectUninitialized)?;
4415 if callee.type_index(store.0) != start_func_ty {
4416 bail!(Trap::ThreadNewIndirectInvalidType);
4417 }
4418
4419 let token = StoreToken::new(store.as_context_mut());
4420 let start_func = Box::new(
4421 move |store: &mut dyn VMStore, guest_thread: QualifiedThreadId| -> Result<()> {
4422 let old_thread = store.set_thread(guest_thread)?;
4423 log::trace!(
4424 "thread start: replaced {old_thread:?} with {guest_thread:?} as current thread"
4425 );
4426
4427 let mut store = token.as_context_mut(store);
4428 let mut params = [ValRaw::i32(context)];
4429 unsafe { callee.call_unchecked(store.as_context_mut(), &mut params)? };
4432
4433 store.0.set_thread(old_thread)?;
4434
4435 let runtime_instance = self.runtime_instance(runtime_instance);
4436
4437 store
4440 .0
4441 .switch_or_trap_if_may_not_suspend(runtime_instance)?;
4442
4443 store.0.cleanup_thread(guest_thread, runtime_instance)?;
4444
4445 log::trace!("explicit thread {guest_thread:?} completed");
4446 let state = store.0.concurrent_state_mut()?;
4447 if let Some(t) = old_thread.guest() {
4448 state.set_thread_running(t.thread)?;
4449 }
4450 log::trace!("thread start: restored {old_thread:?} as current thread");
4451
4452 Ok(())
4453 },
4454 );
4455
4456 let current_thread = store.0.current_guest_thread()?;
4457 let state = store.0.concurrent_state_mut()?;
4458 let parent_task = current_thread.task;
4459
4460 let new_thread = GuestThread::new_explicit(state, parent_task, start_func)?;
4461 let thread_id = state.push(new_thread)?;
4462 state.get_mut(parent_task)?.threads.insert(thread_id);
4463
4464 log::trace!("new thread with id {thread_id:?} created");
4465
4466 self.add_guest_thread_to_instance_table(thread_id, store.0, runtime_instance)
4467 }
4468
4469 pub(crate) fn resume_thread(
4470 self,
4471 store: &mut StoreOpaque,
4472 runtime_instance: RuntimeComponentInstanceIndex,
4473 thread_idx: u32,
4474 how: ResumeThread,
4475 ) -> Result<bool> {
4476 let thread_id =
4477 GuestThread::from_instance(self.id().get_mut(store), runtime_instance, thread_idx)?;
4478 let state = store.concurrent_state_mut()?;
4479 let guest_thread = QualifiedThreadId::qualify(state, thread_id)?;
4480
4481 if store.current_guest_thread()? == guest_thread {
4482 bail!(Trap::CannotResumeThread);
4483 }
4484
4485 let priority = match how {
4486 ResumeThread::Resume => Priority::Switch,
4487 ResumeThread::ResumeLater => Priority::Low,
4488
4489 ResumeThread::Promote => {
4492 let instance = self.runtime_instance(runtime_instance);
4493 let do_not_enter = store
4494 .instance_state(instance)
4495 .concurrent_state()
4496 .do_not_enter;
4497 return store.concurrent_state_mut()?.promote_work_item_matching(
4498 |item: &WorkItem| match item {
4499 WorkItem::ResumeThread { thread, .. }
4500 | WorkItem::ResumeFiber { thread, .. } => *thread == guest_thread,
4501
4502 WorkItem::GuestCall {
4503 call: GuestCall { thread, kind },
4504 ..
4505 } => {
4506 *thread == guest_thread
4507 && match kind {
4508 GuestCallKind::DeliverEvent { .. } => !do_not_enter,
4512
4513 GuestCallKind::StartExplicit(_) => true,
4516
4517 GuestCallKind::StartImplicit { .. } => false,
4521 }
4522 }
4523
4524 WorkItem::PushFuture(_) | WorkItem::WorkerFunction(_) => false,
4525 },
4526 );
4527 }
4528 };
4529
4530 let state = store.concurrent_state_mut()?;
4531 let thread = state.get_mut(guest_thread.thread)?;
4532
4533 match &thread.state {
4536 GuestThreadState::NotStartedExplicit(_) => {}
4537 GuestThreadState::Suspended(_) => {
4538 log::trace!("resuming thread {thread_id:?} that was suspended");
4539 state.schedule_suspended_thread(
4540 guest_thread,
4541 self.runtime_instance(runtime_instance),
4542 priority,
4543 )?;
4544 return Ok(true);
4545 }
4546 _ => bail!(Trap::CannotResumeThread),
4547 }
4548 match mem::replace(&mut thread.state, GuestThreadState::Running) {
4549 GuestThreadState::NotStartedExplicit(start_func) => {
4550 log::trace!("starting thread {guest_thread:?}");
4551 let guest_call = WorkItem::GuestCall {
4552 instance: self.runtime_instance(runtime_instance),
4553 call: GuestCall {
4554 thread: guest_thread,
4555 kind: GuestCallKind::StartExplicit(Box::new(move |store| {
4556 start_func(store, guest_thread)
4557 })),
4558 },
4559 };
4560 state.push_work_item(guest_call, priority)?;
4561 }
4562 other => {
4563 thread.state = other;
4564 bail_bug!("thread state checked to be resumable above");
4565 }
4566 }
4567 Ok(true)
4568 }
4569
4570 fn add_guest_thread_to_instance_table(
4571 self,
4572 thread_id: TableId<GuestThread>,
4573 store: &mut StoreOpaque,
4574 runtime_instance: RuntimeComponentInstanceIndex,
4575 ) -> Result<u32> {
4576 let (state, counter) =
4577 store.instance_state_and_handle_counter(self.runtime_instance(runtime_instance));
4578 let guest_id = state
4579 .thread_handle_table()
4580 .guest_thread_insert(counter, thread_id.rep())?;
4581 store
4582 .concurrent_state_mut()?
4583 .get_mut(thread_id)?
4584 .instance_rep = Some(guest_id);
4585 Ok(guest_id)
4586 }
4587
4588 pub(crate) fn suspension_intrinsic(
4592 self,
4593 store: &mut StoreOpaque,
4594 caller: RuntimeComponentInstanceIndex,
4595 yielding: bool,
4596 to_thread: SuspensionTarget,
4597 ) -> Result<WaitResult> {
4598 let check_suspend = match to_thread {
4599 SuspensionTarget::Promote(thread) => {
4600 !self.resume_thread(store, caller, thread, ResumeThread::Promote)?
4601 }
4602 SuspensionTarget::Resume(thread) => {
4603 if !self.resume_thread(store, caller, thread, ResumeThread::Resume)? {
4604 bail_bug!(
4605 "`resume_thread` should only ever return false \
4606 when `ResumeThread::Promote` is passed to it"
4607 );
4608 }
4609 false
4610 }
4611 SuspensionTarget::None => true,
4612 };
4613
4614 if check_suspend && !store.switch_if_may_not_suspend(self.runtime_instance(caller))? {
4615 return if yielding {
4616 Ok(WaitResult::Completed)
4617 } else {
4618 Err(Trap::CannotBlockSyncTask.into())
4619 };
4620 }
4621
4622 let guest_thread = store.current_guest_thread()?;
4623
4624 let reason = if yielding {
4625 SuspendReason::Yielding {
4626 thread: guest_thread,
4627 }
4628 } else {
4629 SuspendReason::ExplicitlySuspending {
4630 thread: guest_thread,
4631 }
4632 };
4633
4634 store.suspend(reason)?;
4635
4636 Ok(WaitResult::Completed)
4637 }
4638
4639 fn waitable_check(
4641 self,
4642 store: &mut StoreOpaque,
4643 caller: RuntimeInstance,
4644 check: WaitableCheck,
4645 params: WaitableCheckParams,
4646 ) -> Result<u32> {
4647 let guest_thread = store.current_guest_thread()?;
4648
4649 log::trace!("waitable check for {guest_thread:?}; set {:?}", params.set);
4650
4651 match &check {
4654 WaitableCheck::Wait => {
4655 let set = params.set;
4656
4657 loop {
4662 let state = store.concurrent_state_mut()?;
4663 let task = state.get_mut(guest_thread.task)?;
4664 if !(task.event.is_none() || matches!(task.event, Some(Event::Cancelled)))
4665 || !state.get_mut(set)?.ready.is_empty()
4666 {
4667 break;
4668 }
4669
4670 store.switch_or_trap_if_may_not_suspend(caller)?;
4671
4672 store.suspend(SuspendReason::Waiting {
4673 set,
4674 thread: guest_thread,
4675 })?;
4676 }
4677 }
4678 WaitableCheck::Poll => {}
4679 }
4680
4681 log::trace!(
4682 "waitable check for {guest_thread:?}; set {:?}, part two",
4683 params.set
4684 );
4685
4686 let event = self.get_event(store, guest_thread.task, Some(params.set), false)?;
4688
4689 let (ordinal, handle, result) = match &check {
4690 WaitableCheck::Wait => {
4691 let (event, waitable) = match event {
4692 Some(p) => p,
4693 None => bail_bug!("event expected to be present"),
4694 };
4695 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4696 let (ordinal, result) = event.parts();
4697 (ordinal, handle, result)
4698 }
4699 WaitableCheck::Poll => {
4700 if let Some((event, waitable)) = event {
4701 let handle = waitable.map(|(_, v)| v).unwrap_or(0);
4702 let (ordinal, result) = event.parts();
4703 (ordinal, handle, result)
4704 } else {
4705 log::trace!(
4706 "no events ready to deliver via waitable-set.poll to {:?}; set {:?}",
4707 guest_thread.task,
4708 params.set
4709 );
4710 let (ordinal, result) = Event::None.parts();
4711 (ordinal, 0, result)
4712 }
4713 }
4714 };
4715 let ptr = crate::component::func::validate_inbounds_dynamic(
4716 &CanonicalAbiInfo::POINTER_PAIR,
4717 self.options_memory(store, params.options),
4718 &ValRaw::u32(params.payload),
4719 )?;
4720 let memory = self
4721 .options_memory_range_mut(store, params.options, ptr, 8)
4722 .unwrap();
4723 memory[0..4].copy_from_slice(&handle.to_le_bytes());
4724 memory[4..8].copy_from_slice(&result.to_le_bytes());
4725 Ok(ordinal)
4726 }
4727
4728 pub(crate) fn subtask_cancel(
4730 self,
4731 store: &mut StoreOpaque,
4732 caller_instance: RuntimeComponentInstanceIndex,
4733 async_: bool,
4734 task_id: u32,
4735 ) -> Result<u32> {
4736 let (rep, is_host) = store
4737 .instance_state(self.runtime_instance(caller_instance))
4738 .handle_table()
4739 .subtask_rep(task_id)?;
4740 let waitable = if is_host {
4741 Waitable::Host(TableId::<HostTask>::new(rep))
4742 } else {
4743 Waitable::Guest(TableId::<GuestTask>::new(rep))
4744 };
4745 let concurrent_state = store.concurrent_state_mut()?;
4746
4747 log::trace!("subtask_cancel {waitable:?} (handle {task_id}; async {async_})");
4748
4749 waitable.trap_if_in_waitable_set(concurrent_state)?;
4750
4751 let needs_block;
4752 if let Waitable::Host(host_task) = waitable {
4753 let state = &mut concurrent_state.get_mut(host_task)?.state;
4754 match state {
4755 HostTaskState::CalleeRunning(handle) => {
4762 handle.abort();
4763 *state = HostTaskState::CalleeCancelling;
4764 needs_block = true;
4765 }
4766
4767 HostTaskState::CalleeCancelling | HostTaskState::CalleeDone { cancelled: true } => {
4770 bail!(Trap::SubtaskCancelAfterTerminal);
4771 }
4772 HostTaskState::CalleeDone { cancelled: false } => {
4773 *state = HostTaskState::CalleeDone { cancelled: true };
4776 needs_block = false;
4777 }
4778
4779 HostTaskState::CalleeStarted | HostTaskState::CalleeFinished(_) => {
4782 bail_bug!("invalid states for host callee")
4783 }
4784 }
4785 } else {
4786 let guest_task = TableId::<GuestTask>::new(rep);
4787 let task = concurrent_state.get_mut(guest_task)?;
4788
4789 if task.cancel_requested {
4792 bail!(Trap::SubtaskCancelAfterTerminal);
4793 }
4794
4795 if !task.already_lowered_parameters() {
4796 store.cancel_guest_subtask_without_lowered_parameters(
4797 self.runtime_instance(caller_instance),
4798 guest_task,
4799 )?;
4800 return Ok(Status::StartCancelled as u32);
4801 } else if !task.returned_or_cancelled() {
4802 task.cancel_requested = true;
4809 task.event = Some(Event::Cancelled);
4810 let runtime_instance = task.instance;
4811 for thread in task.threads.clone() {
4812 let thread = QualifiedThreadId {
4813 task: guest_task,
4814 thread,
4815 };
4816 let concurrent_state = store.concurrent_state_mut()?;
4817 let thread_mut = concurrent_state.get_mut(thread.thread)?;
4818
4819 let yield_ = |store: &mut StoreOpaque| {
4820 let state = store.instance_state(runtime_instance).concurrent_state();
4825 let old_do_not_suspend = state.do_not_suspend;
4826 state.do_not_suspend = false;
4827
4828 let caller = store.current_guest_thread()?;
4829
4830 let state = store.concurrent_state_mut()?;
4835 let set = state.get_mut(caller.thread)?.sync_call_set;
4836 waitable.join(state, Some(set))?;
4837
4838 store.suspend(SuspendReason::YieldingToSubtask { thread: caller })?;
4839
4840 let state = store.concurrent_state_mut()?;
4841 waitable.join(state, None)?;
4842
4843 store
4844 .instance_state(runtime_instance)
4845 .concurrent_state()
4846 .do_not_suspend = old_do_not_suspend;
4847
4848 Ok::<(), crate::Error>(())
4849 };
4850
4851 match thread_mut.wake_on_cancel.take() {
4852 WakeOnCancel::Waiting(set) => {
4853 let instance = concurrent_state.take_callback_waiter(set, thread)?;
4855 let item = WorkItem::GuestCall {
4856 instance: runtime_instance,
4857 call: GuestCall {
4858 thread,
4859 kind: GuestCallKind::DeliverEvent {
4860 instance,
4861 set: Some(set),
4862 },
4863 },
4864 };
4865 concurrent_state.set_switch_item(item)?;
4866
4867 yield_(store)?;
4868
4869 break;
4870 }
4871 WakeOnCancel::Yielding => {
4872 if concurrent_state.promote_thread_work_item(thread)? {
4873 yield_(store)?;
4874 break;
4875 } else if store
4876 .instance_state(runtime_instance)
4877 .concurrent_state()
4878 .pending
4879 .contains_key(&thread)
4880 {
4881 store
4891 .concurrent_state_mut()?
4892 .get_mut(thread.thread)?
4893 .wake_on_cancel = WakeOnCancel::Yielding;
4894 } else {
4895 bail_bug!("thread with `WakeOnCancel::Yielding` not promotable");
4896 }
4897 }
4898 WakeOnCancel::None => {}
4899 }
4900 }
4901
4902 needs_block = !store
4905 .concurrent_state_mut()?
4906 .get_mut(guest_task)?
4907 .returned_or_cancelled()
4908 } else {
4909 needs_block = false;
4910 }
4911 };
4912
4913 if needs_block {
4917 if async_ {
4918 return Ok(BLOCKED);
4919 }
4920
4921 let old_next_switch_item = if is_host {
4931 Some(store.concurrent_state_mut()?.save_next_switch_item()?)
4932 } else {
4933 None
4934 };
4935
4936 store.wait_for_event(self.runtime_instance(caller_instance), waitable)?;
4939
4940 if let Some(item) = old_next_switch_item {
4941 let state = store.concurrent_state_mut()?;
4942 state.restore_next_switch_item(item)?;
4943 }
4944
4945 }
4947
4948 let event = waitable.take_event(store.concurrent_state_mut()?)?;
4949 if let Some(Event::Subtask {
4950 status: status @ (Status::Returned | Status::ReturnCancelled),
4951 }) = event
4952 {
4953 Ok(status as u32)
4954 } else {
4955 bail!(Trap::SubtaskCancelAfterTerminal);
4956 }
4957 }
4958}
4959
4960pub trait VMComponentAsyncStore {
4968 unsafe fn prepare_call(
4974 &mut self,
4975 instance: Instance,
4976 memory: *mut VMMemoryDefinition,
4977 start: NonNull<VMFuncRef>,
4978 return_: NonNull<VMFuncRef>,
4979 caller_instance: RuntimeComponentInstanceIndex,
4980 callee_instance: RuntimeComponentInstanceIndex,
4981 task_return_type: TypeTupleIndex,
4982 callee_async: bool,
4983 string_encoding: StringEncoding,
4984 result_count: u32,
4985 storage: *mut ValRaw,
4986 storage_len: usize,
4987 ) -> Result<()>;
4988
4989 unsafe fn start_call(
4992 &mut self,
4993 instance: Instance,
4994 callback: *mut VMFuncRef,
4995 post_return: *mut VMFuncRef,
4996 callee: NonNull<VMFuncRef>,
4997 param_count: u32,
4998 result_count: u32,
4999 flags: u32,
5000 storage: *mut MaybeUninit<ValRaw>,
5001 storage_len: usize,
5002 ) -> Result<()>;
5003
5004 fn future_write(
5006 &mut self,
5007 instance: Instance,
5008 caller: RuntimeComponentInstanceIndex,
5009 ty: TypeFutureTableIndex,
5010 options: OptionsIndex,
5011 future: u32,
5012 address: u32,
5013 ) -> Result<u32>;
5014
5015 fn future_read(
5017 &mut self,
5018 instance: Instance,
5019 caller: RuntimeComponentInstanceIndex,
5020 ty: TypeFutureTableIndex,
5021 options: OptionsIndex,
5022 future: u32,
5023 address: u32,
5024 ) -> Result<u32>;
5025
5026 fn future_drop_writable(
5028 &mut self,
5029 instance: Instance,
5030 ty: TypeFutureTableIndex,
5031 writer: u32,
5032 ) -> Result<()>;
5033
5034 fn stream_write(
5036 &mut self,
5037 instance: Instance,
5038 caller: RuntimeComponentInstanceIndex,
5039 ty: TypeStreamTableIndex,
5040 options: OptionsIndex,
5041 stream: u32,
5042 address: u32,
5043 count: u32,
5044 ) -> Result<u32>;
5045
5046 fn stream_read(
5048 &mut self,
5049 instance: Instance,
5050 caller: RuntimeComponentInstanceIndex,
5051 ty: TypeStreamTableIndex,
5052 options: OptionsIndex,
5053 stream: u32,
5054 address: u32,
5055 count: u32,
5056 ) -> Result<u32>;
5057
5058 fn flat_stream_write(
5061 &mut self,
5062 instance: Instance,
5063 caller: RuntimeComponentInstanceIndex,
5064 ty: TypeStreamTableIndex,
5065 options: OptionsIndex,
5066 payload_size: u32,
5067 payload_align: u32,
5068 stream: u32,
5069 address: u32,
5070 count: u32,
5071 ) -> Result<u32>;
5072
5073 fn flat_stream_read(
5076 &mut self,
5077 instance: Instance,
5078 caller: RuntimeComponentInstanceIndex,
5079 ty: TypeStreamTableIndex,
5080 options: OptionsIndex,
5081 payload_size: u32,
5082 payload_align: u32,
5083 stream: u32,
5084 address: u32,
5085 count: u32,
5086 ) -> Result<u32>;
5087
5088 fn stream_drop_writable(
5090 &mut self,
5091 instance: Instance,
5092 ty: TypeStreamTableIndex,
5093 writer: u32,
5094 ) -> Result<()>;
5095
5096 fn error_context_debug_message(
5098 &mut self,
5099 instance: Instance,
5100 ty: TypeComponentLocalErrorContextTableIndex,
5101 options: OptionsIndex,
5102 err_ctx_handle: u32,
5103 debug_msg_address: u32,
5104 ) -> Result<()>;
5105
5106 fn thread_new_indirect(
5108 &mut self,
5109 instance: Instance,
5110 caller: RuntimeComponentInstanceIndex,
5111 func_ty_idx: ModuleInternedTypeIndex,
5112 start_func_table_idx: RuntimeTableIndex,
5113 start_func_idx: u32,
5114 context: i32,
5115 ) -> Result<u32>;
5116}
5117
5118impl<T: 'static> VMComponentAsyncStore for StoreInner<T> {
5120 unsafe fn prepare_call(
5121 &mut self,
5122 instance: Instance,
5123 memory: *mut VMMemoryDefinition,
5124 start: NonNull<VMFuncRef>,
5125 return_: NonNull<VMFuncRef>,
5126 caller_instance: RuntimeComponentInstanceIndex,
5127 callee_instance: RuntimeComponentInstanceIndex,
5128 task_return_type: TypeTupleIndex,
5129 callee_async: bool,
5130 string_encoding: StringEncoding,
5131 result_count_or_max_if_async: u32,
5132 storage: *mut ValRaw,
5133 storage_len: usize,
5134 ) -> Result<()> {
5135 let params = unsafe { core::slice::from_raw_parts(storage, storage_len) }.to_vec();
5139
5140 unsafe {
5141 instance.prepare_call(
5142 StoreContextMut(self),
5143 start,
5144 return_,
5145 caller_instance,
5146 callee_instance,
5147 task_return_type,
5148 callee_async,
5149 memory,
5150 string_encoding,
5151 match result_count_or_max_if_async {
5152 PREPARE_ASYNC_NO_RESULT => CallerInfo::Async {
5153 params,
5154 has_result: false,
5155 },
5156 PREPARE_ASYNC_WITH_RESULT => CallerInfo::Async {
5157 params,
5158 has_result: true,
5159 },
5160 result_count => CallerInfo::Sync {
5161 params,
5162 result_count,
5163 },
5164 },
5165 )
5166 }
5167 }
5168
5169 unsafe fn start_call(
5170 &mut self,
5171 instance: Instance,
5172 callback: *mut VMFuncRef,
5173 post_return: *mut VMFuncRef,
5174 callee: NonNull<VMFuncRef>,
5175 param_count: u32,
5176 result_count: u32,
5177 flags: u32,
5178 storage: *mut MaybeUninit<ValRaw>,
5179 storage_len: usize,
5180 ) -> Result<()> {
5181 unsafe {
5182 instance.start_call(
5183 StoreContextMut(self),
5184 callback,
5185 post_return,
5186 callee,
5187 param_count,
5188 result_count,
5189 flags,
5190 core::slice::from_raw_parts_mut(storage, storage_len),
5194 )
5195 }
5196 }
5197
5198 fn future_write(
5199 &mut self,
5200 instance: Instance,
5201 caller: RuntimeComponentInstanceIndex,
5202 ty: TypeFutureTableIndex,
5203 options: OptionsIndex,
5204 future: u32,
5205 address: u32,
5206 ) -> Result<u32> {
5207 instance
5208 .guest_write(
5209 StoreContextMut(self),
5210 caller,
5211 TransmitIndex::Future(ty),
5212 options,
5213 None,
5214 future,
5215 address,
5216 1,
5217 )
5218 .map(|result| result.encode())
5219 }
5220
5221 fn future_read(
5222 &mut self,
5223 instance: Instance,
5224 caller: RuntimeComponentInstanceIndex,
5225 ty: TypeFutureTableIndex,
5226 options: OptionsIndex,
5227 future: u32,
5228 address: u32,
5229 ) -> Result<u32> {
5230 instance
5231 .guest_read(
5232 StoreContextMut(self),
5233 caller,
5234 TransmitIndex::Future(ty),
5235 options,
5236 None,
5237 future,
5238 address,
5239 1,
5240 )
5241 .map(|result| result.encode())
5242 }
5243
5244 fn stream_write(
5245 &mut self,
5246 instance: Instance,
5247 caller: RuntimeComponentInstanceIndex,
5248 ty: TypeStreamTableIndex,
5249 options: OptionsIndex,
5250 stream: u32,
5251 address: u32,
5252 count: u32,
5253 ) -> Result<u32> {
5254 instance
5255 .guest_write(
5256 StoreContextMut(self),
5257 caller,
5258 TransmitIndex::Stream(ty),
5259 options,
5260 None,
5261 stream,
5262 address,
5263 count,
5264 )
5265 .map(|result| result.encode())
5266 }
5267
5268 fn stream_read(
5269 &mut self,
5270 instance: Instance,
5271 caller: RuntimeComponentInstanceIndex,
5272 ty: TypeStreamTableIndex,
5273 options: OptionsIndex,
5274 stream: u32,
5275 address: u32,
5276 count: u32,
5277 ) -> Result<u32> {
5278 instance
5279 .guest_read(
5280 StoreContextMut(self),
5281 caller,
5282 TransmitIndex::Stream(ty),
5283 options,
5284 None,
5285 stream,
5286 address,
5287 count,
5288 )
5289 .map(|result| result.encode())
5290 }
5291
5292 fn future_drop_writable(
5293 &mut self,
5294 instance: Instance,
5295 ty: TypeFutureTableIndex,
5296 writer: u32,
5297 ) -> Result<()> {
5298 instance.guest_drop_writable(self, TransmitIndex::Future(ty), writer)
5299 }
5300
5301 fn flat_stream_write(
5302 &mut self,
5303 instance: Instance,
5304 caller: RuntimeComponentInstanceIndex,
5305 ty: TypeStreamTableIndex,
5306 options: OptionsIndex,
5307 payload_size: u32,
5308 payload_align: u32,
5309 stream: u32,
5310 address: u32,
5311 count: u32,
5312 ) -> Result<u32> {
5313 instance
5314 .guest_write(
5315 StoreContextMut(self),
5316 caller,
5317 TransmitIndex::Stream(ty),
5318 options,
5319 Some(FlatAbi {
5320 size: payload_size,
5321 align: payload_align,
5322 }),
5323 stream,
5324 address,
5325 count,
5326 )
5327 .map(|result| result.encode())
5328 }
5329
5330 fn flat_stream_read(
5331 &mut self,
5332 instance: Instance,
5333 caller: RuntimeComponentInstanceIndex,
5334 ty: TypeStreamTableIndex,
5335 options: OptionsIndex,
5336 payload_size: u32,
5337 payload_align: u32,
5338 stream: u32,
5339 address: u32,
5340 count: u32,
5341 ) -> Result<u32> {
5342 instance
5343 .guest_read(
5344 StoreContextMut(self),
5345 caller,
5346 TransmitIndex::Stream(ty),
5347 options,
5348 Some(FlatAbi {
5349 size: payload_size,
5350 align: payload_align,
5351 }),
5352 stream,
5353 address,
5354 count,
5355 )
5356 .map(|result| result.encode())
5357 }
5358
5359 fn stream_drop_writable(
5360 &mut self,
5361 instance: Instance,
5362 ty: TypeStreamTableIndex,
5363 writer: u32,
5364 ) -> Result<()> {
5365 instance.guest_drop_writable(self, TransmitIndex::Stream(ty), writer)
5366 }
5367
5368 fn error_context_debug_message(
5369 &mut self,
5370 instance: Instance,
5371 ty: TypeComponentLocalErrorContextTableIndex,
5372 options: OptionsIndex,
5373 err_ctx_handle: u32,
5374 debug_msg_address: u32,
5375 ) -> Result<()> {
5376 instance.error_context_debug_message(
5377 StoreContextMut(self),
5378 ty,
5379 options,
5380 err_ctx_handle,
5381 debug_msg_address,
5382 )
5383 }
5384
5385 fn thread_new_indirect(
5386 &mut self,
5387 instance: Instance,
5388 caller: RuntimeComponentInstanceIndex,
5389 func_ty_idx: ModuleInternedTypeIndex,
5390 start_func_table_idx: RuntimeTableIndex,
5391 start_func_idx: u32,
5392 context: i32,
5393 ) -> Result<u32> {
5394 instance.thread_new_indirect(
5395 StoreContextMut(self),
5396 caller,
5397 func_ty_idx,
5398 start_func_table_idx,
5399 start_func_idx,
5400 context,
5401 )
5402 }
5403}
5404
5405type HostTaskFuture = Pin<Box<dyn Future<Output = Result<()>> + Send + 'static>>;
5406
5407async fn run_with_host_task_set<F>(task: TableId<HostTask>, future: F) -> Result<F::Output>
5410where
5411 F: Future,
5412{
5413 let mut future = pin!(future);
5414 future::poll_fn(|cx| {
5415 let old_thread = match tls::get(|store| store.set_thread(task)) {
5416 Ok(thread) => thread,
5417 Err(error) => return Poll::Ready(Err(error)),
5418 };
5419 let result = future.as_mut().poll(cx);
5420 match tls::get(|store| store.set_thread(old_thread)) {
5421 Ok(_) => result.map(Ok),
5422 Err(error) => Poll::Ready(Err(error)),
5423 }
5424 })
5425 .await
5426}
5427
5428pub(crate) struct HostTask {
5432 common: WaitableCommon,
5433
5434 call_context: CallContext,
5437
5438 state: HostTaskState,
5439
5440 group: TaskGroupId,
5441}
5442
5443enum HostTaskState {
5444 CalleeStarted,
5449
5450 CalleeRunning(JoinHandle),
5455
5456 CalleeCancelling,
5460
5461 CalleeFinished(LiftedResult),
5465
5466 CalleeDone { cancelled: bool },
5469}
5470
5471impl HostTask {
5472 fn new(
5473 concurrent_state: &mut ConcurrentState,
5474 state: HostTaskState,
5475 caller: QualifiedThreadId,
5476 ) -> Result<Self> {
5477 let group = concurrent_state.get_mut(caller.task)?.group;
5478 concurrent_state.increment_group_ref_count(group)?;
5479
5480 Ok(Self {
5481 common: WaitableCommon::default(),
5482 call_context: CallContext::default(),
5483 state,
5484 group,
5485 })
5486 }
5487}
5488
5489impl TableDebug for HostTask {
5490 fn type_name() -> &'static str {
5491 "HostTask"
5492 }
5493}
5494
5495type CallbackFn = Box<dyn Fn(&mut dyn VMStore, Event, u32) -> Result<u32> + Send + Sync + 'static>;
5496
5497enum Caller {
5499 Host {
5501 tx: Option<oneshot::Sender<LiftedResult>>,
5503 caller: Option<TableId<HostTask>>,
5507 },
5508 Guest {
5510 thread: QualifiedThreadId,
5512 },
5513}
5514
5515struct LiftResult {
5518 lift: RawLift,
5519 ty: TypeTupleIndex,
5520 memory: Option<SendSyncPtr<VMMemoryDefinition>>,
5521 string_encoding: StringEncoding,
5522}
5523
5524#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5529pub(crate) struct QualifiedThreadId {
5530 task: TableId<GuestTask>,
5531 thread: TableId<GuestThread>,
5532}
5533
5534impl QualifiedThreadId {
5535 fn qualify(
5536 state: &mut ConcurrentState,
5537 thread: TableId<GuestThread>,
5538 ) -> Result<QualifiedThreadId> {
5539 Ok(QualifiedThreadId {
5540 task: state.get_mut(thread)?.parent_task,
5541 thread,
5542 })
5543 }
5544}
5545
5546impl fmt::Debug for QualifiedThreadId {
5547 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5548 f.debug_tuple("QualifiedThreadId")
5549 .field(&self.task.rep())
5550 .field(&self.thread.rep())
5551 .finish()
5552 }
5553}
5554
5555enum GuestThreadState {
5556 NotStartedImplicit,
5557 NotStartedExplicit(
5558 Box<dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync>,
5559 ),
5560 Running,
5561 Suspended(StoreFiber<'static>),
5562 Ready {
5563 fiber: StoreFiber<'static>,
5564 },
5565 Completed,
5566}
5567
5568impl GuestThreadState {
5569 fn check_no_fiber(&self) -> Result<()> {
5570 match self {
5571 Self::Suspended(_) | Self::Ready { .. } => bail_bug!("thread already owns a fiber"),
5572 Self::NotStartedImplicit
5573 | Self::NotStartedExplicit(_)
5574 | Self::Running
5575 | Self::Completed => Ok(()),
5576 }
5577 }
5578}
5579
5580impl fmt::Debug for GuestThreadState {
5581 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
5582 match self {
5583 Self::NotStartedImplicit => f.debug_tuple("NotStartedImplicit").finish(),
5584 Self::NotStartedExplicit(_) => f.debug_tuple("NotStartedExplicit").finish(),
5585 Self::Running => f.debug_tuple("Running").finish(),
5586 Self::Suspended(_) => f.debug_tuple("Suspended").finish(),
5587 Self::Ready { .. } => f.debug_struct("Ready").finish(),
5588 Self::Completed => f.debug_tuple("Completed").finish(),
5589 }
5590 }
5591}
5592
5593#[derive(Copy, Clone, PartialEq, Eq, Debug)]
5594enum WakeOnCancel {
5595 None,
5596 Waiting(TableId<WaitableSet>),
5597 Yielding,
5598}
5599
5600impl WakeOnCancel {
5601 fn is_none(self) -> bool {
5602 matches!(self, WakeOnCancel::None)
5603 }
5604
5605 fn replace(&mut self, other: WakeOnCancel) -> Self {
5606 let old = *self;
5607 *self = other;
5608 old
5609 }
5610
5611 fn take(&mut self) -> Self {
5612 self.replace(WakeOnCancel::None)
5613 }
5614}
5615
5616pub struct GuestThread {
5617 context: [u32; NUM_COMPONENT_CONTEXT_SLOTS],
5620 parent_task: TableId<GuestTask>,
5622 wake_on_cancel: WakeOnCancel,
5625 state: GuestThreadState,
5627 instance_rep: Option<u32>,
5630 sync_call_set: TableId<WaitableSet>,
5632 old_do_not_suspend: Option<bool>,
5635}
5636
5637impl GuestThread {
5638 fn from_instance(
5641 state: Pin<&mut ComponentInstance>,
5642 caller_instance: RuntimeComponentInstanceIndex,
5643 guest_thread: u32,
5644 ) -> Result<TableId<Self>> {
5645 let rep = state.instance_states().0[caller_instance]
5646 .thread_handle_table()
5647 .guest_thread_rep(guest_thread)?;
5648 Ok(TableId::new(rep))
5649 }
5650
5651 fn new_implicit(state: &mut ConcurrentState, parent_task: TableId<GuestTask>) -> Result<Self> {
5652 let sync_call_set = state.push(WaitableSet {
5653 is_sync_call_set: true,
5654 ..WaitableSet::default()
5655 })?;
5656 Ok(Self {
5657 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5658 parent_task,
5659 wake_on_cancel: WakeOnCancel::None,
5660 state: GuestThreadState::NotStartedImplicit,
5661 instance_rep: None,
5662 sync_call_set,
5663 old_do_not_suspend: None,
5664 })
5665 }
5666
5667 fn new_explicit(
5668 state: &mut ConcurrentState,
5669 parent_task: TableId<GuestTask>,
5670 start_func: Box<
5671 dyn FnOnce(&mut dyn VMStore, QualifiedThreadId) -> Result<()> + Send + Sync,
5672 >,
5673 ) -> Result<Self> {
5674 state.increment_task_ref_count(parent_task)?;
5675 let sync_call_set = state.push(WaitableSet {
5676 is_sync_call_set: true,
5677 ..WaitableSet::default()
5678 })?;
5679 Ok(Self {
5680 context: [0; NUM_COMPONENT_CONTEXT_SLOTS],
5681 parent_task,
5682 wake_on_cancel: WakeOnCancel::None,
5683 state: GuestThreadState::NotStartedExplicit(start_func),
5684 instance_rep: None,
5685 sync_call_set,
5686 old_do_not_suspend: None,
5687 })
5688 }
5689}
5690
5691impl TableDebug for GuestThread {
5692 fn type_name() -> &'static str {
5693 "GuestThread"
5694 }
5695}
5696
5697enum SyncResult {
5698 NotProduced,
5699 Produced(Option<ValRaw>),
5700 Taken,
5701}
5702
5703impl SyncResult {
5704 fn take(&mut self) -> Result<Option<Option<ValRaw>>> {
5705 Ok(match mem::replace(self, SyncResult::Taken) {
5706 SyncResult::NotProduced => None,
5707 SyncResult::Produced(val) => Some(val),
5708 SyncResult::Taken => {
5709 bail_bug!("attempted to take a synchronous result that was already taken")
5710 }
5711 })
5712 }
5713}
5714
5715pub(crate) struct GuestTask {
5717 common: WaitableCommon,
5719 lower_params: Option<RawLower>,
5721 lift_result: Option<LiftResult>,
5723 result: Option<LiftedResult>,
5726 callback: Option<CallbackFn>,
5729 caller: Caller,
5731 call_context: CallContext,
5736 sync_result: SyncResult,
5739 cancel_request_delivered: bool,
5743 cancel_requested: bool,
5748 starting_sent: bool,
5751 instance: RuntimeInstance,
5758 event: Option<Event>,
5760 exited: bool,
5762 threads: HashSet<TableId<GuestThread>>,
5764 async_typed: bool,
5767 async_lifted: bool,
5770 decremented_interesting_task_count: bool,
5773 group: TaskGroupId,
5775 ref_count: usize,
5780}
5781
5782impl GuestTask {
5783 fn already_lowered_parameters(&self) -> bool {
5784 self.lower_params.is_none()
5786 }
5787
5788 fn returned_or_cancelled(&self) -> bool {
5789 self.lift_result.is_none()
5791 }
5792
5793 fn new(
5794 state: &mut ConcurrentState,
5795 lower_params: RawLower,
5796 lift_result: LiftResult,
5797 caller: Caller,
5798 callback: Option<CallbackFn>,
5799 instance: RuntimeInstance,
5800 async_typed: bool,
5801 async_lifted: bool,
5802 ) -> Result<QualifiedThreadId> {
5803 let group = match caller {
5804 Caller::Guest { thread } => {
5805 let group = state.get_mut(thread.task)?.group;
5806 state.increment_group_ref_count(group)?;
5807 group
5808 }
5809 Caller::Host { .. } => state.make_task_group()?,
5810 };
5811
5812 let task = state.push(Self {
5813 common: WaitableCommon::default(),
5814 lower_params: Some(lower_params),
5815 lift_result: Some(lift_result),
5816 result: None,
5817 callback,
5818 caller,
5819 call_context: CallContext::default(),
5820 sync_result: SyncResult::NotProduced,
5821 cancel_request_delivered: false,
5822 cancel_requested: false,
5823 starting_sent: false,
5824 instance,
5825 event: None,
5826 exited: false,
5827 threads: HashSet::new(),
5828 async_typed,
5829 async_lifted,
5830 decremented_interesting_task_count: false,
5831 group,
5832 ref_count: 2,
5835 })?;
5836 let new_thread = GuestThread::new_implicit(state, task)?;
5837 let thread = state.push(new_thread)?;
5838 state.get_mut(task)?.threads.insert(thread);
5839 state.interesting_tasks += 1;
5840 let thread = QualifiedThreadId { task, thread };
5841 log::trace!("new implicit thread {thread:?} for instance {instance:?}");
5842 Ok(thread)
5843 }
5844}
5845
5846impl TableDebug for GuestTask {
5847 fn type_name() -> &'static str {
5848 "GuestTask"
5849 }
5850}
5851
5852#[derive(Default)]
5854struct WaitableCommon {
5855 event: Option<Event>,
5857 set: Option<TableId<WaitableSet>>,
5859 handle: Option<u32>,
5861}
5862
5863#[derive(Copy, Clone, Ord, PartialOrd, Eq, PartialEq)]
5865enum Waitable {
5866 Host(TableId<HostTask>),
5868 Guest(TableId<GuestTask>),
5870 Transmit(TableId<TransmitHandle>),
5872}
5873
5874impl Waitable {
5875 fn from_instance(
5878 state: Pin<&mut ComponentInstance>,
5879 caller_instance: RuntimeComponentInstanceIndex,
5880 waitable: u32,
5881 ) -> Result<Self> {
5882 use crate::runtime::vm::component::Waitable;
5883
5884 let (waitable, kind) = state.instance_states().0[caller_instance]
5885 .handle_table()
5886 .waitable_rep(waitable)?;
5887
5888 Ok(match kind {
5889 Waitable::Subtask { is_host: true } => Self::Host(TableId::new(waitable)),
5890 Waitable::Subtask { is_host: false } => Self::Guest(TableId::new(waitable)),
5891 Waitable::Stream | Waitable::Future => Self::Transmit(TableId::new(waitable)),
5892 })
5893 }
5894
5895 fn rep(&self) -> u32 {
5897 match self {
5898 Self::Host(id) => id.rep(),
5899 Self::Guest(id) => id.rep(),
5900 Self::Transmit(id) => id.rep(),
5901 }
5902 }
5903
5904 fn join(&self, state: &mut ConcurrentState, set: Option<TableId<WaitableSet>>) -> Result<()> {
5908 log::trace!("waitable {self:?} join set {set:?}");
5909
5910 let old = mem::replace(&mut self.common(state)?.set, set);
5911
5912 if let Some(old) = old {
5913 match *self {
5914 Waitable::Host(id) => state.remove_child(id, old),
5915 Waitable::Guest(id) => state.remove_child(id, old),
5916 Waitable::Transmit(id) => state.remove_child(id, old),
5917 }?;
5918
5919 state.get_mut(old)?.ready.remove(self);
5920 }
5921
5922 if let Some(set) = set {
5923 match *self {
5924 Waitable::Host(id) => state.add_child(id, set),
5925 Waitable::Guest(id) => state.add_child(id, set),
5926 Waitable::Transmit(id) => state.add_child(id, set),
5927 }?;
5928
5929 if self.common(state)?.event.is_some() {
5930 self.mark_ready(state)?;
5931 }
5932 }
5933
5934 Ok(())
5935 }
5936
5937 fn common<'a>(&self, state: &'a mut ConcurrentState) -> Result<&'a mut WaitableCommon> {
5939 Ok(match self {
5940 Self::Host(id) => &mut state.get_mut(*id)?.common,
5941 Self::Guest(id) => &mut state.get_mut(*id)?.common,
5942 Self::Transmit(id) => &mut state.get_mut(*id)?.common,
5943 })
5944 }
5945
5946 fn trap_if_in_waitable_set(&self, state: &mut ConcurrentState) -> Result<()> {
5952 if self.common(state)?.set.is_some() {
5953 bail!(Trap::WaitableSyncAndAsync);
5954 }
5955 Ok(())
5956 }
5957
5958 fn trap_if_has_sync_waiter(&self, state: &mut ConcurrentState) -> Result<()> {
5964 if let Some(set) = self.common(state)?.set
5965 && state.get_mut(set)?.is_sync_call_set
5966 {
5967 bail!(Trap::WaitableSyncAndAsync);
5968 }
5969 Ok(())
5970 }
5971
5972 fn set_event(&self, state: &mut ConcurrentState, event: Option<Event>) -> Result<()> {
5976 log::trace!("set event for {self:?}: {event:?}");
5977 self.common(state)?.event = event;
5978 self.mark_ready(state)
5979 }
5980
5981 fn take_event(&self, state: &mut ConcurrentState) -> Result<Option<Event>> {
5983 let common = self.common(state)?;
5984 let event = common.event.take();
5985 if let Some(set) = self.common(state)?.set {
5986 state.get_mut(set)?.ready.remove(self);
5987 }
5988
5989 Ok(event)
5990 }
5991
5992 fn mark_ready(&self, state: &mut ConcurrentState) -> Result<()> {
5996 if let Some(set) = self.common(state)?.set {
5997 state.get_mut(set)?.ready.insert(*self);
5998 state.wake_waiter(set)?;
5999 }
6000 Ok(())
6001 }
6002
6003 fn delete_from(&self, state: &mut ConcurrentState) -> Result<()> {
6006 match self {
6007 Self::Host(task) => state.delete_host_task(*task),
6008 Self::Guest(task) => state.decrement_task_ref_count(*task),
6009 Self::Transmit(task) => Ok(state.delete(*task).map(drop)?),
6010 }
6011 }
6012}
6013
6014impl fmt::Debug for Waitable {
6015 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6016 match self {
6017 Self::Host(id) => write!(f, "{id:?}"),
6018 Self::Guest(id) => write!(f, "{id:?}"),
6019 Self::Transmit(id) => write!(f, "{id:?}"),
6020 }
6021 }
6022}
6023
6024#[derive(Default)]
6026struct WaitableSet {
6027 ready: BTreeSet<Waitable>,
6029 waiting: BTreeMap<QualifiedThreadId, WaitMode>,
6031 num_waiting: usize,
6034 is_sync_call_set: bool,
6037}
6038
6039impl WaitableSet {
6040 fn stop_waiting(&mut self) -> Result<()> {
6042 self.num_waiting = match self.num_waiting.checked_sub(1) {
6043 Some(n) => n,
6044 None => bail_bug!("waiter not accounted for in waitable set"),
6045 };
6046 Ok(())
6047 }
6048}
6049
6050impl TableDebug for WaitableSet {
6051 fn type_name() -> &'static str {
6052 "WaitableSet"
6053 }
6054}
6055
6056type RawLower =
6058 Box<dyn FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync>;
6059
6060type RawLift = Box<
6062 dyn FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6063>;
6064
6065type LiftedResult = Box<dyn Any + Send + Sync>;
6069
6070struct DummyResult;
6073
6074#[derive(Default)]
6076pub struct ConcurrentInstanceState {
6077 backpressure: u16,
6079 do_not_enter: bool,
6081 do_not_suspend: bool,
6084 pending: BTreeMap<QualifiedThreadId, GuestCallKind>,
6087 pending_start_count: usize,
6090}
6091
6092impl ConcurrentInstanceState {
6093 pub fn pending_is_empty(&self) -> bool {
6094 self.pending.is_empty()
6095 }
6096}
6097
6098#[derive(Debug, Copy, Clone)]
6099pub(crate) enum CurrentThread {
6100 Guest(QualifiedThreadId),
6103 Host(TableId<HostTask>),
6105 DeferredHost(QualifiedThreadId),
6108 None,
6111}
6112
6113impl CurrentThread {
6114 fn guest(&self) -> Option<&QualifiedThreadId> {
6115 match self {
6116 Self::Guest(id) => Some(id),
6117 _ => None,
6118 }
6119 }
6120
6121 fn guest_task(&self) -> Option<TableId<GuestTask>> {
6122 match self {
6123 Self::Guest(id) => Some(id.task),
6124 _ => None,
6125 }
6126 }
6127
6128 fn is_none(&self) -> bool {
6129 matches!(self, Self::None)
6130 }
6131}
6132
6133impl From<QualifiedThreadId> for CurrentThread {
6134 fn from(id: QualifiedThreadId) -> Self {
6135 Self::Guest(id)
6136 }
6137}
6138
6139impl From<TableId<HostTask>> for CurrentThread {
6140 fn from(id: TableId<HostTask>) -> Self {
6141 Self::Host(id)
6142 }
6143}
6144
6145enum Priority {
6146 Switch,
6147 High,
6148 Low,
6149}
6150
6151pub struct ConcurrentState {
6153 unforced_current_thread: CurrentThread,
6159
6160 deferred_host_call_context: Option<CallContext>,
6166
6167 futures: AlwaysMut<Option<FuturesUnordered<HostTaskFuture>>>,
6172 table: AlwaysMut<ResourceTable>,
6174 switch_item: Option<WorkItem>,
6182 next_switch_item: Option<WorkItem>,
6188 high_priority: VecDeque<WorkItem>,
6190 low_priority: VecDeque<WorkItem>,
6192 suspend_reason: Option<SuspendReason>,
6196 worker: Option<StoreFiber<'static>>,
6200 worker_item: Option<WorkerItem>,
6202
6203 global_error_context_ref_counts:
6216 BTreeMap<TypeComponentGlobalErrorContextTableIndex, GlobalErrorContextRefCount>,
6217
6218 interesting_tasks: usize,
6231
6232 interesting_tasks_empty_waker: Option<Waker>,
6236
6237 ready_for_concurrent_call_waker: Option<Waker>,
6242
6243 event_loop_running: bool,
6245
6246 saved_next_switch_items: Vec<Option<WorkItem>>,
6249
6250 #[cfg(feature = "task-group-hook")]
6252 task_group_hook: Option<Box<dyn TaskGroupHook>>,
6253}
6254
6255impl Default for ConcurrentState {
6256 fn default() -> Self {
6257 Self {
6258 unforced_current_thread: CurrentThread::None,
6259 deferred_host_call_context: None,
6260 table: AlwaysMut::new(ResourceTable::new()),
6261 futures: AlwaysMut::new(Some(FuturesUnordered::new())),
6262 switch_item: None,
6263 next_switch_item: None,
6264 high_priority: VecDeque::new(),
6265 low_priority: VecDeque::new(),
6266 suspend_reason: None,
6267 worker: None,
6268 worker_item: None,
6269 global_error_context_ref_counts: BTreeMap::new(),
6270 interesting_tasks: 0,
6271 interesting_tasks_empty_waker: None,
6272 ready_for_concurrent_call_waker: None,
6273 event_loop_running: false,
6274 saved_next_switch_items: Vec::new(),
6275 #[cfg(feature = "task-group-hook")]
6276 task_group_hook: None,
6277 }
6278 }
6279}
6280
6281impl ConcurrentState {
6282 pub(crate) fn take_fibers_and_futures(
6299 &mut self,
6300 fibers: &mut Vec<StoreFiber<'static>>,
6301 futures: &mut Vec<FuturesUnordered<HostTaskFuture>>,
6302 ) {
6303 let ConcurrentState {
6304 table,
6305 worker,
6306 switch_item,
6307 next_switch_item,
6308 high_priority,
6309 low_priority,
6310 saved_next_switch_items,
6311 futures: pending_futures,
6312 worker_item: _,
6313 unforced_current_thread: _,
6314 deferred_host_call_context: _,
6315 suspend_reason: _,
6316 global_error_context_ref_counts: _,
6317 interesting_tasks: _,
6318 interesting_tasks_empty_waker: _,
6319 ready_for_concurrent_call_waker: _,
6320 event_loop_running: _,
6321 #[cfg(feature = "task-group-hook")]
6322 task_group_hook: _,
6323 } = self;
6324
6325 let mut items = Vec::new();
6326 for (_, entry) in table.get_mut().iter_mut() {
6327 let entry = entry.into_entry_mut();
6328 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6329 for mode in mem::take(&mut set.waiting).into_values() {
6330 match mode {
6331 WaitMode::Fiber(fiber) => {
6332 fibers.push(fiber);
6333 }
6334 WaitMode::Callback(_) => {}
6335 }
6336 }
6337 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6338 match mem::replace(&mut thread.state, GuestThreadState::Completed) {
6339 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6340 fibers.push(fiber);
6341 }
6342 GuestThreadState::NotStartedImplicit
6343 | GuestThreadState::NotStartedExplicit(_)
6344 | GuestThreadState::Running
6345 | GuestThreadState::Completed => {}
6346 }
6347 } else if let Some(item) = entry.downcast_mut::<Option<WorkItem>>() {
6348 if let Some(item) = item.take() {
6349 items.push(item);
6350 }
6351 }
6352 }
6353
6354 if let Some(fiber) = worker.take() {
6355 fibers.push(fiber);
6356 }
6357
6358 let mut handle_item = |item| match item {
6359 WorkItem::ResumeFiber { fiber, .. } => {
6360 fibers.push(fiber);
6361 }
6362 WorkItem::PushFuture(future) => {
6363 pending_futures
6364 .get_mut()
6365 .as_mut()
6366 .unwrap()
6367 .push(future.into_inner());
6368 }
6369 WorkItem::ResumeThread { .. }
6370 | WorkItem::GuestCall { .. }
6371 | WorkItem::WorkerFunction(_) => {}
6372 };
6373
6374 for item in items {
6375 handle_item(item);
6376 }
6377 if let Some(item) = switch_item.take() {
6378 handle_item(item);
6379 }
6380 if let Some(item) = next_switch_item.take() {
6381 handle_item(item);
6382 }
6383 for item in mem::take(high_priority) {
6384 handle_item(item);
6385 }
6386 for item in mem::take(low_priority) {
6387 handle_item(item);
6388 }
6389 for item in mem::take(saved_next_switch_items)
6390 .into_iter()
6391 .filter_map(|v| v)
6392 {
6393 handle_item(item);
6394 }
6395
6396 if let Some(them) = pending_futures.get_mut().take() {
6397 futures.push(them);
6398 }
6399 }
6400
6401 #[cfg(feature = "gc")]
6402 pub(crate) fn trace_fiber_roots(
6403 &mut self,
6404 modules: &ModuleRegistry,
6405 unwind: &dyn Unwind,
6406 gc_roots_list: &mut GcRootsList,
6407 ) {
6408 let ConcurrentState {
6409 table,
6410 worker,
6411 switch_item,
6412 next_switch_item,
6413 high_priority,
6414 low_priority,
6415 saved_next_switch_items,
6416
6417 futures: _,
6421
6422 worker_item: _,
6424 unforced_current_thread: _,
6425 deferred_host_call_context: _,
6426 suspend_reason: _,
6427 global_error_context_ref_counts: _,
6428 interesting_tasks: _,
6429 interesting_tasks_empty_waker: _,
6430 ready_for_concurrent_call_waker: _,
6431 event_loop_running: _,
6432 #[cfg(feature = "task-group-hook")]
6433 task_group_hook: _,
6434 } = self;
6435
6436 for (_, entry) in table.get_mut().iter_mut() {
6437 let entry = entry.into_entry_mut();
6438 if let Some(set) = entry.downcast_mut::<WaitableSet>() {
6439 for mode in set.waiting.values_mut() {
6440 match mode {
6441 WaitMode::Fiber(fiber) => {
6442 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6443 }
6444 WaitMode::Callback(_) => {}
6445 }
6446 }
6447 } else if let Some(thread) = entry.downcast_mut::<GuestThread>() {
6448 match &mut thread.state {
6449 GuestThreadState::Suspended(fiber) | GuestThreadState::Ready { fiber } => {
6450 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6451 }
6452 GuestThreadState::NotStartedImplicit
6453 | GuestThreadState::NotStartedExplicit(_)
6454 | GuestThreadState::Running
6455 | GuestThreadState::Completed => {}
6456 }
6457 } else if let Some(Some(WorkItem::ResumeFiber { fiber, .. })) =
6458 entry.downcast_mut::<Option<WorkItem>>()
6459 {
6460 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6461 }
6462 }
6463
6464 if let Some(fiber) = worker {
6465 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6466 }
6467
6468 let mut handle_item = |item: &mut WorkItem| match item {
6469 WorkItem::ResumeFiber { fiber, .. } => {
6470 fiber.trace_gc_roots(modules, unwind, gc_roots_list);
6471 }
6472 WorkItem::PushFuture(_future) => {
6473 }
6476 WorkItem::ResumeThread { .. }
6477 | WorkItem::GuestCall { .. }
6478 | WorkItem::WorkerFunction(_) => {}
6479 };
6480
6481 if let Some(item) = switch_item {
6482 handle_item(item);
6483 }
6484 if let Some(item) = next_switch_item {
6485 handle_item(item);
6486 }
6487 for item in high_priority {
6488 handle_item(item);
6489 }
6490 for item in low_priority {
6491 handle_item(item);
6492 }
6493 for item in saved_next_switch_items
6494 .iter_mut()
6495 .filter_map(|v| v.as_mut())
6496 {
6497 handle_item(item);
6498 }
6499 }
6500
6501 fn schedule_suspended_thread(
6502 &mut self,
6503 thread: QualifiedThreadId,
6504 instance: RuntimeInstance,
6505 priority: Priority,
6506 ) -> Result<()> {
6507 if matches!(priority, Priority::Switch) && self.switch_item.is_some() {
6509 bail_bug!("switch item already set");
6510 }
6511 let slot = &mut self.get_mut(thread.thread)?.state;
6512 if !matches!(slot, GuestThreadState::Suspended(_)) {
6513 bail!(Trap::CannotResumeThread);
6514 }
6515 let GuestThreadState::Suspended(fiber) = mem::replace(slot, GuestThreadState::Running)
6516 else {
6517 unreachable!()
6518 };
6519 let item = WorkItem::ResumeFiber {
6520 instance,
6521 thread,
6522 fiber,
6523 };
6524 match priority {
6525 Priority::Switch => self.switch_item = Some(item),
6526 Priority::High => self.push_high_priority(item),
6527 Priority::Low => self.push_low_priority(item),
6528 }
6529 Ok(())
6530 }
6531
6532 fn set_thread_running(&mut self, thread: TableId<GuestThread>) -> Result<()> {
6533 let slot = &mut self.get_mut(thread)?.state;
6534 slot.check_no_fiber()?;
6535 *slot = GuestThreadState::Running;
6536 Ok(())
6537 }
6538
6539 fn save_next_switch_item(&mut self) -> Result<TableId<Option<WorkItem>>> {
6540 let saved = self.push(None::<WorkItem>)?;
6543 let slot = self.table.get_mut().get_mut(&Resource::from(saved))?;
6544 *slot = self.next_switch_item.take();
6545 Ok(saved)
6546 }
6547
6548 fn restore_next_switch_item(&mut self, saved: TableId<Option<WorkItem>>) -> Result<()> {
6549 if self.next_switch_item.is_some() {
6550 bail_bug!("next switch item already set when restoring");
6551 }
6552 self.next_switch_item = self.delete(saved)?;
6553 Ok(())
6554 }
6555
6556 fn take_callback_waiter(
6557 &mut self,
6558 set: TableId<WaitableSet>,
6559 thread: QualifiedThreadId,
6560 ) -> Result<Instance> {
6561 let waiting = &mut self.get_mut(set)?.waiting;
6562 let instance = match waiting.get(&thread) {
6563 Some(WaitMode::Callback(instance)) => *instance,
6564 other => bail_bug!("expected `Some(WaitMode::Callback(_))`; got `{other:?}`"),
6565 };
6566 waiting.remove(&thread);
6567 Ok(instance)
6568 }
6569
6570 fn push<V: Send + Sync + 'static>(
6571 &mut self,
6572 value: V,
6573 ) -> Result<TableId<V>, ResourceTableError> {
6574 self.table.get_mut().push(value).map(TableId::from)
6575 }
6576
6577 fn get_mut<V: 'static>(&mut self, id: TableId<V>) -> Result<&mut V, ResourceTableError> {
6578 self.table.get_mut().get_mut(&Resource::from(id))
6579 }
6580
6581 pub fn add_child<T: 'static, U: 'static>(
6582 &mut self,
6583 child: TableId<T>,
6584 parent: TableId<U>,
6585 ) -> Result<(), ResourceTableError> {
6586 self.table
6587 .get_mut()
6588 .add_child(Resource::from(child), Resource::from(parent))
6589 }
6590
6591 pub fn remove_child<T: 'static, U: 'static>(
6592 &mut self,
6593 child: TableId<T>,
6594 parent: TableId<U>,
6595 ) -> Result<(), ResourceTableError> {
6596 self.table
6597 .get_mut()
6598 .remove_child(Resource::from(child), Resource::from(parent))
6599 }
6600
6601 fn delete<V: 'static>(&mut self, id: TableId<V>) -> Result<V, ResourceTableError> {
6602 self.table.get_mut().delete(Resource::from(id))
6603 }
6604
6605 fn push_future(&mut self, future: HostTaskFuture) {
6606 self.push_high_priority(WorkItem::PushFuture(AlwaysMut::new(future)));
6613 }
6614
6615 fn set_switch_item(&mut self, item: WorkItem) -> Result<()> {
6616 if self.switch_item.is_some() {
6617 self.push_high_priority(item);
6620 bail_bug!("switch item already set");
6621 }
6622
6623 self.switch_item = Some(item);
6624 log::trace!("set switch item: {:?}", self.switch_item.as_ref().unwrap());
6625
6626 Ok(())
6627 }
6628
6629 fn take_next_switch_item(&mut self) -> Result<()> {
6630 if let Some(item) = self.next_switch_item.take() {
6631 self.set_switch_item(item)?;
6632 }
6633 Ok(())
6634 }
6635
6636 fn wake_waiter(&mut self, set: TableId<WaitableSet>) -> Result<()> {
6639 let Some((&thread, _)) = self.get_mut(set)?.waiting.first_key_value() else {
6642 return Ok(());
6643 };
6644 let wake_on_cancel = self.get_mut(thread.thread)?.wake_on_cancel;
6645 if wake_on_cancel != WakeOnCancel::None && wake_on_cancel != WakeOnCancel::Waiting(set) {
6646 bail_bug!("thread {thread:?} has unexpected wake_on_cancel value {wake_on_cancel:?}");
6647 }
6648 let instance = self.get_mut(thread.task)?.instance;
6649 self.get_mut(thread.thread)?.wake_on_cancel.take();
6650 let (_, mode) = self.get_mut(set)?.waiting.pop_first().unwrap();
6651
6652 let item = match mode {
6653 WaitMode::Fiber(fiber) => WorkItem::ResumeFiber {
6654 instance,
6655 thread,
6656 fiber,
6657 },
6658 WaitMode::Callback(callback_instance) => WorkItem::GuestCall {
6659 instance,
6660 call: GuestCall {
6661 thread,
6662 kind: GuestCallKind::DeliverEvent {
6663 instance: callback_instance,
6664 set: Some(set),
6665 },
6666 },
6667 },
6668 };
6669 self.push_high_priority(item);
6670 Ok(())
6671 }
6672
6673 fn push_high_priority(&mut self, item: WorkItem) {
6674 self.high_priority.push_front(item);
6675 log::trace!(
6676 "push high priority: {:?}",
6677 self.high_priority.front().unwrap()
6678 );
6679 }
6680
6681 fn push_low_priority(&mut self, item: WorkItem) {
6682 self.low_priority.push_front(item);
6683 log::trace!(
6684 "push low priority: {:?}",
6685 self.low_priority.front().unwrap()
6686 );
6687 }
6688
6689 fn push_work_item(&mut self, item: WorkItem, priority: Priority) -> Result<()> {
6690 match priority {
6691 Priority::Switch => self.set_switch_item(item)?,
6692 Priority::High => self.push_high_priority(item),
6693 Priority::Low => self.push_low_priority(item),
6694 }
6695
6696 Ok(())
6697 }
6698
6699 fn promote_instance_local_thread_work_item(
6700 &mut self,
6701 current_instance: RuntimeInstance,
6702 ) -> Result<bool> {
6703 log::trace!("promote thread work items for {current_instance:?}");
6704
6705 self.promote_work_item_matching(|item: &WorkItem| {
6706 let result = match item {
6707 WorkItem::ResumeThread { instance, .. }
6708 | WorkItem::ResumeFiber { instance, .. }
6709 | WorkItem::GuestCall { instance, .. } => *instance == current_instance,
6710 _ => false,
6711 };
6712
6713 log::trace!("candidate {item:?}: {result}");
6714 result
6715 })
6716 }
6717
6718 fn promote_thread_work_item(&mut self, thread: QualifiedThreadId) -> Result<bool> {
6719 self.promote_work_item_matching(|item: &WorkItem| match item {
6720 WorkItem::ResumeThread {
6721 thread: item_thread,
6722 ..
6723 }
6724 | WorkItem::GuestCall {
6725 call:
6726 GuestCall {
6727 thread: item_thread,
6728 ..
6729 },
6730 ..
6731 } => *item_thread == thread,
6732 _ => false,
6733 })
6734 }
6735
6736 fn promote_work_item_matching<F>(&mut self, mut predicate: F) -> Result<bool>
6737 where
6738 F: FnMut(&WorkItem) -> bool,
6739 {
6740 if self.switch_item.is_none() {
6747 if let Some(index) = self.high_priority.iter().rposition(&mut predicate) {
6748 self.switch_item = self.high_priority.remove(index);
6749 }
6750 }
6751 if self.switch_item.is_none() {
6752 if let Some(index) = self.low_priority.iter().rposition(predicate) {
6753 self.switch_item = self.low_priority.remove(index);
6754 }
6755 }
6756 Ok(self.switch_item.is_some())
6757 }
6758
6759 pub fn call_context(&mut self, task: VMScope) -> Result<&mut CallContext> {
6762 match task.kind() {
6763 ScopeKind::HostId(task) => {
6764 let task: TableId<HostTask> = TableId::new(task);
6765 Ok(&mut self.get_mut(task)?.call_context)
6766 }
6767 ScopeKind::Id(task) => {
6768 let task: TableId<GuestTask> = TableId::new(task);
6769 Ok(&mut self.get_mut(task)?.call_context)
6770 }
6771 }
6772 }
6773
6774 pub(crate) fn deferred_host_call_context(&mut self) -> Option<&mut CallContext> {
6775 self.deferred_host_call_context.as_mut()
6776 }
6777
6778 fn futures_mut(&mut self) -> Result<&mut FuturesUnordered<HostTaskFuture>> {
6779 match self.futures.get_mut().as_mut() {
6780 Some(f) => Ok(f),
6781 None => bail_bug!("futures field of concurrent state is currently taken"),
6782 }
6783 }
6784
6785 pub(crate) fn table(&mut self) -> &mut ResourceTable {
6786 self.table.get_mut()
6787 }
6788
6789 fn debug_assert_deferred_host_invariant(&self) {
6790 debug_assert_eq!(
6791 self.deferred_host_call_context.is_some(),
6792 matches!(self.unforced_current_thread, CurrentThread::DeferredHost(_)),
6793 "a deferred host thread and call context must exist together",
6794 );
6795 }
6796
6797 fn materialize_host_task(&mut self) -> Result<CurrentThread> {
6798 self.debug_assert_deferred_host_invariant();
6799 let caller = match self.unforced_current_thread {
6800 CurrentThread::DeferredHost(caller) => caller,
6801 thread => return Ok(thread),
6802 };
6803
6804 let task = HostTask::new(self, HostTaskState::CalleeStarted, caller)?;
6806 let task = self.push(task)?;
6807 let call_context = self
6808 .deferred_host_call_context
6809 .take()
6810 .expect("deferred host call context should be present");
6811 self.get_mut(task)
6812 .expect("newly inserted host task should be present")
6813 .call_context = call_context;
6814 self.unforced_current_thread = CurrentThread::Host(task);
6815 self.debug_assert_deferred_host_invariant();
6816 log::trace!("new host task materialized {task:?}");
6817 Ok(CurrentThread::Host(task))
6818 }
6819
6820 fn materialize_current_host_task_id(&mut self) -> Result<Option<TableId<HostTask>>> {
6821 match self.materialize_host_task()? {
6822 CurrentThread::Host(id) => Ok(Some(id)),
6823 CurrentThread::None => Ok(None),
6824 CurrentThread::Guest(_) => {
6825 bail_bug!("tried to materialize a host task id from a guest thread")
6826 }
6827 CurrentThread::DeferredHost(_) => {
6828 bail_bug!(
6829 "current thread is a deferred host thread which should have been materialized"
6830 )
6831 }
6832 }
6833 }
6834
6835 pub(crate) fn materialize_current_scope(&mut self) -> Result<VMScope> {
6836 match self.materialize_host_task()? {
6837 CurrentThread::Host(id) => Ok(VMScope::host_id(id.rep())),
6838 _ => bail_bug!("current scope is not a deferred host scope"),
6839 }
6840 }
6841
6842 fn increment_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6843 let count = &mut self.get_mut(task)?.ref_count;
6844 *count += 1;
6845 log::trace!("increment {task:?} to {count}");
6846 Ok(())
6847 }
6848
6849 fn decrement_task_ref_count(&mut self, task: TableId<GuestTask>) -> Result<()> {
6850 let count = &mut self.get_mut(task)?.ref_count;
6851 if *count == 0 {
6852 bail_bug!("can't decrement task ref count that's already zero");
6853 }
6854 *count -= 1;
6855 log::trace!("decrement {task:?} to {count}");
6856 if *count == 0 {
6857 let task = self.delete(task)?;
6858
6859 self.decrement_group_ref_count(task.group)?;
6860
6861 debug_assert!(task.decremented_interesting_task_count);
6868 }
6869 Ok(())
6870 }
6871
6872 fn delete_host_task(&mut self, task: TableId<HostTask>) -> Result<()> {
6873 log::trace!("delete host task {task:?}");
6874 let task = self.delete(task)?;
6875
6876 self.decrement_group_ref_count(task.group)
6877 }
6878}
6879
6880fn for_any_lower<
6883 F: FnOnce(&mut dyn VMStore, &mut [MaybeUninit<ValRaw>]) -> Result<()> + Send + Sync,
6884>(
6885 fun: F,
6886) -> F {
6887 fun
6888}
6889
6890fn for_any_lift<
6892 F: FnOnce(&mut dyn VMStore, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>> + Send + Sync,
6893>(
6894 fun: F,
6895) -> F {
6896 fun
6897}
6898
6899fn check_ambient_store(id: StoreId) {
6900 let message = "\
6901 `Future`s which depend on asynchronous component tasks, streams, or \
6902 futures to complete may only be polled from the event loop of the \
6903 store to which they belong. Please use \
6904 `StoreContextMut::{run_concurrent,spawn}` to poll or await them.\
6905 ";
6906 tls::try_get(|store| {
6907 let matched = match store {
6908 tls::TryGet::Some(store) => store.id() == id,
6909 tls::TryGet::Taken | tls::TryGet::None => false,
6910 };
6911
6912 if !matched {
6913 panic!("{message}")
6914 }
6915 });
6916}
6917
6918fn unpack_callback_code(code: u32) -> (u32, u32) {
6919 (code & 0xF, code >> 4)
6920}
6921
6922struct WaitableCheckParams {
6926 set: TableId<WaitableSet>,
6927 options: OptionsIndex,
6928 payload: u32,
6929}
6930
6931enum WaitableCheck {
6934 Wait,
6935 Poll,
6936}
6937
6938pub(crate) struct PreparedCall<R> {
6940 handle: Func,
6942 thread: QualifiedThreadId,
6944 param_count: usize,
6946 rx: oneshot::Receiver<LiftedResult>,
6949 _phantom: PhantomData<R>,
6950}
6951
6952impl<R> PreparedCall<R> {
6953 pub(crate) fn acquire_task_id(&self, store: &mut StoreOpaque) -> Result<TaskId> {
6960 store
6961 .concurrent_state_mut()?
6962 .increment_task_ref_count(self.thread.task)?;
6963
6964 Ok(TaskId {
6965 task: self.thread.task,
6966 })
6967 }
6968}
6969
6970pub(crate) struct TaskId {
6972 task: TableId<GuestTask>,
6973}
6974
6975impl TaskId {
6976 pub(crate) fn host_future_dropped(&self, store: &mut StoreOpaque) -> Result<()> {
6988 let task = store.concurrent_state_mut()?.get_mut(self.task)?;
6989 if !task.already_lowered_parameters() {
6990 task.lower_params = Some(Box::new(|store, _| {
6995 store.set_trapped();
6996 bail!("the host future for this call was dropped before it started")
6997 }));
6998 } else {
6999 _ = store
7004 .concurrent_state_mut()?
7005 .decrement_task_ref_count(self.task);
7006 }
7007
7008 Ok(())
7009 }
7010}
7011
7012pub(crate) fn prepare_call<T, R>(
7018 mut store: StoreContextMut<T>,
7019 handle: Func,
7020 param_count: usize,
7021 lower_params: impl FnOnce(StoreContextMut<T>, &mut [MaybeUninit<ValRaw>]) -> Result<()>
7022 + Send
7023 + Sync
7024 + 'static,
7025 lift_result: impl FnOnce(&mut StoreOpaque, &[ValRaw]) -> Result<Box<dyn Any + Send + Sync>>
7026 + Send
7027 + Sync
7028 + 'static,
7029) -> Result<PreparedCall<R>> {
7030 if !store.0.may_enter() {
7034 bail!(Trap::CannotEnterComponent);
7035 }
7036
7037 let (options, _flags, ty, raw_options) = handle.abi_info(store.0);
7038
7039 let instance = handle.instance().id().get(store.0);
7040 let options = &instance.component().env_component().options[options];
7041 let ty = &instance.component().types()[ty];
7042 let async_typed = ty.async_;
7043 let async_lifted = raw_options.async_;
7044 let task_return_type = ty.results;
7045 let component_instance = raw_options.instance;
7046 let callback = options.callback.map(|i| instance.runtime_callback(i));
7047 let memory = options
7048 .memory()
7049 .map(|i| instance.runtime_memory(i))
7050 .map(SendSyncPtr::new);
7051 let string_encoding = options.string_encoding;
7052 let token = StoreToken::new(store.as_context_mut());
7053 let caller = store.0.materialize_host_task_id()?;
7054 let state = store.0.concurrent_state_mut()?;
7055
7056 let (tx, rx) = oneshot::channel();
7057
7058 let instance = handle.instance().runtime_instance(component_instance);
7059 let thread = GuestTask::new(
7060 state,
7061 Box::new(for_any_lower(move |store, params| {
7062 lower_params(token.as_context_mut(store), params)
7063 })),
7064 LiftResult {
7065 lift: Box::new(for_any_lift(move |store, result| {
7066 lift_result(store, result)
7067 })),
7068 ty: task_return_type,
7069 memory,
7070 string_encoding,
7071 },
7072 Caller::Host {
7073 tx: Some(tx),
7074 caller,
7075 },
7076 callback.map(|callback| {
7077 let callback = SendSyncPtr::new(callback);
7078 let instance = handle.instance();
7079 Box::new(move |store: &mut dyn VMStore, event, handle| {
7080 let store = token.as_context_mut(store);
7081 unsafe { instance.call_callback(store, callback, event, handle) }
7084 }) as CallbackFn
7085 }),
7086 instance,
7087 async_typed,
7088 async_lifted,
7089 )?;
7090
7091 Ok(PreparedCall {
7092 handle,
7093 thread,
7094 param_count,
7095 rx,
7096 _phantom: PhantomData,
7097 })
7098}
7099
7100pub(crate) struct StagedCall<R> {
7101 store: StoreId,
7102 rx: oneshot::Receiver<LiftedResult>,
7103 _marker: PhantomData<fn() -> R>,
7104 group: TaskGroupId,
7112}
7113
7114impl<R> StagedCall<R> {
7115 pub(crate) fn new<T: 'static>(
7126 mut store: StoreContextMut<T>,
7127 prepared: PreparedCall<R>,
7128 ) -> Result<StagedCall<R>> {
7129 let PreparedCall {
7130 handle,
7131 thread,
7132 param_count,
7133 rx,
7134 ..
7135 } = prepared;
7136
7137 stage_call0(store.as_context_mut(), handle, thread, param_count)?;
7138
7139 let state = store.0.concurrent_state_mut()?;
7140 let group = state.get_mut(thread.task)?.group;
7141 state.decrement_task_ref_count(thread.task)?;
7142
7143 Ok(StagedCall {
7144 store: store.0.id(),
7145 rx,
7146 _marker: PhantomData,
7147 group,
7148 })
7149 }
7150}
7151
7152impl<R> Future for StagedCall<R>
7153where
7154 R: 'static,
7155{
7156 type Output = Result<R>;
7157
7158 fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
7159 check_ambient_store(self.store);
7160 Pin::new(&mut self.rx).poll(cx).map(|result| match result {
7161 Ok(r) => match r.downcast() {
7162 Ok(r) => Ok(*r),
7163 Err(_) => bail_bug!("wrong type of value produced"),
7164 },
7165 Err(oneshot::Canceled) => bail_bug!("channel erroneously dropped"),
7166 })
7167 }
7168}
7169
7170fn stage_call0<T: 'static>(
7173 store: StoreContextMut<T>,
7174 handle: Func,
7175 guest_thread: QualifiedThreadId,
7176 param_count: usize,
7177) -> Result<()> {
7178 let (_options, _, _ty, raw_options) = handle.abi_info(store.0);
7179 let is_concurrent = raw_options.async_;
7180 let callback = raw_options.callback;
7181 let instance = handle.instance();
7182 let callee = handle.lifted_core_func(store.0);
7183 let post_return = raw_options
7184 .post_return
7185 .map(|i| instance.id().get(store.0).runtime_post_return(i));
7186 let callback = callback.map(|i| {
7187 let instance = instance.id().get(store.0);
7188 SendSyncPtr::new(instance.runtime_callback(i))
7189 });
7190
7191 log::trace!("queueing call {guest_thread:?}");
7192
7193 unsafe {
7197 instance.stage_call(
7198 store,
7199 guest_thread,
7200 SendSyncPtr::new(callee),
7201 param_count,
7202 1,
7203 is_concurrent,
7204 callback,
7205 post_return.map(SendSyncPtr::new),
7206 true,
7207 )
7208 }
7209}
7210
7211#[cfg(all(test, feature = "cranelift", feature = "wat"))]
7212mod tests {
7213 use super::*;
7214 use crate::component::{Component, Linker};
7215 use crate::store::AsStoreOpaque;
7216 use crate::{Config, Engine};
7217
7218 fn host_subtask(
7219 state: HostTaskState,
7220 event: Option<Event>,
7221 ) -> Result<(Store<()>, Instance, TableId<HostTask>, u32)> {
7222 let mut config = Config::new();
7223 config.wasm_component_model_async(true);
7224 let engine = Engine::new(&config)?;
7225 let component = Component::new(&engine, "(component)")?;
7226 let mut store = Store::new(&engine, ());
7227 let instance = Linker::new(&engine).instantiate(&mut store, &component)?;
7228 let store_opaque = store.as_store_opaque();
7229 let concurrent_state = store_opaque.concurrent_state_mut()?;
7230 let group = concurrent_state.make_task_group()?;
7233 let task = concurrent_state.push(HostTask {
7234 common: WaitableCommon::default(),
7235 call_context: CallContext::default(),
7236 state,
7237 group,
7238 })?;
7239 let handle = store_opaque
7240 .handle_table_insertion(
7241 instance.runtime_instance(RuntimeComponentInstanceIndex::from_u32(0)),
7242 )
7243 .subtask_host(task.rep())?;
7244 let common = &mut store_opaque.concurrent_state_mut()?.get_mut(task)?.common;
7245 common.handle = Some(handle);
7246 common.event = event;
7247 Ok((store, instance, task, handle))
7248 }
7249
7250 #[test]
7251 fn host_subtask_drop_during_cancellation() -> Result<()> {
7252 for abort_completed in [false, true] {
7253 let (handle, future) = JoinHandle::run(future::pending::<()>());
7254 let mut future = pin!(future);
7255 let (mut store, instance, task, handle) =
7256 host_subtask(HostTaskState::CalleeRunning(handle), None)?;
7257 let store = store.as_store_opaque();
7258 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7259 assert_eq!(
7260 instance.subtask_cancel(store, caller, true, handle)?,
7261 BLOCKED
7262 );
7263 if abort_completed {
7264 assert!(matches!(
7267 future
7268 .as_mut()
7269 .poll(&mut Context::from_waker(Waker::noop())),
7270 Poll::Ready(None),
7271 ));
7272 }
7273 for async_ in [false, true] {
7274 let err = instance
7275 .subtask_cancel(store, caller, async_, handle)
7276 .unwrap_err();
7277 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7278 }
7279 let err = instance.subtask_drop(store, caller, handle).unwrap_err();
7280 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7281 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7282 }
7283 Ok(())
7284 }
7285
7286 #[test]
7287 fn host_subtask_cancel_after_completion() -> Result<()> {
7288 for async_ in [false, true] {
7289 let (mut store, instance, task, handle) = host_subtask(
7290 HostTaskState::CalleeDone { cancelled: false },
7291 Some(Event::Subtask {
7292 status: Status::Returned,
7293 }),
7294 )?;
7295 let store = store.as_store_opaque();
7296 let caller = RuntimeComponentInstanceIndex::from_u32(0);
7297 assert_eq!(
7298 instance.subtask_cancel(store, caller, async_, handle)?,
7299 Status::Returned as u32,
7300 );
7301 let err = instance
7302 .subtask_cancel(store, caller, async_, handle)
7303 .unwrap_err();
7304 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskCancelAfterTerminal);
7305 instance.subtask_drop(store, caller, handle)?;
7306 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7307 }
7308 Ok(())
7309 }
7310
7311 #[test]
7312 fn host_subtask_drop_requires_terminal_event_delivery() -> Result<()> {
7313 for (cancelled, status) in [
7314 (false, Status::Returned),
7315 (true, Status::Returned),
7316 (true, Status::ReturnCancelled),
7317 ] {
7318 for delivered in [false, true] {
7319 let event = if delivered {
7320 None
7321 } else {
7322 Some(Event::Subtask { status })
7323 };
7324 let (mut store, instance, task, handle) =
7325 host_subtask(HostTaskState::CalleeDone { cancelled }, event)?;
7326 let store = store.as_store_opaque();
7327 let result = instance.subtask_drop(
7328 store,
7329 RuntimeComponentInstanceIndex::from_u32(0),
7330 handle,
7331 );
7332 if delivered {
7333 result?;
7334 assert!(store.concurrent_state_mut()?.get_mut(task).is_err());
7335 } else {
7336 let err = result.unwrap_err();
7337 assert_eq!(err.downcast::<Trap>()?, Trap::SubtaskDropNotResolved);
7338 assert!(store.concurrent_state_mut()?.get_mut(task).is_ok());
7339 }
7340 }
7341 }
7342 Ok(())
7343 }
7344}